moving
This commit is contained in:
@@ -109,6 +109,7 @@ export interface StartTraceInput {
|
||||
export interface FinalizeTraceInput {
|
||||
outcome: TraceOutcome;
|
||||
output: string;
|
||||
model?: string | null;
|
||||
reasoning?: TraceReasoningStep[];
|
||||
toolCalls?: TraceToolCall[];
|
||||
artifacts?: string[];
|
||||
@@ -158,6 +159,30 @@ const EXECUTION_TRACES_DDL: string[] = [
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_persona ON execution_traces (persona_id, outcome)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_outcome ON execution_traces (outcome, created_at DESC)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_traces_workspace ON execution_traces (workspace_id, created_at DESC)`,
|
||||
`CREATE TABLE IF NOT EXISTS execution_trace_spend (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
|
||||
cost_usd REAL NOT NULL CHECK (cost_usd > 0),
|
||||
settled_at TEXT NOT NULL
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_trace_spend_settled ON execution_trace_spend (settled_at, trace_id)`,
|
||||
`CREATE TABLE IF NOT EXISTS execution_trace_spend_reservations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
trace_id INTEGER NOT NULL REFERENCES execution_traces(id) ON DELETE CASCADE,
|
||||
estimated_cost_usd REAL NOT NULL CHECK (estimated_cost_usd > 0),
|
||||
actual_cost_usd REAL CHECK (actual_cost_usd IS NULL OR actual_cost_usd >= 0),
|
||||
state TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK (state IN ('pending', 'settled', 'released')),
|
||||
reserved_at TEXT NOT NULL,
|
||||
resolved_at TEXT,
|
||||
CHECK (
|
||||
(state = 'pending' AND actual_cost_usd IS NULL AND resolved_at IS NULL) OR
|
||||
(state = 'settled' AND actual_cost_usd IS NOT NULL AND resolved_at IS NOT NULL) OR
|
||||
(state = 'released' AND actual_cost_usd IS NULL AND resolved_at IS NOT NULL)
|
||||
)
|
||||
)`,
|
||||
`CREATE INDEX IF NOT EXISTS idx_trace_spend_reservations_time
|
||||
ON execution_trace_spend_reservations (reserved_at, trace_id, state)`,
|
||||
];
|
||||
|
||||
/** Exported DDL concatenated — kept for anyone who needs the full table SQL. */
|
||||
@@ -175,10 +200,6 @@ export class ExecutionTraceStore {
|
||||
private ensureTable(): void {
|
||||
try {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='execution_traces'",
|
||||
).get();
|
||||
if (exists) return;
|
||||
for (const stmt of EXECUTION_TRACES_DDL) {
|
||||
raw.prepare(stmt).run();
|
||||
}
|
||||
@@ -249,6 +270,141 @@ export class ExecutionTraceStore {
|
||||
.run(JSON.stringify(payload), id);
|
||||
}
|
||||
|
||||
/** Persist one settled model charge while a long-running trace is pending. */
|
||||
recordCost(id: number, costUsd: number, timestamp = new Date().toISOString()): void {
|
||||
if (!Number.isFinite(costUsd) || costUsd <= 0) {
|
||||
throw new RangeError('Trace cost entry must be a positive finite number');
|
||||
}
|
||||
const settledMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(settledMs)) {
|
||||
throw new RangeError('Trace cost timestamp must be a valid ISO date');
|
||||
}
|
||||
const settledAt = new Date(settledMs).toISOString();
|
||||
const raw = this.db.getDatabase();
|
||||
raw.transaction(() => {
|
||||
const updated = raw.prepare(`
|
||||
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
|
||||
`).run(costUsd, id);
|
||||
if (updated.changes === 0) return;
|
||||
raw.prepare(`
|
||||
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(id, costUsd, settledAt);
|
||||
})();
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist a conservative model-spend reservation before provider dispatch.
|
||||
* Pending reservations intentionally have no settled-spend row, so restart
|
||||
* recovery counts the estimate through the reservation ledger.
|
||||
*/
|
||||
reserveCost(
|
||||
id: number,
|
||||
estimatedCostUsd: number,
|
||||
timestamp = new Date().toISOString(),
|
||||
): number {
|
||||
if (!Number.isFinite(estimatedCostUsd) || estimatedCostUsd <= 0) {
|
||||
throw new RangeError('Trace cost reservation must be positive and finite');
|
||||
}
|
||||
const reservedMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(reservedMs)) {
|
||||
throw new RangeError('Trace cost reservation timestamp must be valid ISO date');
|
||||
}
|
||||
const result = this.db.getDatabase().prepare(`
|
||||
INSERT INTO execution_trace_spend_reservations
|
||||
(trace_id, estimated_cost_usd, state, reserved_at)
|
||||
SELECT id, ?, 'pending', ?
|
||||
FROM execution_traces
|
||||
WHERE id = ? AND outcome = 'pending'
|
||||
`).run(estimatedCostUsd, new Date(reservedMs).toISOString(), id);
|
||||
if (result.changes !== 1) {
|
||||
throw new Error(`Pending execution trace ${id} does not exist`);
|
||||
}
|
||||
return Number(result.lastInsertRowid);
|
||||
}
|
||||
|
||||
/** Replace a pending estimate with one authoritative settled charge. */
|
||||
settleReservedCost(
|
||||
reservationId: number,
|
||||
actualCostUsd: number,
|
||||
timestamp = new Date().toISOString(),
|
||||
): boolean {
|
||||
if (!Number.isFinite(actualCostUsd) || actualCostUsd < 0) {
|
||||
throw new RangeError('Settled trace cost must be non-negative and finite');
|
||||
}
|
||||
const settledMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(settledMs)) {
|
||||
throw new RangeError('Trace cost timestamp must be valid ISO date');
|
||||
}
|
||||
const settledAt = new Date(settledMs).toISOString();
|
||||
const raw = this.db.getDatabase();
|
||||
return raw.transaction(() => {
|
||||
const current = raw.prepare(`
|
||||
SELECT trace_id AS traceId, reserved_at AS reservedAt,
|
||||
state, actual_cost_usd AS actualCostUsd
|
||||
FROM execution_trace_spend_reservations
|
||||
WHERE id = ?
|
||||
`).get(reservationId) as {
|
||||
traceId: number;
|
||||
reservedAt: string;
|
||||
state: 'pending' | 'settled' | 'released';
|
||||
actualCostUsd: number | null;
|
||||
} | undefined;
|
||||
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
|
||||
if (current.state === 'settled' && current.actualCostUsd === actualCostUsd) return false;
|
||||
if (current.state !== 'pending') {
|
||||
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
|
||||
}
|
||||
const updated = raw.prepare(`
|
||||
UPDATE execution_trace_spend_reservations
|
||||
SET state = 'settled', actual_cost_usd = ?, resolved_at = ?
|
||||
WHERE id = ? AND state = 'pending'
|
||||
`).run(actualCostUsd, settledAt, reservationId);
|
||||
if (updated.changes !== 1) return false;
|
||||
if (actualCostUsd > 0) {
|
||||
raw.prepare(`
|
||||
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(current.traceId, actualCostUsd, current.reservedAt);
|
||||
raw.prepare(`
|
||||
UPDATE execution_traces SET cost_usd = cost_usd + ? WHERE id = ?
|
||||
`).run(actualCostUsd, current.traceId);
|
||||
}
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
/** Release a definitely pre-inference reservation without recording spend. */
|
||||
releaseReservedCost(reservationId: number, timestamp = new Date().toISOString()): boolean {
|
||||
const resolvedMs = Date.parse(timestamp);
|
||||
if (!Number.isFinite(resolvedMs)) {
|
||||
throw new RangeError('Trace cost timestamp must be valid ISO date');
|
||||
}
|
||||
const raw = this.db.getDatabase();
|
||||
return raw.transaction(() => {
|
||||
const current = raw.prepare(`
|
||||
SELECT id, state
|
||||
FROM execution_trace_spend_reservations
|
||||
WHERE id = ?
|
||||
`).get(reservationId) as {
|
||||
id: number;
|
||||
state: 'pending' | 'settled' | 'released';
|
||||
} | undefined;
|
||||
if (!current) throw new Error(`Cost reservation ${reservationId} does not exist`);
|
||||
if (current.state === 'released') return false;
|
||||
if (current.state !== 'pending') {
|
||||
throw new Error(`Cost reservation ${reservationId} is already ${current.state}`);
|
||||
}
|
||||
const result = raw.prepare(`
|
||||
UPDATE execution_trace_spend_reservations
|
||||
SET state = 'released', actual_cost_usd = NULL, resolved_at = ?
|
||||
WHERE id = ? AND state = 'pending'
|
||||
`).run(new Date(resolvedMs).toISOString(), reservationId);
|
||||
if (result.changes !== 1) return false;
|
||||
return true;
|
||||
})();
|
||||
}
|
||||
|
||||
/** Finalize a trace — set outcome, merge payload, record cost + duration. */
|
||||
finalize(id: number, input: FinalizeTraceInput): ExecutionTrace | undefined {
|
||||
const current = this.get(id);
|
||||
@@ -270,22 +426,42 @@ export class ExecutionTraceStore {
|
||||
const createdMs = Date.parse(current.created_at + 'Z');
|
||||
const now = Date.now();
|
||||
const durationMs = Number.isFinite(createdMs) ? Math.max(0, now - createdMs) : 0;
|
||||
const finalModel = input.model === undefined ? current.model : input.model;
|
||||
|
||||
this.db.getDatabase().prepare(`
|
||||
UPDATE execution_traces
|
||||
SET outcome = ?,
|
||||
trace_json = ?,
|
||||
cost_usd = ?,
|
||||
duration_ms = ?,
|
||||
finalized_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
input.outcome,
|
||||
JSON.stringify(merged),
|
||||
input.costUsd ?? current.cost_usd,
|
||||
durationMs,
|
||||
id,
|
||||
);
|
||||
const raw = this.db.getDatabase();
|
||||
raw.transaction(() => {
|
||||
const finalCost = Math.max(current.cost_usd, input.costUsd ?? current.cost_usd);
|
||||
if (input.costUsd !== undefined && finalCost > 0) {
|
||||
const ledger = raw.prepare(`
|
||||
SELECT COUNT(*) AS entries, COALESCE(SUM(cost_usd), 0) AS total
|
||||
FROM execution_trace_spend WHERE trace_id = ?
|
||||
`).get(id) as { entries: number; total: number | null };
|
||||
const missingCost = Math.max(0, finalCost - Number(ledger.total ?? 0));
|
||||
if (ledger.entries > 0 && missingCost > 0) {
|
||||
raw.prepare(`
|
||||
INSERT INTO execution_trace_spend (trace_id, cost_usd, settled_at)
|
||||
VALUES (?, ?, ?)
|
||||
`).run(id, missingCost, new Date().toISOString());
|
||||
}
|
||||
}
|
||||
raw.prepare(`
|
||||
UPDATE execution_traces
|
||||
SET outcome = ?,
|
||||
model = ?,
|
||||
trace_json = ?,
|
||||
cost_usd = ?,
|
||||
duration_ms = ?,
|
||||
finalized_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(
|
||||
input.outcome,
|
||||
finalModel,
|
||||
JSON.stringify(merged),
|
||||
finalCost,
|
||||
durationMs,
|
||||
id,
|
||||
);
|
||||
})();
|
||||
|
||||
return this.get(id);
|
||||
}
|
||||
@@ -320,6 +496,42 @@ export class ExecutionTraceStore {
|
||||
return row ? toParsed(row) : undefined;
|
||||
}
|
||||
|
||||
/** Highest trace id present when a consumer starts its process-local ledger. */
|
||||
getLatestId(): number {
|
||||
const row = this.db.getDatabase().prepare(
|
||||
'SELECT COALESCE(MAX(id), 0) AS id FROM execution_traces',
|
||||
).get() as { id: number | null };
|
||||
return Number(row.id ?? 0);
|
||||
}
|
||||
|
||||
/** Sum persisted model cost from a timestamp through an inclusive trace-id boundary. */
|
||||
getTotalCostSince(since: string, throughId: number = Number.MAX_SAFE_INTEGER): number {
|
||||
const settledSince = new Date(since).toISOString();
|
||||
const raw = this.db.getDatabase();
|
||||
const ledger = raw.prepare(`
|
||||
SELECT COALESCE(SUM(s.cost_usd), 0) AS total
|
||||
FROM execution_trace_spend s
|
||||
JOIN execution_traces t ON t.id = s.trace_id
|
||||
WHERE s.settled_at >= ? AND t.id <= ?
|
||||
`).get(settledSince, throughId) as { total: number | null };
|
||||
const pending = raw.prepare(`
|
||||
SELECT COALESCE(SUM(estimated_cost_usd), 0) AS total
|
||||
FROM execution_trace_spend_reservations
|
||||
WHERE state = 'pending' AND reserved_at >= ? AND trace_id <= ?
|
||||
`).get(settledSince, throughId) as { total: number | null };
|
||||
const legacy = raw.prepare(`
|
||||
SELECT COALESCE(SUM(t.cost_usd), 0) AS total
|
||||
FROM execution_traces t
|
||||
WHERE t.created_at >= datetime(?) AND t.id <= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM execution_trace_spend s WHERE s.trace_id = t.id
|
||||
)
|
||||
`).get(since, throughId) as { total: number | null };
|
||||
return Number(ledger.total ?? 0)
|
||||
+ Number(pending.total ?? 0)
|
||||
+ Number(legacy.total ?? 0);
|
||||
}
|
||||
|
||||
/** Query traces with optional filters. */
|
||||
query(filter: TraceQueryFilter = {}): ExecutionTrace[] {
|
||||
const clauses: string[] = [];
|
||||
|
||||
@@ -70,6 +70,22 @@ export class FrameStore {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a write unit atomically. Top-level callers acquire the write lock up
|
||||
* front and retry the whole closure on transient cross-process contention.
|
||||
* Nested callers use better-sqlite3's savepoint behavior and never retry an
|
||||
* inner closure against the same ambient snapshot.
|
||||
*/
|
||||
runInTransaction<T>(fn: () => T): T {
|
||||
if (typeof fn !== 'function') {
|
||||
throw new TypeError('FrameStore.runInTransaction requires a function');
|
||||
}
|
||||
const raw = this.db.getDatabase();
|
||||
const transaction = raw.transaction(fn);
|
||||
if (raw.inTransaction) return transaction();
|
||||
return this.db.runWithBusyRetry(() => transaction.immediate());
|
||||
}
|
||||
|
||||
createIFrame(
|
||||
gopId: string,
|
||||
content: string,
|
||||
@@ -154,6 +170,12 @@ export class FrameStore {
|
||||
return this.db.getDatabase().prepare('SELECT * FROM memory_frames WHERE id = ?').get(id) as MemoryFrame | undefined;
|
||||
}
|
||||
|
||||
hasSession(gopId: string): boolean {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT 1 FROM sessions WHERE gop_id = ? LIMIT 1',
|
||||
).get(gopId) !== undefined;
|
||||
}
|
||||
|
||||
getLatestIFrame(gopId: string): MemoryFrame | undefined {
|
||||
return this.db.getDatabase().prepare(`
|
||||
SELECT * FROM memory_frames
|
||||
@@ -287,6 +309,13 @@ export class FrameStore {
|
||||
if (!existing) return undefined;
|
||||
|
||||
const newImportance = importance ?? existing.importance;
|
||||
if (content === existing.content) {
|
||||
if (newImportance !== existing.importance) {
|
||||
raw.prepare('UPDATE memory_frames SET importance = ? WHERE id = ?')
|
||||
.run(newImportance, id);
|
||||
}
|
||||
return this.getById(id);
|
||||
}
|
||||
|
||||
// Update main table (content_hash maintained — oss-drift D3)
|
||||
raw.prepare(`
|
||||
|
||||
@@ -2,13 +2,14 @@
|
||||
* In-process embedder using @huggingface/transformers (ONNX Runtime).
|
||||
* Default provider for ALL desktop users — zero config, works offline.
|
||||
* Model: Xenova/all-MiniLM-L6-v2 (384 native dims, normalized to target dims).
|
||||
* Downloads ~23MB model on first use, cached in ~/.waggle/models/.
|
||||
* Downloads ~90MB fp32 model on first use, cached in ~/.waggle/models/.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import type { Embedder } from './embeddings.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { withTransformersModelLoad } from './transformers-model-load.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-embedder');
|
||||
|
||||
@@ -32,13 +33,18 @@ export async function createInProcessEmbedder(config?: Partial<InProcessEmbedder
|
||||
const cacheDir = config?.cacheDir ?? path.join(os.homedir(), '.waggle', 'models');
|
||||
const targetDims = config?.targetDimensions ?? 1024;
|
||||
|
||||
log.info(`Loading in-process embedding model: ${model} (~23MB first download)`);
|
||||
log.info(`Loading in-process embedding model: ${model} (~90MB fp32 first download)`);
|
||||
|
||||
const { pipeline, env } = await import('@huggingface/transformers');
|
||||
env.cacheDir = cacheDir;
|
||||
env.allowRemoteModels = true;
|
||||
|
||||
const extractor = await pipeline('feature-extraction', model, { dtype: 'fp32' });
|
||||
const { pipeline } = await import('@huggingface/transformers');
|
||||
const extractor = await withTransformersModelLoad({
|
||||
cacheDir,
|
||||
model,
|
||||
load: (canonicalCacheDir) => pipeline('feature-extraction', model, {
|
||||
dtype: 'fp32',
|
||||
cache_dir: canonicalCacheDir,
|
||||
}),
|
||||
onQuarantine: () => log.warn(`Quarantined corrupt embedding model cache: ${model}`),
|
||||
});
|
||||
const nativeDims = 384; // all-MiniLM-L6-v2 output dimensions
|
||||
|
||||
log.info(`In-process embedder ready (${nativeDims} native dims → ${targetDims} normalized)`);
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { withTransformersModelLoad } from './transformers-model-load.js';
|
||||
|
||||
const log = createCoreLogger('inprocess-reranker');
|
||||
|
||||
@@ -56,18 +57,25 @@ export async function createInProcessReranker(
|
||||
|
||||
log.info(`Loading in-process reranker: ${model} (~22MB first download)`);
|
||||
|
||||
const { AutoTokenizer, AutoModelForSequenceClassification, env } = await import(
|
||||
const { AutoTokenizer, AutoModelForSequenceClassification } = await import(
|
||||
'@huggingface/transformers'
|
||||
);
|
||||
env.cacheDir = cacheDir;
|
||||
env.allowRemoteModels = true;
|
||||
|
||||
// Cross-encoders need direct tokenizer + model access — pipeline API
|
||||
// doesn't expose the (text, text_pair) input pattern cleanly across
|
||||
// all transformers.js versions. Calling the model directly with
|
||||
// tokenized pairs is the stable path.
|
||||
const tokenizer = await AutoTokenizer.from_pretrained(model);
|
||||
const seqModel = await AutoModelForSequenceClassification.from_pretrained(model, { dtype: 'fp32' });
|
||||
const { tokenizer, seqModel } = await withTransformersModelLoad({
|
||||
cacheDir,
|
||||
model,
|
||||
load: async (canonicalCacheDir) => ({
|
||||
// Cross-encoders need direct tokenizer + model access — pipeline API
|
||||
// doesn't expose the (text, text_pair) input pattern cleanly across
|
||||
// all transformers.js versions. Calling the model directly with
|
||||
// tokenized pairs is the stable path.
|
||||
tokenizer: await AutoTokenizer.from_pretrained(model, { cache_dir: canonicalCacheDir }),
|
||||
seqModel: await AutoModelForSequenceClassification.from_pretrained(model, {
|
||||
dtype: 'fp32',
|
||||
cache_dir: canonicalCacheDir,
|
||||
}),
|
||||
}),
|
||||
onQuarantine: () => log.warn(`Quarantined corrupt reranker model cache: ${model}`),
|
||||
});
|
||||
|
||||
log.info(`In-process reranker ready: ${model}`);
|
||||
|
||||
@@ -77,7 +85,7 @@ export async function createInProcessReranker(
|
||||
text_pair: doc,
|
||||
padding: true,
|
||||
truncation: true,
|
||||
return_tensors: 'pt',
|
||||
return_tensor: true,
|
||||
});
|
||||
const out = await seqModel(inputs);
|
||||
// ms-marco-MiniLM outputs a single logit per pair (1-class regression).
|
||||
@@ -105,7 +113,7 @@ export async function createInProcessReranker(
|
||||
text_pair: docs,
|
||||
padding: true,
|
||||
truncation: true,
|
||||
return_tensors: 'pt',
|
||||
return_tensor: true,
|
||||
});
|
||||
const out = await seqModel(inputs);
|
||||
const logits = out.logits ?? out[0];
|
||||
|
||||
@@ -59,6 +59,17 @@ export class KnowledgeGraph {
|
||||
this.db = db;
|
||||
}
|
||||
|
||||
/** Run a graph write unit atomically, nesting as a savepoint when needed. */
|
||||
runInTransaction<T>(fn: () => T): T {
|
||||
if (typeof fn !== 'function') {
|
||||
throw new TypeError('KnowledgeGraph.runInTransaction requires a function');
|
||||
}
|
||||
const raw = this.db.getDatabase();
|
||||
const transaction = raw.transaction(fn);
|
||||
if (raw.inTransaction) return transaction();
|
||||
return this.db.runWithBusyRetry(() => transaction.immediate());
|
||||
}
|
||||
|
||||
setValidationSchema(schema: ValidationSchema): void {
|
||||
this.schema = schema;
|
||||
}
|
||||
@@ -381,11 +392,16 @@ export class KnowledgeGraph {
|
||||
/** Link an entity to a frame it was extracted from (kg_entity_frames bridge).
|
||||
* Powers the 'contextual' scoring signal. Idempotent per (entity, frame). */
|
||||
linkEntityToFrame(entityId: number, frameId: number): void {
|
||||
try {
|
||||
this.db.getDatabase().prepare(
|
||||
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
|
||||
).run(entityId, frameId);
|
||||
} catch { /* bridge table absent on a pre-migration DB — best-effort */ }
|
||||
try { this.linkEntityToFrameStrict(entityId, frameId); } catch {
|
||||
/* bridge table absent on a pre-migration DB — best-effort */
|
||||
}
|
||||
}
|
||||
|
||||
/** Strict provenance link for atomic writers; true only for a new link. */
|
||||
linkEntityToFrameStrict(entityId: number, frameId: number): boolean {
|
||||
return this.db.getDatabase().prepare(
|
||||
'INSERT OR IGNORE INTO kg_entity_frames (entity_id, frame_id) VALUES (?, ?)'
|
||||
).run(entityId, frameId).changes === 1;
|
||||
}
|
||||
|
||||
/** Seed entities whose name appears in free text (case-insensitive, name ≥3
|
||||
|
||||
@@ -123,7 +123,7 @@ export async function fetchRawDetailLane(
|
||||
const excludeIds = opts.excludeIds ?? new Set<number>();
|
||||
|
||||
// ── Pool ──────────────────────────────────────────────────────────────
|
||||
let pool: FrameRow[] = [];
|
||||
let pool: FrameRow[];
|
||||
if (opts.window) {
|
||||
pool = windowPool(db, opts.window.since, opts.window.until);
|
||||
if (pool.length > WINDOW_POOL_MAX) {
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { Embedder } from './embeddings.js';
|
||||
import type { MemoryFrame, Importance } from './frames.js';
|
||||
import type { Reranker } from './inprocess-reranker.js';
|
||||
import { chunkText, type ChunkOptions } from './chunker.js';
|
||||
import { buildFtsOrQuery, hasUnsegmentedScript, sanitizeFtsToken } from './fts-sanitize.js';
|
||||
import { buildFtsOrQuery, FTS_STOP_WORDS, hasUnsegmentedScript } from './fts-sanitize.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import {
|
||||
computeRelevance,
|
||||
@@ -86,6 +86,7 @@ export function assessRetrievalConfidence(
|
||||
}
|
||||
|
||||
const RRF_K = 60;
|
||||
const MAX_PUNCTUATED_FTS_TERMS = 16;
|
||||
|
||||
const log = createCoreLogger('hybrid-search');
|
||||
|
||||
@@ -121,6 +122,27 @@ function escapeLikeTerm(term: string): string {
|
||||
return term.replace(/[\\%_]/g, ch => `\\${ch}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a strict fallback MATCH expression from punctuation-delimited terms.
|
||||
* Unlike the primary recall-oriented OR query, every surviving term is
|
||||
* required. Refuse overlong expressions instead of truncating them into a
|
||||
* broader query.
|
||||
*/
|
||||
function buildPunctuatedFtsAndQuery(query: string, minimumTerms = 2): string {
|
||||
const tokens = query.match(/[\p{L}\p{N}_]+/gu) ?? [];
|
||||
const terms = tokens.filter((token) => (
|
||||
token.length > 2
|
||||
&& !FTS_STOP_WORDS.has(token.toLowerCase())
|
||||
&& !hasUnsegmentedScript(token)
|
||||
));
|
||||
|
||||
if (terms.length < minimumTerms || terms.length > MAX_PUNCTUATED_FTS_TERMS) return '';
|
||||
|
||||
const uniqueTerms = [...new Set(terms)];
|
||||
if (uniqueTerms.length < minimumTerms) return '';
|
||||
return uniqueTerms.map(term => `"${term}"`).join(' AND ');
|
||||
}
|
||||
|
||||
export class HybridSearch {
|
||||
private db: MindDB;
|
||||
private embedder: Embedder;
|
||||
@@ -180,13 +202,13 @@ export class HybridSearch {
|
||||
// Flag off → chunkResults is null without touching the chunk tables,
|
||||
// so the lane below is byte-identical to pre-D1.
|
||||
const chunkResults = chunkRetrievalEnabled()
|
||||
? await this.vectorSearchChunks(query, laneFetch, gopId)
|
||||
? await this.vectorSearchChunks(query, laneFetch, gopId, options.excludeDeprecated)
|
||||
: null;
|
||||
const [keywordResults, vectorResults] = await Promise.all([
|
||||
this.keywordSearch(query, laneFetch, gopId),
|
||||
this.keywordSearch(query, laneFetch, gopId, options.excludeDeprecated),
|
||||
chunkResults !== null
|
||||
? Promise.resolve(chunkResults)
|
||||
: this.vectorSearch(query, laneFetch, gopId),
|
||||
: this.vectorSearch(query, laneFetch, gopId, options.excludeDeprecated),
|
||||
]);
|
||||
|
||||
// RRF fusion
|
||||
@@ -321,7 +343,12 @@ export class HybridSearch {
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
async keywordSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
|
||||
async keywordSearch(
|
||||
query: string,
|
||||
limit: number,
|
||||
gopId?: string,
|
||||
excludeDeprecated = false,
|
||||
): Promise<number[]> {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
// W3.6: Sanitize query for FTS5 with OR-based matching for better recall.
|
||||
@@ -339,21 +366,29 @@ export class HybridSearch {
|
||||
// unsegmented script (CJK) the emptiness is a sanitizer artifact, not a
|
||||
// lack of signal: unicode61 cannot token-match CJK prose, but LIKE
|
||||
// substring matching can, so route those to the fallback lane.
|
||||
return hasUnsegmentedScript(query) ? this.likeFallbackSearch(query, limit, gopId) : [];
|
||||
return hasUnsegmentedScript(query)
|
||||
? this.likeFallbackSearch(query, limit, gopId, excludeDeprecated)
|
||||
: [];
|
||||
}
|
||||
|
||||
let sql: string;
|
||||
let params: unknown[];
|
||||
|
||||
if (gopId) {
|
||||
sql = `
|
||||
SELECT mf.id FROM memory_frames_fts fts
|
||||
JOIN memory_frames mf ON mf.id = fts.rowid
|
||||
WHERE fts.content MATCH ? AND mf.gop_id = ?
|
||||
ORDER BY rank
|
||||
WHERE fts.content MATCH ? AND mf.gop_id = ?${excludeDeprecated ? " AND mf.importance != 'deprecated'" : ''}
|
||||
ORDER BY fts.rank
|
||||
LIMIT ?
|
||||
`;
|
||||
} else if (excludeDeprecated) {
|
||||
sql = `
|
||||
SELECT mf.id FROM memory_frames_fts fts
|
||||
JOIN memory_frames mf ON mf.id = fts.rowid
|
||||
WHERE fts.content MATCH ? AND mf.importance != 'deprecated'
|
||||
ORDER BY fts.rank
|
||||
LIMIT ?
|
||||
`;
|
||||
params = [safeQuery, gopId, limit];
|
||||
} else {
|
||||
sql = `
|
||||
SELECT rowid as id FROM memory_frames_fts
|
||||
@@ -361,68 +396,112 @@ export class HybridSearch {
|
||||
ORDER BY rank
|
||||
LIMIT ?
|
||||
`;
|
||||
params = [safeQuery, limit];
|
||||
}
|
||||
|
||||
try {
|
||||
const runMatch = (matchQuery: string): number[] => {
|
||||
const params = gopId
|
||||
? [matchQuery, gopId, limit]
|
||||
: [matchQuery, limit];
|
||||
const rows = raw.prepare(sql).all(...params) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
return rows.map(row => row.id);
|
||||
};
|
||||
|
||||
const runPunctuationFallback = (allowSingleTerm = false): number[] => {
|
||||
// Preserve the complete identifier first. This is the most precise lane
|
||||
// and the only safe behavior when the token count exceeds the FTS bound.
|
||||
const literalIds = this.likeFallbackSearch(query, limit, gopId, excludeDeprecated);
|
||||
if (literalIds.length > 0) return literalIds;
|
||||
|
||||
// SQLite LIKE only case-folds ASCII. A strict unicode61 MATCH over every
|
||||
// punctuation-delimited term supplies Unicode case-insensitive recall
|
||||
// without broad OR matches.
|
||||
const boundaryQuery = buildPunctuatedFtsAndQuery(query, allowSingleTerm ? 1 : 2);
|
||||
if (!boundaryQuery) return [];
|
||||
try {
|
||||
return runMatch(boundaryQuery);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const ids = runMatch(safeQuery);
|
||||
if (ids.length === 0 && /[^\p{L}\p{N}_\s]/u.test(query)) {
|
||||
return runPunctuationFallback();
|
||||
}
|
||||
return ids;
|
||||
} catch {
|
||||
// FTS5 parse error (e.g. user query with FTS5-special chars that survived
|
||||
// sanitization) — fall back to a LIKE keyword scan over the same column so
|
||||
// we return best-effort matches instead of a false "no memory found".
|
||||
return this.likeFallbackSearch(query, limit, gopId);
|
||||
// FTS5 parse error (for example, an unmatched quote): retry through the
|
||||
// same precise literal-plus-strict-boundary fallback used for zero hits.
|
||||
return runPunctuationFallback(true);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* LIKE-based keyword fallback over memory_frames.content. Used when the FTS5
|
||||
* MATCH query throws a parse error (e.g. an unbalanced quote or other FTS5
|
||||
* operator the user typed literally). The raw query is split into word tokens
|
||||
* — stripping the punctuation that caused the FTS5 error, mirroring the
|
||||
* primary sanitizer — and matched with OR-ed LIKE clauses for best-effort
|
||||
* recall. Bound parameters only (the term is never interpolated) and LIKE
|
||||
* metachars (`%`, `_`, `\`) are escaped with an ESCAPE clause so each token
|
||||
* matches literally. If no usable token survives, a single literal LIKE over
|
||||
* the whole escaped query is used.
|
||||
* Whole-query LIKE fallback over memory_frames.content. Bound parameters only
|
||||
* (the term is never interpolated), with LIKE metachars (`%`, `_`, `\`)
|
||||
* escaped so punctuation-delimited identifiers stay literal. Unicode
|
||||
* case-insensitive fallback is handled separately by strict unicode61 FTS.
|
||||
*/
|
||||
private likeFallbackSearch(query: string, limit: number, gopId?: string): number[] {
|
||||
private likeFallbackSearch(
|
||||
query: string,
|
||||
limit: number,
|
||||
gopId?: string,
|
||||
excludeDeprecated = false,
|
||||
): number[] {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
const tokens = query
|
||||
.split(/\s+/)
|
||||
.map(sanitizeFtsToken) // strip punctuation (incl. FTS5 operators), Unicode-aware
|
||||
.filter(w => w.length > 0);
|
||||
const terms = (tokens.length > 0 ? tokens : [query]).map(t => `%${escapeLikeTerm(t)}%`);
|
||||
|
||||
const likeClause = terms.map(() => `content LIKE ? ESCAPE '\\'`).join(' OR ');
|
||||
const term = `%${escapeLikeTerm(query)}%`;
|
||||
const deprecatedFilter = excludeDeprecated ? " AND importance != 'deprecated'" : '';
|
||||
|
||||
try {
|
||||
if (gopId) {
|
||||
const rows = raw.prepare(
|
||||
`SELECT id FROM memory_frames
|
||||
WHERE (${likeClause}) AND gop_id = ?
|
||||
WHERE content LIKE ? ESCAPE '\\' AND gop_id = ?${deprecatedFilter}
|
||||
ORDER BY created_at DESC LIMIT ?`
|
||||
).all(...terms, gopId, limit) as { id: number }[];
|
||||
).all(term, gopId, limit) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
}
|
||||
const rows = raw.prepare(
|
||||
`SELECT id FROM memory_frames
|
||||
WHERE (${likeClause})
|
||||
WHERE content LIKE ? ESCAPE '\\'${deprecatedFilter}
|
||||
ORDER BY created_at DESC LIMIT ?`
|
||||
).all(...terms, limit) as { id: number }[];
|
||||
).all(term, limit) as { id: number }[];
|
||||
return rows.map(r => r.id);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async vectorSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
|
||||
async vectorSearch(
|
||||
query: string,
|
||||
limit: number,
|
||||
gopId?: string,
|
||||
excludeDeprecated = false,
|
||||
): Promise<number[]> {
|
||||
this.ensureFingerprint();
|
||||
const embedding = await this.embedder.embed(query);
|
||||
const blob = f32ToBlob(embedding);
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
if (excludeDeprecated) {
|
||||
const gopFilter = gopId ? ' AND gop_id = ?' : '';
|
||||
try {
|
||||
const rows = raw.prepare(`
|
||||
SELECT rowid as id FROM memory_frames_vec
|
||||
WHERE embedding MATCH ? AND k = ?
|
||||
AND rowid IN (
|
||||
SELECT id FROM memory_frames
|
||||
WHERE importance != 'deprecated'${gopFilter}
|
||||
)
|
||||
ORDER BY distance
|
||||
`).all(blob, limit, ...(gopId ? [gopId] : [])) as { id: number }[];
|
||||
return rows.map((row) => row.id);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
if (gopId) {
|
||||
// Two-step: get candidates from vec, then filter by GOP
|
||||
try {
|
||||
@@ -597,7 +676,12 @@ export class HybridSearch {
|
||||
* chunk index is empty (or the tables are missing), returns null so callers
|
||||
* can cleanly fall back to the whole-frame vectorSearch path.
|
||||
*/
|
||||
async vectorSearchChunks(query: string, limit: number, gopId?: string): Promise<number[] | null> {
|
||||
async vectorSearchChunks(
|
||||
query: string,
|
||||
limit: number,
|
||||
gopId?: string,
|
||||
excludeDeprecated = false,
|
||||
): Promise<number[] | null> {
|
||||
this.ensureFingerprint();
|
||||
const raw = this.db.getDatabase();
|
||||
// Cheap probe — avoid embedding the query when chunks aren't populated.
|
||||
@@ -618,15 +702,28 @@ export class HybridSearch {
|
||||
// Over-fetch chunks (limit * 5) so dedup-to-frame still leaves enough
|
||||
// candidates after collapsing multiple chunks of the same frame.
|
||||
try {
|
||||
const gopFilter = gopId ? ' AND mf.gop_id = ?' : '';
|
||||
const candidateFilter = excludeDeprecated
|
||||
? ` AND v.rowid IN (
|
||||
SELECT c2.id
|
||||
FROM memory_frame_chunks c2
|
||||
JOIN memory_frames mf ON mf.id = c2.frame_id
|
||||
WHERE mf.importance != 'deprecated'${gopFilter}
|
||||
)`
|
||||
: '';
|
||||
const chunkRows = raw
|
||||
.prepare(
|
||||
`SELECT v.rowid AS chunk_id, c.frame_id
|
||||
FROM memory_frame_chunks_vec v
|
||||
JOIN memory_frame_chunks c ON c.id = v.rowid
|
||||
WHERE v.embedding MATCH ? AND k = ?
|
||||
WHERE v.embedding MATCH ? AND k = ?${candidateFilter}
|
||||
ORDER BY distance`
|
||||
)
|
||||
.all(blob, Math.max(limit * 5, 25)) as Array<{ chunk_id: number; frame_id: number }>;
|
||||
.all(
|
||||
blob,
|
||||
Math.max(limit * 5, 25),
|
||||
...(excludeDeprecated && gopId ? [gopId] : []),
|
||||
) as Array<{ chunk_id: number; frame_id: number }>;
|
||||
|
||||
if (chunkRows.length === 0) return [];
|
||||
|
||||
@@ -640,7 +737,7 @@ export class HybridSearch {
|
||||
if (frameIds.length >= limit) break;
|
||||
}
|
||||
|
||||
if (gopId) {
|
||||
if (gopId && !excludeDeprecated) {
|
||||
const placeholders = frameIds.map(() => '?').join(',');
|
||||
const filtered = raw
|
||||
.prepare(
|
||||
|
||||
@@ -51,6 +51,19 @@
|
||||
|
||||
import type { MindDB } from './db.js';
|
||||
import type { FrameStore, FrameSource, MemoryFrame } from './frames.js';
|
||||
import { evaluateExternalMemoryIngress } from '../memory-ingress-guard.js';
|
||||
|
||||
export const MAX_CONSOLIDATION_OBSERVATIONS = 400;
|
||||
const MAX_PROMPT_CHARS = 100_000;
|
||||
const MAX_RESPONSE_CHARS = 100_000;
|
||||
const MAX_LABEL_CHARS = 256;
|
||||
const MAX_CURRENT_VALUE_CHARS = 4_000;
|
||||
const CREATED_AT_SORT_EXPR = `julianday(CASE
|
||||
WHEN created_at GLOB '*[+-][0-9][0-9][0-9][0-9]'
|
||||
THEN substr(created_at, 1, length(created_at) - 5)
|
||||
|| substr(created_at, -5, 3) || ':' || substr(created_at, -2)
|
||||
ELSE created_at
|
||||
END)`;
|
||||
|
||||
/**
|
||||
* LLM callback the consolidation passes inject. Given a system + user message,
|
||||
@@ -98,7 +111,7 @@ export interface ConsolidationResult {
|
||||
export interface CollectObservationsOptions {
|
||||
/** Scope to a single GOP session; omit for the whole mind. */
|
||||
gopId?: string;
|
||||
/** Cap the number of observations (keeps the LLM prompt bounded). */
|
||||
/** Select the latest N eligible observations, returned chronologically. */
|
||||
limit?: number;
|
||||
/**
|
||||
* Which frame source to include. Defaults to 'agent_inferred' (the
|
||||
@@ -124,19 +137,26 @@ const GROUP_SYSTEM =
|
||||
* an empty object when nothing parses — the callers treat "no intents" as a
|
||||
* valid, non-fatal outcome rather than throwing on model chatter.
|
||||
*/
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function parseLlmJson(raw: string): Record<string, unknown> {
|
||||
if (typeof raw !== 'string') return {};
|
||||
if (raw.length > MAX_RESPONSE_CHARS) return {};
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return {};
|
||||
try {
|
||||
return JSON.parse(trimmed) as Record<string, unknown>;
|
||||
const parsed: unknown = JSON.parse(trimmed);
|
||||
return isRecord(parsed) ? parsed : {};
|
||||
} catch {
|
||||
// Not a bare JSON document — try to recover an embedded object below.
|
||||
}
|
||||
const match = trimmed.match(/\{[\s\S]*\}/);
|
||||
if (match) {
|
||||
try {
|
||||
return JSON.parse(match[0]) as Record<string, unknown>;
|
||||
const parsed: unknown = JSON.parse(match[0]);
|
||||
return isRecord(parsed) ? parsed : {};
|
||||
} catch {
|
||||
// Embedded block was also malformed — fall through to the empty result.
|
||||
}
|
||||
@@ -154,20 +174,83 @@ function assertObservations(observations: unknown): asserts observations is Obse
|
||||
throw new TypeError('consolidate: each observation must be an object');
|
||||
}
|
||||
const rec = o as Record<string, unknown>;
|
||||
if (!Number.isInteger(rec.id)) {
|
||||
throw new TypeError('consolidate: observation.id must be an integer');
|
||||
if (!Number.isSafeInteger(rec.id) || Number(rec.id) <= 0) {
|
||||
throw new TypeError('consolidate: observation.id must be a positive safe integer');
|
||||
}
|
||||
if (typeof rec.content !== 'string') {
|
||||
throw new TypeError('consolidate: observation.content must be a string');
|
||||
}
|
||||
if (typeof rec.created_at !== 'string') {
|
||||
throw new TypeError('consolidate: observation.created_at must be a string');
|
||||
}
|
||||
if (!Number.isFinite(observationTime(rec.created_at))) {
|
||||
throw new TypeError('consolidate: observation.created_at must be a valid timestamp');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Render the observations as a `N. [YYYY-MM-DD] content` numbered list. */
|
||||
function numberObservations(observations: Observation[]): string {
|
||||
return observations
|
||||
.map((o, idx) => `${idx + 1}. [${String(o.created_at ?? '').slice(0, 10)}] ${o.content}`)
|
||||
.join('\n');
|
||||
const SQLITE_TIMESTAMP_RE =
|
||||
/^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})(?:\.(\d+))?(?:Z|([+-])(\d{2}):?(\d{2}))?$/;
|
||||
|
||||
function observationTime(value: string): number {
|
||||
const match = SQLITE_TIMESTAMP_RE.exec(value.trim());
|
||||
if (!match) return Number.NaN;
|
||||
|
||||
const [, date, time, fraction = '', sign, hour, minute] = match;
|
||||
if (hour && (Number(hour) > 14 || Number(minute) > 59)) return Number.NaN;
|
||||
|
||||
const firstThree = Number((fraction + '000').slice(0, 3));
|
||||
const millis = Math.min(999, firstThree + ((fraction[3] ?? '0') >= '5' ? 1 : 0));
|
||||
const zone = sign ? `${sign}${hour}:${minute}` : 'Z';
|
||||
return Date.parse(`${date}T${time}.${String(millis).padStart(3, '0')}${zone}`);
|
||||
}
|
||||
|
||||
function normalizeObservations(observations: Observation[]): Observation[] {
|
||||
const sorted = [...observations].sort((a, b) => {
|
||||
const timeDelta = observationTime(a.created_at) - observationTime(b.created_at);
|
||||
return timeDelta || a.id - b.id;
|
||||
});
|
||||
const seen = new Set<number>();
|
||||
return sorted.filter(({ id }) => {
|
||||
if (seen.has(id)) return false;
|
||||
seen.add(id);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function prepareDetectionPrompt(
|
||||
observations: Observation[],
|
||||
system: string,
|
||||
operation: string,
|
||||
): { normalized: Observation[]; user: string } {
|
||||
assertObservations(observations);
|
||||
if (observations.length > MAX_CONSOLIDATION_OBSERVATIONS) {
|
||||
throw new RangeError(
|
||||
`${operation}: at most ${MAX_CONSOLIDATION_OBSERVATIONS} observations are allowed`,
|
||||
);
|
||||
}
|
||||
const normalized = normalizeObservations(observations);
|
||||
const lines: string[] = [];
|
||||
let userChars = 0;
|
||||
for (let index = 0; index < normalized.length; index += 1) {
|
||||
const observation = normalized[index];
|
||||
const prefix = `${index + 1}. [${observation.created_at.slice(0, 10)}] `;
|
||||
const addedChars = (index > 0 ? 1 : 0) + prefix.length + observation.content.length;
|
||||
if (system.length + userChars + addedChars > MAX_PROMPT_CHARS) {
|
||||
throw new RangeError(`${operation}: prompt exceeds ${MAX_PROMPT_CHARS} characters`);
|
||||
}
|
||||
lines.push(prefix + observation.content);
|
||||
userChars += addedChars;
|
||||
}
|
||||
return { normalized, user: lines.join('\n') };
|
||||
}
|
||||
|
||||
function safeModelText(value: unknown, maxChars: number): string | null {
|
||||
if (typeof value !== 'string') return '';
|
||||
const text = value.trim();
|
||||
if (text.length > maxChars) return null;
|
||||
if (text && evaluateExternalMemoryIngress({ content: text }).action !== 'allow') return null;
|
||||
return text;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -186,8 +269,9 @@ function mapNumbersToFrameIds(
|
||||
const indices: number[] = [];
|
||||
const seen = new Set<number>();
|
||||
for (const n of numbers) {
|
||||
const idx = Number(n);
|
||||
if (!Number.isInteger(idx) || idx < 1 || idx > observations.length) continue;
|
||||
if (typeof n !== 'number' || !Number.isSafeInteger(n)) continue;
|
||||
const idx = n;
|
||||
if (idx < 1 || idx > observations.length) continue;
|
||||
if (seen.has(idx)) continue;
|
||||
seen.add(idx);
|
||||
indices.push(idx);
|
||||
@@ -208,13 +292,17 @@ export async function detectSupersessionChains(
|
||||
observations: Observation[],
|
||||
llm: ConsolidationLlm,
|
||||
): Promise<SupersessionChain[]> {
|
||||
assertObservations(observations);
|
||||
if (typeof llm !== 'function') {
|
||||
throw new TypeError('detectSupersessionChains: llm must be a function');
|
||||
}
|
||||
if (observations.length < 2) return [];
|
||||
const { normalized, user } = prepareDetectionPrompt(
|
||||
observations,
|
||||
SUPERSESSION_SYSTEM,
|
||||
'detectSupersessionChains',
|
||||
);
|
||||
if (normalized.length < 2) return [];
|
||||
|
||||
const raw = await llm(SUPERSESSION_SYSTEM, numberObservations(observations));
|
||||
const raw = await llm(SUPERSESSION_SYSTEM, user);
|
||||
const parsed = parseLlmJson(raw);
|
||||
const rawChains = Array.isArray(parsed.chains) ? parsed.chains : [];
|
||||
|
||||
@@ -222,10 +310,11 @@ export async function detectSupersessionChains(
|
||||
for (const entry of rawChains) {
|
||||
if (!entry || typeof entry !== 'object') continue;
|
||||
const rec = entry as Record<string, unknown>;
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, observations, true);
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, normalized, true);
|
||||
if (frameIds.length < 2) continue;
|
||||
const attribute = typeof rec.attribute === 'string' ? rec.attribute.trim() : '';
|
||||
const currentValue = typeof rec.current_value === 'string' ? rec.current_value.trim() : '';
|
||||
const attribute = safeModelText(rec.attribute, MAX_LABEL_CHARS);
|
||||
const currentValue = safeModelText(rec.current_value, MAX_CURRENT_VALUE_CHARS);
|
||||
if (attribute === null || currentValue === null) continue;
|
||||
chains.push({ attribute: attribute || 'value', currentValue, frameIds });
|
||||
}
|
||||
return chains;
|
||||
@@ -240,13 +329,17 @@ export async function detectEntityGroups(
|
||||
observations: Observation[],
|
||||
llm: ConsolidationLlm,
|
||||
): Promise<EntityGroup[]> {
|
||||
assertObservations(observations);
|
||||
if (typeof llm !== 'function') {
|
||||
throw new TypeError('detectEntityGroups: llm must be a function');
|
||||
}
|
||||
if (observations.length < 2) return [];
|
||||
const { normalized, user } = prepareDetectionPrompt(
|
||||
observations,
|
||||
GROUP_SYSTEM,
|
||||
'detectEntityGroups',
|
||||
);
|
||||
if (normalized.length < 2) return [];
|
||||
|
||||
const raw = await llm(GROUP_SYSTEM, numberObservations(observations));
|
||||
const raw = await llm(GROUP_SYSTEM, user);
|
||||
const parsed = parseLlmJson(raw);
|
||||
const rawGroups = Array.isArray(parsed.groups) ? parsed.groups : [];
|
||||
|
||||
@@ -254,9 +347,10 @@ export async function detectEntityGroups(
|
||||
for (const entry of rawGroups) {
|
||||
if (!entry || typeof entry !== 'object') continue;
|
||||
const rec = entry as Record<string, unknown>;
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, observations, false);
|
||||
const frameIds = mapNumbersToFrameIds(rec.ids, normalized, false);
|
||||
if (frameIds.length < 2) continue;
|
||||
const label = typeof rec.label === 'string' ? rec.label.trim() : '';
|
||||
const label = safeModelText(rec.label, MAX_LABEL_CHARS);
|
||||
if (label === null) continue;
|
||||
groups.push({ label: label || 'group', frameIds });
|
||||
}
|
||||
return groups;
|
||||
@@ -284,54 +378,201 @@ export function applyConsolidation(
|
||||
groups: EntityGroup[],
|
||||
gopId: string,
|
||||
): ConsolidationResult {
|
||||
if (!frames || typeof frames.createPFrame !== 'function') {
|
||||
if (
|
||||
!frames
|
||||
|| typeof frames.createPFrame !== 'function'
|
||||
|| typeof frames.runInTransaction !== 'function'
|
||||
) {
|
||||
throw new TypeError('applyConsolidation: frames must be a FrameStore');
|
||||
}
|
||||
if (typeof gopId !== 'string' || !gopId) {
|
||||
throw new Error('applyConsolidation: gopId is required');
|
||||
}
|
||||
|
||||
const pframes: MemoryFrame[] = [];
|
||||
const bframes: MemoryFrame[] = [];
|
||||
const deprecated: number[] = [];
|
||||
|
||||
for (const chain of chains ?? []) {
|
||||
const ids = chain.frameIds;
|
||||
if (!Array.isArray(ids) || ids.length < 2) continue;
|
||||
|
||||
const baseId = ids[0];
|
||||
const newestId = ids[ids.length - 1];
|
||||
const newest = frames.getById(newestId);
|
||||
if (!newest) continue; // newest member gone — cannot anchor a current value
|
||||
|
||||
// Deprecate every stale member (all but the newest).
|
||||
for (const staleId of ids.slice(0, -1)) {
|
||||
const stale = frames.getById(staleId);
|
||||
if (!stale) continue;
|
||||
frames.update(staleId, stale.content, 'deprecated');
|
||||
deprecated.push(staleId);
|
||||
return frames.runInTransaction(() => {
|
||||
if (!Array.isArray(chains) || chains.length > MAX_CONSOLIDATION_OBSERVATIONS) {
|
||||
throw new TypeError('applyConsolidation: chains must be a bounded array');
|
||||
}
|
||||
if (!Array.isArray(groups) || groups.length > MAX_CONSOLIDATION_OBSERVATIONS) {
|
||||
throw new TypeError('applyConsolidation: groups must be a bounded array');
|
||||
}
|
||||
if (!frames.hasSession(gopId)) {
|
||||
throw new Error(`applyConsolidation: destination session does not exist: ${gopId}`);
|
||||
}
|
||||
// Boost the surviving newest so it wins recall ties.
|
||||
frames.update(newestId, newest.content, 'critical');
|
||||
|
||||
// Emit the current-value P-frame (base = oldest), preferring the model's
|
||||
// clean value and falling back to the newest frame's raw content.
|
||||
const cleanValue = chain.currentValue.trim() ? chain.currentValue.trim() : newest.content;
|
||||
const attribute = chain.attribute.trim() ? chain.attribute.trim() : 'value';
|
||||
const asOf = String(newest.created_at).slice(0, 10);
|
||||
const pContent = `[current] ${attribute}: ${cleanValue} (as of ${asOf})`;
|
||||
pframes.push(frames.createPFrame(gopId, pContent, baseId, 'critical', 'agent_inferred'));
|
||||
}
|
||||
const frameCache = new Map<number, MemoryFrame>();
|
||||
const requireFrame = (id: number): MemoryFrame => {
|
||||
const cached = frameCache.get(id);
|
||||
if (cached) return cached;
|
||||
const frame = frames.getById(id);
|
||||
if (!frame) throw new Error(`applyConsolidation: missing frame ${id}`);
|
||||
if (frame.importance === 'deprecated') {
|
||||
throw new Error(`applyConsolidation: frame ${id} is deprecated`);
|
||||
}
|
||||
frameCache.set(id, frame);
|
||||
return frame;
|
||||
};
|
||||
const requireIds = (value: unknown, kind: string): number[] => {
|
||||
if (
|
||||
!Array.isArray(value)
|
||||
|| value.length < 2
|
||||
|| value.length > MAX_CONSOLIDATION_OBSERVATIONS
|
||||
) {
|
||||
throw new TypeError(`applyConsolidation: ${kind}.frameIds must contain 2-${MAX_CONSOLIDATION_OBSERVATIONS} ids`);
|
||||
}
|
||||
const seen = new Set<number>();
|
||||
for (const id of value) {
|
||||
if (typeof id !== 'number' || !Number.isSafeInteger(id) || id <= 0) {
|
||||
throw new TypeError(`applyConsolidation: ${kind}.frameIds must be positive safe integers`);
|
||||
}
|
||||
if (seen.has(id)) {
|
||||
throw new TypeError(`applyConsolidation: ${kind}.frameIds must be unique`);
|
||||
}
|
||||
seen.add(id);
|
||||
}
|
||||
return [...value] as number[];
|
||||
};
|
||||
|
||||
for (const group of groups ?? []) {
|
||||
const ids = group.frameIds;
|
||||
if (!Array.isArray(ids) || ids.length < 2) continue;
|
||||
const label = group.label.trim() ? group.label.trim() : 'group';
|
||||
const desc = `${label} (${ids.length} members)`;
|
||||
bframes.push(frames.createBFrame(gopId, desc, ids[0], ids));
|
||||
}
|
||||
const chainPlans: Array<{
|
||||
baseId: number;
|
||||
pContent: string;
|
||||
}> = [];
|
||||
const groupPlans: Array<{ ids: number[]; desc: string }> = [];
|
||||
const chainPlanByIdentity = new Map<string, { pContent: string }>();
|
||||
const groupPlanByIdentity = new Map<string, { desc: string }>();
|
||||
const staleRoles = new Set<number>();
|
||||
const newestRoles = new Set<number>();
|
||||
|
||||
return { pframes, bframes, deprecated };
|
||||
for (const entry of chains) {
|
||||
if (!isRecord(entry)) {
|
||||
throw new TypeError('applyConsolidation: each chain must be an object');
|
||||
}
|
||||
if (typeof entry.attribute !== 'string' || typeof entry.currentValue !== 'string') {
|
||||
throw new TypeError('applyConsolidation: chain labels and values must be strings');
|
||||
}
|
||||
const ids = requireIds(entry.frameIds, 'chain');
|
||||
const members = ids.map(requireFrame);
|
||||
for (let index = 1; index < members.length; index += 1) {
|
||||
const previous = members[index - 1];
|
||||
const current = members[index];
|
||||
const previousTime = observationTime(previous.created_at);
|
||||
const currentTime = observationTime(current.created_at);
|
||||
if (
|
||||
!Number.isFinite(previousTime)
|
||||
|| !Number.isFinite(currentTime)
|
||||
|| previousTime > currentTime
|
||||
|| (previousTime === currentTime && previous.id >= current.id)
|
||||
) {
|
||||
throw new Error('applyConsolidation: chain ids must be chronological oldest to newest');
|
||||
}
|
||||
}
|
||||
|
||||
const newest = members[members.length - 1];
|
||||
const cleanValue = safeModelText(entry.currentValue, MAX_CURRENT_VALUE_CHARS);
|
||||
const cleanAttribute = safeModelText(entry.attribute, MAX_LABEL_CHARS);
|
||||
if (cleanValue === null || cleanAttribute === null) {
|
||||
throw new Error('applyConsolidation: unsafe chain label or value');
|
||||
}
|
||||
const value = cleanValue || newest.content;
|
||||
const attribute = cleanAttribute || 'value';
|
||||
const asOf = String(newest.created_at).slice(0, 10);
|
||||
const pContent = `[current] ${attribute}: ${value} (as of ${asOf})`;
|
||||
if (evaluateExternalMemoryIngress({ content: pContent }).action !== 'allow') {
|
||||
throw new Error('applyConsolidation: unsafe P-frame payload');
|
||||
}
|
||||
|
||||
const chainIdentity = JSON.stringify([
|
||||
attribute.normalize('NFKC').replace(/\s+/g, ' ').toLowerCase(),
|
||||
ids,
|
||||
]);
|
||||
const existingChain = chainPlanByIdentity.get(chainIdentity);
|
||||
if (existingChain) {
|
||||
if (existingChain.pContent !== pContent) {
|
||||
throw new Error('applyConsolidation: conflicting duplicate chain');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
chainPlanByIdentity.set(chainIdentity, { pContent });
|
||||
|
||||
const staleIds = ids.slice(0, -1);
|
||||
for (const staleId of staleIds) staleRoles.add(staleId);
|
||||
newestRoles.add(newest.id);
|
||||
chainPlans.push({ baseId: ids[0], pContent });
|
||||
}
|
||||
|
||||
for (const staleId of staleRoles) {
|
||||
if (newestRoles.has(staleId)) {
|
||||
throw new Error(`applyConsolidation: conflicting stale/newest role for frame ${staleId}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const entry of groups) {
|
||||
if (!isRecord(entry)) {
|
||||
throw new TypeError('applyConsolidation: each group must be an object');
|
||||
}
|
||||
if (typeof entry.label !== 'string') {
|
||||
throw new TypeError('applyConsolidation: group label must be a string');
|
||||
}
|
||||
const ids = requireIds(entry.frameIds, 'group');
|
||||
ids.forEach(requireFrame);
|
||||
const canonicalIds = [...ids].sort((a, b) => a - b);
|
||||
const cleanLabel = safeModelText(entry.label, MAX_LABEL_CHARS);
|
||||
if (cleanLabel === null) {
|
||||
throw new Error('applyConsolidation: unsafe group label');
|
||||
}
|
||||
const desc = `${cleanLabel || 'group'} (${ids.length} members)`;
|
||||
const persisted = JSON.stringify({ description: desc, references: canonicalIds });
|
||||
if (
|
||||
evaluateExternalMemoryIngress({ content: desc }).action !== 'allow'
|
||||
|| evaluateExternalMemoryIngress({ content: persisted }).action !== 'allow'
|
||||
) {
|
||||
throw new Error('applyConsolidation: unsafe B-frame payload');
|
||||
}
|
||||
|
||||
const groupIdentity = JSON.stringify([
|
||||
(cleanLabel || 'group').normalize('NFKC').replace(/\s+/g, ' ').toLowerCase(),
|
||||
canonicalIds,
|
||||
]);
|
||||
const existingGroup = groupPlanByIdentity.get(groupIdentity);
|
||||
if (existingGroup) {
|
||||
if (existingGroup.desc !== desc) {
|
||||
throw new Error('applyConsolidation: conflicting duplicate group');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
groupPlanByIdentity.set(groupIdentity, { desc });
|
||||
groupPlans.push({ ids: canonicalIds, desc });
|
||||
}
|
||||
|
||||
const pframes: MemoryFrame[] = [];
|
||||
const bframes: MemoryFrame[] = [];
|
||||
const deprecated = [...staleRoles];
|
||||
for (const staleId of staleRoles) {
|
||||
const stale = frameCache.get(staleId)!;
|
||||
if (!frames.update(staleId, stale.content, 'deprecated')) {
|
||||
throw new Error(`applyConsolidation: frame ${staleId} disappeared during update`);
|
||||
}
|
||||
}
|
||||
for (const newestId of newestRoles) {
|
||||
const newest = frameCache.get(newestId)!;
|
||||
if (!frames.update(newestId, newest.content, 'critical')) {
|
||||
throw new Error(`applyConsolidation: frame ${newestId} disappeared during update`);
|
||||
}
|
||||
}
|
||||
for (const plan of chainPlans) {
|
||||
pframes.push(frames.createPFrame(
|
||||
gopId,
|
||||
plan.pContent,
|
||||
plan.baseId,
|
||||
'critical',
|
||||
'agent_inferred',
|
||||
));
|
||||
}
|
||||
for (const plan of groupPlans) {
|
||||
bframes.push(frames.createBFrame(gopId, plan.desc, plan.ids[0], plan.ids));
|
||||
}
|
||||
return { pframes, bframes, deprecated };
|
||||
});
|
||||
}
|
||||
|
||||
// ── Read helpers ───────────────────────────────────────────────────────────
|
||||
@@ -359,11 +600,22 @@ export function collectObservations(
|
||||
params.push(options.gopId);
|
||||
}
|
||||
|
||||
let sql = `SELECT id, content, created_at FROM memory_frames WHERE ${conditions.join(' AND ')} ORDER BY created_at, id`;
|
||||
if (options.limit && options.limit > 0) {
|
||||
sql += ' LIMIT ?';
|
||||
params.push(options.limit);
|
||||
const sql = `
|
||||
SELECT id, content, created_at
|
||||
FROM (
|
||||
SELECT id, content, created_at
|
||||
FROM memory_frames
|
||||
WHERE ${conditions.join(' AND ')}
|
||||
ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC
|
||||
LIMIT ?
|
||||
)
|
||||
ORDER BY ${CREATED_AT_SORT_EXPR} ASC, id ASC
|
||||
`;
|
||||
return raw.prepare(sql).all(...params) as Observation[];
|
||||
}
|
||||
const sql = `SELECT id, content, created_at FROM memory_frames WHERE ${conditions.join(' AND ')} ORDER BY ${CREATED_AT_SORT_EXPR}, id`;
|
||||
return raw.prepare(sql).all(...params) as Observation[];
|
||||
}
|
||||
|
||||
@@ -379,15 +631,25 @@ export function getCurrentValues(db: MindDB, gopId?: string): string[] {
|
||||
gopId
|
||||
? raw
|
||||
.prepare(
|
||||
"SELECT content FROM memory_frames WHERE frame_type = 'P' AND gop_id = ? ORDER BY created_at, id",
|
||||
`SELECT content, importance FROM memory_frames WHERE frame_type = 'P' AND substr(content, 1, 10) = '[current] ' AND gop_id = ? ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC`,
|
||||
)
|
||||
.all(gopId)
|
||||
: raw
|
||||
.prepare("SELECT content FROM memory_frames WHERE frame_type = 'P' ORDER BY created_at, id")
|
||||
.prepare(`SELECT content, importance FROM memory_frames WHERE frame_type = 'P' AND substr(content, 1, 10) = '[current] ' ORDER BY ${CREATED_AT_SORT_EXPR} DESC, id DESC`)
|
||||
.all()
|
||||
) as Array<{ content: string }>;
|
||||
) as Array<{ content: string; importance: string }>;
|
||||
|
||||
return rows
|
||||
.map((r) => String(r.content).replace(/^\[current\]\s*/, '').trim())
|
||||
.filter(Boolean);
|
||||
const seen = new Set<string>();
|
||||
const current: string[] = [];
|
||||
for (const row of rows) {
|
||||
const line = String(row.content).replace(/^\[current\]\s*/, '').trim();
|
||||
const colon = line.indexOf(':');
|
||||
if (colon <= 0) continue;
|
||||
const key = line.slice(0, colon).trim().replace(/\s+/g, ' ').toLowerCase();
|
||||
if (!key || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
if (row.importance === 'deprecated') continue;
|
||||
current.push(line);
|
||||
}
|
||||
return current.reverse();
|
||||
}
|
||||
|
||||
258
packages/hive-mind-core/src/mind/transformers-model-load.ts
Normal file
258
packages/hive-mind-core/src/mind/transformers-model-load.ts
Normal file
@@ -0,0 +1,258 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { performance } from 'node:perf_hooks';
|
||||
import Database from 'better-sqlite3';
|
||||
|
||||
const LOCK_WAIT_TIMEOUT_MS = 12 * 60 * 1_000;
|
||||
const LOCK_RETRY_MIN_MS = 35;
|
||||
const LOCK_RETRY_JITTER_MS = 30;
|
||||
const SAFE_HUGGING_FACE_MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)?$/;
|
||||
const CORRUPT_ONNX_ERROR = /^Load model from (.+\.onnx) failed:\s*Protobuf parsing failed\.?$/i;
|
||||
const LOCK_DIRECTORY = '.waggle-model-locks';
|
||||
|
||||
export interface TransformersModelLoadOptions<T> {
|
||||
cacheDir: string;
|
||||
model: string;
|
||||
load: (canonicalCacheDir: string) => Promise<T>;
|
||||
lockTimeoutMs?: number;
|
||||
onQuarantine?: (quarantineDir: string) => void | Promise<void>;
|
||||
}
|
||||
|
||||
function normalizeLockKey(value: string): string {
|
||||
return process.platform === 'win32' ? value.toLocaleLowerCase('en-US') : value;
|
||||
}
|
||||
|
||||
export function modelLoadLockPath(cacheDir: string, model: string): string {
|
||||
const modelHash = createHash('sha256')
|
||||
.update(normalizeLockKey(model))
|
||||
.digest('hex')
|
||||
.slice(0, 16);
|
||||
return path.join(path.resolve(cacheDir), LOCK_DIRECTORY, `${modelHash}.sqlite`);
|
||||
}
|
||||
|
||||
function isSqliteBusy(error: unknown): boolean {
|
||||
if (!(error instanceof Error) || !('code' in error)) return false;
|
||||
const code = String((error as Error & { code?: unknown }).code);
|
||||
return code === 'SQLITE_BUSY'
|
||||
|| code === 'SQLITE_BUSY_SNAPSHOT'
|
||||
|| code === 'SQLITE_LOCKED';
|
||||
}
|
||||
|
||||
async function acquireCrossProcessLock(
|
||||
database: Database.Database,
|
||||
timeoutMs: number,
|
||||
): Promise<void> {
|
||||
const deadline = performance.now() + timeoutMs;
|
||||
|
||||
for (;;) {
|
||||
try {
|
||||
database.exec('BEGIN IMMEDIATE');
|
||||
return;
|
||||
} catch (error) {
|
||||
if (!isSqliteBusy(error)) throw error;
|
||||
if (performance.now() >= deadline) {
|
||||
throw new Error(
|
||||
`Timed out waiting ${timeoutMs}ms for local model cache lock`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const retryMs = LOCK_RETRY_MIN_MS + Math.random() * LOCK_RETRY_JITTER_MS;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, retryMs));
|
||||
}
|
||||
}
|
||||
|
||||
function isWithin(root: string, candidate: string): boolean {
|
||||
const relative = path.relative(root, candidate);
|
||||
return relative !== ''
|
||||
&& relative !== '..'
|
||||
&& !relative.startsWith(`..${path.sep}`)
|
||||
&& !path.isAbsolute(relative);
|
||||
}
|
||||
|
||||
function isWithinOrEqual(root: string, candidate: string): boolean {
|
||||
return root === candidate || isWithin(root, candidate);
|
||||
}
|
||||
|
||||
function assertUnlinkedPath(root: string, target: string): void {
|
||||
const relative = path.relative(root, target);
|
||||
if (relative === '' || relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
||||
throw new Error('Model cache path escapes its cache root');
|
||||
}
|
||||
|
||||
let current = root;
|
||||
for (const segment of relative.split(path.sep)) {
|
||||
current = path.join(current, segment);
|
||||
const stats = fs.lstatSync(current);
|
||||
if (stats.isSymbolicLink()) {
|
||||
throw new Error(`Model cache path crosses a filesystem link: ${current}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateQuarantinePaths(
|
||||
canonicalCacheDir: string,
|
||||
modelDir: string,
|
||||
reportedOnnxPath: string,
|
||||
): { modelDir: string; quarantineRoot: string } | null {
|
||||
if (!path.isAbsolute(reportedOnnxPath)) return null;
|
||||
if (!fs.existsSync(modelDir) || !fs.existsSync(reportedOnnxPath)) return null;
|
||||
|
||||
assertUnlinkedPath(canonicalCacheDir, modelDir);
|
||||
assertUnlinkedPath(canonicalCacheDir, reportedOnnxPath);
|
||||
|
||||
const quarantineRoot = path.dirname(modelDir);
|
||||
const modelStats = fs.lstatSync(modelDir);
|
||||
const reportStats = fs.lstatSync(reportedOnnxPath);
|
||||
const rootStats = fs.lstatSync(quarantineRoot);
|
||||
if (!modelStats.isDirectory() || !reportStats.isFile() || !rootStats.isDirectory()) return null;
|
||||
|
||||
const realCacheDir = fs.realpathSync.native(canonicalCacheDir);
|
||||
const realQuarantineRoot = fs.realpathSync.native(quarantineRoot);
|
||||
const realModelDir = fs.realpathSync.native(modelDir);
|
||||
const realOnnxPath = fs.realpathSync.native(reportedOnnxPath);
|
||||
if (!isWithinOrEqual(realCacheDir, realQuarantineRoot)
|
||||
|| !isWithin(realQuarantineRoot, realModelDir)
|
||||
|| path.dirname(realModelDir) !== realQuarantineRoot
|
||||
|| !isWithin(realModelDir, realOnnxPath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { modelDir: realModelDir, quarantineRoot: realQuarantineRoot };
|
||||
}
|
||||
|
||||
function quarantineCorruptModel(
|
||||
canonicalCacheDir: string,
|
||||
model: string,
|
||||
reportedOnnxPath: string,
|
||||
): string | null {
|
||||
if (!SAFE_HUGGING_FACE_MODEL_ID.test(model)) return null;
|
||||
|
||||
const modelDir = path.join(canonicalCacheDir, ...model.split('/'));
|
||||
const firstValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
|
||||
if (!firstValidation) return null;
|
||||
|
||||
// Re-resolve immediately before the move so a changed link/path cannot redirect it.
|
||||
const finalValidation = validateQuarantinePaths(canonicalCacheDir, modelDir, reportedOnnxPath);
|
||||
if (!finalValidation) return null;
|
||||
|
||||
const quarantineDir = path.join(
|
||||
finalValidation.quarantineRoot,
|
||||
`${path.basename(finalValidation.modelDir)}.corrupt-${Date.now()}-${randomUUID()}`,
|
||||
);
|
||||
fs.renameSync(finalValidation.modelDir, quarantineDir);
|
||||
return quarantineDir;
|
||||
}
|
||||
|
||||
function reportedCorruptOnnxPath(error: unknown): string | null {
|
||||
if (!(error instanceof Error)) return null;
|
||||
return error.message.trim().match(CORRUPT_ONNX_ERROR)?.[1] ?? null;
|
||||
}
|
||||
|
||||
function notifyQuarantine(
|
||||
callback: TransformersModelLoadOptions<unknown>['onQuarantine'],
|
||||
quarantineDir: string,
|
||||
): void {
|
||||
try {
|
||||
const notification = callback?.(quarantineDir);
|
||||
if (notification) {
|
||||
void Promise.resolve(notification).catch(() => undefined);
|
||||
}
|
||||
} catch {
|
||||
// Notification is advisory and must not alter model recovery control flow.
|
||||
}
|
||||
}
|
||||
|
||||
function prepareLockDatabase(canonicalCacheDir: string, model: string): Database.Database {
|
||||
const lockPath = modelLoadLockPath(canonicalCacheDir, model);
|
||||
const lockDir = path.dirname(lockPath);
|
||||
fs.mkdirSync(lockDir, { recursive: true });
|
||||
|
||||
const lockDirStats = fs.lstatSync(lockDir);
|
||||
if (!lockDirStats.isDirectory() || lockDirStats.isSymbolicLink()) {
|
||||
throw new Error('Local model lock path is not a regular directory');
|
||||
}
|
||||
const realLockDir = fs.realpathSync.native(lockDir);
|
||||
if (!isWithin(canonicalCacheDir, realLockDir)) {
|
||||
throw new Error('Local model lock path escapes its cache root');
|
||||
}
|
||||
if (fs.existsSync(lockPath)) {
|
||||
const lockStats = fs.lstatSync(lockPath);
|
||||
if (!lockStats.isFile() || lockStats.isSymbolicLink()) {
|
||||
throw new Error('Local model lock database is not a regular file');
|
||||
}
|
||||
}
|
||||
|
||||
return new Database(lockPath, { timeout: 0 });
|
||||
}
|
||||
|
||||
async function runModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
|
||||
const timeoutMs = options.lockTimeoutMs ?? LOCK_WAIT_TIMEOUT_MS;
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs < 0) {
|
||||
throw new Error('Local model cache lock timeout must be a non-negative finite number');
|
||||
}
|
||||
|
||||
const requestedCacheDir = path.resolve(options.cacheDir);
|
||||
fs.mkdirSync(requestedCacheDir, { recursive: true });
|
||||
const canonicalCacheDir = fs.realpathSync.native(requestedCacheDir);
|
||||
const lockDatabase = prepareLockDatabase(canonicalCacheDir, options.model);
|
||||
let result!: T;
|
||||
let primaryError: unknown;
|
||||
let hasPrimaryError = false;
|
||||
|
||||
try {
|
||||
await acquireCrossProcessLock(lockDatabase, timeoutMs);
|
||||
|
||||
try {
|
||||
result = await options.load(canonicalCacheDir);
|
||||
} catch (firstError) {
|
||||
const reportedOnnxPath = reportedCorruptOnnxPath(firstError);
|
||||
if (!reportedOnnxPath) throw firstError;
|
||||
|
||||
let quarantineDir: string | null = null;
|
||||
try {
|
||||
quarantineDir = quarantineCorruptModel(canonicalCacheDir, options.model, reportedOnnxPath);
|
||||
} catch {
|
||||
// Preserve the original loader error if safe quarantine cannot complete.
|
||||
}
|
||||
if (!quarantineDir) throw firstError;
|
||||
|
||||
notifyQuarantine(options.onQuarantine, quarantineDir);
|
||||
result = await options.load(canonicalCacheDir);
|
||||
}
|
||||
} catch (error) {
|
||||
primaryError = error;
|
||||
hasPrimaryError = true;
|
||||
}
|
||||
|
||||
let cleanupError: unknown;
|
||||
let hasCleanupError = false;
|
||||
if (lockDatabase.inTransaction) {
|
||||
try {
|
||||
lockDatabase.exec('ROLLBACK');
|
||||
} catch (error) {
|
||||
cleanupError = error;
|
||||
hasCleanupError = true;
|
||||
}
|
||||
}
|
||||
try {
|
||||
lockDatabase.close();
|
||||
} catch (error) {
|
||||
if (!hasCleanupError) cleanupError = error;
|
||||
hasCleanupError = true;
|
||||
}
|
||||
|
||||
if (hasPrimaryError) throw primaryError;
|
||||
if (hasCleanupError) throw cleanupError;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes one model/cache load across workers and processes. SQLite owns
|
||||
* the OS lock, so process termination releases it without PID/age heuristics.
|
||||
*/
|
||||
export function withTransformersModelLoad<T>(options: TransformersModelLoadOptions<T>): Promise<T> {
|
||||
return runModelLoad(options);
|
||||
}
|
||||
Reference in New Issue
Block a user