moving

packages/hive-mind-core/src/mind/search.ts

@@ -0,0 +1,698 @@
+import type { MindDB } from './db.js';
+import type { Embedder } from './embeddings.js';
+import type { MemoryFrame, Importance } from './frames.js';
+import type { Reranker } from './inprocess-reranker.js';
+import { chunkText, type ChunkOptions } from './chunker.js';
+import { buildFtsOrQuery, hasUnsegmentedScript, sanitizeFtsToken } from './fts-sanitize.js';
+import { createCoreLogger } from '../logger.js';
+import {
+  computeRelevance,
+  SCORING_PROFILES,
+  type ScoringProfile,
+  type ScoringContext,
+  type ScoredResult,
+} from './scoring.js';
+import { KnowledgeGraph } from './knowledge.js';
+
+export interface SearchOptions {
+  limit?: number;
+  gopId?: string; // scope to a specific session
+  profile?: ScoringProfile;
+  context?: ScoringContext;
+  /** F20: Only include frames created on or after this ISO date string. */
+  since?: string;
+  /** F20: Only include frames created on or before this ISO date string. */
+  until?: string;
+  /**
+   * W4.2: cross-encoder reranker invoked AFTER RRF on the top-`rerankPoolSize`
+   * candidates. When provided, results are sorted by reranker score
+   * (jointly attentive over query+doc). RRF still selects the candidate
+   * pool; the reranker only re-orders the survivors. Soft-fails to RRF
+   * ordering on any reranker error.
+   */
+  reranker?: Reranker;
+  /** How many candidates to send to the reranker (default 30). */
+  rerankPoolSize?: number;
+  /**
+   * Hard-exclude frames with importance='deprecated' from results. Default OFF
+   * for back-compat: deprecated frames still surface, merely down-weighted 0.3×
+   * by the scoring layer. Turn ON where a superseded value must NEVER leak into
+   * the read context — e.g. after supersession consolidation (see supersede.ts),
+   * where a 0.3× multiplier still let stale values surface via the focus lane.
+   */
+  excludeDeprecated?: boolean;
+}
+
+export interface SearchResult {
+  frame: MemoryFrame;
+  rrfScore: number;
+  relevanceScore: number;
+  finalScore: number;
+}
+
+// Ported from hive-mind a99ea0e.
+/**
+ * Retrieval-confidence verdict for the abstain path (LongMemEval's
+ * "insufficient evidence" ability). Pure + side-effect-free so callers
+ * (MCP recall_memory, CLI, eval harness) can decide whether to answer or
+ * abstain without re-running search.
+ */
+export interface RetrievalConfidence {
+  /** True when the top result clears the threshold (safe to answer). */
+  sufficient: boolean;
+  /** The top finalScore observed (0 when there were no results). */
+  topScore: number;
+  /** The threshold it was compared against. */
+  threshold: number;
+}
+
+// Ported from hive-mind a99ea0e.
+/**
+ * Assess whether a result set carries enough signal to answer, or whether the
+ * caller should abstain ("insufficient evidence"). A scaffold for the abstain
+ * path: it does NOT change `search()` output — callers opt in by passing the
+ * results plus a τ threshold. `sufficient` is true iff the top finalScore is
+ * strictly greater than τ; an empty set is always insufficient.
+ *
+ * Threshold semantics intentionally mirror the recall-stress edge-query rule
+ * (a low top score means "nothing relevant surfaced").
+ */
+export function assessRetrievalConfidence(
+  results: readonly SearchResult[],
+  threshold: number,
+): RetrievalConfidence {
+  const topScore = results.length ? results[0].finalScore : 0;
+  return { sufficient: topScore > threshold, topScore, threshold };
+}
+
+const RRF_K = 60;
+
+const log = createCoreLogger('hybrid-search');
+
+// Reverse-ported from OSS hive-mind chunker (oss-drift triage D1, 2026-06-11).
+/**
+ * Chunk-level retrieval flag — DEFAULT ON since the 2026-06-12 long-frame
+ * needle probe (benchmarks/chunk-probe/): on a copy of the real production
+ * personal mind, paired hit@5 = chunk 46/52 vs whole-frame 17/52 (discordant
+ * pairs 30-vs-1, McNemar p≈2e-8); chunk led even within the embed cap
+ * (17/20 vs 13/20) and dominated beyond it (29/32 vs 4/32 — content past the
+ * embedder's true token context is structurally invisible to whole-frame
+ * vectors). LoCoMo was rejected as the ruler: its frames sit below the
+ * 2000-char chunk threshold, so an A/B there measures noise by construction.
+ * Kill switch: WAGGLE_CHUNK_RETRIEVAL=0. Gates BOTH the write side
+ * (indexFrame / indexFramesBatch also chunk-index the frame) and the read
+ * side (search() queries memory_frame_chunks_vec, falling back to whole-frame
+ * vectors while the chunk index is empty). `indexChunksForFrame` /
+ * `rechunkAllFrames` stay callable regardless of the flag (backfill + eval).
+ */
+export function chunkRetrievalEnabled(): boolean {
+  return process.env.WAGGLE_CHUNK_RETRIEVAL !== '0';
+}
+
+function f32ToBlob(f32: Float32Array): Uint8Array {
+  return new Uint8Array(f32.buffer, f32.byteOffset, f32.byteLength);
+}
+
+/**
+ * Escape LIKE metacharacters (`%`, `_`) and the escape char itself (`\`) so the
+ * keyword-fallback term is matched literally. Pair with `ESCAPE '\'` on the LIKE.
+ */
+function escapeLikeTerm(term: string): string {
+  return term.replace(/[\\%_]/g, ch => `\\${ch}`);
+}
+
+export class HybridSearch {
+  private db: MindDB;
+  private embedder: Embedder;
+  private fingerprintChecked = false;
+
+  constructor(db: MindDB, embedder: Embedder) {
+    this.db = db;
+    this.embedder = embedder;
+  }
+
+  // Reverse-ported from OSS hive-mind (oss-drift triage R7, 2026-06-11).
+  /**
+   * Guard the .mind's embedding fingerprint before vector reads/writes. Throws
+   * EmbeddingDimMismatchError if the active embedder's dim differs from what
+   * the .mind's vectors were written at; warns (but allows) on a same-dim model
+   * change. Memoized on success so it costs one meta read per instance lifetime.
+   * Must be called BEFORE any try/catch that would swallow the error.
+   */
+  private ensureFingerprint(): void {
+    if (this.fingerprintChecked) return;
+    const e = this.embedder as Embedder & {
+      getActiveProvider?(): string;
+      getStatus?(): { modelName?: string };
+    };
+    const provider = e.getActiveProvider?.() ?? 'unknown';
+    const model = e.getStatus?.().modelName ?? 'unknown';
+    const result = this.db.ensureEmbeddingFingerprint({ provider, model, dim: this.embedder.dimensions });
+    // Only memoize after a non-throwing check (a dim mismatch must keep throwing).
+    this.fingerprintChecked = true;
+    if (result.status === 'model-changed') {
+      log.warn(
+        `Embedding model changed for this .mind (${result.storedProvider}/${result.storedModel} → ` +
+          `${provider}/${model}, same ${this.embedder.dimensions}-dim). Existing vectors stay searchable, ` +
+          `but cross-model similarity is degraded — consider re-embedding all frames.`
+      );
+    }
+  }
+
+  async search(query: string, options: SearchOptions = {}): Promise<SearchResult[]> {
+    const { limit = 20, gopId, profile = 'balanced', context = {}, since, until, reranker, rerankPoolSize } = options;
+    const weights = SCORING_PROFILES[profile];
+
+    // Run keyword and vector searches in parallel.
+    // W4.1b slot-consumption fix: the since/until filter applies AFTER the
+    // lanes run (as a WHERE over candidate ids), so out-of-window candidates
+    // would otherwise consume lane slots and shrink results below `limit`
+    // even when in-window frames exist deeper in the lanes. Over-fetch the
+    // lanes when a temporal window is active so the post-filter has depth.
+    const laneFetch = (since || until) ? limit * 10 : limit * 2;
+
+    // D1 chunk lane (flag-gated, default OFF): prefer chunk-level vector
+    // search when WAGGLE_CHUNK_RETRIEVAL=1 AND chunks_vec is populated —
+    // chunk embeddings discriminate better on domain-homogeneous corpora
+    // than whole-frame embeddings. vectorSearchChunks returns null when no
+    // chunks exist, signalling clean fallback to the whole-frame path. Both
+    // paths return frame IDs so the RRF + scoring pipeline is unchanged.
+    // Flag off → chunkResults is null without touching the chunk tables,
+    // so the lane below is byte-identical to pre-D1.
+    const chunkResults = chunkRetrievalEnabled()
+      ? await this.vectorSearchChunks(query, laneFetch, gopId)
+      : null;
+    const [keywordResults, vectorResults] = await Promise.all([
+      this.keywordSearch(query, laneFetch, gopId),
+      chunkResults !== null
+        ? Promise.resolve(chunkResults)
+        : this.vectorSearch(query, laneFetch, gopId),
+    ]);
+
+    // RRF fusion
+    const rrfScores = new Map<number, number>();
+
+    keywordResults.forEach((id, rank) => {
+      rrfScores.set(id, (rrfScores.get(id) ?? 0) + 1 / (RRF_K + rank));
+    });
+
+    vectorResults.forEach((id, rank) => {
+      rrfScores.set(id, (rrfScores.get(id) ?? 0) + 1 / (RRF_K + rank));
+    });
+
+    // Get all unique frame IDs
+    const frameIds = [...rrfScores.keys()];
+    if (frameIds.length === 0) return [];
+
+    // F20: Fetch frames with optional temporal filtering
+    const raw = this.db.getDatabase();
+    const placeholders = frameIds.map(() => '?').join(',');
+    const temporalConditions: string[] = [];
+    const temporalParams: unknown[] = [...frameIds];
+
+    // W4.1b fencepost fix: `created_at` carries mixed formats across write
+    // paths — `datetime('now')` ("YYYY-MM-DD HH:MM:SS") vs harvest ISO
+    // ("YYYY-MM-DDT…Z"). A date-only `until` string-compares BELOW any
+    // same-day timestamp ("2026-03-21T10:00" > "2026-03-21"), silently
+    // excluding the whole final day. Compare date-only bounds on the
+    // 10-char date prefix instead — format-agnostic and inclusive.
+    if (since) {
+      if (since.length === 10) {
+        temporalConditions.push('substr(created_at, 1, 10) >= ?');
+      } else {
+        temporalConditions.push('created_at >= ?');
+      }
+      temporalParams.push(since);
+    }
+    if (until) {
+      if (until.length === 10) {
+        temporalConditions.push('substr(created_at, 1, 10) <= ?');
+      } else {
+        temporalConditions.push('created_at <= ?');
+      }
+      temporalParams.push(until);
+    }
+
+    const whereExtra = temporalConditions.length > 0
+      ? ` AND ${temporalConditions.join(' AND ')}`
+      : '';
+    // Hard-exclude deprecated frames when requested (no param needed — literal
+    // condition). Dropping them from `frames` removes them from frameMap, so
+    // they never enter the result set OR the reranker pool.
+    const deprecatedExtra = options.excludeDeprecated ? " AND importance != 'deprecated'" : '';
+
+    const frames = raw.prepare(
+      `SELECT * FROM memory_frames WHERE id IN (${placeholders})${whereExtra}${deprecatedExtra}`
+    ).all(...temporalParams) as MemoryFrame[];
+
+    const frameMap = new Map(frames.map(f => [f.id, f]));
+
+    // W4.1: turn on the 'contextual' scoring signal. Seed graph distance from
+    // entities the caller flagged (context.recentEntityIds) plus entities named
+    // in the query, BFS the KG, and map to frames via the kg_entity_frames bridge.
+    // Best-effort: a graph hiccup must never fail the search.
+    let scoringContext = context;
+    if (!scoringContext.graphDistances) {
+      try {
+        const kg = new KnowledgeGraph(this.db);
+        const seeds = new Set<number>(scoringContext.recentEntityIds ?? []);
+        for (const id of kg.findEntitiesInText(query)) seeds.add(id);
+        if (seeds.size > 0) {
+          const graphDistances = kg.frameDistancesFromEntities([...seeds], 3);
+          if (graphDistances.size > 0) scoringContext = { ...scoringContext, graphDistances };
+        }
+      } catch { /* contextual signal is optional */ }
+    }
+
+    // Compute final scores
+    const results: SearchResult[] = [];
+    for (const [frameId, rrfScore] of rrfScores) {
+      const frame = frameMap.get(frameId);
+      if (!frame) continue;
+
+      const relevanceScore = computeRelevance(
+        {
+          id: frame.id,
+          // W4.2 bug #3: temporal decay anchors on write time, not access time.
+          created_at: frame.created_at,
+          last_accessed: frame.last_accessed,
+          access_count: frame.access_count,
+          importance: frame.importance as Importance,
+        },
+        weights,
+        scoringContext
+      );
+
+      results.push({
+        frame,
+        rrfScore,
+        relevanceScore,
+        finalScore: rrfScore * relevanceScore,
+      });
+    }
+
+    results.sort((a, b) => b.finalScore - a.finalScore);
+
+    // W4.2: optional cross-encoder reranking on the top pool (reverse-ported
+    // from the OSS benchmark-proven stack). Reranker scoring is jointly
+    // attentive over (query, doc), so it discriminates much better than
+    // vector dot products on densely-homogeneous corpora. RRF still selects
+    // the candidate pool; the reranker only re-orders the survivors.
+    if (reranker) {
+      const poolSize = Math.min(rerankPoolSize ?? 30, results.length);
+      const pool = results.slice(0, poolSize);
+      try {
+        const docs = pool.map((r) => r.frame.content);
+        const scores = await reranker.scoreBatch(query, docs);
+        // Pair (result, rerank score), sort desc, replace finalScore so the
+        // shape stays the same for downstream consumers.
+        const reranked = pool.map((r, i) => ({ ...r, finalScore: scores[i] }));
+        reranked.sort((a, b) => b.finalScore - a.finalScore);
+        // Append any pool tail items beyond rerankPoolSize so a small limit
+        // doesn't suddenly contract the result set.
+        return reranked.concat(results.slice(poolSize)).slice(0, limit);
+      } catch {
+        // Reranker failure (model load, OOM, dim mismatch) — fall back to
+        // RRF ordering. Soft-fail so a misconfigured reranker doesn't
+        // kill recall entirely.
+      }
+    }
+
+    return results.slice(0, limit);
+  }
+
+  async keywordSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
+    const raw = this.db.getDatabase();
+
+    // W3.6: Sanitize query for FTS5 with OR-based matching for better recall.
+    // Old: implicit AND (all terms required) → fails on "hiring decisions this month"
+    // New: OR between terms (any term matches) → FTS5 rank orders by relevance
+    // S1: sanitizer unified in fts-sanitize.ts, Unicode-aware (Cyrillic and
+    // diacritic terms survive; ASCII output is byte-identical to before).
+    const safeQuery = query.includes('"')
+      ? query // already quoted by caller
+      : buildFtsOrQuery(query);
+
+    if (!safeQuery) {
+      // Empty MATCH string. For ASCII queries that means stop words / short
+      // tokens only — keep returning [] (regression lock). For queries in an
+      // unsegmented script (CJK) the emptiness is a sanitizer artifact, not a
+      // lack of signal: unicode61 cannot token-match CJK prose, but LIKE
+      // substring matching can, so route those to the fallback lane.
+      return hasUnsegmentedScript(query) ? this.likeFallbackSearch(query, limit, gopId) : [];
+    }
+
+    let sql: string;
+    let params: unknown[];
+
+    if (gopId) {
+      sql = `
+        SELECT mf.id FROM memory_frames_fts fts
+        JOIN memory_frames mf ON mf.id = fts.rowid
+        WHERE fts.content MATCH ? AND mf.gop_id = ?
+        ORDER BY rank
+        LIMIT ?
+      `;
+      params = [safeQuery, gopId, limit];
+    } else {
+      sql = `
+        SELECT rowid as id FROM memory_frames_fts
+        WHERE content MATCH ?
+        ORDER BY rank
+        LIMIT ?
+      `;
+      params = [safeQuery, limit];
+    }
+
+    try {
+      const rows = raw.prepare(sql).all(...params) as { id: number }[];
+      return rows.map(r => r.id);
+    } catch {
+      // FTS5 parse error (e.g. user query with FTS5-special chars that survived
+      // sanitization) — fall back to a LIKE keyword scan over the same column so
+      // we return best-effort matches instead of a false "no memory found".
+      return this.likeFallbackSearch(query, limit, gopId);
+    }
+  }
+
+  /**
+   * LIKE-based keyword fallback over memory_frames.content. Used when the FTS5
+   * MATCH query throws a parse error (e.g. an unbalanced quote or other FTS5
+   * operator the user typed literally). The raw query is split into word tokens
+   * — stripping the punctuation that caused the FTS5 error, mirroring the
+   * primary sanitizer — and matched with OR-ed LIKE clauses for best-effort
+   * recall. Bound parameters only (the term is never interpolated) and LIKE
+   * metachars (`%`, `_`, `\`) are escaped with an ESCAPE clause so each token
+   * matches literally. If no usable token survives, a single literal LIKE over
+   * the whole escaped query is used.
+   */
+  private likeFallbackSearch(query: string, limit: number, gopId?: string): number[] {
+    const raw = this.db.getDatabase();
+
+    const tokens = query
+      .split(/\s+/)
+      .map(sanitizeFtsToken) // strip punctuation (incl. FTS5 operators), Unicode-aware
+      .filter(w => w.length > 0);
+    const terms = (tokens.length > 0 ? tokens : [query]).map(t => `%${escapeLikeTerm(t)}%`);
+
+    const likeClause = terms.map(() => `content LIKE ? ESCAPE '\\'`).join(' OR ');
+
+    try {
+      if (gopId) {
+        const rows = raw.prepare(
+          `SELECT id FROM memory_frames
+           WHERE (${likeClause}) AND gop_id = ?
+           ORDER BY created_at DESC LIMIT ?`
+        ).all(...terms, gopId, limit) as { id: number }[];
+        return rows.map(r => r.id);
+      }
+      const rows = raw.prepare(
+        `SELECT id FROM memory_frames
+         WHERE (${likeClause})
+         ORDER BY created_at DESC LIMIT ?`
+      ).all(...terms, limit) as { id: number }[];
+      return rows.map(r => r.id);
+    } catch {
+      return [];
+    }
+  }
+
+  async vectorSearch(query: string, limit: number, gopId?: string): Promise<number[]> {
+    this.ensureFingerprint();
+    const embedding = await this.embedder.embed(query);
+    const blob = f32ToBlob(embedding);
+    const raw = this.db.getDatabase();
+
+    if (gopId) {
+      // Two-step: get candidates from vec, then filter by GOP
+      try {
+        const rows = raw.prepare(`
+          SELECT v.rowid as id FROM memory_frames_vec v
+          WHERE v.embedding MATCH ? AND k = ?
+          ORDER BY distance
+        `).all(blob, limit * 3) as { id: number }[];
+
+        // Filter by GOP
+        if (rows.length === 0) return [];
+        const placeholders = rows.map(() => '?').join(',');
+        const filtered = raw.prepare(`
+          SELECT id FROM memory_frames WHERE id IN (${placeholders}) AND gop_id = ?
+        `).all(...rows.map(r => r.id), gopId) as { id: number }[];
+
+        return filtered.map(r => r.id).slice(0, limit);
+      } catch {
+        return [];
+      }
+    } else {
+      try {
+        const rows = raw.prepare(`
+          SELECT rowid as id FROM memory_frames_vec
+          WHERE embedding MATCH ? AND k = ?
+          ORDER BY distance
+        `).all(blob, limit) as { id: number }[];
+        return rows.map(r => r.id);
+      } catch {
+        return [];
+      }
+    }
+  }
+
+  async indexFrame(frameId: number, content: string): Promise<void> {
+    this.ensureFingerprint();
+    if (!Number.isFinite(frameId)) {
+      throw new Error('Invalid frame ID for vector indexing');
+    }
+    const embedding = await this.embedder.embed(content);
+    const raw = this.db.getDatabase();
+    // sqlite-vec vec0 requires rowid as SQL literal (parameterized rowid not supported)
+    const id = Math.trunc(frameId);
+    raw.prepare(
+      `INSERT INTO memory_frames_vec (rowid, embedding) VALUES (${id}, ?)`
+    ).run(f32ToBlob(embedding));
+
+    // D1 (flag-gated, default OFF): keep the chunk index in lockstep with
+    // live frame writes. Soft-fail — a chunk-indexing error must never break
+    // the primary whole-frame write (mirrors the reranker soft-fail stance).
+    if (chunkRetrievalEnabled()) {
+      try {
+        await this.indexChunksForFrame(frameId, content);
+      } catch (err) {
+        log.warn(
+          `chunk indexing failed for frame ${id} (whole-frame vector written): ` +
+            `${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+    }
+  }
+
+  async indexFramesBatch(frames: { id: number; content: string }[]): Promise<void> {
+    if (frames.length === 0) return;
+    this.ensureFingerprint();
+    for (const f of frames) {
+      if (!Number.isFinite(f.id)) {
+        throw new Error('Invalid frame ID for vector indexing');
+      }
+    }
+    const contents = frames.map(f => f.content);
+    const embeddings = await this.embedder.embedBatch(contents);
+    const raw = this.db.getDatabase();
+    // sqlite-vec vec0 requires rowid as SQL literal (parameterized rowid not supported)
+    const insertAll = raw.transaction(() => {
+      for (let i = 0; i < frames.length; i++) {
+        const id = Math.trunc(frames[i].id);
+        raw.prepare(
+          `INSERT INTO memory_frames_vec (rowid, embedding) VALUES (${id}, ?)`
+        ).run(f32ToBlob(embeddings[i]));
+      }
+    });
+    insertAll();
+
+    // D1 (flag-gated, default OFF): chunk-index batch writes too, so frames
+    // ingested via the batch path (harvest) aren't invisible to the chunk
+    // lane. Soft-fail per frame — see indexFrame.
+    if (chunkRetrievalEnabled()) {
+      for (const f of frames) {
+        try {
+          await this.indexChunksForFrame(f.id, f.content);
+        } catch (err) {
+          log.warn(
+            `chunk indexing failed for frame ${Math.trunc(f.id)} (whole-frame vector written): ` +
+              `${err instanceof Error ? err.message : String(err)}`
+          );
+        }
+      }
+    }
+  }
+
+  // ── Chunk-level indexing (oss-drift triage D1, 2026-06-11) ─────────────
+  // Reverse-ported from OSS hive-mind "Phase 3b-3 chunking". Whole-frame
+  // embeddings cluster too tightly on a domain-homogeneous corpus (every
+  // frame is "about the same project"), so retrieval can't discriminate.
+  // Chunking decomposes a frame into ~500-token paragraph-level pieces,
+  // each with its own embedding — search returns the chunk, we map back to
+  // the parent frame for the final result.
+  // ─────────────────────────────────────────────────────────────────────
+
+  /**
+   * Replace all chunks for a frame: clears existing chunks/vec rows for the
+   * frame, re-chunks the content, embeds each chunk, inserts both rows.
+   * Idempotent — safe to call repeatedly. Used by rechunkAllFrames and by
+   * the flag-gated indexFrame path. NOT itself gated on
+   * WAGGLE_CHUNK_RETRIEVAL (backfill + eval call it directly).
+   */
+  async indexChunksForFrame(
+    frameId: number,
+    content: string,
+    opts: ChunkOptions = {},
+  ): Promise<number> {
+    if (!Number.isFinite(frameId) || frameId <= 0) {
+      throw new Error('Invalid frame ID for chunk indexing');
+    }
+    this.ensureFingerprint();
+    const raw = this.db.getDatabase();
+    const id = Math.trunc(frameId);
+
+    const chunks = chunkText(content, opts);
+    if (chunks.length === 0) return 0;
+
+    // Embed all chunks. embedBatch amortises HTTP overhead on Ollama/API providers.
+    const texts = chunks.map((c) => c.text);
+    const embeddings = await this.embedder.embedBatch(texts);
+
+    // Single tx so partial failure leaves the frame's chunks empty
+    // (next rechunk pass will re-fill from scratch — same end state).
+    const tx = raw.transaction(() => {
+      // Find existing chunk_ids for this frame so we can drop their vec rows.
+      // Foreign-key cascade handles memory_frame_chunks deletion when the
+      // parent frame is deleted, but for re-indexing we're keeping the
+      // frame and just replacing its chunks.
+      const existing = raw
+        .prepare('SELECT id FROM memory_frame_chunks WHERE frame_id = ?')
+        .all(id) as Array<{ id: number }>;
+      for (const row of existing) {
+        // sqlite-vec rowid must be SQL literal.
+        raw.prepare(`DELETE FROM memory_frame_chunks_vec WHERE rowid = ${Math.trunc(row.id)}`).run();
+      }
+      raw.prepare('DELETE FROM memory_frame_chunks WHERE frame_id = ?').run(id);
+
+      const insertChunk = raw.prepare(
+        'INSERT INTO memory_frame_chunks (frame_id, chunk_idx, content, char_start, char_end) VALUES (?, ?, ?, ?, ?)'
+      );
+      for (let i = 0; i < chunks.length; i++) {
+        const c = chunks[i];
+        const result = insertChunk.run(id, i, c.text, c.charStart, c.charEnd);
+        const chunkId = Math.trunc(Number(result.lastInsertRowid));
+        raw
+          .prepare(`INSERT INTO memory_frame_chunks_vec (rowid, embedding) VALUES (${chunkId}, ?)`)
+          .run(f32ToBlob(embeddings[i]));
+      }
+    });
+    tx();
+    return chunks.length;
+  }
+
+  /**
+   * Vector search over chunks. Returns parent frame IDs deduped (best-chunk-
+   * per-frame wins — first-seen order under ORDER BY distance). When the
+   * chunk index is empty (or the tables are missing), returns null so callers
+   * can cleanly fall back to the whole-frame vectorSearch path.
+   */
+  async vectorSearchChunks(query: string, limit: number, gopId?: string): Promise<number[] | null> {
+    this.ensureFingerprint();
+    const raw = this.db.getDatabase();
+    // Cheap probe — avoid embedding the query when chunks aren't populated.
+    let chunkCount: number;
+    try {
+      const row = raw.prepare('SELECT COUNT(*) AS n FROM memory_frame_chunks').get() as
+        | { n: number }
+        | undefined;
+      chunkCount = row?.n ?? 0;
+    } catch {
+      return null;
+    }
+    if (chunkCount === 0) return null;
+
+    const embedding = await this.embedder.embed(query);
+    const blob = f32ToBlob(embedding);
+
+    // Over-fetch chunks (limit * 5) so dedup-to-frame still leaves enough
+    // candidates after collapsing multiple chunks of the same frame.
+    try {
+      const chunkRows = raw
+        .prepare(
+          `SELECT v.rowid AS chunk_id, c.frame_id
+             FROM memory_frame_chunks_vec v
+             JOIN memory_frame_chunks c ON c.id = v.rowid
+            WHERE v.embedding MATCH ? AND k = ?
+            ORDER BY distance`
+        )
+        .all(blob, Math.max(limit * 5, 25)) as Array<{ chunk_id: number; frame_id: number }>;
+
+      if (chunkRows.length === 0) return [];
+
+      // Dedup by frame_id, preserving first-seen order (best-distance chunk).
+      const seen = new Set<number>();
+      const frameIds: number[] = [];
+      for (const r of chunkRows) {
+        if (seen.has(r.frame_id)) continue;
+        seen.add(r.frame_id);
+        frameIds.push(r.frame_id);
+        if (frameIds.length >= limit) break;
+      }
+
+      if (gopId) {
+        const placeholders = frameIds.map(() => '?').join(',');
+        const filtered = raw
+          .prepare(
+            `SELECT id FROM memory_frames WHERE id IN (${placeholders}) AND gop_id = ?`
+          )
+          .all(...frameIds, gopId) as { id: number }[];
+        return filtered.map((r) => r.id).slice(0, limit);
+      }
+      return frameIds;
+    } catch {
+      return null;
+    }
+  }
+}
+
+// Reverse-ported from OSS hive-mind chunker (oss-drift triage D1, 2026-06-11);
+// follows the OSS `maintenance --rechunk-all` per-mind logic.
+export interface RechunkResult {
+  framesProcessed: number;
+  chunksCreated: number;
+  framesFailed: number;
+}
+
+/**
+ * (Re)chunk + chunk-index every non-deprecated frame in the .mind. Idempotent
+ * per-frame — indexChunksForFrame deletes a frame's existing chunks before
+ * re-inserting. One bad frame doesn't abort the batch (logged + counted).
+ * Backfill/eval helper only — no CLI/route wiring yet, and NOT gated on
+ * WAGGLE_CHUNK_RETRIEVAL (it must be runnable before any flag flip).
+ */
+export async function rechunkAllFrames(db: MindDB, search: HybridSearch): Promise<RechunkResult> {
+  const raw = db.getDatabase();
+  const frames = raw
+    .prepare("SELECT id, content FROM memory_frames WHERE importance != 'deprecated' ORDER BY id ASC")
+    .all() as Array<{ id: number; content: string }>;
+
+  let framesProcessed = 0;
+  let chunksCreated = 0;
+  let framesFailed = 0;
+
+  for (const f of frames) {
+    try {
+      const n = await search.indexChunksForFrame(f.id, f.content);
+      framesProcessed++;
+      chunksCreated += n;
+    } catch (err) {
+      framesFailed++;
+      log.warn(
+        `rechunkAllFrames: frame ${f.id} failed: ${err instanceof Error ? err.message : String(err)}`
+      );
+    }
+  }
+
+  return { framesProcessed, chunksCreated, framesFailed };
+}