This commit is contained in:
162
packages/hive-mind-core/src/harvest/chatgpt-adapter.ts
Normal file
162
packages/hive-mind-core/src/harvest/chatgpt-adapter.ts
Normal file
@@ -0,0 +1,162 @@
|
||||
/**
|
||||
* ChatGPT Adapter — parse ChatGPT JSON export into UniversalImportItems.
|
||||
*
|
||||
* ChatGPT export format uses a `mapping` object with node IDs containing
|
||||
* messages. Each conversation has a title, create_time, and the mapping tree.
|
||||
*/
|
||||
|
||||
import { stableHarvestId } from './stable-id.js';
|
||||
import type { SourceAdapter, UniversalImportItem, ConversationMessage } from './types.js';
|
||||
import { asRecord, getArray, getNumber, getString, type RawRecord } from './raw-types.js';
|
||||
|
||||
export class ChatGPTAdapter implements SourceAdapter {
|
||||
readonly sourceType = 'chatgpt' as const;
|
||||
readonly displayName = 'ChatGPT';
|
||||
|
||||
parse(input: unknown): UniversalImportItem[] {
|
||||
const root = asRecord(input);
|
||||
const conversations = Array.isArray(input) ? input : root && getArray(root, 'conversations');
|
||||
if (!Array.isArray(conversations)) return [];
|
||||
|
||||
const items: UniversalImportItem[] = [];
|
||||
|
||||
for (const rawConv of conversations) {
|
||||
const conv = asRecord(rawConv);
|
||||
if (!conv) continue;
|
||||
const title = getString(conv, 'title') || 'Untitled';
|
||||
const messages: ConversationMessage[] = [];
|
||||
|
||||
// ChatGPT uses a mapping object with node IDs
|
||||
const mapping = asRecord(conv.mapping);
|
||||
if (mapping) {
|
||||
const nodes = Object.values(mapping)
|
||||
.map(asRecord)
|
||||
.filter((n): n is RawRecord => n !== null);
|
||||
const sorted = nodes
|
||||
.filter(n => {
|
||||
const msg = asRecord(n.message);
|
||||
const content = msg && asRecord(msg.content);
|
||||
const parts = content && getArray(content, 'parts');
|
||||
return (parts?.length ?? 0) > 0;
|
||||
})
|
||||
.sort((a, b) => {
|
||||
const am = asRecord(a.message);
|
||||
const bm = asRecord(b.message);
|
||||
return (am && getNumber(am, 'create_time') ? getNumber(am, 'create_time')! : 0)
|
||||
- (bm && getNumber(bm, 'create_time') ? getNumber(bm, 'create_time')! : 0);
|
||||
});
|
||||
|
||||
for (const node of sorted) {
|
||||
const msg = asRecord(node.message);
|
||||
const author = msg && asRecord(msg.author);
|
||||
const authorRole = author && getString(author, 'role');
|
||||
if (!msg || !authorRole) continue;
|
||||
if (authorRole === 'system') continue;
|
||||
|
||||
const role = authorRole === 'user' ? 'user' as const : 'assistant' as const;
|
||||
const content = asRecord(msg.content);
|
||||
const parts = content ? getArray(content, 'parts') : undefined;
|
||||
// W4.4 (caption parity): multimodal object parts were silently
|
||||
// dropped — DALL-E image parts carry their generation prompt
|
||||
// (metadata.dalle.prompt), the only text-bearing image field in
|
||||
// ChatGPT exports. Render as "[Shared image: …]" (Memori's
|
||||
// convention; W3.3 measured dropped captions at 4.5pp single-hop).
|
||||
const textParts: string[] = [];
|
||||
for (const p of parts ?? []) {
|
||||
if (typeof p === 'string') { textParts.push(p); continue; }
|
||||
const rec = asRecord(p);
|
||||
if (!rec) continue;
|
||||
const ct = getString(rec, 'content_type') ?? '';
|
||||
if (ct.includes('image')) {
|
||||
const meta = asRecord(rec.metadata);
|
||||
const dalle = meta ? asRecord(meta.dalle) : null;
|
||||
const prompt = dalle ? getString(dalle, 'prompt') : undefined;
|
||||
if (prompt) textParts.push(`[Shared image: ${prompt}]`);
|
||||
}
|
||||
}
|
||||
// Message-level attachments: names are text-bearing presence signals.
|
||||
const msgMeta = asRecord(msg.metadata);
|
||||
for (const rawAtt of (msgMeta ? getArray(msgMeta, 'attachments') : undefined) ?? []) {
|
||||
const att = asRecord(rawAtt);
|
||||
const name = att ? getString(att, 'name') : undefined;
|
||||
if (name) textParts.push(`[Attached: ${name}]`);
|
||||
}
|
||||
const text = textParts.join('\n').trim();
|
||||
if (!text) continue;
|
||||
|
||||
const createTime = getNumber(msg, 'create_time');
|
||||
messages.push({
|
||||
role,
|
||||
text,
|
||||
timestamp: createTime ? new Date(createTime * 1000).toISOString() : undefined,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (messages.length === 0) continue;
|
||||
|
||||
// Also check for custom instructions in conversation metadata
|
||||
const customInstructions = conv.custom_instructions;
|
||||
const createTime = getNumber(conv, 'create_time');
|
||||
|
||||
items.push({
|
||||
// #7 sticky erasure: stable per-conversation id (keyed on the export's own
|
||||
// conversation id, NOT content, so a grown conversation keeps its id).
|
||||
id: stableHarvestId('chatgpt', getString(conv, 'id') ?? getString(conv, 'conversation_id') ?? `conv\x00${title}\x00${createTime ?? ''}`),
|
||||
source: 'chatgpt',
|
||||
type: 'conversation',
|
||||
title,
|
||||
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
|
||||
messages,
|
||||
timestamp: createTime ? new Date(createTime * 1000).toISOString() : new Date().toISOString(),
|
||||
metadata: {
|
||||
conversationId: getString(conv, 'id') ?? getString(conv, 'conversation_id'),
|
||||
messageCount: messages.length,
|
||||
...(customInstructions ? { customInstructions } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Also extract custom instructions / memory as separate items
|
||||
if (root?.user_custom_instructions) {
|
||||
items.push({
|
||||
// 'singleton' discriminator (2 parts) so this can't collide with a
|
||||
// conversation whose conv.id is literally the string 'custom_instructions'.
|
||||
id: stableHarvestId('chatgpt', 'singleton', 'custom_instructions'),
|
||||
source: 'chatgpt',
|
||||
type: 'instruction',
|
||||
title: 'ChatGPT Custom Instructions',
|
||||
content: typeof root.user_custom_instructions === 'string'
|
||||
? root.user_custom_instructions
|
||||
: JSON.stringify(root.user_custom_instructions),
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: { type: 'custom_instructions' },
|
||||
});
|
||||
}
|
||||
|
||||
const memories = root && getArray(root, 'memories');
|
||||
if (memories) {
|
||||
for (const rawMem of memories) {
|
||||
const mem = asRecord(rawMem);
|
||||
const content = typeof rawMem === 'string'
|
||||
? rawMem
|
||||
: (mem && (getString(mem, 'content') ?? getString(mem, 'text'))) ?? JSON.stringify(rawMem);
|
||||
items.push({
|
||||
// No stable per-memory id exists in the export, so key on created_at+content
|
||||
// (the best available surrogate). Caveat: ChatGPT memories are user-editable,
|
||||
// so an EDIT changes the id → erasure isn't sticky across an edit (bounded,
|
||||
// documented tradeoff — same class as the universal-text content-keyed path).
|
||||
id: stableHarvestId('chatgpt', 'memory', (mem && getString(mem, 'created_at')) ?? '', content),
|
||||
source: 'chatgpt',
|
||||
type: 'memory',
|
||||
title: 'ChatGPT Memory',
|
||||
content,
|
||||
timestamp: (mem && getString(mem, 'created_at')) ?? new Date().toISOString(),
|
||||
metadata: { type: 'chatgpt_memory' },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
}
|
||||
30
packages/hive-mind-core/src/harvest/chunk-utils.ts
Normal file
30
packages/hive-mind-core/src/harvest/chunk-utils.ts
Normal file
@@ -0,0 +1,30 @@
|
||||
/**
|
||||
* Shared text chunking utility for harvest adapters.
|
||||
*
|
||||
* Splits text into chunks by paragraph boundaries (double-newline),
|
||||
* respecting a configurable maximum character length per chunk.
|
||||
*/
|
||||
|
||||
const DEFAULT_MAX_LENGTH = 2000;
|
||||
|
||||
export function chunkByParagraphs(text: string, maxLen: number = DEFAULT_MAX_LENGTH): string[] {
|
||||
const paragraphs = text.split(/\n\s*\n/).filter(p => p.trim().length > 0);
|
||||
const chunks: string[] = [];
|
||||
let current = '';
|
||||
|
||||
for (const para of paragraphs) {
|
||||
const trimmed = para.trim();
|
||||
if (current.length + trimmed.length + 2 > maxLen && current.length > 0) {
|
||||
chunks.push(current.trim());
|
||||
current = trimmed;
|
||||
} else {
|
||||
current += (current ? '\n\n' : '') + trimmed;
|
||||
}
|
||||
}
|
||||
|
||||
if (current.trim()) {
|
||||
chunks.push(current.trim());
|
||||
}
|
||||
|
||||
return chunks;
|
||||
}
|
||||
271
packages/hive-mind-core/src/harvest/claude-adapter.ts
Normal file
271
packages/hive-mind-core/src/harvest/claude-adapter.ts
Normal file
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* Claude Adapter — parse Claude web/desktop JSON export into UniversalImportItems.
|
||||
*
|
||||
* Handles multiple shapes of the Claude export:
|
||||
* - Raw `chat_messages` array per conversation
|
||||
* - Structured content-block format where `msg.content` is typed blocks
|
||||
* - `projects[].docs[]` project-knowledge documents → type='artifact'
|
||||
* - `memories` (conversations_memory + project_memories map) → type='memory'
|
||||
* - `design_chats[]` Claude design workspace threads → type='conversation'
|
||||
*
|
||||
* The memories / design_chats streams (and the enriched project-docs
|
||||
* parsing) came online in the 2026-04-22 Claude.ai export refresh.
|
||||
* Reverse-ported from OSS hive-mind (oss-drift triage R6, 2026-06-11).
|
||||
*
|
||||
* W4.4 (caption parity, mono-only): message-level `attachments` /
|
||||
* `files` arrays surface their text content — `extracted_content` is
|
||||
* inlined as "[Attached: name] …" (capped 500 chars), bare names as
|
||||
* "[Shared file: name]".
|
||||
*/
|
||||
|
||||
import { stableHarvestId } from './stable-id.js';
|
||||
import type { SourceAdapter, UniversalImportItem, ConversationMessage } from './types.js';
|
||||
import { asRecord, firstString, getArray, getString, type RawRecord } from './raw-types.js';
|
||||
|
||||
export class ClaudeAdapter implements SourceAdapter {
|
||||
readonly sourceType = 'claude' as const;
|
||||
readonly displayName = 'Claude';
|
||||
|
||||
parse(input: unknown): UniversalImportItem[] {
|
||||
const items: UniversalImportItem[] = [];
|
||||
const root = asRecord(input);
|
||||
|
||||
// ── Conversations (historical default path) ───────────────────────
|
||||
const conversations = Array.isArray(input) ? input : root && getArray(root, 'conversations');
|
||||
if (Array.isArray(conversations)) {
|
||||
for (const conv of conversations) {
|
||||
items.push(...this.parseConversation(conv));
|
||||
}
|
||||
}
|
||||
|
||||
// ── Project-knowledge docs → artifact items ──────────────────────
|
||||
const projects = root && getArray(root, 'projects');
|
||||
if (projects) {
|
||||
for (const project of projects) {
|
||||
items.push(...this.parseProjectDocs(project));
|
||||
}
|
||||
}
|
||||
|
||||
// ── Memories stream (conversations_memory + project_memories) ────
|
||||
if (root && root.memories !== undefined) {
|
||||
items.push(...this.parseMemories(root.memories));
|
||||
}
|
||||
|
||||
// ── Design chats stream ──────────────────────────────────────────
|
||||
const designChats = root && getArray(root, 'design_chats');
|
||||
if (designChats) {
|
||||
for (const dc of designChats) {
|
||||
items.push(...this.parseDesignChat(dc));
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse one Claude chat message (conversation or design-chat shape) into a
|
||||
* ConversationMessage, applying the W4.4 attachment/caption extraction.
|
||||
* Returns null for messages with no surfaceable text.
|
||||
*/
|
||||
private parseMessage(rawMsg: unknown): ConversationMessage | null {
|
||||
const msg = asRecord(rawMsg);
|
||||
if (!msg) return null;
|
||||
const role = (getString(msg, 'sender') === 'human' || getString(msg, 'role') === 'user')
|
||||
? 'user' as const
|
||||
: 'assistant' as const;
|
||||
|
||||
// Handle content blocks (Claude format)
|
||||
let text: string;
|
||||
const blocks = getArray(msg, 'content');
|
||||
if (blocks) {
|
||||
text = blocks
|
||||
.map(asRecord)
|
||||
.filter((b): b is RawRecord => b !== null && b.type === 'text')
|
||||
.map(b => getString(b, 'text') ?? '')
|
||||
.join('\n')
|
||||
.trim();
|
||||
} else {
|
||||
text = (getString(msg, 'text') ?? getString(msg, 'content') ?? '').trim();
|
||||
}
|
||||
|
||||
// W4.4 (caption parity): Claude exports carry message-level
|
||||
// `attachments` (with extracted_content — text already extracted
|
||||
// from images/docs) and `files` arrays; both were never accessed.
|
||||
const extras: string[] = [];
|
||||
for (const key of ['attachments', 'files'] as const) {
|
||||
for (const rawAtt of getArray(msg, key) ?? []) {
|
||||
const att = asRecord(rawAtt);
|
||||
if (!att) continue;
|
||||
const name = getString(att, 'file_name') ?? getString(att, 'name');
|
||||
const extracted = getString(att, 'extracted_content');
|
||||
if (extracted && extracted.trim()) {
|
||||
extras.push(`[Attached: ${name ?? 'file'}] ${extracted.trim().slice(0, 500)}`);
|
||||
} else if (name) {
|
||||
extras.push(`[Shared file: ${name}]`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (extras.length > 0) text = [text, ...extras].filter(Boolean).join('\n').trim();
|
||||
if (!text) return null;
|
||||
|
||||
return {
|
||||
role,
|
||||
text,
|
||||
timestamp: getString(msg, 'created_at') ?? getString(msg, 'timestamp'),
|
||||
};
|
||||
}
|
||||
|
||||
private parseConversation(rawConv: unknown): UniversalImportItem[] {
|
||||
const conv = asRecord(rawConv);
|
||||
if (!conv) return [];
|
||||
const title = firstString(conv, 'name', 'title') || 'Untitled';
|
||||
const messages: ConversationMessage[] = [];
|
||||
|
||||
const chatMessages = getArray(conv, 'chat_messages') ?? getArray(conv, 'messages') ?? [];
|
||||
for (const rawMsg of chatMessages) {
|
||||
const parsed = this.parseMessage(rawMsg);
|
||||
if (parsed) messages.push(parsed);
|
||||
}
|
||||
|
||||
if (messages.length === 0) return [];
|
||||
|
||||
return [{
|
||||
// #7 sticky erasure: stable per-conversation id (conv uuid, not content).
|
||||
id: stableHarvestId('claude', firstString(conv, 'uuid', 'id') ?? `conv\x00${title}\x00${firstString(conv, 'created_at', 'create_time') ?? ''}`),
|
||||
source: 'claude',
|
||||
type: 'conversation',
|
||||
title,
|
||||
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
|
||||
messages,
|
||||
timestamp: firstString(conv, 'created_at', 'create_time') ?? new Date().toISOString(),
|
||||
metadata: {
|
||||
conversationId: firstString(conv, 'uuid', 'id'),
|
||||
messageCount: messages.length,
|
||||
projectId: getString(conv, 'project_uuid') ?? undefined,
|
||||
},
|
||||
}];
|
||||
}
|
||||
|
||||
private parseProjectDocs(rawProject: unknown): UniversalImportItem[] {
|
||||
const project = asRecord(rawProject);
|
||||
const docs = project && getArray(project, 'docs');
|
||||
if (!project || !docs) return [];
|
||||
const out: UniversalImportItem[] = [];
|
||||
for (const rawDoc of docs) {
|
||||
const doc = asRecord(rawDoc);
|
||||
if (!doc) continue;
|
||||
const content = getString(doc, 'content') ?? '';
|
||||
if (content.length === 0) continue;
|
||||
out.push({
|
||||
id: stableHarvestId('claude', 'projdoc', getString(project, 'uuid') ?? '', getString(doc, 'uuid') ?? firstString(doc, 'filename', 'title') ?? ''),
|
||||
source: 'claude',
|
||||
type: 'artifact',
|
||||
title: firstString(doc, 'filename', 'title') ?? 'Project Document',
|
||||
content,
|
||||
timestamp: getString(doc, 'created_at')
|
||||
?? getString(project, 'updated_at')
|
||||
?? getString(project, 'created_at')
|
||||
?? new Date().toISOString(),
|
||||
metadata: {
|
||||
projectName: getString(project, 'name'),
|
||||
projectUuid: getString(project, 'uuid'),
|
||||
type: 'project_knowledge',
|
||||
docUuid: getString(doc, 'uuid'),
|
||||
filename: getString(doc, 'filename'),
|
||||
},
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private parseMemories(input: unknown): UniversalImportItem[] {
|
||||
// `memories.json` is a single-entry array per the 2026-04-22 export
|
||||
// shape: `[{ conversations_memory, project_memories, account_uuid }]`.
|
||||
// Accept both the array-wrapped form and a bare object for flexibility.
|
||||
const arr = Array.isArray(input) ? input : [input];
|
||||
const out: UniversalImportItem[] = [];
|
||||
|
||||
for (const rawEntry of arr) {
|
||||
const entry = asRecord(rawEntry);
|
||||
if (!entry) continue;
|
||||
const accountUuid = getString(entry, 'account_uuid');
|
||||
|
||||
// conversations_memory — usually a single long string of user-about facts
|
||||
const convMem = entry.conversations_memory;
|
||||
if (convMem !== undefined && convMem !== null) {
|
||||
const content = typeof convMem === 'string' ? convMem : JSON.stringify(convMem);
|
||||
if (content.length > 0) {
|
||||
out.push({
|
||||
// account-level singleton — fixed discriminator, NOT content (memory grows).
|
||||
id: stableHarvestId('claude', 'memory', 'conversations_memory', accountUuid ?? ''),
|
||||
source: 'claude',
|
||||
type: 'memory',
|
||||
title: 'Claude Memory — Conversations',
|
||||
content,
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
memoryKind: 'conversations_memory',
|
||||
accountUuid,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// project_memories — map of project_uuid -> memory string
|
||||
const projectMemories = asRecord(entry.project_memories);
|
||||
if (projectMemories) {
|
||||
for (const [projUuid, memValue] of Object.entries(projectMemories)) {
|
||||
const content = typeof memValue === 'string' ? memValue : JSON.stringify(memValue);
|
||||
if (content.length > 0) {
|
||||
out.push({
|
||||
// per-project memory keyed on the project uuid map key (not content).
|
||||
id: stableHarvestId('claude', 'memory', 'project_memory', accountUuid ?? '', projUuid),
|
||||
source: 'claude',
|
||||
type: 'memory',
|
||||
title: `Claude Memory — Project ${projUuid}`,
|
||||
content,
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
memoryKind: 'project_memory',
|
||||
projectUuid: projUuid,
|
||||
accountUuid,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
private parseDesignChat(input: unknown): UniversalImportItem[] {
|
||||
const dc = asRecord(input);
|
||||
if (!dc) return [];
|
||||
const messages: ConversationMessage[] = [];
|
||||
const msgs = getArray(dc, 'messages') ?? getArray(dc, 'chat_messages') ?? [];
|
||||
|
||||
for (const rawMsg of msgs) {
|
||||
const parsed = this.parseMessage(rawMsg);
|
||||
if (parsed) messages.push(parsed);
|
||||
}
|
||||
|
||||
if (messages.length === 0) return [];
|
||||
|
||||
return [{
|
||||
id: stableHarvestId('claude', getString(dc, 'uuid') ?? `designchat\x00${getString(dc, 'project') ?? ''}\x00${getString(dc, 'title') ?? ''}\x00${getString(dc, 'created_at') ?? ''}`),
|
||||
source: 'claude',
|
||||
type: 'conversation',
|
||||
title: getString(dc, 'title') ?? 'Claude Design Chat',
|
||||
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
|
||||
messages,
|
||||
timestamp: getString(dc, 'created_at') ?? new Date().toISOString(),
|
||||
metadata: {
|
||||
designChatUuid: getString(dc, 'uuid'),
|
||||
projectUuid: getString(dc, 'project') ?? undefined,
|
||||
messageCount: messages.length,
|
||||
stream: 'design_chats',
|
||||
},
|
||||
}];
|
||||
}
|
||||
}
|
||||
359
packages/hive-mind-core/src/harvest/claude-code-adapter.ts
Normal file
359
packages/hive-mind-core/src/harvest/claude-code-adapter.ts
Normal file
@@ -0,0 +1,359 @@
|
||||
/**
|
||||
* Claude Code Filesystem Adapter — reads ~/.claude/ directory structure.
|
||||
*
|
||||
* Extracts:
|
||||
* - memory/*.md files (with frontmatter: type, name, description)
|
||||
* - rules/**\/*.md files (coding standards, workflow rules)
|
||||
* - plans/*.md files (implementation plans)
|
||||
* - settings.json (model preferences, tool config)
|
||||
* - CLAUDE.md project files (architectural decisions)
|
||||
* - .mind/*.md session handoffs (decisions, directions)
|
||||
* - Decision extraction from memory content (pattern matching)
|
||||
*
|
||||
* This is a FilesystemAdapter — it reads directly from disk.
|
||||
*/
|
||||
|
||||
import { stableHarvestId } from './stable-id.js';
|
||||
import { decisionOfSubjectId } from './decision-derivation.js';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { FilesystemAdapter, UniversalImportItem, ImportItemType } from './types.js';
|
||||
|
||||
// Map Claude Code memory types to our import types
|
||||
const MEMORY_TYPE_MAP: Record<string, ImportItemType> = {
|
||||
user: 'preference',
|
||||
feedback: 'decision',
|
||||
project: 'memory',
|
||||
reference: 'memory',
|
||||
};
|
||||
|
||||
// Patterns that indicate user decisions in text
|
||||
const DECISION_PATTERNS = [
|
||||
/\bwe (?:decided|chose|picked|went with|agreed|confirmed)\b/i,
|
||||
/\blet'?s (?:go with|use|do|keep|drop|switch|move)\b/i,
|
||||
/\bdecision:\s/i,
|
||||
/\bconfirmed:\s/i,
|
||||
/\bapproved:\s/i,
|
||||
/\brejected:\s/i,
|
||||
/\bwon'?t (?:do|use|implement|add|need)\b/i,
|
||||
/\bmust (?:use|have|be|support|include)\b/i,
|
||||
/\bnon-negotiable\b/i,
|
||||
/\brequirement:\s/i,
|
||||
/\bconstraint:\s/i,
|
||||
/\bchose .+ (?:over|instead of|rather than)\b/i,
|
||||
/\bdropped?\b.+\bin favo(?:u)?r of\b/i,
|
||||
];
|
||||
|
||||
interface MemoryFrontmatter {
|
||||
name?: string;
|
||||
description?: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
function parseFrontmatter(content: string): { frontmatter: MemoryFrontmatter; body: string } {
|
||||
const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
|
||||
if (!match) return { frontmatter: {}, body: content };
|
||||
|
||||
const raw = match[1];
|
||||
const body = match[2].trim();
|
||||
const frontmatter: MemoryFrontmatter = {};
|
||||
|
||||
for (const line of raw.split('\n')) {
|
||||
const colonIdx = line.indexOf(':');
|
||||
if (colonIdx < 0) continue;
|
||||
const key = line.slice(0, colonIdx).trim();
|
||||
const value = line.slice(colonIdx + 1).trim();
|
||||
if (key === 'name') frontmatter.name = value;
|
||||
if (key === 'description') frontmatter.description = value;
|
||||
if (key === 'type') frontmatter.type = value;
|
||||
}
|
||||
|
||||
return { frontmatter, body };
|
||||
}
|
||||
|
||||
function readFilesRecursive(dir: string, ext: string): { filePath: string; content: string }[] {
|
||||
const results: { filePath: string; content: string }[] = [];
|
||||
if (!fs.existsSync(dir)) return results;
|
||||
|
||||
const entries = fs.readdirSync(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
results.push(...readFilesRecursive(fullPath, ext));
|
||||
} else if (entry.name.endsWith(ext)) {
|
||||
try {
|
||||
results.push({ filePath: fullPath, content: fs.readFileSync(fullPath, 'utf-8') });
|
||||
} catch { /* skip unreadable files */ }
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export class ClaudeCodeAdapter implements FilesystemAdapter {
|
||||
readonly sourceType = 'claude-code' as const;
|
||||
readonly displayName = 'Claude Code';
|
||||
|
||||
parse(input: unknown): UniversalImportItem[] {
|
||||
// For the SourceAdapter interface — parse JSON if provided
|
||||
if (typeof input === 'string') {
|
||||
return this.scan(input);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
scan(dirPath: string): UniversalImportItem[] {
|
||||
if (!fs.existsSync(dirPath)) return [];
|
||||
const items: UniversalImportItem[] = [];
|
||||
|
||||
// 1. Scan all project memory directories
|
||||
const projectsDir = path.join(dirPath, 'projects');
|
||||
if (fs.existsSync(projectsDir)) {
|
||||
const projectEntries = fs.readdirSync(projectsDir, { withFileTypes: true });
|
||||
for (const projEntry of projectEntries) {
|
||||
if (!projEntry.isDirectory()) continue;
|
||||
const memoryDir = path.join(projectsDir, projEntry.name, 'memory');
|
||||
items.push(...this.scanMemoryDir(memoryDir, projEntry.name));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Scan rules
|
||||
const rulesDir = path.join(dirPath, 'rules');
|
||||
const ruleFiles = readFilesRecursive(rulesDir, '.md');
|
||||
for (const { filePath, content } of ruleFiles) {
|
||||
const relPath = path.relative(dirPath, filePath);
|
||||
items.push({
|
||||
// #7 sticky erasure: file path is the stable id (survives content growth).
|
||||
// Normalize the OS separator so the SAME tree scanned on win32 vs POSIX
|
||||
// yields the SAME id (stableHarvestId's cross-process determinism contract).
|
||||
id: stableHarvestId('claude-code', relPath.split(path.sep).join('/')),
|
||||
source: 'claude-code',
|
||||
type: 'rule',
|
||||
title: `Rule: ${path.basename(filePath, '.md')}`,
|
||||
content,
|
||||
timestamp: this.getFileMtime(filePath),
|
||||
metadata: { filePath: relPath, category: 'rule' },
|
||||
});
|
||||
}
|
||||
|
||||
// 3. Scan plans
|
||||
const plansDir = path.join(dirPath, 'plans');
|
||||
if (fs.existsSync(plansDir)) {
|
||||
const planFiles = fs.readdirSync(plansDir).filter(f => f.endsWith('.md'));
|
||||
for (const planFile of planFiles) {
|
||||
const fullPath = path.join(plansDir, planFile);
|
||||
try {
|
||||
const content = fs.readFileSync(fullPath, 'utf-8');
|
||||
items.push({
|
||||
id: stableHarvestId('claude-code', `plans/${planFile}`),
|
||||
source: 'claude-code',
|
||||
type: 'artifact',
|
||||
title: `Plan: ${planFile.replace('.md', '')}`,
|
||||
content,
|
||||
timestamp: this.getFileMtime(fullPath),
|
||||
metadata: { filePath: `plans/${planFile}`, category: 'plan' },
|
||||
});
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Read settings.json for preferences
|
||||
const settingsPath = path.join(dirPath, 'settings.json');
|
||||
if (fs.existsSync(settingsPath)) {
|
||||
try {
|
||||
const raw = fs.readFileSync(settingsPath, 'utf-8');
|
||||
const settings = JSON.parse(raw);
|
||||
const prefs: string[] = [];
|
||||
if (settings.model) prefs.push(`Preferred model: ${settings.model}`);
|
||||
if (settings.alwaysThinkingEnabled) prefs.push('Extended thinking: enabled');
|
||||
if (Array.isArray(settings.allowedTools)) {
|
||||
prefs.push(`Allowed tools: ${settings.allowedTools.length} configured`);
|
||||
}
|
||||
if (prefs.length > 0) {
|
||||
items.push({
|
||||
id: stableHarvestId('claude-code', 'settings.json'),
|
||||
source: 'claude-code',
|
||||
type: 'preference',
|
||||
title: 'Claude Code Settings',
|
||||
content: prefs.join('\n'),
|
||||
timestamp: this.getFileMtime(settingsPath),
|
||||
metadata: { filePath: 'settings.json', category: 'settings' },
|
||||
});
|
||||
}
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
|
||||
// 5. Scan CLAUDE.md files from project directories (architectural decisions)
|
||||
if (fs.existsSync(projectsDir)) {
|
||||
const projectEntries = fs.readdirSync(projectsDir, { withFileTypes: true });
|
||||
for (const projEntry of projectEntries) {
|
||||
if (!projEntry.isDirectory()) continue;
|
||||
const claudeMdPath = path.join(projectsDir, projEntry.name, 'CLAUDE.md');
|
||||
if (fs.existsSync(claudeMdPath)) {
|
||||
try {
|
||||
const content = fs.readFileSync(claudeMdPath, 'utf-8');
|
||||
if (content.trim().length > 50) {
|
||||
items.push({
|
||||
id: stableHarvestId('claude-code', `projects/${projEntry.name}/CLAUDE.md`),
|
||||
source: 'claude-code',
|
||||
type: 'artifact',
|
||||
title: `Project CLAUDE.md (${projEntry.name})`,
|
||||
content: content.slice(0, 4000),
|
||||
timestamp: this.getFileMtime(claudeMdPath),
|
||||
metadata: {
|
||||
filePath: `projects/${projEntry.name}/CLAUDE.md`,
|
||||
category: 'project-config',
|
||||
},
|
||||
});
|
||||
}
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Scan .mind/ directories for session handoffs and decisions
|
||||
if (fs.existsSync(projectsDir)) {
|
||||
const projectEntries = fs.readdirSync(projectsDir, { withFileTypes: true });
|
||||
for (const projEntry of projectEntries) {
|
||||
if (!projEntry.isDirectory()) continue;
|
||||
const mindDir = path.join(projectsDir, projEntry.name, '.mind');
|
||||
items.push(...this.scanMindDir(mindDir, projEntry.name));
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Extract decisions from memory items that contain decision language
|
||||
items.push(...this.extractDecisions(items));
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
/** Scan .mind/ directory for session handoffs with decisions. */
|
||||
private scanMindDir(mindDir: string, projectHash: string): UniversalImportItem[] {
|
||||
if (!fs.existsSync(mindDir)) return [];
|
||||
const items: UniversalImportItem[] = [];
|
||||
|
||||
try {
|
||||
const files = fs.readdirSync(mindDir).filter(f => f.endsWith('.md'));
|
||||
for (const file of files) {
|
||||
const fullPath = path.join(mindDir, file);
|
||||
try {
|
||||
const content = fs.readFileSync(fullPath, 'utf-8');
|
||||
if (content.trim().length < 50) continue;
|
||||
|
||||
// Determine type from filename
|
||||
const lowerFile = file.toLowerCase();
|
||||
const isDecision = lowerFile.includes('decision');
|
||||
const isState = lowerFile.includes('state');
|
||||
|
||||
items.push({
|
||||
id: stableHarvestId('claude-code', `projects/${projectHash}/.mind/${file}`),
|
||||
source: 'claude-code',
|
||||
type: isDecision ? 'decision' : 'artifact',
|
||||
title: `${isDecision ? 'Decisions' : isState ? 'State' : 'Session'}: ${file.replace('.md', '')}`,
|
||||
content: content.slice(0, 4000),
|
||||
timestamp: this.getFileMtime(fullPath),
|
||||
metadata: {
|
||||
filePath: `projects/${projectHash}/.mind/${file}`,
|
||||
category: isDecision ? 'decision' : 'session-handoff',
|
||||
},
|
||||
});
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
} catch { /* skip */ }
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract decision items from existing memory/preference/artifact items.
|
||||
* Scans content for decision patterns and creates separate decision items
|
||||
* for statements that match. Avoids duplicating items already typed as 'decision'.
|
||||
*/
|
||||
private extractDecisions(existingItems: readonly UniversalImportItem[]): UniversalImportItem[] {
|
||||
const decisions: UniversalImportItem[] = [];
|
||||
|
||||
for (const item of existingItems) {
|
||||
// Skip items already categorized as decisions
|
||||
if (item.type === 'decision' || item.type === 'rule') continue;
|
||||
|
||||
const lines = item.content.split('\n');
|
||||
const decisionLines: string[] = [];
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (trimmed.length < 10) continue;
|
||||
|
||||
for (const pattern of DECISION_PATTERNS) {
|
||||
if (pattern.test(trimmed)) {
|
||||
decisionLines.push(trimmed);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (decisionLines.length > 0) {
|
||||
decisions.push({
|
||||
// derived item — namespaced by the (now-stable) parent id so it never
|
||||
// collides with the parent's own id. decisionOfSubjectId is the SHARED
|
||||
// derivation MindErasure.eraseBySourceRef recomputes to reach + suppress
|
||||
// this derived subject on erasure (#7 P2), so the two sites cannot drift.
|
||||
id: decisionOfSubjectId(item.id),
|
||||
source: 'claude-code',
|
||||
type: 'decision',
|
||||
title: `Decisions from: ${item.title}`,
|
||||
content: decisionLines.join('\n'),
|
||||
timestamp: item.timestamp,
|
||||
metadata: {
|
||||
...item.metadata,
|
||||
category: 'decision',
|
||||
extractedFrom: item.id,
|
||||
decisionCount: decisionLines.length,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return decisions;
|
||||
}
|
||||
|
||||
private scanMemoryDir(memoryDir: string, projectHash: string): UniversalImportItem[] {
|
||||
if (!fs.existsSync(memoryDir)) return [];
|
||||
const items: UniversalImportItem[] = [];
|
||||
|
||||
const files = fs.readdirSync(memoryDir).filter(f => f.endsWith('.md') && f !== 'MEMORY.md');
|
||||
for (const file of files) {
|
||||
const fullPath = path.join(memoryDir, file);
|
||||
try {
|
||||
const raw = fs.readFileSync(fullPath, 'utf-8');
|
||||
const { frontmatter, body } = parseFrontmatter(raw);
|
||||
if (!body) continue;
|
||||
|
||||
const importType = MEMORY_TYPE_MAP[frontmatter.type ?? ''] ?? 'memory';
|
||||
|
||||
items.push({
|
||||
id: stableHarvestId('claude-code', `projects/${projectHash}/memory/${file}`),
|
||||
source: 'claude-code',
|
||||
type: importType,
|
||||
title: frontmatter.name ?? file.replace('.md', ''),
|
||||
content: body,
|
||||
timestamp: this.getFileMtime(fullPath),
|
||||
metadata: {
|
||||
filePath: `projects/${projectHash}/memory/${file}`,
|
||||
memoryType: frontmatter.type,
|
||||
description: frontmatter.description,
|
||||
category: 'memory',
|
||||
},
|
||||
});
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private getFileMtime(filePath: string): string {
|
||||
try {
|
||||
return fs.statSync(filePath).mtime.toISOString();
|
||||
} catch {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
}
|
||||
}
|
||||
31
packages/hive-mind-core/src/harvest/decision-derivation.ts
Normal file
31
packages/hive-mind-core/src/harvest/decision-derivation.ts
Normal file
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* decision-derivation.ts — the single source of truth for claude-code's derived
|
||||
* `decision-of` subject key.
|
||||
*
|
||||
* claude-code harvest (claude-code-adapter extractDecisions) fans a scanned item
|
||||
* out into a SEPARATE "Decisions from: …" item that quotes the parent's decision
|
||||
* lines. That derived item lands as its OWN GDPR Art.17 subject, keyed on
|
||||
* decisionOfSubjectId(parentId). Two layers must agree on that key EXACTLY:
|
||||
* - the adapter, which MINTS the derived item's id at harvest time;
|
||||
* - MindErasure.eraseBySourceRef, which RECOMPUTES it to erase + suppress the
|
||||
* derived subject when the parent is erased (else it survives erasure and
|
||||
* re-materializes on re-import — the #7 P2 gap).
|
||||
* Duplicating the derivation across those two sites is precisely the drift that
|
||||
* created the gap, so both import from here. Changing the token also invalidates
|
||||
* every already-persisted derived subject id, so treat it as a data contract.
|
||||
*/
|
||||
import { stableHarvestId } from './stable-id.js';
|
||||
|
||||
/** The only harvest source that derives a separate `decision-of` subject. */
|
||||
export const CLAUDE_CODE_DECISION_SOURCE = 'claude-code';
|
||||
/** The derivation-kind token the derived id is namespaced under. */
|
||||
export const DECISION_OF_KIND = 'decision-of';
|
||||
|
||||
/**
|
||||
* Deterministic id for the `decision-of` subject derived from a claude-code
|
||||
* parent item. `parentId` is the parent's stable harvest id (== its raw_archive
|
||||
* source_ref), so the id is stable across re-imports of the same parent.
|
||||
*/
|
||||
export function decisionOfSubjectId(parentId: string): string {
|
||||
return stableHarvestId(CLAUDE_CODE_DECISION_SOURCE, DECISION_OF_KIND, parentId);
|
||||
}
|
||||
BIN
packages/hive-mind-core/src/harvest/dedup.ts
Normal file
BIN
packages/hive-mind-core/src/harvest/dedup.ts
Normal file
Binary file not shown.
267
packages/hive-mind-core/src/harvest/extract-kg-entities.ts
Normal file
267
packages/hive-mind-core/src/harvest/extract-kg-entities.ts
Normal file
@@ -0,0 +1,267 @@
|
||||
// Reverse-ported from OSS hive-mind llm-extractor (oss-drift triage D2, 2026-06-11); executors rehomed onto LLMCallFn.
|
||||
/**
|
||||
* extract-kg-entities.ts — LLM-based knowledge-graph entity extraction.
|
||||
*
|
||||
* The heuristic capitalized-n-gram regex (packages/agent/src/entity-extractor.ts
|
||||
* via CognifyPipeline) produces high noise: sentence-starts, log prefixes, and
|
||||
* fragments. This pass replaces the regex with an LLM that understands semantics
|
||||
* and returns typed entities (person/project/file/decision/bug/concept/tool/
|
||||
* location) keyed by frame id.
|
||||
*
|
||||
* Ported core = PROMPT + JSONL PARSER + BATCHING + noise filter. The OSS
|
||||
* executors ('cc' subprocess spawn, raw Anthropic POST) are dropped — the
|
||||
* monorepo drives all LLM calls through `LLMCallFn` ('fast' tier), exactly
|
||||
* like extract-memory-lanes.ts.
|
||||
*
|
||||
* Failure model mirrors extract-memory-lanes: per-batch failures are collected
|
||||
* into `errors`, never thrown — partial results beat zero results when one
|
||||
* frame confuses the model. All extracted names are injection-scanned before
|
||||
* being returned (LLM output over possibly-tainted harvested content).
|
||||
*/
|
||||
|
||||
import type { LLMCallFn } from './pipeline.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import { isNoiseName, normalizeEntityName } from '../mind/entity-normalizer.js';
|
||||
import type { KnowledgeGraph } from '../mind/knowledge.js';
|
||||
|
||||
const log = createCoreLogger('extract-kg-entities');
|
||||
|
||||
/** Canonical entity types the prompt asks the model to choose from. */
|
||||
export const KG_ENTITY_TYPES = [
|
||||
'person',
|
||||
'project',
|
||||
'file',
|
||||
'decision',
|
||||
'bug',
|
||||
'concept',
|
||||
'tool',
|
||||
'location',
|
||||
] as const;
|
||||
|
||||
export type KgEntityType = (typeof KG_ENTITY_TYPES)[number];
|
||||
|
||||
/** One extracted entity attributed back to its source frame. */
|
||||
export interface KgEntity {
|
||||
frameId: number;
|
||||
type: KgEntityType;
|
||||
name: string;
|
||||
}
|
||||
|
||||
export interface KgEntityExtraction {
|
||||
entities: KgEntity[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Frames per batch. OSS defaulted to 3 because `claude -p` subprocess
|
||||
* wall-clock blew past 90s on batch=5 with raw frames; LLMCallFn has no
|
||||
* subprocess pressure, so 5 frames/batch with the per-frame content cap
|
||||
* keeps prompts bounded while halving call count.
|
||||
*/
|
||||
const BATCH_SIZE = 5;
|
||||
|
||||
/**
|
||||
* Per-frame content cap before sending to the model (OSS finding: the first
|
||||
* ~2-3KB of a frame carries the named entities; long wiki-synth frames can
|
||||
* exceed 8KB and add nothing but latency).
|
||||
*/
|
||||
const MAX_FRAME_CHARS = 2500;
|
||||
|
||||
/** The instructions block sent to the model. Stable across batches. */
|
||||
const PROMPT_INSTRUCTIONS = `Extract named entities from the FRAMES below. For each entity, output ONE JSON object on its own line.
|
||||
|
||||
OUTPUT FORMAT (JSONL — one object per line, no other text):
|
||||
{"frame_id": <number>, "name": "<entity name>", "type": "<type>"}
|
||||
|
||||
VALID TYPES (pick the closest fit):
|
||||
- person a specific human (e.g. "Marko", "Alice Chen")
|
||||
- project a named project, repo, codebase, or product (e.g. "hive-mind", "Phase 3")
|
||||
- file a specific file path or filename (e.g. "synth-drain.js", "PHASE-3-PLAN.md")
|
||||
- decision a specific architectural or strategic choice with a name (e.g. "open-core boundary")
|
||||
- bug a known issue, incident, or failure mode (e.g. "subprocess feedback loop")
|
||||
- tool a CLI tool, library, framework, or service (e.g. "Ollama", "Voyage", "sqlite-vec")
|
||||
- concept a domain concept that doesn't fit above (e.g. "watermark", "reranker")
|
||||
- location a directory or workspace path (e.g. "D:/Projects/hive-mind")
|
||||
|
||||
DO NOT EXTRACT:
|
||||
- pronouns, demonstratives ("this", "that", "these")
|
||||
- generic verbs at sentence start ("Add", "Update", "Run")
|
||||
- standalone acronyms shorter than 4 chars ("API", "CLI", "MCP", "JSON")
|
||||
- weekdays, months, dates
|
||||
- common English words
|
||||
- fragments — if you'd struggle to write a wiki page about it, skip it
|
||||
|
||||
QUALITY BAR: ~3-8 high-signal entities per frame is typical. If a frame is short or non-substantive, return zero entities for it (just don't emit lines for it).
|
||||
|
||||
Output JSONL only. No prose, no markdown fences, no commentary.`;
|
||||
|
||||
interface FrameInput {
|
||||
id: number;
|
||||
content: string;
|
||||
}
|
||||
|
||||
/** Builds the user-message text for one batch. */
|
||||
function buildBatchPrompt(frames: ReadonlyArray<FrameInput>): string {
|
||||
const sep = '='.repeat(60);
|
||||
const blocks = frames.map((f) => {
|
||||
const trimmed = f.content.trim();
|
||||
const body = trimmed.length > MAX_FRAME_CHARS
|
||||
? `${trimmed.slice(0, MAX_FRAME_CHARS)}\n[...frame truncated for extraction; ${trimmed.length - MAX_FRAME_CHARS} chars omitted]`
|
||||
: trimmed;
|
||||
return `${sep}\nFRAME id=${f.id}\n${sep}\n${body}`;
|
||||
}).join('\n\n');
|
||||
return `${PROMPT_INSTRUCTIONS}\n\n${blocks}\n\n=== END OF FRAMES ===\n\nNow output JSONL:`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strips a fenced code block the model sometimes adds despite instructions.
|
||||
* Returns the inner content if a single \`\`\`...\`\`\` block wraps everything,
|
||||
* else the original text.
|
||||
*/
|
||||
function unwrapFencedBlock(text: string): string {
|
||||
const trimmed = text.trim();
|
||||
const fenceMatch = trimmed.match(/^```(?:json|jsonl)?\s*\n([\s\S]*?)\n```\s*$/);
|
||||
return fenceMatch ? fenceMatch[1] : trimmed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses LLM JSONL output into typed entities. Per-line tolerance: a single
|
||||
* malformed line never aborts the batch. Dropped lines:
|
||||
* - unparseable JSON
|
||||
* - frame_id not in this batch (the model invented one — unattributable)
|
||||
* - type outside KG_ENTITY_TYPES (stricter than OSS, which coerced to
|
||||
* 'concept' — validate at the boundary instead of laundering junk types)
|
||||
* - names failing the write-time noise filter (isNoiseName)
|
||||
* - names carrying an injection payload (scanned BEFORE returning — LLM
|
||||
* output over harvested content is tainted input)
|
||||
*/
|
||||
function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEntity[] {
|
||||
const entities: KgEntity[] = [];
|
||||
const cleaned = unwrapFencedBlock(raw);
|
||||
|
||||
for (const line of cleaned.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || !trimmed.startsWith('{')) continue;
|
||||
|
||||
let parsed: Record<string, unknown>;
|
||||
try {
|
||||
parsed = JSON.parse(trimmed) as Record<string, unknown>;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const frameId = Number(parsed.frame_id);
|
||||
if (!Number.isFinite(frameId) || !validFrameIds.has(frameId)) continue;
|
||||
|
||||
const name = typeof parsed.name === 'string' ? parsed.name.trim() : '';
|
||||
if (name.length < 2) continue;
|
||||
// Write-time noise filter (oss-drift R3 — first wiring): stop tokens,
|
||||
// sub-4-char names, single-word acronyms never enter the graph.
|
||||
if (isNoiseName(name)) continue;
|
||||
|
||||
const rawType = typeof parsed.type === 'string' ? parsed.type.toLowerCase().trim() : '';
|
||||
if (!(KG_ENTITY_TYPES as readonly string[]).includes(rawType)) continue;
|
||||
|
||||
const scan = scanForInjection(name, 'tool_output');
|
||||
if (!scan.safe) {
|
||||
log.warn('dropping extracted entity name with injection payload', { flags: scan.flags.join(',') });
|
||||
continue;
|
||||
}
|
||||
|
||||
entities.push({ frameId, name, type: rawType as KgEntityType });
|
||||
}
|
||||
|
||||
return entities;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract typed KG entities from N frames via the LLM. Internally batches
|
||||
* into groups of BATCH_SIZE, one 'fast'-tier call per batch. Per-batch
|
||||
* failures are collected into `errors`, never thrown.
|
||||
*/
|
||||
export async function extractKgEntities(
|
||||
datedFrames: ReadonlyArray<FrameInput>,
|
||||
llmCall: LLMCallFn,
|
||||
): Promise<KgEntityExtraction> {
|
||||
const out: KgEntityExtraction = { entities: [], errors: [] };
|
||||
if (datedFrames.length === 0) return out;
|
||||
|
||||
for (let i = 0; i < datedFrames.length; i += BATCH_SIZE) {
|
||||
const batch = datedFrames.slice(i, i + BATCH_SIZE);
|
||||
const validIds = new Set(batch.map((f) => f.id));
|
||||
try {
|
||||
const raw = await llmCall(buildBatchPrompt(batch), 'fast');
|
||||
out.entities.push(...parseJsonlOutput(raw, validIds));
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
out.errors.push(`kg-entities batch ${i / BATCH_SIZE} (frames ${batch[0].id}..${batch[batch.length - 1].id}): ${msg}`);
|
||||
log.warn('kg-entity extraction batch failed', { batch: i / BATCH_SIZE, error: msg });
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Graph writing ────────────────────────────────────────────────────────────
|
||||
|
||||
export interface WriteKgEntitiesResult {
|
||||
/** New knowledge_entities rows. */
|
||||
created: number;
|
||||
/** Existing entities whose seen_count was bumped (exact-name dedup hit). */
|
||||
updated: number;
|
||||
}
|
||||
|
||||
function safeParseProps(raw: string | undefined | null): Record<string, unknown> {
|
||||
if (!raw) return {};
|
||||
try { return JSON.parse(raw) as Record<string, unknown>; } catch { return {}; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist extracted entities into the knowledge graph.
|
||||
*
|
||||
* Dedup is exact-name via `kg.findEntityByName()` (oss-drift R2) — NEVER the
|
||||
* LIKE-based `searchEntities` top-K, which silently drops the exact match once
|
||||
* enough similarly-named entities accumulate (3506 duplicate "Phase" rows
|
||||
* observed in the OSS repo). On a hit, seen_count is bumped the same way the
|
||||
* cognify CLI does; on a miss, a new row is created with source 'cognify-llm'
|
||||
* (the tag that distinguishes LLM-grade entities from heuristic noise).
|
||||
*/
|
||||
export function writeKgEntities(
|
||||
kg: KnowledgeGraph,
|
||||
extraction: KgEntityExtraction,
|
||||
): WriteKgEntitiesResult {
|
||||
const result: WriteKgEntitiesResult = { created: 0, updated: 0 };
|
||||
|
||||
for (const entity of extraction.entities) {
|
||||
// Defense at the write seam (mirrors the cognify CLI): callers other than
|
||||
// extractKgEntities may not have noise-filtered.
|
||||
if (isNoiseName(entity.name)) continue;
|
||||
if (normalizeEntityName(entity.name).length < 3) continue;
|
||||
|
||||
const existing = kg.findEntityByName(entity.name);
|
||||
if (existing) {
|
||||
const existingProps = safeParseProps(existing.properties);
|
||||
const seenCount = Number(existingProps.seen_count ?? 1) + 1;
|
||||
kg.updateEntity(existing.id, {
|
||||
properties: { ...existingProps, seen_count: seenCount },
|
||||
});
|
||||
kg.linkEntityToFrame(existing.id, entity.frameId);
|
||||
result.updated++;
|
||||
} else {
|
||||
try {
|
||||
const created = kg.createEntity(entity.type, entity.name, { seen_count: 1, source: 'cognify-llm' });
|
||||
kg.linkEntityToFrame(created.id, entity.frameId);
|
||||
result.created++;
|
||||
} catch (e: unknown) {
|
||||
// Ontology validation may reject — skip this entity, never abort the pass.
|
||||
log.warn('createEntity rejected extracted entity', {
|
||||
name: entity.name,
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
335
packages/hive-mind-core/src/harvest/extract-memory-lanes.ts
Normal file
335
packages/hive-mind-core/src/harvest/extract-memory-lanes.ts
Normal file
@@ -0,0 +1,335 @@
|
||||
/**
|
||||
* extract-memory-lanes.ts — W4.3 extraction passes for the benchmark-proven
|
||||
* recall lanes (W4-PRODUCTION-PORT-PLAN-2026-06-11.md components #5/#6/#8).
|
||||
*
|
||||
* Three single-call LLM passes over conversation text, production
|
||||
* generalizations of the LoCoMo-validated extraction scripts (28/31 dense
|
||||
* facts, 33 episodic events, 35 profile cards — evidence: 87.66 overall,
|
||||
* benchmarks/results/memori-phase22-RESULT.md):
|
||||
*
|
||||
* - DENSE FACTS → `[mind-fact]` cross-session syntheses
|
||||
* - EPISODIC → `[mind-event]` datable events, created_at =
|
||||
* LLM-resolved EVENT date
|
||||
* - PROFILES → `[mind-profile <name>]` per-person persona cards,
|
||||
* importance DELIBERATELY 'normal'
|
||||
* (stays out of the K5 lane)
|
||||
*
|
||||
* Frames are prefix-tagged on their first line so recall lanes fetch them by
|
||||
* `content LIKE '[mind-… %'` — the same convention the benchmark proved.
|
||||
* All LLM output is injection-scanned before any frame write (LLM passes run
|
||||
* over possibly-tainted harvested content).
|
||||
*
|
||||
* Model: LLMCallFn 'fast' tier (benchmark used gpt-4o-mini, temp 0). Ollama
|
||||
* is an optional routing target via LiteLLM — no hard dependency.
|
||||
*/
|
||||
|
||||
import type { LLMCallFn } from './pipeline.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
import type { FrameStore } from '../mind/frames.js';
|
||||
|
||||
const log = createCoreLogger('extract-memory-lanes');
|
||||
|
||||
/** First-line content prefixes for the three lanes (recall fetches by these). */
|
||||
export const MIND_FACT_PREFIX = '[mind-fact]';
|
||||
export const MIND_EVENT_PREFIX = '[mind-event]';
|
||||
export const MIND_PROFILE_PREFIX = '[mind-profile';
|
||||
|
||||
export interface ExtractedEvent {
|
||||
/** ISO date of the session/source the event was narrated in. */
|
||||
session_date: string;
|
||||
/** Relative cue found in the utterance ("yesterday"…), or "none". */
|
||||
cue: string;
|
||||
/** LLM-resolved date the event actually happened (YYYY-MM-DD). */
|
||||
event_date: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface ExtractedFact {
|
||||
category: 'preference' | 'decision' | 'trait' | 'theme';
|
||||
speaker: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export interface ExtractedProfile {
|
||||
speaker: string;
|
||||
card: string;
|
||||
}
|
||||
|
||||
export interface MemoryLaneExtraction {
|
||||
facts: ExtractedFact[];
|
||||
events: ExtractedEvent[];
|
||||
profiles: ExtractedProfile[];
|
||||
errors: string[];
|
||||
}
|
||||
|
||||
// ── Prompts (benchmark scripts 31/33/35, generalized off the speaker pair) ──
|
||||
|
||||
const FACTS_SYSTEM =
|
||||
'You extract many synthesis-level memory facts from long-term conversations. ' +
|
||||
'Be exhaustive — cover preferences, decisions, traits, life-stances, beliefs, ' +
|
||||
'progressions, themes, hobbies, fears, joys, opinions, family relationships, ' +
|
||||
'professional details. Output ONLY the JSON, no preamble.';
|
||||
|
||||
function factsPrompt(text: string): string {
|
||||
return `${FACTS_SYSTEM}
|
||||
|
||||
Conversation/source material:
|
||||
|
||||
${text}
|
||||
|
||||
---
|
||||
|
||||
Extract a DENSE set of synthesis-level memory facts (scale the count to the material; up to ~60 for long conversations). Aim for HIGH COVERAGE — include facts relevant to inference questions like "would X be considered Y?" or "how does X feel about Y?".
|
||||
|
||||
Categories:
|
||||
1. preference — "User preference: <Name> [values/likes/dislikes/prefers/believes] <thing>[ because <reason>]"
|
||||
2. decision — "Decision: <Name> decided to/plans to <action>[, because <reason>]"
|
||||
3. trait — "Trait: <Name> is <trait/orientation>[, as shown by <pattern>]"
|
||||
4. theme — "Theme: <topic/arc> — <insight or progression across sessions>"
|
||||
|
||||
Be SPECIFIC and CONCRETE. Each fact stands alone. Cover all participants.
|
||||
|
||||
Output STRICT JSON:
|
||||
{"facts": [{"category": "preference|decision|trait|theme", "speaker": "Name|both", "text": "<full prefix-tagged sentence>"}, ...]}`;
|
||||
}
|
||||
|
||||
const EVENTS_SYSTEM =
|
||||
'You extract specific datable events from long-term conversations AND resolve WHEN each ' +
|
||||
'event actually happened. Source material carries dates (session headers, timestamps). ' +
|
||||
'Events are often recounted in PAST tense with relative time cues ("yesterday", "last ' +
|
||||
'week", "two months ago", "last year"). You MUST compute the ACTUAL event date by applying ' +
|
||||
'the cue to that passage\'s date — do NOT just copy the source date. If an event has no ' +
|
||||
'relative cue (happening now / present tense / planned for the future), use the source date. ' +
|
||||
'Focus on concrete things that HAPPENED: activities, places visited, milestones, purchases, ' +
|
||||
'meetings, projects, health events, travel. Do NOT include timeless preferences or ' +
|
||||
'personality traits — only events. Output ONLY the JSON.';
|
||||
|
||||
function eventsPrompt(text: string): string {
|
||||
return `${EVENTS_SYSTEM}
|
||||
|
||||
Source material (dated):
|
||||
|
||||
${text}
|
||||
|
||||
---
|
||||
|
||||
Extract the specific datable events (scale the count to the material). For EACH event output:
|
||||
- session_date: the ISO date of the passage the event was narrated in (YYYY-MM-DD)
|
||||
- cue: the exact relative time phrase ("yesterday", "last week", "two months ago"), or "none"
|
||||
- event_date: the RESOLVED actual date the event happened (YYYY-MM-DD)
|
||||
- text: a concise sentence about what happened (names, titles, exact activities)
|
||||
|
||||
Resolution rules (apply cue to session_date):
|
||||
- "yesterday" -> session_date − 1 day
|
||||
- "the day before yesterday" -> session_date − 2 days
|
||||
- "last week" / "a week ago" -> session_date − 7 days
|
||||
- "N days/weeks ago" -> subtract that many days/weeks
|
||||
- "last month" / "a month ago" -> session_date − 1 month
|
||||
- "N months ago" -> subtract N months
|
||||
- "last year" -> same month/day, year − 1
|
||||
- "none" (present tense/now) -> event_date = session_date
|
||||
|
||||
Worked example: passage dated 2023-05-08, "I went to the support group yesterday"
|
||||
-> {"session_date":"2023-05-08","cue":"yesterday","event_date":"2023-05-07","text":"Caroline attended the LGBTQ support group"}
|
||||
|
||||
Output STRICT JSON:
|
||||
{"events": [{"session_date":"YYYY-MM-DD","cue":"...","event_date":"YYYY-MM-DD","text":"..."}, ...]}`;
|
||||
}
|
||||
|
||||
const PROFILES_SYSTEM =
|
||||
'You build dense persona profile cards from long-term conversations, aggregating ' +
|
||||
'dispersed weak signals (activities, choices, stated values, recurring themes) into a ' +
|
||||
'coherent portrait. The card must support INFERENCE questions like "would X be ' +
|
||||
'considered religious?" or "what would X\'s likely preference be?". Aggregate signals; ' +
|
||||
'include world-knowledge hooks (specific titles, brand names, place names — verbatim, ' +
|
||||
'never generalized). Output ONLY the JSON.';
|
||||
|
||||
function profilesPrompt(text: string): string {
|
||||
return `${PROFILES_SYSTEM}
|
||||
|
||||
Source material:
|
||||
|
||||
${text}
|
||||
|
||||
---
|
||||
|
||||
Build one profile card per main participant (120-180 words each). Each card MUST cover, compactly:
|
||||
- Identity & life situation (job/role, family, relationships, location if stated)
|
||||
- Interests & habits with SPECIFIC named items (exact titles, brands, activities)
|
||||
- Values, beliefs, personality leanings AS EVIDENCED
|
||||
- Major life arc events with rough dates
|
||||
- People mentioned around them and who those people likely are
|
||||
- Current state at the end of the material (latest job, plans, status)
|
||||
|
||||
Write declarative, signal-dense prose. No hedging filler. Keep verbatim named entities.
|
||||
|
||||
Output STRICT JSON:
|
||||
{"profiles": [{"speaker": "<Name>", "card": "..."}, ...]}`;
|
||||
}
|
||||
|
||||
// ── Parsing helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
||||
|
||||
function parseJsonObject(raw: string): Record<string, unknown> | null {
|
||||
const cleaned = raw.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(cleaned);
|
||||
return parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Extraction ──────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Run the three lane-extraction passes over one body of conversation text.
|
||||
* Each pass fails independently — a parse failure on one lane never blocks
|
||||
* the others (errors are collected, not thrown).
|
||||
*/
|
||||
export async function extractMemoryLanes(
|
||||
text: string,
|
||||
llmCall: LLMCallFn,
|
||||
): Promise<MemoryLaneExtraction> {
|
||||
const out: MemoryLaneExtraction = { facts: [], events: [], profiles: [], errors: [] };
|
||||
|
||||
const passes: Array<{ name: string; run: () => Promise<void> }> = [
|
||||
{
|
||||
name: 'facts',
|
||||
run: async () => {
|
||||
const obj = parseJsonObject(await llmCall(factsPrompt(text), 'fast'));
|
||||
const facts = Array.isArray(obj?.facts) ? obj.facts : [];
|
||||
for (const f of facts as Array<Record<string, unknown>>) {
|
||||
if (typeof f?.text === 'string' && f.text.trim().length > 0) {
|
||||
out.facts.push({
|
||||
category: (['preference', 'decision', 'trait', 'theme'].includes(String(f.category))
|
||||
? String(f.category)
|
||||
: 'preference') as ExtractedFact['category'],
|
||||
speaker: typeof f.speaker === 'string' ? f.speaker : 'unknown',
|
||||
text: f.text.trim(),
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'events',
|
||||
run: async () => {
|
||||
const obj = parseJsonObject(await llmCall(eventsPrompt(text), 'fast'));
|
||||
const events = Array.isArray(obj?.events) ? obj.events : [];
|
||||
for (const e of events as Array<Record<string, unknown>>) {
|
||||
const eventDate = typeof e?.event_date === 'string' ? e.event_date : '';
|
||||
if (typeof e?.text === 'string' && e.text.trim().length > 0 && ISO_DATE_RE.test(eventDate)) {
|
||||
out.events.push({
|
||||
session_date: typeof e.session_date === 'string' ? e.session_date : eventDate,
|
||||
cue: typeof e.cue === 'string' ? e.cue : 'none',
|
||||
event_date: eventDate,
|
||||
text: e.text.trim(),
|
||||
});
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'profiles',
|
||||
run: async () => {
|
||||
const obj = parseJsonObject(await llmCall(profilesPrompt(text), 'fast'));
|
||||
const profiles = Array.isArray(obj?.profiles) ? obj.profiles : [];
|
||||
for (const p of profiles as Array<Record<string, unknown>>) {
|
||||
if (typeof p?.speaker === 'string' && typeof p?.card === 'string' && p.card.trim().length > 0) {
|
||||
out.profiles.push({ speaker: p.speaker.trim(), card: p.card.trim() });
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
for (const pass of passes) {
|
||||
try {
|
||||
await pass.run();
|
||||
} catch (e: unknown) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
out.errors.push(`${pass.name}: ${msg}`);
|
||||
log.warn(`memory-lane extraction pass failed`, { pass: pass.name, error: msg });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Frame writing ────────────────────────────────────────────────────────────
|
||||
|
||||
export interface WriteLaneFramesResult {
|
||||
factsWritten: number;
|
||||
eventsWritten: number;
|
||||
profilesWritten: number;
|
||||
injectionDropped: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist an extraction as prefix-tagged frames.
|
||||
*
|
||||
* - facts: `[mind-fact]\n<text>` importance 'normal'
|
||||
* - events: `[mind-event]\n[YYYY-MM-DD] <text>` importance 'normal',
|
||||
* created_at = the RESOLVED event date (write-time temporal
|
||||
* anchoring — the production counterpart of the LoCoMo P4 win)
|
||||
* - profiles: `[mind-profile <name>]\n<card>` importance 'normal' —
|
||||
* DELIBERATELY normal so cards stay out of the importance-K5
|
||||
* lane (benchmark design decision); prior card for the same
|
||||
* person is replaced (profiles evolve, facts accumulate).
|
||||
*
|
||||
* createIFrame's content dedup makes fact/event writes idempotent across
|
||||
* re-runs. Every item is injection-scanned before write — extraction output
|
||||
* derives from possibly-tainted harvested content.
|
||||
*/
|
||||
export function writeMemoryLaneFrames(
|
||||
frames: FrameStore,
|
||||
gopId: string,
|
||||
extraction: MemoryLaneExtraction,
|
||||
): WriteLaneFramesResult {
|
||||
const result: WriteLaneFramesResult = {
|
||||
factsWritten: 0, eventsWritten: 0, profilesWritten: 0, injectionDropped: 0,
|
||||
};
|
||||
|
||||
const safe = (text: string): boolean => {
|
||||
const scan = scanForInjection(text, 'tool_output');
|
||||
if (!scan.safe) {
|
||||
result.injectionDropped++;
|
||||
log.warn('dropping extracted item with injection payload', { flags: scan.flags.join(',') });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
for (const f of extraction.facts) {
|
||||
if (!safe(f.text)) continue;
|
||||
frames.createIFrame(gopId, `${MIND_FACT_PREFIX}\n${f.text}`, 'normal', 'system');
|
||||
result.factsWritten++;
|
||||
}
|
||||
|
||||
for (const e of extraction.events) {
|
||||
if (!safe(e.text)) continue;
|
||||
frames.createIFrame(
|
||||
gopId,
|
||||
`${MIND_EVENT_PREFIX}\n[${e.event_date}] ${e.text}`,
|
||||
'normal',
|
||||
'system',
|
||||
`${e.event_date}T00:00:00.000Z`,
|
||||
);
|
||||
result.eventsWritten++;
|
||||
}
|
||||
|
||||
for (const p of extraction.profiles) {
|
||||
if (!safe(p.card)) continue;
|
||||
const header = `${MIND_PROFILE_PREFIX} ${p.speaker}]`;
|
||||
// Replace-on-update: a person's card supersedes the previous one.
|
||||
frames.deleteByContentPrefix(header);
|
||||
frames.createIFrame(gopId, `${header}\n${p.card}`, 'normal', 'system');
|
||||
result.profilesWritten++;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
164
packages/hive-mind-core/src/harvest/gemini-adapter.ts
Normal file
164
packages/hive-mind-core/src/harvest/gemini-adapter.ts
Normal file
@@ -0,0 +1,164 @@
|
||||
/**
|
||||
* Gemini Adapter — parse Google Takeout / Gemini conversation exports.
|
||||
*
|
||||
* Supports both Google Takeout format and direct Gemini API export.
|
||||
*/
|
||||
|
||||
import { stableHarvestId } from './stable-id.js';
|
||||
import type { SourceAdapter, UniversalImportItem, ConversationMessage } from './types.js';
|
||||
import { asRecord, firstString, getArray, getString, type RawRecord } from './raw-types.js';
|
||||
|
||||
export class GeminiAdapter implements SourceAdapter {
|
||||
readonly sourceType = 'gemini' as const;
|
||||
readonly displayName = 'Gemini';
|
||||
|
||||
parse(input: unknown): UniversalImportItem[] {
|
||||
// Handle different Gemini export formats
|
||||
if (Array.isArray(input)) {
|
||||
return this.parseConversationArray(input);
|
||||
}
|
||||
|
||||
const root = asRecord(input);
|
||||
if (!root) return [];
|
||||
|
||||
// Google Takeout format: { conversations: [...] }
|
||||
const conversations = getArray(root, 'conversations');
|
||||
if (conversations) {
|
||||
return this.parseConversationArray(conversations);
|
||||
}
|
||||
|
||||
// Gemini API history format: { history: [...] }
|
||||
if (getArray(root, 'history')) {
|
||||
return this.parseSingleConversation(root);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
private parseConversationArray(conversations: unknown[]): UniversalImportItem[] {
|
||||
const items: UniversalImportItem[] = [];
|
||||
|
||||
for (const rawConv of conversations) {
|
||||
const conv = asRecord(rawConv);
|
||||
if (!conv) continue;
|
||||
const title = firstString(conv, 'title', 'name') ?? 'Untitled';
|
||||
const messages: ConversationMessage[] = [];
|
||||
|
||||
const entries = getArray(conv, 'messages') ?? getArray(conv, 'turns') ?? getArray(conv, 'history') ?? [];
|
||||
for (const rawEntry of entries) {
|
||||
const entry = asRecord(rawEntry);
|
||||
if (!entry) continue;
|
||||
const role = this.resolveRole(entry);
|
||||
if (!role || role === 'system') continue;
|
||||
|
||||
const text = this.extractText(entry);
|
||||
if (!text) continue;
|
||||
|
||||
messages.push({
|
||||
role,
|
||||
text,
|
||||
timestamp: firstString(entry, 'createTime', 'create_time', 'timestamp'),
|
||||
});
|
||||
}
|
||||
|
||||
if (messages.length === 0) continue;
|
||||
|
||||
items.push({
|
||||
// #7 sticky erasure: conversation id (Takeout export id), not content.
|
||||
id: stableHarvestId('gemini', firstString(conv, 'id', 'conversationId') ?? `${title}\x00${firstString(conv, 'createTime', 'create_time', 'created_at') ?? ''}`),
|
||||
source: 'gemini',
|
||||
type: 'conversation',
|
||||
title,
|
||||
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
|
||||
messages,
|
||||
timestamp: firstString(conv, 'createTime', 'create_time', 'created_at') ?? new Date().toISOString(),
|
||||
metadata: {
|
||||
conversationId: firstString(conv, 'id', 'conversationId'),
|
||||
messageCount: messages.length,
|
||||
model: firstString(conv, 'model', 'modelVersion'),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private parseSingleConversation(conv: RawRecord): UniversalImportItem[] {
|
||||
const messages: ConversationMessage[] = [];
|
||||
const entries = getArray(conv, 'history') ?? [];
|
||||
|
||||
for (const rawEntry of entries) {
|
||||
const entry = asRecord(rawEntry);
|
||||
if (!entry) continue;
|
||||
const role = this.resolveRole(entry);
|
||||
if (!role || role === 'system') continue;
|
||||
const text = this.extractText(entry);
|
||||
if (!text) continue;
|
||||
messages.push({ role, text });
|
||||
}
|
||||
|
||||
if (messages.length === 0) return [];
|
||||
|
||||
return [{
|
||||
// {history} API dump carries no id — title+model is the only stable surrogate
|
||||
// (documented collision risk for two same-title+model dumps; no better anchor).
|
||||
id: stableHarvestId('gemini', getString(conv, 'title') ?? 'Gemini Conversation', getString(conv, 'model') ?? ''),
|
||||
source: 'gemini',
|
||||
type: 'conversation',
|
||||
title: getString(conv, 'title') ?? 'Gemini Conversation',
|
||||
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
|
||||
messages,
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: { model: getString(conv, 'model') },
|
||||
}];
|
||||
}
|
||||
|
||||
private resolveRole(entry: RawRecord): 'user' | 'assistant' | 'system' | null {
|
||||
const role = firstString(entry, 'role', 'author', 'sender');
|
||||
if (!role) return null;
|
||||
const r = role.toLowerCase();
|
||||
if (r === 'user' || r === 'human') return 'user';
|
||||
if (r === 'model' || r === 'assistant' || r === 'gemini') return 'assistant';
|
||||
if (r === 'system') return 'system';
|
||||
return null;
|
||||
}
|
||||
|
||||
private extractText(entry: RawRecord): string {
|
||||
// Gemini parts format: { parts: [{ text: "..." }] }
|
||||
const parts = getArray(entry, 'parts');
|
||||
if (parts) {
|
||||
return parts
|
||||
.map(asRecord)
|
||||
.map(p => {
|
||||
if (!p) return undefined;
|
||||
const t = getString(p, 'text');
|
||||
if (t !== undefined) return t;
|
||||
// W4.4 (caption parity): media parts were silently dropped.
|
||||
// Surface the text-bearing fields the export carries — file
|
||||
// URI/name for fileData, mime type as a presence signal for
|
||||
// inline images ("did X share a photo?" questions).
|
||||
const fd = asRecord(p.fileData) ?? asRecord(p.file_data);
|
||||
if (fd) {
|
||||
const uri = getString(fd, 'fileUri') ?? getString(fd, 'file_uri') ?? getString(fd, 'displayName');
|
||||
const mime = getString(fd, 'mimeType') ?? getString(fd, 'mime_type');
|
||||
return `[Shared file: ${uri ?? mime ?? 'media'}]`;
|
||||
}
|
||||
const il = asRecord(p.inlineData) ?? asRecord(p.inline_data);
|
||||
if (il) {
|
||||
const mime = getString(il, 'mimeType') ?? getString(il, 'mime_type');
|
||||
return mime ? `[Shared media: ${mime}]` : undefined;
|
||||
}
|
||||
return undefined;
|
||||
})
|
||||
.filter((t): t is string => typeof t === 'string')
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
// Simple text field
|
||||
const text = getString(entry, 'text');
|
||||
if (text !== undefined) return text.trim();
|
||||
const content = getString(entry, 'content');
|
||||
if (content !== undefined) return content.trim();
|
||||
return '';
|
||||
}
|
||||
}
|
||||
45
packages/hive-mind-core/src/harvest/index.ts
Normal file
45
packages/hive-mind-core/src/harvest/index.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
export * from './types.js';
|
||||
export { chunkByParagraphs } from './chunk-utils.js';
|
||||
export { CLASSIFY_PROMPT, EXTRACT_PROMPT, SYNTHESIZE_PROMPT } from './prompts.js';
|
||||
export { HarvestSourceStore } from './source-store.js';
|
||||
export { HarvestRunStore, type HarvestRun, type HarvestRunStatus } from './run-store.js';
|
||||
export { dedup, harvestSetHash, type DedupResult } from './dedup.js';
|
||||
export { asRecord, getString, getNumber, getArray, firstString, type RawRecord } from './raw-types.js';
|
||||
export { ChatGPTAdapter } from './chatgpt-adapter.js';
|
||||
export { ClaudeAdapter } from './claude-adapter.js';
|
||||
export { ClaudeCodeAdapter } from './claude-code-adapter.js';
|
||||
export { GeminiAdapter } from './gemini-adapter.js';
|
||||
export { PerplexityAdapter } from './perplexity-adapter.js';
|
||||
export { UniversalAdapter } from './universal-adapter.js';
|
||||
export { MarkdownAdapter } from './markdown-adapter.js';
|
||||
export { PlaintextAdapter } from './plaintext-adapter.js';
|
||||
export { UrlAdapter } from './url-adapter.js';
|
||||
export { PdfAdapter } from './pdf-adapter.js';
|
||||
export { HarvestPipeline, type LLMCallFn, type PipelineOptions } from './pipeline.js';
|
||||
|
||||
// Memory-lane extraction passes (facts / events / profiles). Ported from hive-mind a99ea0e.
|
||||
export {
|
||||
extractMemoryLanes,
|
||||
writeMemoryLaneFrames,
|
||||
MIND_FACT_PREFIX,
|
||||
MIND_EVENT_PREFIX,
|
||||
MIND_PROFILE_PREFIX,
|
||||
type ExtractedFact,
|
||||
type ExtractedEvent,
|
||||
type ExtractedProfile,
|
||||
type MemoryLaneExtraction,
|
||||
type WriteLaneFramesResult,
|
||||
} from './extract-memory-lanes.js';
|
||||
|
||||
// Per-turn verbatim dialogue storage (raw-detail lane, write side). Ported from hive-mind a99ea0e.
|
||||
export {
|
||||
writeRawTurnFrames,
|
||||
rawTurnHeader,
|
||||
parseRawTurnHeader,
|
||||
rawTurnConvKey,
|
||||
MIND_RAWTURN_PREFIX,
|
||||
MAX_TURNS_PER_ITEM,
|
||||
RAWDETAIL_KILL_SWITCH,
|
||||
type WriteRawTurnsResult,
|
||||
type ParsedRawTurnHeader,
|
||||
} from './raw-turns.js';
|
||||
142
packages/hive-mind-core/src/harvest/markdown-adapter.ts
Normal file
142
packages/hive-mind-core/src/harvest/markdown-adapter.ts
Normal file
@@ -0,0 +1,142 @@
|
||||
/**
|
||||
* Markdown Source Adapter — parses .md files into importable items.
|
||||
*
|
||||
* Splits markdown by top-level headings (# or ##).
|
||||
* Each section becomes a separate UniversalImportItem.
|
||||
* Extracts entities from heading names and bold terms.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import type { SourceAdapter, UniversalImportItem } from './types.js';
|
||||
|
||||
interface MarkdownSection {
|
||||
heading: string;
|
||||
level: number;
|
||||
content: string;
|
||||
}
|
||||
|
||||
function splitByHeadings(text: string): MarkdownSection[] {
|
||||
const lines = text.split('\n');
|
||||
const sections: MarkdownSection[] = [];
|
||||
let currentHeading = '';
|
||||
let currentLevel = 0;
|
||||
let currentLines: string[] = [];
|
||||
|
||||
for (const line of lines) {
|
||||
const headingMatch = line.match(/^(#{1,3})\s+(.+)/);
|
||||
if (headingMatch) {
|
||||
// Flush previous section
|
||||
if (currentLines.length > 0 || currentHeading) {
|
||||
sections.push({
|
||||
heading: currentHeading,
|
||||
level: currentLevel,
|
||||
content: currentLines.join('\n').trim(),
|
||||
});
|
||||
}
|
||||
currentHeading = headingMatch[2].trim();
|
||||
currentLevel = headingMatch[1].length;
|
||||
currentLines = [];
|
||||
} else {
|
||||
currentLines.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
// Flush last section
|
||||
if (currentLines.length > 0 || currentHeading) {
|
||||
sections.push({
|
||||
heading: currentHeading,
|
||||
level: currentLevel,
|
||||
content: currentLines.join('\n').trim(),
|
||||
});
|
||||
}
|
||||
|
||||
return sections.filter(s => s.content.length > 0);
|
||||
}
|
||||
|
||||
function extractBoldTerms(text: string): string[] {
|
||||
const matches = text.matchAll(/\*\*([^*]+)\*\*/g);
|
||||
const terms: string[] = [];
|
||||
for (const m of matches) {
|
||||
const term = m[1].trim();
|
||||
if (term.length > 1 && term.length < 80) {
|
||||
terms.push(term);
|
||||
}
|
||||
}
|
||||
return [...new Set(terms)];
|
||||
}
|
||||
|
||||
export class MarkdownAdapter implements SourceAdapter {
|
||||
readonly sourceType = 'markdown' as const;
|
||||
readonly displayName = 'Markdown';
|
||||
|
||||
parse(input: unknown): UniversalImportItem[] {
|
||||
if (typeof input !== 'string') return [];
|
||||
|
||||
// Input can be a file path or raw markdown content
|
||||
let content: string;
|
||||
let sourcePath: string | undefined;
|
||||
|
||||
if (input.length < 500 && !input.includes('\n')) {
|
||||
// Likely a file path
|
||||
try {
|
||||
if (fs.existsSync(input)) {
|
||||
content = fs.readFileSync(input, 'utf-8');
|
||||
sourcePath = input;
|
||||
} else {
|
||||
// Treat as raw content
|
||||
content = input;
|
||||
}
|
||||
} catch {
|
||||
content = input;
|
||||
}
|
||||
} else {
|
||||
content = input;
|
||||
}
|
||||
|
||||
if (!content.trim()) return [];
|
||||
|
||||
const sections = splitByHeadings(content);
|
||||
|
||||
// If no headings found, treat entire content as one item
|
||||
if (sections.length === 0) {
|
||||
return [{
|
||||
id: randomUUID(),
|
||||
source: 'markdown',
|
||||
type: 'document',
|
||||
title: sourcePath ? sourcePath.split(/[\\/]/).pop()?.replace('.md', '') ?? 'Document' : 'Document',
|
||||
content: content.slice(0, 4000),
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
...(sourcePath && { filePath: sourcePath }),
|
||||
contentType: 'note',
|
||||
},
|
||||
}];
|
||||
}
|
||||
|
||||
const items: UniversalImportItem[] = [];
|
||||
const docTitle = sourcePath?.split(/[\\/]/).pop()?.replace('.md', '');
|
||||
|
||||
for (const section of sections) {
|
||||
const boldTerms = extractBoldTerms(section.content);
|
||||
const entities = boldTerms.slice(0, 10).map(t => ({ name: t, type: 'concept' }));
|
||||
|
||||
items.push({
|
||||
id: randomUUID(),
|
||||
source: 'markdown',
|
||||
type: 'document',
|
||||
title: section.heading || docTitle || 'Untitled section',
|
||||
content: section.content.slice(0, 4000),
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
...(sourcePath && { filePath: sourcePath }),
|
||||
headingLevel: section.level,
|
||||
contentType: 'note',
|
||||
...(entities.length > 0 && { entities }),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
}
|
||||
97
packages/hive-mind-core/src/harvest/pdf-adapter.ts
Normal file
97
packages/hive-mind-core/src/harvest/pdf-adapter.ts
Normal file
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* PDF Source Adapter — extracts text from PDF files.
|
||||
*
|
||||
* Uses pdf-parse as an optional dependency.
|
||||
* If pdf-parse is not installed, provides a clear error message.
|
||||
*
|
||||
* Splits PDF text by pages, groups into ~3000 char chunks.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import type { SourceAdapter, UniversalImportItem } from './types.js';
|
||||
import { chunkByParagraphs } from './chunk-utils.js';
|
||||
|
||||
const MAX_CHUNK_LENGTH = 3000;
|
||||
|
||||
export class PdfAdapter implements SourceAdapter {
|
||||
readonly sourceType = 'pdf' as const;
|
||||
readonly displayName = 'PDF Document';
|
||||
|
||||
parse(_input: unknown): UniversalImportItem[] {
|
||||
// Synchronous parse not supported for PDF — use parseFile()
|
||||
return [];
|
||||
}
|
||||
|
||||
/** Parse a PDF file from a file path. */
|
||||
async parseFile(filePath: string): Promise<UniversalImportItem[]> {
|
||||
// Dynamic import — pdf-parse is optional
|
||||
let PDFParseClass: unknown;
|
||||
try {
|
||||
const mod = await import('pdf-parse');
|
||||
PDFParseClass = mod.PDFParse;
|
||||
} catch {
|
||||
throw new Error(
|
||||
'pdf-parse is not installed. Install it with: npm install pdf-parse\n'
|
||||
+ 'Then retry the import.',
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof PDFParseClass !== 'function') {
|
||||
throw new Error('pdf-parse module found but PDFParse class not available.');
|
||||
}
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
throw new Error(`PDF file not found: ${filePath}`);
|
||||
}
|
||||
|
||||
const buffer = fs.readFileSync(filePath);
|
||||
// PDFParse constructor takes { data: Buffer|Uint8Array }
|
||||
const parser = new (PDFParseClass as new (opts: { data: Buffer }) => {
|
||||
load(): Promise<void>;
|
||||
getText(params?: object): Promise<{ text: string; pages: { text: string }[] }>;
|
||||
getInfo(params?: object): Promise<{ info: Record<string, string>; numPages: number }>;
|
||||
destroy(): Promise<void>;
|
||||
})({ data: buffer });
|
||||
|
||||
await parser.load();
|
||||
|
||||
const textResult = await parser.getText();
|
||||
let infoResult: { info: Record<string, string>; numPages: number } | undefined;
|
||||
try {
|
||||
infoResult = await parser.getInfo();
|
||||
} catch { /* info extraction is non-fatal */ }
|
||||
|
||||
await parser.destroy();
|
||||
|
||||
const fullText = textResult.text ?? '';
|
||||
if (fullText.trim().length < 10) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const info = infoResult?.info ?? {};
|
||||
const numPages = infoResult?.numPages ?? 0;
|
||||
const docTitle = info['Title']
|
||||
?? filePath.split(/[\\/]/).pop()?.replace('.pdf', '')
|
||||
?? 'PDF Document';
|
||||
|
||||
const chunks = chunkByParagraphs(fullText, MAX_CHUNK_LENGTH);
|
||||
|
||||
return chunks.map((chunk, i) => ({
|
||||
id: randomUUID(),
|
||||
source: 'pdf' as const,
|
||||
type: 'document' as const,
|
||||
title: chunks.length > 1 ? `${docTitle} (part ${i + 1})` : docTitle,
|
||||
content: chunk.slice(0, 4000),
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
filePath,
|
||||
contentType: 'paper' as const,
|
||||
pages: numPages,
|
||||
...(info['Author'] && { author: info['Author'] }),
|
||||
part: i + 1,
|
||||
totalParts: chunks.length,
|
||||
},
|
||||
}));
|
||||
}
|
||||
}
|
||||
163
packages/hive-mind-core/src/harvest/perplexity-adapter.ts
Normal file
163
packages/hive-mind-core/src/harvest/perplexity-adapter.ts
Normal file
@@ -0,0 +1,163 @@
|
||||
/**
|
||||
* Perplexity Adapter — parse Perplexity conversation exports.
|
||||
*
|
||||
* Perplexity's export (via account → settings → data export) delivers
|
||||
* threads as JSON. Two shapes seen in the wild:
|
||||
*
|
||||
* {
|
||||
* "threads": [{ "id", "title", "created_at", "messages": [...] }]
|
||||
* }
|
||||
*
|
||||
* or a bare array of threads. Also handles a per-thread "messages"
|
||||
* variant where each message has: role, content, sources? (citations).
|
||||
*
|
||||
* Sources/citations per assistant message are flattened into the text
|
||||
* body as "Sources: <url1>, <url2>" — they're the distinguishing
|
||||
* feature of Perplexity answers and should survive into the harvest
|
||||
* pipeline for downstream attribution.
|
||||
*/
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { SourceAdapter, UniversalImportItem, ConversationMessage } from './types.js';
|
||||
|
||||
export class PerplexityAdapter implements SourceAdapter {
|
||||
readonly sourceType = 'perplexity' as const;
|
||||
readonly displayName = 'Perplexity';
|
||||
|
||||
parse(input: unknown): UniversalImportItem[] {
|
||||
if (Array.isArray(input)) {
|
||||
return this.parseThreadArray(input);
|
||||
}
|
||||
|
||||
const root = input as Record<string, unknown> | null;
|
||||
if (!root || typeof root !== 'object') return [];
|
||||
|
||||
// Common wrapper keys observed in Perplexity exports
|
||||
if (Array.isArray(root.threads)) {
|
||||
return this.parseThreadArray(root.threads as unknown[]);
|
||||
}
|
||||
if (Array.isArray(root.conversations)) {
|
||||
return this.parseThreadArray(root.conversations as unknown[]);
|
||||
}
|
||||
if (Array.isArray(root.items)) {
|
||||
return this.parseThreadArray(root.items as unknown[]);
|
||||
}
|
||||
|
||||
// Single-thread shape: the root itself has messages
|
||||
if (Array.isArray(root.messages)) {
|
||||
return this.parseSingleThread(root);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
private parseThreadArray(threads: unknown[]): UniversalImportItem[] {
|
||||
const items: UniversalImportItem[] = [];
|
||||
for (const raw of threads) {
|
||||
const thread = raw as Record<string, unknown> | null;
|
||||
if (!thread || typeof thread !== 'object') continue;
|
||||
const built = this.buildItem(thread);
|
||||
if (built) items.push(built);
|
||||
}
|
||||
return items;
|
||||
}
|
||||
|
||||
private parseSingleThread(thread: Record<string, unknown>): UniversalImportItem[] {
|
||||
const built = this.buildItem(thread);
|
||||
return built ? [built] : [];
|
||||
}
|
||||
|
||||
private buildItem(thread: Record<string, unknown>): UniversalImportItem | null {
|
||||
const rawMessages = (thread.messages ?? thread.turns ?? []) as unknown[];
|
||||
if (!Array.isArray(rawMessages) || rawMessages.length === 0) return null;
|
||||
|
||||
const messages: ConversationMessage[] = [];
|
||||
for (const rawMsg of rawMessages) {
|
||||
const msg = rawMsg as Record<string, unknown> | null;
|
||||
if (!msg) continue;
|
||||
const role = this.resolveRole(msg);
|
||||
if (!role || role === 'system') continue;
|
||||
|
||||
const text = this.extractText(msg);
|
||||
if (!text) continue;
|
||||
|
||||
// Flatten citations/sources into the text so they survive the pipeline.
|
||||
const sources = this.extractSources(msg);
|
||||
const textWithSources = sources.length > 0
|
||||
? `${text}\n\nSources: ${sources.join(', ')}`
|
||||
: text;
|
||||
|
||||
messages.push({
|
||||
role,
|
||||
text: textWithSources,
|
||||
timestamp: this.extractTimestamp(msg),
|
||||
});
|
||||
}
|
||||
|
||||
if (messages.length === 0) return null;
|
||||
|
||||
const title = (thread.title as string) ?? (thread.name as string) ?? 'Perplexity Thread';
|
||||
|
||||
return {
|
||||
id: randomUUID(),
|
||||
source: 'perplexity',
|
||||
type: 'conversation',
|
||||
title,
|
||||
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
|
||||
messages,
|
||||
timestamp: this.extractTimestamp(thread) ?? new Date().toISOString(),
|
||||
metadata: {
|
||||
threadId: thread.id ?? thread.threadId,
|
||||
messageCount: messages.length,
|
||||
hasCitations: messages.some(m => m.text.includes('Sources:')),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private resolveRole(entry: Record<string, unknown>): 'user' | 'assistant' | 'system' | null {
|
||||
const raw = entry.role ?? entry.author ?? entry.sender ?? entry.type;
|
||||
if (!raw) return null;
|
||||
const r = String(raw).toLowerCase();
|
||||
if (r === 'user' || r === 'human' || r === 'question') return 'user';
|
||||
if (r === 'assistant' || r === 'ai' || r === 'perplexity' || r === 'answer') return 'assistant';
|
||||
if (r === 'system') return 'system';
|
||||
return null;
|
||||
}
|
||||
|
||||
private extractText(entry: Record<string, unknown>): string {
|
||||
// Try common text field names
|
||||
if (typeof entry.content === 'string') return entry.content.trim();
|
||||
if (typeof entry.text === 'string') return entry.text.trim();
|
||||
if (typeof entry.answer === 'string') return entry.answer.trim();
|
||||
if (typeof entry.query === 'string') return entry.query.trim();
|
||||
|
||||
// ChatGPT-like structured content: { parts: [...] }
|
||||
const content = entry.content as Record<string, unknown> | undefined;
|
||||
if (content && Array.isArray(content.parts)) {
|
||||
return content.parts.filter(p => typeof p === 'string').join('\n').trim();
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
private extractSources(entry: Record<string, unknown>): string[] {
|
||||
const raw = entry.sources ?? entry.citations ?? entry.web_results;
|
||||
if (!Array.isArray(raw)) return [];
|
||||
const urls: string[] = [];
|
||||
for (const src of raw) {
|
||||
if (typeof src === 'string') {
|
||||
urls.push(src);
|
||||
} else if (src && typeof src === 'object') {
|
||||
const s = src as Record<string, unknown>;
|
||||
const url = s.url ?? s.link ?? s.href;
|
||||
if (typeof url === 'string') urls.push(url);
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
private extractTimestamp(entry: Record<string, unknown>): string | undefined {
|
||||
const raw = entry.timestamp ?? entry.created_at ?? entry.createdAt ?? entry.time;
|
||||
if (typeof raw === 'string') return raw;
|
||||
if (typeof raw === 'number') return new Date(raw).toISOString();
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
339
packages/hive-mind-core/src/harvest/pipeline.ts
Normal file
339
packages/hive-mind-core/src/harvest/pipeline.ts
Normal file
@@ -0,0 +1,339 @@
|
||||
/**
|
||||
* HarvestPipeline — orchestrates the 4-pass distillation pipeline.
|
||||
*
|
||||
* Pass 1: Classify (Haiku — cheap)
|
||||
* Pass 2: Extract (Sonnet — accurate)
|
||||
* Pass 3: Synthesize (Sonnet — accurate)
|
||||
* Pass 4: Dedup (local — no LLM)
|
||||
*
|
||||
* The pipeline accepts UniversalImportItems and produces DistilledKnowledge[].
|
||||
* LLM calls are batched (20 items per call) to optimize cost.
|
||||
*/
|
||||
|
||||
import type {
|
||||
UniversalImportItem, ClassifiedItem, ExtractedContent,
|
||||
DistilledKnowledge, HarvestPipelineResult, ImportSourceType,
|
||||
} from './types.js';
|
||||
import { CLASSIFY_PROMPT, EXTRACT_PROMPT, SYNTHESIZE_PROMPT } from './prompts.js';
|
||||
import { dedup } from './dedup.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('harvest-pipeline');
|
||||
|
||||
const BATCH_SIZE = 20;
|
||||
const CONCURRENCY_CAP = 3;
|
||||
|
||||
export interface LLMCallFn {
|
||||
(prompt: string, model: 'fast' | 'accurate'): Promise<string>;
|
||||
}
|
||||
|
||||
export interface PipelineOptions {
|
||||
llmCall: LLMCallFn;
|
||||
existingContents?: string[];
|
||||
onProgress?: (stage: string, current: number, total: number) => void;
|
||||
/** Items per LLM batch call (default: 20). */
|
||||
batchSize?: number;
|
||||
/** Max concurrent LLM batch calls (default: 3). */
|
||||
concurrency?: number;
|
||||
/**
|
||||
* Fallback behavior when the Pass 1 (classify) LLM call throws.
|
||||
* - `'skip'` (default, safer): drop the batch. Under-inclusion beats cost/noise inflation.
|
||||
* - `'pass-through-medium'` (legacy): promote every item to `value: 'medium'`. This is what
|
||||
* the pipeline did historically but it runs extract + synthesize on junk and pollutes
|
||||
* memory with trivial greetings/debugging loops when the classify model hiccups.
|
||||
*/
|
||||
classifyFailureFallback?: 'skip' | 'pass-through-medium';
|
||||
}
|
||||
|
||||
/** Safely parse JSON from LLM response, handling markdown code fences. */
|
||||
function parseLLMJson<T>(raw: string): T[] {
|
||||
const cleaned = raw.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
|
||||
try {
|
||||
const parsed = JSON.parse(cleaned);
|
||||
return Array.isArray(parsed) ? parsed : [parsed];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Split array into batches. */
|
||||
function batch<T>(items: T[], size: number): T[][] {
|
||||
const batches: T[][] = [];
|
||||
for (let i = 0; i < items.length; i += size) {
|
||||
batches.push(items.slice(i, i + size));
|
||||
}
|
||||
return batches;
|
||||
}
|
||||
|
||||
/** Run async tasks with a concurrency cap (tumbling window — waits for full batch before next). */
|
||||
async function runWithConcurrency<T>(
|
||||
tasks: (() => Promise<T>)[],
|
||||
cap: number,
|
||||
): Promise<T[]> {
|
||||
const results: T[] = [];
|
||||
for (let i = 0; i < tasks.length; i += cap) {
|
||||
const window = tasks.slice(i, i + cap);
|
||||
const windowResults = await Promise.all(window.map(fn => fn()));
|
||||
results.push(...windowResults);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
export class HarvestPipeline {
|
||||
private llmCall: LLMCallFn;
|
||||
private existingContents: string[];
|
||||
private onProgress?: (stage: string, current: number, total: number) => void;
|
||||
private classifyFailureFallback: 'skip' | 'pass-through-medium';
|
||||
private batchSize: number;
|
||||
private concurrency: number;
|
||||
|
||||
constructor(options: PipelineOptions) {
|
||||
this.llmCall = options.llmCall;
|
||||
this.existingContents = options.existingContents ?? [];
|
||||
this.onProgress = options.onProgress;
|
||||
this.classifyFailureFallback = options.classifyFailureFallback ?? 'skip';
|
||||
this.batchSize = options.batchSize ?? BATCH_SIZE;
|
||||
this.concurrency = options.concurrency ?? CONCURRENCY_CAP;
|
||||
}
|
||||
|
||||
async run(items: UniversalImportItem[], source: ImportSourceType): Promise<HarvestPipelineResult> {
|
||||
const startTime = Date.now();
|
||||
const errors: string[] = [];
|
||||
log.info('harvest pipeline starting', { source, itemCount: items.length, batchSize: this.batchSize, concurrency: this.concurrency });
|
||||
|
||||
// Pass 0: Injection scan — drop any item whose title or content carries a
|
||||
// prompt-injection payload (role_override / prompt_extraction / instruction_injection).
|
||||
// Harvest ingests UNTRUSTED external exports (ChatGPT/Claude/Gemini JSON dumps,
|
||||
// Perplexity shares, URL fetches). A hostile file must not flow through to the
|
||||
// LLM passes or into memory frames.
|
||||
const originalCount = items.length;
|
||||
items = items.filter((item) => {
|
||||
// Scan title + first 4KB of content — enough to catch payloads hidden in either field.
|
||||
// Using 'tool_output' context since imports are external data, weighted like tool output.
|
||||
const probe = `${item.title ?? ''}\n${(item.content ?? '').slice(0, 4000)}`;
|
||||
const scan = scanForInjection(probe, 'tool_output');
|
||||
if (!scan.safe) {
|
||||
const reason = scan.flags.join(',');
|
||||
log.warn('dropping harvest item with injection payload', {
|
||||
itemId: item.id,
|
||||
title: item.title?.slice(0, 80),
|
||||
flags: scan.flags,
|
||||
score: scan.score,
|
||||
});
|
||||
errors.push(`Blocked item "${item.title?.slice(0, 40) ?? item.id}" — injection detected (${reason})`);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
const blockedCount = originalCount - items.length;
|
||||
if (blockedCount > 0) {
|
||||
log.info(`harvest security: blocked ${blockedCount} of ${originalCount} items for injection patterns`);
|
||||
}
|
||||
|
||||
// Pass 1: Classify
|
||||
this.onProgress?.('classify', 0, items.length);
|
||||
const classified = await this.classify(items, errors);
|
||||
const valuable = classified.filter(c => c.value !== 'skip');
|
||||
|
||||
// Pass 2: Extract
|
||||
this.onProgress?.('extract', 0, valuable.length);
|
||||
const extracted = await this.extract(valuable, errors);
|
||||
|
||||
// Pass 3: Synthesize
|
||||
this.onProgress?.('synthesize', 0, extracted.length);
|
||||
const distilled = await this.synthesize(extracted, source, errors);
|
||||
|
||||
// Pass 4: Dedup
|
||||
this.onProgress?.('dedup', 0, distilled.length);
|
||||
const dedupResult = dedup(distilled, this.existingContents);
|
||||
|
||||
const durationMs = Date.now() - startTime;
|
||||
log.info('harvest pipeline complete', {
|
||||
source, itemsReceived: originalCount, classified: classified.length,
|
||||
extracted: extracted.length, distilled: distilled.length,
|
||||
unique: dedupResult.unique.length, dupsSkipped: dedupResult.duplicatesSkipped,
|
||||
errors: errors.length, durationMs,
|
||||
});
|
||||
|
||||
return {
|
||||
source, itemsReceived: originalCount,
|
||||
itemsClassified: classified.length,
|
||||
itemsSkipped: classified.length - valuable.length,
|
||||
itemsExtracted: extracted.length,
|
||||
knowledgeDistilled: dedupResult.unique,
|
||||
framesSaved: 0, // Caller handles persistence
|
||||
entitiesCreated: 0,
|
||||
relationsCreated: 0,
|
||||
identityUpdates: dedupResult.unique.filter(k => k.targetLayer === 'identity').length,
|
||||
duplicatesSkipped: dedupResult.duplicatesSkipped,
|
||||
errors,
|
||||
costUsd: 0, // Caller tracks cost
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
private async classify(items: UniversalImportItem[], errors: string[]): Promise<ClassifiedItem[]> {
|
||||
const results: ClassifiedItem[] = [];
|
||||
const batches = batch(items, this.batchSize);
|
||||
|
||||
// M3: run batches with concurrency cap instead of sequentially
|
||||
const tasks = batches.map((b, i) => async () => {
|
||||
this.onProgress?.('classify', i * this.batchSize, items.length);
|
||||
const prompt = CLASSIFY_PROMPT + b.map((item, idx) => (
|
||||
`\n--- Item ${idx} (id: ${item.id}) ---\nTitle: ${item.title}\nSource: ${item.source}\nType: ${item.type}\nContent (first 500 chars): ${item.content.slice(0, 500)}\n`
|
||||
)).join('');
|
||||
|
||||
try {
|
||||
const response = await this.llmCall(prompt, 'fast');
|
||||
const parsed = parseLLMJson<{ itemId?: string; domain?: string; value?: string; categories?: string[] }>(response);
|
||||
const batchResults: ClassifiedItem[] = [];
|
||||
|
||||
for (const entry of parsed) {
|
||||
if (!entry.itemId) {
|
||||
log.warn('classify entry missing itemId — skipping', { batchIndex: i });
|
||||
continue;
|
||||
}
|
||||
const item = b.find(it => it.id === entry.itemId);
|
||||
if (!item) {
|
||||
log.warn('classify entry itemId does not match any batch item — skipping', { itemId: entry.itemId, batchIndex: i });
|
||||
continue;
|
||||
}
|
||||
batchResults.push({
|
||||
item,
|
||||
domain: (entry.domain as ClassifiedItem['domain']) ?? 'mixed',
|
||||
value: (entry.value as ClassifiedItem['value']) ?? 'medium',
|
||||
categories: entry.categories ?? [],
|
||||
});
|
||||
}
|
||||
return batchResults;
|
||||
} catch (err) {
|
||||
errors.push(`Classify batch ${i} failed: ${err instanceof Error ? err.message : 'unknown'}`);
|
||||
if (this.classifyFailureFallback === 'pass-through-medium') {
|
||||
return b.map(item => ({ item, domain: 'mixed' as const, value: 'medium' as const, categories: [] as string[] }));
|
||||
}
|
||||
return [] as ClassifiedItem[];
|
||||
}
|
||||
});
|
||||
|
||||
const batchResults = await runWithConcurrency(tasks, this.concurrency);
|
||||
for (const br of batchResults) results.push(...br);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private async extract(classified: ClassifiedItem[], errors: string[]): Promise<ExtractedContent[]> {
|
||||
const results: ExtractedContent[] = [];
|
||||
const batches = batch(classified, this.batchSize);
|
||||
|
||||
// M3: concurrent batches with cap
|
||||
const tasks = batches.map((b, i) => async () => {
|
||||
this.onProgress?.('extract', i * this.batchSize, classified.length);
|
||||
const prompt = EXTRACT_PROMPT + b.map((c, idx) => (
|
||||
`\n--- Conversation ${idx} (id: ${c.item.id}, value: ${c.value}, categories: ${c.categories.join(',')}) ---\nTitle: ${c.item.title}\n${c.item.content.slice(0, 2000)}\n`
|
||||
)).join('');
|
||||
|
||||
try {
|
||||
const response = await this.llmCall(prompt, 'accurate');
|
||||
const parsed = parseLLMJson<{
|
||||
itemId?: string;
|
||||
decisions?: unknown[]; preferences?: unknown[]; facts?: unknown[];
|
||||
knowledge?: unknown[]; entities?: unknown[]; relations?: unknown[];
|
||||
}>(response);
|
||||
const batchResults: ExtractedContent[] = [];
|
||||
|
||||
for (const entry of parsed) {
|
||||
if (!entry.itemId) {
|
||||
log.warn('extract entry missing itemId — skipping', { batchIndex: i });
|
||||
continue;
|
||||
}
|
||||
if (!b.some(c => c.item.id === entry.itemId)) {
|
||||
log.warn('extract entry itemId does not match batch — skipping', { itemId: entry.itemId, batchIndex: i });
|
||||
continue;
|
||||
}
|
||||
batchResults.push({
|
||||
itemId: entry.itemId,
|
||||
decisions: (entry.decisions ?? []) as ExtractedContent['decisions'],
|
||||
preferences: (entry.preferences ?? []) as ExtractedContent['preferences'],
|
||||
facts: (entry.facts ?? []) as ExtractedContent['facts'],
|
||||
knowledge: (entry.knowledge ?? []) as ExtractedContent['knowledge'],
|
||||
entities: (entry.entities ?? []) as ExtractedContent['entities'],
|
||||
relations: (entry.relations ?? []) as ExtractedContent['relations'],
|
||||
});
|
||||
}
|
||||
return batchResults;
|
||||
} catch (err) {
|
||||
errors.push(`Extract batch ${i} failed: ${err instanceof Error ? err.message : 'unknown'}`);
|
||||
return [] as ExtractedContent[];
|
||||
}
|
||||
});
|
||||
|
||||
const batchResults = await runWithConcurrency(tasks, this.concurrency);
|
||||
for (const br of batchResults) results.push(...br);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private async synthesize(
|
||||
extracted: ExtractedContent[],
|
||||
source: ImportSourceType,
|
||||
errors: string[],
|
||||
): Promise<DistilledKnowledge[]> {
|
||||
const results: DistilledKnowledge[] = [];
|
||||
const batches = batch(extracted, this.batchSize);
|
||||
|
||||
// Review C2: per-item serialization with individual budget. The old flat
|
||||
// `JSON.stringify(b, null, 2).slice(0, 8000)` truncated the *middle* of the last
|
||||
// item's JSON on a long batch; parseLLMJson returned [] on the malformed tail and
|
||||
// every subsequent item silently vanished with no error. Now each item gets its
|
||||
// own PER_ITEM_BUDGET and survives regardless of batch size.
|
||||
const PER_ITEM_BUDGET = 1200;
|
||||
|
||||
// M3: concurrent batches with cap
|
||||
const tasks = batches.map((b, i) => async () => {
|
||||
this.onProgress?.('synthesize', i * this.batchSize, extracted.length);
|
||||
const serialized = b.map((ec, idx) => {
|
||||
const json = JSON.stringify(ec, null, 2);
|
||||
const trimmed = json.length > PER_ITEM_BUDGET
|
||||
? json.slice(0, PER_ITEM_BUDGET) + '\n ... (truncated — full item in trace)'
|
||||
: json;
|
||||
return `\n--- Item ${idx} (id: ${ec.itemId}) ---\n${trimmed}`;
|
||||
}).join('');
|
||||
const prompt = SYNTHESIZE_PROMPT + serialized;
|
||||
|
||||
try {
|
||||
const response = await this.llmCall(prompt, 'accurate');
|
||||
const parsed = parseLLMJson<{
|
||||
targetLayer?: string; frameType?: string; importance?: string;
|
||||
content?: string; confidence?: number;
|
||||
}>(response);
|
||||
const batchResults: DistilledKnowledge[] = [];
|
||||
|
||||
for (const entry of parsed) {
|
||||
batchResults.push({
|
||||
targetLayer: (entry.targetLayer as DistilledKnowledge['targetLayer']) ?? 'frame',
|
||||
frameType: (entry.frameType as DistilledKnowledge['frameType']) ?? 'I',
|
||||
importance: (entry.importance as DistilledKnowledge['importance']) ?? 'normal',
|
||||
content: entry.content ?? '',
|
||||
provenance: {
|
||||
originalSource: source,
|
||||
importedAt: new Date().toISOString(),
|
||||
distillationModel: 'accurate',
|
||||
confidence: entry.confidence ?? 0.7,
|
||||
pass: 3,
|
||||
},
|
||||
});
|
||||
}
|
||||
return batchResults;
|
||||
} catch (err) {
|
||||
errors.push(`Synthesize batch ${i} failed: ${err instanceof Error ? err.message : 'unknown'}`);
|
||||
return [] as DistilledKnowledge[];
|
||||
}
|
||||
});
|
||||
|
||||
const batchResults = await runWithConcurrency(tasks, this.concurrency);
|
||||
for (const br of batchResults) results.push(...br);
|
||||
|
||||
return results;
|
||||
}
|
||||
}
|
||||
61
packages/hive-mind-core/src/harvest/plaintext-adapter.ts
Normal file
61
packages/hive-mind-core/src/harvest/plaintext-adapter.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Plaintext Source Adapter — parses .txt files into importable items.
|
||||
*
|
||||
* Splits text by double-newline paragraphs.
|
||||
* Groups paragraphs into chunks of ~2000 chars max.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import * as fs from 'node:fs';
|
||||
import type { SourceAdapter, UniversalImportItem } from './types.js';
|
||||
import { chunkByParagraphs } from './chunk-utils.js';
|
||||
|
||||
export class PlaintextAdapter implements SourceAdapter {
|
||||
readonly sourceType = 'plaintext' as const;
|
||||
readonly displayName = 'Plain Text';
|
||||
|
||||
parse(input: unknown): UniversalImportItem[] {
|
||||
if (typeof input !== 'string') return [];
|
||||
|
||||
let content: string;
|
||||
let sourcePath: string | undefined;
|
||||
|
||||
// Check if input is a file path
|
||||
if (input.length < 500 && !input.includes('\n')) {
|
||||
try {
|
||||
if (fs.existsSync(input)) {
|
||||
content = fs.readFileSync(input, 'utf-8');
|
||||
sourcePath = input;
|
||||
} else {
|
||||
content = input;
|
||||
}
|
||||
} catch {
|
||||
content = input;
|
||||
}
|
||||
} else {
|
||||
content = input;
|
||||
}
|
||||
|
||||
if (!content.trim()) return [];
|
||||
|
||||
const chunks = chunkByParagraphs(content);
|
||||
const docTitle = sourcePath?.split(/[\\/]/).pop()?.replace(/\.\w+$/, '');
|
||||
|
||||
return chunks.map((chunk, i) => ({
|
||||
id: randomUUID(),
|
||||
source: 'plaintext' as const,
|
||||
type: 'document' as const,
|
||||
title: docTitle
|
||||
? (chunks.length > 1 ? `${docTitle} (part ${i + 1})` : docTitle)
|
||||
: `Text fragment ${i + 1}`,
|
||||
content: chunk.slice(0, 4000),
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
...(sourcePath && { filePath: sourcePath }),
|
||||
contentType: 'note',
|
||||
part: i + 1,
|
||||
totalParts: chunks.length,
|
||||
},
|
||||
}));
|
||||
}
|
||||
}
|
||||
72
packages/hive-mind-core/src/harvest/prompts.ts
Normal file
72
packages/hive-mind-core/src/harvest/prompts.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Harvest Pipeline Prompts — LLM prompt templates for each distillation pass.
|
||||
*/
|
||||
|
||||
export const CLASSIFY_PROMPT = `You are a knowledge classifier. For each conversation/item below, classify it.
|
||||
|
||||
Return a JSON array with one entry per item:
|
||||
[{
|
||||
"itemId": "...",
|
||||
"domain": "work" | "personal" | "technical" | "mixed",
|
||||
"value": "high" | "medium" | "low" | "skip",
|
||||
"categories": ["decision", "preference", "fact", "knowledge", "project", "identity", "trivial"]
|
||||
}]
|
||||
|
||||
Rules:
|
||||
- "skip" = greetings, trivial exchanges, "hello", "thanks", debugging loops with no insight
|
||||
- "high" = decisions, preferences, personal facts, project context, technical architecture
|
||||
- "medium" = general knowledge, research, learning
|
||||
- "low" = routine questions with generic answers
|
||||
|
||||
Items:
|
||||
`;
|
||||
|
||||
export const EXTRACT_PROMPT = `You are a knowledge extractor. For each classified conversation, extract structured knowledge.
|
||||
|
||||
Return a JSON array:
|
||||
[{
|
||||
"itemId": "...",
|
||||
"decisions": ["chose X over Y because Z"],
|
||||
"preferences": ["prefers dark mode", "likes concise responses"],
|
||||
"facts": ["works at Egzakta Group", "role is CEO"],
|
||||
"knowledge": ["React 18 concurrent features improve perceived performance"],
|
||||
"entities": [{"name": "Egzakta Group", "type": "organization"}],
|
||||
"relations": [{"source": "Marko", "target": "Egzakta Group", "relation": "works_at"}]
|
||||
}]
|
||||
|
||||
Rules:
|
||||
- Only extract what the USER stated or decided, not what the AI suggested
|
||||
- Decisions must include the reason if one was given
|
||||
- Preferences must be actionable (not "I like good code" — too vague)
|
||||
- Facts must be verifiable or specific (names, roles, companies, tech stack)
|
||||
- Entities: types are person, organization, project, technology, concept, location, event
|
||||
- Relations: use lowercase_snake_case for relation types
|
||||
|
||||
Conversations:
|
||||
`;
|
||||
|
||||
export const SYNTHESIZE_PROMPT = `You are a memory synthesizer. Convert extracted knowledge into structured memory frames.
|
||||
|
||||
For each extraction, produce frames suitable for a persistent memory system:
|
||||
|
||||
Return a JSON array:
|
||||
[{
|
||||
"targetLayer": "identity" | "frame" | "kg_entity" | "kg_relation",
|
||||
"frameType": "I",
|
||||
"importance": "critical" | "important" | "normal",
|
||||
"content": "The actual memory content, written as a clear statement",
|
||||
"confidence": 0.0-1.0
|
||||
}]
|
||||
|
||||
Rules:
|
||||
- "identity" = personal facts (name, role, company, capabilities, personality traits)
|
||||
- "frame" with importance "important" = decisions and preferences
|
||||
- "frame" with importance "normal" = general knowledge and facts
|
||||
- "kg_entity" = entities to add to the knowledge graph
|
||||
- "kg_relation" = relationships between entities
|
||||
- Content should be self-contained — readable without the original conversation
|
||||
- Deduplicate: if two items say the same thing, pick the most complete version
|
||||
- confidence: 1.0 = user explicitly stated, 0.7 = strongly implied, 0.5 = inferred
|
||||
|
||||
Extractions:
|
||||
`;
|
||||
169
packages/hive-mind-core/src/harvest/raw-turns.ts
Normal file
169
packages/hive-mind-core/src/harvest/raw-turns.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* raw-turns.ts — W4.6 per-turn verbatim dialogue storage (write side).
|
||||
*
|
||||
* The W3.4 ablation attributed the benchmark's single-hop win to the
|
||||
* RAWDETAIL escalation lane (+2.40 z=1.95; captions alone +0.26 ns) —
|
||||
* fine-grained perceptual detail survives ONLY in verbatim turns; the
|
||||
* distillation passes carry it generically. This module stores each
|
||||
* conversation turn as its own frame so the recall-side lane
|
||||
* (`mind/raw-detail-lane.ts`) can pool / CE-rerank / neighbor-expand them.
|
||||
*
|
||||
* Frame convention (first line is the lane tag; body is the verbatim turn):
|
||||
*
|
||||
* `[mind-rawturn conv:<key> turn:<n> speaker:<s>]\n<turn text>`
|
||||
*
|
||||
* - `conv:<key>` sanitized item id — groups turns of one conversation
|
||||
* - `turn:<n>` contiguous index over STORED turns (dialogue order) —
|
||||
* the ±1 adjacency key. Production frames interleave
|
||||
* across sources, so the benchmark's id-ordering trick
|
||||
* does not transfer; the index makes adjacency explicit.
|
||||
* - `speaker:<s>` sanitized role/name (no whitespace, no `]`)
|
||||
*
|
||||
* `[mind-` prefixing keeps raw turns out of the memory-lane extraction
|
||||
* cron's source material (its `NOT LIKE '[mind-%'` self-feeding guard) and
|
||||
* lets recall dedup them against the snippet lanes by content prefix.
|
||||
*
|
||||
* Every turn is injection-scanned BEFORE write: verbatim dialogue is the
|
||||
* most injection-prone frame class, and recallMemory blocks the ENTIRE
|
||||
* recall block on a scan hit — poisoned turns must die here, not there.
|
||||
*
|
||||
* Storage growth is the accepted tradeoff (Marko GO 2026-06-11, plan §6.2).
|
||||
*/
|
||||
|
||||
import type { FrameStore } from '../mind/frames.js';
|
||||
import type { UniversalImportItem } from './types.js';
|
||||
import { HARVEST_FRAME_CONTENT_CAP } from './types.js';
|
||||
import { scanForInjection } from '../injection-scanner.js';
|
||||
import { createCoreLogger } from '../logger.js';
|
||||
|
||||
const log = createCoreLogger('raw-turns');
|
||||
|
||||
/** First-line content prefix for raw-turn frames (recall lane fetches by this). */
|
||||
export const MIND_RAWTURN_PREFIX = '[mind-rawturn';
|
||||
|
||||
/** Hard per-conversation cap — backstop against pathological exports.
|
||||
* LoCoMo conversations run ~600 turns; 2000 leaves generous headroom. */
|
||||
export const MAX_TURNS_PER_ITEM = 2000;
|
||||
|
||||
/** Env kill switch (checked by CALLERS, mirrored here for the recall lane). */
|
||||
export const RAWDETAIL_KILL_SWITCH = 'WAGGLE_RAWDETAIL';
|
||||
|
||||
export interface WriteRawTurnsResult {
|
||||
written: number;
|
||||
skippedEmpty: number;
|
||||
injectionDropped: number;
|
||||
/** true when MAX_TURNS_PER_ITEM truncated the conversation (logged, never silent). */
|
||||
capped: boolean;
|
||||
}
|
||||
|
||||
/** Strict ISO-8601 gate (same contract as FrameStore.createIFrame). */
|
||||
function isIsoTimestamp(value: string | undefined): value is string {
|
||||
return typeof value === 'string'
|
||||
&& /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})$/.test(value)
|
||||
&& Number.isFinite(Date.parse(value));
|
||||
}
|
||||
|
||||
/** Sanitize a header token: keep [A-Za-z0-9_-], collapse everything else to '-'.
|
||||
* Removes `]`, whitespace, and LIKE metacharacters (% _ kept — they're safe
|
||||
* in equality-style prefix lookups because the recall lane LIKE-escapes). */
|
||||
function sanitizeToken(value: string, maxLen: number): string {
|
||||
const cleaned = value.replace(/[^A-Za-z0-9_-]+/g, '-').replace(/^-+|-+$/g, '');
|
||||
return (cleaned || 'unknown').slice(0, maxLen);
|
||||
}
|
||||
|
||||
/** Build the raw-turn header for conv/turn/speaker. Exported for the recall
|
||||
* lane's neighbor lookups (single source of truth for the format). */
|
||||
export function rawTurnHeader(convKey: string, turn: number, speaker: string): string {
|
||||
return `${MIND_RAWTURN_PREFIX} conv:${convKey} turn:${turn} speaker:${speaker}]`;
|
||||
}
|
||||
|
||||
export interface ParsedRawTurnHeader {
|
||||
conv: string;
|
||||
turn: number;
|
||||
speaker: string;
|
||||
}
|
||||
|
||||
/** Parse a raw-turn frame's first line. Returns null for non-rawturn content. */
|
||||
export function parseRawTurnHeader(content: string): ParsedRawTurnHeader | null {
|
||||
const m = content.match(/^\[mind-rawturn conv:([A-Za-z0-9_-]+) turn:(\d+) speaker:([A-Za-z0-9_-]+)\]/);
|
||||
if (!m) return null;
|
||||
return { conv: m[1], turn: parseInt(m[2], 10), speaker: m[3] };
|
||||
}
|
||||
|
||||
/** Conversation key for an import item (sanitized, stable across re-imports).
|
||||
* Params are widened to plain strings so the GDPR erasure path (which knows a
|
||||
* subject only as free-string source + source_ref) can reconstruct the exact
|
||||
* same key; UniversalImportItem's narrower fields still satisfy it. */
|
||||
export function rawTurnConvKey(item: { source: string; id: string }): string {
|
||||
return sanitizeToken(`${item.source}-${item.id}`, 64);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store each user/assistant turn of `item.messages` as a `[mind-rawturn …]`
|
||||
* frame. No-op (all zeros) when the item carries no messages.
|
||||
*
|
||||
* - turn index is contiguous over STORED turns (skips don't leave gaps —
|
||||
* ±1 adjacency stays meaningful over the stored dialogue)
|
||||
* - system messages are skipped (boilerplate, not dialogue evidence)
|
||||
* - created_at: message timestamp if valid ISO, else item timestamp, else
|
||||
* schema default (createIFrame validates again — never writes junk)
|
||||
* - importance 'normal' (stays out of the K5 importance lane), source 'import'
|
||||
* - createIFrame content-dedup makes re-imports idempotent within its
|
||||
* 500-frame recency window; source-level content hashing in the harvest
|
||||
* routes guards the wider case (unchanged exports never reach here)
|
||||
*/
|
||||
export function writeRawTurnFrames(
|
||||
frames: FrameStore,
|
||||
gopId: string,
|
||||
item: UniversalImportItem,
|
||||
): WriteRawTurnsResult {
|
||||
const result: WriteRawTurnsResult = {
|
||||
written: 0, skippedEmpty: 0, injectionDropped: 0, capped: false,
|
||||
};
|
||||
const messages = item.messages;
|
||||
if (!Array.isArray(messages) || messages.length === 0) return result;
|
||||
|
||||
const convKey = rawTurnConvKey(item);
|
||||
const itemTs = isIsoTimestamp(item.timestamp) ? item.timestamp : undefined;
|
||||
|
||||
let turn = 0;
|
||||
for (const msg of messages) {
|
||||
if (msg.role !== 'user' && msg.role !== 'assistant') continue;
|
||||
const text = (msg.text ?? '').trim();
|
||||
if (text.length === 0) {
|
||||
result.skippedEmpty++;
|
||||
continue;
|
||||
}
|
||||
if (turn >= MAX_TURNS_PER_ITEM) {
|
||||
result.capped = true;
|
||||
break;
|
||||
}
|
||||
// Scan first 4KB — same probe budget as the harvest pipeline's Pass 0.
|
||||
const scan = scanForInjection(text.slice(0, 4000), 'tool_output');
|
||||
if (!scan.safe) {
|
||||
result.injectionDropped++;
|
||||
log.warn('dropping raw turn with injection payload', {
|
||||
conv: convKey, turn, flags: scan.flags.join(','),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const speaker = sanitizeToken(msg.role, 24);
|
||||
const createdAt = isIsoTimestamp(msg.timestamp) ? msg.timestamp : itemTs;
|
||||
frames.createIFrame(
|
||||
gopId,
|
||||
`${rawTurnHeader(convKey, turn, speaker)}\n${text.slice(0, HARVEST_FRAME_CONTENT_CAP)}`,
|
||||
'normal',
|
||||
'import',
|
||||
createdAt,
|
||||
);
|
||||
result.written++;
|
||||
turn++;
|
||||
}
|
||||
|
||||
if (result.capped) {
|
||||
log.warn('raw-turn storage capped — conversation exceeds MAX_TURNS_PER_ITEM', {
|
||||
conv: convKey, stored: result.written, totalMessages: messages.length,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
45
packages/hive-mind-core/src/harvest/raw-types.ts
Normal file
45
packages/hive-mind-core/src/harvest/raw-types.ts
Normal file
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Raw external export shapes — loosely-typed structures as they arrive from
|
||||
* third-party JSON exports (ChatGPT / Claude / Gemini / generic).
|
||||
*
|
||||
* These adapters parse UNTRUSTED external data, so every field is optional and
|
||||
* widened. Property access goes through the narrowing helpers below rather than
|
||||
* casting to `any`, so a malformed export degrades to "skip" instead of throwing.
|
||||
*/
|
||||
|
||||
/** A JSON object whose keys are unknown until narrowed. */
|
||||
export type RawRecord = Record<string, unknown>;
|
||||
|
||||
/** Narrow an unknown value to a plain object, or null. */
|
||||
export function asRecord(value: unknown): RawRecord | null {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? (value as RawRecord)
|
||||
: null;
|
||||
}
|
||||
|
||||
/** Read a string property, or undefined if absent / wrong type. */
|
||||
export function getString(obj: RawRecord, key: string): string | undefined {
|
||||
const v = obj[key];
|
||||
return typeof v === 'string' ? v : undefined;
|
||||
}
|
||||
|
||||
/** Read a number property, or undefined if absent / wrong type. */
|
||||
export function getNumber(obj: RawRecord, key: string): number | undefined {
|
||||
const v = obj[key];
|
||||
return typeof v === 'number' ? v : undefined;
|
||||
}
|
||||
|
||||
/** Read an array property as unknown[], or undefined if absent / wrong type. */
|
||||
export function getArray(obj: RawRecord, key: string): unknown[] | undefined {
|
||||
const v = obj[key];
|
||||
return Array.isArray(v) ? v : undefined;
|
||||
}
|
||||
|
||||
/** First defined string among the given keys (export shapes vary). */
|
||||
export function firstString(obj: RawRecord, ...keys: string[]): string | undefined {
|
||||
for (const key of keys) {
|
||||
const v = getString(obj, key);
|
||||
if (v !== undefined) return v;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
191
packages/hive-mind-core/src/harvest/run-store.ts
Normal file
191
packages/hive-mind-core/src/harvest/run-store.ts
Normal file
@@ -0,0 +1,191 @@
|
||||
/**
|
||||
* HarvestRunStore — tracks the lifecycle of individual harvest commit runs
|
||||
* so the UI can surface interrupted runs and offer to resume them.
|
||||
*
|
||||
* A run is a single POST /api/harvest/commit invocation. States:
|
||||
* running — route is executing
|
||||
* completed — finished successfully
|
||||
* failed — explicit error; `error_message` is populated
|
||||
* abandoned — user chose to discard; cache is deleted
|
||||
*
|
||||
* A "interrupted" run from the UI's perspective is any `running` or `failed`
|
||||
* row with a surviving `input_cache_path`. The route never transitions
|
||||
* `running` -> `interrupted` — it simply never finalizes when the client
|
||||
* disconnects, so the row stays `running` forever. getLatestInterrupted()
|
||||
* surfaces the latest such row.
|
||||
*
|
||||
* Resume = replay the same input payload; FrameStore.createIFrame dedups
|
||||
* on content so already-saved frames become no-ops. No fine-grained
|
||||
* checkpoint-offset math needed.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '../mind/db.js';
|
||||
import type { ImportSourceType } from './types.js';
|
||||
|
||||
const HARVEST_RUNS_DDL = `
|
||||
CREATE TABLE IF NOT EXISTS harvest_runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
total_items INTEGER NOT NULL DEFAULT 0,
|
||||
items_saved INTEGER NOT NULL DEFAULT 0,
|
||||
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
finished_at TEXT,
|
||||
error_message TEXT,
|
||||
input_cache_path TEXT
|
||||
);
|
||||
`;
|
||||
|
||||
export type HarvestRunStatus = 'running' | 'completed' | 'failed' | 'abandoned';
|
||||
|
||||
export interface HarvestRun {
|
||||
id: number;
|
||||
source: ImportSourceType;
|
||||
status: HarvestRunStatus;
|
||||
totalItems: number;
|
||||
itemsSaved: number;
|
||||
startedAt: string;
|
||||
updatedAt: string;
|
||||
finishedAt: string | null;
|
||||
errorMessage: string | null;
|
||||
inputCachePath: string | null;
|
||||
}
|
||||
|
||||
export class HarvestRunStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const existsRow = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='harvest_runs'",
|
||||
).get();
|
||||
if (!existsRow) {
|
||||
raw.exec(HARVEST_RUNS_DDL);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new `running` run record. Caller holds the id for subsequent
|
||||
* heartbeat/complete/fail calls.
|
||||
*/
|
||||
start(source: ImportSourceType, totalItems: number, inputCachePath: string | null = null): HarvestRun {
|
||||
const raw = this.db.getDatabase();
|
||||
const result = raw.prepare(`
|
||||
INSERT INTO harvest_runs (source, status, total_items, input_cache_path)
|
||||
VALUES (?, 'running', ?, ?)
|
||||
`).run(source, totalItems, inputCachePath);
|
||||
return this.getById(Number(result.lastInsertRowid))!;
|
||||
}
|
||||
|
||||
/** Update items_saved + updated_at on a running row. No-op on terminal rows. */
|
||||
heartbeat(id: number, itemsSaved: number): void {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE harvest_runs SET
|
||||
items_saved = ?,
|
||||
updated_at = datetime('now')
|
||||
WHERE id = ? AND status = 'running'
|
||||
`).run(itemsSaved, id);
|
||||
}
|
||||
|
||||
/** Mark a run completed. Idempotent — no-op on terminal rows. */
|
||||
complete(id: number, itemsSaved: number): void {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE harvest_runs SET
|
||||
status = 'completed',
|
||||
items_saved = ?,
|
||||
updated_at = datetime('now'),
|
||||
finished_at = datetime('now')
|
||||
WHERE id = ? AND status = 'running'
|
||||
`).run(itemsSaved, id);
|
||||
}
|
||||
|
||||
/** Mark a run failed with an error message. Idempotent — no-op on terminal rows. */
|
||||
fail(id: number, errorMessage: string, itemsSaved?: number): void {
|
||||
const raw = this.db.getDatabase();
|
||||
if (typeof itemsSaved === 'number') {
|
||||
raw.prepare(`
|
||||
UPDATE harvest_runs SET
|
||||
status = 'failed',
|
||||
items_saved = ?,
|
||||
error_message = ?,
|
||||
updated_at = datetime('now'),
|
||||
finished_at = datetime('now')
|
||||
WHERE id = ? AND status = 'running'
|
||||
`).run(itemsSaved, errorMessage.slice(0, 2000), id);
|
||||
} else {
|
||||
raw.prepare(`
|
||||
UPDATE harvest_runs SET
|
||||
status = 'failed',
|
||||
error_message = ?,
|
||||
updated_at = datetime('now'),
|
||||
finished_at = datetime('now')
|
||||
WHERE id = ? AND status = 'running'
|
||||
`).run(errorMessage.slice(0, 2000), id);
|
||||
}
|
||||
}
|
||||
|
||||
/** Mark a run abandoned. Used when the user discards an interrupted run. */
|
||||
abandon(id: number): void {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE harvest_runs SET
|
||||
status = 'abandoned',
|
||||
updated_at = datetime('now'),
|
||||
finished_at = datetime('now')
|
||||
WHERE id = ? AND status IN ('running', 'failed')
|
||||
`).run(id);
|
||||
}
|
||||
|
||||
getById(id: number): HarvestRun | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM harvest_runs WHERE id = ?').get(id) as Record<string, unknown> | undefined;
|
||||
return row ? this.rowToRun(row) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Latest `running` or `failed` run with a surviving cache path — the UI
|
||||
* uses this to offer resume on next page load.
|
||||
*/
|
||||
getLatestInterrupted(): HarvestRun | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare(`
|
||||
SELECT * FROM harvest_runs
|
||||
WHERE status IN ('running', 'failed')
|
||||
AND input_cache_path IS NOT NULL
|
||||
ORDER BY started_at DESC
|
||||
LIMIT 1
|
||||
`).get() as Record<string, unknown> | undefined;
|
||||
return row ? this.rowToRun(row) : null;
|
||||
}
|
||||
|
||||
/** All runs, newest first — useful for debugging and future history views. */
|
||||
getAll(limit = 50): HarvestRun[] {
|
||||
const raw = this.db.getDatabase();
|
||||
return (raw.prepare(`
|
||||
SELECT * FROM harvest_runs ORDER BY started_at DESC LIMIT ?
|
||||
`).all(limit) as Record<string, unknown>[]).map(r => this.rowToRun(r));
|
||||
}
|
||||
|
||||
private rowToRun(row: Record<string, unknown>): HarvestRun {
|
||||
return {
|
||||
id: row.id as number,
|
||||
source: row.source as ImportSourceType,
|
||||
status: row.status as HarvestRunStatus,
|
||||
totalItems: row.total_items as number,
|
||||
itemsSaved: row.items_saved as number,
|
||||
startedAt: row.started_at as string,
|
||||
updatedAt: row.updated_at as string,
|
||||
finishedAt: (row.finished_at as string | null) ?? null,
|
||||
errorMessage: (row.error_message as string | null) ?? null,
|
||||
inputCachePath: (row.input_cache_path as string | null) ?? null,
|
||||
};
|
||||
}
|
||||
}
|
||||
140
packages/hive-mind-core/src/harvest/source-store.ts
Normal file
140
packages/hive-mind-core/src/harvest/source-store.ts
Normal file
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* HarvestSourceStore — tracks connected harvest sources and sync state.
|
||||
*/
|
||||
|
||||
import type { MindDB } from '../mind/db.js';
|
||||
import type { HarvestSource, ImportSourceType } from './types.js';
|
||||
|
||||
const HARVEST_SOURCES_DDL = `
|
||||
CREATE TABLE IF NOT EXISTS harvest_sources (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
source_path TEXT,
|
||||
last_synced_at TEXT,
|
||||
items_imported INTEGER NOT NULL DEFAULT 0,
|
||||
frames_created INTEGER NOT NULL DEFAULT 0,
|
||||
auto_sync INTEGER NOT NULL DEFAULT 0,
|
||||
sync_interval_hours INTEGER NOT NULL DEFAULT 24,
|
||||
last_content_hash TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
`;
|
||||
|
||||
export class HarvestSourceStore {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureTable();
|
||||
}
|
||||
|
||||
private ensureTable(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
const exists = raw.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='harvest_sources'",
|
||||
).get();
|
||||
if (!exists) {
|
||||
raw.exec(HARVEST_SOURCES_DDL);
|
||||
}
|
||||
}
|
||||
|
||||
/** Register or update a harvest source. */
|
||||
upsert(source: ImportSourceType, displayName: string, sourcePath?: string): HarvestSource {
|
||||
const raw = this.db.getDatabase();
|
||||
|
||||
raw.prepare(`
|
||||
INSERT INTO harvest_sources (source, display_name, source_path)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(source) DO UPDATE SET
|
||||
display_name = excluded.display_name,
|
||||
source_path = COALESCE(excluded.source_path, harvest_sources.source_path)
|
||||
`).run(source, displayName, sourcePath ?? null);
|
||||
|
||||
return this.getBySource(source)!;
|
||||
}
|
||||
|
||||
/** Record a completed sync. */
|
||||
recordSync(source: ImportSourceType, itemsImported: number, framesCreated: number, contentHash?: string): void {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE harvest_sources SET
|
||||
last_synced_at = datetime('now'),
|
||||
items_imported = items_imported + ?,
|
||||
frames_created = frames_created + ?,
|
||||
last_content_hash = COALESCE(?, last_content_hash)
|
||||
WHERE source = ?
|
||||
`).run(itemsImported, framesCreated, contentHash ?? null, source);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the R3-004 unchanged-set skip hash for a source, forcing the NEXT harvest
|
||||
* of it to run the full per-item loop instead of short-circuiting as "unchanged".
|
||||
* Needed after GDPR re-consent (#7): lifting a subject's suppression must let an
|
||||
* IDENTICAL re-import re-materialize it — but the set-hash skip would otherwise
|
||||
* skip the whole run before the per-item loop ever re-adds the re-consented subject.
|
||||
* No-op if the source row does not exist.
|
||||
*/
|
||||
clearContentHash(source: ImportSourceType): void {
|
||||
this.db.getDatabase()
|
||||
.prepare('UPDATE harvest_sources SET last_content_hash = NULL WHERE source = ?')
|
||||
.run(source);
|
||||
}
|
||||
|
||||
/** Enable or disable auto-sync for a source. */
|
||||
setAutoSync(source: ImportSourceType, enabled: boolean, intervalHours?: number): void {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
UPDATE harvest_sources SET
|
||||
auto_sync = ?,
|
||||
sync_interval_hours = COALESCE(?, sync_interval_hours)
|
||||
WHERE source = ?
|
||||
`).run(enabled ? 1 : 0, intervalHours ?? null, source);
|
||||
}
|
||||
|
||||
/** Get a specific source. */
|
||||
getBySource(source: ImportSourceType): HarvestSource | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM harvest_sources WHERE source = ?').get(source) as Record<string, unknown> | undefined;
|
||||
return row ? this.rowToSource(row) : null;
|
||||
}
|
||||
|
||||
/** Get all registered sources. */
|
||||
getAll(): HarvestSource[] {
|
||||
const raw = this.db.getDatabase();
|
||||
return (raw.prepare('SELECT * FROM harvest_sources ORDER BY last_synced_at DESC').all() as Record<string, unknown>[])
|
||||
.map(r => this.rowToSource(r));
|
||||
}
|
||||
|
||||
/** Get sources that need syncing (auto_sync enabled and interval elapsed). */
|
||||
getStale(): HarvestSource[] {
|
||||
const raw = this.db.getDatabase();
|
||||
return (raw.prepare(`
|
||||
SELECT * FROM harvest_sources
|
||||
WHERE auto_sync = 1
|
||||
AND (last_synced_at IS NULL
|
||||
OR datetime(last_synced_at, '+' || sync_interval_hours || ' hours') <= datetime('now'))
|
||||
`).all() as Record<string, unknown>[]).map(r => this.rowToSource(r));
|
||||
}
|
||||
|
||||
/** Remove a source. */
|
||||
remove(source: ImportSourceType): void {
|
||||
this.db.getDatabase().prepare('DELETE FROM harvest_sources WHERE source = ?').run(source);
|
||||
}
|
||||
|
||||
private rowToSource(row: Record<string, unknown>): HarvestSource {
|
||||
return {
|
||||
id: row.id as number,
|
||||
source: row.source as ImportSourceType,
|
||||
displayName: row.display_name as string,
|
||||
sourcePath: row.source_path as string | null,
|
||||
lastSyncedAt: row.last_synced_at as string | null,
|
||||
itemsImported: row.items_imported as number,
|
||||
framesCreated: row.frames_created as number,
|
||||
autoSync: (row.auto_sync as number) === 1,
|
||||
syncIntervalHours: row.sync_interval_hours as number,
|
||||
lastContentHash: row.last_content_hash as string | null,
|
||||
createdAt: row.created_at as string,
|
||||
};
|
||||
}
|
||||
}
|
||||
48
packages/hive-mind-core/src/harvest/stable-id.ts
Normal file
48
packages/hive-mind-core/src/harvest/stable-id.ts
Normal file
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* stable-id.ts — deterministic, collision-safe id for harvest import items.
|
||||
*
|
||||
* WHY: a harvest item's `id` becomes the GDPR Art.17 subject key. The pipeline
|
||||
* runs it through `rawTurnConvKey → sanitizeToken(`${source}-${id}`, 64)`
|
||||
* (raw-turns.ts) to form `source_ref`, the key the erasure path reconstructs
|
||||
* from (source, source_ref) alone. `randomUUID()` mints a fresh id every
|
||||
* re-import, so a re-imported conversation lands under a NEW source_ref and
|
||||
* "sticky erasure" (erase-once-stays-erased across re-import) silently breaks.
|
||||
* A deterministic id keyed on the export's OWN stable identifiers fixes this —
|
||||
* and makes raw_archive idempotent across re-imports of the same subject.
|
||||
*
|
||||
* CONTRACT:
|
||||
* - deterministic: same (source, ...parts) -> same id, forever, cross-process.
|
||||
* - sanitize-stable: output is lowercase sha256 hex ([0-9a-f]) only, so it
|
||||
* survives sanitizeToken(...,64) unchanged (no '-' collapse, no truncation
|
||||
* collision within the 64-char budget).
|
||||
* - collision-safe: a NUL ('\x00') separator between parts prevents field-
|
||||
* boundary ambiguity (e.g. 'a' + 'bc' vs 'ab' + 'c'); NUL cannot appear in
|
||||
* any real title/id/path, so it is an unambiguous delimiter. Undefined parts
|
||||
* collapse to '' but STILL emit a separator, so a present-vs-absent field
|
||||
* never aliases a shifted field.
|
||||
* - never keys on growing content: callers pass stable identifiers (conv uuid,
|
||||
* file path, map key) so a conversation that GAINS turns keeps its id.
|
||||
*
|
||||
* Reference impl: mind/content-hash.ts (same createHash('sha256') pattern).
|
||||
*/
|
||||
import { createHash } from 'node:crypto';
|
||||
|
||||
/**
|
||||
* Deterministic short id for a harvest item.
|
||||
*
|
||||
* @param source the ImportSourceType discriminator (e.g. 'chatgpt') — the first
|
||||
* hashed field, so ids from different adapters that share an
|
||||
* otherwise-identical key can never collide.
|
||||
* @param parts the stable identity fields. string|number|undefined accepted;
|
||||
* number is stringified, undefined becomes '' (separator still
|
||||
* emitted). At least one meaningful part SHOULD be passed.
|
||||
* @returns 40-char lowercase hex (sanitizeToken-stable, well under the 64 cap).
|
||||
*/
|
||||
export function stableHarvestId(
|
||||
source: string,
|
||||
...parts: ReadonlyArray<string | number | undefined>
|
||||
): string {
|
||||
// NUL-join: source is field 0; every part gets its own field even when ''.
|
||||
const key = [source, ...parts.map(p => (p === undefined ? '' : String(p)))].join('\x00');
|
||||
return createHash('sha256').update(key).digest('hex').slice(0, 40);
|
||||
}
|
||||
140
packages/hive-mind-core/src/harvest/types.ts
Normal file
140
packages/hive-mind-core/src/harvest/types.ts
Normal file
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Harvest Types — universal import format and distillation pipeline types.
|
||||
*/
|
||||
|
||||
// ── Source Types ──
|
||||
|
||||
export type ImportSourceType =
|
||||
| 'chatgpt' | 'claude' | 'claude-code' | 'claude-desktop'
|
||||
| 'gemini' | 'google-ai-studio' | 'perplexity' | 'grok'
|
||||
| 'cursor' | 'copilot' | 'manus' | 'genspark'
|
||||
| 'qwen' | 'minimax' | 'z-ai' | 'openclaw' | 'cowork'
|
||||
| 'elevenlabs' | 'google-flow'
|
||||
| 'markdown' | 'plaintext' | 'pdf' | 'url'
|
||||
| 'unknown';
|
||||
|
||||
export type ImportItemType =
|
||||
| 'conversation' | 'memory' | 'instruction'
|
||||
| 'preference' | 'artifact' | 'rule'
|
||||
| 'decision' | 'document';
|
||||
|
||||
// ── Universal Import Item (adapter output) ──
|
||||
|
||||
export interface ConversationMessage {
|
||||
role: 'user' | 'assistant' | 'system';
|
||||
text: string;
|
||||
timestamp?: string;
|
||||
}
|
||||
|
||||
export interface UniversalImportItem {
|
||||
id: string;
|
||||
source: ImportSourceType;
|
||||
type: ImportItemType;
|
||||
title: string;
|
||||
content: string;
|
||||
messages?: ConversationMessage[];
|
||||
timestamp: string;
|
||||
metadata: Record<string, unknown>;
|
||||
}
|
||||
|
||||
// ── Distilled Knowledge (pipeline output) ──
|
||||
|
||||
export type DistillTargetLayer = 'identity' | 'frame' | 'kg_entity' | 'kg_relation' | 'awareness';
|
||||
|
||||
export interface KnowledgeProvenance {
|
||||
originalSource: ImportSourceType;
|
||||
originalId?: string;
|
||||
conversationTitle?: string;
|
||||
importedAt: string;
|
||||
distillationModel: string;
|
||||
confidence: number;
|
||||
pass: number;
|
||||
}
|
||||
|
||||
export interface DistilledKnowledge {
|
||||
targetLayer: DistillTargetLayer;
|
||||
frameType?: 'I' | 'P';
|
||||
importance: 'critical' | 'important' | 'normal' | 'temporary';
|
||||
content: string;
|
||||
entities?: { name: string; type: string }[];
|
||||
relations?: { source: string; target: string; relation: string }[];
|
||||
provenance: KnowledgeProvenance;
|
||||
}
|
||||
|
||||
// ── Classification (Pass 1 output) ──
|
||||
|
||||
export type ClassificationDomain = 'work' | 'personal' | 'technical' | 'mixed';
|
||||
export type ClassificationValue = 'high' | 'medium' | 'low' | 'skip';
|
||||
|
||||
export interface ClassifiedItem {
|
||||
item: UniversalImportItem;
|
||||
domain: ClassificationDomain;
|
||||
value: ClassificationValue;
|
||||
categories: string[];
|
||||
}
|
||||
|
||||
// ── Extraction (Pass 2 output) ──
|
||||
|
||||
export interface ExtractedContent {
|
||||
itemId: string;
|
||||
decisions: string[];
|
||||
preferences: string[];
|
||||
facts: string[];
|
||||
knowledge: string[];
|
||||
entities: { name: string; type: string }[];
|
||||
relations: { source: string; target: string; relation: string }[];
|
||||
}
|
||||
|
||||
// ── Pipeline Result ──
|
||||
|
||||
export interface HarvestPipelineResult {
|
||||
source: ImportSourceType;
|
||||
itemsReceived: number;
|
||||
itemsClassified: number;
|
||||
itemsSkipped: number;
|
||||
itemsExtracted: number;
|
||||
knowledgeDistilled: DistilledKnowledge[];
|
||||
framesSaved: number;
|
||||
entitiesCreated: number;
|
||||
relationsCreated: number;
|
||||
identityUpdates: number;
|
||||
duplicatesSkipped: number;
|
||||
errors: string[];
|
||||
costUsd: number;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
// ── Harvest Source (tracking table) ──
|
||||
|
||||
export interface HarvestSource {
|
||||
id: number;
|
||||
source: ImportSourceType;
|
||||
displayName: string;
|
||||
sourcePath: string | null;
|
||||
lastSyncedAt: string | null;
|
||||
itemsImported: number;
|
||||
framesCreated: number;
|
||||
autoSync: boolean;
|
||||
syncIntervalHours: number;
|
||||
lastContentHash: string | null;
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
// ── Adapter Interface ──
|
||||
|
||||
export interface SourceAdapter {
|
||||
readonly sourceType: ImportSourceType;
|
||||
readonly displayName: string;
|
||||
parse(input: unknown): UniversalImportItem[];
|
||||
}
|
||||
|
||||
export interface FilesystemAdapter extends SourceAdapter {
|
||||
scan(dirPath: string): UniversalImportItem[];
|
||||
}
|
||||
|
||||
/**
|
||||
* W4.4: unified per-frame content cap for harvest imports. The three ingest
|
||||
* surfaces had drifted (MCP tools 2,000 vs sidecar 10,000) — a 5x divergence
|
||||
* in what the same export preserved depending on the door it came through.
|
||||
*/
|
||||
export const HARVEST_FRAME_CONTENT_CAP = 10_000;
|
||||
248
packages/hive-mind-core/src/harvest/universal-adapter.ts
Normal file
248
packages/hive-mind-core/src/harvest/universal-adapter.ts
Normal file
@@ -0,0 +1,248 @@
|
||||
/**
|
||||
* Universal Adapter — accepts any text, JSON, or Markdown input.
|
||||
*
|
||||
* For Tier 2 platforms (Perplexity, Grok, Manus, Genspark, Qwen, Minimax,
|
||||
* z.ai, OpenClaw, Cowork, ElevenLabs, Google Flow) where formats vary.
|
||||
*
|
||||
* Strategy:
|
||||
* 1. Detect if input is a known JSON format
|
||||
* 2. If JSON with recognizable structure, parse as conversations
|
||||
* 3. Otherwise, treat as raw text and create a single import item
|
||||
*/
|
||||
|
||||
import { stableHarvestId } from './stable-id.js';
|
||||
import type { SourceAdapter, UniversalImportItem, ImportSourceType, ConversationMessage } from './types.js';
|
||||
import { asRecord, firstString, getArray, getString, type RawRecord } from './raw-types.js';
|
||||
|
||||
/** Heuristic source detection from content cues. */
|
||||
function detectSource(input: unknown): ImportSourceType {
|
||||
if (typeof input === 'string') {
|
||||
const lower = input.toLowerCase();
|
||||
if (lower.includes('perplexity')) return 'perplexity';
|
||||
if (lower.includes('grok') || lower.includes('x.ai')) return 'grok';
|
||||
if (lower.includes('manus')) return 'manus';
|
||||
if (lower.includes('genspark')) return 'genspark';
|
||||
if (lower.includes('qwen') || lower.includes('tongyi')) return 'qwen';
|
||||
if (lower.includes('minimax')) return 'minimax';
|
||||
if (lower.includes('elevenlabs')) return 'elevenlabs';
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
const obj = asRecord(input);
|
||||
if (obj) {
|
||||
const keys = Object.keys(obj);
|
||||
const source = getString(obj, 'source');
|
||||
if (keys.includes('perplexity') || source === 'perplexity') return 'perplexity';
|
||||
if (keys.includes('grok') || source === 'grok') return 'grok';
|
||||
if (getString(obj, 'provider') === 'qwen') return 'qwen';
|
||||
}
|
||||
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
/** Try to find conversations in any JSON structure. */
|
||||
function findConversations(obj: unknown): RawRecord[] | null {
|
||||
if (Array.isArray(obj)) {
|
||||
const first = asRecord(obj[0]);
|
||||
if (obj.length > 0 && first && (first.messages || first.chat_messages || first.turns || first.history)) {
|
||||
return obj.map(asRecord).filter((c): c is RawRecord => c !== null);
|
||||
}
|
||||
if (obj.length > 0 && first && (first.role || first.sender || first.author)) {
|
||||
return [{ title: 'Imported Conversation', messages: obj }];
|
||||
}
|
||||
}
|
||||
|
||||
const record = asRecord(obj);
|
||||
if (record) {
|
||||
for (const key of ['conversations', 'chats', 'threads', 'sessions', 'history', 'data']) {
|
||||
const nested = getArray(record, key);
|
||||
if (nested) {
|
||||
return findConversations(nested);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export class UniversalAdapter implements SourceAdapter {
|
||||
readonly sourceType = 'unknown' as const;
|
||||
readonly displayName = 'Universal (Auto-detect)';
|
||||
|
||||
parse(input: unknown): UniversalImportItem[] {
|
||||
if (typeof input === 'string') {
|
||||
return this.parseText(input);
|
||||
}
|
||||
if (typeof input === 'object' && input !== null) {
|
||||
return this.parseJson(input);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
private parseText(text: string): UniversalImportItem[] {
|
||||
const source = detectSource(text);
|
||||
const items: UniversalImportItem[] = [];
|
||||
const conversations = this.splitConversations(text);
|
||||
|
||||
for (const conv of conversations) {
|
||||
const messages = this.extractMessagesFromText(conv.content);
|
||||
|
||||
items.push({
|
||||
// raw text paste has no id — content is the only surrogate (NOT growth-stable;
|
||||
// documented tradeoff, no better anchor exists for free-text).
|
||||
id: stableHarvestId('universal-text', source, conv.content),
|
||||
source,
|
||||
type: messages.length > 0 ? 'conversation' : 'memory',
|
||||
title: conv.title,
|
||||
content: conv.content,
|
||||
messages: messages.length > 0 ? messages : undefined,
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: { parseMethod: 'universal-text', detectedSource: source },
|
||||
});
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private parseJson(input: object): UniversalImportItem[] {
|
||||
const source = detectSource(input);
|
||||
const items: UniversalImportItem[] = [];
|
||||
const conversations = findConversations(input);
|
||||
|
||||
if (conversations) {
|
||||
for (const conv of conversations) {
|
||||
const title = firstString(conv, 'title', 'name', 'subject') ?? 'Imported Conversation';
|
||||
const rawMessages = getArray(conv, 'messages') ?? getArray(conv, 'chat_messages')
|
||||
?? getArray(conv, 'turns') ?? getArray(conv, 'history') ?? [];
|
||||
const messages: ConversationMessage[] = [];
|
||||
|
||||
for (const rawMsg of rawMessages) {
|
||||
const msg = asRecord(rawMsg);
|
||||
if (!msg) continue;
|
||||
const role = this.resolveRole(msg);
|
||||
if (!role) continue;
|
||||
const text = this.extractText(msg);
|
||||
if (!text) continue;
|
||||
messages.push({ role, text, timestamp: firstString(msg, 'timestamp', 'created_at', 'createTime') });
|
||||
}
|
||||
|
||||
if (messages.length === 0) continue;
|
||||
|
||||
const convContent = messages.map(m => `${m.role}: ${m.text}`).join('\n\n');
|
||||
const convId = getString(conv, 'id');
|
||||
items.push({
|
||||
// Stable per-conversation id when the export gives one (growth-stable). Else
|
||||
// fall back to source+title+created_at PLUS content: a bare message-array paste
|
||||
// has no id/timestamp and a CONSTANT synthetic title ('Imported Conversation'),
|
||||
// so without content every such paste collapses to ONE (source, source_ref)
|
||||
// subject key → cross-subject over-suppression / co-erasure. Content makes them
|
||||
// distinct (id-less → not growth-stable, the documented universal-text tradeoff).
|
||||
id: stableHarvestId('universal-json', convId ?? `${source}\x00${title}\x00${firstString(conv, 'created_at', 'createTime', 'timestamp') ?? ''}\x00${convContent}`),
|
||||
source,
|
||||
type: 'conversation',
|
||||
title,
|
||||
content: convContent,
|
||||
messages,
|
||||
timestamp: firstString(conv, 'created_at', 'createTime', 'timestamp') ?? new Date().toISOString(),
|
||||
metadata: { parseMethod: 'universal-json', detectedSource: source, conversationId: convId },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (items.length === 0) {
|
||||
const record = asRecord(input);
|
||||
items.push({
|
||||
id: stableHarvestId('universal-json-raw', source, JSON.stringify(input)),
|
||||
source,
|
||||
type: 'memory',
|
||||
title: (record && getString(record, 'title')) ?? 'Imported Data',
|
||||
content: JSON.stringify(input, null, 2).slice(0, 50000),
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: { parseMethod: 'universal-json-raw', detectedSource: source },
|
||||
});
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private splitConversations(text: string): { title: string; content: string }[] {
|
||||
const separators = [
|
||||
/^#{1,3}\s+/gm,
|
||||
/^={3,}$/gm,
|
||||
/^-{3,}$/gm,
|
||||
/^Conversation \d+/gim,
|
||||
];
|
||||
|
||||
for (const sep of separators) {
|
||||
const parts = text.split(sep).filter(p => p.trim().length > 20);
|
||||
if (parts.length > 1) {
|
||||
return parts.map((p, i) => ({
|
||||
title: `Conversation ${i + 1}`,
|
||||
content: p.trim(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
return [{ title: 'Imported Text', content: text.trim() }];
|
||||
}
|
||||
|
||||
private extractMessagesFromText(text: string): ConversationMessage[] {
|
||||
const messages: ConversationMessage[] = [];
|
||||
const pattern = /^(User|Human|Me|Assistant|AI|Bot|Claude|ChatGPT|Gemini|Grok):\s*([\s\S]*?)(?=^(?:User|Human|Me|Assistant|AI|Bot|Claude|ChatGPT|Gemini|Grok):|$)/gim;
|
||||
let match;
|
||||
|
||||
while ((match = pattern.exec(text)) !== null) {
|
||||
const speaker = match[1].toLowerCase();
|
||||
const content = match[2].trim();
|
||||
if (!content) continue;
|
||||
|
||||
const role = ['user', 'human', 'me'].includes(speaker) ? 'user' as const : 'assistant' as const;
|
||||
messages.push({ role, text: content });
|
||||
}
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
private resolveRole(msg: RawRecord): 'user' | 'assistant' | null {
|
||||
const role = firstString(msg, 'role', 'sender', 'author', 'type');
|
||||
if (!role) return null;
|
||||
const r = role.toLowerCase();
|
||||
if (['user', 'human', 'me'].includes(r)) return 'user';
|
||||
if (['assistant', 'ai', 'bot', 'model', 'system'].includes(r)) return 'assistant';
|
||||
return null;
|
||||
}
|
||||
|
||||
private extractText(msg: RawRecord): string {
|
||||
const directText = getString(msg, 'text');
|
||||
if (directText !== undefined) return directText.trim();
|
||||
const directContent = getString(msg, 'content');
|
||||
if (directContent !== undefined) return directContent.trim();
|
||||
const contentBlocks = getArray(msg, 'content');
|
||||
if (contentBlocks) {
|
||||
return contentBlocks
|
||||
.map(b => {
|
||||
if (typeof b === 'string') return b;
|
||||
const rec = asRecord(b);
|
||||
if (!rec) return undefined;
|
||||
if (rec.type === 'text') return getString(rec, 'text') ?? '';
|
||||
// W4.4 (caption parity): generic text-bearing image fields.
|
||||
const caption = getString(rec, 'caption') ?? getString(rec, 'alt') ?? getString(rec, 'description');
|
||||
if (caption) return `[Shared image: ${caption}]`;
|
||||
return undefined;
|
||||
})
|
||||
.filter((t): t is string => typeof t === 'string')
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
const parts = getArray(msg, 'parts');
|
||||
if (parts) {
|
||||
return parts
|
||||
.map(asRecord)
|
||||
.map(p => (p ? getString(p, 'text') : undefined))
|
||||
.filter((t): t is string => typeof t === 'string')
|
||||
.join('\n')
|
||||
.trim();
|
||||
}
|
||||
return '';
|
||||
}
|
||||
}
|
||||
186
packages/hive-mind-core/src/harvest/url-adapter.ts
Normal file
186
packages/hive-mind-core/src/harvest/url-adapter.ts
Normal file
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* URL Source Adapter — fetches web pages and extracts readable content.
|
||||
*
|
||||
* Uses built-in fetch + basic HTML stripping.
|
||||
* Splits content by sections if headings are present.
|
||||
*/
|
||||
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { SourceAdapter, UniversalImportItem } from './types.js';
|
||||
import { safeFetch, allowLocalFromEnv } from './url-egress-guard.js';
|
||||
|
||||
/** Strip HTML tags and decode common entities. Returns plain text. */
|
||||
function stripHtml(html: string): string {
|
||||
// Remove script and style blocks entirely
|
||||
let text = html.replace(/<script[\s\S]*?<\/script>/gi, '');
|
||||
text = text.replace(/<style[\s\S]*?<\/style>/gi, '');
|
||||
text = text.replace(/<nav[\s\S]*?<\/nav>/gi, '');
|
||||
text = text.replace(/<footer[\s\S]*?<\/footer>/gi, '');
|
||||
text = text.replace(/<header[\s\S]*?<\/header>/gi, '');
|
||||
|
||||
// Convert headings to markdown-style
|
||||
text = text.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n');
|
||||
text = text.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n');
|
||||
text = text.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n');
|
||||
|
||||
// Convert paragraphs and breaks to newlines
|
||||
text = text.replace(/<\/p>/gi, '\n\n');
|
||||
text = text.replace(/<br\s*\/?>/gi, '\n');
|
||||
text = text.replace(/<li[^>]*>/gi, '\n- ');
|
||||
|
||||
// Strip remaining tags
|
||||
text = text.replace(/<[^>]+>/g, '');
|
||||
|
||||
// Decode common HTML entities
|
||||
text = text.replace(/&/g, '&');
|
||||
text = text.replace(/</g, '<');
|
||||
text = text.replace(/>/g, '>');
|
||||
text = text.replace(/"/g, '"');
|
||||
text = text.replace(/'/g, "'");
|
||||
text = text.replace(/ /g, ' ');
|
||||
|
||||
// Collapse excessive whitespace
|
||||
text = text.replace(/[ \t]+/g, ' ');
|
||||
text = text.replace(/\n{3,}/g, '\n\n');
|
||||
|
||||
return text.trim();
|
||||
}
|
||||
|
||||
/** Extract <title> from HTML. */
|
||||
function extractTitle(html: string): string | undefined {
|
||||
const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
|
||||
return match ? match[1].trim().replace(/\s+/g, ' ') : undefined;
|
||||
}
|
||||
|
||||
/** Extract meta description from HTML. */
|
||||
function extractDescription(html: string): string | undefined {
|
||||
const match = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([\s\S]*?)["'][^>]*>/i)
|
||||
?? html.match(/<meta[^>]*content=["']([\s\S]*?)["'][^>]*name=["']description["'][^>]*>/i);
|
||||
return match ? match[1].trim() : undefined;
|
||||
}
|
||||
|
||||
export class UrlAdapter implements SourceAdapter {
|
||||
readonly sourceType = 'url' as const;
|
||||
readonly displayName = 'Web URL';
|
||||
|
||||
parse(input: unknown): UniversalImportItem[] {
|
||||
// Synchronous parse — for pre-fetched HTML content
|
||||
if (typeof input !== 'string') return [];
|
||||
|
||||
// If input looks like HTML, parse it directly
|
||||
if (input.includes('<html') || input.includes('<body') || input.includes('<div')) {
|
||||
return this.parseHtml(input, undefined);
|
||||
}
|
||||
|
||||
// If input is a URL, return empty — caller should use fetchAndParse()
|
||||
if (input.startsWith('http://') || input.startsWith('https://')) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
/** Fetch a URL and parse its content. Async because of network I/O.
|
||||
* Routed through the SSRF egress guard — the target and every redirect hop
|
||||
* must resolve to a public address (loopback allowed only when
|
||||
* WAGGLE_ALLOW_LOCAL_FETCH is set). Blocked targets throw EgressBlockedError,
|
||||
* surfaced to the MCP ingest caller as an error result. */
|
||||
async fetchAndParse(url: string): Promise<UniversalImportItem[]> {
|
||||
const response = await safeFetch(
|
||||
url,
|
||||
{
|
||||
headers: {
|
||||
'User-Agent': 'Waggle-Memory/1.0 (knowledge harvester)',
|
||||
'Accept': 'text/html,application/xhtml+xml,text/plain',
|
||||
},
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
},
|
||||
{ allowLocal: allowLocalFromEnv() },
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type') ?? '';
|
||||
const body = await response.text();
|
||||
|
||||
if (contentType.includes('text/html') || contentType.includes('xhtml')) {
|
||||
return this.parseHtml(body, url);
|
||||
}
|
||||
|
||||
// Plain text or other — treat as plaintext
|
||||
return [{
|
||||
id: randomUUID(),
|
||||
source: 'url',
|
||||
type: 'document',
|
||||
title: url,
|
||||
content: body.slice(0, 4000),
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: { sourceUrl: url, contentType: 'article' },
|
||||
}];
|
||||
}
|
||||
|
||||
private parseHtml(html: string, sourceUrl: string | undefined): UniversalImportItem[] {
|
||||
const title = extractTitle(html) ?? sourceUrl ?? 'Web page';
|
||||
const description = extractDescription(html);
|
||||
const plainText = stripHtml(html);
|
||||
|
||||
if (!plainText || plainText.length < 50) return [];
|
||||
|
||||
// For short pages, return as single item
|
||||
if (plainText.length <= 4000) {
|
||||
return [{
|
||||
id: randomUUID(),
|
||||
source: 'url',
|
||||
type: 'document',
|
||||
title,
|
||||
content: plainText,
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
...(sourceUrl && { sourceUrl }),
|
||||
...(description && { description }),
|
||||
contentType: 'article',
|
||||
},
|
||||
}];
|
||||
}
|
||||
|
||||
// For longer pages, split by headings
|
||||
const sections = plainText.split(/\n(?=#{1,3}\s)/);
|
||||
const items: UniversalImportItem[] = [];
|
||||
|
||||
for (const section of sections) {
|
||||
const trimmed = section.trim();
|
||||
if (trimmed.length < 30) continue;
|
||||
|
||||
const headingMatch = trimmed.match(/^#{1,3}\s+(.+)/);
|
||||
const sectionTitle = headingMatch ? headingMatch[1].trim() : title;
|
||||
|
||||
items.push({
|
||||
id: randomUUID(),
|
||||
source: 'url',
|
||||
type: 'document',
|
||||
title: sectionTitle === title ? title : `${title} — ${sectionTitle}`,
|
||||
content: trimmed.slice(0, 4000),
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
...(sourceUrl && { sourceUrl }),
|
||||
contentType: 'article',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return items.length > 0 ? items : [{
|
||||
id: randomUUID(),
|
||||
source: 'url',
|
||||
type: 'document',
|
||||
title,
|
||||
content: plainText.slice(0, 4000),
|
||||
timestamp: new Date().toISOString(),
|
||||
metadata: {
|
||||
...(sourceUrl && { sourceUrl }),
|
||||
contentType: 'article',
|
||||
},
|
||||
}];
|
||||
}
|
||||
}
|
||||
298
packages/hive-mind-core/src/harvest/url-egress-guard.ts
Normal file
298
packages/hive-mind-core/src/harvest/url-egress-guard.ts
Normal file
@@ -0,0 +1,298 @@
|
||||
/**
|
||||
* SSRF egress guard for the URL harvest adapter.
|
||||
*
|
||||
* The harvest URL adapter fetches attacker-influenceable URLs (a user or an
|
||||
* MCP client naming a URL to ingest). Without this guard an ingest of
|
||||
* `http://169.254.169.254/latest/meta-data/` or an RFC1918 host reaches
|
||||
* internal services on the cloud/TEAMS deploy (sidecar binds 0.0.0.0) and can
|
||||
* exfiltrate instance-metadata IAM credentials.
|
||||
*
|
||||
* This module is a self-contained, dependency-free (node builtins only) copy of
|
||||
* the guard spec shared with `packages/agent/src/url-egress-guard.ts`. It lives
|
||||
* here — rather than importing from @waggle/agent — because hive-mind-core is
|
||||
* OSS-mirrored and must not depend on Waggle-proprietary packages. Keep the two
|
||||
* implementations in sync; they share one spec.
|
||||
*
|
||||
* Obfuscated IP literals (octal / decimal / hex) are normalized by the OS
|
||||
* resolver: `net.isIP` rejects them as literals, so they fall through to
|
||||
* `dns.lookup`, which returns the canonical dotted form we classify.
|
||||
*/
|
||||
|
||||
import { lookup as dnsLookup } from 'node:dns/promises';
|
||||
import { isIP } from 'node:net';
|
||||
|
||||
export type AddressClass =
|
||||
| 'public'
|
||||
| 'loopback'
|
||||
| 'private'
|
||||
| 'link-local'
|
||||
| 'unique-local'
|
||||
| 'multicast'
|
||||
| 'reserved'
|
||||
| 'unspecified'
|
||||
| 'invalid';
|
||||
|
||||
export interface ResolvedAddress {
|
||||
address: string;
|
||||
family: number;
|
||||
}
|
||||
|
||||
export type LookupFn = (hostname: string) => Promise<ResolvedAddress[]>;
|
||||
|
||||
export interface EgressGuardOptions {
|
||||
/** Permit loopback targets (default false). Only loopback is unlocked. */
|
||||
allowLocal?: boolean;
|
||||
/** Injectable resolver (tests). Defaults to node:dns/promises lookup(all). */
|
||||
lookup?: LookupFn;
|
||||
}
|
||||
|
||||
export class EgressBlockedError extends Error {
|
||||
public readonly url: string;
|
||||
public readonly addressClass?: AddressClass;
|
||||
constructor(message: string, url: string, addressClass?: AddressClass) {
|
||||
super(message);
|
||||
this.name = 'EgressBlockedError';
|
||||
this.url = url;
|
||||
this.addressClass = addressClass;
|
||||
}
|
||||
}
|
||||
|
||||
function parseIpv4Octets(ip: string): [number, number, number, number] | null {
|
||||
const parts = ip.split('.');
|
||||
if (parts.length !== 4) return null;
|
||||
const octets: number[] = [];
|
||||
for (const part of parts) {
|
||||
if (!/^\d{1,3}$/.test(part)) return null;
|
||||
const n = Number(part);
|
||||
if (n < 0 || n > 255) return null;
|
||||
octets.push(n);
|
||||
}
|
||||
return [octets[0], octets[1], octets[2], octets[3]];
|
||||
}
|
||||
|
||||
function classifyIpv4(ip: string): AddressClass {
|
||||
const octets = parseIpv4Octets(ip);
|
||||
if (!octets) return 'invalid';
|
||||
const [a, b, c] = octets;
|
||||
if (a === 0) return 'unspecified'; // 0.0.0.0/8
|
||||
if (a === 127) return 'loopback'; // 127.0.0.0/8
|
||||
if (a === 10) return 'private'; // 10.0.0.0/8
|
||||
if (a === 172 && b >= 16 && b <= 31) return 'private'; // 172.16.0.0/12
|
||||
if (a === 192 && b === 168) return 'private'; // 192.168.0.0/16
|
||||
if (a === 100 && b >= 64 && b <= 127) return 'private'; // 100.64.0.0/10 CGNAT
|
||||
if (a === 169 && b === 254) return 'link-local'; // 169.254.0.0/16 (metadata)
|
||||
if (a >= 224 && a <= 239) return 'multicast'; // 224.0.0.0/4
|
||||
if (a >= 240) return 'reserved'; // 240.0.0.0/4 + 255.255.255.255
|
||||
if (a === 192 && b === 0 && c === 0) return 'reserved'; // 192.0.0.0/24
|
||||
if (a === 192 && b === 0 && c === 2) return 'reserved'; // TEST-NET-1
|
||||
if (a === 198 && (b === 18 || b === 19)) return 'reserved'; // 198.18.0.0/15
|
||||
if (a === 198 && b === 51 && c === 100) return 'reserved'; // TEST-NET-2
|
||||
if (a === 203 && b === 0 && c === 113) return 'reserved'; // TEST-NET-3
|
||||
return 'public';
|
||||
}
|
||||
|
||||
function parseIpv6Hextets(ip: string): number[] | null {
|
||||
let s = ip.toLowerCase();
|
||||
const zoneAt = s.indexOf('%');
|
||||
if (zoneAt !== -1) s = s.slice(0, zoneAt);
|
||||
|
||||
const dotMatch = s.match(/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
|
||||
if (dotMatch) {
|
||||
const v4 = parseIpv4Octets(dotMatch[1]);
|
||||
if (!v4) return null;
|
||||
const hi = ((v4[0] << 8) | v4[1]).toString(16);
|
||||
const lo = ((v4[2] << 8) | v4[3]).toString(16);
|
||||
s = s.slice(0, dotMatch.index) + hi + ':' + lo;
|
||||
}
|
||||
|
||||
const halves = s.split('::');
|
||||
if (halves.length > 2) return null;
|
||||
|
||||
const head = halves[0] ? halves[0].split(':') : [];
|
||||
let groups: string[];
|
||||
if (halves.length === 2) {
|
||||
const tail = halves[1] ? halves[1].split(':') : [];
|
||||
const missing = 8 - head.length - tail.length;
|
||||
if (missing < 0) return null;
|
||||
groups = [...head, ...Array(missing).fill('0'), ...tail];
|
||||
} else {
|
||||
groups = head;
|
||||
}
|
||||
if (groups.length !== 8) return null;
|
||||
|
||||
const hextets: number[] = [];
|
||||
for (const g of groups) {
|
||||
if (!/^[0-9a-f]{1,4}$/.test(g)) return null;
|
||||
hextets.push(parseInt(g, 16));
|
||||
}
|
||||
return hextets;
|
||||
}
|
||||
|
||||
function classifyIpv6(ip: string): AddressClass {
|
||||
const h = parseIpv6Hextets(ip);
|
||||
if (!h) return 'invalid';
|
||||
|
||||
const firstFive = h[0] | h[1] | h[2] | h[3] | h[4];
|
||||
if (firstFive === 0 && (h[5] === 0xffff || h[5] === 0)) {
|
||||
const embedded = `${h[6] >> 8}.${h[6] & 0xff}.${h[7] >> 8}.${h[7] & 0xff}`;
|
||||
const v4Class = classifyIpv4(embedded);
|
||||
if (h[5] === 0 && h[6] === 0 && h[7] === 0) return 'unspecified';
|
||||
if (h[5] === 0 && h[6] === 0 && h[7] === 1) return 'loopback'; // ::1
|
||||
return v4Class;
|
||||
}
|
||||
|
||||
if ((h[0] & 0xffc0) === 0xfe80) return 'link-local'; // fe80::/10
|
||||
if ((h[0] & 0xfe00) === 0xfc00) return 'unique-local'; // fc00::/7 (ULA)
|
||||
if ((h[0] & 0xff00) === 0xff00) return 'multicast'; // ff00::/8
|
||||
if (h[0] === 0x2001 && h[1] === 0x0db8) return 'reserved'; // 2001:db8::/32 docs
|
||||
if (h[0] === 0x0064 && h[1] === 0xff9b) return 'reserved'; // 64:ff9b::/96 NAT64
|
||||
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 0) return 'reserved'; // 100::/64 discard
|
||||
return 'public';
|
||||
}
|
||||
|
||||
/** Classify a single IP-literal address. Fail-closed: unknown -> 'invalid'. */
|
||||
export function classifyAddress(ip: string): AddressClass {
|
||||
const family = isIP(ip);
|
||||
if (family === 4) return classifyIpv4(ip);
|
||||
if (family === 6) return classifyIpv6(ip);
|
||||
return 'invalid';
|
||||
}
|
||||
|
||||
function isAllowed(cls: AddressClass, allowLocal: boolean): boolean {
|
||||
if (cls === 'public') return true;
|
||||
if (cls === 'loopback' && allowLocal) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
async function defaultLookup(hostname: string): Promise<ResolvedAddress[]> {
|
||||
const results = await dnsLookup(hostname, { all: true, verbatim: true });
|
||||
return results.map((r) => ({ address: r.address, family: r.family }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that `rawUrl` is an http(s) URL whose host resolves only to public
|
||||
* addresses. Throws {@link EgressBlockedError} otherwise. Returns parsed URL.
|
||||
*/
|
||||
export async function assertUrlAllowed(
|
||||
rawUrl: string,
|
||||
options: EgressGuardOptions = {},
|
||||
): Promise<URL> {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(rawUrl);
|
||||
} catch {
|
||||
throw new EgressBlockedError('Invalid URL', rawUrl);
|
||||
}
|
||||
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||
throw new EgressBlockedError(
|
||||
`Blocked non-http(s) scheme "${parsed.protocol}"`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
|
||||
// url.hostname keeps the surrounding brackets on an IPv6 literal ("[::1]"),
|
||||
// which isIP() does not recognize — strip them so the literal is classified
|
||||
// directly (loopback/private/link-local/…) instead of falling through to a DNS
|
||||
// lookup that fails ENOTFOUND on Linux (and only accidentally resolves on
|
||||
// Windows). Without this, bracketed-IPv6 URLs bypass classification entirely.
|
||||
const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
|
||||
const literalFamily = isIP(hostname);
|
||||
|
||||
let addresses: ResolvedAddress[];
|
||||
if (literalFamily !== 0) {
|
||||
addresses = [{ address: hostname, family: literalFamily }];
|
||||
} else {
|
||||
const lookupFn = options.lookup ?? defaultLookup;
|
||||
try {
|
||||
addresses = await lookupFn(hostname);
|
||||
} catch (err) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution failed for "${hostname}": ${detail}`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
if (!addresses || addresses.length === 0) {
|
||||
throw new EgressBlockedError(
|
||||
`DNS resolution returned no addresses for "${hostname}"`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const allowLocal = options.allowLocal ?? false;
|
||||
for (const { address } of addresses) {
|
||||
const cls = classifyAddress(address);
|
||||
if (!isAllowed(cls, allowLocal)) {
|
||||
throw new EgressBlockedError(
|
||||
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
|
||||
rawUrl,
|
||||
cls,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export interface SafeFetchOptions extends EgressGuardOptions {
|
||||
/** Maximum redirect hops to follow (default 5). */
|
||||
maxRedirects?: number;
|
||||
/** Injectable fetch (tests). Defaults to globalThis.fetch. */
|
||||
fetchImpl?: typeof globalThis.fetch;
|
||||
}
|
||||
|
||||
/**
|
||||
* SSRF-safe fetch. Validates before the request and re-validates every redirect
|
||||
* hop (`redirect: 'manual'`). Caller-supplied `redirect` in `init` is ignored.
|
||||
*/
|
||||
export async function safeFetch(
|
||||
rawUrl: string,
|
||||
init: RequestInit = {},
|
||||
options: SafeFetchOptions = {},
|
||||
): Promise<Response> {
|
||||
const maxRedirects = options.maxRedirects ?? 5;
|
||||
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
|
||||
|
||||
let currentUrl = rawUrl;
|
||||
for (let hop = 0; hop <= maxRedirects; hop++) {
|
||||
await assertUrlAllowed(currentUrl, options);
|
||||
|
||||
const response = await fetchImpl(currentUrl, { ...init, redirect: 'manual' });
|
||||
|
||||
const isRedirect = response.status >= 300 && response.status < 400;
|
||||
const location = isRedirect ? response.headers.get('location') : null;
|
||||
if (!location) {
|
||||
return response;
|
||||
}
|
||||
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
/* best-effort; ignore */
|
||||
}
|
||||
|
||||
let nextUrl: string;
|
||||
try {
|
||||
nextUrl = new URL(location, currentUrl).toString();
|
||||
} catch {
|
||||
throw new EgressBlockedError(
|
||||
`Invalid redirect target "${location}"`,
|
||||
currentUrl,
|
||||
);
|
||||
}
|
||||
currentUrl = nextUrl;
|
||||
}
|
||||
|
||||
throw new EgressBlockedError(
|
||||
`Exceeded maximum redirects (${maxRedirects})`,
|
||||
rawUrl,
|
||||
);
|
||||
}
|
||||
|
||||
/** True when local (loopback) fetches are opted in via env. */
|
||||
export function allowLocalFromEnv(): boolean {
|
||||
const v = process.env.WAGGLE_ALLOW_LOCAL_FETCH;
|
||||
return v === '1' || v === 'true';
|
||||
}
|
||||
Reference in New Issue
Block a user