moving
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled

This commit is contained in:
Oleg Maslov
2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions

View File

@@ -0,0 +1,162 @@
/**
* ChatGPT Adapter — parse ChatGPT JSON export into UniversalImportItems.
*
* ChatGPT export format uses a `mapping` object with node IDs containing
* messages. Each conversation has a title, create_time, and the mapping tree.
*/
import { stableHarvestId } from './stable-id.js';
import type { SourceAdapter, UniversalImportItem, ConversationMessage } from './types.js';
import { asRecord, getArray, getNumber, getString, type RawRecord } from './raw-types.js';
export class ChatGPTAdapter implements SourceAdapter {
readonly sourceType = 'chatgpt' as const;
readonly displayName = 'ChatGPT';
parse(input: unknown): UniversalImportItem[] {
const root = asRecord(input);
const conversations = Array.isArray(input) ? input : root && getArray(root, 'conversations');
if (!Array.isArray(conversations)) return [];
const items: UniversalImportItem[] = [];
for (const rawConv of conversations) {
const conv = asRecord(rawConv);
if (!conv) continue;
const title = getString(conv, 'title') || 'Untitled';
const messages: ConversationMessage[] = [];
// ChatGPT uses a mapping object with node IDs
const mapping = asRecord(conv.mapping);
if (mapping) {
const nodes = Object.values(mapping)
.map(asRecord)
.filter((n): n is RawRecord => n !== null);
const sorted = nodes
.filter(n => {
const msg = asRecord(n.message);
const content = msg && asRecord(msg.content);
const parts = content && getArray(content, 'parts');
return (parts?.length ?? 0) > 0;
})
.sort((a, b) => {
const am = asRecord(a.message);
const bm = asRecord(b.message);
return (am && getNumber(am, 'create_time') ? getNumber(am, 'create_time')! : 0)
- (bm && getNumber(bm, 'create_time') ? getNumber(bm, 'create_time')! : 0);
});
for (const node of sorted) {
const msg = asRecord(node.message);
const author = msg && asRecord(msg.author);
const authorRole = author && getString(author, 'role');
if (!msg || !authorRole) continue;
if (authorRole === 'system') continue;
const role = authorRole === 'user' ? 'user' as const : 'assistant' as const;
const content = asRecord(msg.content);
const parts = content ? getArray(content, 'parts') : undefined;
// W4.4 (caption parity): multimodal object parts were silently
// dropped — DALL-E image parts carry their generation prompt
// (metadata.dalle.prompt), the only text-bearing image field in
// ChatGPT exports. Render as "[Shared image: …]" (Memori's
// convention; W3.3 measured dropped captions at 4.5pp single-hop).
const textParts: string[] = [];
for (const p of parts ?? []) {
if (typeof p === 'string') { textParts.push(p); continue; }
const rec = asRecord(p);
if (!rec) continue;
const ct = getString(rec, 'content_type') ?? '';
if (ct.includes('image')) {
const meta = asRecord(rec.metadata);
const dalle = meta ? asRecord(meta.dalle) : null;
const prompt = dalle ? getString(dalle, 'prompt') : undefined;
if (prompt) textParts.push(`[Shared image: ${prompt}]`);
}
}
// Message-level attachments: names are text-bearing presence signals.
const msgMeta = asRecord(msg.metadata);
for (const rawAtt of (msgMeta ? getArray(msgMeta, 'attachments') : undefined) ?? []) {
const att = asRecord(rawAtt);
const name = att ? getString(att, 'name') : undefined;
if (name) textParts.push(`[Attached: ${name}]`);
}
const text = textParts.join('\n').trim();
if (!text) continue;
const createTime = getNumber(msg, 'create_time');
messages.push({
role,
text,
timestamp: createTime ? new Date(createTime * 1000).toISOString() : undefined,
});
}
}
if (messages.length === 0) continue;
// Also check for custom instructions in conversation metadata
const customInstructions = conv.custom_instructions;
const createTime = getNumber(conv, 'create_time');
items.push({
// #7 sticky erasure: stable per-conversation id (keyed on the export's own
// conversation id, NOT content, so a grown conversation keeps its id).
id: stableHarvestId('chatgpt', getString(conv, 'id') ?? getString(conv, 'conversation_id') ?? `conv\x00${title}\x00${createTime ?? ''}`),
source: 'chatgpt',
type: 'conversation',
title,
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
messages,
timestamp: createTime ? new Date(createTime * 1000).toISOString() : new Date().toISOString(),
metadata: {
conversationId: getString(conv, 'id') ?? getString(conv, 'conversation_id'),
messageCount: messages.length,
...(customInstructions ? { customInstructions } : {}),
},
});
}
// Also extract custom instructions / memory as separate items
if (root?.user_custom_instructions) {
items.push({
// 'singleton' discriminator (2 parts) so this can't collide with a
// conversation whose conv.id is literally the string 'custom_instructions'.
id: stableHarvestId('chatgpt', 'singleton', 'custom_instructions'),
source: 'chatgpt',
type: 'instruction',
title: 'ChatGPT Custom Instructions',
content: typeof root.user_custom_instructions === 'string'
? root.user_custom_instructions
: JSON.stringify(root.user_custom_instructions),
timestamp: new Date().toISOString(),
metadata: { type: 'custom_instructions' },
});
}
const memories = root && getArray(root, 'memories');
if (memories) {
for (const rawMem of memories) {
const mem = asRecord(rawMem);
const content = typeof rawMem === 'string'
? rawMem
: (mem && (getString(mem, 'content') ?? getString(mem, 'text'))) ?? JSON.stringify(rawMem);
items.push({
// No stable per-memory id exists in the export, so key on created_at+content
// (the best available surrogate). Caveat: ChatGPT memories are user-editable,
// so an EDIT changes the id → erasure isn't sticky across an edit (bounded,
// documented tradeoff — same class as the universal-text content-keyed path).
id: stableHarvestId('chatgpt', 'memory', (mem && getString(mem, 'created_at')) ?? '', content),
source: 'chatgpt',
type: 'memory',
title: 'ChatGPT Memory',
content,
timestamp: (mem && getString(mem, 'created_at')) ?? new Date().toISOString(),
metadata: { type: 'chatgpt_memory' },
});
}
}
return items;
}
}

View File

@@ -0,0 +1,30 @@
/**
* Shared text chunking utility for harvest adapters.
*
* Splits text into chunks by paragraph boundaries (double-newline),
* respecting a configurable maximum character length per chunk.
*/
const DEFAULT_MAX_LENGTH = 2000;
export function chunkByParagraphs(text: string, maxLen: number = DEFAULT_MAX_LENGTH): string[] {
const paragraphs = text.split(/\n\s*\n/).filter(p => p.trim().length > 0);
const chunks: string[] = [];
let current = '';
for (const para of paragraphs) {
const trimmed = para.trim();
if (current.length + trimmed.length + 2 > maxLen && current.length > 0) {
chunks.push(current.trim());
current = trimmed;
} else {
current += (current ? '\n\n' : '') + trimmed;
}
}
if (current.trim()) {
chunks.push(current.trim());
}
return chunks;
}

View File

@@ -0,0 +1,271 @@
/**
* Claude Adapter — parse Claude web/desktop JSON export into UniversalImportItems.
*
* Handles multiple shapes of the Claude export:
* - Raw `chat_messages` array per conversation
* - Structured content-block format where `msg.content` is typed blocks
* - `projects[].docs[]` project-knowledge documents → type='artifact'
* - `memories` (conversations_memory + project_memories map) → type='memory'
* - `design_chats[]` Claude design workspace threads → type='conversation'
*
* The memories / design_chats streams (and the enriched project-docs
* parsing) came online in the 2026-04-22 Claude.ai export refresh.
* Reverse-ported from OSS hive-mind (oss-drift triage R6, 2026-06-11).
*
* W4.4 (caption parity, mono-only): message-level `attachments` /
* `files` arrays surface their text content — `extracted_content` is
* inlined as "[Attached: name] …" (capped 500 chars), bare names as
* "[Shared file: name]".
*/
import { stableHarvestId } from './stable-id.js';
import type { SourceAdapter, UniversalImportItem, ConversationMessage } from './types.js';
import { asRecord, firstString, getArray, getString, type RawRecord } from './raw-types.js';
export class ClaudeAdapter implements SourceAdapter {
readonly sourceType = 'claude' as const;
readonly displayName = 'Claude';
parse(input: unknown): UniversalImportItem[] {
const items: UniversalImportItem[] = [];
const root = asRecord(input);
// ── Conversations (historical default path) ───────────────────────
const conversations = Array.isArray(input) ? input : root && getArray(root, 'conversations');
if (Array.isArray(conversations)) {
for (const conv of conversations) {
items.push(...this.parseConversation(conv));
}
}
// ── Project-knowledge docs → artifact items ──────────────────────
const projects = root && getArray(root, 'projects');
if (projects) {
for (const project of projects) {
items.push(...this.parseProjectDocs(project));
}
}
// ── Memories stream (conversations_memory + project_memories) ────
if (root && root.memories !== undefined) {
items.push(...this.parseMemories(root.memories));
}
// ── Design chats stream ──────────────────────────────────────────
const designChats = root && getArray(root, 'design_chats');
if (designChats) {
for (const dc of designChats) {
items.push(...this.parseDesignChat(dc));
}
}
return items;
}
/**
* Parse one Claude chat message (conversation or design-chat shape) into a
* ConversationMessage, applying the W4.4 attachment/caption extraction.
* Returns null for messages with no surfaceable text.
*/
private parseMessage(rawMsg: unknown): ConversationMessage | null {
const msg = asRecord(rawMsg);
if (!msg) return null;
const role = (getString(msg, 'sender') === 'human' || getString(msg, 'role') === 'user')
? 'user' as const
: 'assistant' as const;
// Handle content blocks (Claude format)
let text: string;
const blocks = getArray(msg, 'content');
if (blocks) {
text = blocks
.map(asRecord)
.filter((b): b is RawRecord => b !== null && b.type === 'text')
.map(b => getString(b, 'text') ?? '')
.join('\n')
.trim();
} else {
text = (getString(msg, 'text') ?? getString(msg, 'content') ?? '').trim();
}
// W4.4 (caption parity): Claude exports carry message-level
// `attachments` (with extracted_content — text already extracted
// from images/docs) and `files` arrays; both were never accessed.
const extras: string[] = [];
for (const key of ['attachments', 'files'] as const) {
for (const rawAtt of getArray(msg, key) ?? []) {
const att = asRecord(rawAtt);
if (!att) continue;
const name = getString(att, 'file_name') ?? getString(att, 'name');
const extracted = getString(att, 'extracted_content');
if (extracted && extracted.trim()) {
extras.push(`[Attached: ${name ?? 'file'}] ${extracted.trim().slice(0, 500)}`);
} else if (name) {
extras.push(`[Shared file: ${name}]`);
}
}
}
if (extras.length > 0) text = [text, ...extras].filter(Boolean).join('\n').trim();
if (!text) return null;
return {
role,
text,
timestamp: getString(msg, 'created_at') ?? getString(msg, 'timestamp'),
};
}
private parseConversation(rawConv: unknown): UniversalImportItem[] {
const conv = asRecord(rawConv);
if (!conv) return [];
const title = firstString(conv, 'name', 'title') || 'Untitled';
const messages: ConversationMessage[] = [];
const chatMessages = getArray(conv, 'chat_messages') ?? getArray(conv, 'messages') ?? [];
for (const rawMsg of chatMessages) {
const parsed = this.parseMessage(rawMsg);
if (parsed) messages.push(parsed);
}
if (messages.length === 0) return [];
return [{
// #7 sticky erasure: stable per-conversation id (conv uuid, not content).
id: stableHarvestId('claude', firstString(conv, 'uuid', 'id') ?? `conv\x00${title}\x00${firstString(conv, 'created_at', 'create_time') ?? ''}`),
source: 'claude',
type: 'conversation',
title,
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
messages,
timestamp: firstString(conv, 'created_at', 'create_time') ?? new Date().toISOString(),
metadata: {
conversationId: firstString(conv, 'uuid', 'id'),
messageCount: messages.length,
projectId: getString(conv, 'project_uuid') ?? undefined,
},
}];
}
private parseProjectDocs(rawProject: unknown): UniversalImportItem[] {
const project = asRecord(rawProject);
const docs = project && getArray(project, 'docs');
if (!project || !docs) return [];
const out: UniversalImportItem[] = [];
for (const rawDoc of docs) {
const doc = asRecord(rawDoc);
if (!doc) continue;
const content = getString(doc, 'content') ?? '';
if (content.length === 0) continue;
out.push({
id: stableHarvestId('claude', 'projdoc', getString(project, 'uuid') ?? '', getString(doc, 'uuid') ?? firstString(doc, 'filename', 'title') ?? ''),
source: 'claude',
type: 'artifact',
title: firstString(doc, 'filename', 'title') ?? 'Project Document',
content,
timestamp: getString(doc, 'created_at')
?? getString(project, 'updated_at')
?? getString(project, 'created_at')
?? new Date().toISOString(),
metadata: {
projectName: getString(project, 'name'),
projectUuid: getString(project, 'uuid'),
type: 'project_knowledge',
docUuid: getString(doc, 'uuid'),
filename: getString(doc, 'filename'),
},
});
}
return out;
}
private parseMemories(input: unknown): UniversalImportItem[] {
// `memories.json` is a single-entry array per the 2026-04-22 export
// shape: `[{ conversations_memory, project_memories, account_uuid }]`.
// Accept both the array-wrapped form and a bare object for flexibility.
const arr = Array.isArray(input) ? input : [input];
const out: UniversalImportItem[] = [];
for (const rawEntry of arr) {
const entry = asRecord(rawEntry);
if (!entry) continue;
const accountUuid = getString(entry, 'account_uuid');
// conversations_memory — usually a single long string of user-about facts
const convMem = entry.conversations_memory;
if (convMem !== undefined && convMem !== null) {
const content = typeof convMem === 'string' ? convMem : JSON.stringify(convMem);
if (content.length > 0) {
out.push({
// account-level singleton — fixed discriminator, NOT content (memory grows).
id: stableHarvestId('claude', 'memory', 'conversations_memory', accountUuid ?? ''),
source: 'claude',
type: 'memory',
title: 'Claude Memory — Conversations',
content,
timestamp: new Date().toISOString(),
metadata: {
memoryKind: 'conversations_memory',
accountUuid,
},
});
}
}
// project_memories — map of project_uuid -> memory string
const projectMemories = asRecord(entry.project_memories);
if (projectMemories) {
for (const [projUuid, memValue] of Object.entries(projectMemories)) {
const content = typeof memValue === 'string' ? memValue : JSON.stringify(memValue);
if (content.length > 0) {
out.push({
// per-project memory keyed on the project uuid map key (not content).
id: stableHarvestId('claude', 'memory', 'project_memory', accountUuid ?? '', projUuid),
source: 'claude',
type: 'memory',
title: `Claude Memory — Project ${projUuid}`,
content,
timestamp: new Date().toISOString(),
metadata: {
memoryKind: 'project_memory',
projectUuid: projUuid,
accountUuid,
},
});
}
}
}
}
return out;
}
private parseDesignChat(input: unknown): UniversalImportItem[] {
const dc = asRecord(input);
if (!dc) return [];
const messages: ConversationMessage[] = [];
const msgs = getArray(dc, 'messages') ?? getArray(dc, 'chat_messages') ?? [];
for (const rawMsg of msgs) {
const parsed = this.parseMessage(rawMsg);
if (parsed) messages.push(parsed);
}
if (messages.length === 0) return [];
return [{
id: stableHarvestId('claude', getString(dc, 'uuid') ?? `designchat\x00${getString(dc, 'project') ?? ''}\x00${getString(dc, 'title') ?? ''}\x00${getString(dc, 'created_at') ?? ''}`),
source: 'claude',
type: 'conversation',
title: getString(dc, 'title') ?? 'Claude Design Chat',
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
messages,
timestamp: getString(dc, 'created_at') ?? new Date().toISOString(),
metadata: {
designChatUuid: getString(dc, 'uuid'),
projectUuid: getString(dc, 'project') ?? undefined,
messageCount: messages.length,
stream: 'design_chats',
},
}];
}
}

View File

@@ -0,0 +1,359 @@
/**
* Claude Code Filesystem Adapter — reads ~/.claude/ directory structure.
*
* Extracts:
* - memory/*.md files (with frontmatter: type, name, description)
* - rules/**\/*.md files (coding standards, workflow rules)
* - plans/*.md files (implementation plans)
* - settings.json (model preferences, tool config)
* - CLAUDE.md project files (architectural decisions)
* - .mind/*.md session handoffs (decisions, directions)
* - Decision extraction from memory content (pattern matching)
*
* This is a FilesystemAdapter — it reads directly from disk.
*/
import { stableHarvestId } from './stable-id.js';
import { decisionOfSubjectId } from './decision-derivation.js';
import * as fs from 'node:fs';
import * as path from 'node:path';
import type { FilesystemAdapter, UniversalImportItem, ImportItemType } from './types.js';
// Map Claude Code memory types to our import types
const MEMORY_TYPE_MAP: Record<string, ImportItemType> = {
user: 'preference',
feedback: 'decision',
project: 'memory',
reference: 'memory',
};
// Patterns that indicate user decisions in text
const DECISION_PATTERNS = [
/\bwe (?:decided|chose|picked|went with|agreed|confirmed)\b/i,
/\blet'?s (?:go with|use|do|keep|drop|switch|move)\b/i,
/\bdecision:\s/i,
/\bconfirmed:\s/i,
/\bapproved:\s/i,
/\brejected:\s/i,
/\bwon'?t (?:do|use|implement|add|need)\b/i,
/\bmust (?:use|have|be|support|include)\b/i,
/\bnon-negotiable\b/i,
/\brequirement:\s/i,
/\bconstraint:\s/i,
/\bchose .+ (?:over|instead of|rather than)\b/i,
/\bdropped?\b.+\bin favo(?:u)?r of\b/i,
];
interface MemoryFrontmatter {
name?: string;
description?: string;
type?: string;
}
function parseFrontmatter(content: string): { frontmatter: MemoryFrontmatter; body: string } {
const match = content.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
if (!match) return { frontmatter: {}, body: content };
const raw = match[1];
const body = match[2].trim();
const frontmatter: MemoryFrontmatter = {};
for (const line of raw.split('\n')) {
const colonIdx = line.indexOf(':');
if (colonIdx < 0) continue;
const key = line.slice(0, colonIdx).trim();
const value = line.slice(colonIdx + 1).trim();
if (key === 'name') frontmatter.name = value;
if (key === 'description') frontmatter.description = value;
if (key === 'type') frontmatter.type = value;
}
return { frontmatter, body };
}
function readFilesRecursive(dir: string, ext: string): { filePath: string; content: string }[] {
const results: { filePath: string; content: string }[] = [];
if (!fs.existsSync(dir)) return results;
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
results.push(...readFilesRecursive(fullPath, ext));
} else if (entry.name.endsWith(ext)) {
try {
results.push({ filePath: fullPath, content: fs.readFileSync(fullPath, 'utf-8') });
} catch { /* skip unreadable files */ }
}
}
return results;
}
export class ClaudeCodeAdapter implements FilesystemAdapter {
readonly sourceType = 'claude-code' as const;
readonly displayName = 'Claude Code';
parse(input: unknown): UniversalImportItem[] {
// For the SourceAdapter interface — parse JSON if provided
if (typeof input === 'string') {
return this.scan(input);
}
return [];
}
scan(dirPath: string): UniversalImportItem[] {
if (!fs.existsSync(dirPath)) return [];
const items: UniversalImportItem[] = [];
// 1. Scan all project memory directories
const projectsDir = path.join(dirPath, 'projects');
if (fs.existsSync(projectsDir)) {
const projectEntries = fs.readdirSync(projectsDir, { withFileTypes: true });
for (const projEntry of projectEntries) {
if (!projEntry.isDirectory()) continue;
const memoryDir = path.join(projectsDir, projEntry.name, 'memory');
items.push(...this.scanMemoryDir(memoryDir, projEntry.name));
}
}
// 2. Scan rules
const rulesDir = path.join(dirPath, 'rules');
const ruleFiles = readFilesRecursive(rulesDir, '.md');
for (const { filePath, content } of ruleFiles) {
const relPath = path.relative(dirPath, filePath);
items.push({
// #7 sticky erasure: file path is the stable id (survives content growth).
// Normalize the OS separator so the SAME tree scanned on win32 vs POSIX
// yields the SAME id (stableHarvestId's cross-process determinism contract).
id: stableHarvestId('claude-code', relPath.split(path.sep).join('/')),
source: 'claude-code',
type: 'rule',
title: `Rule: ${path.basename(filePath, '.md')}`,
content,
timestamp: this.getFileMtime(filePath),
metadata: { filePath: relPath, category: 'rule' },
});
}
// 3. Scan plans
const plansDir = path.join(dirPath, 'plans');
if (fs.existsSync(plansDir)) {
const planFiles = fs.readdirSync(plansDir).filter(f => f.endsWith('.md'));
for (const planFile of planFiles) {
const fullPath = path.join(plansDir, planFile);
try {
const content = fs.readFileSync(fullPath, 'utf-8');
items.push({
id: stableHarvestId('claude-code', `plans/${planFile}`),
source: 'claude-code',
type: 'artifact',
title: `Plan: ${planFile.replace('.md', '')}`,
content,
timestamp: this.getFileMtime(fullPath),
metadata: { filePath: `plans/${planFile}`, category: 'plan' },
});
} catch { /* skip */ }
}
}
// 4. Read settings.json for preferences
const settingsPath = path.join(dirPath, 'settings.json');
if (fs.existsSync(settingsPath)) {
try {
const raw = fs.readFileSync(settingsPath, 'utf-8');
const settings = JSON.parse(raw);
const prefs: string[] = [];
if (settings.model) prefs.push(`Preferred model: ${settings.model}`);
if (settings.alwaysThinkingEnabled) prefs.push('Extended thinking: enabled');
if (Array.isArray(settings.allowedTools)) {
prefs.push(`Allowed tools: ${settings.allowedTools.length} configured`);
}
if (prefs.length > 0) {
items.push({
id: stableHarvestId('claude-code', 'settings.json'),
source: 'claude-code',
type: 'preference',
title: 'Claude Code Settings',
content: prefs.join('\n'),
timestamp: this.getFileMtime(settingsPath),
metadata: { filePath: 'settings.json', category: 'settings' },
});
}
} catch { /* skip */ }
}
// 5. Scan CLAUDE.md files from project directories (architectural decisions)
if (fs.existsSync(projectsDir)) {
const projectEntries = fs.readdirSync(projectsDir, { withFileTypes: true });
for (const projEntry of projectEntries) {
if (!projEntry.isDirectory()) continue;
const claudeMdPath = path.join(projectsDir, projEntry.name, 'CLAUDE.md');
if (fs.existsSync(claudeMdPath)) {
try {
const content = fs.readFileSync(claudeMdPath, 'utf-8');
if (content.trim().length > 50) {
items.push({
id: stableHarvestId('claude-code', `projects/${projEntry.name}/CLAUDE.md`),
source: 'claude-code',
type: 'artifact',
title: `Project CLAUDE.md (${projEntry.name})`,
content: content.slice(0, 4000),
timestamp: this.getFileMtime(claudeMdPath),
metadata: {
filePath: `projects/${projEntry.name}/CLAUDE.md`,
category: 'project-config',
},
});
}
} catch { /* skip */ }
}
}
}
// 6. Scan .mind/ directories for session handoffs and decisions
if (fs.existsSync(projectsDir)) {
const projectEntries = fs.readdirSync(projectsDir, { withFileTypes: true });
for (const projEntry of projectEntries) {
if (!projEntry.isDirectory()) continue;
const mindDir = path.join(projectsDir, projEntry.name, '.mind');
items.push(...this.scanMindDir(mindDir, projEntry.name));
}
}
// 7. Extract decisions from memory items that contain decision language
items.push(...this.extractDecisions(items));
return items;
}
/** Scan .mind/ directory for session handoffs with decisions. */
private scanMindDir(mindDir: string, projectHash: string): UniversalImportItem[] {
if (!fs.existsSync(mindDir)) return [];
const items: UniversalImportItem[] = [];
try {
const files = fs.readdirSync(mindDir).filter(f => f.endsWith('.md'));
for (const file of files) {
const fullPath = path.join(mindDir, file);
try {
const content = fs.readFileSync(fullPath, 'utf-8');
if (content.trim().length < 50) continue;
// Determine type from filename
const lowerFile = file.toLowerCase();
const isDecision = lowerFile.includes('decision');
const isState = lowerFile.includes('state');
items.push({
id: stableHarvestId('claude-code', `projects/${projectHash}/.mind/${file}`),
source: 'claude-code',
type: isDecision ? 'decision' : 'artifact',
title: `${isDecision ? 'Decisions' : isState ? 'State' : 'Session'}: ${file.replace('.md', '')}`,
content: content.slice(0, 4000),
timestamp: this.getFileMtime(fullPath),
metadata: {
filePath: `projects/${projectHash}/.mind/${file}`,
category: isDecision ? 'decision' : 'session-handoff',
},
});
} catch { /* skip */ }
}
} catch { /* skip */ }
return items;
}
/**
* Extract decision items from existing memory/preference/artifact items.
* Scans content for decision patterns and creates separate decision items
* for statements that match. Avoids duplicating items already typed as 'decision'.
*/
private extractDecisions(existingItems: readonly UniversalImportItem[]): UniversalImportItem[] {
const decisions: UniversalImportItem[] = [];
for (const item of existingItems) {
// Skip items already categorized as decisions
if (item.type === 'decision' || item.type === 'rule') continue;
const lines = item.content.split('\n');
const decisionLines: string[] = [];
for (const line of lines) {
const trimmed = line.trim();
if (trimmed.length < 10) continue;
for (const pattern of DECISION_PATTERNS) {
if (pattern.test(trimmed)) {
decisionLines.push(trimmed);
break;
}
}
}
if (decisionLines.length > 0) {
decisions.push({
// derived item — namespaced by the (now-stable) parent id so it never
// collides with the parent's own id. decisionOfSubjectId is the SHARED
// derivation MindErasure.eraseBySourceRef recomputes to reach + suppress
// this derived subject on erasure (#7 P2), so the two sites cannot drift.
id: decisionOfSubjectId(item.id),
source: 'claude-code',
type: 'decision',
title: `Decisions from: ${item.title}`,
content: decisionLines.join('\n'),
timestamp: item.timestamp,
metadata: {
...item.metadata,
category: 'decision',
extractedFrom: item.id,
decisionCount: decisionLines.length,
},
});
}
}
return decisions;
}
private scanMemoryDir(memoryDir: string, projectHash: string): UniversalImportItem[] {
if (!fs.existsSync(memoryDir)) return [];
const items: UniversalImportItem[] = [];
const files = fs.readdirSync(memoryDir).filter(f => f.endsWith('.md') && f !== 'MEMORY.md');
for (const file of files) {
const fullPath = path.join(memoryDir, file);
try {
const raw = fs.readFileSync(fullPath, 'utf-8');
const { frontmatter, body } = parseFrontmatter(raw);
if (!body) continue;
const importType = MEMORY_TYPE_MAP[frontmatter.type ?? ''] ?? 'memory';
items.push({
id: stableHarvestId('claude-code', `projects/${projectHash}/memory/${file}`),
source: 'claude-code',
type: importType,
title: frontmatter.name ?? file.replace('.md', ''),
content: body,
timestamp: this.getFileMtime(fullPath),
metadata: {
filePath: `projects/${projectHash}/memory/${file}`,
memoryType: frontmatter.type,
description: frontmatter.description,
category: 'memory',
},
});
} catch { /* skip */ }
}
return items;
}
private getFileMtime(filePath: string): string {
try {
return fs.statSync(filePath).mtime.toISOString();
} catch {
return new Date().toISOString();
}
}
}

View File

@@ -0,0 +1,31 @@
/**
* decision-derivation.ts — the single source of truth for claude-code's derived
* `decision-of` subject key.
*
* claude-code harvest (claude-code-adapter extractDecisions) fans a scanned item
* out into a SEPARATE "Decisions from: …" item that quotes the parent's decision
* lines. That derived item lands as its OWN GDPR Art.17 subject, keyed on
* decisionOfSubjectId(parentId). Two layers must agree on that key EXACTLY:
* - the adapter, which MINTS the derived item's id at harvest time;
* - MindErasure.eraseBySourceRef, which RECOMPUTES it to erase + suppress the
* derived subject when the parent is erased (else it survives erasure and
* re-materializes on re-import — the #7 P2 gap).
* Duplicating the derivation across those two sites is precisely the drift that
* created the gap, so both import from here. Changing the token also invalidates
* every already-persisted derived subject id, so treat it as a data contract.
*/
import { stableHarvestId } from './stable-id.js';
/** The only harvest source that derives a separate `decision-of` subject. */
export const CLAUDE_CODE_DECISION_SOURCE = 'claude-code';
/** The derivation-kind token the derived id is namespaced under. */
export const DECISION_OF_KIND = 'decision-of';
/**
* Deterministic id for the `decision-of` subject derived from a claude-code
* parent item. `parentId` is the parent's stable harvest id (== its raw_archive
* source_ref), so the id is stable across re-imports of the same parent.
*/
export function decisionOfSubjectId(parentId: string): string {
return stableHarvestId(CLAUDE_CODE_DECISION_SOURCE, DECISION_OF_KIND, parentId);
}

Binary file not shown.

View File

@@ -0,0 +1,267 @@
// Reverse-ported from OSS hive-mind llm-extractor (oss-drift triage D2, 2026-06-11); executors rehomed onto LLMCallFn.
/**
* extract-kg-entities.ts — LLM-based knowledge-graph entity extraction.
*
* The heuristic capitalized-n-gram regex (packages/agent/src/entity-extractor.ts
* via CognifyPipeline) produces high noise: sentence-starts, log prefixes, and
* fragments. This pass replaces the regex with an LLM that understands semantics
* and returns typed entities (person/project/file/decision/bug/concept/tool/
* location) keyed by frame id.
*
* Ported core = PROMPT + JSONL PARSER + BATCHING + noise filter. The OSS
* executors ('cc' subprocess spawn, raw Anthropic POST) are dropped — the
* monorepo drives all LLM calls through `LLMCallFn` ('fast' tier), exactly
* like extract-memory-lanes.ts.
*
* Failure model mirrors extract-memory-lanes: per-batch failures are collected
* into `errors`, never thrown — partial results beat zero results when one
* frame confuses the model. All extracted names are injection-scanned before
* being returned (LLM output over possibly-tainted harvested content).
*/
import type { LLMCallFn } from './pipeline.js';
import { scanForInjection } from '../injection-scanner.js';
import { createCoreLogger } from '../logger.js';
import { isNoiseName, normalizeEntityName } from '../mind/entity-normalizer.js';
import type { KnowledgeGraph } from '../mind/knowledge.js';
const log = createCoreLogger('extract-kg-entities');
/** Canonical entity types the prompt asks the model to choose from. */
export const KG_ENTITY_TYPES = [
'person',
'project',
'file',
'decision',
'bug',
'concept',
'tool',
'location',
] as const;
export type KgEntityType = (typeof KG_ENTITY_TYPES)[number];
/** One extracted entity attributed back to its source frame. */
export interface KgEntity {
frameId: number;
type: KgEntityType;
name: string;
}
export interface KgEntityExtraction {
entities: KgEntity[];
errors: string[];
}
/**
* Frames per batch. OSS defaulted to 3 because `claude -p` subprocess
* wall-clock blew past 90s on batch=5 with raw frames; LLMCallFn has no
* subprocess pressure, so 5 frames/batch with the per-frame content cap
* keeps prompts bounded while halving call count.
*/
const BATCH_SIZE = 5;
/**
* Per-frame content cap before sending to the model (OSS finding: the first
* ~2-3KB of a frame carries the named entities; long wiki-synth frames can
* exceed 8KB and add nothing but latency).
*/
const MAX_FRAME_CHARS = 2500;
/** The instructions block sent to the model. Stable across batches. */
const PROMPT_INSTRUCTIONS = `Extract named entities from the FRAMES below. For each entity, output ONE JSON object on its own line.
OUTPUT FORMAT (JSONL — one object per line, no other text):
{"frame_id": <number>, "name": "<entity name>", "type": "<type>"}
VALID TYPES (pick the closest fit):
- person a specific human (e.g. "Marko", "Alice Chen")
- project a named project, repo, codebase, or product (e.g. "hive-mind", "Phase 3")
- file a specific file path or filename (e.g. "synth-drain.js", "PHASE-3-PLAN.md")
- decision a specific architectural or strategic choice with a name (e.g. "open-core boundary")
- bug a known issue, incident, or failure mode (e.g. "subprocess feedback loop")
- tool a CLI tool, library, framework, or service (e.g. "Ollama", "Voyage", "sqlite-vec")
- concept a domain concept that doesn't fit above (e.g. "watermark", "reranker")
- location a directory or workspace path (e.g. "D:/Projects/hive-mind")
DO NOT EXTRACT:
- pronouns, demonstratives ("this", "that", "these")
- generic verbs at sentence start ("Add", "Update", "Run")
- standalone acronyms shorter than 4 chars ("API", "CLI", "MCP", "JSON")
- weekdays, months, dates
- common English words
- fragments — if you'd struggle to write a wiki page about it, skip it
QUALITY BAR: ~3-8 high-signal entities per frame is typical. If a frame is short or non-substantive, return zero entities for it (just don't emit lines for it).
Output JSONL only. No prose, no markdown fences, no commentary.`;
interface FrameInput {
id: number;
content: string;
}
/** Builds the user-message text for one batch. */
function buildBatchPrompt(frames: ReadonlyArray<FrameInput>): string {
const sep = '='.repeat(60);
const blocks = frames.map((f) => {
const trimmed = f.content.trim();
const body = trimmed.length > MAX_FRAME_CHARS
? `${trimmed.slice(0, MAX_FRAME_CHARS)}\n[...frame truncated for extraction; ${trimmed.length - MAX_FRAME_CHARS} chars omitted]`
: trimmed;
return `${sep}\nFRAME id=${f.id}\n${sep}\n${body}`;
}).join('\n\n');
return `${PROMPT_INSTRUCTIONS}\n\n${blocks}\n\n=== END OF FRAMES ===\n\nNow output JSONL:`;
}
/**
* Strips a fenced code block the model sometimes adds despite instructions.
* Returns the inner content if a single \`\`\`...\`\`\` block wraps everything,
* else the original text.
*/
function unwrapFencedBlock(text: string): string {
const trimmed = text.trim();
const fenceMatch = trimmed.match(/^```(?:json|jsonl)?\s*\n([\s\S]*?)\n```\s*$/);
return fenceMatch ? fenceMatch[1] : trimmed;
}
/**
* Parses LLM JSONL output into typed entities. Per-line tolerance: a single
* malformed line never aborts the batch. Dropped lines:
* - unparseable JSON
* - frame_id not in this batch (the model invented one — unattributable)
* - type outside KG_ENTITY_TYPES (stricter than OSS, which coerced to
* 'concept' — validate at the boundary instead of laundering junk types)
* - names failing the write-time noise filter (isNoiseName)
* - names carrying an injection payload (scanned BEFORE returning — LLM
* output over harvested content is tainted input)
*/
function parseJsonlOutput(raw: string, validFrameIds: ReadonlySet<number>): KgEntity[] {
const entities: KgEntity[] = [];
const cleaned = unwrapFencedBlock(raw);
for (const line of cleaned.split('\n')) {
const trimmed = line.trim();
if (!trimmed || !trimmed.startsWith('{')) continue;
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(trimmed) as Record<string, unknown>;
} catch {
continue;
}
const frameId = Number(parsed.frame_id);
if (!Number.isFinite(frameId) || !validFrameIds.has(frameId)) continue;
const name = typeof parsed.name === 'string' ? parsed.name.trim() : '';
if (name.length < 2) continue;
// Write-time noise filter (oss-drift R3 — first wiring): stop tokens,
// sub-4-char names, single-word acronyms never enter the graph.
if (isNoiseName(name)) continue;
const rawType = typeof parsed.type === 'string' ? parsed.type.toLowerCase().trim() : '';
if (!(KG_ENTITY_TYPES as readonly string[]).includes(rawType)) continue;
const scan = scanForInjection(name, 'tool_output');
if (!scan.safe) {
log.warn('dropping extracted entity name with injection payload', { flags: scan.flags.join(',') });
continue;
}
entities.push({ frameId, name, type: rawType as KgEntityType });
}
return entities;
}
/**
* Extract typed KG entities from N frames via the LLM. Internally batches
* into groups of BATCH_SIZE, one 'fast'-tier call per batch. Per-batch
* failures are collected into `errors`, never thrown.
*/
export async function extractKgEntities(
datedFrames: ReadonlyArray<FrameInput>,
llmCall: LLMCallFn,
): Promise<KgEntityExtraction> {
const out: KgEntityExtraction = { entities: [], errors: [] };
if (datedFrames.length === 0) return out;
for (let i = 0; i < datedFrames.length; i += BATCH_SIZE) {
const batch = datedFrames.slice(i, i + BATCH_SIZE);
const validIds = new Set(batch.map((f) => f.id));
try {
const raw = await llmCall(buildBatchPrompt(batch), 'fast');
out.entities.push(...parseJsonlOutput(raw, validIds));
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
out.errors.push(`kg-entities batch ${i / BATCH_SIZE} (frames ${batch[0].id}..${batch[batch.length - 1].id}): ${msg}`);
log.warn('kg-entity extraction batch failed', { batch: i / BATCH_SIZE, error: msg });
}
}
return out;
}
// ── Graph writing ────────────────────────────────────────────────────────────
export interface WriteKgEntitiesResult {
/** New knowledge_entities rows. */
created: number;
/** Existing entities whose seen_count was bumped (exact-name dedup hit). */
updated: number;
}
function safeParseProps(raw: string | undefined | null): Record<string, unknown> {
if (!raw) return {};
try { return JSON.parse(raw) as Record<string, unknown>; } catch { return {}; }
}
/**
* Persist extracted entities into the knowledge graph.
*
* Dedup is exact-name via `kg.findEntityByName()` (oss-drift R2) — NEVER the
* LIKE-based `searchEntities` top-K, which silently drops the exact match once
* enough similarly-named entities accumulate (3506 duplicate "Phase" rows
* observed in the OSS repo). On a hit, seen_count is bumped the same way the
* cognify CLI does; on a miss, a new row is created with source 'cognify-llm'
* (the tag that distinguishes LLM-grade entities from heuristic noise).
*/
export function writeKgEntities(
kg: KnowledgeGraph,
extraction: KgEntityExtraction,
): WriteKgEntitiesResult {
const result: WriteKgEntitiesResult = { created: 0, updated: 0 };
for (const entity of extraction.entities) {
// Defense at the write seam (mirrors the cognify CLI): callers other than
// extractKgEntities may not have noise-filtered.
if (isNoiseName(entity.name)) continue;
if (normalizeEntityName(entity.name).length < 3) continue;
const existing = kg.findEntityByName(entity.name);
if (existing) {
const existingProps = safeParseProps(existing.properties);
const seenCount = Number(existingProps.seen_count ?? 1) + 1;
kg.updateEntity(existing.id, {
properties: { ...existingProps, seen_count: seenCount },
});
kg.linkEntityToFrame(existing.id, entity.frameId);
result.updated++;
} else {
try {
const created = kg.createEntity(entity.type, entity.name, { seen_count: 1, source: 'cognify-llm' });
kg.linkEntityToFrame(created.id, entity.frameId);
result.created++;
} catch (e: unknown) {
// Ontology validation may reject — skip this entity, never abort the pass.
log.warn('createEntity rejected extracted entity', {
name: entity.name,
error: e instanceof Error ? e.message : String(e),
});
}
}
}
return result;
}

View File

@@ -0,0 +1,335 @@
/**
* extract-memory-lanes.ts — W4.3 extraction passes for the benchmark-proven
* recall lanes (W4-PRODUCTION-PORT-PLAN-2026-06-11.md components #5/#6/#8).
*
* Three single-call LLM passes over conversation text, production
* generalizations of the LoCoMo-validated extraction scripts (28/31 dense
* facts, 33 episodic events, 35 profile cards — evidence: 87.66 overall,
* benchmarks/results/memori-phase22-RESULT.md):
*
* - DENSE FACTS → `[mind-fact]` cross-session syntheses
* - EPISODIC → `[mind-event]` datable events, created_at =
* LLM-resolved EVENT date
* - PROFILES → `[mind-profile <name>]` per-person persona cards,
* importance DELIBERATELY 'normal'
* (stays out of the K5 lane)
*
* Frames are prefix-tagged on their first line so recall lanes fetch them by
* `content LIKE '[mind-… %'` — the same convention the benchmark proved.
* All LLM output is injection-scanned before any frame write (LLM passes run
* over possibly-tainted harvested content).
*
* Model: LLMCallFn 'fast' tier (benchmark used gpt-4o-mini, temp 0). Ollama
* is an optional routing target via LiteLLM — no hard dependency.
*/
import type { LLMCallFn } from './pipeline.js';
import { scanForInjection } from '../injection-scanner.js';
import { createCoreLogger } from '../logger.js';
import type { FrameStore } from '../mind/frames.js';
const log = createCoreLogger('extract-memory-lanes');
/** First-line content prefixes for the three lanes (recall fetches by these). */
export const MIND_FACT_PREFIX = '[mind-fact]';
export const MIND_EVENT_PREFIX = '[mind-event]';
export const MIND_PROFILE_PREFIX = '[mind-profile';
export interface ExtractedEvent {
/** ISO date of the session/source the event was narrated in. */
session_date: string;
/** Relative cue found in the utterance ("yesterday"…), or "none". */
cue: string;
/** LLM-resolved date the event actually happened (YYYY-MM-DD). */
event_date: string;
text: string;
}
export interface ExtractedFact {
category: 'preference' | 'decision' | 'trait' | 'theme';
speaker: string;
text: string;
}
export interface ExtractedProfile {
speaker: string;
card: string;
}
export interface MemoryLaneExtraction {
facts: ExtractedFact[];
events: ExtractedEvent[];
profiles: ExtractedProfile[];
errors: string[];
}
// ── Prompts (benchmark scripts 31/33/35, generalized off the speaker pair) ──
const FACTS_SYSTEM =
'You extract many synthesis-level memory facts from long-term conversations. ' +
'Be exhaustive — cover preferences, decisions, traits, life-stances, beliefs, ' +
'progressions, themes, hobbies, fears, joys, opinions, family relationships, ' +
'professional details. Output ONLY the JSON, no preamble.';
function factsPrompt(text: string): string {
return `${FACTS_SYSTEM}
Conversation/source material:
${text}
---
Extract a DENSE set of synthesis-level memory facts (scale the count to the material; up to ~60 for long conversations). Aim for HIGH COVERAGE — include facts relevant to inference questions like "would X be considered Y?" or "how does X feel about Y?".
Categories:
1. preference — "User preference: <Name> [values/likes/dislikes/prefers/believes] <thing>[ because <reason>]"
2. decision — "Decision: <Name> decided to/plans to <action>[, because <reason>]"
3. trait — "Trait: <Name> is <trait/orientation>[, as shown by <pattern>]"
4. theme — "Theme: <topic/arc> — <insight or progression across sessions>"
Be SPECIFIC and CONCRETE. Each fact stands alone. Cover all participants.
Output STRICT JSON:
{"facts": [{"category": "preference|decision|trait|theme", "speaker": "Name|both", "text": "<full prefix-tagged sentence>"}, ...]}`;
}
const EVENTS_SYSTEM =
'You extract specific datable events from long-term conversations AND resolve WHEN each ' +
'event actually happened. Source material carries dates (session headers, timestamps). ' +
'Events are often recounted in PAST tense with relative time cues ("yesterday", "last ' +
'week", "two months ago", "last year"). You MUST compute the ACTUAL event date by applying ' +
'the cue to that passage\'s date — do NOT just copy the source date. If an event has no ' +
'relative cue (happening now / present tense / planned for the future), use the source date. ' +
'Focus on concrete things that HAPPENED: activities, places visited, milestones, purchases, ' +
'meetings, projects, health events, travel. Do NOT include timeless preferences or ' +
'personality traits — only events. Output ONLY the JSON.';
function eventsPrompt(text: string): string {
return `${EVENTS_SYSTEM}
Source material (dated):
${text}
---
Extract the specific datable events (scale the count to the material). For EACH event output:
- session_date: the ISO date of the passage the event was narrated in (YYYY-MM-DD)
- cue: the exact relative time phrase ("yesterday", "last week", "two months ago"), or "none"
- event_date: the RESOLVED actual date the event happened (YYYY-MM-DD)
- text: a concise sentence about what happened (names, titles, exact activities)
Resolution rules (apply cue to session_date):
- "yesterday" -> session_date − 1 day
- "the day before yesterday" -> session_date − 2 days
- "last week" / "a week ago" -> session_date − 7 days
- "N days/weeks ago" -> subtract that many days/weeks
- "last month" / "a month ago" -> session_date − 1 month
- "N months ago" -> subtract N months
- "last year" -> same month/day, year − 1
- "none" (present tense/now) -> event_date = session_date
Worked example: passage dated 2023-05-08, "I went to the support group yesterday"
-> {"session_date":"2023-05-08","cue":"yesterday","event_date":"2023-05-07","text":"Caroline attended the LGBTQ support group"}
Output STRICT JSON:
{"events": [{"session_date":"YYYY-MM-DD","cue":"...","event_date":"YYYY-MM-DD","text":"..."}, ...]}`;
}
const PROFILES_SYSTEM =
'You build dense persona profile cards from long-term conversations, aggregating ' +
'dispersed weak signals (activities, choices, stated values, recurring themes) into a ' +
'coherent portrait. The card must support INFERENCE questions like "would X be ' +
'considered religious?" or "what would X\'s likely preference be?". Aggregate signals; ' +
'include world-knowledge hooks (specific titles, brand names, place names — verbatim, ' +
'never generalized). Output ONLY the JSON.';
function profilesPrompt(text: string): string {
return `${PROFILES_SYSTEM}
Source material:
${text}
---
Build one profile card per main participant (120-180 words each). Each card MUST cover, compactly:
- Identity & life situation (job/role, family, relationships, location if stated)
- Interests & habits with SPECIFIC named items (exact titles, brands, activities)
- Values, beliefs, personality leanings AS EVIDENCED
- Major life arc events with rough dates
- People mentioned around them and who those people likely are
- Current state at the end of the material (latest job, plans, status)
Write declarative, signal-dense prose. No hedging filler. Keep verbatim named entities.
Output STRICT JSON:
{"profiles": [{"speaker": "<Name>", "card": "..."}, ...]}`;
}
// ── Parsing helpers ──────────────────────────────────────────────────────────
const ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
function parseJsonObject(raw: string): Record<string, unknown> | null {
const cleaned = raw.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
try {
const parsed: unknown = JSON.parse(cleaned);
return parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)
? (parsed as Record<string, unknown>)
: null;
} catch {
return null;
}
}
// ── Extraction ──────────────────────────────────────────────────────────────
/**
* Run the three lane-extraction passes over one body of conversation text.
* Each pass fails independently — a parse failure on one lane never blocks
* the others (errors are collected, not thrown).
*/
export async function extractMemoryLanes(
text: string,
llmCall: LLMCallFn,
): Promise<MemoryLaneExtraction> {
const out: MemoryLaneExtraction = { facts: [], events: [], profiles: [], errors: [] };
const passes: Array<{ name: string; run: () => Promise<void> }> = [
{
name: 'facts',
run: async () => {
const obj = parseJsonObject(await llmCall(factsPrompt(text), 'fast'));
const facts = Array.isArray(obj?.facts) ? obj.facts : [];
for (const f of facts as Array<Record<string, unknown>>) {
if (typeof f?.text === 'string' && f.text.trim().length > 0) {
out.facts.push({
category: (['preference', 'decision', 'trait', 'theme'].includes(String(f.category))
? String(f.category)
: 'preference') as ExtractedFact['category'],
speaker: typeof f.speaker === 'string' ? f.speaker : 'unknown',
text: f.text.trim(),
});
}
}
},
},
{
name: 'events',
run: async () => {
const obj = parseJsonObject(await llmCall(eventsPrompt(text), 'fast'));
const events = Array.isArray(obj?.events) ? obj.events : [];
for (const e of events as Array<Record<string, unknown>>) {
const eventDate = typeof e?.event_date === 'string' ? e.event_date : '';
if (typeof e?.text === 'string' && e.text.trim().length > 0 && ISO_DATE_RE.test(eventDate)) {
out.events.push({
session_date: typeof e.session_date === 'string' ? e.session_date : eventDate,
cue: typeof e.cue === 'string' ? e.cue : 'none',
event_date: eventDate,
text: e.text.trim(),
});
}
}
},
},
{
name: 'profiles',
run: async () => {
const obj = parseJsonObject(await llmCall(profilesPrompt(text), 'fast'));
const profiles = Array.isArray(obj?.profiles) ? obj.profiles : [];
for (const p of profiles as Array<Record<string, unknown>>) {
if (typeof p?.speaker === 'string' && typeof p?.card === 'string' && p.card.trim().length > 0) {
out.profiles.push({ speaker: p.speaker.trim(), card: p.card.trim() });
}
}
},
},
];
for (const pass of passes) {
try {
await pass.run();
} catch (e: unknown) {
const msg = e instanceof Error ? e.message : String(e);
out.errors.push(`${pass.name}: ${msg}`);
log.warn(`memory-lane extraction pass failed`, { pass: pass.name, error: msg });
}
}
return out;
}
// ── Frame writing ────────────────────────────────────────────────────────────
export interface WriteLaneFramesResult {
factsWritten: number;
eventsWritten: number;
profilesWritten: number;
injectionDropped: number;
}
/**
* Persist an extraction as prefix-tagged frames.
*
* - facts: `[mind-fact]\n<text>` importance 'normal'
* - events: `[mind-event]\n[YYYY-MM-DD] <text>` importance 'normal',
* created_at = the RESOLVED event date (write-time temporal
* anchoring — the production counterpart of the LoCoMo P4 win)
* - profiles: `[mind-profile <name>]\n<card>` importance 'normal' —
* DELIBERATELY normal so cards stay out of the importance-K5
* lane (benchmark design decision); prior card for the same
* person is replaced (profiles evolve, facts accumulate).
*
* createIFrame's content dedup makes fact/event writes idempotent across
* re-runs. Every item is injection-scanned before write — extraction output
* derives from possibly-tainted harvested content.
*/
export function writeMemoryLaneFrames(
frames: FrameStore,
gopId: string,
extraction: MemoryLaneExtraction,
): WriteLaneFramesResult {
const result: WriteLaneFramesResult = {
factsWritten: 0, eventsWritten: 0, profilesWritten: 0, injectionDropped: 0,
};
const safe = (text: string): boolean => {
const scan = scanForInjection(text, 'tool_output');
if (!scan.safe) {
result.injectionDropped++;
log.warn('dropping extracted item with injection payload', { flags: scan.flags.join(',') });
return false;
}
return true;
};
for (const f of extraction.facts) {
if (!safe(f.text)) continue;
frames.createIFrame(gopId, `${MIND_FACT_PREFIX}\n${f.text}`, 'normal', 'system');
result.factsWritten++;
}
for (const e of extraction.events) {
if (!safe(e.text)) continue;
frames.createIFrame(
gopId,
`${MIND_EVENT_PREFIX}\n[${e.event_date}] ${e.text}`,
'normal',
'system',
`${e.event_date}T00:00:00.000Z`,
);
result.eventsWritten++;
}
for (const p of extraction.profiles) {
if (!safe(p.card)) continue;
const header = `${MIND_PROFILE_PREFIX} ${p.speaker}]`;
// Replace-on-update: a person's card supersedes the previous one.
frames.deleteByContentPrefix(header);
frames.createIFrame(gopId, `${header}\n${p.card}`, 'normal', 'system');
result.profilesWritten++;
}
return result;
}

View File

@@ -0,0 +1,164 @@
/**
* Gemini Adapter — parse Google Takeout / Gemini conversation exports.
*
* Supports both Google Takeout format and direct Gemini API export.
*/
import { stableHarvestId } from './stable-id.js';
import type { SourceAdapter, UniversalImportItem, ConversationMessage } from './types.js';
import { asRecord, firstString, getArray, getString, type RawRecord } from './raw-types.js';
export class GeminiAdapter implements SourceAdapter {
readonly sourceType = 'gemini' as const;
readonly displayName = 'Gemini';
parse(input: unknown): UniversalImportItem[] {
// Handle different Gemini export formats
if (Array.isArray(input)) {
return this.parseConversationArray(input);
}
const root = asRecord(input);
if (!root) return [];
// Google Takeout format: { conversations: [...] }
const conversations = getArray(root, 'conversations');
if (conversations) {
return this.parseConversationArray(conversations);
}
// Gemini API history format: { history: [...] }
if (getArray(root, 'history')) {
return this.parseSingleConversation(root);
}
return [];
}
private parseConversationArray(conversations: unknown[]): UniversalImportItem[] {
const items: UniversalImportItem[] = [];
for (const rawConv of conversations) {
const conv = asRecord(rawConv);
if (!conv) continue;
const title = firstString(conv, 'title', 'name') ?? 'Untitled';
const messages: ConversationMessage[] = [];
const entries = getArray(conv, 'messages') ?? getArray(conv, 'turns') ?? getArray(conv, 'history') ?? [];
for (const rawEntry of entries) {
const entry = asRecord(rawEntry);
if (!entry) continue;
const role = this.resolveRole(entry);
if (!role || role === 'system') continue;
const text = this.extractText(entry);
if (!text) continue;
messages.push({
role,
text,
timestamp: firstString(entry, 'createTime', 'create_time', 'timestamp'),
});
}
if (messages.length === 0) continue;
items.push({
// #7 sticky erasure: conversation id (Takeout export id), not content.
id: stableHarvestId('gemini', firstString(conv, 'id', 'conversationId') ?? `${title}\x00${firstString(conv, 'createTime', 'create_time', 'created_at') ?? ''}`),
source: 'gemini',
type: 'conversation',
title,
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
messages,
timestamp: firstString(conv, 'createTime', 'create_time', 'created_at') ?? new Date().toISOString(),
metadata: {
conversationId: firstString(conv, 'id', 'conversationId'),
messageCount: messages.length,
model: firstString(conv, 'model', 'modelVersion'),
},
});
}
return items;
}
private parseSingleConversation(conv: RawRecord): UniversalImportItem[] {
const messages: ConversationMessage[] = [];
const entries = getArray(conv, 'history') ?? [];
for (const rawEntry of entries) {
const entry = asRecord(rawEntry);
if (!entry) continue;
const role = this.resolveRole(entry);
if (!role || role === 'system') continue;
const text = this.extractText(entry);
if (!text) continue;
messages.push({ role, text });
}
if (messages.length === 0) return [];
return [{
// {history} API dump carries no id — title+model is the only stable surrogate
// (documented collision risk for two same-title+model dumps; no better anchor).
id: stableHarvestId('gemini', getString(conv, 'title') ?? 'Gemini Conversation', getString(conv, 'model') ?? ''),
source: 'gemini',
type: 'conversation',
title: getString(conv, 'title') ?? 'Gemini Conversation',
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
messages,
timestamp: new Date().toISOString(),
metadata: { model: getString(conv, 'model') },
}];
}
private resolveRole(entry: RawRecord): 'user' | 'assistant' | 'system' | null {
const role = firstString(entry, 'role', 'author', 'sender');
if (!role) return null;
const r = role.toLowerCase();
if (r === 'user' || r === 'human') return 'user';
if (r === 'model' || r === 'assistant' || r === 'gemini') return 'assistant';
if (r === 'system') return 'system';
return null;
}
private extractText(entry: RawRecord): string {
// Gemini parts format: { parts: [{ text: "..." }] }
const parts = getArray(entry, 'parts');
if (parts) {
return parts
.map(asRecord)
.map(p => {
if (!p) return undefined;
const t = getString(p, 'text');
if (t !== undefined) return t;
// W4.4 (caption parity): media parts were silently dropped.
// Surface the text-bearing fields the export carries — file
// URI/name for fileData, mime type as a presence signal for
// inline images ("did X share a photo?" questions).
const fd = asRecord(p.fileData) ?? asRecord(p.file_data);
if (fd) {
const uri = getString(fd, 'fileUri') ?? getString(fd, 'file_uri') ?? getString(fd, 'displayName');
const mime = getString(fd, 'mimeType') ?? getString(fd, 'mime_type');
return `[Shared file: ${uri ?? mime ?? 'media'}]`;
}
const il = asRecord(p.inlineData) ?? asRecord(p.inline_data);
if (il) {
const mime = getString(il, 'mimeType') ?? getString(il, 'mime_type');
return mime ? `[Shared media: ${mime}]` : undefined;
}
return undefined;
})
.filter((t): t is string => typeof t === 'string')
.join('\n')
.trim();
}
// Simple text field
const text = getString(entry, 'text');
if (text !== undefined) return text.trim();
const content = getString(entry, 'content');
if (content !== undefined) return content.trim();
return '';
}
}

View File

@@ -0,0 +1,45 @@
export * from './types.js';
export { chunkByParagraphs } from './chunk-utils.js';
export { CLASSIFY_PROMPT, EXTRACT_PROMPT, SYNTHESIZE_PROMPT } from './prompts.js';
export { HarvestSourceStore } from './source-store.js';
export { HarvestRunStore, type HarvestRun, type HarvestRunStatus } from './run-store.js';
export { dedup, harvestSetHash, type DedupResult } from './dedup.js';
export { asRecord, getString, getNumber, getArray, firstString, type RawRecord } from './raw-types.js';
export { ChatGPTAdapter } from './chatgpt-adapter.js';
export { ClaudeAdapter } from './claude-adapter.js';
export { ClaudeCodeAdapter } from './claude-code-adapter.js';
export { GeminiAdapter } from './gemini-adapter.js';
export { PerplexityAdapter } from './perplexity-adapter.js';
export { UniversalAdapter } from './universal-adapter.js';
export { MarkdownAdapter } from './markdown-adapter.js';
export { PlaintextAdapter } from './plaintext-adapter.js';
export { UrlAdapter } from './url-adapter.js';
export { PdfAdapter } from './pdf-adapter.js';
export { HarvestPipeline, type LLMCallFn, type PipelineOptions } from './pipeline.js';
// Memory-lane extraction passes (facts / events / profiles). Ported from hive-mind a99ea0e.
export {
extractMemoryLanes,
writeMemoryLaneFrames,
MIND_FACT_PREFIX,
MIND_EVENT_PREFIX,
MIND_PROFILE_PREFIX,
type ExtractedFact,
type ExtractedEvent,
type ExtractedProfile,
type MemoryLaneExtraction,
type WriteLaneFramesResult,
} from './extract-memory-lanes.js';
// Per-turn verbatim dialogue storage (raw-detail lane, write side). Ported from hive-mind a99ea0e.
export {
writeRawTurnFrames,
rawTurnHeader,
parseRawTurnHeader,
rawTurnConvKey,
MIND_RAWTURN_PREFIX,
MAX_TURNS_PER_ITEM,
RAWDETAIL_KILL_SWITCH,
type WriteRawTurnsResult,
type ParsedRawTurnHeader,
} from './raw-turns.js';

View File

@@ -0,0 +1,142 @@
/**
* Markdown Source Adapter — parses .md files into importable items.
*
* Splits markdown by top-level headings (# or ##).
* Each section becomes a separate UniversalImportItem.
* Extracts entities from heading names and bold terms.
*/
import { randomUUID } from 'node:crypto';
import * as fs from 'node:fs';
import type { SourceAdapter, UniversalImportItem } from './types.js';
interface MarkdownSection {
heading: string;
level: number;
content: string;
}
function splitByHeadings(text: string): MarkdownSection[] {
const lines = text.split('\n');
const sections: MarkdownSection[] = [];
let currentHeading = '';
let currentLevel = 0;
let currentLines: string[] = [];
for (const line of lines) {
const headingMatch = line.match(/^(#{1,3})\s+(.+)/);
if (headingMatch) {
// Flush previous section
if (currentLines.length > 0 || currentHeading) {
sections.push({
heading: currentHeading,
level: currentLevel,
content: currentLines.join('\n').trim(),
});
}
currentHeading = headingMatch[2].trim();
currentLevel = headingMatch[1].length;
currentLines = [];
} else {
currentLines.push(line);
}
}
// Flush last section
if (currentLines.length > 0 || currentHeading) {
sections.push({
heading: currentHeading,
level: currentLevel,
content: currentLines.join('\n').trim(),
});
}
return sections.filter(s => s.content.length > 0);
}
function extractBoldTerms(text: string): string[] {
const matches = text.matchAll(/\*\*([^*]+)\*\*/g);
const terms: string[] = [];
for (const m of matches) {
const term = m[1].trim();
if (term.length > 1 && term.length < 80) {
terms.push(term);
}
}
return [...new Set(terms)];
}
export class MarkdownAdapter implements SourceAdapter {
readonly sourceType = 'markdown' as const;
readonly displayName = 'Markdown';
parse(input: unknown): UniversalImportItem[] {
if (typeof input !== 'string') return [];
// Input can be a file path or raw markdown content
let content: string;
let sourcePath: string | undefined;
if (input.length < 500 && !input.includes('\n')) {
// Likely a file path
try {
if (fs.existsSync(input)) {
content = fs.readFileSync(input, 'utf-8');
sourcePath = input;
} else {
// Treat as raw content
content = input;
}
} catch {
content = input;
}
} else {
content = input;
}
if (!content.trim()) return [];
const sections = splitByHeadings(content);
// If no headings found, treat entire content as one item
if (sections.length === 0) {
return [{
id: randomUUID(),
source: 'markdown',
type: 'document',
title: sourcePath ? sourcePath.split(/[\\/]/).pop()?.replace('.md', '') ?? 'Document' : 'Document',
content: content.slice(0, 4000),
timestamp: new Date().toISOString(),
metadata: {
...(sourcePath && { filePath: sourcePath }),
contentType: 'note',
},
}];
}
const items: UniversalImportItem[] = [];
const docTitle = sourcePath?.split(/[\\/]/).pop()?.replace('.md', '');
for (const section of sections) {
const boldTerms = extractBoldTerms(section.content);
const entities = boldTerms.slice(0, 10).map(t => ({ name: t, type: 'concept' }));
items.push({
id: randomUUID(),
source: 'markdown',
type: 'document',
title: section.heading || docTitle || 'Untitled section',
content: section.content.slice(0, 4000),
timestamp: new Date().toISOString(),
metadata: {
...(sourcePath && { filePath: sourcePath }),
headingLevel: section.level,
contentType: 'note',
...(entities.length > 0 && { entities }),
},
});
}
return items;
}
}

View File

@@ -0,0 +1,97 @@
/**
* PDF Source Adapter — extracts text from PDF files.
*
* Uses pdf-parse as an optional dependency.
* If pdf-parse is not installed, provides a clear error message.
*
* Splits PDF text by pages, groups into ~3000 char chunks.
*/
import { randomUUID } from 'node:crypto';
import * as fs from 'node:fs';
import type { SourceAdapter, UniversalImportItem } from './types.js';
import { chunkByParagraphs } from './chunk-utils.js';
const MAX_CHUNK_LENGTH = 3000;
export class PdfAdapter implements SourceAdapter {
readonly sourceType = 'pdf' as const;
readonly displayName = 'PDF Document';
parse(_input: unknown): UniversalImportItem[] {
// Synchronous parse not supported for PDF — use parseFile()
return [];
}
/** Parse a PDF file from a file path. */
async parseFile(filePath: string): Promise<UniversalImportItem[]> {
// Dynamic import — pdf-parse is optional
let PDFParseClass: unknown;
try {
const mod = await import('pdf-parse');
PDFParseClass = mod.PDFParse;
} catch {
throw new Error(
'pdf-parse is not installed. Install it with: npm install pdf-parse\n'
+ 'Then retry the import.',
);
}
if (typeof PDFParseClass !== 'function') {
throw new Error('pdf-parse module found but PDFParse class not available.');
}
if (!fs.existsSync(filePath)) {
throw new Error(`PDF file not found: ${filePath}`);
}
const buffer = fs.readFileSync(filePath);
// PDFParse constructor takes { data: Buffer|Uint8Array }
const parser = new (PDFParseClass as new (opts: { data: Buffer }) => {
load(): Promise<void>;
getText(params?: object): Promise<{ text: string; pages: { text: string }[] }>;
getInfo(params?: object): Promise<{ info: Record<string, string>; numPages: number }>;
destroy(): Promise<void>;
})({ data: buffer });
await parser.load();
const textResult = await parser.getText();
let infoResult: { info: Record<string, string>; numPages: number } | undefined;
try {
infoResult = await parser.getInfo();
} catch { /* info extraction is non-fatal */ }
await parser.destroy();
const fullText = textResult.text ?? '';
if (fullText.trim().length < 10) {
return [];
}
const info = infoResult?.info ?? {};
const numPages = infoResult?.numPages ?? 0;
const docTitle = info['Title']
?? filePath.split(/[\\/]/).pop()?.replace('.pdf', '')
?? 'PDF Document';
const chunks = chunkByParagraphs(fullText, MAX_CHUNK_LENGTH);
return chunks.map((chunk, i) => ({
id: randomUUID(),
source: 'pdf' as const,
type: 'document' as const,
title: chunks.length > 1 ? `${docTitle} (part ${i + 1})` : docTitle,
content: chunk.slice(0, 4000),
timestamp: new Date().toISOString(),
metadata: {
filePath,
contentType: 'paper' as const,
pages: numPages,
...(info['Author'] && { author: info['Author'] }),
part: i + 1,
totalParts: chunks.length,
},
}));
}
}

View File

@@ -0,0 +1,163 @@
/**
* Perplexity Adapter — parse Perplexity conversation exports.
*
* Perplexity's export (via account → settings → data export) delivers
* threads as JSON. Two shapes seen in the wild:
*
* {
* "threads": [{ "id", "title", "created_at", "messages": [...] }]
* }
*
* or a bare array of threads. Also handles a per-thread "messages"
* variant where each message has: role, content, sources? (citations).
*
* Sources/citations per assistant message are flattened into the text
* body as "Sources: <url1>, <url2>" — they're the distinguishing
* feature of Perplexity answers and should survive into the harvest
* pipeline for downstream attribution.
*/
import { randomUUID } from 'node:crypto';
import type { SourceAdapter, UniversalImportItem, ConversationMessage } from './types.js';
export class PerplexityAdapter implements SourceAdapter {
readonly sourceType = 'perplexity' as const;
readonly displayName = 'Perplexity';
parse(input: unknown): UniversalImportItem[] {
if (Array.isArray(input)) {
return this.parseThreadArray(input);
}
const root = input as Record<string, unknown> | null;
if (!root || typeof root !== 'object') return [];
// Common wrapper keys observed in Perplexity exports
if (Array.isArray(root.threads)) {
return this.parseThreadArray(root.threads as unknown[]);
}
if (Array.isArray(root.conversations)) {
return this.parseThreadArray(root.conversations as unknown[]);
}
if (Array.isArray(root.items)) {
return this.parseThreadArray(root.items as unknown[]);
}
// Single-thread shape: the root itself has messages
if (Array.isArray(root.messages)) {
return this.parseSingleThread(root);
}
return [];
}
private parseThreadArray(threads: unknown[]): UniversalImportItem[] {
const items: UniversalImportItem[] = [];
for (const raw of threads) {
const thread = raw as Record<string, unknown> | null;
if (!thread || typeof thread !== 'object') continue;
const built = this.buildItem(thread);
if (built) items.push(built);
}
return items;
}
private parseSingleThread(thread: Record<string, unknown>): UniversalImportItem[] {
const built = this.buildItem(thread);
return built ? [built] : [];
}
private buildItem(thread: Record<string, unknown>): UniversalImportItem | null {
const rawMessages = (thread.messages ?? thread.turns ?? []) as unknown[];
if (!Array.isArray(rawMessages) || rawMessages.length === 0) return null;
const messages: ConversationMessage[] = [];
for (const rawMsg of rawMessages) {
const msg = rawMsg as Record<string, unknown> | null;
if (!msg) continue;
const role = this.resolveRole(msg);
if (!role || role === 'system') continue;
const text = this.extractText(msg);
if (!text) continue;
// Flatten citations/sources into the text so they survive the pipeline.
const sources = this.extractSources(msg);
const textWithSources = sources.length > 0
? `${text}\n\nSources: ${sources.join(', ')}`
: text;
messages.push({
role,
text: textWithSources,
timestamp: this.extractTimestamp(msg),
});
}
if (messages.length === 0) return null;
const title = (thread.title as string) ?? (thread.name as string) ?? 'Perplexity Thread';
return {
id: randomUUID(),
source: 'perplexity',
type: 'conversation',
title,
content: messages.map(m => `${m.role}: ${m.text}`).join('\n\n'),
messages,
timestamp: this.extractTimestamp(thread) ?? new Date().toISOString(),
metadata: {
threadId: thread.id ?? thread.threadId,
messageCount: messages.length,
hasCitations: messages.some(m => m.text.includes('Sources:')),
},
};
}
private resolveRole(entry: Record<string, unknown>): 'user' | 'assistant' | 'system' | null {
const raw = entry.role ?? entry.author ?? entry.sender ?? entry.type;
if (!raw) return null;
const r = String(raw).toLowerCase();
if (r === 'user' || r === 'human' || r === 'question') return 'user';
if (r === 'assistant' || r === 'ai' || r === 'perplexity' || r === 'answer') return 'assistant';
if (r === 'system') return 'system';
return null;
}
private extractText(entry: Record<string, unknown>): string {
// Try common text field names
if (typeof entry.content === 'string') return entry.content.trim();
if (typeof entry.text === 'string') return entry.text.trim();
if (typeof entry.answer === 'string') return entry.answer.trim();
if (typeof entry.query === 'string') return entry.query.trim();
// ChatGPT-like structured content: { parts: [...] }
const content = entry.content as Record<string, unknown> | undefined;
if (content && Array.isArray(content.parts)) {
return content.parts.filter(p => typeof p === 'string').join('\n').trim();
}
return '';
}
private extractSources(entry: Record<string, unknown>): string[] {
const raw = entry.sources ?? entry.citations ?? entry.web_results;
if (!Array.isArray(raw)) return [];
const urls: string[] = [];
for (const src of raw) {
if (typeof src === 'string') {
urls.push(src);
} else if (src && typeof src === 'object') {
const s = src as Record<string, unknown>;
const url = s.url ?? s.link ?? s.href;
if (typeof url === 'string') urls.push(url);
}
}
return urls;
}
private extractTimestamp(entry: Record<string, unknown>): string | undefined {
const raw = entry.timestamp ?? entry.created_at ?? entry.createdAt ?? entry.time;
if (typeof raw === 'string') return raw;
if (typeof raw === 'number') return new Date(raw).toISOString();
return undefined;
}
}

View File

@@ -0,0 +1,339 @@
/**
* HarvestPipeline — orchestrates the 4-pass distillation pipeline.
*
* Pass 1: Classify (Haiku — cheap)
* Pass 2: Extract (Sonnet — accurate)
* Pass 3: Synthesize (Sonnet — accurate)
* Pass 4: Dedup (local — no LLM)
*
* The pipeline accepts UniversalImportItems and produces DistilledKnowledge[].
* LLM calls are batched (20 items per call) to optimize cost.
*/
import type {
UniversalImportItem, ClassifiedItem, ExtractedContent,
DistilledKnowledge, HarvestPipelineResult, ImportSourceType,
} from './types.js';
import { CLASSIFY_PROMPT, EXTRACT_PROMPT, SYNTHESIZE_PROMPT } from './prompts.js';
import { dedup } from './dedup.js';
import { scanForInjection } from '../injection-scanner.js';
import { createCoreLogger } from '../logger.js';
const log = createCoreLogger('harvest-pipeline');
const BATCH_SIZE = 20;
const CONCURRENCY_CAP = 3;
export interface LLMCallFn {
(prompt: string, model: 'fast' | 'accurate'): Promise<string>;
}
export interface PipelineOptions {
llmCall: LLMCallFn;
existingContents?: string[];
onProgress?: (stage: string, current: number, total: number) => void;
/** Items per LLM batch call (default: 20). */
batchSize?: number;
/** Max concurrent LLM batch calls (default: 3). */
concurrency?: number;
/**
* Fallback behavior when the Pass 1 (classify) LLM call throws.
* - `'skip'` (default, safer): drop the batch. Under-inclusion beats cost/noise inflation.
* - `'pass-through-medium'` (legacy): promote every item to `value: 'medium'`. This is what
* the pipeline did historically but it runs extract + synthesize on junk and pollutes
* memory with trivial greetings/debugging loops when the classify model hiccups.
*/
classifyFailureFallback?: 'skip' | 'pass-through-medium';
}
/** Safely parse JSON from LLM response, handling markdown code fences. */
function parseLLMJson<T>(raw: string): T[] {
const cleaned = raw.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
try {
const parsed = JSON.parse(cleaned);
return Array.isArray(parsed) ? parsed : [parsed];
} catch {
return [];
}
}
/** Split array into batches. */
function batch<T>(items: T[], size: number): T[][] {
const batches: T[][] = [];
for (let i = 0; i < items.length; i += size) {
batches.push(items.slice(i, i + size));
}
return batches;
}
/** Run async tasks with a concurrency cap (tumbling window — waits for full batch before next). */
async function runWithConcurrency<T>(
tasks: (() => Promise<T>)[],
cap: number,
): Promise<T[]> {
const results: T[] = [];
for (let i = 0; i < tasks.length; i += cap) {
const window = tasks.slice(i, i + cap);
const windowResults = await Promise.all(window.map(fn => fn()));
results.push(...windowResults);
}
return results;
}
export class HarvestPipeline {
private llmCall: LLMCallFn;
private existingContents: string[];
private onProgress?: (stage: string, current: number, total: number) => void;
private classifyFailureFallback: 'skip' | 'pass-through-medium';
private batchSize: number;
private concurrency: number;
constructor(options: PipelineOptions) {
this.llmCall = options.llmCall;
this.existingContents = options.existingContents ?? [];
this.onProgress = options.onProgress;
this.classifyFailureFallback = options.classifyFailureFallback ?? 'skip';
this.batchSize = options.batchSize ?? BATCH_SIZE;
this.concurrency = options.concurrency ?? CONCURRENCY_CAP;
}
async run(items: UniversalImportItem[], source: ImportSourceType): Promise<HarvestPipelineResult> {
const startTime = Date.now();
const errors: string[] = [];
log.info('harvest pipeline starting', { source, itemCount: items.length, batchSize: this.batchSize, concurrency: this.concurrency });
// Pass 0: Injection scan — drop any item whose title or content carries a
// prompt-injection payload (role_override / prompt_extraction / instruction_injection).
// Harvest ingests UNTRUSTED external exports (ChatGPT/Claude/Gemini JSON dumps,
// Perplexity shares, URL fetches). A hostile file must not flow through to the
// LLM passes or into memory frames.
const originalCount = items.length;
items = items.filter((item) => {
// Scan title + first 4KB of content — enough to catch payloads hidden in either field.
// Using 'tool_output' context since imports are external data, weighted like tool output.
const probe = `${item.title ?? ''}\n${(item.content ?? '').slice(0, 4000)}`;
const scan = scanForInjection(probe, 'tool_output');
if (!scan.safe) {
const reason = scan.flags.join(',');
log.warn('dropping harvest item with injection payload', {
itemId: item.id,
title: item.title?.slice(0, 80),
flags: scan.flags,
score: scan.score,
});
errors.push(`Blocked item "${item.title?.slice(0, 40) ?? item.id}" — injection detected (${reason})`);
return false;
}
return true;
});
const blockedCount = originalCount - items.length;
if (blockedCount > 0) {
log.info(`harvest security: blocked ${blockedCount} of ${originalCount} items for injection patterns`);
}
// Pass 1: Classify
this.onProgress?.('classify', 0, items.length);
const classified = await this.classify(items, errors);
const valuable = classified.filter(c => c.value !== 'skip');
// Pass 2: Extract
this.onProgress?.('extract', 0, valuable.length);
const extracted = await this.extract(valuable, errors);
// Pass 3: Synthesize
this.onProgress?.('synthesize', 0, extracted.length);
const distilled = await this.synthesize(extracted, source, errors);
// Pass 4: Dedup
this.onProgress?.('dedup', 0, distilled.length);
const dedupResult = dedup(distilled, this.existingContents);
const durationMs = Date.now() - startTime;
log.info('harvest pipeline complete', {
source, itemsReceived: originalCount, classified: classified.length,
extracted: extracted.length, distilled: distilled.length,
unique: dedupResult.unique.length, dupsSkipped: dedupResult.duplicatesSkipped,
errors: errors.length, durationMs,
});
return {
source, itemsReceived: originalCount,
itemsClassified: classified.length,
itemsSkipped: classified.length - valuable.length,
itemsExtracted: extracted.length,
knowledgeDistilled: dedupResult.unique,
framesSaved: 0, // Caller handles persistence
entitiesCreated: 0,
relationsCreated: 0,
identityUpdates: dedupResult.unique.filter(k => k.targetLayer === 'identity').length,
duplicatesSkipped: dedupResult.duplicatesSkipped,
errors,
costUsd: 0, // Caller tracks cost
durationMs,
};
}
private async classify(items: UniversalImportItem[], errors: string[]): Promise<ClassifiedItem[]> {
const results: ClassifiedItem[] = [];
const batches = batch(items, this.batchSize);
// M3: run batches with concurrency cap instead of sequentially
const tasks = batches.map((b, i) => async () => {
this.onProgress?.('classify', i * this.batchSize, items.length);
const prompt = CLASSIFY_PROMPT + b.map((item, idx) => (
`\n--- Item ${idx} (id: ${item.id}) ---\nTitle: ${item.title}\nSource: ${item.source}\nType: ${item.type}\nContent (first 500 chars): ${item.content.slice(0, 500)}\n`
)).join('');
try {
const response = await this.llmCall(prompt, 'fast');
const parsed = parseLLMJson<{ itemId?: string; domain?: string; value?: string; categories?: string[] }>(response);
const batchResults: ClassifiedItem[] = [];
for (const entry of parsed) {
if (!entry.itemId) {
log.warn('classify entry missing itemId — skipping', { batchIndex: i });
continue;
}
const item = b.find(it => it.id === entry.itemId);
if (!item) {
log.warn('classify entry itemId does not match any batch item — skipping', { itemId: entry.itemId, batchIndex: i });
continue;
}
batchResults.push({
item,
domain: (entry.domain as ClassifiedItem['domain']) ?? 'mixed',
value: (entry.value as ClassifiedItem['value']) ?? 'medium',
categories: entry.categories ?? [],
});
}
return batchResults;
} catch (err) {
errors.push(`Classify batch ${i} failed: ${err instanceof Error ? err.message : 'unknown'}`);
if (this.classifyFailureFallback === 'pass-through-medium') {
return b.map(item => ({ item, domain: 'mixed' as const, value: 'medium' as const, categories: [] as string[] }));
}
return [] as ClassifiedItem[];
}
});
const batchResults = await runWithConcurrency(tasks, this.concurrency);
for (const br of batchResults) results.push(...br);
return results;
}
private async extract(classified: ClassifiedItem[], errors: string[]): Promise<ExtractedContent[]> {
const results: ExtractedContent[] = [];
const batches = batch(classified, this.batchSize);
// M3: concurrent batches with cap
const tasks = batches.map((b, i) => async () => {
this.onProgress?.('extract', i * this.batchSize, classified.length);
const prompt = EXTRACT_PROMPT + b.map((c, idx) => (
`\n--- Conversation ${idx} (id: ${c.item.id}, value: ${c.value}, categories: ${c.categories.join(',')}) ---\nTitle: ${c.item.title}\n${c.item.content.slice(0, 2000)}\n`
)).join('');
try {
const response = await this.llmCall(prompt, 'accurate');
const parsed = parseLLMJson<{
itemId?: string;
decisions?: unknown[]; preferences?: unknown[]; facts?: unknown[];
knowledge?: unknown[]; entities?: unknown[]; relations?: unknown[];
}>(response);
const batchResults: ExtractedContent[] = [];
for (const entry of parsed) {
if (!entry.itemId) {
log.warn('extract entry missing itemId — skipping', { batchIndex: i });
continue;
}
if (!b.some(c => c.item.id === entry.itemId)) {
log.warn('extract entry itemId does not match batch — skipping', { itemId: entry.itemId, batchIndex: i });
continue;
}
batchResults.push({
itemId: entry.itemId,
decisions: (entry.decisions ?? []) as ExtractedContent['decisions'],
preferences: (entry.preferences ?? []) as ExtractedContent['preferences'],
facts: (entry.facts ?? []) as ExtractedContent['facts'],
knowledge: (entry.knowledge ?? []) as ExtractedContent['knowledge'],
entities: (entry.entities ?? []) as ExtractedContent['entities'],
relations: (entry.relations ?? []) as ExtractedContent['relations'],
});
}
return batchResults;
} catch (err) {
errors.push(`Extract batch ${i} failed: ${err instanceof Error ? err.message : 'unknown'}`);
return [] as ExtractedContent[];
}
});
const batchResults = await runWithConcurrency(tasks, this.concurrency);
for (const br of batchResults) results.push(...br);
return results;
}
private async synthesize(
extracted: ExtractedContent[],
source: ImportSourceType,
errors: string[],
): Promise<DistilledKnowledge[]> {
const results: DistilledKnowledge[] = [];
const batches = batch(extracted, this.batchSize);
// Review C2: per-item serialization with individual budget. The old flat
// `JSON.stringify(b, null, 2).slice(0, 8000)` truncated the *middle* of the last
// item's JSON on a long batch; parseLLMJson returned [] on the malformed tail and
// every subsequent item silently vanished with no error. Now each item gets its
// own PER_ITEM_BUDGET and survives regardless of batch size.
const PER_ITEM_BUDGET = 1200;
// M3: concurrent batches with cap
const tasks = batches.map((b, i) => async () => {
this.onProgress?.('synthesize', i * this.batchSize, extracted.length);
const serialized = b.map((ec, idx) => {
const json = JSON.stringify(ec, null, 2);
const trimmed = json.length > PER_ITEM_BUDGET
? json.slice(0, PER_ITEM_BUDGET) + '\n ... (truncated — full item in trace)'
: json;
return `\n--- Item ${idx} (id: ${ec.itemId}) ---\n${trimmed}`;
}).join('');
const prompt = SYNTHESIZE_PROMPT + serialized;
try {
const response = await this.llmCall(prompt, 'accurate');
const parsed = parseLLMJson<{
targetLayer?: string; frameType?: string; importance?: string;
content?: string; confidence?: number;
}>(response);
const batchResults: DistilledKnowledge[] = [];
for (const entry of parsed) {
batchResults.push({
targetLayer: (entry.targetLayer as DistilledKnowledge['targetLayer']) ?? 'frame',
frameType: (entry.frameType as DistilledKnowledge['frameType']) ?? 'I',
importance: (entry.importance as DistilledKnowledge['importance']) ?? 'normal',
content: entry.content ?? '',
provenance: {
originalSource: source,
importedAt: new Date().toISOString(),
distillationModel: 'accurate',
confidence: entry.confidence ?? 0.7,
pass: 3,
},
});
}
return batchResults;
} catch (err) {
errors.push(`Synthesize batch ${i} failed: ${err instanceof Error ? err.message : 'unknown'}`);
return [] as DistilledKnowledge[];
}
});
const batchResults = await runWithConcurrency(tasks, this.concurrency);
for (const br of batchResults) results.push(...br);
return results;
}
}

View File

@@ -0,0 +1,61 @@
/**
* Plaintext Source Adapter — parses .txt files into importable items.
*
* Splits text by double-newline paragraphs.
* Groups paragraphs into chunks of ~2000 chars max.
*/
import { randomUUID } from 'node:crypto';
import * as fs from 'node:fs';
import type { SourceAdapter, UniversalImportItem } from './types.js';
import { chunkByParagraphs } from './chunk-utils.js';
export class PlaintextAdapter implements SourceAdapter {
readonly sourceType = 'plaintext' as const;
readonly displayName = 'Plain Text';
parse(input: unknown): UniversalImportItem[] {
if (typeof input !== 'string') return [];
let content: string;
let sourcePath: string | undefined;
// Check if input is a file path
if (input.length < 500 && !input.includes('\n')) {
try {
if (fs.existsSync(input)) {
content = fs.readFileSync(input, 'utf-8');
sourcePath = input;
} else {
content = input;
}
} catch {
content = input;
}
} else {
content = input;
}
if (!content.trim()) return [];
const chunks = chunkByParagraphs(content);
const docTitle = sourcePath?.split(/[\\/]/).pop()?.replace(/\.\w+$/, '');
return chunks.map((chunk, i) => ({
id: randomUUID(),
source: 'plaintext' as const,
type: 'document' as const,
title: docTitle
? (chunks.length > 1 ? `${docTitle} (part ${i + 1})` : docTitle)
: `Text fragment ${i + 1}`,
content: chunk.slice(0, 4000),
timestamp: new Date().toISOString(),
metadata: {
...(sourcePath && { filePath: sourcePath }),
contentType: 'note',
part: i + 1,
totalParts: chunks.length,
},
}));
}
}

View File

@@ -0,0 +1,72 @@
/**
* Harvest Pipeline Prompts — LLM prompt templates for each distillation pass.
*/
export const CLASSIFY_PROMPT = `You are a knowledge classifier. For each conversation/item below, classify it.
Return a JSON array with one entry per item:
[{
"itemId": "...",
"domain": "work" | "personal" | "technical" | "mixed",
"value": "high" | "medium" | "low" | "skip",
"categories": ["decision", "preference", "fact", "knowledge", "project", "identity", "trivial"]
}]
Rules:
- "skip" = greetings, trivial exchanges, "hello", "thanks", debugging loops with no insight
- "high" = decisions, preferences, personal facts, project context, technical architecture
- "medium" = general knowledge, research, learning
- "low" = routine questions with generic answers
Items:
`;
export const EXTRACT_PROMPT = `You are a knowledge extractor. For each classified conversation, extract structured knowledge.
Return a JSON array:
[{
"itemId": "...",
"decisions": ["chose X over Y because Z"],
"preferences": ["prefers dark mode", "likes concise responses"],
"facts": ["works at Egzakta Group", "role is CEO"],
"knowledge": ["React 18 concurrent features improve perceived performance"],
"entities": [{"name": "Egzakta Group", "type": "organization"}],
"relations": [{"source": "Marko", "target": "Egzakta Group", "relation": "works_at"}]
}]
Rules:
- Only extract what the USER stated or decided, not what the AI suggested
- Decisions must include the reason if one was given
- Preferences must be actionable (not "I like good code" — too vague)
- Facts must be verifiable or specific (names, roles, companies, tech stack)
- Entities: types are person, organization, project, technology, concept, location, event
- Relations: use lowercase_snake_case for relation types
Conversations:
`;
export const SYNTHESIZE_PROMPT = `You are a memory synthesizer. Convert extracted knowledge into structured memory frames.
For each extraction, produce frames suitable for a persistent memory system:
Return a JSON array:
[{
"targetLayer": "identity" | "frame" | "kg_entity" | "kg_relation",
"frameType": "I",
"importance": "critical" | "important" | "normal",
"content": "The actual memory content, written as a clear statement",
"confidence": 0.0-1.0
}]
Rules:
- "identity" = personal facts (name, role, company, capabilities, personality traits)
- "frame" with importance "important" = decisions and preferences
- "frame" with importance "normal" = general knowledge and facts
- "kg_entity" = entities to add to the knowledge graph
- "kg_relation" = relationships between entities
- Content should be self-contained — readable without the original conversation
- Deduplicate: if two items say the same thing, pick the most complete version
- confidence: 1.0 = user explicitly stated, 0.7 = strongly implied, 0.5 = inferred
Extractions:
`;

View File

@@ -0,0 +1,169 @@
/**
* raw-turns.ts — W4.6 per-turn verbatim dialogue storage (write side).
*
* The W3.4 ablation attributed the benchmark's single-hop win to the
* RAWDETAIL escalation lane (+2.40 z=1.95; captions alone +0.26 ns) —
* fine-grained perceptual detail survives ONLY in verbatim turns; the
* distillation passes carry it generically. This module stores each
* conversation turn as its own frame so the recall-side lane
* (`mind/raw-detail-lane.ts`) can pool / CE-rerank / neighbor-expand them.
*
* Frame convention (first line is the lane tag; body is the verbatim turn):
*
* `[mind-rawturn conv:<key> turn:<n> speaker:<s>]\n<turn text>`
*
* - `conv:<key>` sanitized item id — groups turns of one conversation
* - `turn:<n>` contiguous index over STORED turns (dialogue order) —
* the ±1 adjacency key. Production frames interleave
* across sources, so the benchmark's id-ordering trick
* does not transfer; the index makes adjacency explicit.
* - `speaker:<s>` sanitized role/name (no whitespace, no `]`)
*
* `[mind-` prefixing keeps raw turns out of the memory-lane extraction
* cron's source material (its `NOT LIKE '[mind-%'` self-feeding guard) and
* lets recall dedup them against the snippet lanes by content prefix.
*
* Every turn is injection-scanned BEFORE write: verbatim dialogue is the
* most injection-prone frame class, and recallMemory blocks the ENTIRE
* recall block on a scan hit — poisoned turns must die here, not there.
*
* Storage growth is the accepted tradeoff (Marko GO 2026-06-11, plan §6.2).
*/
import type { FrameStore } from '../mind/frames.js';
import type { UniversalImportItem } from './types.js';
import { HARVEST_FRAME_CONTENT_CAP } from './types.js';
import { scanForInjection } from '../injection-scanner.js';
import { createCoreLogger } from '../logger.js';
const log = createCoreLogger('raw-turns');
/** First-line content prefix for raw-turn frames (recall lane fetches by this). */
export const MIND_RAWTURN_PREFIX = '[mind-rawturn';
/** Hard per-conversation cap — backstop against pathological exports.
* LoCoMo conversations run ~600 turns; 2000 leaves generous headroom. */
export const MAX_TURNS_PER_ITEM = 2000;
/** Env kill switch (checked by CALLERS, mirrored here for the recall lane). */
export const RAWDETAIL_KILL_SWITCH = 'WAGGLE_RAWDETAIL';
export interface WriteRawTurnsResult {
written: number;
skippedEmpty: number;
injectionDropped: number;
/** true when MAX_TURNS_PER_ITEM truncated the conversation (logged, never silent). */
capped: boolean;
}
/** Strict ISO-8601 gate (same contract as FrameStore.createIFrame). */
function isIsoTimestamp(value: string | undefined): value is string {
return typeof value === 'string'
&& /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})$/.test(value)
&& Number.isFinite(Date.parse(value));
}
/** Sanitize a header token: keep [A-Za-z0-9_-], collapse everything else to '-'.
* Removes `]`, whitespace, and LIKE metacharacters (% _ kept — they're safe
* in equality-style prefix lookups because the recall lane LIKE-escapes). */
function sanitizeToken(value: string, maxLen: number): string {
const cleaned = value.replace(/[^A-Za-z0-9_-]+/g, '-').replace(/^-+|-+$/g, '');
return (cleaned || 'unknown').slice(0, maxLen);
}
/** Build the raw-turn header for conv/turn/speaker. Exported for the recall
* lane's neighbor lookups (single source of truth for the format). */
export function rawTurnHeader(convKey: string, turn: number, speaker: string): string {
return `${MIND_RAWTURN_PREFIX} conv:${convKey} turn:${turn} speaker:${speaker}]`;
}
export interface ParsedRawTurnHeader {
conv: string;
turn: number;
speaker: string;
}
/** Parse a raw-turn frame's first line. Returns null for non-rawturn content. */
export function parseRawTurnHeader(content: string): ParsedRawTurnHeader | null {
const m = content.match(/^\[mind-rawturn conv:([A-Za-z0-9_-]+) turn:(\d+) speaker:([A-Za-z0-9_-]+)\]/);
if (!m) return null;
return { conv: m[1], turn: parseInt(m[2], 10), speaker: m[3] };
}
/** Conversation key for an import item (sanitized, stable across re-imports).
* Params are widened to plain strings so the GDPR erasure path (which knows a
* subject only as free-string source + source_ref) can reconstruct the exact
* same key; UniversalImportItem's narrower fields still satisfy it. */
export function rawTurnConvKey(item: { source: string; id: string }): string {
return sanitizeToken(`${item.source}-${item.id}`, 64);
}
/**
* Store each user/assistant turn of `item.messages` as a `[mind-rawturn …]`
* frame. No-op (all zeros) when the item carries no messages.
*
* - turn index is contiguous over STORED turns (skips don't leave gaps —
* ±1 adjacency stays meaningful over the stored dialogue)
* - system messages are skipped (boilerplate, not dialogue evidence)
* - created_at: message timestamp if valid ISO, else item timestamp, else
* schema default (createIFrame validates again — never writes junk)
* - importance 'normal' (stays out of the K5 importance lane), source 'import'
* - createIFrame content-dedup makes re-imports idempotent within its
* 500-frame recency window; source-level content hashing in the harvest
* routes guards the wider case (unchanged exports never reach here)
*/
export function writeRawTurnFrames(
frames: FrameStore,
gopId: string,
item: UniversalImportItem,
): WriteRawTurnsResult {
const result: WriteRawTurnsResult = {
written: 0, skippedEmpty: 0, injectionDropped: 0, capped: false,
};
const messages = item.messages;
if (!Array.isArray(messages) || messages.length === 0) return result;
const convKey = rawTurnConvKey(item);
const itemTs = isIsoTimestamp(item.timestamp) ? item.timestamp : undefined;
let turn = 0;
for (const msg of messages) {
if (msg.role !== 'user' && msg.role !== 'assistant') continue;
const text = (msg.text ?? '').trim();
if (text.length === 0) {
result.skippedEmpty++;
continue;
}
if (turn >= MAX_TURNS_PER_ITEM) {
result.capped = true;
break;
}
// Scan first 4KB — same probe budget as the harvest pipeline's Pass 0.
const scan = scanForInjection(text.slice(0, 4000), 'tool_output');
if (!scan.safe) {
result.injectionDropped++;
log.warn('dropping raw turn with injection payload', {
conv: convKey, turn, flags: scan.flags.join(','),
});
continue;
}
const speaker = sanitizeToken(msg.role, 24);
const createdAt = isIsoTimestamp(msg.timestamp) ? msg.timestamp : itemTs;
frames.createIFrame(
gopId,
`${rawTurnHeader(convKey, turn, speaker)}\n${text.slice(0, HARVEST_FRAME_CONTENT_CAP)}`,
'normal',
'import',
createdAt,
);
result.written++;
turn++;
}
if (result.capped) {
log.warn('raw-turn storage capped — conversation exceeds MAX_TURNS_PER_ITEM', {
conv: convKey, stored: result.written, totalMessages: messages.length,
});
}
return result;
}

View File

@@ -0,0 +1,45 @@
/**
* Raw external export shapes — loosely-typed structures as they arrive from
* third-party JSON exports (ChatGPT / Claude / Gemini / generic).
*
* These adapters parse UNTRUSTED external data, so every field is optional and
* widened. Property access goes through the narrowing helpers below rather than
* casting to `any`, so a malformed export degrades to "skip" instead of throwing.
*/
/** A JSON object whose keys are unknown until narrowed. */
export type RawRecord = Record<string, unknown>;
/** Narrow an unknown value to a plain object, or null. */
export function asRecord(value: unknown): RawRecord | null {
return value && typeof value === 'object' && !Array.isArray(value)
? (value as RawRecord)
: null;
}
/** Read a string property, or undefined if absent / wrong type. */
export function getString(obj: RawRecord, key: string): string | undefined {
const v = obj[key];
return typeof v === 'string' ? v : undefined;
}
/** Read a number property, or undefined if absent / wrong type. */
export function getNumber(obj: RawRecord, key: string): number | undefined {
const v = obj[key];
return typeof v === 'number' ? v : undefined;
}
/** Read an array property as unknown[], or undefined if absent / wrong type. */
export function getArray(obj: RawRecord, key: string): unknown[] | undefined {
const v = obj[key];
return Array.isArray(v) ? v : undefined;
}
/** First defined string among the given keys (export shapes vary). */
export function firstString(obj: RawRecord, ...keys: string[]): string | undefined {
for (const key of keys) {
const v = getString(obj, key);
if (v !== undefined) return v;
}
return undefined;
}

View File

@@ -0,0 +1,191 @@
/**
* HarvestRunStore — tracks the lifecycle of individual harvest commit runs
* so the UI can surface interrupted runs and offer to resume them.
*
* A run is a single POST /api/harvest/commit invocation. States:
* running — route is executing
* completed — finished successfully
* failed — explicit error; `error_message` is populated
* abandoned — user chose to discard; cache is deleted
*
* A "interrupted" run from the UI's perspective is any `running` or `failed`
* row with a surviving `input_cache_path`. The route never transitions
* `running` -> `interrupted` — it simply never finalizes when the client
* disconnects, so the row stays `running` forever. getLatestInterrupted()
* surfaces the latest such row.
*
* Resume = replay the same input payload; FrameStore.createIFrame dedups
* on content so already-saved frames become no-ops. No fine-grained
* checkpoint-offset math needed.
*/
import type { MindDB } from '../mind/db.js';
import type { ImportSourceType } from './types.js';
const HARVEST_RUNS_DDL = `
CREATE TABLE IF NOT EXISTS harvest_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'running',
total_items INTEGER NOT NULL DEFAULT 0,
items_saved INTEGER NOT NULL DEFAULT 0,
started_at TEXT NOT NULL DEFAULT (datetime('now')),
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
finished_at TEXT,
error_message TEXT,
input_cache_path TEXT
);
`;
export type HarvestRunStatus = 'running' | 'completed' | 'failed' | 'abandoned';
export interface HarvestRun {
id: number;
source: ImportSourceType;
status: HarvestRunStatus;
totalItems: number;
itemsSaved: number;
startedAt: string;
updatedAt: string;
finishedAt: string | null;
errorMessage: string | null;
inputCachePath: string | null;
}
export class HarvestRunStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const existsRow = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='harvest_runs'",
).get();
if (!existsRow) {
raw.exec(HARVEST_RUNS_DDL);
}
}
/**
* Create a new `running` run record. Caller holds the id for subsequent
* heartbeat/complete/fail calls.
*/
start(source: ImportSourceType, totalItems: number, inputCachePath: string | null = null): HarvestRun {
const raw = this.db.getDatabase();
const result = raw.prepare(`
INSERT INTO harvest_runs (source, status, total_items, input_cache_path)
VALUES (?, 'running', ?, ?)
`).run(source, totalItems, inputCachePath);
return this.getById(Number(result.lastInsertRowid))!;
}
/** Update items_saved + updated_at on a running row. No-op on terminal rows. */
heartbeat(id: number, itemsSaved: number): void {
const raw = this.db.getDatabase();
raw.prepare(`
UPDATE harvest_runs SET
items_saved = ?,
updated_at = datetime('now')
WHERE id = ? AND status = 'running'
`).run(itemsSaved, id);
}
/** Mark a run completed. Idempotent — no-op on terminal rows. */
complete(id: number, itemsSaved: number): void {
const raw = this.db.getDatabase();
raw.prepare(`
UPDATE harvest_runs SET
status = 'completed',
items_saved = ?,
updated_at = datetime('now'),
finished_at = datetime('now')
WHERE id = ? AND status = 'running'
`).run(itemsSaved, id);
}
/** Mark a run failed with an error message. Idempotent — no-op on terminal rows. */
fail(id: number, errorMessage: string, itemsSaved?: number): void {
const raw = this.db.getDatabase();
if (typeof itemsSaved === 'number') {
raw.prepare(`
UPDATE harvest_runs SET
status = 'failed',
items_saved = ?,
error_message = ?,
updated_at = datetime('now'),
finished_at = datetime('now')
WHERE id = ? AND status = 'running'
`).run(itemsSaved, errorMessage.slice(0, 2000), id);
} else {
raw.prepare(`
UPDATE harvest_runs SET
status = 'failed',
error_message = ?,
updated_at = datetime('now'),
finished_at = datetime('now')
WHERE id = ? AND status = 'running'
`).run(errorMessage.slice(0, 2000), id);
}
}
/** Mark a run abandoned. Used when the user discards an interrupted run. */
abandon(id: number): void {
const raw = this.db.getDatabase();
raw.prepare(`
UPDATE harvest_runs SET
status = 'abandoned',
updated_at = datetime('now'),
finished_at = datetime('now')
WHERE id = ? AND status IN ('running', 'failed')
`).run(id);
}
getById(id: number): HarvestRun | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM harvest_runs WHERE id = ?').get(id) as Record<string, unknown> | undefined;
return row ? this.rowToRun(row) : null;
}
/**
* Latest `running` or `failed` run with a surviving cache path — the UI
* uses this to offer resume on next page load.
*/
getLatestInterrupted(): HarvestRun | null {
const raw = this.db.getDatabase();
const row = raw.prepare(`
SELECT * FROM harvest_runs
WHERE status IN ('running', 'failed')
AND input_cache_path IS NOT NULL
ORDER BY started_at DESC
LIMIT 1
`).get() as Record<string, unknown> | undefined;
return row ? this.rowToRun(row) : null;
}
/** All runs, newest first — useful for debugging and future history views. */
getAll(limit = 50): HarvestRun[] {
const raw = this.db.getDatabase();
return (raw.prepare(`
SELECT * FROM harvest_runs ORDER BY started_at DESC LIMIT ?
`).all(limit) as Record<string, unknown>[]).map(r => this.rowToRun(r));
}
private rowToRun(row: Record<string, unknown>): HarvestRun {
return {
id: row.id as number,
source: row.source as ImportSourceType,
status: row.status as HarvestRunStatus,
totalItems: row.total_items as number,
itemsSaved: row.items_saved as number,
startedAt: row.started_at as string,
updatedAt: row.updated_at as string,
finishedAt: (row.finished_at as string | null) ?? null,
errorMessage: (row.error_message as string | null) ?? null,
inputCachePath: (row.input_cache_path as string | null) ?? null,
};
}
}

View File

@@ -0,0 +1,140 @@
/**
* HarvestSourceStore — tracks connected harvest sources and sync state.
*/
import type { MindDB } from '../mind/db.js';
import type { HarvestSource, ImportSourceType } from './types.js';
const HARVEST_SOURCES_DDL = `
CREATE TABLE IF NOT EXISTS harvest_sources (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL UNIQUE,
display_name TEXT NOT NULL,
source_path TEXT,
last_synced_at TEXT,
items_imported INTEGER NOT NULL DEFAULT 0,
frames_created INTEGER NOT NULL DEFAULT 0,
auto_sync INTEGER NOT NULL DEFAULT 0,
sync_interval_hours INTEGER NOT NULL DEFAULT 24,
last_content_hash TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
);
`;
export class HarvestSourceStore {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureTable();
}
private ensureTable(): void {
const raw = this.db.getDatabase();
const exists = raw.prepare(
"SELECT name FROM sqlite_master WHERE type='table' AND name='harvest_sources'",
).get();
if (!exists) {
raw.exec(HARVEST_SOURCES_DDL);
}
}
/** Register or update a harvest source. */
upsert(source: ImportSourceType, displayName: string, sourcePath?: string): HarvestSource {
const raw = this.db.getDatabase();
raw.prepare(`
INSERT INTO harvest_sources (source, display_name, source_path)
VALUES (?, ?, ?)
ON CONFLICT(source) DO UPDATE SET
display_name = excluded.display_name,
source_path = COALESCE(excluded.source_path, harvest_sources.source_path)
`).run(source, displayName, sourcePath ?? null);
return this.getBySource(source)!;
}
/** Record a completed sync. */
recordSync(source: ImportSourceType, itemsImported: number, framesCreated: number, contentHash?: string): void {
const raw = this.db.getDatabase();
raw.prepare(`
UPDATE harvest_sources SET
last_synced_at = datetime('now'),
items_imported = items_imported + ?,
frames_created = frames_created + ?,
last_content_hash = COALESCE(?, last_content_hash)
WHERE source = ?
`).run(itemsImported, framesCreated, contentHash ?? null, source);
}
/**
* Clear the R3-004 unchanged-set skip hash for a source, forcing the NEXT harvest
* of it to run the full per-item loop instead of short-circuiting as "unchanged".
* Needed after GDPR re-consent (#7): lifting a subject's suppression must let an
* IDENTICAL re-import re-materialize it — but the set-hash skip would otherwise
* skip the whole run before the per-item loop ever re-adds the re-consented subject.
* No-op if the source row does not exist.
*/
clearContentHash(source: ImportSourceType): void {
this.db.getDatabase()
.prepare('UPDATE harvest_sources SET last_content_hash = NULL WHERE source = ?')
.run(source);
}
/** Enable or disable auto-sync for a source. */
setAutoSync(source: ImportSourceType, enabled: boolean, intervalHours?: number): void {
const raw = this.db.getDatabase();
raw.prepare(`
UPDATE harvest_sources SET
auto_sync = ?,
sync_interval_hours = COALESCE(?, sync_interval_hours)
WHERE source = ?
`).run(enabled ? 1 : 0, intervalHours ?? null, source);
}
/** Get a specific source. */
getBySource(source: ImportSourceType): HarvestSource | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM harvest_sources WHERE source = ?').get(source) as Record<string, unknown> | undefined;
return row ? this.rowToSource(row) : null;
}
/** Get all registered sources. */
getAll(): HarvestSource[] {
const raw = this.db.getDatabase();
return (raw.prepare('SELECT * FROM harvest_sources ORDER BY last_synced_at DESC').all() as Record<string, unknown>[])
.map(r => this.rowToSource(r));
}
/** Get sources that need syncing (auto_sync enabled and interval elapsed). */
getStale(): HarvestSource[] {
const raw = this.db.getDatabase();
return (raw.prepare(`
SELECT * FROM harvest_sources
WHERE auto_sync = 1
AND (last_synced_at IS NULL
OR datetime(last_synced_at, '+' || sync_interval_hours || ' hours') <= datetime('now'))
`).all() as Record<string, unknown>[]).map(r => this.rowToSource(r));
}
/** Remove a source. */
remove(source: ImportSourceType): void {
this.db.getDatabase().prepare('DELETE FROM harvest_sources WHERE source = ?').run(source);
}
private rowToSource(row: Record<string, unknown>): HarvestSource {
return {
id: row.id as number,
source: row.source as ImportSourceType,
displayName: row.display_name as string,
sourcePath: row.source_path as string | null,
lastSyncedAt: row.last_synced_at as string | null,
itemsImported: row.items_imported as number,
framesCreated: row.frames_created as number,
autoSync: (row.auto_sync as number) === 1,
syncIntervalHours: row.sync_interval_hours as number,
lastContentHash: row.last_content_hash as string | null,
createdAt: row.created_at as string,
};
}
}

View File

@@ -0,0 +1,48 @@
/**
* stable-id.ts — deterministic, collision-safe id for harvest import items.
*
* WHY: a harvest item's `id` becomes the GDPR Art.17 subject key. The pipeline
* runs it through `rawTurnConvKey → sanitizeToken(`${source}-${id}`, 64)`
* (raw-turns.ts) to form `source_ref`, the key the erasure path reconstructs
* from (source, source_ref) alone. `randomUUID()` mints a fresh id every
* re-import, so a re-imported conversation lands under a NEW source_ref and
* "sticky erasure" (erase-once-stays-erased across re-import) silently breaks.
* A deterministic id keyed on the export's OWN stable identifiers fixes this —
* and makes raw_archive idempotent across re-imports of the same subject.
*
* CONTRACT:
* - deterministic: same (source, ...parts) -> same id, forever, cross-process.
* - sanitize-stable: output is lowercase sha256 hex ([0-9a-f]) only, so it
* survives sanitizeToken(...,64) unchanged (no '-' collapse, no truncation
* collision within the 64-char budget).
* - collision-safe: a NUL ('\x00') separator between parts prevents field-
* boundary ambiguity (e.g. 'a' + 'bc' vs 'ab' + 'c'); NUL cannot appear in
* any real title/id/path, so it is an unambiguous delimiter. Undefined parts
* collapse to '' but STILL emit a separator, so a present-vs-absent field
* never aliases a shifted field.
* - never keys on growing content: callers pass stable identifiers (conv uuid,
* file path, map key) so a conversation that GAINS turns keeps its id.
*
* Reference impl: mind/content-hash.ts (same createHash('sha256') pattern).
*/
import { createHash } from 'node:crypto';
/**
* Deterministic short id for a harvest item.
*
* @param source the ImportSourceType discriminator (e.g. 'chatgpt') — the first
* hashed field, so ids from different adapters that share an
* otherwise-identical key can never collide.
* @param parts the stable identity fields. string|number|undefined accepted;
* number is stringified, undefined becomes '' (separator still
* emitted). At least one meaningful part SHOULD be passed.
* @returns 40-char lowercase hex (sanitizeToken-stable, well under the 64 cap).
*/
export function stableHarvestId(
source: string,
...parts: ReadonlyArray<string | number | undefined>
): string {
// NUL-join: source is field 0; every part gets its own field even when ''.
const key = [source, ...parts.map(p => (p === undefined ? '' : String(p)))].join('\x00');
return createHash('sha256').update(key).digest('hex').slice(0, 40);
}

View File

@@ -0,0 +1,140 @@
/**
* Harvest Types — universal import format and distillation pipeline types.
*/
// ── Source Types ──
export type ImportSourceType =
| 'chatgpt' | 'claude' | 'claude-code' | 'claude-desktop'
| 'gemini' | 'google-ai-studio' | 'perplexity' | 'grok'
| 'cursor' | 'copilot' | 'manus' | 'genspark'
| 'qwen' | 'minimax' | 'z-ai' | 'openclaw' | 'cowork'
| 'elevenlabs' | 'google-flow'
| 'markdown' | 'plaintext' | 'pdf' | 'url'
| 'unknown';
export type ImportItemType =
| 'conversation' | 'memory' | 'instruction'
| 'preference' | 'artifact' | 'rule'
| 'decision' | 'document';
// ── Universal Import Item (adapter output) ──
export interface ConversationMessage {
role: 'user' | 'assistant' | 'system';
text: string;
timestamp?: string;
}
export interface UniversalImportItem {
id: string;
source: ImportSourceType;
type: ImportItemType;
title: string;
content: string;
messages?: ConversationMessage[];
timestamp: string;
metadata: Record<string, unknown>;
}
// ── Distilled Knowledge (pipeline output) ──
export type DistillTargetLayer = 'identity' | 'frame' | 'kg_entity' | 'kg_relation' | 'awareness';
export interface KnowledgeProvenance {
originalSource: ImportSourceType;
originalId?: string;
conversationTitle?: string;
importedAt: string;
distillationModel: string;
confidence: number;
pass: number;
}
export interface DistilledKnowledge {
targetLayer: DistillTargetLayer;
frameType?: 'I' | 'P';
importance: 'critical' | 'important' | 'normal' | 'temporary';
content: string;
entities?: { name: string; type: string }[];
relations?: { source: string; target: string; relation: string }[];
provenance: KnowledgeProvenance;
}
// ── Classification (Pass 1 output) ──
export type ClassificationDomain = 'work' | 'personal' | 'technical' | 'mixed';
export type ClassificationValue = 'high' | 'medium' | 'low' | 'skip';
export interface ClassifiedItem {
item: UniversalImportItem;
domain: ClassificationDomain;
value: ClassificationValue;
categories: string[];
}
// ── Extraction (Pass 2 output) ──
export interface ExtractedContent {
itemId: string;
decisions: string[];
preferences: string[];
facts: string[];
knowledge: string[];
entities: { name: string; type: string }[];
relations: { source: string; target: string; relation: string }[];
}
// ── Pipeline Result ──
export interface HarvestPipelineResult {
source: ImportSourceType;
itemsReceived: number;
itemsClassified: number;
itemsSkipped: number;
itemsExtracted: number;
knowledgeDistilled: DistilledKnowledge[];
framesSaved: number;
entitiesCreated: number;
relationsCreated: number;
identityUpdates: number;
duplicatesSkipped: number;
errors: string[];
costUsd: number;
durationMs: number;
}
// ── Harvest Source (tracking table) ──
export interface HarvestSource {
id: number;
source: ImportSourceType;
displayName: string;
sourcePath: string | null;
lastSyncedAt: string | null;
itemsImported: number;
framesCreated: number;
autoSync: boolean;
syncIntervalHours: number;
lastContentHash: string | null;
createdAt: string;
}
// ── Adapter Interface ──
export interface SourceAdapter {
readonly sourceType: ImportSourceType;
readonly displayName: string;
parse(input: unknown): UniversalImportItem[];
}
export interface FilesystemAdapter extends SourceAdapter {
scan(dirPath: string): UniversalImportItem[];
}
/**
* W4.4: unified per-frame content cap for harvest imports. The three ingest
* surfaces had drifted (MCP tools 2,000 vs sidecar 10,000) — a 5x divergence
* in what the same export preserved depending on the door it came through.
*/
export const HARVEST_FRAME_CONTENT_CAP = 10_000;

View File

@@ -0,0 +1,248 @@
/**
* Universal Adapter — accepts any text, JSON, or Markdown input.
*
* For Tier 2 platforms (Perplexity, Grok, Manus, Genspark, Qwen, Minimax,
* z.ai, OpenClaw, Cowork, ElevenLabs, Google Flow) where formats vary.
*
* Strategy:
* 1. Detect if input is a known JSON format
* 2. If JSON with recognizable structure, parse as conversations
* 3. Otherwise, treat as raw text and create a single import item
*/
import { stableHarvestId } from './stable-id.js';
import type { SourceAdapter, UniversalImportItem, ImportSourceType, ConversationMessage } from './types.js';
import { asRecord, firstString, getArray, getString, type RawRecord } from './raw-types.js';
/** Heuristic source detection from content cues. */
function detectSource(input: unknown): ImportSourceType {
if (typeof input === 'string') {
const lower = input.toLowerCase();
if (lower.includes('perplexity')) return 'perplexity';
if (lower.includes('grok') || lower.includes('x.ai')) return 'grok';
if (lower.includes('manus')) return 'manus';
if (lower.includes('genspark')) return 'genspark';
if (lower.includes('qwen') || lower.includes('tongyi')) return 'qwen';
if (lower.includes('minimax')) return 'minimax';
if (lower.includes('elevenlabs')) return 'elevenlabs';
return 'unknown';
}
const obj = asRecord(input);
if (obj) {
const keys = Object.keys(obj);
const source = getString(obj, 'source');
if (keys.includes('perplexity') || source === 'perplexity') return 'perplexity';
if (keys.includes('grok') || source === 'grok') return 'grok';
if (getString(obj, 'provider') === 'qwen') return 'qwen';
}
return 'unknown';
}
/** Try to find conversations in any JSON structure. */
function findConversations(obj: unknown): RawRecord[] | null {
if (Array.isArray(obj)) {
const first = asRecord(obj[0]);
if (obj.length > 0 && first && (first.messages || first.chat_messages || first.turns || first.history)) {
return obj.map(asRecord).filter((c): c is RawRecord => c !== null);
}
if (obj.length > 0 && first && (first.role || first.sender || first.author)) {
return [{ title: 'Imported Conversation', messages: obj }];
}
}
const record = asRecord(obj);
if (record) {
for (const key of ['conversations', 'chats', 'threads', 'sessions', 'history', 'data']) {
const nested = getArray(record, key);
if (nested) {
return findConversations(nested);
}
}
}
return null;
}
export class UniversalAdapter implements SourceAdapter {
readonly sourceType = 'unknown' as const;
readonly displayName = 'Universal (Auto-detect)';
parse(input: unknown): UniversalImportItem[] {
if (typeof input === 'string') {
return this.parseText(input);
}
if (typeof input === 'object' && input !== null) {
return this.parseJson(input);
}
return [];
}
private parseText(text: string): UniversalImportItem[] {
const source = detectSource(text);
const items: UniversalImportItem[] = [];
const conversations = this.splitConversations(text);
for (const conv of conversations) {
const messages = this.extractMessagesFromText(conv.content);
items.push({
// raw text paste has no id — content is the only surrogate (NOT growth-stable;
// documented tradeoff, no better anchor exists for free-text).
id: stableHarvestId('universal-text', source, conv.content),
source,
type: messages.length > 0 ? 'conversation' : 'memory',
title: conv.title,
content: conv.content,
messages: messages.length > 0 ? messages : undefined,
timestamp: new Date().toISOString(),
metadata: { parseMethod: 'universal-text', detectedSource: source },
});
}
return items;
}
private parseJson(input: object): UniversalImportItem[] {
const source = detectSource(input);
const items: UniversalImportItem[] = [];
const conversations = findConversations(input);
if (conversations) {
for (const conv of conversations) {
const title = firstString(conv, 'title', 'name', 'subject') ?? 'Imported Conversation';
const rawMessages = getArray(conv, 'messages') ?? getArray(conv, 'chat_messages')
?? getArray(conv, 'turns') ?? getArray(conv, 'history') ?? [];
const messages: ConversationMessage[] = [];
for (const rawMsg of rawMessages) {
const msg = asRecord(rawMsg);
if (!msg) continue;
const role = this.resolveRole(msg);
if (!role) continue;
const text = this.extractText(msg);
if (!text) continue;
messages.push({ role, text, timestamp: firstString(msg, 'timestamp', 'created_at', 'createTime') });
}
if (messages.length === 0) continue;
const convContent = messages.map(m => `${m.role}: ${m.text}`).join('\n\n');
const convId = getString(conv, 'id');
items.push({
// Stable per-conversation id when the export gives one (growth-stable). Else
// fall back to source+title+created_at PLUS content: a bare message-array paste
// has no id/timestamp and a CONSTANT synthetic title ('Imported Conversation'),
// so without content every such paste collapses to ONE (source, source_ref)
// subject key → cross-subject over-suppression / co-erasure. Content makes them
// distinct (id-less → not growth-stable, the documented universal-text tradeoff).
id: stableHarvestId('universal-json', convId ?? `${source}\x00${title}\x00${firstString(conv, 'created_at', 'createTime', 'timestamp') ?? ''}\x00${convContent}`),
source,
type: 'conversation',
title,
content: convContent,
messages,
timestamp: firstString(conv, 'created_at', 'createTime', 'timestamp') ?? new Date().toISOString(),
metadata: { parseMethod: 'universal-json', detectedSource: source, conversationId: convId },
});
}
}
if (items.length === 0) {
const record = asRecord(input);
items.push({
id: stableHarvestId('universal-json-raw', source, JSON.stringify(input)),
source,
type: 'memory',
title: (record && getString(record, 'title')) ?? 'Imported Data',
content: JSON.stringify(input, null, 2).slice(0, 50000),
timestamp: new Date().toISOString(),
metadata: { parseMethod: 'universal-json-raw', detectedSource: source },
});
}
return items;
}
private splitConversations(text: string): { title: string; content: string }[] {
const separators = [
/^#{1,3}\s+/gm,
/^={3,}$/gm,
/^-{3,}$/gm,
/^Conversation \d+/gim,
];
for (const sep of separators) {
const parts = text.split(sep).filter(p => p.trim().length > 20);
if (parts.length > 1) {
return parts.map((p, i) => ({
title: `Conversation ${i + 1}`,
content: p.trim(),
}));
}
}
return [{ title: 'Imported Text', content: text.trim() }];
}
private extractMessagesFromText(text: string): ConversationMessage[] {
const messages: ConversationMessage[] = [];
const pattern = /^(User|Human|Me|Assistant|AI|Bot|Claude|ChatGPT|Gemini|Grok):\s*([\s\S]*?)(?=^(?:User|Human|Me|Assistant|AI|Bot|Claude|ChatGPT|Gemini|Grok):|$)/gim;
let match;
while ((match = pattern.exec(text)) !== null) {
const speaker = match[1].toLowerCase();
const content = match[2].trim();
if (!content) continue;
const role = ['user', 'human', 'me'].includes(speaker) ? 'user' as const : 'assistant' as const;
messages.push({ role, text: content });
}
return messages;
}
private resolveRole(msg: RawRecord): 'user' | 'assistant' | null {
const role = firstString(msg, 'role', 'sender', 'author', 'type');
if (!role) return null;
const r = role.toLowerCase();
if (['user', 'human', 'me'].includes(r)) return 'user';
if (['assistant', 'ai', 'bot', 'model', 'system'].includes(r)) return 'assistant';
return null;
}
private extractText(msg: RawRecord): string {
const directText = getString(msg, 'text');
if (directText !== undefined) return directText.trim();
const directContent = getString(msg, 'content');
if (directContent !== undefined) return directContent.trim();
const contentBlocks = getArray(msg, 'content');
if (contentBlocks) {
return contentBlocks
.map(b => {
if (typeof b === 'string') return b;
const rec = asRecord(b);
if (!rec) return undefined;
if (rec.type === 'text') return getString(rec, 'text') ?? '';
// W4.4 (caption parity): generic text-bearing image fields.
const caption = getString(rec, 'caption') ?? getString(rec, 'alt') ?? getString(rec, 'description');
if (caption) return `[Shared image: ${caption}]`;
return undefined;
})
.filter((t): t is string => typeof t === 'string')
.join('\n')
.trim();
}
const parts = getArray(msg, 'parts');
if (parts) {
return parts
.map(asRecord)
.map(p => (p ? getString(p, 'text') : undefined))
.filter((t): t is string => typeof t === 'string')
.join('\n')
.trim();
}
return '';
}
}

View File

@@ -0,0 +1,186 @@
/**
* URL Source Adapter — fetches web pages and extracts readable content.
*
* Uses built-in fetch + basic HTML stripping.
* Splits content by sections if headings are present.
*/
import { randomUUID } from 'node:crypto';
import type { SourceAdapter, UniversalImportItem } from './types.js';
import { safeFetch, allowLocalFromEnv } from './url-egress-guard.js';
/** Strip HTML tags and decode common entities. Returns plain text. */
function stripHtml(html: string): string {
// Remove script and style blocks entirely
let text = html.replace(/<script[\s\S]*?<\/script>/gi, '');
text = text.replace(/<style[\s\S]*?<\/style>/gi, '');
text = text.replace(/<nav[\s\S]*?<\/nav>/gi, '');
text = text.replace(/<footer[\s\S]*?<\/footer>/gi, '');
text = text.replace(/<header[\s\S]*?<\/header>/gi, '');
// Convert headings to markdown-style
text = text.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, '\n# $1\n');
text = text.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, '\n## $1\n');
text = text.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, '\n### $1\n');
// Convert paragraphs and breaks to newlines
text = text.replace(/<\/p>/gi, '\n\n');
text = text.replace(/<br\s*\/?>/gi, '\n');
text = text.replace(/<li[^>]*>/gi, '\n- ');
// Strip remaining tags
text = text.replace(/<[^>]+>/g, '');
// Decode common HTML entities
text = text.replace(/&amp;/g, '&');
text = text.replace(/&lt;/g, '<');
text = text.replace(/&gt;/g, '>');
text = text.replace(/&quot;/g, '"');
text = text.replace(/&#39;/g, "'");
text = text.replace(/&nbsp;/g, ' ');
// Collapse excessive whitespace
text = text.replace(/[ \t]+/g, ' ');
text = text.replace(/\n{3,}/g, '\n\n');
return text.trim();
}
/** Extract <title> from HTML. */
function extractTitle(html: string): string | undefined {
const match = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
return match ? match[1].trim().replace(/\s+/g, ' ') : undefined;
}
/** Extract meta description from HTML. */
function extractDescription(html: string): string | undefined {
const match = html.match(/<meta[^>]*name=["']description["'][^>]*content=["']([\s\S]*?)["'][^>]*>/i)
?? html.match(/<meta[^>]*content=["']([\s\S]*?)["'][^>]*name=["']description["'][^>]*>/i);
return match ? match[1].trim() : undefined;
}
export class UrlAdapter implements SourceAdapter {
readonly sourceType = 'url' as const;
readonly displayName = 'Web URL';
parse(input: unknown): UniversalImportItem[] {
// Synchronous parse — for pre-fetched HTML content
if (typeof input !== 'string') return [];
// If input looks like HTML, parse it directly
if (input.includes('<html') || input.includes('<body') || input.includes('<div')) {
return this.parseHtml(input, undefined);
}
// If input is a URL, return empty — caller should use fetchAndParse()
if (input.startsWith('http://') || input.startsWith('https://')) {
return [];
}
return [];
}
/** Fetch a URL and parse its content. Async because of network I/O.
* Routed through the SSRF egress guard — the target and every redirect hop
* must resolve to a public address (loopback allowed only when
* WAGGLE_ALLOW_LOCAL_FETCH is set). Blocked targets throw EgressBlockedError,
* surfaced to the MCP ingest caller as an error result. */
async fetchAndParse(url: string): Promise<UniversalImportItem[]> {
const response = await safeFetch(
url,
{
headers: {
'User-Agent': 'Waggle-Memory/1.0 (knowledge harvester)',
'Accept': 'text/html,application/xhtml+xml,text/plain',
},
signal: AbortSignal.timeout(15_000),
},
{ allowLocal: allowLocalFromEnv() },
);
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
}
const contentType = response.headers.get('content-type') ?? '';
const body = await response.text();
if (contentType.includes('text/html') || contentType.includes('xhtml')) {
return this.parseHtml(body, url);
}
// Plain text or other — treat as plaintext
return [{
id: randomUUID(),
source: 'url',
type: 'document',
title: url,
content: body.slice(0, 4000),
timestamp: new Date().toISOString(),
metadata: { sourceUrl: url, contentType: 'article' },
}];
}
private parseHtml(html: string, sourceUrl: string | undefined): UniversalImportItem[] {
const title = extractTitle(html) ?? sourceUrl ?? 'Web page';
const description = extractDescription(html);
const plainText = stripHtml(html);
if (!plainText || plainText.length < 50) return [];
// For short pages, return as single item
if (plainText.length <= 4000) {
return [{
id: randomUUID(),
source: 'url',
type: 'document',
title,
content: plainText,
timestamp: new Date().toISOString(),
metadata: {
...(sourceUrl && { sourceUrl }),
...(description && { description }),
contentType: 'article',
},
}];
}
// For longer pages, split by headings
const sections = plainText.split(/\n(?=#{1,3}\s)/);
const items: UniversalImportItem[] = [];
for (const section of sections) {
const trimmed = section.trim();
if (trimmed.length < 30) continue;
const headingMatch = trimmed.match(/^#{1,3}\s+(.+)/);
const sectionTitle = headingMatch ? headingMatch[1].trim() : title;
items.push({
id: randomUUID(),
source: 'url',
type: 'document',
title: sectionTitle === title ? title : `${title} — ${sectionTitle}`,
content: trimmed.slice(0, 4000),
timestamp: new Date().toISOString(),
metadata: {
...(sourceUrl && { sourceUrl }),
contentType: 'article',
},
});
}
return items.length > 0 ? items : [{
id: randomUUID(),
source: 'url',
type: 'document',
title,
content: plainText.slice(0, 4000),
timestamp: new Date().toISOString(),
metadata: {
...(sourceUrl && { sourceUrl }),
contentType: 'article',
},
}];
}
}

View File

@@ -0,0 +1,298 @@
/**
* SSRF egress guard for the URL harvest adapter.
*
* The harvest URL adapter fetches attacker-influenceable URLs (a user or an
* MCP client naming a URL to ingest). Without this guard an ingest of
* `http://169.254.169.254/latest/meta-data/` or an RFC1918 host reaches
* internal services on the cloud/TEAMS deploy (sidecar binds 0.0.0.0) and can
* exfiltrate instance-metadata IAM credentials.
*
* This module is a self-contained, dependency-free (node builtins only) copy of
* the guard spec shared with `packages/agent/src/url-egress-guard.ts`. It lives
* here — rather than importing from @waggle/agent — because hive-mind-core is
* OSS-mirrored and must not depend on Waggle-proprietary packages. Keep the two
* implementations in sync; they share one spec.
*
* Obfuscated IP literals (octal / decimal / hex) are normalized by the OS
* resolver: `net.isIP` rejects them as literals, so they fall through to
* `dns.lookup`, which returns the canonical dotted form we classify.
*/
import { lookup as dnsLookup } from 'node:dns/promises';
import { isIP } from 'node:net';
export type AddressClass =
| 'public'
| 'loopback'
| 'private'
| 'link-local'
| 'unique-local'
| 'multicast'
| 'reserved'
| 'unspecified'
| 'invalid';
export interface ResolvedAddress {
address: string;
family: number;
}
export type LookupFn = (hostname: string) => Promise<ResolvedAddress[]>;
export interface EgressGuardOptions {
/** Permit loopback targets (default false). Only loopback is unlocked. */
allowLocal?: boolean;
/** Injectable resolver (tests). Defaults to node:dns/promises lookup(all). */
lookup?: LookupFn;
}
export class EgressBlockedError extends Error {
public readonly url: string;
public readonly addressClass?: AddressClass;
constructor(message: string, url: string, addressClass?: AddressClass) {
super(message);
this.name = 'EgressBlockedError';
this.url = url;
this.addressClass = addressClass;
}
}
function parseIpv4Octets(ip: string): [number, number, number, number] | null {
const parts = ip.split('.');
if (parts.length !== 4) return null;
const octets: number[] = [];
for (const part of parts) {
if (!/^\d{1,3}$/.test(part)) return null;
const n = Number(part);
if (n < 0 || n > 255) return null;
octets.push(n);
}
return [octets[0], octets[1], octets[2], octets[3]];
}
function classifyIpv4(ip: string): AddressClass {
const octets = parseIpv4Octets(ip);
if (!octets) return 'invalid';
const [a, b, c] = octets;
if (a === 0) return 'unspecified'; // 0.0.0.0/8
if (a === 127) return 'loopback'; // 127.0.0.0/8
if (a === 10) return 'private'; // 10.0.0.0/8
if (a === 172 && b >= 16 && b <= 31) return 'private'; // 172.16.0.0/12
if (a === 192 && b === 168) return 'private'; // 192.168.0.0/16
if (a === 100 && b >= 64 && b <= 127) return 'private'; // 100.64.0.0/10 CGNAT
if (a === 169 && b === 254) return 'link-local'; // 169.254.0.0/16 (metadata)
if (a >= 224 && a <= 239) return 'multicast'; // 224.0.0.0/4
if (a >= 240) return 'reserved'; // 240.0.0.0/4 + 255.255.255.255
if (a === 192 && b === 0 && c === 0) return 'reserved'; // 192.0.0.0/24
if (a === 192 && b === 0 && c === 2) return 'reserved'; // TEST-NET-1
if (a === 198 && (b === 18 || b === 19)) return 'reserved'; // 198.18.0.0/15
if (a === 198 && b === 51 && c === 100) return 'reserved'; // TEST-NET-2
if (a === 203 && b === 0 && c === 113) return 'reserved'; // TEST-NET-3
return 'public';
}
function parseIpv6Hextets(ip: string): number[] | null {
let s = ip.toLowerCase();
const zoneAt = s.indexOf('%');
if (zoneAt !== -1) s = s.slice(0, zoneAt);
const dotMatch = s.match(/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
if (dotMatch) {
const v4 = parseIpv4Octets(dotMatch[1]);
if (!v4) return null;
const hi = ((v4[0] << 8) | v4[1]).toString(16);
const lo = ((v4[2] << 8) | v4[3]).toString(16);
s = s.slice(0, dotMatch.index) + hi + ':' + lo;
}
const halves = s.split('::');
if (halves.length > 2) return null;
const head = halves[0] ? halves[0].split(':') : [];
let groups: string[];
if (halves.length === 2) {
const tail = halves[1] ? halves[1].split(':') : [];
const missing = 8 - head.length - tail.length;
if (missing < 0) return null;
groups = [...head, ...Array(missing).fill('0'), ...tail];
} else {
groups = head;
}
if (groups.length !== 8) return null;
const hextets: number[] = [];
for (const g of groups) {
if (!/^[0-9a-f]{1,4}$/.test(g)) return null;
hextets.push(parseInt(g, 16));
}
return hextets;
}
function classifyIpv6(ip: string): AddressClass {
const h = parseIpv6Hextets(ip);
if (!h) return 'invalid';
const firstFive = h[0] | h[1] | h[2] | h[3] | h[4];
if (firstFive === 0 && (h[5] === 0xffff || h[5] === 0)) {
const embedded = `${h[6] >> 8}.${h[6] & 0xff}.${h[7] >> 8}.${h[7] & 0xff}`;
const v4Class = classifyIpv4(embedded);
if (h[5] === 0 && h[6] === 0 && h[7] === 0) return 'unspecified';
if (h[5] === 0 && h[6] === 0 && h[7] === 1) return 'loopback'; // ::1
return v4Class;
}
if ((h[0] & 0xffc0) === 0xfe80) return 'link-local'; // fe80::/10
if ((h[0] & 0xfe00) === 0xfc00) return 'unique-local'; // fc00::/7 (ULA)
if ((h[0] & 0xff00) === 0xff00) return 'multicast'; // ff00::/8
if (h[0] === 0x2001 && h[1] === 0x0db8) return 'reserved'; // 2001:db8::/32 docs
if (h[0] === 0x0064 && h[1] === 0xff9b) return 'reserved'; // 64:ff9b::/96 NAT64
if (h[0] === 0x0100 && h[1] === 0 && h[2] === 0 && h[3] === 0) return 'reserved'; // 100::/64 discard
return 'public';
}
/** Classify a single IP-literal address. Fail-closed: unknown -> 'invalid'. */
export function classifyAddress(ip: string): AddressClass {
const family = isIP(ip);
if (family === 4) return classifyIpv4(ip);
if (family === 6) return classifyIpv6(ip);
return 'invalid';
}
function isAllowed(cls: AddressClass, allowLocal: boolean): boolean {
if (cls === 'public') return true;
if (cls === 'loopback' && allowLocal) return true;
return false;
}
async function defaultLookup(hostname: string): Promise<ResolvedAddress[]> {
const results = await dnsLookup(hostname, { all: true, verbatim: true });
return results.map((r) => ({ address: r.address, family: r.family }));
}
/**
* Validate that `rawUrl` is an http(s) URL whose host resolves only to public
* addresses. Throws {@link EgressBlockedError} otherwise. Returns parsed URL.
*/
export async function assertUrlAllowed(
rawUrl: string,
options: EgressGuardOptions = {},
): Promise<URL> {
let parsed: URL;
try {
parsed = new URL(rawUrl);
} catch {
throw new EgressBlockedError('Invalid URL', rawUrl);
}
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
throw new EgressBlockedError(
`Blocked non-http(s) scheme "${parsed.protocol}"`,
rawUrl,
);
}
// url.hostname keeps the surrounding brackets on an IPv6 literal ("[::1]"),
// which isIP() does not recognize — strip them so the literal is classified
// directly (loopback/private/link-local/…) instead of falling through to a DNS
// lookup that fails ENOTFOUND on Linux (and only accidentally resolves on
// Windows). Without this, bracketed-IPv6 URLs bypass classification entirely.
const hostname = parsed.hostname.replace(/^\[|\]$/g, '');
const literalFamily = isIP(hostname);
let addresses: ResolvedAddress[];
if (literalFamily !== 0) {
addresses = [{ address: hostname, family: literalFamily }];
} else {
const lookupFn = options.lookup ?? defaultLookup;
try {
addresses = await lookupFn(hostname);
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
throw new EgressBlockedError(
`DNS resolution failed for "${hostname}": ${detail}`,
rawUrl,
);
}
if (!addresses || addresses.length === 0) {
throw new EgressBlockedError(
`DNS resolution returned no addresses for "${hostname}"`,
rawUrl,
);
}
}
const allowLocal = options.allowLocal ?? false;
for (const { address } of addresses) {
const cls = classifyAddress(address);
if (!isAllowed(cls, allowLocal)) {
throw new EgressBlockedError(
`Blocked egress to ${cls} address ${address} (host "${hostname}")`,
rawUrl,
cls,
);
}
}
return parsed;
}
export interface SafeFetchOptions extends EgressGuardOptions {
/** Maximum redirect hops to follow (default 5). */
maxRedirects?: number;
/** Injectable fetch (tests). Defaults to globalThis.fetch. */
fetchImpl?: typeof globalThis.fetch;
}
/**
* SSRF-safe fetch. Validates before the request and re-validates every redirect
* hop (`redirect: 'manual'`). Caller-supplied `redirect` in `init` is ignored.
*/
export async function safeFetch(
rawUrl: string,
init: RequestInit = {},
options: SafeFetchOptions = {},
): Promise<Response> {
const maxRedirects = options.maxRedirects ?? 5;
const fetchImpl = options.fetchImpl ?? globalThis.fetch;
let currentUrl = rawUrl;
for (let hop = 0; hop <= maxRedirects; hop++) {
await assertUrlAllowed(currentUrl, options);
const response = await fetchImpl(currentUrl, { ...init, redirect: 'manual' });
const isRedirect = response.status >= 300 && response.status < 400;
const location = isRedirect ? response.headers.get('location') : null;
if (!location) {
return response;
}
try {
await response.body?.cancel();
} catch {
/* best-effort; ignore */
}
let nextUrl: string;
try {
nextUrl = new URL(location, currentUrl).toString();
} catch {
throw new EgressBlockedError(
`Invalid redirect target "${location}"`,
currentUrl,
);
}
currentUrl = nextUrl;
}
throw new EgressBlockedError(
`Exceeded maximum redirects (${maxRedirects})`,
rawUrl,
);
}
/** True when local (loopback) fetches are opted in via env. */
export function allowLocalFromEnv(): boolean {
const v = process.env.WAGGLE_ALLOW_LOCAL_FETCH;
return v === '1' || v === 'true';
}