refactor: remove the weaver package and the chat-export import (P1.5)

Weaver consolidated and decayed frames of the legacy memory engine; mem0
updates and merges memories itself. The ChatGPT/Claude export import wrote
frames and its screen is hidden in v1 (ADR-0002).
This commit is contained in:
shumeiko.it
2026-10-24 12:01:22 +02:00
parent 101523e463
commit f4d2a62b24
17 changed files with 2 additions and 1718 deletions

View File

@@ -161,7 +161,5 @@ export {
SkillHashStore, computeSkillHash, SKILL_HASHES_TABLE_SQL,
type SkillHash,
} from './skill-hashes.js';
export { processImport, parseChatGPTExport, parseClaudeExport, extractKnowledge } from './memory-import.js';
export { createFileStore, LocalFileStore, LinkedDirStore, S3FileStore, isSensitiveFilePath, type FileStore, type FileEntry, type StorageInfo, type S3Config } from './file-store.js';
export { FileIndexer, MAX_CONTENT_BYTES, type FileIndexRow, type FileIndexResult } from './file-indexer.js';
export type { ImportSource, ImportResult, ExtractedKnowledge, ParsedConversation, ConversationMessage } from './memory-import.js';

View File

@@ -1,292 +0,0 @@
/**
* Memory Import — parse ChatGPT and Claude exports, extract knowledge.
*
* Flow: raw JSON → parse conversations → extract knowledge items → ImportResult
*/
export type ImportSource = 'chatgpt' | 'claude';
export interface ConversationMessage {
role: 'user' | 'assistant';
text: string;
timestamp?: string;
}
export interface ParsedConversation {
title: string;
messages: ConversationMessage[];
createdAt?: string;
source: ImportSource;
}
export interface ExtractedKnowledge {
type: 'decision' | 'fact' | 'preference' | 'topic';
content: string;
source: ImportSource;
conversationTitle: string;
importance: 'important' | 'normal';
}
export interface ImportResult {
source: ImportSource;
conversationsFound: number;
conversationsParsed: number;
knowledgeExtracted: ExtractedKnowledge[];
errors: string[];
}
// ── Raw export shapes (untrusted external JSON; all fields optional) ──
/** A ChatGPT export node carries an optional message in its `mapping` entry. */
interface ChatGPTMessage {
author?: { role?: string };
content?: { parts?: unknown[] };
create_time?: number;
}
interface ChatGPTNode {
message?: ChatGPTMessage;
}
interface ChatGPTConversation {
title?: string;
mapping?: Record<string, ChatGPTNode>;
create_time?: number;
}
interface ClaudeMessage {
sender?: string;
role?: string;
text?: string;
content?: string;
created_at?: string;
timestamp?: string;
}
interface ClaudeConversation {
name?: string;
title?: string;
chat_messages?: ClaudeMessage[];
messages?: ClaudeMessage[];
created_at?: string;
create_time?: string;
}
/** Pull the `conversations` array out of an export that may be the array itself. */
function extractConversations(json: unknown): unknown[] {
if (Array.isArray(json)) return json;
if (json && typeof json === 'object' && 'conversations' in json) {
const c = (json as { conversations?: unknown }).conversations;
if (Array.isArray(c)) return c;
}
return [];
}
// ── Parsers ───────────────────────────────────────
export function parseChatGPTExport(json: unknown): ParsedConversation[] {
const conversations = extractConversations(json);
return conversations.map((raw) => {
const conv = raw as ChatGPTConversation;
const title = conv.title || 'Untitled';
const messages: ConversationMessage[] = [];
// ChatGPT uses a mapping object with node IDs
if (conv.mapping && typeof conv.mapping === 'object') {
const nodes = Object.values(conv.mapping);
// Sort by create_time for chronological order
const sorted = nodes
.filter((n) => (n?.message?.content?.parts?.length ?? 0) > 0)
.sort((a, b) => (a.message?.create_time ?? 0) - (b.message?.create_time ?? 0));
for (const node of sorted) {
const msg = node.message;
if (!msg || !msg.author?.role) continue;
const role = msg.author.role === 'user' ? 'user' : 'assistant';
if (msg.author.role === 'system') continue; // Skip system messages
const text = (msg.content?.parts ?? [])
.filter((p): p is string => typeof p === 'string')
.join('\n')
.trim();
if (!text) continue;
messages.push({
role,
text,
timestamp: msg.create_time ? new Date(msg.create_time * 1000).toISOString() : undefined,
});
}
}
return {
title,
messages,
createdAt: conv.create_time ? new Date(conv.create_time * 1000).toISOString() : undefined,
source: 'chatgpt' as ImportSource,
};
}).filter(c => c.messages.length > 0);
}
export function parseClaudeExport(json: unknown): ParsedConversation[] {
const conversations = extractConversations(json);
return conversations.map((raw) => {
const conv = raw as ClaudeConversation;
const title = conv.name || conv.title || 'Untitled';
const messages: ConversationMessage[] = [];
const chatMessages = conv.chat_messages ?? conv.messages ?? [];
for (const msg of chatMessages) {
const role = (msg.sender === 'human' || msg.role === 'user') ? 'user' : 'assistant';
const text = (msg.text ?? msg.content ?? '').trim();
if (!text) continue;
messages.push({
role,
text,
timestamp: msg.created_at ?? msg.timestamp,
});
}
return {
title,
messages,
createdAt: conv.created_at ?? conv.create_time,
source: 'claude' as ImportSource,
};
}).filter(c => c.messages.length > 0);
}
// ── Knowledge Extraction ──────────────────────────
const DECISION_PATTERNS = [
/\b(?:decided|chose|going with|we'll use|confirmed|agreed|settled on|picked|selected)\b/i,
/\b(?:decision|choice):\s/i,
/\blet's go with\b/i,
];
const PREFERENCE_PATTERNS = [
/\b(?:I prefer|I like|I always|I never|I don't like|don't use|use .+ instead)\b/i,
/\b(?:my preference|I'd rather|please always|please never|from now on)\b/i,
/\b(?:my style is|I want you to|format it as)\b/i,
];
const FACT_PATTERNS = [
/\b(?:my name is|I work at|I'm a|I am a|my company|my team|my role|our product|our stack)\b/i,
/\b(?:we use|our tech|we're building|the project is|the codebase)\b/i,
];
export function extractKnowledge(conversations: ParsedConversation[]): ExtractedKnowledge[] {
const knowledge: ExtractedKnowledge[] = [];
const seen = new Set<string>(); // Deduplicate
for (const conv of conversations) {
// Extract conversation topic
if (conv.title && conv.title !== 'Untitled' && conv.title.length > 5) {
const topicKey = `topic:${conv.title.toLowerCase().trim()}`;
if (!seen.has(topicKey)) {
seen.add(topicKey);
knowledge.push({
type: 'topic',
content: `Conversation topic: ${conv.title}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'normal',
});
}
}
// Scan user messages for decisions, preferences, facts
for (const msg of conv.messages) {
if (msg.role !== 'user') continue;
if (msg.text.length < 10 || msg.text.length > 500) continue; // Skip too short/long
const sentences = msg.text.split(/[.!?\n]+/).map(s => s.trim()).filter(s => s.length > 10);
for (const sentence of sentences) {
const contentKey = sentence.toLowerCase().slice(0, 80);
if (seen.has(contentKey)) continue;
// Check for decisions
if (DECISION_PATTERNS.some(p => p.test(sentence))) {
seen.add(contentKey);
knowledge.push({
type: 'decision',
content: `Decision: ${sentence}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'important',
});
continue;
}
// Check for preferences
if (PREFERENCE_PATTERNS.some(p => p.test(sentence))) {
seen.add(contentKey);
knowledge.push({
type: 'preference',
content: `Preference: ${sentence}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'important',
});
continue;
}
// Check for facts
if (FACT_PATTERNS.some(p => p.test(sentence))) {
seen.add(contentKey);
knowledge.push({
type: 'fact',
content: `${sentence}`,
source: conv.source,
conversationTitle: conv.title,
importance: 'normal',
});
continue;
}
}
}
}
// Cap at 100 items to avoid polluting memory
return knowledge.slice(0, 100);
}
// ── Main Import Function ──────────────────────────
export function processImport(jsonData: unknown, source: ImportSource): ImportResult {
const errors: string[] = [];
let conversations: ParsedConversation[];
try {
conversations = source === 'chatgpt'
? parseChatGPTExport(jsonData)
: parseClaudeExport(jsonData);
} catch (err: unknown) {
const message = err instanceof Error ? err.message : String(err);
return {
source,
conversationsFound: 0,
conversationsParsed: 0,
knowledgeExtracted: [],
errors: [`Parse error: ${message}`],
};
}
if (conversations.length === 0) {
errors.push('No conversations found in export');
}
const knowledge = extractKnowledge(conversations);
return {
source,
conversationsFound: conversations.length,
conversationsParsed: conversations.filter(c => c.messages.length > 0).length,
knowledgeExtracted: knowledge,
errors,
};
}