/** * Universal Adapter — accepts any text, JSON, or Markdown input. * * For Tier 2 platforms (Perplexity, Grok, Manus, Genspark, Qwen, Minimax, * z.ai, OpenClaw, Cowork, ElevenLabs, Google Flow) where formats vary. * * Strategy: * 1. Detect if input is a known JSON format * 2. If JSON with recognizable structure, parse as conversations * 3. Otherwise, treat as raw text and create a single import item */ import { stableHarvestId } from './stable-id.js'; import type { SourceAdapter, UniversalImportItem, ImportSourceType, ConversationMessage } from './types.js'; import { asRecord, firstString, getArray, getString, type RawRecord } from './raw-types.js'; /** Heuristic source detection from content cues. */ function detectSource(input: unknown): ImportSourceType { if (typeof input === 'string') { const lower = input.toLowerCase(); if (lower.includes('perplexity')) return 'perplexity'; if (lower.includes('grok') || lower.includes('x.ai')) return 'grok'; if (lower.includes('manus')) return 'manus'; if (lower.includes('genspark')) return 'genspark'; if (lower.includes('qwen') || lower.includes('tongyi')) return 'qwen'; if (lower.includes('minimax')) return 'minimax'; if (lower.includes('elevenlabs')) return 'elevenlabs'; return 'unknown'; } const obj = asRecord(input); if (obj) { const keys = Object.keys(obj); const source = getString(obj, 'source'); if (keys.includes('perplexity') || source === 'perplexity') return 'perplexity'; if (keys.includes('grok') || source === 'grok') return 'grok'; if (getString(obj, 'provider') === 'qwen') return 'qwen'; } return 'unknown'; } /** Try to find conversations in any JSON structure. */ function findConversations(obj: unknown): RawRecord[] | null { if (Array.isArray(obj)) { const first = asRecord(obj[0]); if (obj.length > 0 && first && (first.messages || first.chat_messages || first.turns || first.history)) { return obj.map(asRecord).filter((c): c is RawRecord => c !== null); } if (obj.length > 0 && first && (first.role || first.sender || first.author)) { return [{ title: 'Imported Conversation', messages: obj }]; } } const record = asRecord(obj); if (record) { for (const key of ['conversations', 'chats', 'threads', 'sessions', 'history', 'data']) { const nested = getArray(record, key); if (nested) { return findConversations(nested); } } } return null; } export class UniversalAdapter implements SourceAdapter { readonly sourceType = 'unknown' as const; readonly displayName = 'Universal (Auto-detect)'; parse(input: unknown): UniversalImportItem[] { if (typeof input === 'string') { return this.parseText(input); } if (typeof input === 'object' && input !== null) { return this.parseJson(input); } return []; } private parseText(text: string): UniversalImportItem[] { const source = detectSource(text); const items: UniversalImportItem[] = []; const conversations = this.splitConversations(text); for (const conv of conversations) { const messages = this.extractMessagesFromText(conv.content); items.push({ // raw text paste has no id — content is the only surrogate (NOT growth-stable; // documented tradeoff, no better anchor exists for free-text). id: stableHarvestId('universal-text', source, conv.content), source, type: messages.length > 0 ? 'conversation' : 'memory', title: conv.title, content: conv.content, messages: messages.length > 0 ? messages : undefined, timestamp: new Date().toISOString(), metadata: { parseMethod: 'universal-text', detectedSource: source }, }); } return items; } private parseJson(input: object): UniversalImportItem[] { const source = detectSource(input); const items: UniversalImportItem[] = []; const conversations = findConversations(input); if (conversations) { for (const conv of conversations) { const title = firstString(conv, 'title', 'name', 'subject') ?? 'Imported Conversation'; const rawMessages = getArray(conv, 'messages') ?? getArray(conv, 'chat_messages') ?? getArray(conv, 'turns') ?? getArray(conv, 'history') ?? []; const messages: ConversationMessage[] = []; for (const rawMsg of rawMessages) { const msg = asRecord(rawMsg); if (!msg) continue; const role = this.resolveRole(msg); if (!role) continue; const text = this.extractText(msg); if (!text) continue; messages.push({ role, text, timestamp: firstString(msg, 'timestamp', 'created_at', 'createTime') }); } if (messages.length === 0) continue; const convContent = messages.map(m => `${m.role}: ${m.text}`).join('\n\n'); const convId = getString(conv, 'id'); items.push({ // Stable per-conversation id when the export gives one (growth-stable). Else // fall back to source+title+created_at PLUS content: a bare message-array paste // has no id/timestamp and a CONSTANT synthetic title ('Imported Conversation'), // so without content every such paste collapses to ONE (source, source_ref) // subject key → cross-subject over-suppression / co-erasure. Content makes them // distinct (id-less → not growth-stable, the documented universal-text tradeoff). id: stableHarvestId('universal-json', convId ?? `${source}\x00${title}\x00${firstString(conv, 'created_at', 'createTime', 'timestamp') ?? ''}\x00${convContent}`), source, type: 'conversation', title, content: convContent, messages, timestamp: firstString(conv, 'created_at', 'createTime', 'timestamp') ?? new Date().toISOString(), metadata: { parseMethod: 'universal-json', detectedSource: source, conversationId: convId }, }); } } if (items.length === 0) { const record = asRecord(input); items.push({ id: stableHarvestId('universal-json-raw', source, JSON.stringify(input)), source, type: 'memory', title: (record && getString(record, 'title')) ?? 'Imported Data', content: JSON.stringify(input, null, 2).slice(0, 50000), timestamp: new Date().toISOString(), metadata: { parseMethod: 'universal-json-raw', detectedSource: source }, }); } return items; } private splitConversations(text: string): { title: string; content: string }[] { const separators = [ /^#{1,3}\s+/gm, /^={3,}$/gm, /^-{3,}$/gm, /^Conversation \d+/gim, ]; for (const sep of separators) { const parts = text.split(sep).filter(p => p.trim().length > 20); if (parts.length > 1) { return parts.map((p, i) => ({ title: `Conversation ${i + 1}`, content: p.trim(), })); } } return [{ title: 'Imported Text', content: text.trim() }]; } private extractMessagesFromText(text: string): ConversationMessage[] { const messages: ConversationMessage[] = []; const pattern = /^(User|Human|Me|Assistant|AI|Bot|Claude|ChatGPT|Gemini|Grok):\s*([\s\S]*?)(?=^(?:User|Human|Me|Assistant|AI|Bot|Claude|ChatGPT|Gemini|Grok):|$)/gim; let match; while ((match = pattern.exec(text)) !== null) { const speaker = match[1].toLowerCase(); const content = match[2].trim(); if (!content) continue; const role = ['user', 'human', 'me'].includes(speaker) ? 'user' as const : 'assistant' as const; messages.push({ role, text: content }); } return messages; } private resolveRole(msg: RawRecord): 'user' | 'assistant' | null { const role = firstString(msg, 'role', 'sender', 'author', 'type'); if (!role) return null; const r = role.toLowerCase(); if (['user', 'human', 'me'].includes(r)) return 'user'; if (['assistant', 'ai', 'bot', 'model', 'system'].includes(r)) return 'assistant'; return null; } private extractText(msg: RawRecord): string { const directText = getString(msg, 'text'); if (directText !== undefined) return directText.trim(); const directContent = getString(msg, 'content'); if (directContent !== undefined) return directContent.trim(); const contentBlocks = getArray(msg, 'content'); if (contentBlocks) { return contentBlocks .map(b => { if (typeof b === 'string') return b; const rec = asRecord(b); if (!rec) return undefined; if (rec.type === 'text') return getString(rec, 'text') ?? ''; // W4.4 (caption parity): generic text-bearing image fields. const caption = getString(rec, 'caption') ?? getString(rec, 'alt') ?? getString(rec, 'description'); if (caption) return `[Shared image: ${caption}]`; return undefined; }) .filter((t): t is string => typeof t === 'string') .join('\n') .trim(); } const parts = getArray(msg, 'parts'); if (parts) { return parts .map(asRecord) .map(p => (p ? getString(p, 'text') : undefined)) .filter((t): t is string => typeof t === 'string') .join('\n') .trim(); } return ''; } }