Files
waggle-os/packages/hive-mind-core/src/harvest/universal-adapter.ts
Oleg Maslov 0c3e2ead3b
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled
moving
2026-09-02 10:10:29 +02:00

249 lines
9.2 KiB
TypeScript

/**
* Universal Adapter — accepts any text, JSON, or Markdown input.
*
* For Tier 2 platforms (Perplexity, Grok, Manus, Genspark, Qwen, Minimax,
* z.ai, OpenClaw, Cowork, ElevenLabs, Google Flow) where formats vary.
*
* Strategy:
* 1. Detect if input is a known JSON format
* 2. If JSON with recognizable structure, parse as conversations
* 3. Otherwise, treat as raw text and create a single import item
*/
import { stableHarvestId } from './stable-id.js';
import type { SourceAdapter, UniversalImportItem, ImportSourceType, ConversationMessage } from './types.js';
import { asRecord, firstString, getArray, getString, type RawRecord } from './raw-types.js';
/** Heuristic source detection from content cues. */
function detectSource(input: unknown): ImportSourceType {
if (typeof input === 'string') {
const lower = input.toLowerCase();
if (lower.includes('perplexity')) return 'perplexity';
if (lower.includes('grok') || lower.includes('x.ai')) return 'grok';
if (lower.includes('manus')) return 'manus';
if (lower.includes('genspark')) return 'genspark';
if (lower.includes('qwen') || lower.includes('tongyi')) return 'qwen';
if (lower.includes('minimax')) return 'minimax';
if (lower.includes('elevenlabs')) return 'elevenlabs';
return 'unknown';
}
const obj = asRecord(input);
if (obj) {
const keys = Object.keys(obj);
const source = getString(obj, 'source');
if (keys.includes('perplexity') || source === 'perplexity') return 'perplexity';
if (keys.includes('grok') || source === 'grok') return 'grok';
if (getString(obj, 'provider') === 'qwen') return 'qwen';
}
return 'unknown';
}
/** Try to find conversations in any JSON structure. */
function findConversations(obj: unknown): RawRecord[] | null {
if (Array.isArray(obj)) {
const first = asRecord(obj[0]);
if (obj.length > 0 && first && (first.messages || first.chat_messages || first.turns || first.history)) {
return obj.map(asRecord).filter((c): c is RawRecord => c !== null);
}
if (obj.length > 0 && first && (first.role || first.sender || first.author)) {
return [{ title: 'Imported Conversation', messages: obj }];
}
}
const record = asRecord(obj);
if (record) {
for (const key of ['conversations', 'chats', 'threads', 'sessions', 'history', 'data']) {
const nested = getArray(record, key);
if (nested) {
return findConversations(nested);
}
}
}
return null;
}
export class UniversalAdapter implements SourceAdapter {
readonly sourceType = 'unknown' as const;
readonly displayName = 'Universal (Auto-detect)';
parse(input: unknown): UniversalImportItem[] {
if (typeof input === 'string') {
return this.parseText(input);
}
if (typeof input === 'object' && input !== null) {
return this.parseJson(input);
}
return [];
}
private parseText(text: string): UniversalImportItem[] {
const source = detectSource(text);
const items: UniversalImportItem[] = [];
const conversations = this.splitConversations(text);
for (const conv of conversations) {
const messages = this.extractMessagesFromText(conv.content);
items.push({
// raw text paste has no id — content is the only surrogate (NOT growth-stable;
// documented tradeoff, no better anchor exists for free-text).
id: stableHarvestId('universal-text', source, conv.content),
source,
type: messages.length > 0 ? 'conversation' : 'memory',
title: conv.title,
content: conv.content,
messages: messages.length > 0 ? messages : undefined,
timestamp: new Date().toISOString(),
metadata: { parseMethod: 'universal-text', detectedSource: source },
});
}
return items;
}
private parseJson(input: object): UniversalImportItem[] {
const source = detectSource(input);
const items: UniversalImportItem[] = [];
const conversations = findConversations(input);
if (conversations) {
for (const conv of conversations) {
const title = firstString(conv, 'title', 'name', 'subject') ?? 'Imported Conversation';
const rawMessages = getArray(conv, 'messages') ?? getArray(conv, 'chat_messages')
?? getArray(conv, 'turns') ?? getArray(conv, 'history') ?? [];
const messages: ConversationMessage[] = [];
for (const rawMsg of rawMessages) {
const msg = asRecord(rawMsg);
if (!msg) continue;
const role = this.resolveRole(msg);
if (!role) continue;
const text = this.extractText(msg);
if (!text) continue;
messages.push({ role, text, timestamp: firstString(msg, 'timestamp', 'created_at', 'createTime') });
}
if (messages.length === 0) continue;
const convContent = messages.map(m => `${m.role}: ${m.text}`).join('\n\n');
const convId = getString(conv, 'id');
items.push({
// Stable per-conversation id when the export gives one (growth-stable). Else
// fall back to source+title+created_at PLUS content: a bare message-array paste
// has no id/timestamp and a CONSTANT synthetic title ('Imported Conversation'),
// so without content every such paste collapses to ONE (source, source_ref)
// subject key → cross-subject over-suppression / co-erasure. Content makes them
// distinct (id-less → not growth-stable, the documented universal-text tradeoff).
id: stableHarvestId('universal-json', convId ?? `${source}\x00${title}\x00${firstString(conv, 'created_at', 'createTime', 'timestamp') ?? ''}\x00${convContent}`),
source,
type: 'conversation',
title,
content: convContent,
messages,
timestamp: firstString(conv, 'created_at', 'createTime', 'timestamp') ?? new Date().toISOString(),
metadata: { parseMethod: 'universal-json', detectedSource: source, conversationId: convId },
});
}
}
if (items.length === 0) {
const record = asRecord(input);
items.push({
id: stableHarvestId('universal-json-raw', source, JSON.stringify(input)),
source,
type: 'memory',
title: (record && getString(record, 'title')) ?? 'Imported Data',
content: JSON.stringify(input, null, 2).slice(0, 50000),
timestamp: new Date().toISOString(),
metadata: { parseMethod: 'universal-json-raw', detectedSource: source },
});
}
return items;
}
private splitConversations(text: string): { title: string; content: string }[] {
const separators = [
/^#{1,3}\s+/gm,
/^={3,}$/gm,
/^-{3,}$/gm,
/^Conversation \d+/gim,
];
for (const sep of separators) {
const parts = text.split(sep).filter(p => p.trim().length > 20);
if (parts.length > 1) {
return parts.map((p, i) => ({
title: `Conversation ${i + 1}`,
content: p.trim(),
}));
}
}
return [{ title: 'Imported Text', content: text.trim() }];
}
private extractMessagesFromText(text: string): ConversationMessage[] {
const messages: ConversationMessage[] = [];
const pattern = /^(User|Human|Me|Assistant|AI|Bot|Claude|ChatGPT|Gemini|Grok):\s*([\s\S]*?)(?=^(?:User|Human|Me|Assistant|AI|Bot|Claude|ChatGPT|Gemini|Grok):|$)/gim;
let match;
while ((match = pattern.exec(text)) !== null) {
const speaker = match[1].toLowerCase();
const content = match[2].trim();
if (!content) continue;
const role = ['user', 'human', 'me'].includes(speaker) ? 'user' as const : 'assistant' as const;
messages.push({ role, text: content });
}
return messages;
}
private resolveRole(msg: RawRecord): 'user' | 'assistant' | null {
const role = firstString(msg, 'role', 'sender', 'author', 'type');
if (!role) return null;
const r = role.toLowerCase();
if (['user', 'human', 'me'].includes(r)) return 'user';
if (['assistant', 'ai', 'bot', 'model', 'system'].includes(r)) return 'assistant';
return null;
}
private extractText(msg: RawRecord): string {
const directText = getString(msg, 'text');
if (directText !== undefined) return directText.trim();
const directContent = getString(msg, 'content');
if (directContent !== undefined) return directContent.trim();
const contentBlocks = getArray(msg, 'content');
if (contentBlocks) {
return contentBlocks
.map(b => {
if (typeof b === 'string') return b;
const rec = asRecord(b);
if (!rec) return undefined;
if (rec.type === 'text') return getString(rec, 'text') ?? '';
// W4.4 (caption parity): generic text-bearing image fields.
const caption = getString(rec, 'caption') ?? getString(rec, 'alt') ?? getString(rec, 'description');
if (caption) return `[Shared image: ${caption}]`;
return undefined;
})
.filter((t): t is string => typeof t === 'string')
.join('\n')
.trim();
}
const parts = getArray(msg, 'parts');
if (parts) {
return parts
.map(asRecord)
.map(p => (p ? getString(p, 'text') : undefined))
.filter((t): t is string => typeof t === 'string')
.join('\n')
.trim();
}
return '';
}
}