This commit is contained in:
334
packages/wiki-compiler/src/adapters/notion.ts
Normal file
334
packages/wiki-compiler/src/adapters/notion.ts
Normal file
@@ -0,0 +1,334 @@
|
||||
/**
|
||||
* Notion workspace export adapter (M-13)
|
||||
*
|
||||
* Writes compiled wiki pages to a user's Notion workspace as child pages
|
||||
* under a user-chosen root. Uses the Notion REST API directly (no SDK)
|
||||
* so we don't add @notionhq/client to the dep graph.
|
||||
*
|
||||
* Re-run strategy (per M-13 decision memo — delta):
|
||||
* - Unchanged (content_hash matches existing notion_page_id) -> skip API call
|
||||
* - New page (no notion_page_id) -> POST /v1/pages
|
||||
* - Changed page (content_hash differs) -> archive old + create new
|
||||
*
|
||||
* Keeping the converter standalone (no library dep) is intentional — our
|
||||
* markdown output is narrow and predictable (H1-H3, paragraphs, bullets,
|
||||
* blockquotes, inline links). A dependency would inherit edge-case
|
||||
* handling for features we don't emit.
|
||||
*/
|
||||
|
||||
import type { PageRecord, WikiPageType } from '../types.js';
|
||||
|
||||
const NOTION_API = 'https://api.notion.com/v1';
|
||||
const NOTION_VERSION = '2022-06-28';
|
||||
/** Notion API caps block children per request at 100. */
|
||||
const BLOCK_BATCH_SIZE = 100;
|
||||
|
||||
export interface NotionExportOptions {
|
||||
/** Integration token (from notion.so/my-integrations — Internal Integration). */
|
||||
token: string;
|
||||
/** Root page ID under which to create child pages. UUIDs with or without dashes accepted. */
|
||||
rootPageId: string;
|
||||
}
|
||||
|
||||
export interface NotionExportStats {
|
||||
byType: Record<WikiPageType, number>;
|
||||
pagesCreated: number;
|
||||
pagesUpdated: number;
|
||||
pagesUnchanged: number;
|
||||
pagesFailed: number;
|
||||
errors: { slug: string; message: string }[];
|
||||
}
|
||||
|
||||
export interface NotionStateHelpers {
|
||||
getNotionPageId(slug: string): string | null;
|
||||
setNotionPageId(slug: string, pageId: string): void;
|
||||
clearNotionPageId(slug: string): void;
|
||||
/** Used to detect "did content change since last export?". */
|
||||
getPageContentHash(slug: string): string | null;
|
||||
setPageContentHash?(slug: string, hash: string): void;
|
||||
}
|
||||
|
||||
export interface NotionBlock {
|
||||
object: 'block';
|
||||
type: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface RichText {
|
||||
type: 'text';
|
||||
text: { content: string; link?: { url: string } | null };
|
||||
annotations?: {
|
||||
bold?: boolean;
|
||||
italic?: boolean;
|
||||
code?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The per-type payload a {@link NotionBlock} carries under its dynamic
|
||||
* `[block.type]` key (e.g. `block.heading_1`, `block.paragraph`). Each holds
|
||||
* the converted rich-text run. Exposed for narrowing the block's indexed
|
||||
* `unknown` value at consumer/test boundaries.
|
||||
*/
|
||||
export interface NotionBlockPayload {
|
||||
rich_text: RichText[];
|
||||
}
|
||||
|
||||
/** Strip a leading YAML frontmatter block (--- ... ---) if present. */
|
||||
export function stripFrontmatter(markdown: string): { body: string; title?: string } {
|
||||
const match = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
|
||||
if (!match) return { body: markdown };
|
||||
const yaml = match[1];
|
||||
const body = match[2];
|
||||
const nameMatch = yaml.match(/^name:\s*(.+)$/m);
|
||||
return { body, title: nameMatch?.[1]?.trim() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a markdown-text segment into Notion rich_text items, preserving
|
||||
* inline links `[text](url)`, `**bold**`, `*italic*`, and `` `code` ``.
|
||||
* Intentionally narrow — our wiki pages do not use more than this.
|
||||
*/
|
||||
export function toRichText(text: string): RichText[] {
|
||||
if (!text) return [];
|
||||
const out: RichText[] = [];
|
||||
const linkRe = /\[([^\]]+)\]\(([^)]+)\)/g;
|
||||
let lastIdx = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = linkRe.exec(text)) !== null) {
|
||||
if (match.index > lastIdx) {
|
||||
const slice = text.slice(lastIdx, match.index);
|
||||
out.push(...parseEmphasis(slice));
|
||||
}
|
||||
out.push({
|
||||
type: 'text',
|
||||
text: { content: match[1], link: { url: match[2] } },
|
||||
});
|
||||
lastIdx = match.index + match[0].length;
|
||||
}
|
||||
if (lastIdx < text.length) {
|
||||
out.push(...parseEmphasis(text.slice(lastIdx)));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Handle **bold**, *italic*, `code`. No nesting — our markdown does not nest these. */
|
||||
function parseEmphasis(text: string): RichText[] {
|
||||
if (!text) return [];
|
||||
const re = /(\*\*[^*]+\*\*|\*[^*]+\*|`[^`]+`)/g;
|
||||
const parts = text.split(re);
|
||||
return parts.filter(p => p.length > 0).map(p => {
|
||||
if (p.startsWith('**') && p.endsWith('**')) {
|
||||
return { type: 'text' as const, text: { content: p.slice(2, -2) }, annotations: { bold: true } };
|
||||
}
|
||||
if (p.startsWith('*') && p.endsWith('*')) {
|
||||
return { type: 'text' as const, text: { content: p.slice(1, -1) }, annotations: { italic: true } };
|
||||
}
|
||||
if (p.startsWith('`') && p.endsWith('`')) {
|
||||
return { type: 'text' as const, text: { content: p.slice(1, -1) }, annotations: { code: true } };
|
||||
}
|
||||
return { type: 'text' as const, text: { content: p } };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert our wiki markdown body (frontmatter already stripped) into an
|
||||
* ordered array of Notion blocks. Only handles: H1-H3, paragraphs, bullets,
|
||||
* blockquotes. Lines that do not match become paragraphs.
|
||||
*/
|
||||
export function markdownToBlocks(body: string): NotionBlock[] {
|
||||
const blocks: NotionBlock[] = [];
|
||||
const lines = body.split(/\r?\n/);
|
||||
let paragraph: string[] = [];
|
||||
|
||||
const flushParagraph = () => {
|
||||
const content = paragraph.join(' ').trim();
|
||||
if (content) {
|
||||
blocks.push({
|
||||
object: 'block',
|
||||
type: 'paragraph',
|
||||
paragraph: { rich_text: toRichText(content) },
|
||||
});
|
||||
}
|
||||
paragraph = [];
|
||||
};
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trimEnd();
|
||||
if (trimmed === '') {
|
||||
flushParagraph();
|
||||
continue;
|
||||
}
|
||||
|
||||
const h = trimmed.match(/^(#{1,3})\s+(.+)$/);
|
||||
if (h) {
|
||||
flushParagraph();
|
||||
const level = h[1].length as 1 | 2 | 3;
|
||||
const text = h[2];
|
||||
const type = `heading_${level}` as const;
|
||||
blocks.push({
|
||||
object: 'block',
|
||||
type,
|
||||
[type]: { rich_text: toRichText(text) },
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const bullet = trimmed.match(/^[-*]\s+(.+)$/);
|
||||
if (bullet) {
|
||||
flushParagraph();
|
||||
blocks.push({
|
||||
object: 'block',
|
||||
type: 'bulleted_list_item',
|
||||
bulleted_list_item: { rich_text: toRichText(bullet[1]) },
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const quote = trimmed.match(/^>\s?(.*)$/);
|
||||
if (quote) {
|
||||
flushParagraph();
|
||||
blocks.push({
|
||||
object: 'block',
|
||||
type: 'quote',
|
||||
quote: { rich_text: toRichText(quote[1]) },
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
paragraph.push(trimmed);
|
||||
}
|
||||
flushParagraph();
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a UUID page id from either a plain id (abc123...) or a Notion
|
||||
* page URL. Returns the id with or without dashes — Notion accepts both.
|
||||
*/
|
||||
export function extractNotionPageId(urlOrId: string): string | null {
|
||||
const trimmed = urlOrId.trim();
|
||||
const hexMatch = trimmed.match(/([a-f0-9]{8}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{12})/i);
|
||||
return hexMatch ? hexMatch[1] : null;
|
||||
}
|
||||
|
||||
function notionHeaders(token: string): Record<string, string> {
|
||||
return {
|
||||
Authorization: `Bearer ${token}`,
|
||||
'Notion-Version': NOTION_VERSION,
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
}
|
||||
|
||||
async function createNotionPage(
|
||||
token: string,
|
||||
rootPageId: string,
|
||||
title: string,
|
||||
blocks: NotionBlock[],
|
||||
): Promise<{ id: string }> {
|
||||
const firstBatch = blocks.slice(0, BLOCK_BATCH_SIZE);
|
||||
const res = await fetch(`${NOTION_API}/pages`, {
|
||||
method: 'POST',
|
||||
headers: notionHeaders(token),
|
||||
body: JSON.stringify({
|
||||
parent: { page_id: rootPageId },
|
||||
properties: {
|
||||
title: {
|
||||
title: [{ text: { content: title.slice(0, 200) } }],
|
||||
},
|
||||
},
|
||||
children: firstBatch,
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.text().catch(() => res.statusText);
|
||||
throw new Error(`Notion create failed (${res.status}): ${err.slice(0, 300)}`);
|
||||
}
|
||||
const data = await res.json() as { id: string };
|
||||
|
||||
let cursor = BLOCK_BATCH_SIZE;
|
||||
while (cursor < blocks.length) {
|
||||
const batch = blocks.slice(cursor, cursor + BLOCK_BATCH_SIZE);
|
||||
const appendRes = await fetch(`${NOTION_API}/blocks/${data.id}/children`, {
|
||||
method: 'PATCH',
|
||||
headers: notionHeaders(token),
|
||||
body: JSON.stringify({ children: batch }),
|
||||
});
|
||||
if (!appendRes.ok) {
|
||||
const err = await appendRes.text().catch(() => appendRes.statusText);
|
||||
throw new Error(`Notion append failed (${appendRes.status}): ${err.slice(0, 300)}`);
|
||||
}
|
||||
cursor += BLOCK_BATCH_SIZE;
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
async function archiveNotionPage(token: string, pageId: string): Promise<void> {
|
||||
const res = await fetch(`${NOTION_API}/pages/${pageId}`, {
|
||||
method: 'PATCH',
|
||||
headers: notionHeaders(token),
|
||||
body: JSON.stringify({ archived: true }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
// Archive-failure is tolerable — worst case the user sees two pages.
|
||||
// We swallow it here so the caller can still create the new page.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export every compiled page to Notion. Skips index/health virtual pages.
|
||||
* Runs sequentially to respect Notion's rate limit (~3 req/sec).
|
||||
*/
|
||||
export async function writeToNotionWorkspace(
|
||||
pages: PageRecord[],
|
||||
opts: NotionExportOptions,
|
||||
state: NotionStateHelpers,
|
||||
): Promise<NotionExportStats> {
|
||||
const stats: NotionExportStats = {
|
||||
byType: {} as Record<WikiPageType, number>,
|
||||
pagesCreated: 0,
|
||||
pagesUpdated: 0,
|
||||
pagesUnchanged: 0,
|
||||
pagesFailed: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
for (const page of pages) {
|
||||
if (page.pageType === 'index' || page.pageType === 'health') continue;
|
||||
|
||||
try {
|
||||
const existingId = state.getNotionPageId(page.slug);
|
||||
const existingHash = state.getPageContentHash(page.slug);
|
||||
if (existingId && existingHash === page.contentHash) {
|
||||
stats.pagesUnchanged++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const { body, title: frontmatterTitle } = stripFrontmatter(page.markdown);
|
||||
const blocks = markdownToBlocks(body);
|
||||
const title = frontmatterTitle ?? page.name;
|
||||
|
||||
if (existingId) {
|
||||
await archiveNotionPage(opts.token, existingId);
|
||||
state.clearNotionPageId(page.slug);
|
||||
}
|
||||
|
||||
const created = await createNotionPage(opts.token, opts.rootPageId, title, blocks);
|
||||
state.setNotionPageId(page.slug, created.id);
|
||||
if (existingId) stats.pagesUpdated++;
|
||||
else stats.pagesCreated++;
|
||||
stats.byType[page.pageType] = (stats.byType[page.pageType] ?? 0) + 1;
|
||||
} catch (err) {
|
||||
stats.pagesFailed++;
|
||||
stats.errors.push({
|
||||
slug: page.slug,
|
||||
message: err instanceof Error ? err.message : 'Unknown error',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
128
packages/wiki-compiler/src/adapters/obsidian.ts
Normal file
128
packages/wiki-compiler/src/adapters/obsidian.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* Obsidian Vault Adapter (M-12)
|
||||
*
|
||||
* Writes compiled wiki pages to a user-specified directory in a shape
|
||||
* Obsidian reads natively:
|
||||
* {outDir}/
|
||||
* _index.md — table of contents grouped by page type
|
||||
* entity/{slug}.md — one file per compiled entity page
|
||||
* concept/{slug}.md — one file per compiled concept page
|
||||
* synthesis/{slug}.md — one file per synthesis page
|
||||
*
|
||||
* YAML frontmatter is preserved as-is — Obsidian reads `type:`, `confidence:`,
|
||||
* etc. The body is rewritten only to convert raw `[[Display Name]]` wikilinks
|
||||
* into the `[[slug|Display Name]]` alias form so the links resolve against
|
||||
* our slug-based filenames.
|
||||
*
|
||||
* This adapter does NOT call an LLM, does NOT mutate state.compile watermarks,
|
||||
* and is safe to call repeatedly — files are overwritten on each export.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { PageRecord } from '../types.js';
|
||||
|
||||
export interface ObsidianExportResult {
|
||||
outDir: string;
|
||||
filesWritten: number;
|
||||
indexPath: string;
|
||||
byType: Record<string, number>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform `[[Display Name]]` wikilinks to `[[slug|Display Name]]` so they
|
||||
* resolve against slug-named files. Leaves `[[already-a-slug]]` and
|
||||
* `[[slug|Display]]` forms alone.
|
||||
*/
|
||||
function transformWikilinks(markdown: string, nameToSlug: Map<string, string>): string {
|
||||
return markdown.replace(/\[\[([^\]|]+?)\]\]/g, (match, inner: string) => {
|
||||
const trimmed = inner.trim();
|
||||
// If the target is already a known slug (present in nameToSlug values),
|
||||
// leave it alone.
|
||||
for (const slug of nameToSlug.values()) {
|
||||
if (slug === trimmed) return match;
|
||||
}
|
||||
// Otherwise try to resolve as a display name.
|
||||
const slug = nameToSlug.get(trimmed.toLowerCase());
|
||||
return slug ? `[[${slug}|${trimmed}]]` : match;
|
||||
});
|
||||
}
|
||||
|
||||
function buildIndex(pages: PageRecord[]): string {
|
||||
const byType: Record<string, PageRecord[]> = {};
|
||||
for (const page of pages) {
|
||||
(byType[page.pageType] ??= []).push(page);
|
||||
}
|
||||
const lines: string[] = [];
|
||||
lines.push('---');
|
||||
lines.push('type: index');
|
||||
lines.push(`generated_at: ${new Date().toISOString()}`);
|
||||
lines.push(`total_pages: ${pages.length}`);
|
||||
lines.push('---');
|
||||
lines.push('');
|
||||
lines.push('# Waggle Wiki Index');
|
||||
lines.push('');
|
||||
lines.push(`Exported ${pages.length} page(s) from your Waggle memory.`);
|
||||
lines.push('');
|
||||
|
||||
const order: Array<[string, string]> = [
|
||||
['entity', 'Entities'],
|
||||
['concept', 'Concepts'],
|
||||
['synthesis', 'Cross-Source Syntheses'],
|
||||
];
|
||||
|
||||
for (const [type, heading] of order) {
|
||||
const group = byType[type];
|
||||
if (!group || group.length === 0) continue;
|
||||
lines.push(`## ${heading} (${group.length})`);
|
||||
lines.push('');
|
||||
for (const page of group) {
|
||||
lines.push(`- [[${page.slug}|${page.name}]] — ${page.sourceCount} source(s)`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the given pages to a directory shaped for Obsidian.
|
||||
*
|
||||
* @param pages page records from CompilationState.getAllPages()
|
||||
* @param outDir absolute directory path — created if missing. Must be writable.
|
||||
* @returns paths actually written + per-type counts
|
||||
*/
|
||||
export function writeToObsidianVault(pages: PageRecord[], outDir: string): ObsidianExportResult {
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
|
||||
// Build name → slug map for the wikilink transform. Case-insensitive
|
||||
// lookup so `[[Project Alpha]]` resolves to `project-alpha`.
|
||||
const nameToSlug = new Map<string, string>();
|
||||
for (const page of pages) {
|
||||
nameToSlug.set(page.name.toLowerCase(), page.slug);
|
||||
}
|
||||
|
||||
const byType: Record<string, number> = {};
|
||||
let filesWritten = 0;
|
||||
|
||||
for (const page of pages) {
|
||||
// Index + health pages are virtual — skip them; we write our own index.
|
||||
if (page.pageType === 'index' || page.pageType === 'health') continue;
|
||||
|
||||
const typeDir = path.join(outDir, page.pageType);
|
||||
fs.mkdirSync(typeDir, { recursive: true });
|
||||
|
||||
const content = transformWikilinks(page.markdown, nameToSlug);
|
||||
const filePath = path.join(typeDir, `${page.slug}.md`);
|
||||
fs.writeFileSync(filePath, content, 'utf-8');
|
||||
|
||||
filesWritten++;
|
||||
byType[page.pageType] = (byType[page.pageType] ?? 0) + 1;
|
||||
}
|
||||
|
||||
const indexPath = path.join(outDir, '_index.md');
|
||||
fs.writeFileSync(indexPath, buildIndex(pages), 'utf-8');
|
||||
filesWritten++;
|
||||
|
||||
return { outDir, filesWritten, indexPath, byType };
|
||||
}
|
||||
Reference in New Issue
Block a user