This commit is contained in:
27
packages/wiki-compiler/package.json
Normal file
27
packages/wiki-compiler/package.json
Normal file
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"name": "@waggle/wiki-compiler",
|
||||
"version": "0.1.0",
|
||||
"description": "Wiki Compiler — LLM-powered knowledge synthesis engine. Compiles memory frames + KG into interlinked wiki pages.",
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"import": "./dist/index.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "node ../../node_modules/vitest/vitest.mjs run --root ../.. --config vitest.config.ts packages/wiki-compiler/tests"
|
||||
},
|
||||
"dependencies": {
|
||||
"@waggle/core": "*",
|
||||
"@waggle/shared": "*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^22.0.0",
|
||||
"typescript": "^5.8.3"
|
||||
},
|
||||
"license": "MIT"
|
||||
}
|
||||
334
packages/wiki-compiler/src/adapters/notion.ts
Normal file
334
packages/wiki-compiler/src/adapters/notion.ts
Normal file
@@ -0,0 +1,334 @@
|
||||
/**
|
||||
* Notion workspace export adapter (M-13)
|
||||
*
|
||||
* Writes compiled wiki pages to a user's Notion workspace as child pages
|
||||
* under a user-chosen root. Uses the Notion REST API directly (no SDK)
|
||||
* so we don't add @notionhq/client to the dep graph.
|
||||
*
|
||||
* Re-run strategy (per M-13 decision memo — delta):
|
||||
* - Unchanged (content_hash matches existing notion_page_id) -> skip API call
|
||||
* - New page (no notion_page_id) -> POST /v1/pages
|
||||
* - Changed page (content_hash differs) -> archive old + create new
|
||||
*
|
||||
* Keeping the converter standalone (no library dep) is intentional — our
|
||||
* markdown output is narrow and predictable (H1-H3, paragraphs, bullets,
|
||||
* blockquotes, inline links). A dependency would inherit edge-case
|
||||
* handling for features we don't emit.
|
||||
*/
|
||||
|
||||
import type { PageRecord, WikiPageType } from '../types.js';
|
||||
|
||||
const NOTION_API = 'https://api.notion.com/v1';
|
||||
const NOTION_VERSION = '2022-06-28';
|
||||
/** Notion API caps block children per request at 100. */
|
||||
const BLOCK_BATCH_SIZE = 100;
|
||||
|
||||
export interface NotionExportOptions {
|
||||
/** Integration token (from notion.so/my-integrations — Internal Integration). */
|
||||
token: string;
|
||||
/** Root page ID under which to create child pages. UUIDs with or without dashes accepted. */
|
||||
rootPageId: string;
|
||||
}
|
||||
|
||||
export interface NotionExportStats {
|
||||
byType: Record<WikiPageType, number>;
|
||||
pagesCreated: number;
|
||||
pagesUpdated: number;
|
||||
pagesUnchanged: number;
|
||||
pagesFailed: number;
|
||||
errors: { slug: string; message: string }[];
|
||||
}
|
||||
|
||||
export interface NotionStateHelpers {
|
||||
getNotionPageId(slug: string): string | null;
|
||||
setNotionPageId(slug: string, pageId: string): void;
|
||||
clearNotionPageId(slug: string): void;
|
||||
/** Used to detect "did content change since last export?". */
|
||||
getPageContentHash(slug: string): string | null;
|
||||
setPageContentHash?(slug: string, hash: string): void;
|
||||
}
|
||||
|
||||
export interface NotionBlock {
|
||||
object: 'block';
|
||||
type: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface RichText {
|
||||
type: 'text';
|
||||
text: { content: string; link?: { url: string } | null };
|
||||
annotations?: {
|
||||
bold?: boolean;
|
||||
italic?: boolean;
|
||||
code?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The per-type payload a {@link NotionBlock} carries under its dynamic
|
||||
* `[block.type]` key (e.g. `block.heading_1`, `block.paragraph`). Each holds
|
||||
* the converted rich-text run. Exposed for narrowing the block's indexed
|
||||
* `unknown` value at consumer/test boundaries.
|
||||
*/
|
||||
export interface NotionBlockPayload {
|
||||
rich_text: RichText[];
|
||||
}
|
||||
|
||||
/** Strip a leading YAML frontmatter block (--- ... ---) if present. */
|
||||
export function stripFrontmatter(markdown: string): { body: string; title?: string } {
|
||||
const match = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
|
||||
if (!match) return { body: markdown };
|
||||
const yaml = match[1];
|
||||
const body = match[2];
|
||||
const nameMatch = yaml.match(/^name:\s*(.+)$/m);
|
||||
return { body, title: nameMatch?.[1]?.trim() };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a markdown-text segment into Notion rich_text items, preserving
|
||||
* inline links `[text](url)`, `**bold**`, `*italic*`, and `` `code` ``.
|
||||
* Intentionally narrow — our wiki pages do not use more than this.
|
||||
*/
|
||||
export function toRichText(text: string): RichText[] {
|
||||
if (!text) return [];
|
||||
const out: RichText[] = [];
|
||||
const linkRe = /\[([^\]]+)\]\(([^)]+)\)/g;
|
||||
let lastIdx = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
while ((match = linkRe.exec(text)) !== null) {
|
||||
if (match.index > lastIdx) {
|
||||
const slice = text.slice(lastIdx, match.index);
|
||||
out.push(...parseEmphasis(slice));
|
||||
}
|
||||
out.push({
|
||||
type: 'text',
|
||||
text: { content: match[1], link: { url: match[2] } },
|
||||
});
|
||||
lastIdx = match.index + match[0].length;
|
||||
}
|
||||
if (lastIdx < text.length) {
|
||||
out.push(...parseEmphasis(text.slice(lastIdx)));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Handle **bold**, *italic*, `code`. No nesting — our markdown does not nest these. */
|
||||
function parseEmphasis(text: string): RichText[] {
|
||||
if (!text) return [];
|
||||
const re = /(\*\*[^*]+\*\*|\*[^*]+\*|`[^`]+`)/g;
|
||||
const parts = text.split(re);
|
||||
return parts.filter(p => p.length > 0).map(p => {
|
||||
if (p.startsWith('**') && p.endsWith('**')) {
|
||||
return { type: 'text' as const, text: { content: p.slice(2, -2) }, annotations: { bold: true } };
|
||||
}
|
||||
if (p.startsWith('*') && p.endsWith('*')) {
|
||||
return { type: 'text' as const, text: { content: p.slice(1, -1) }, annotations: { italic: true } };
|
||||
}
|
||||
if (p.startsWith('`') && p.endsWith('`')) {
|
||||
return { type: 'text' as const, text: { content: p.slice(1, -1) }, annotations: { code: true } };
|
||||
}
|
||||
return { type: 'text' as const, text: { content: p } };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert our wiki markdown body (frontmatter already stripped) into an
|
||||
* ordered array of Notion blocks. Only handles: H1-H3, paragraphs, bullets,
|
||||
* blockquotes. Lines that do not match become paragraphs.
|
||||
*/
|
||||
export function markdownToBlocks(body: string): NotionBlock[] {
|
||||
const blocks: NotionBlock[] = [];
|
||||
const lines = body.split(/\r?\n/);
|
||||
let paragraph: string[] = [];
|
||||
|
||||
const flushParagraph = () => {
|
||||
const content = paragraph.join(' ').trim();
|
||||
if (content) {
|
||||
blocks.push({
|
||||
object: 'block',
|
||||
type: 'paragraph',
|
||||
paragraph: { rich_text: toRichText(content) },
|
||||
});
|
||||
}
|
||||
paragraph = [];
|
||||
};
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trimEnd();
|
||||
if (trimmed === '') {
|
||||
flushParagraph();
|
||||
continue;
|
||||
}
|
||||
|
||||
const h = trimmed.match(/^(#{1,3})\s+(.+)$/);
|
||||
if (h) {
|
||||
flushParagraph();
|
||||
const level = h[1].length as 1 | 2 | 3;
|
||||
const text = h[2];
|
||||
const type = `heading_${level}` as const;
|
||||
blocks.push({
|
||||
object: 'block',
|
||||
type,
|
||||
[type]: { rich_text: toRichText(text) },
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const bullet = trimmed.match(/^[-*]\s+(.+)$/);
|
||||
if (bullet) {
|
||||
flushParagraph();
|
||||
blocks.push({
|
||||
object: 'block',
|
||||
type: 'bulleted_list_item',
|
||||
bulleted_list_item: { rich_text: toRichText(bullet[1]) },
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const quote = trimmed.match(/^>\s?(.*)$/);
|
||||
if (quote) {
|
||||
flushParagraph();
|
||||
blocks.push({
|
||||
object: 'block',
|
||||
type: 'quote',
|
||||
quote: { rich_text: toRichText(quote[1]) },
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
paragraph.push(trimmed);
|
||||
}
|
||||
flushParagraph();
|
||||
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a UUID page id from either a plain id (abc123...) or a Notion
|
||||
* page URL. Returns the id with or without dashes — Notion accepts both.
|
||||
*/
|
||||
export function extractNotionPageId(urlOrId: string): string | null {
|
||||
const trimmed = urlOrId.trim();
|
||||
const hexMatch = trimmed.match(/([a-f0-9]{8}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{12})/i);
|
||||
return hexMatch ? hexMatch[1] : null;
|
||||
}
|
||||
|
||||
function notionHeaders(token: string): Record<string, string> {
|
||||
return {
|
||||
Authorization: `Bearer ${token}`,
|
||||
'Notion-Version': NOTION_VERSION,
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
}
|
||||
|
||||
async function createNotionPage(
|
||||
token: string,
|
||||
rootPageId: string,
|
||||
title: string,
|
||||
blocks: NotionBlock[],
|
||||
): Promise<{ id: string }> {
|
||||
const firstBatch = blocks.slice(0, BLOCK_BATCH_SIZE);
|
||||
const res = await fetch(`${NOTION_API}/pages`, {
|
||||
method: 'POST',
|
||||
headers: notionHeaders(token),
|
||||
body: JSON.stringify({
|
||||
parent: { page_id: rootPageId },
|
||||
properties: {
|
||||
title: {
|
||||
title: [{ text: { content: title.slice(0, 200) } }],
|
||||
},
|
||||
},
|
||||
children: firstBatch,
|
||||
}),
|
||||
});
|
||||
if (!res.ok) {
|
||||
const err = await res.text().catch(() => res.statusText);
|
||||
throw new Error(`Notion create failed (${res.status}): ${err.slice(0, 300)}`);
|
||||
}
|
||||
const data = await res.json() as { id: string };
|
||||
|
||||
let cursor = BLOCK_BATCH_SIZE;
|
||||
while (cursor < blocks.length) {
|
||||
const batch = blocks.slice(cursor, cursor + BLOCK_BATCH_SIZE);
|
||||
const appendRes = await fetch(`${NOTION_API}/blocks/${data.id}/children`, {
|
||||
method: 'PATCH',
|
||||
headers: notionHeaders(token),
|
||||
body: JSON.stringify({ children: batch }),
|
||||
});
|
||||
if (!appendRes.ok) {
|
||||
const err = await appendRes.text().catch(() => appendRes.statusText);
|
||||
throw new Error(`Notion append failed (${appendRes.status}): ${err.slice(0, 300)}`);
|
||||
}
|
||||
cursor += BLOCK_BATCH_SIZE;
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
async function archiveNotionPage(token: string, pageId: string): Promise<void> {
|
||||
const res = await fetch(`${NOTION_API}/pages/${pageId}`, {
|
||||
method: 'PATCH',
|
||||
headers: notionHeaders(token),
|
||||
body: JSON.stringify({ archived: true }),
|
||||
});
|
||||
if (!res.ok) {
|
||||
// Archive-failure is tolerable — worst case the user sees two pages.
|
||||
// We swallow it here so the caller can still create the new page.
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Export every compiled page to Notion. Skips index/health virtual pages.
|
||||
* Runs sequentially to respect Notion's rate limit (~3 req/sec).
|
||||
*/
|
||||
export async function writeToNotionWorkspace(
|
||||
pages: PageRecord[],
|
||||
opts: NotionExportOptions,
|
||||
state: NotionStateHelpers,
|
||||
): Promise<NotionExportStats> {
|
||||
const stats: NotionExportStats = {
|
||||
byType: {} as Record<WikiPageType, number>,
|
||||
pagesCreated: 0,
|
||||
pagesUpdated: 0,
|
||||
pagesUnchanged: 0,
|
||||
pagesFailed: 0,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
for (const page of pages) {
|
||||
if (page.pageType === 'index' || page.pageType === 'health') continue;
|
||||
|
||||
try {
|
||||
const existingId = state.getNotionPageId(page.slug);
|
||||
const existingHash = state.getPageContentHash(page.slug);
|
||||
if (existingId && existingHash === page.contentHash) {
|
||||
stats.pagesUnchanged++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const { body, title: frontmatterTitle } = stripFrontmatter(page.markdown);
|
||||
const blocks = markdownToBlocks(body);
|
||||
const title = frontmatterTitle ?? page.name;
|
||||
|
||||
if (existingId) {
|
||||
await archiveNotionPage(opts.token, existingId);
|
||||
state.clearNotionPageId(page.slug);
|
||||
}
|
||||
|
||||
const created = await createNotionPage(opts.token, opts.rootPageId, title, blocks);
|
||||
state.setNotionPageId(page.slug, created.id);
|
||||
if (existingId) stats.pagesUpdated++;
|
||||
else stats.pagesCreated++;
|
||||
stats.byType[page.pageType] = (stats.byType[page.pageType] ?? 0) + 1;
|
||||
} catch (err) {
|
||||
stats.pagesFailed++;
|
||||
stats.errors.push({
|
||||
slug: page.slug,
|
||||
message: err instanceof Error ? err.message : 'Unknown error',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
128
packages/wiki-compiler/src/adapters/obsidian.ts
Normal file
128
packages/wiki-compiler/src/adapters/obsidian.ts
Normal file
@@ -0,0 +1,128 @@
|
||||
/**
|
||||
* Obsidian Vault Adapter (M-12)
|
||||
*
|
||||
* Writes compiled wiki pages to a user-specified directory in a shape
|
||||
* Obsidian reads natively:
|
||||
* {outDir}/
|
||||
* _index.md — table of contents grouped by page type
|
||||
* entity/{slug}.md — one file per compiled entity page
|
||||
* concept/{slug}.md — one file per compiled concept page
|
||||
* synthesis/{slug}.md — one file per synthesis page
|
||||
*
|
||||
* YAML frontmatter is preserved as-is — Obsidian reads `type:`, `confidence:`,
|
||||
* etc. The body is rewritten only to convert raw `[[Display Name]]` wikilinks
|
||||
* into the `[[slug|Display Name]]` alias form so the links resolve against
|
||||
* our slug-based filenames.
|
||||
*
|
||||
* This adapter does NOT call an LLM, does NOT mutate state.compile watermarks,
|
||||
* and is safe to call repeatedly — files are overwritten on each export.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import type { PageRecord } from '../types.js';
|
||||
|
||||
export interface ObsidianExportResult {
|
||||
outDir: string;
|
||||
filesWritten: number;
|
||||
indexPath: string;
|
||||
byType: Record<string, number>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transform `[[Display Name]]` wikilinks to `[[slug|Display Name]]` so they
|
||||
* resolve against slug-named files. Leaves `[[already-a-slug]]` and
|
||||
* `[[slug|Display]]` forms alone.
|
||||
*/
|
||||
function transformWikilinks(markdown: string, nameToSlug: Map<string, string>): string {
|
||||
return markdown.replace(/\[\[([^\]|]+?)\]\]/g, (match, inner: string) => {
|
||||
const trimmed = inner.trim();
|
||||
// If the target is already a known slug (present in nameToSlug values),
|
||||
// leave it alone.
|
||||
for (const slug of nameToSlug.values()) {
|
||||
if (slug === trimmed) return match;
|
||||
}
|
||||
// Otherwise try to resolve as a display name.
|
||||
const slug = nameToSlug.get(trimmed.toLowerCase());
|
||||
return slug ? `[[${slug}|${trimmed}]]` : match;
|
||||
});
|
||||
}
|
||||
|
||||
function buildIndex(pages: PageRecord[]): string {
|
||||
const byType: Record<string, PageRecord[]> = {};
|
||||
for (const page of pages) {
|
||||
(byType[page.pageType] ??= []).push(page);
|
||||
}
|
||||
const lines: string[] = [];
|
||||
lines.push('---');
|
||||
lines.push('type: index');
|
||||
lines.push(`generated_at: ${new Date().toISOString()}`);
|
||||
lines.push(`total_pages: ${pages.length}`);
|
||||
lines.push('---');
|
||||
lines.push('');
|
||||
lines.push('# Waggle Wiki Index');
|
||||
lines.push('');
|
||||
lines.push(`Exported ${pages.length} page(s) from your Waggle memory.`);
|
||||
lines.push('');
|
||||
|
||||
const order: Array<[string, string]> = [
|
||||
['entity', 'Entities'],
|
||||
['concept', 'Concepts'],
|
||||
['synthesis', 'Cross-Source Syntheses'],
|
||||
];
|
||||
|
||||
for (const [type, heading] of order) {
|
||||
const group = byType[type];
|
||||
if (!group || group.length === 0) continue;
|
||||
lines.push(`## ${heading} (${group.length})`);
|
||||
lines.push('');
|
||||
for (const page of group) {
|
||||
lines.push(`- [[${page.slug}|${page.name}]] — ${page.sourceCount} source(s)`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the given pages to a directory shaped for Obsidian.
|
||||
*
|
||||
* @param pages page records from CompilationState.getAllPages()
|
||||
* @param outDir absolute directory path — created if missing. Must be writable.
|
||||
* @returns paths actually written + per-type counts
|
||||
*/
|
||||
export function writeToObsidianVault(pages: PageRecord[], outDir: string): ObsidianExportResult {
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
|
||||
// Build name → slug map for the wikilink transform. Case-insensitive
|
||||
// lookup so `[[Project Alpha]]` resolves to `project-alpha`.
|
||||
const nameToSlug = new Map<string, string>();
|
||||
for (const page of pages) {
|
||||
nameToSlug.set(page.name.toLowerCase(), page.slug);
|
||||
}
|
||||
|
||||
const byType: Record<string, number> = {};
|
||||
let filesWritten = 0;
|
||||
|
||||
for (const page of pages) {
|
||||
// Index + health pages are virtual — skip them; we write our own index.
|
||||
if (page.pageType === 'index' || page.pageType === 'health') continue;
|
||||
|
||||
const typeDir = path.join(outDir, page.pageType);
|
||||
fs.mkdirSync(typeDir, { recursive: true });
|
||||
|
||||
const content = transformWikilinks(page.markdown, nameToSlug);
|
||||
const filePath = path.join(typeDir, `${page.slug}.md`);
|
||||
fs.writeFileSync(filePath, content, 'utf-8');
|
||||
|
||||
filesWritten++;
|
||||
byType[page.pageType] = (byType[page.pageType] ?? 0) + 1;
|
||||
}
|
||||
|
||||
const indexPath = path.join(outDir, '_index.md');
|
||||
fs.writeFileSync(indexPath, buildIndex(pages), 'utf-8');
|
||||
filesWritten++;
|
||||
|
||||
return { outDir, filesWritten, indexPath, byType };
|
||||
}
|
||||
596
packages/wiki-compiler/src/compiler.ts
Normal file
596
packages/wiki-compiler/src/compiler.ts
Normal file
@@ -0,0 +1,596 @@
|
||||
/**
|
||||
* Wiki Compiler — compiles memory frames + KG into interlinked wiki pages.
|
||||
*
|
||||
* Core compilation functions:
|
||||
* - compileEntityPage() — entity → frames + relations → LLM → markdown
|
||||
* - compileConceptPage() — topic → search → LLM → markdown
|
||||
* - compileSynthesisPage() — cross-source pattern detection
|
||||
* - compileIndex() — navigable catalog
|
||||
* - compileHealth() — contradictions, gaps, data quality
|
||||
*/
|
||||
|
||||
import type {
|
||||
KnowledgeGraph,
|
||||
FrameStore,
|
||||
HybridSearch,
|
||||
Entity,
|
||||
} from '@waggle/core';
|
||||
import type {
|
||||
WikiPage,
|
||||
WikiPageType,
|
||||
CompilerConfig,
|
||||
CompilationResult,
|
||||
HealthReport,
|
||||
HealthIssue,
|
||||
} from './types.js';
|
||||
import { CompilationState, contentHash } from './state.js';
|
||||
import { entityPagePrompt, conceptPagePrompt, synthesisPagePrompt } from './prompts.js';
|
||||
|
||||
function slugify(name: string): string {
|
||||
return name
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '')
|
||||
.slice(0, 80);
|
||||
}
|
||||
|
||||
function buildFrontmatter(
|
||||
type: WikiPageType,
|
||||
name: string,
|
||||
frameIds: number[],
|
||||
relatedEntities: string[],
|
||||
confidence: number,
|
||||
entityType?: string,
|
||||
): string {
|
||||
const lines = [
|
||||
'---',
|
||||
`type: ${type}`,
|
||||
...(entityType ? [`entity_type: ${entityType}`] : []),
|
||||
`name: "${name.replace(/"/g, '\\"')}"`,
|
||||
`confidence: ${confidence.toFixed(2)}`,
|
||||
`sources: ${frameIds.length}`,
|
||||
`last_compiled: ${new Date().toISOString()}`,
|
||||
`frame_ids: [${frameIds.join(', ')}]`,
|
||||
`related_entities: [${relatedEntities.map(e => `"${e}"`).join(', ')}]`,
|
||||
'---',
|
||||
];
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export class WikiCompiler {
|
||||
private kg: KnowledgeGraph;
|
||||
private frames: FrameStore;
|
||||
private search: HybridSearch;
|
||||
private state: CompilationState;
|
||||
private config: Required<CompilerConfig>;
|
||||
|
||||
constructor(
|
||||
kg: KnowledgeGraph,
|
||||
frames: FrameStore,
|
||||
search: HybridSearch,
|
||||
state: CompilationState,
|
||||
config: CompilerConfig,
|
||||
) {
|
||||
this.kg = kg;
|
||||
this.frames = frames;
|
||||
this.search = search;
|
||||
this.state = state;
|
||||
this.config = {
|
||||
synthesize: config.synthesize,
|
||||
outputDir: config.outputDir ?? 'wiki',
|
||||
minFramesPerPage: config.minFramesPerPage ?? 2,
|
||||
maxFramesPerCall: config.maxFramesPerCall ?? 30,
|
||||
minConfidence: config.minConfidence ?? 0.3,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Entity Page ───────────────────────────────────────────────
|
||||
|
||||
async compileEntityPage(entity: Entity): Promise<WikiPage | null> {
|
||||
// Gather frames mentioning this entity
|
||||
const searchResults = await this.search.search(entity.name, {
|
||||
limit: this.config.maxFramesPerCall,
|
||||
});
|
||||
|
||||
const frameData = searchResults.map(r => ({
|
||||
id: r.frame.id,
|
||||
content: r.frame.content,
|
||||
created_at: r.frame.created_at,
|
||||
}));
|
||||
|
||||
if (frameData.length < this.config.minFramesPerPage) {
|
||||
return null; // Not enough data for a page
|
||||
}
|
||||
|
||||
// Gather relations
|
||||
const outRels = this.kg.getRelationsFrom(entity.id);
|
||||
const inRels = this.kg.getRelationsTo(entity.id);
|
||||
const relations: { target: string; relationType: string; confidence: number }[] = [];
|
||||
const relatedEntities: string[] = [];
|
||||
|
||||
for (const rel of outRels) {
|
||||
const target = this.kg.getEntity(rel.target_id);
|
||||
if (target && target.valid_to === null) {
|
||||
relations.push({ target: target.name, relationType: rel.relation_type, confidence: rel.confidence });
|
||||
relatedEntities.push(target.name);
|
||||
}
|
||||
}
|
||||
for (const rel of inRels) {
|
||||
const source = this.kg.getEntity(rel.source_id);
|
||||
if (source && source.valid_to === null) {
|
||||
relations.push({ target: source.name, relationType: `${rel.relation_type} (inbound)`, confidence: rel.confidence });
|
||||
if (!relatedEntities.includes(source.name)) {
|
||||
relatedEntities.push(source.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Synthesize via LLM
|
||||
const prompt = entityPagePrompt(entity.name, entity.entity_type, frameData, relations);
|
||||
const body = await this.config.synthesize(prompt);
|
||||
|
||||
const frameIds = frameData.map(f => f.id);
|
||||
const confidence = frameData.length > 5 ? 0.9 : frameData.length > 2 ? 0.7 : 0.5;
|
||||
const slug = slugify(entity.name);
|
||||
|
||||
const frontmatter = buildFrontmatter('entity', entity.name, frameIds, relatedEntities, confidence, entity.entity_type);
|
||||
const markdown = `${frontmatter}\n\n# ${entity.name}\n\n${body}`;
|
||||
|
||||
return {
|
||||
slug,
|
||||
frontmatter: {
|
||||
type: 'entity',
|
||||
entity_type: entity.entity_type,
|
||||
name: entity.name,
|
||||
confidence,
|
||||
sources: frameIds.length,
|
||||
last_compiled: new Date().toISOString(),
|
||||
frame_ids: frameIds,
|
||||
related_entities: relatedEntities,
|
||||
},
|
||||
markdown,
|
||||
contentHash: contentHash(markdown),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Concept Page ──────────────────────────────────────────────
|
||||
|
||||
async compileConceptPage(conceptName: string): Promise<WikiPage | null> {
|
||||
const searchResults = await this.search.search(conceptName, {
|
||||
limit: this.config.maxFramesPerCall,
|
||||
});
|
||||
|
||||
const frameData = searchResults.map(r => ({
|
||||
id: r.frame.id,
|
||||
content: r.frame.content,
|
||||
created_at: r.frame.created_at,
|
||||
}));
|
||||
|
||||
if (frameData.length < this.config.minFramesPerPage) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Find related entities via KG
|
||||
const entityResults = this.kg.searchEntities(conceptName, 10);
|
||||
const relatedEntities = entityResults.map(e => e.name);
|
||||
|
||||
const prompt = conceptPagePrompt(conceptName, frameData, relatedEntities);
|
||||
const body = await this.config.synthesize(prompt);
|
||||
|
||||
const frameIds = frameData.map(f => f.id);
|
||||
const confidence = frameData.length > 5 ? 0.85 : 0.6;
|
||||
const slug = slugify(conceptName);
|
||||
|
||||
const frontmatter = buildFrontmatter('concept', conceptName, frameIds, relatedEntities, confidence);
|
||||
const markdown = `${frontmatter}\n\n# ${conceptName}\n\n${body}`;
|
||||
|
||||
return {
|
||||
slug,
|
||||
frontmatter: {
|
||||
type: 'concept',
|
||||
name: conceptName,
|
||||
confidence,
|
||||
sources: frameIds.length,
|
||||
last_compiled: new Date().toISOString(),
|
||||
frame_ids: frameIds,
|
||||
related_entities: relatedEntities,
|
||||
},
|
||||
markdown,
|
||||
contentHash: contentHash(markdown),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Synthesis Page ────────────────────────────────────────────
|
||||
|
||||
async compileSynthesisPage(topic: string): Promise<WikiPage | null> {
|
||||
const searchResults = await this.search.search(topic, {
|
||||
limit: this.config.maxFramesPerCall * 2, // more context for synthesis
|
||||
});
|
||||
|
||||
// Group frames by source to detect cross-source patterns
|
||||
const bySource = new Map<string, typeof searchResults>();
|
||||
for (const r of searchResults) {
|
||||
const source = r.frame.source ?? 'unknown';
|
||||
let group = bySource.get(source);
|
||||
if (!group) {
|
||||
group = [];
|
||||
bySource.set(source, group);
|
||||
}
|
||||
group.push(r);
|
||||
}
|
||||
|
||||
// Need frames from at least 2 sources for synthesis
|
||||
if (bySource.size < 2) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const crossSourceFrames = searchResults.slice(0, this.config.maxFramesPerCall).map(r => ({
|
||||
id: r.frame.id,
|
||||
content: r.frame.content,
|
||||
source: r.frame.source ?? 'unknown',
|
||||
created_at: r.frame.created_at,
|
||||
}));
|
||||
|
||||
const prompt = synthesisPagePrompt(topic, crossSourceFrames);
|
||||
const body = await this.config.synthesize(prompt);
|
||||
|
||||
const frameIds = crossSourceFrames.map(f => f.id);
|
||||
const confidence = bySource.size > 3 ? 0.85 : 0.65;
|
||||
const slug = `synthesis-${slugify(topic)}`;
|
||||
const sources = Array.from(bySource.keys());
|
||||
|
||||
const frontmatter = buildFrontmatter('synthesis', `Synthesis: ${topic}`, frameIds, sources, confidence);
|
||||
const markdown = `${frontmatter}\n\n# Synthesis: ${topic}\n\n${body}`;
|
||||
|
||||
return {
|
||||
slug,
|
||||
frontmatter: {
|
||||
type: 'synthesis',
|
||||
name: `Synthesis: ${topic}`,
|
||||
confidence,
|
||||
sources: frameIds.length,
|
||||
last_compiled: new Date().toISOString(),
|
||||
frame_ids: frameIds,
|
||||
related_entities: sources,
|
||||
},
|
||||
markdown,
|
||||
contentHash: contentHash(markdown),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Index Page ────────────────────────────────────────────────
|
||||
|
||||
compileIndex(): WikiPage {
|
||||
const allPages = this.state.getAllPages();
|
||||
|
||||
const entityPages = allPages.filter(p => p.pageType === 'entity');
|
||||
const conceptPages = allPages.filter(p => p.pageType === 'concept');
|
||||
const synthesisPages = allPages.filter(p => p.pageType === 'synthesis');
|
||||
|
||||
const lines: string[] = [
|
||||
'# Wiki Index',
|
||||
'',
|
||||
`*${allPages.length} pages compiled — last updated ${new Date().toISOString().slice(0, 10)}*`,
|
||||
'',
|
||||
];
|
||||
|
||||
if (entityPages.length > 0) {
|
||||
lines.push('## Entities', '');
|
||||
for (const p of entityPages) {
|
||||
lines.push(`- [[${p.name}]] — ${p.sourceCount} source${p.sourceCount === 1 ? '' : 's'} (${p.compiledAt.slice(0, 10)})`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
if (conceptPages.length > 0) {
|
||||
lines.push('## Concepts', '');
|
||||
for (const p of conceptPages) {
|
||||
lines.push(`- [[${p.name}]] — ${p.sourceCount} source${p.sourceCount === 1 ? '' : 's'} (${p.compiledAt.slice(0, 10)})`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
if (synthesisPages.length > 0) {
|
||||
lines.push('## Cross-Source Synthesis', '');
|
||||
for (const p of synthesisPages) {
|
||||
lines.push(`- [[${p.name}]] — ${p.sourceCount} source${p.sourceCount === 1 ? '' : 's'} (${p.compiledAt.slice(0, 10)})`);
|
||||
}
|
||||
lines.push('');
|
||||
}
|
||||
|
||||
const markdown = lines.join('\n');
|
||||
|
||||
return {
|
||||
slug: 'index',
|
||||
frontmatter: {
|
||||
type: 'index',
|
||||
name: 'Wiki Index',
|
||||
confidence: 1.0,
|
||||
sources: allPages.length,
|
||||
last_compiled: new Date().toISOString(),
|
||||
frame_ids: [],
|
||||
related_entities: [],
|
||||
},
|
||||
markdown,
|
||||
contentHash: contentHash(markdown),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Health Report ─────────────────────────────────────────────
|
||||
|
||||
compileHealth(): HealthReport {
|
||||
const issues: HealthIssue[] = [];
|
||||
const allPages = this.state.getAllPages();
|
||||
const entityCount = this.kg.getEntityCount();
|
||||
const frameStats = this.frames.getStats();
|
||||
|
||||
// Check for orphan entities (KG entities with no wiki page)
|
||||
const entities = this.kg.getEntities(1000);
|
||||
const pageNames = new Set(allPages.map(p => p.name.toLowerCase()));
|
||||
|
||||
for (const entity of entities) {
|
||||
if (!pageNames.has(entity.name.toLowerCase())) {
|
||||
// Check if entity has enough frames to justify a page
|
||||
const rels = this.kg.getRelationsFrom(entity.id);
|
||||
if (rels.length > 0 || entity.entity_type === 'person' || entity.entity_type === 'project') {
|
||||
issues.push({
|
||||
type: 'missing_page',
|
||||
severity: rels.length > 2 ? 'high' : 'medium',
|
||||
description: `Entity "${entity.name}" (${entity.entity_type}) has no wiki page`,
|
||||
entity: entity.name,
|
||||
suggestion: `Run: compileEntityPage("${entity.name}")`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for weak pages (few sources)
|
||||
for (const page of allPages) {
|
||||
if (page.sourceCount < 2 && page.pageType !== 'index') {
|
||||
issues.push({
|
||||
type: 'weak_confidence',
|
||||
severity: 'low',
|
||||
description: `Page "${page.name}" has only ${page.sourceCount} source(s)`,
|
||||
entity: page.name,
|
||||
suggestion: 'Gather more data about this topic',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check for orphan entities in KG (no relations at all)
|
||||
for (const entity of entities) {
|
||||
const outRels = this.kg.getRelationsFrom(entity.id);
|
||||
const inRels = this.kg.getRelationsTo(entity.id);
|
||||
if (outRels.length === 0 && inRels.length === 0) {
|
||||
issues.push({
|
||||
type: 'orphan_entity',
|
||||
severity: 'low',
|
||||
description: `Entity "${entity.name}" (${entity.entity_type}) has no relations`,
|
||||
entity: entity.name,
|
||||
suggestion: 'Consider adding relations or retiring this entity',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// M-14: stale_page check. A page is stale when its compiledAt is older
|
||||
// than STALE_DAYS ago AND the watermark shows new frames have arrived
|
||||
// since. The watermark-vs-frame-count heuristic avoids false positives
|
||||
// for pages whose topic genuinely hasn't changed — they're not stale,
|
||||
// just stable.
|
||||
const STALE_DAYS = 30;
|
||||
const staleCutoff = Date.now() - STALE_DAYS * 24 * 60 * 60 * 1000;
|
||||
const latestFrameId = Math.max(0, ...allPages.flatMap(p => {
|
||||
try { return (JSON.parse(p.frameIds) as number[]) ?? []; } catch { return []; }
|
||||
}));
|
||||
const newestIndexedFrameId = this.state.getWatermark().lastFrameId;
|
||||
const hasNewerFramesOverall = newestIndexedFrameId > latestFrameId;
|
||||
|
||||
for (const page of allPages) {
|
||||
if (page.pageType === 'index' || page.pageType === 'health') continue;
|
||||
const compiledMs = Date.parse(page.compiledAt);
|
||||
if (Number.isFinite(compiledMs) && compiledMs < staleCutoff && hasNewerFramesOverall) {
|
||||
issues.push({
|
||||
type: 'stale_page',
|
||||
severity: 'medium',
|
||||
description: `Page "${page.name}" was compiled ${Math.round((Date.now() - compiledMs) / (24 * 60 * 60 * 1000))} days ago and new frames have arrived since`,
|
||||
entity: page.name,
|
||||
suggestion: 'Recompile to refresh with recent context',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// M-14: coverage ratio — pages / compilable entities (entities with
|
||||
// ≥1 relation OR entity_type person/project, matching the missing_page
|
||||
// threshold above). Reports a 0-1 fraction; UI renders it as %.
|
||||
const compilableEntities = entities.filter(e => {
|
||||
const rels = this.kg.getRelationsFrom(e.id);
|
||||
return rels.length > 0 || e.entity_type === 'person' || e.entity_type === 'project';
|
||||
}).length;
|
||||
const coverage = compilableEntities > 0
|
||||
? Math.min(1, allPages.filter(p => p.pageType === 'entity').length / compilableEntities)
|
||||
: 0;
|
||||
const stalePageCount = issues.filter(i => i.type === 'stale_page').length;
|
||||
|
||||
// Data quality score
|
||||
const hasEntities = entityCount > 0 ? 20 : 0;
|
||||
const hasFrames = frameStats.total > 10 ? 20 : frameStats.total > 0 ? 10 : 0;
|
||||
const hasPages = allPages.length > 5 ? 20 : allPages.length > 0 ? 10 : 0;
|
||||
const lowIssues = issues.filter(i => i.severity === 'high').length;
|
||||
const issueDeduction = Math.min(40, lowIssues * 10);
|
||||
const dataQualityScore = Math.max(0, hasEntities + hasFrames + hasPages + 40 - issueDeduction);
|
||||
|
||||
return {
|
||||
totalEntities: entityCount,
|
||||
totalFrames: frameStats.total,
|
||||
totalPages: allPages.length,
|
||||
coverage,
|
||||
stalePageCount,
|
||||
issues,
|
||||
dataQualityScore,
|
||||
compiledAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Full Compilation ──────────────────────────────────────────
|
||||
|
||||
async compile(options?: { incremental?: boolean; concepts?: string[] }): Promise<CompilationResult> {
|
||||
const startTime = Date.now();
|
||||
const incremental = options?.incremental ?? true;
|
||||
const watermark = this.state.getWatermark();
|
||||
|
||||
let pagesCreated = 0;
|
||||
let pagesUpdated = 0;
|
||||
let pagesUnchanged = 0;
|
||||
const entityPageNames: string[] = [];
|
||||
const conceptPageNames: string[] = [];
|
||||
const synthesisPageNames: string[] = [];
|
||||
|
||||
// 1. Compile entity pages for all significant entities
|
||||
const entities = this.kg.getEntities(200);
|
||||
for (const entity of entities) {
|
||||
// Skip entities we've already compiled unless new frames exist
|
||||
if (incremental && watermark.lastFrameId > 0) {
|
||||
const existingPage = this.state.getPage(slugify(entity.name));
|
||||
if (existingPage) {
|
||||
// Check if new frames mention this entity
|
||||
const newFrames = this.state.getFramesSince(watermark.lastFrameId, 100);
|
||||
const mentionsEntity = newFrames.some(f =>
|
||||
f.content.toLowerCase().includes(entity.name.toLowerCase()),
|
||||
);
|
||||
if (!mentionsEntity) {
|
||||
pagesUnchanged++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const page = await this.compileEntityPage(entity);
|
||||
if (page) {
|
||||
const result = this.state.upsertPage(
|
||||
page.slug, 'entity', entity.name,
|
||||
page.contentHash, page.frontmatter.frame_ids, page.frontmatter.sources,
|
||||
page.markdown,
|
||||
);
|
||||
if (result.action === 'created') pagesCreated++;
|
||||
else if (result.action === 'updated') pagesUpdated++;
|
||||
else pagesUnchanged++;
|
||||
entityPageNames.push(entity.name);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Compile concept pages (user-specified or auto-detected)
|
||||
const concepts = options?.concepts ?? this.detectConcepts(entities);
|
||||
for (const concept of concepts) {
|
||||
const page = await this.compileConceptPage(concept);
|
||||
if (page) {
|
||||
const result = this.state.upsertPage(
|
||||
page.slug, 'concept', concept,
|
||||
page.contentHash, page.frontmatter.frame_ids, page.frontmatter.sources,
|
||||
page.markdown,
|
||||
);
|
||||
if (result.action === 'created') pagesCreated++;
|
||||
else if (result.action === 'updated') pagesUpdated++;
|
||||
else pagesUnchanged++;
|
||||
conceptPageNames.push(concept);
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Compile synthesis pages for topics with cross-source data
|
||||
for (const concept of concepts) {
|
||||
const page = await this.compileSynthesisPage(concept);
|
||||
if (page) {
|
||||
const result = this.state.upsertPage(
|
||||
page.slug, 'synthesis', `Synthesis: ${concept}`,
|
||||
page.contentHash, page.frontmatter.frame_ids, page.frontmatter.sources,
|
||||
page.markdown,
|
||||
);
|
||||
if (result.action === 'created') pagesCreated++;
|
||||
else if (result.action === 'updated') pagesUpdated++;
|
||||
else pagesUnchanged++;
|
||||
synthesisPageNames.push(concept);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Compile index
|
||||
const indexPage = this.compileIndex();
|
||||
this.state.upsertPage('index', 'index', 'Wiki Index', indexPage.contentHash, [], 0, indexPage.markdown);
|
||||
|
||||
// 5. Update watermark
|
||||
const maxFrameId = this.state.getMaxFrameId();
|
||||
const totalCompiled = pagesCreated + pagesUpdated;
|
||||
this.state.updateWatermark(maxFrameId, totalCompiled);
|
||||
|
||||
// 6. Health check
|
||||
const health = this.compileHealth();
|
||||
|
||||
return {
|
||||
pagesCreated,
|
||||
pagesUpdated,
|
||||
pagesUnchanged,
|
||||
entityPages: entityPageNames,
|
||||
conceptPages: conceptPageNames,
|
||||
synthesisPages: synthesisPageNames,
|
||||
healthIssues: health.issues.length,
|
||||
watermark: { lastFrameId: maxFrameId, lastCompiledAt: new Date().toISOString(), pagesCompiled: totalCompiled },
|
||||
durationMs: Date.now() - startTime,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Helpers ───────────────────────────────────────────────────
|
||||
|
||||
/** Auto-detect concepts from entity types and common topics. */
|
||||
private detectConcepts(entities: Entity[]): string[] {
|
||||
const typeCounts = new Map<string, number>();
|
||||
for (const e of entities) {
|
||||
typeCounts.set(e.entity_type, (typeCounts.get(e.entity_type) ?? 0) + 1);
|
||||
}
|
||||
|
||||
const concepts: string[] = [];
|
||||
|
||||
// Add entity types that have multiple entries as concepts
|
||||
for (const [type, count] of typeCounts) {
|
||||
if (count >= 3 && type !== 'concept') {
|
||||
concepts.push(type);
|
||||
}
|
||||
}
|
||||
|
||||
// Add 'concept' type entities directly as concepts
|
||||
const conceptEntities = entities.filter(e => e.entity_type === 'concept');
|
||||
for (const e of conceptEntities) {
|
||||
if (!concepts.includes(e.name)) {
|
||||
concepts.push(e.name);
|
||||
}
|
||||
}
|
||||
|
||||
return concepts.slice(0, 20); // Cap at 20 concepts
|
||||
}
|
||||
|
||||
/**
|
||||
* Export all wiki pages as a flat markdown bundle.
|
||||
* Returns a map of slug → markdown content, suitable for writing to disk.
|
||||
*/
|
||||
exportToMarkdown(): Map<string, string> {
|
||||
const pages = this.state.getAllPages();
|
||||
const result = new Map<string, string>();
|
||||
for (const page of pages) {
|
||||
result.set(page.slug, page.markdown || `# ${page.name}\n\n(no content compiled yet)`);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Export all wiki pages to a directory as individual .md files.
|
||||
* Creates the directory if it doesn't exist.
|
||||
*/
|
||||
async exportToDirectory(dir: string): Promise<{ filesWritten: number }> {
|
||||
const { mkdirSync, writeFileSync } = await import('node:fs');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const pages = this.exportToMarkdown();
|
||||
let count = 0;
|
||||
for (const [slug, markdown] of pages) {
|
||||
writeFileSync(`${dir}/${slug}.md`, markdown, 'utf-8');
|
||||
count++;
|
||||
}
|
||||
return { filesWritten: count };
|
||||
}
|
||||
}
|
||||
29
packages/wiki-compiler/src/index.ts
Normal file
29
packages/wiki-compiler/src/index.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
export { WikiCompiler } from './compiler.js';
|
||||
export { CompilationState, contentHash } from './state.js';
|
||||
export { resolveSynthesizer, type ResolvedSynthesizer, type SynthesizerConfig } from './synthesizer.js';
|
||||
export { entityPagePrompt, conceptPagePrompt, synthesisPagePrompt } from './prompts.js';
|
||||
export { writeToObsidianVault, type ObsidianExportResult } from './adapters/obsidian.js';
|
||||
export {
|
||||
writeToNotionWorkspace,
|
||||
markdownToBlocks,
|
||||
stripFrontmatter,
|
||||
toRichText,
|
||||
extractNotionPageId,
|
||||
type NotionExportOptions,
|
||||
type NotionExportStats,
|
||||
type NotionStateHelpers,
|
||||
type NotionBlock,
|
||||
} from './adapters/notion.js';
|
||||
export type {
|
||||
WikiPage,
|
||||
WikiPageType,
|
||||
WikiPageFrontmatter,
|
||||
CompilationWatermark,
|
||||
PageRecord,
|
||||
CompilerConfig,
|
||||
LLMSynthesizeFn,
|
||||
CompilationResult,
|
||||
HealthReport,
|
||||
HealthIssue,
|
||||
HealthIssueType,
|
||||
} from './types.js';
|
||||
94
packages/wiki-compiler/src/prompts.ts
Normal file
94
packages/wiki-compiler/src/prompts.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
/**
|
||||
* LLM prompts for wiki page compilation.
|
||||
*/
|
||||
|
||||
export function entityPagePrompt(
|
||||
entityName: string,
|
||||
entityType: string,
|
||||
frames: { id: number; content: string; created_at: string }[],
|
||||
relations: { target: string; relationType: string; confidence: number }[],
|
||||
): string {
|
||||
const frameList = frames
|
||||
.map(f => `[Frame #${f.id}, ${f.created_at}]: ${f.content}`)
|
||||
.join('\n\n');
|
||||
|
||||
const relationList = relations.length > 0
|
||||
? relations.map(r => `- ${r.target} (${r.relationType}, confidence: ${r.confidence})`).join('\n')
|
||||
: 'No relations found.';
|
||||
|
||||
return `You are a wiki compiler. Synthesize the following memory frames about "${entityName}" (${entityType}) into a wiki page.
|
||||
|
||||
## Source Frames (${frames.length} total)
|
||||
${frameList}
|
||||
|
||||
## Known Relations
|
||||
${relationList}
|
||||
|
||||
## Instructions
|
||||
1. Write a **Summary** section (2-3 sentences synthesizing all knowledge)
|
||||
2. Write a **Key Facts** section with bullet points citing frame IDs (e.g., "from frame #42")
|
||||
3. Write a **Timeline** section if temporal events are present (table: Date | Event | Source)
|
||||
4. Write a **Relations** section listing connected entities with [[wiki links]]
|
||||
5. Write an **Open Questions** section noting gaps or unresolved contradictions
|
||||
6. If frames contradict each other, note it in a **Contradictions** section with confidence assessment
|
||||
|
||||
Be concise. Cite frame IDs for every claim. Use markdown formatting.
|
||||
Output ONLY the page body (no frontmatter — that's added automatically).`;
|
||||
}
|
||||
|
||||
export function conceptPagePrompt(
|
||||
conceptName: string,
|
||||
frames: { id: number; content: string; created_at: string }[],
|
||||
relatedEntities: string[],
|
||||
): string {
|
||||
const frameList = frames
|
||||
.map(f => `[Frame #${f.id}, ${f.created_at}]: ${f.content}`)
|
||||
.join('\n\n');
|
||||
|
||||
const entityList = relatedEntities.length > 0
|
||||
? relatedEntities.map(e => `- [[${e}]]`).join('\n')
|
||||
: 'None identified.';
|
||||
|
||||
return `You are a wiki compiler. Synthesize the following memory frames about the concept "${conceptName}" into a wiki page.
|
||||
|
||||
## Source Frames (${frames.length} total)
|
||||
${frameList}
|
||||
|
||||
## Related Entities
|
||||
${entityList}
|
||||
|
||||
## Instructions
|
||||
1. Write a **TL;DR** (2 sentences max)
|
||||
2. Write a **What We Know** section synthesizing all frames
|
||||
3. Write a **Sources & Evolution** section showing how understanding evolved over time
|
||||
4. Write a **Related Topics** section with [[wiki links]]
|
||||
5. Note any gaps or areas needing more data
|
||||
|
||||
Be concise. Cite frame IDs. Use markdown formatting.
|
||||
Output ONLY the page body (no frontmatter).`;
|
||||
}
|
||||
|
||||
export function synthesisPagePrompt(
|
||||
topic: string,
|
||||
crossSourceFrames: { id: number; content: string; source: string; created_at: string }[],
|
||||
): string {
|
||||
const frameList = crossSourceFrames
|
||||
.map(f => `[Frame #${f.id}, source: ${f.source}, ${f.created_at}]: ${f.content}`)
|
||||
.join('\n\n');
|
||||
|
||||
return `You are a wiki compiler performing cross-source synthesis. Multiple independent sources discuss "${topic}". Find patterns, agreements, and contradictions.
|
||||
|
||||
## Frames from Multiple Sources (${crossSourceFrames.length} total)
|
||||
${frameList}
|
||||
|
||||
## Instructions
|
||||
1. Write a **Cross-Source Summary** — what do multiple sources agree on?
|
||||
2. Write a **Patterns** section — recurring themes across sources
|
||||
3. Write a **Contradictions** section — where sources disagree (with frame IDs)
|
||||
4. Write an **Insights** section — what can we conclude that no single source stated?
|
||||
5. Write a **Confidence Assessment** — how reliable is this synthesis?
|
||||
|
||||
This is the most valuable page type. Focus on insights that emerge from combining sources.
|
||||
Be concise. Cite frame IDs. Use markdown formatting.
|
||||
Output ONLY the page body (no frontmatter).`;
|
||||
}
|
||||
193
packages/wiki-compiler/src/state.ts
Normal file
193
packages/wiki-compiler/src/state.ts
Normal file
@@ -0,0 +1,193 @@
|
||||
/**
|
||||
* Compilation State Tracker — SQLite-backed watermarks and page hashes.
|
||||
*
|
||||
* Tracks what has been compiled and when, enabling incremental compilation.
|
||||
* Uses the same MindDB instance as the memory system.
|
||||
*/
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { MindDB } from '@waggle/core';
|
||||
import type { WikiPageType, CompilationWatermark, PageRecord } from './types.js';
|
||||
|
||||
const PAGES_TABLE = `
|
||||
CREATE TABLE IF NOT EXISTS wiki_pages (
|
||||
slug TEXT PRIMARY KEY,
|
||||
page_type TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
markdown TEXT NOT NULL DEFAULT '',
|
||||
frame_ids TEXT NOT NULL DEFAULT '[]',
|
||||
compiled_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
source_count INTEGER NOT NULL DEFAULT 0
|
||||
)`;
|
||||
|
||||
const WATERMARK_TABLE = `
|
||||
CREATE TABLE IF NOT EXISTS wiki_watermark (
|
||||
id INTEGER PRIMARY KEY CHECK (id = 1),
|
||||
last_frame_id INTEGER NOT NULL DEFAULT 0,
|
||||
last_compiled_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
pages_compiled INTEGER NOT NULL DEFAULT 0
|
||||
)`;
|
||||
|
||||
export function contentHash(content: string): string {
|
||||
return createHash('sha256').update(content).digest('hex').slice(0, 16);
|
||||
}
|
||||
|
||||
export class CompilationState {
|
||||
private db: MindDB;
|
||||
|
||||
constructor(db: MindDB) {
|
||||
this.db = db;
|
||||
this.ensureSchema();
|
||||
}
|
||||
|
||||
private ensureSchema(): void {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(PAGES_TABLE).run();
|
||||
raw.prepare(WATERMARK_TABLE).run();
|
||||
// Migration: add markdown column if missing (for databases created before v1.1)
|
||||
try {
|
||||
raw.prepare("SELECT markdown FROM wiki_pages LIMIT 0").get();
|
||||
} catch {
|
||||
raw.prepare("ALTER TABLE wiki_pages ADD COLUMN markdown TEXT NOT NULL DEFAULT ''").run();
|
||||
}
|
||||
// M-13: Notion export delta tracking — add notion_page_id column if
|
||||
// missing. Nullable because most pages haven't been exported.
|
||||
try {
|
||||
raw.prepare("SELECT notion_page_id FROM wiki_pages LIMIT 0").get();
|
||||
} catch {
|
||||
raw.prepare("ALTER TABLE wiki_pages ADD COLUMN notion_page_id TEXT").run();
|
||||
}
|
||||
}
|
||||
|
||||
/** M-13: read the Notion page id previously written for `slug`, or null. */
|
||||
getNotionPageId(slug: string): string | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT notion_page_id FROM wiki_pages WHERE slug = ?').get(slug) as { notion_page_id: string | null } | undefined;
|
||||
return row?.notion_page_id ?? null;
|
||||
}
|
||||
|
||||
/** M-13: record the Notion page id created for `slug`. */
|
||||
setNotionPageId(slug: string, pageId: string): void {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare('UPDATE wiki_pages SET notion_page_id = ? WHERE slug = ?').run(pageId, slug);
|
||||
}
|
||||
|
||||
/** M-13: clear stored Notion page id (used when we archive + recreate on change). */
|
||||
clearNotionPageId(slug: string): void {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare('UPDATE wiki_pages SET notion_page_id = NULL WHERE slug = ?').run(slug);
|
||||
}
|
||||
|
||||
/** M-13: get a page's current content_hash (used by Notion exporter for delta). */
|
||||
getPageContentHash(slug: string): string | null {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT content_hash FROM wiki_pages WHERE slug = ?').get(slug) as { content_hash: string } | undefined;
|
||||
return row?.content_hash ?? null;
|
||||
}
|
||||
|
||||
// ── Watermark ─────────────────────────────────────────────────
|
||||
|
||||
getWatermark(): CompilationWatermark {
|
||||
const raw = this.db.getDatabase();
|
||||
const row = raw.prepare('SELECT * FROM wiki_watermark WHERE id = 1').get() as {
|
||||
last_frame_id: number;
|
||||
last_compiled_at: string;
|
||||
pages_compiled: number;
|
||||
} | undefined;
|
||||
|
||||
if (!row) {
|
||||
return { lastFrameId: 0, lastCompiledAt: '', pagesCompiled: 0 };
|
||||
}
|
||||
|
||||
return {
|
||||
lastFrameId: row.last_frame_id,
|
||||
lastCompiledAt: row.last_compiled_at,
|
||||
pagesCompiled: row.pages_compiled,
|
||||
};
|
||||
}
|
||||
|
||||
updateWatermark(lastFrameId: number, pagesCompiled: number): void {
|
||||
const raw = this.db.getDatabase();
|
||||
raw.prepare(`
|
||||
INSERT INTO wiki_watermark (id, last_frame_id, last_compiled_at, pages_compiled)
|
||||
VALUES (1, ?, datetime('now'), ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
last_frame_id = excluded.last_frame_id,
|
||||
last_compiled_at = excluded.last_compiled_at,
|
||||
pages_compiled = excluded.pages_compiled
|
||||
`).run(lastFrameId, pagesCompiled);
|
||||
}
|
||||
|
||||
// ── Page Records ──────────────────────────────────────────────
|
||||
|
||||
getPage(slug: string): PageRecord | undefined {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT slug, page_type as pageType, name, content_hash as contentHash, markdown, frame_ids as frameIds, compiled_at as compiledAt, source_count as sourceCount FROM wiki_pages WHERE slug = ?',
|
||||
).get(slug) as PageRecord | undefined;
|
||||
}
|
||||
|
||||
getAllPages(): PageRecord[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT slug, page_type as pageType, name, content_hash as contentHash, markdown, frame_ids as frameIds, compiled_at as compiledAt, source_count as sourceCount FROM wiki_pages ORDER BY name',
|
||||
).all() as PageRecord[];
|
||||
}
|
||||
|
||||
getPagesByType(pageType: WikiPageType): PageRecord[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT slug, page_type as pageType, name, content_hash as contentHash, markdown, frame_ids as frameIds, compiled_at as compiledAt, source_count as sourceCount FROM wiki_pages WHERE page_type = ? ORDER BY name',
|
||||
).all(pageType) as PageRecord[];
|
||||
}
|
||||
|
||||
upsertPage(
|
||||
slug: string,
|
||||
pageType: WikiPageType,
|
||||
name: string,
|
||||
hash: string,
|
||||
frameIds: number[],
|
||||
sourceCount: number,
|
||||
markdown = '',
|
||||
): { action: 'created' | 'updated' | 'unchanged' } {
|
||||
const raw = this.db.getDatabase();
|
||||
const existing = this.getPage(slug);
|
||||
|
||||
if (existing && existing.contentHash === hash) {
|
||||
return { action: 'unchanged' };
|
||||
}
|
||||
|
||||
raw.prepare(`
|
||||
INSERT INTO wiki_pages (slug, page_type, name, content_hash, markdown, frame_ids, compiled_at, source_count)
|
||||
VALUES (?, ?, ?, ?, ?, ?, datetime('now'), ?)
|
||||
ON CONFLICT(slug) DO UPDATE SET
|
||||
content_hash = excluded.content_hash,
|
||||
markdown = excluded.markdown,
|
||||
frame_ids = excluded.frame_ids,
|
||||
compiled_at = excluded.compiled_at,
|
||||
source_count = excluded.source_count
|
||||
`).run(slug, pageType, name, hash, markdown, JSON.stringify(frameIds), sourceCount);
|
||||
|
||||
return { action: existing ? 'updated' : 'created' };
|
||||
}
|
||||
|
||||
deletePage(slug: string): boolean {
|
||||
const result = this.db.getDatabase().prepare(
|
||||
'DELETE FROM wiki_pages WHERE slug = ?',
|
||||
).run(slug);
|
||||
return result.changes > 0;
|
||||
}
|
||||
|
||||
/** Get the highest frame ID in the database. */
|
||||
getMaxFrameId(): number {
|
||||
const row = this.db.getDatabase().prepare(
|
||||
'SELECT COALESCE(MAX(id), 0) as max_id FROM memory_frames',
|
||||
).get() as { max_id: number };
|
||||
return row.max_id;
|
||||
}
|
||||
|
||||
/** Get frames newer than a given ID. */
|
||||
getFramesSince(frameId: number, limit = 500): { id: number; content: string; importance: string; source: string; created_at: string }[] {
|
||||
return this.db.getDatabase().prepare(
|
||||
'SELECT id, content, importance, source, created_at FROM memory_frames WHERE id > ? ORDER BY id ASC LIMIT ?',
|
||||
).all(frameId, limit) as { id: number; content: string; importance: string; source: string; created_at: string }[];
|
||||
}
|
||||
}
|
||||
156
packages/wiki-compiler/src/synthesizer.ts
Normal file
156
packages/wiki-compiler/src/synthesizer.ts
Normal file
@@ -0,0 +1,156 @@
|
||||
/**
|
||||
* LLM Synthesizer — resolves the best available LLM for wiki page synthesis.
|
||||
*
|
||||
* Priority chain:
|
||||
* 1. Anthropic Haiku (cheapest, fastest, best for synthesis)
|
||||
* 2. Ollama (free, local)
|
||||
* 3. Echo (no LLM, returns structured stub)
|
||||
*
|
||||
* All synthesizers implement LLMSynthesizeFn: (prompt: string) => Promise<string>
|
||||
*/
|
||||
|
||||
import type { LLMSynthesizeFn } from './types.js';
|
||||
|
||||
export interface SynthesizerConfig {
|
||||
/** Anthropic API key. Checked from env if not provided. */
|
||||
anthropicApiKey?: string;
|
||||
/** Ollama base URL. Checked from env if not provided. */
|
||||
ollamaUrl?: string;
|
||||
/** Ollama model name (default: llama3.2) */
|
||||
ollamaModel?: string;
|
||||
/** Max tokens for synthesis output (default: 1500) */
|
||||
maxTokens?: number;
|
||||
}
|
||||
|
||||
// ── Anthropic (Haiku) ───────────────────────────────────────────
|
||||
|
||||
function createAnthropicSynthesizer(apiKey: string, maxTokens: number): LLMSynthesizeFn {
|
||||
return async (prompt: string): Promise<string> => {
|
||||
// Dynamic import — @anthropic-ai/sdk is optional
|
||||
const { default: Anthropic } = await import('@anthropic-ai/sdk');
|
||||
const client = new Anthropic({ apiKey });
|
||||
|
||||
const response = await client.messages.create({
|
||||
model: 'claude-haiku-4-5-20251001',
|
||||
max_tokens: maxTokens,
|
||||
messages: [{ role: 'user', content: prompt }],
|
||||
});
|
||||
|
||||
const textBlock = response.content.find(b => b.type === 'text');
|
||||
return textBlock?.text ?? '';
|
||||
};
|
||||
}
|
||||
|
||||
// ── Ollama ──────────────────────────────────────────────────────
|
||||
|
||||
function createOllamaSynthesizer(baseUrl: string, model: string, maxTokens: number): LLMSynthesizeFn {
|
||||
return async (prompt: string): Promise<string> => {
|
||||
const url = `${baseUrl.replace(/\/$/, '')}/api/generate`;
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
prompt,
|
||||
stream: false,
|
||||
options: { num_predict: maxTokens },
|
||||
}),
|
||||
signal: AbortSignal.timeout(60_000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Ollama error: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json() as { response: string };
|
||||
return data.response ?? '';
|
||||
};
|
||||
}
|
||||
|
||||
// ── Echo (fallback) ─────────────────────────────────────────────
|
||||
|
||||
function createEchoSynthesizer(): LLMSynthesizeFn {
|
||||
return async (prompt: string): Promise<string> => {
|
||||
const frameMatch = prompt.match(/\((\d+) total\)/);
|
||||
const frameCount = frameMatch ? frameMatch[1] : '?';
|
||||
const nameMatch = prompt.match(/about "([^"]+)"/) ?? prompt.match(/concept "([^"]+)"/);
|
||||
const name = nameMatch ? nameMatch[1] : 'this topic';
|
||||
|
||||
// Extract actual frame content for a basic summary
|
||||
const frameLines = (prompt.match(/\[Frame #\d+.*?\]: .+/g) || []);
|
||||
const facts = frameLines.slice(0, 8).map(line => {
|
||||
const m = line.match(/\[Frame (#\d+).*?\]: (.+)/);
|
||||
return m ? `- ${m[2].slice(0, 200)} *(${m[1]})*` : null;
|
||||
}).filter(Boolean);
|
||||
|
||||
return `## Summary\nCompiled from ${frameCount} source frames about ${name}.\n\n` +
|
||||
(facts.length > 0 ? `## Key Facts\n${facts.join('\n')}\n\n` : '') +
|
||||
`> *Compiled with echo synthesizer. Connect an LLM for richer synthesis.*\n` +
|
||||
`> Set ANTHROPIC_API_KEY or WAGGLE_OLLAMA_URL in your environment.`;
|
||||
};
|
||||
}
|
||||
|
||||
// ── Resolver ────────────────────────────────────────────────────
|
||||
|
||||
export interface ResolvedSynthesizer {
|
||||
synthesize: LLMSynthesizeFn;
|
||||
provider: 'anthropic' | 'ollama' | 'echo';
|
||||
model: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the best available LLM synthesizer.
|
||||
* Checks env vars and config, returns the first working option.
|
||||
*/
|
||||
export async function resolveSynthesizer(config?: SynthesizerConfig): Promise<ResolvedSynthesizer> {
|
||||
const maxTokens = config?.maxTokens ?? 1500;
|
||||
|
||||
// 1. Try Anthropic
|
||||
const anthropicKey = config?.anthropicApiKey
|
||||
?? process.env.ANTHROPIC_API_KEY
|
||||
?? process.env.WAGGLE_ANTHROPIC_API_KEY;
|
||||
|
||||
if (anthropicKey) {
|
||||
try {
|
||||
// Verify the SDK is importable
|
||||
await import('@anthropic-ai/sdk');
|
||||
return {
|
||||
synthesize: createAnthropicSynthesizer(anthropicKey, maxTokens),
|
||||
provider: 'anthropic',
|
||||
model: 'claude-haiku-4-5-20251001',
|
||||
};
|
||||
} catch {
|
||||
// SDK not available — fall through
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Try Ollama
|
||||
const ollamaUrl = config?.ollamaUrl ?? process.env.WAGGLE_OLLAMA_URL;
|
||||
const ollamaModel = config?.ollamaModel ?? process.env.WAGGLE_OLLAMA_MODEL ?? 'llama3.2';
|
||||
|
||||
if (ollamaUrl) {
|
||||
try {
|
||||
// Quick health check
|
||||
const health = await fetch(`${ollamaUrl.replace(/\/$/, '')}/api/tags`, {
|
||||
signal: AbortSignal.timeout(3_000),
|
||||
});
|
||||
if (health.ok) {
|
||||
return {
|
||||
synthesize: createOllamaSynthesizer(ollamaUrl, ollamaModel, maxTokens),
|
||||
provider: 'ollama',
|
||||
model: ollamaModel,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Ollama not reachable — fall through
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Echo fallback
|
||||
return {
|
||||
synthesize: createEchoSynthesizer(),
|
||||
provider: 'echo',
|
||||
model: 'echo',
|
||||
};
|
||||
}
|
||||
119
packages/wiki-compiler/src/types.ts
Normal file
119
packages/wiki-compiler/src/types.ts
Normal file
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* Wiki Compiler Types — page definitions, compilation config, and state tracking.
|
||||
*/
|
||||
|
||||
// ── Page Types ──────────────────────────────────────────────────
|
||||
|
||||
export type WikiPageType =
|
||||
| 'entity'
|
||||
| 'concept'
|
||||
| 'synthesis'
|
||||
| 'index'
|
||||
| 'health';
|
||||
|
||||
export interface WikiPageFrontmatter {
|
||||
type: WikiPageType;
|
||||
name: string;
|
||||
entity_type?: string;
|
||||
confidence: number;
|
||||
sources: number;
|
||||
last_compiled: string;
|
||||
frame_ids: number[];
|
||||
related_entities: string[];
|
||||
}
|
||||
|
||||
export interface WikiPage {
|
||||
/** URL-safe slug, e.g. "project-alpha" */
|
||||
slug: string;
|
||||
/** Page frontmatter */
|
||||
frontmatter: WikiPageFrontmatter;
|
||||
/** Full markdown content (including frontmatter as YAML) */
|
||||
markdown: string;
|
||||
/** SHA-256 hash of content for change detection */
|
||||
contentHash: string;
|
||||
}
|
||||
|
||||
// ── Compilation State ───────────────────────────────────────────
|
||||
|
||||
export interface CompilationWatermark {
|
||||
/** Highest frame ID processed in last compilation */
|
||||
lastFrameId: number;
|
||||
/** ISO timestamp of last compilation */
|
||||
lastCompiledAt: string;
|
||||
/** Number of pages generated/updated */
|
||||
pagesCompiled: number;
|
||||
}
|
||||
|
||||
export interface PageRecord {
|
||||
slug: string;
|
||||
pageType: WikiPageType;
|
||||
name: string;
|
||||
contentHash: string;
|
||||
markdown: string;
|
||||
frameIds: string; // JSON array
|
||||
compiledAt: string;
|
||||
sourceCount: number;
|
||||
}
|
||||
|
||||
// ── Compiler Configuration ──────────────────────────────────────
|
||||
|
||||
export type LLMSynthesizeFn = (prompt: string) => Promise<string>;
|
||||
|
||||
export interface CompilerConfig {
|
||||
/** Function to call the LLM for synthesis */
|
||||
synthesize: LLMSynthesizeFn;
|
||||
/** Output directory for wiki pages (default: wiki/) */
|
||||
outputDir?: string;
|
||||
/** Minimum frames to justify a page (default: 2) */
|
||||
minFramesPerPage?: number;
|
||||
/** Maximum frames to send as context per LLM call (default: 30) */
|
||||
maxFramesPerCall?: number;
|
||||
/** Minimum confidence for entity pages (default: 0.3) */
|
||||
minConfidence?: number;
|
||||
}
|
||||
|
||||
// ── Compilation Result ──────────────────────────────────────────
|
||||
|
||||
export interface CompilationResult {
|
||||
pagesCreated: number;
|
||||
pagesUpdated: number;
|
||||
pagesUnchanged: number;
|
||||
entityPages: string[];
|
||||
conceptPages: string[];
|
||||
synthesisPages: string[];
|
||||
healthIssues: number;
|
||||
watermark: CompilationWatermark;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
// ── Health Report ───────────────────────────────────────────────
|
||||
|
||||
export type HealthIssueType =
|
||||
| 'contradiction'
|
||||
| 'gap'
|
||||
| 'orphan_entity'
|
||||
| 'weak_confidence'
|
||||
| 'stale_page'
|
||||
| 'missing_page';
|
||||
|
||||
export interface HealthIssue {
|
||||
type: HealthIssueType;
|
||||
severity: 'high' | 'medium' | 'low';
|
||||
description: string;
|
||||
entity?: string;
|
||||
frameIds?: number[];
|
||||
suggestion?: string;
|
||||
}
|
||||
|
||||
export interface HealthReport {
|
||||
totalEntities: number;
|
||||
totalFrames: number;
|
||||
totalPages: number;
|
||||
/** M-14: entity pages / compilable entities, 0..1. UI renders as %. */
|
||||
coverage: number;
|
||||
/** M-14: count of pages flagged with `stale_page` issue. */
|
||||
stalePageCount: number;
|
||||
issues: HealthIssue[];
|
||||
dataQualityScore: number; // 0-100
|
||||
compiledAt: string;
|
||||
}
|
||||
169
packages/wiki-compiler/tests/notion.test.ts
Normal file
169
packages/wiki-compiler/tests/notion.test.ts
Normal file
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* Notion adapter unit tests (M-13)
|
||||
*
|
||||
* Pure-function coverage for the markdown→blocks converter, rich-text
|
||||
* builder, frontmatter stripper, and page-id extractor. Network paths
|
||||
* (`createNotionPage`, `writeToNotionWorkspace`) are not unit-tested —
|
||||
* they need a mocked fetch and belong in an integration test.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import {
|
||||
markdownToBlocks,
|
||||
toRichText,
|
||||
stripFrontmatter,
|
||||
extractNotionPageId,
|
||||
type NotionBlock,
|
||||
type NotionBlockPayload,
|
||||
} from '../src/adapters/notion.js';
|
||||
|
||||
/**
|
||||
* Read the rich-text payload a block carries under its dynamic `block.type`
|
||||
* key. The block stores it as `unknown` (open-ended Notion shape), so narrow
|
||||
* it to the known {@link NotionBlockPayload} at the test boundary.
|
||||
*/
|
||||
function payloadOf(block: NotionBlock): NotionBlockPayload {
|
||||
return block[block.type] as NotionBlockPayload;
|
||||
}
|
||||
|
||||
describe('stripFrontmatter', () => {
|
||||
it('removes a leading YAML block and extracts name', () => {
|
||||
const input = `---
|
||||
type: entity
|
||||
name: Project Alpha
|
||||
confidence: 0.9
|
||||
---
|
||||
|
||||
# Body starts here`;
|
||||
const out = stripFrontmatter(input);
|
||||
expect(out.title).toBe('Project Alpha');
|
||||
expect(out.body.trimStart()).toBe('# Body starts here');
|
||||
});
|
||||
|
||||
it('returns full markdown and no title when frontmatter is absent', () => {
|
||||
const input = `# Just a header\n\nNo frontmatter.`;
|
||||
const out = stripFrontmatter(input);
|
||||
expect(out.title).toBeUndefined();
|
||||
expect(out.body).toBe(input);
|
||||
});
|
||||
|
||||
it('handles frontmatter without a name field', () => {
|
||||
const input = `---\ntype: concept\n---\n\nBody`;
|
||||
const out = stripFrontmatter(input);
|
||||
expect(out.title).toBeUndefined();
|
||||
expect(out.body.trimStart()).toBe('Body');
|
||||
});
|
||||
});
|
||||
|
||||
describe('toRichText', () => {
|
||||
it('returns empty array for empty input', () => {
|
||||
expect(toRichText('')).toEqual([]);
|
||||
});
|
||||
|
||||
it('wraps plain text in a single rich_text item', () => {
|
||||
const out = toRichText('Hello world');
|
||||
expect(out).toEqual([{ type: 'text', text: { content: 'Hello world' } }]);
|
||||
});
|
||||
|
||||
it('converts [text](url) into a linked rich_text item', () => {
|
||||
const out = toRichText('See [Notion](https://notion.so) docs.');
|
||||
expect(out).toHaveLength(3);
|
||||
expect(out[0]).toEqual({ type: 'text', text: { content: 'See ' } });
|
||||
expect(out[1]).toEqual({
|
||||
type: 'text',
|
||||
text: { content: 'Notion', link: { url: 'https://notion.so' } },
|
||||
});
|
||||
expect(out[2]).toEqual({ type: 'text', text: { content: ' docs.' } });
|
||||
});
|
||||
|
||||
it('applies bold, italic, and code annotations', () => {
|
||||
const out = toRichText('Plain **bold** *italic* `code` end');
|
||||
// split produces: "Plain ", "**bold**", " ", "*italic*", " ", "`code`", " end"
|
||||
const bold = out.find(t => t.text.content === 'bold');
|
||||
const italic = out.find(t => t.text.content === 'italic');
|
||||
const code = out.find(t => t.text.content === 'code');
|
||||
expect(bold?.annotations?.bold).toBe(true);
|
||||
expect(italic?.annotations?.italic).toBe(true);
|
||||
expect(code?.annotations?.code).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('markdownToBlocks', () => {
|
||||
it('maps H1/H2/H3 to heading_1/heading_2/heading_3 blocks', () => {
|
||||
const md = `# H1 Heading\n\n## H2 Heading\n\n### H3 Heading`;
|
||||
const blocks = markdownToBlocks(md);
|
||||
expect(blocks).toHaveLength(3);
|
||||
expect(blocks[0].type).toBe('heading_1');
|
||||
expect(blocks[1].type).toBe('heading_2');
|
||||
expect(blocks[2].type).toBe('heading_3');
|
||||
expect(payloadOf(blocks[0]).rich_text[0].text.content).toBe('H1 Heading');
|
||||
});
|
||||
|
||||
it('maps "- item" and "* item" bullets to bulleted_list_item', () => {
|
||||
const md = `- First\n- Second\n* Third`;
|
||||
const blocks = markdownToBlocks(md);
|
||||
expect(blocks).toHaveLength(3);
|
||||
for (const b of blocks) expect(b.type).toBe('bulleted_list_item');
|
||||
expect(payloadOf(blocks[0]).rich_text[0].text.content).toBe('First');
|
||||
});
|
||||
|
||||
it('maps "> quote" to quote block', () => {
|
||||
const md = `> A quote line`;
|
||||
const blocks = markdownToBlocks(md);
|
||||
expect(blocks).toHaveLength(1);
|
||||
expect(blocks[0].type).toBe('quote');
|
||||
expect(payloadOf(blocks[0]).rich_text[0].text.content).toBe('A quote line');
|
||||
});
|
||||
|
||||
it('consolidates adjacent non-special lines into a single paragraph', () => {
|
||||
const md = `First line.\nSecond line continues.\n\nNew paragraph here.`;
|
||||
const blocks = markdownToBlocks(md);
|
||||
expect(blocks).toHaveLength(2);
|
||||
expect(blocks[0].type).toBe('paragraph');
|
||||
expect(blocks[1].type).toBe('paragraph');
|
||||
expect(payloadOf(blocks[0]).rich_text[0].text.content).toBe('First line. Second line continues.');
|
||||
});
|
||||
|
||||
it('flushes paragraphs when a heading interrupts the block', () => {
|
||||
const md = `A paragraph line.\n# Heading\nNext paragraph.`;
|
||||
const blocks = markdownToBlocks(md);
|
||||
expect(blocks.map(b => b.type)).toEqual(['paragraph', 'heading_1', 'paragraph']);
|
||||
});
|
||||
|
||||
it('preserves inline links inside paragraphs', () => {
|
||||
const md = `See [Notion](https://notion.so) for more.`;
|
||||
const blocks = markdownToBlocks(md);
|
||||
expect(blocks[0].type).toBe('paragraph');
|
||||
const richText = payloadOf(blocks[0]).rich_text;
|
||||
const linkItem = richText.find((t) => t.text.link);
|
||||
expect(linkItem?.text.content).toBe('Notion');
|
||||
expect(linkItem?.text.link?.url).toBe('https://notion.so');
|
||||
});
|
||||
|
||||
it('returns empty array for empty input', () => {
|
||||
expect(markdownToBlocks('')).toEqual([]);
|
||||
expect(markdownToBlocks(' \n\n ')).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractNotionPageId', () => {
|
||||
it('accepts a dashed UUID', () => {
|
||||
const id = '12345678-1234-1234-1234-123456789abc';
|
||||
expect(extractNotionPageId(id)).toBe(id);
|
||||
});
|
||||
|
||||
it('accepts an undashed 32-hex id', () => {
|
||||
const id = '123456781234123412341234567890ab';
|
||||
expect(extractNotionPageId(id)).toBe(id);
|
||||
});
|
||||
|
||||
it('extracts the id from a notion URL', () => {
|
||||
const url = 'https://www.notion.so/Workspace/Some-Page-Title-123456781234123412341234567890ab';
|
||||
expect(extractNotionPageId(url)).toBe('123456781234123412341234567890ab');
|
||||
});
|
||||
|
||||
it('returns null for a non-hex input', () => {
|
||||
expect(extractNotionPageId('not-a-notion-page')).toBeNull();
|
||||
expect(extractNotionPageId('')).toBeNull();
|
||||
});
|
||||
});
|
||||
210
packages/wiki-compiler/tests/obsidian.test.ts
Normal file
210
packages/wiki-compiler/tests/obsidian.test.ts
Normal file
@@ -0,0 +1,210 @@
|
||||
/**
|
||||
* Obsidian adapter unit tests (M-12).
|
||||
*
|
||||
* Covers: filesystem layout, wikilink alias transform, index generation,
|
||||
* idempotent re-run (files overwritten cleanly), skip of index/health pages.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import * as os from 'node:os';
|
||||
import { writeToObsidianVault } from '../src/adapters/obsidian.js';
|
||||
import type { PageRecord } from '../src/types.js';
|
||||
|
||||
function seedPages(): PageRecord[] {
|
||||
return [
|
||||
{
|
||||
slug: 'project-alpha',
|
||||
pageType: 'entity',
|
||||
name: 'Project Alpha',
|
||||
contentHash: 'h1',
|
||||
markdown: `---
|
||||
type: entity
|
||||
name: Project Alpha
|
||||
confidence: 0.9
|
||||
---
|
||||
|
||||
# Project Alpha
|
||||
|
||||
Project Alpha is led by [[Marko]] and involves [[Egzakta Advisory]].
|
||||
|
||||
See also [[Strategy Consulting]] for broader context.
|
||||
`,
|
||||
frameIds: '[1,2,3]',
|
||||
compiledAt: '2026-04-20T00:00:00.000Z',
|
||||
sourceCount: 3,
|
||||
},
|
||||
{
|
||||
slug: 'marko',
|
||||
pageType: 'entity',
|
||||
name: 'Marko',
|
||||
contentHash: 'h2',
|
||||
markdown: `---
|
||||
type: entity
|
||||
name: Marko
|
||||
---
|
||||
|
||||
# Marko
|
||||
|
||||
The lead consultant.`,
|
||||
frameIds: '[1]',
|
||||
compiledAt: '2026-04-20T00:00:00.000Z',
|
||||
sourceCount: 1,
|
||||
},
|
||||
{
|
||||
slug: 'egzakta-advisory',
|
||||
pageType: 'entity',
|
||||
name: 'Egzakta Advisory',
|
||||
contentHash: 'h3',
|
||||
markdown: `---
|
||||
type: entity
|
||||
name: Egzakta Advisory
|
||||
---
|
||||
|
||||
# Egzakta Advisory
|
||||
|
||||
Strategy consulting firm.`,
|
||||
frameIds: '[2]',
|
||||
compiledAt: '2026-04-20T00:00:00.000Z',
|
||||
sourceCount: 1,
|
||||
},
|
||||
{
|
||||
slug: 'strategy-consulting',
|
||||
pageType: 'concept',
|
||||
name: 'Strategy Consulting',
|
||||
contentHash: 'h4',
|
||||
markdown: `---
|
||||
type: concept
|
||||
---
|
||||
|
||||
# Strategy Consulting
|
||||
|
||||
Services offered.`,
|
||||
frameIds: '[3]',
|
||||
compiledAt: '2026-04-20T00:00:00.000Z',
|
||||
sourceCount: 1,
|
||||
},
|
||||
{
|
||||
// Virtual index page — should be SKIPPED by the writer.
|
||||
slug: 'index',
|
||||
pageType: 'index',
|
||||
name: 'Wiki Index',
|
||||
contentHash: 'h5',
|
||||
markdown: 'should not be written',
|
||||
frameIds: '[]',
|
||||
compiledAt: '2026-04-20T00:00:00.000Z',
|
||||
sourceCount: 0,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
describe('writeToObsidianVault', () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-obsidian-test-'));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('writes one file per non-virtual page in a per-type subdirectory', () => {
|
||||
const result = writeToObsidianVault(seedPages(), tmpDir);
|
||||
|
||||
expect(result.outDir).toBe(tmpDir);
|
||||
// 3 entities + 1 concept + 1 _index.md = 5 total (index/health skipped)
|
||||
expect(result.filesWritten).toBe(5);
|
||||
expect(result.byType.entity).toBe(3);
|
||||
expect(result.byType.concept).toBe(1);
|
||||
expect(result.byType.index).toBeUndefined();
|
||||
|
||||
expect(fs.existsSync(path.join(tmpDir, 'entity', 'project-alpha.md'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(tmpDir, 'entity', 'marko.md'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(tmpDir, 'entity', 'egzakta-advisory.md'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(tmpDir, 'concept', 'strategy-consulting.md'))).toBe(true);
|
||||
expect(fs.existsSync(path.join(tmpDir, '_index.md'))).toBe(true);
|
||||
});
|
||||
|
||||
it('transforms [[Display Name]] wikilinks to [[slug|Display Name]]', () => {
|
||||
writeToObsidianVault(seedPages(), tmpDir);
|
||||
const content = fs.readFileSync(path.join(tmpDir, 'entity', 'project-alpha.md'), 'utf-8');
|
||||
|
||||
// Linked-by-name entities get the alias form.
|
||||
expect(content).toContain('[[marko|Marko]]');
|
||||
expect(content).toContain('[[egzakta-advisory|Egzakta Advisory]]');
|
||||
expect(content).toContain('[[strategy-consulting|Strategy Consulting]]');
|
||||
// Raw display-name wikilinks should NOT remain.
|
||||
expect(content).not.toContain('[[Marko]]');
|
||||
expect(content).not.toContain('[[Egzakta Advisory]]');
|
||||
});
|
||||
|
||||
it('leaves already-slug wikilinks alone', () => {
|
||||
const pages: PageRecord[] = [{
|
||||
slug: 'main',
|
||||
pageType: 'entity',
|
||||
name: 'Main',
|
||||
contentHash: 'h',
|
||||
markdown: `# Main\n\nLinks to [[marko]] which is already a slug.`,
|
||||
frameIds: '[]',
|
||||
compiledAt: '2026-04-20T00:00:00.000Z',
|
||||
sourceCount: 1,
|
||||
}, {
|
||||
slug: 'marko',
|
||||
pageType: 'entity',
|
||||
name: 'Marko',
|
||||
contentHash: 'h2',
|
||||
markdown: '# Marko',
|
||||
frameIds: '[]',
|
||||
compiledAt: '2026-04-20T00:00:00.000Z',
|
||||
sourceCount: 1,
|
||||
}];
|
||||
writeToObsidianVault(pages, tmpDir);
|
||||
const content = fs.readFileSync(path.join(tmpDir, 'entity', 'main.md'), 'utf-8');
|
||||
expect(content).toContain('[[marko]]');
|
||||
expect(content).not.toContain('[[marko|marko]]');
|
||||
});
|
||||
|
||||
it('writes _index.md with pages grouped by type', () => {
|
||||
writeToObsidianVault(seedPages(), tmpDir);
|
||||
const indexContent = fs.readFileSync(path.join(tmpDir, '_index.md'), 'utf-8');
|
||||
|
||||
expect(indexContent).toContain('# Waggle Wiki Index');
|
||||
expect(indexContent).toContain('## Entities (3)');
|
||||
expect(indexContent).toContain('## Concepts (1)');
|
||||
expect(indexContent).toContain('[[project-alpha|Project Alpha]]');
|
||||
expect(indexContent).toContain('[[strategy-consulting|Strategy Consulting]]');
|
||||
// The virtual index page should NOT appear in the index itself.
|
||||
expect(indexContent).not.toContain('[[index|');
|
||||
});
|
||||
|
||||
it('preserves YAML frontmatter as-is', () => {
|
||||
writeToObsidianVault(seedPages(), tmpDir);
|
||||
const content = fs.readFileSync(path.join(tmpDir, 'entity', 'project-alpha.md'), 'utf-8');
|
||||
expect(content).toMatch(/^---\n/);
|
||||
expect(content).toMatch(/type: entity\nname: Project Alpha\nconfidence: 0\.9\n---/);
|
||||
});
|
||||
|
||||
it('is idempotent — re-running overwrites files cleanly', () => {
|
||||
writeToObsidianVault(seedPages(), tmpDir);
|
||||
const first = fs.readFileSync(path.join(tmpDir, 'entity', 'project-alpha.md'), 'utf-8');
|
||||
|
||||
// Mutate one page's markdown and rerun.
|
||||
const pages = seedPages();
|
||||
pages[0].markdown = pages[0].markdown.replace('Strategy Consulting', 'Ops Consulting');
|
||||
writeToObsidianVault(pages, tmpDir);
|
||||
const second = fs.readFileSync(path.join(tmpDir, 'entity', 'project-alpha.md'), 'utf-8');
|
||||
|
||||
expect(second).not.toBe(first);
|
||||
expect(second).toContain('Ops Consulting');
|
||||
});
|
||||
|
||||
it('creates outDir if it does not exist', () => {
|
||||
const nested = path.join(tmpDir, 'does', 'not', 'exist', 'yet');
|
||||
expect(fs.existsSync(nested)).toBe(false);
|
||||
const result = writeToObsidianVault(seedPages(), nested);
|
||||
expect(fs.existsSync(nested)).toBe(true);
|
||||
expect(result.filesWritten).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
21
packages/wiki-compiler/tsconfig.json
Normal file
21
packages/wiki-compiler/tsconfig.json
Normal file
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"lib": ["ES2022"],
|
||||
"outDir": "dist",
|
||||
"rootDir": "src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"sourceMap": true,
|
||||
"composite": true
|
||||
},
|
||||
"include": ["src/**/*.ts"],
|
||||
"exclude": ["node_modules", "dist", "tests"]
|
||||
}
|
||||
Reference in New Issue
Block a user