moving
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled

This commit is contained in:
Oleg Maslov
2026-09-02 10:10:29 +02:00
commit 0c3e2ead3b
3841 changed files with 970576 additions and 0 deletions

View File

@@ -0,0 +1,27 @@
{
"name": "@waggle/wiki-compiler",
"version": "0.1.0",
"description": "Wiki Compiler — LLM-powered knowledge synthesis engine. Compiles memory frames + KG into interlinked wiki pages.",
"type": "module",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"exports": {
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
}
},
"scripts": {
"build": "tsc",
"test": "node ../../node_modules/vitest/vitest.mjs run --root ../.. --config vitest.config.ts packages/wiki-compiler/tests"
},
"dependencies": {
"@waggle/core": "*",
"@waggle/shared": "*"
},
"devDependencies": {
"@types/node": "^22.0.0",
"typescript": "^5.8.3"
},
"license": "MIT"
}

View File

@@ -0,0 +1,334 @@
/**
* Notion workspace export adapter (M-13)
*
* Writes compiled wiki pages to a user's Notion workspace as child pages
* under a user-chosen root. Uses the Notion REST API directly (no SDK)
* so we don't add @notionhq/client to the dep graph.
*
* Re-run strategy (per M-13 decision memo — delta):
* - Unchanged (content_hash matches existing notion_page_id) -> skip API call
* - New page (no notion_page_id) -> POST /v1/pages
* - Changed page (content_hash differs) -> archive old + create new
*
* Keeping the converter standalone (no library dep) is intentional — our
* markdown output is narrow and predictable (H1-H3, paragraphs, bullets,
* blockquotes, inline links). A dependency would inherit edge-case
* handling for features we don't emit.
*/
import type { PageRecord, WikiPageType } from '../types.js';
const NOTION_API = 'https://api.notion.com/v1';
const NOTION_VERSION = '2022-06-28';
/** Notion API caps block children per request at 100. */
const BLOCK_BATCH_SIZE = 100;
export interface NotionExportOptions {
/** Integration token (from notion.so/my-integrations — Internal Integration). */
token: string;
/** Root page ID under which to create child pages. UUIDs with or without dashes accepted. */
rootPageId: string;
}
export interface NotionExportStats {
byType: Record<WikiPageType, number>;
pagesCreated: number;
pagesUpdated: number;
pagesUnchanged: number;
pagesFailed: number;
errors: { slug: string; message: string }[];
}
export interface NotionStateHelpers {
getNotionPageId(slug: string): string | null;
setNotionPageId(slug: string, pageId: string): void;
clearNotionPageId(slug: string): void;
/** Used to detect "did content change since last export?". */
getPageContentHash(slug: string): string | null;
setPageContentHash?(slug: string, hash: string): void;
}
export interface NotionBlock {
object: 'block';
type: string;
[key: string]: unknown;
}
export interface RichText {
type: 'text';
text: { content: string; link?: { url: string } | null };
annotations?: {
bold?: boolean;
italic?: boolean;
code?: boolean;
};
}
/**
* The per-type payload a {@link NotionBlock} carries under its dynamic
* `[block.type]` key (e.g. `block.heading_1`, `block.paragraph`). Each holds
* the converted rich-text run. Exposed for narrowing the block's indexed
* `unknown` value at consumer/test boundaries.
*/
export interface NotionBlockPayload {
rich_text: RichText[];
}
/** Strip a leading YAML frontmatter block (--- ... ---) if present. */
export function stripFrontmatter(markdown: string): { body: string; title?: string } {
const match = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
if (!match) return { body: markdown };
const yaml = match[1];
const body = match[2];
const nameMatch = yaml.match(/^name:\s*(.+)$/m);
return { body, title: nameMatch?.[1]?.trim() };
}
/**
* Parse a markdown-text segment into Notion rich_text items, preserving
* inline links `[text](url)`, `**bold**`, `*italic*`, and `` `code` ``.
* Intentionally narrow — our wiki pages do not use more than this.
*/
export function toRichText(text: string): RichText[] {
if (!text) return [];
const out: RichText[] = [];
const linkRe = /\[([^\]]+)\]\(([^)]+)\)/g;
let lastIdx = 0;
let match: RegExpExecArray | null;
while ((match = linkRe.exec(text)) !== null) {
if (match.index > lastIdx) {
const slice = text.slice(lastIdx, match.index);
out.push(...parseEmphasis(slice));
}
out.push({
type: 'text',
text: { content: match[1], link: { url: match[2] } },
});
lastIdx = match.index + match[0].length;
}
if (lastIdx < text.length) {
out.push(...parseEmphasis(text.slice(lastIdx)));
}
return out;
}
/** Handle **bold**, *italic*, `code`. No nesting — our markdown does not nest these. */
function parseEmphasis(text: string): RichText[] {
if (!text) return [];
const re = /(\*\*[^*]+\*\*|\*[^*]+\*|`[^`]+`)/g;
const parts = text.split(re);
return parts.filter(p => p.length > 0).map(p => {
if (p.startsWith('**') && p.endsWith('**')) {
return { type: 'text' as const, text: { content: p.slice(2, -2) }, annotations: { bold: true } };
}
if (p.startsWith('*') && p.endsWith('*')) {
return { type: 'text' as const, text: { content: p.slice(1, -1) }, annotations: { italic: true } };
}
if (p.startsWith('`') && p.endsWith('`')) {
return { type: 'text' as const, text: { content: p.slice(1, -1) }, annotations: { code: true } };
}
return { type: 'text' as const, text: { content: p } };
});
}
/**
* Convert our wiki markdown body (frontmatter already stripped) into an
* ordered array of Notion blocks. Only handles: H1-H3, paragraphs, bullets,
* blockquotes. Lines that do not match become paragraphs.
*/
export function markdownToBlocks(body: string): NotionBlock[] {
const blocks: NotionBlock[] = [];
const lines = body.split(/\r?\n/);
let paragraph: string[] = [];
const flushParagraph = () => {
const content = paragraph.join(' ').trim();
if (content) {
blocks.push({
object: 'block',
type: 'paragraph',
paragraph: { rich_text: toRichText(content) },
});
}
paragraph = [];
};
for (const line of lines) {
const trimmed = line.trimEnd();
if (trimmed === '') {
flushParagraph();
continue;
}
const h = trimmed.match(/^(#{1,3})\s+(.+)$/);
if (h) {
flushParagraph();
const level = h[1].length as 1 | 2 | 3;
const text = h[2];
const type = `heading_${level}` as const;
blocks.push({
object: 'block',
type,
[type]: { rich_text: toRichText(text) },
});
continue;
}
const bullet = trimmed.match(/^[-*]\s+(.+)$/);
if (bullet) {
flushParagraph();
blocks.push({
object: 'block',
type: 'bulleted_list_item',
bulleted_list_item: { rich_text: toRichText(bullet[1]) },
});
continue;
}
const quote = trimmed.match(/^>\s?(.*)$/);
if (quote) {
flushParagraph();
blocks.push({
object: 'block',
type: 'quote',
quote: { rich_text: toRichText(quote[1]) },
});
continue;
}
paragraph.push(trimmed);
}
flushParagraph();
return blocks;
}
/**
* Extract a UUID page id from either a plain id (abc123...) or a Notion
* page URL. Returns the id with or without dashes — Notion accepts both.
*/
export function extractNotionPageId(urlOrId: string): string | null {
const trimmed = urlOrId.trim();
const hexMatch = trimmed.match(/([a-f0-9]{8}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{12})/i);
return hexMatch ? hexMatch[1] : null;
}
function notionHeaders(token: string): Record<string, string> {
return {
Authorization: `Bearer ${token}`,
'Notion-Version': NOTION_VERSION,
'Content-Type': 'application/json',
};
}
async function createNotionPage(
token: string,
rootPageId: string,
title: string,
blocks: NotionBlock[],
): Promise<{ id: string }> {
const firstBatch = blocks.slice(0, BLOCK_BATCH_SIZE);
const res = await fetch(`${NOTION_API}/pages`, {
method: 'POST',
headers: notionHeaders(token),
body: JSON.stringify({
parent: { page_id: rootPageId },
properties: {
title: {
title: [{ text: { content: title.slice(0, 200) } }],
},
},
children: firstBatch,
}),
});
if (!res.ok) {
const err = await res.text().catch(() => res.statusText);
throw new Error(`Notion create failed (${res.status}): ${err.slice(0, 300)}`);
}
const data = await res.json() as { id: string };
let cursor = BLOCK_BATCH_SIZE;
while (cursor < blocks.length) {
const batch = blocks.slice(cursor, cursor + BLOCK_BATCH_SIZE);
const appendRes = await fetch(`${NOTION_API}/blocks/${data.id}/children`, {
method: 'PATCH',
headers: notionHeaders(token),
body: JSON.stringify({ children: batch }),
});
if (!appendRes.ok) {
const err = await appendRes.text().catch(() => appendRes.statusText);
throw new Error(`Notion append failed (${appendRes.status}): ${err.slice(0, 300)}`);
}
cursor += BLOCK_BATCH_SIZE;
}
return data;
}
async function archiveNotionPage(token: string, pageId: string): Promise<void> {
const res = await fetch(`${NOTION_API}/pages/${pageId}`, {
method: 'PATCH',
headers: notionHeaders(token),
body: JSON.stringify({ archived: true }),
});
if (!res.ok) {
// Archive-failure is tolerable — worst case the user sees two pages.
// We swallow it here so the caller can still create the new page.
}
}
/**
* Export every compiled page to Notion. Skips index/health virtual pages.
* Runs sequentially to respect Notion's rate limit (~3 req/sec).
*/
export async function writeToNotionWorkspace(
pages: PageRecord[],
opts: NotionExportOptions,
state: NotionStateHelpers,
): Promise<NotionExportStats> {
const stats: NotionExportStats = {
byType: {} as Record<WikiPageType, number>,
pagesCreated: 0,
pagesUpdated: 0,
pagesUnchanged: 0,
pagesFailed: 0,
errors: [],
};
for (const page of pages) {
if (page.pageType === 'index' || page.pageType === 'health') continue;
try {
const existingId = state.getNotionPageId(page.slug);
const existingHash = state.getPageContentHash(page.slug);
if (existingId && existingHash === page.contentHash) {
stats.pagesUnchanged++;
continue;
}
const { body, title: frontmatterTitle } = stripFrontmatter(page.markdown);
const blocks = markdownToBlocks(body);
const title = frontmatterTitle ?? page.name;
if (existingId) {
await archiveNotionPage(opts.token, existingId);
state.clearNotionPageId(page.slug);
}
const created = await createNotionPage(opts.token, opts.rootPageId, title, blocks);
state.setNotionPageId(page.slug, created.id);
if (existingId) stats.pagesUpdated++;
else stats.pagesCreated++;
stats.byType[page.pageType] = (stats.byType[page.pageType] ?? 0) + 1;
} catch (err) {
stats.pagesFailed++;
stats.errors.push({
slug: page.slug,
message: err instanceof Error ? err.message : 'Unknown error',
});
}
}
return stats;
}

View File

@@ -0,0 +1,128 @@
/**
* Obsidian Vault Adapter (M-12)
*
* Writes compiled wiki pages to a user-specified directory in a shape
* Obsidian reads natively:
* {outDir}/
* _index.md — table of contents grouped by page type
* entity/{slug}.md — one file per compiled entity page
* concept/{slug}.md — one file per compiled concept page
* synthesis/{slug}.md — one file per synthesis page
*
* YAML frontmatter is preserved as-is — Obsidian reads `type:`, `confidence:`,
* etc. The body is rewritten only to convert raw `[[Display Name]]` wikilinks
* into the `[[slug|Display Name]]` alias form so the links resolve against
* our slug-based filenames.
*
* This adapter does NOT call an LLM, does NOT mutate state.compile watermarks,
* and is safe to call repeatedly — files are overwritten on each export.
*/
import * as fs from 'node:fs';
import * as path from 'node:path';
import type { PageRecord } from '../types.js';
export interface ObsidianExportResult {
outDir: string;
filesWritten: number;
indexPath: string;
byType: Record<string, number>;
}
/**
* Transform `[[Display Name]]` wikilinks to `[[slug|Display Name]]` so they
* resolve against slug-named files. Leaves `[[already-a-slug]]` and
* `[[slug|Display]]` forms alone.
*/
function transformWikilinks(markdown: string, nameToSlug: Map<string, string>): string {
return markdown.replace(/\[\[([^\]|]+?)\]\]/g, (match, inner: string) => {
const trimmed = inner.trim();
// If the target is already a known slug (present in nameToSlug values),
// leave it alone.
for (const slug of nameToSlug.values()) {
if (slug === trimmed) return match;
}
// Otherwise try to resolve as a display name.
const slug = nameToSlug.get(trimmed.toLowerCase());
return slug ? `[[${slug}|${trimmed}]]` : match;
});
}
function buildIndex(pages: PageRecord[]): string {
const byType: Record<string, PageRecord[]> = {};
for (const page of pages) {
(byType[page.pageType] ??= []).push(page);
}
const lines: string[] = [];
lines.push('---');
lines.push('type: index');
lines.push(`generated_at: ${new Date().toISOString()}`);
lines.push(`total_pages: ${pages.length}`);
lines.push('---');
lines.push('');
lines.push('# Waggle Wiki Index');
lines.push('');
lines.push(`Exported ${pages.length} page(s) from your Waggle memory.`);
lines.push('');
const order: Array<[string, string]> = [
['entity', 'Entities'],
['concept', 'Concepts'],
['synthesis', 'Cross-Source Syntheses'],
];
for (const [type, heading] of order) {
const group = byType[type];
if (!group || group.length === 0) continue;
lines.push(`## ${heading} (${group.length})`);
lines.push('');
for (const page of group) {
lines.push(`- [[${page.slug}|${page.name}]] — ${page.sourceCount} source(s)`);
}
lines.push('');
}
return lines.join('\n');
}
/**
* Write the given pages to a directory shaped for Obsidian.
*
* @param pages page records from CompilationState.getAllPages()
* @param outDir absolute directory path — created if missing. Must be writable.
* @returns paths actually written + per-type counts
*/
export function writeToObsidianVault(pages: PageRecord[], outDir: string): ObsidianExportResult {
fs.mkdirSync(outDir, { recursive: true });
// Build name → slug map for the wikilink transform. Case-insensitive
// lookup so `[[Project Alpha]]` resolves to `project-alpha`.
const nameToSlug = new Map<string, string>();
for (const page of pages) {
nameToSlug.set(page.name.toLowerCase(), page.slug);
}
const byType: Record<string, number> = {};
let filesWritten = 0;
for (const page of pages) {
// Index + health pages are virtual — skip them; we write our own index.
if (page.pageType === 'index' || page.pageType === 'health') continue;
const typeDir = path.join(outDir, page.pageType);
fs.mkdirSync(typeDir, { recursive: true });
const content = transformWikilinks(page.markdown, nameToSlug);
const filePath = path.join(typeDir, `${page.slug}.md`);
fs.writeFileSync(filePath, content, 'utf-8');
filesWritten++;
byType[page.pageType] = (byType[page.pageType] ?? 0) + 1;
}
const indexPath = path.join(outDir, '_index.md');
fs.writeFileSync(indexPath, buildIndex(pages), 'utf-8');
filesWritten++;
return { outDir, filesWritten, indexPath, byType };
}

View File

@@ -0,0 +1,596 @@
/**
* Wiki Compiler — compiles memory frames + KG into interlinked wiki pages.
*
* Core compilation functions:
* - compileEntityPage() — entity → frames + relations → LLM → markdown
* - compileConceptPage() — topic → search → LLM → markdown
* - compileSynthesisPage() — cross-source pattern detection
* - compileIndex() — navigable catalog
* - compileHealth() — contradictions, gaps, data quality
*/
import type {
KnowledgeGraph,
FrameStore,
HybridSearch,
Entity,
} from '@waggle/core';
import type {
WikiPage,
WikiPageType,
CompilerConfig,
CompilationResult,
HealthReport,
HealthIssue,
} from './types.js';
import { CompilationState, contentHash } from './state.js';
import { entityPagePrompt, conceptPagePrompt, synthesisPagePrompt } from './prompts.js';
function slugify(name: string): string {
return name
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-|-$/g, '')
.slice(0, 80);
}
function buildFrontmatter(
type: WikiPageType,
name: string,
frameIds: number[],
relatedEntities: string[],
confidence: number,
entityType?: string,
): string {
const lines = [
'---',
`type: ${type}`,
...(entityType ? [`entity_type: ${entityType}`] : []),
`name: "${name.replace(/"/g, '\\"')}"`,
`confidence: ${confidence.toFixed(2)}`,
`sources: ${frameIds.length}`,
`last_compiled: ${new Date().toISOString()}`,
`frame_ids: [${frameIds.join(', ')}]`,
`related_entities: [${relatedEntities.map(e => `"${e}"`).join(', ')}]`,
'---',
];
return lines.join('\n');
}
export class WikiCompiler {
private kg: KnowledgeGraph;
private frames: FrameStore;
private search: HybridSearch;
private state: CompilationState;
private config: Required<CompilerConfig>;
constructor(
kg: KnowledgeGraph,
frames: FrameStore,
search: HybridSearch,
state: CompilationState,
config: CompilerConfig,
) {
this.kg = kg;
this.frames = frames;
this.search = search;
this.state = state;
this.config = {
synthesize: config.synthesize,
outputDir: config.outputDir ?? 'wiki',
minFramesPerPage: config.minFramesPerPage ?? 2,
maxFramesPerCall: config.maxFramesPerCall ?? 30,
minConfidence: config.minConfidence ?? 0.3,
};
}
// ── Entity Page ───────────────────────────────────────────────
async compileEntityPage(entity: Entity): Promise<WikiPage | null> {
// Gather frames mentioning this entity
const searchResults = await this.search.search(entity.name, {
limit: this.config.maxFramesPerCall,
});
const frameData = searchResults.map(r => ({
id: r.frame.id,
content: r.frame.content,
created_at: r.frame.created_at,
}));
if (frameData.length < this.config.minFramesPerPage) {
return null; // Not enough data for a page
}
// Gather relations
const outRels = this.kg.getRelationsFrom(entity.id);
const inRels = this.kg.getRelationsTo(entity.id);
const relations: { target: string; relationType: string; confidence: number }[] = [];
const relatedEntities: string[] = [];
for (const rel of outRels) {
const target = this.kg.getEntity(rel.target_id);
if (target && target.valid_to === null) {
relations.push({ target: target.name, relationType: rel.relation_type, confidence: rel.confidence });
relatedEntities.push(target.name);
}
}
for (const rel of inRels) {
const source = this.kg.getEntity(rel.source_id);
if (source && source.valid_to === null) {
relations.push({ target: source.name, relationType: `${rel.relation_type} (inbound)`, confidence: rel.confidence });
if (!relatedEntities.includes(source.name)) {
relatedEntities.push(source.name);
}
}
}
// Synthesize via LLM
const prompt = entityPagePrompt(entity.name, entity.entity_type, frameData, relations);
const body = await this.config.synthesize(prompt);
const frameIds = frameData.map(f => f.id);
const confidence = frameData.length > 5 ? 0.9 : frameData.length > 2 ? 0.7 : 0.5;
const slug = slugify(entity.name);
const frontmatter = buildFrontmatter('entity', entity.name, frameIds, relatedEntities, confidence, entity.entity_type);
const markdown = `${frontmatter}\n\n# ${entity.name}\n\n${body}`;
return {
slug,
frontmatter: {
type: 'entity',
entity_type: entity.entity_type,
name: entity.name,
confidence,
sources: frameIds.length,
last_compiled: new Date().toISOString(),
frame_ids: frameIds,
related_entities: relatedEntities,
},
markdown,
contentHash: contentHash(markdown),
};
}
// ── Concept Page ──────────────────────────────────────────────
async compileConceptPage(conceptName: string): Promise<WikiPage | null> {
const searchResults = await this.search.search(conceptName, {
limit: this.config.maxFramesPerCall,
});
const frameData = searchResults.map(r => ({
id: r.frame.id,
content: r.frame.content,
created_at: r.frame.created_at,
}));
if (frameData.length < this.config.minFramesPerPage) {
return null;
}
// Find related entities via KG
const entityResults = this.kg.searchEntities(conceptName, 10);
const relatedEntities = entityResults.map(e => e.name);
const prompt = conceptPagePrompt(conceptName, frameData, relatedEntities);
const body = await this.config.synthesize(prompt);
const frameIds = frameData.map(f => f.id);
const confidence = frameData.length > 5 ? 0.85 : 0.6;
const slug = slugify(conceptName);
const frontmatter = buildFrontmatter('concept', conceptName, frameIds, relatedEntities, confidence);
const markdown = `${frontmatter}\n\n# ${conceptName}\n\n${body}`;
return {
slug,
frontmatter: {
type: 'concept',
name: conceptName,
confidence,
sources: frameIds.length,
last_compiled: new Date().toISOString(),
frame_ids: frameIds,
related_entities: relatedEntities,
},
markdown,
contentHash: contentHash(markdown),
};
}
// ── Synthesis Page ────────────────────────────────────────────
async compileSynthesisPage(topic: string): Promise<WikiPage | null> {
const searchResults = await this.search.search(topic, {
limit: this.config.maxFramesPerCall * 2, // more context for synthesis
});
// Group frames by source to detect cross-source patterns
const bySource = new Map<string, typeof searchResults>();
for (const r of searchResults) {
const source = r.frame.source ?? 'unknown';
let group = bySource.get(source);
if (!group) {
group = [];
bySource.set(source, group);
}
group.push(r);
}
// Need frames from at least 2 sources for synthesis
if (bySource.size < 2) {
return null;
}
const crossSourceFrames = searchResults.slice(0, this.config.maxFramesPerCall).map(r => ({
id: r.frame.id,
content: r.frame.content,
source: r.frame.source ?? 'unknown',
created_at: r.frame.created_at,
}));
const prompt = synthesisPagePrompt(topic, crossSourceFrames);
const body = await this.config.synthesize(prompt);
const frameIds = crossSourceFrames.map(f => f.id);
const confidence = bySource.size > 3 ? 0.85 : 0.65;
const slug = `synthesis-${slugify(topic)}`;
const sources = Array.from(bySource.keys());
const frontmatter = buildFrontmatter('synthesis', `Synthesis: ${topic}`, frameIds, sources, confidence);
const markdown = `${frontmatter}\n\n# Synthesis: ${topic}\n\n${body}`;
return {
slug,
frontmatter: {
type: 'synthesis',
name: `Synthesis: ${topic}`,
confidence,
sources: frameIds.length,
last_compiled: new Date().toISOString(),
frame_ids: frameIds,
related_entities: sources,
},
markdown,
contentHash: contentHash(markdown),
};
}
// ── Index Page ────────────────────────────────────────────────
compileIndex(): WikiPage {
const allPages = this.state.getAllPages();
const entityPages = allPages.filter(p => p.pageType === 'entity');
const conceptPages = allPages.filter(p => p.pageType === 'concept');
const synthesisPages = allPages.filter(p => p.pageType === 'synthesis');
const lines: string[] = [
'# Wiki Index',
'',
`*${allPages.length} pages compiled — last updated ${new Date().toISOString().slice(0, 10)}*`,
'',
];
if (entityPages.length > 0) {
lines.push('## Entities', '');
for (const p of entityPages) {
lines.push(`- [[${p.name}]] — ${p.sourceCount} source${p.sourceCount === 1 ? '' : 's'} (${p.compiledAt.slice(0, 10)})`);
}
lines.push('');
}
if (conceptPages.length > 0) {
lines.push('## Concepts', '');
for (const p of conceptPages) {
lines.push(`- [[${p.name}]] — ${p.sourceCount} source${p.sourceCount === 1 ? '' : 's'} (${p.compiledAt.slice(0, 10)})`);
}
lines.push('');
}
if (synthesisPages.length > 0) {
lines.push('## Cross-Source Synthesis', '');
for (const p of synthesisPages) {
lines.push(`- [[${p.name}]] — ${p.sourceCount} source${p.sourceCount === 1 ? '' : 's'} (${p.compiledAt.slice(0, 10)})`);
}
lines.push('');
}
const markdown = lines.join('\n');
return {
slug: 'index',
frontmatter: {
type: 'index',
name: 'Wiki Index',
confidence: 1.0,
sources: allPages.length,
last_compiled: new Date().toISOString(),
frame_ids: [],
related_entities: [],
},
markdown,
contentHash: contentHash(markdown),
};
}
// ── Health Report ─────────────────────────────────────────────
compileHealth(): HealthReport {
const issues: HealthIssue[] = [];
const allPages = this.state.getAllPages();
const entityCount = this.kg.getEntityCount();
const frameStats = this.frames.getStats();
// Check for orphan entities (KG entities with no wiki page)
const entities = this.kg.getEntities(1000);
const pageNames = new Set(allPages.map(p => p.name.toLowerCase()));
for (const entity of entities) {
if (!pageNames.has(entity.name.toLowerCase())) {
// Check if entity has enough frames to justify a page
const rels = this.kg.getRelationsFrom(entity.id);
if (rels.length > 0 || entity.entity_type === 'person' || entity.entity_type === 'project') {
issues.push({
type: 'missing_page',
severity: rels.length > 2 ? 'high' : 'medium',
description: `Entity "${entity.name}" (${entity.entity_type}) has no wiki page`,
entity: entity.name,
suggestion: `Run: compileEntityPage("${entity.name}")`,
});
}
}
}
// Check for weak pages (few sources)
for (const page of allPages) {
if (page.sourceCount < 2 && page.pageType !== 'index') {
issues.push({
type: 'weak_confidence',
severity: 'low',
description: `Page "${page.name}" has only ${page.sourceCount} source(s)`,
entity: page.name,
suggestion: 'Gather more data about this topic',
});
}
}
// Check for orphan entities in KG (no relations at all)
for (const entity of entities) {
const outRels = this.kg.getRelationsFrom(entity.id);
const inRels = this.kg.getRelationsTo(entity.id);
if (outRels.length === 0 && inRels.length === 0) {
issues.push({
type: 'orphan_entity',
severity: 'low',
description: `Entity "${entity.name}" (${entity.entity_type}) has no relations`,
entity: entity.name,
suggestion: 'Consider adding relations or retiring this entity',
});
}
}
// M-14: stale_page check. A page is stale when its compiledAt is older
// than STALE_DAYS ago AND the watermark shows new frames have arrived
// since. The watermark-vs-frame-count heuristic avoids false positives
// for pages whose topic genuinely hasn't changed — they're not stale,
// just stable.
const STALE_DAYS = 30;
const staleCutoff = Date.now() - STALE_DAYS * 24 * 60 * 60 * 1000;
const latestFrameId = Math.max(0, ...allPages.flatMap(p => {
try { return (JSON.parse(p.frameIds) as number[]) ?? []; } catch { return []; }
}));
const newestIndexedFrameId = this.state.getWatermark().lastFrameId;
const hasNewerFramesOverall = newestIndexedFrameId > latestFrameId;
for (const page of allPages) {
if (page.pageType === 'index' || page.pageType === 'health') continue;
const compiledMs = Date.parse(page.compiledAt);
if (Number.isFinite(compiledMs) && compiledMs < staleCutoff && hasNewerFramesOverall) {
issues.push({
type: 'stale_page',
severity: 'medium',
description: `Page "${page.name}" was compiled ${Math.round((Date.now() - compiledMs) / (24 * 60 * 60 * 1000))} days ago and new frames have arrived since`,
entity: page.name,
suggestion: 'Recompile to refresh with recent context',
});
}
}
// M-14: coverage ratio — pages / compilable entities (entities with
// ≥1 relation OR entity_type person/project, matching the missing_page
// threshold above). Reports a 0-1 fraction; UI renders it as %.
const compilableEntities = entities.filter(e => {
const rels = this.kg.getRelationsFrom(e.id);
return rels.length > 0 || e.entity_type === 'person' || e.entity_type === 'project';
}).length;
const coverage = compilableEntities > 0
? Math.min(1, allPages.filter(p => p.pageType === 'entity').length / compilableEntities)
: 0;
const stalePageCount = issues.filter(i => i.type === 'stale_page').length;
// Data quality score
const hasEntities = entityCount > 0 ? 20 : 0;
const hasFrames = frameStats.total > 10 ? 20 : frameStats.total > 0 ? 10 : 0;
const hasPages = allPages.length > 5 ? 20 : allPages.length > 0 ? 10 : 0;
const lowIssues = issues.filter(i => i.severity === 'high').length;
const issueDeduction = Math.min(40, lowIssues * 10);
const dataQualityScore = Math.max(0, hasEntities + hasFrames + hasPages + 40 - issueDeduction);
return {
totalEntities: entityCount,
totalFrames: frameStats.total,
totalPages: allPages.length,
coverage,
stalePageCount,
issues,
dataQualityScore,
compiledAt: new Date().toISOString(),
};
}
// ── Full Compilation ──────────────────────────────────────────
async compile(options?: { incremental?: boolean; concepts?: string[] }): Promise<CompilationResult> {
const startTime = Date.now();
const incremental = options?.incremental ?? true;
const watermark = this.state.getWatermark();
let pagesCreated = 0;
let pagesUpdated = 0;
let pagesUnchanged = 0;
const entityPageNames: string[] = [];
const conceptPageNames: string[] = [];
const synthesisPageNames: string[] = [];
// 1. Compile entity pages for all significant entities
const entities = this.kg.getEntities(200);
for (const entity of entities) {
// Skip entities we've already compiled unless new frames exist
if (incremental && watermark.lastFrameId > 0) {
const existingPage = this.state.getPage(slugify(entity.name));
if (existingPage) {
// Check if new frames mention this entity
const newFrames = this.state.getFramesSince(watermark.lastFrameId, 100);
const mentionsEntity = newFrames.some(f =>
f.content.toLowerCase().includes(entity.name.toLowerCase()),
);
if (!mentionsEntity) {
pagesUnchanged++;
continue;
}
}
}
const page = await this.compileEntityPage(entity);
if (page) {
const result = this.state.upsertPage(
page.slug, 'entity', entity.name,
page.contentHash, page.frontmatter.frame_ids, page.frontmatter.sources,
page.markdown,
);
if (result.action === 'created') pagesCreated++;
else if (result.action === 'updated') pagesUpdated++;
else pagesUnchanged++;
entityPageNames.push(entity.name);
}
}
// 2. Compile concept pages (user-specified or auto-detected)
const concepts = options?.concepts ?? this.detectConcepts(entities);
for (const concept of concepts) {
const page = await this.compileConceptPage(concept);
if (page) {
const result = this.state.upsertPage(
page.slug, 'concept', concept,
page.contentHash, page.frontmatter.frame_ids, page.frontmatter.sources,
page.markdown,
);
if (result.action === 'created') pagesCreated++;
else if (result.action === 'updated') pagesUpdated++;
else pagesUnchanged++;
conceptPageNames.push(concept);
}
}
// 3. Compile synthesis pages for topics with cross-source data
for (const concept of concepts) {
const page = await this.compileSynthesisPage(concept);
if (page) {
const result = this.state.upsertPage(
page.slug, 'synthesis', `Synthesis: ${concept}`,
page.contentHash, page.frontmatter.frame_ids, page.frontmatter.sources,
page.markdown,
);
if (result.action === 'created') pagesCreated++;
else if (result.action === 'updated') pagesUpdated++;
else pagesUnchanged++;
synthesisPageNames.push(concept);
}
}
// 4. Compile index
const indexPage = this.compileIndex();
this.state.upsertPage('index', 'index', 'Wiki Index', indexPage.contentHash, [], 0, indexPage.markdown);
// 5. Update watermark
const maxFrameId = this.state.getMaxFrameId();
const totalCompiled = pagesCreated + pagesUpdated;
this.state.updateWatermark(maxFrameId, totalCompiled);
// 6. Health check
const health = this.compileHealth();
return {
pagesCreated,
pagesUpdated,
pagesUnchanged,
entityPages: entityPageNames,
conceptPages: conceptPageNames,
synthesisPages: synthesisPageNames,
healthIssues: health.issues.length,
watermark: { lastFrameId: maxFrameId, lastCompiledAt: new Date().toISOString(), pagesCompiled: totalCompiled },
durationMs: Date.now() - startTime,
};
}
// ── Helpers ───────────────────────────────────────────────────
/** Auto-detect concepts from entity types and common topics. */
private detectConcepts(entities: Entity[]): string[] {
const typeCounts = new Map<string, number>();
for (const e of entities) {
typeCounts.set(e.entity_type, (typeCounts.get(e.entity_type) ?? 0) + 1);
}
const concepts: string[] = [];
// Add entity types that have multiple entries as concepts
for (const [type, count] of typeCounts) {
if (count >= 3 && type !== 'concept') {
concepts.push(type);
}
}
// Add 'concept' type entities directly as concepts
const conceptEntities = entities.filter(e => e.entity_type === 'concept');
for (const e of conceptEntities) {
if (!concepts.includes(e.name)) {
concepts.push(e.name);
}
}
return concepts.slice(0, 20); // Cap at 20 concepts
}
/**
* Export all wiki pages as a flat markdown bundle.
* Returns a map of slug → markdown content, suitable for writing to disk.
*/
exportToMarkdown(): Map<string, string> {
const pages = this.state.getAllPages();
const result = new Map<string, string>();
for (const page of pages) {
result.set(page.slug, page.markdown || `# ${page.name}\n\n(no content compiled yet)`);
}
return result;
}
/**
* Export all wiki pages to a directory as individual .md files.
* Creates the directory if it doesn't exist.
*/
async exportToDirectory(dir: string): Promise<{ filesWritten: number }> {
const { mkdirSync, writeFileSync } = await import('node:fs');
mkdirSync(dir, { recursive: true });
const pages = this.exportToMarkdown();
let count = 0;
for (const [slug, markdown] of pages) {
writeFileSync(`${dir}/${slug}.md`, markdown, 'utf-8');
count++;
}
return { filesWritten: count };
}
}

View File

@@ -0,0 +1,29 @@
export { WikiCompiler } from './compiler.js';
export { CompilationState, contentHash } from './state.js';
export { resolveSynthesizer, type ResolvedSynthesizer, type SynthesizerConfig } from './synthesizer.js';
export { entityPagePrompt, conceptPagePrompt, synthesisPagePrompt } from './prompts.js';
export { writeToObsidianVault, type ObsidianExportResult } from './adapters/obsidian.js';
export {
writeToNotionWorkspace,
markdownToBlocks,
stripFrontmatter,
toRichText,
extractNotionPageId,
type NotionExportOptions,
type NotionExportStats,
type NotionStateHelpers,
type NotionBlock,
} from './adapters/notion.js';
export type {
WikiPage,
WikiPageType,
WikiPageFrontmatter,
CompilationWatermark,
PageRecord,
CompilerConfig,
LLMSynthesizeFn,
CompilationResult,
HealthReport,
HealthIssue,
HealthIssueType,
} from './types.js';

View File

@@ -0,0 +1,94 @@
/**
* LLM prompts for wiki page compilation.
*/
export function entityPagePrompt(
entityName: string,
entityType: string,
frames: { id: number; content: string; created_at: string }[],
relations: { target: string; relationType: string; confidence: number }[],
): string {
const frameList = frames
.map(f => `[Frame #${f.id}, ${f.created_at}]: ${f.content}`)
.join('\n\n');
const relationList = relations.length > 0
? relations.map(r => `- ${r.target} (${r.relationType}, confidence: ${r.confidence})`).join('\n')
: 'No relations found.';
return `You are a wiki compiler. Synthesize the following memory frames about "${entityName}" (${entityType}) into a wiki page.
## Source Frames (${frames.length} total)
${frameList}
## Known Relations
${relationList}
## Instructions
1. Write a **Summary** section (2-3 sentences synthesizing all knowledge)
2. Write a **Key Facts** section with bullet points citing frame IDs (e.g., "from frame #42")
3. Write a **Timeline** section if temporal events are present (table: Date | Event | Source)
4. Write a **Relations** section listing connected entities with [[wiki links]]
5. Write an **Open Questions** section noting gaps or unresolved contradictions
6. If frames contradict each other, note it in a **Contradictions** section with confidence assessment
Be concise. Cite frame IDs for every claim. Use markdown formatting.
Output ONLY the page body (no frontmatter — that's added automatically).`;
}
export function conceptPagePrompt(
conceptName: string,
frames: { id: number; content: string; created_at: string }[],
relatedEntities: string[],
): string {
const frameList = frames
.map(f => `[Frame #${f.id}, ${f.created_at}]: ${f.content}`)
.join('\n\n');
const entityList = relatedEntities.length > 0
? relatedEntities.map(e => `- [[${e}]]`).join('\n')
: 'None identified.';
return `You are a wiki compiler. Synthesize the following memory frames about the concept "${conceptName}" into a wiki page.
## Source Frames (${frames.length} total)
${frameList}
## Related Entities
${entityList}
## Instructions
1. Write a **TL;DR** (2 sentences max)
2. Write a **What We Know** section synthesizing all frames
3. Write a **Sources & Evolution** section showing how understanding evolved over time
4. Write a **Related Topics** section with [[wiki links]]
5. Note any gaps or areas needing more data
Be concise. Cite frame IDs. Use markdown formatting.
Output ONLY the page body (no frontmatter).`;
}
export function synthesisPagePrompt(
topic: string,
crossSourceFrames: { id: number; content: string; source: string; created_at: string }[],
): string {
const frameList = crossSourceFrames
.map(f => `[Frame #${f.id}, source: ${f.source}, ${f.created_at}]: ${f.content}`)
.join('\n\n');
return `You are a wiki compiler performing cross-source synthesis. Multiple independent sources discuss "${topic}". Find patterns, agreements, and contradictions.
## Frames from Multiple Sources (${crossSourceFrames.length} total)
${frameList}
## Instructions
1. Write a **Cross-Source Summary** — what do multiple sources agree on?
2. Write a **Patterns** section — recurring themes across sources
3. Write a **Contradictions** section — where sources disagree (with frame IDs)
4. Write an **Insights** section — what can we conclude that no single source stated?
5. Write a **Confidence Assessment** — how reliable is this synthesis?
This is the most valuable page type. Focus on insights that emerge from combining sources.
Be concise. Cite frame IDs. Use markdown formatting.
Output ONLY the page body (no frontmatter).`;
}

View File

@@ -0,0 +1,193 @@
/**
* Compilation State Tracker — SQLite-backed watermarks and page hashes.
*
* Tracks what has been compiled and when, enabling incremental compilation.
* Uses the same MindDB instance as the memory system.
*/
import { createHash } from 'node:crypto';
import type { MindDB } from '@waggle/core';
import type { WikiPageType, CompilationWatermark, PageRecord } from './types.js';
const PAGES_TABLE = `
CREATE TABLE IF NOT EXISTS wiki_pages (
slug TEXT PRIMARY KEY,
page_type TEXT NOT NULL,
name TEXT NOT NULL,
content_hash TEXT NOT NULL,
markdown TEXT NOT NULL DEFAULT '',
frame_ids TEXT NOT NULL DEFAULT '[]',
compiled_at TEXT NOT NULL DEFAULT (datetime('now')),
source_count INTEGER NOT NULL DEFAULT 0
)`;
const WATERMARK_TABLE = `
CREATE TABLE IF NOT EXISTS wiki_watermark (
id INTEGER PRIMARY KEY CHECK (id = 1),
last_frame_id INTEGER NOT NULL DEFAULT 0,
last_compiled_at TEXT NOT NULL DEFAULT (datetime('now')),
pages_compiled INTEGER NOT NULL DEFAULT 0
)`;
export function contentHash(content: string): string {
return createHash('sha256').update(content).digest('hex').slice(0, 16);
}
export class CompilationState {
private db: MindDB;
constructor(db: MindDB) {
this.db = db;
this.ensureSchema();
}
private ensureSchema(): void {
const raw = this.db.getDatabase();
raw.prepare(PAGES_TABLE).run();
raw.prepare(WATERMARK_TABLE).run();
// Migration: add markdown column if missing (for databases created before v1.1)
try {
raw.prepare("SELECT markdown FROM wiki_pages LIMIT 0").get();
} catch {
raw.prepare("ALTER TABLE wiki_pages ADD COLUMN markdown TEXT NOT NULL DEFAULT ''").run();
}
// M-13: Notion export delta tracking — add notion_page_id column if
// missing. Nullable because most pages haven't been exported.
try {
raw.prepare("SELECT notion_page_id FROM wiki_pages LIMIT 0").get();
} catch {
raw.prepare("ALTER TABLE wiki_pages ADD COLUMN notion_page_id TEXT").run();
}
}
/** M-13: read the Notion page id previously written for `slug`, or null. */
getNotionPageId(slug: string): string | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT notion_page_id FROM wiki_pages WHERE slug = ?').get(slug) as { notion_page_id: string | null } | undefined;
return row?.notion_page_id ?? null;
}
/** M-13: record the Notion page id created for `slug`. */
setNotionPageId(slug: string, pageId: string): void {
const raw = this.db.getDatabase();
raw.prepare('UPDATE wiki_pages SET notion_page_id = ? WHERE slug = ?').run(pageId, slug);
}
/** M-13: clear stored Notion page id (used when we archive + recreate on change). */
clearNotionPageId(slug: string): void {
const raw = this.db.getDatabase();
raw.prepare('UPDATE wiki_pages SET notion_page_id = NULL WHERE slug = ?').run(slug);
}
/** M-13: get a page's current content_hash (used by Notion exporter for delta). */
getPageContentHash(slug: string): string | null {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT content_hash FROM wiki_pages WHERE slug = ?').get(slug) as { content_hash: string } | undefined;
return row?.content_hash ?? null;
}
// ── Watermark ─────────────────────────────────────────────────
getWatermark(): CompilationWatermark {
const raw = this.db.getDatabase();
const row = raw.prepare('SELECT * FROM wiki_watermark WHERE id = 1').get() as {
last_frame_id: number;
last_compiled_at: string;
pages_compiled: number;
} | undefined;
if (!row) {
return { lastFrameId: 0, lastCompiledAt: '', pagesCompiled: 0 };
}
return {
lastFrameId: row.last_frame_id,
lastCompiledAt: row.last_compiled_at,
pagesCompiled: row.pages_compiled,
};
}
updateWatermark(lastFrameId: number, pagesCompiled: number): void {
const raw = this.db.getDatabase();
raw.prepare(`
INSERT INTO wiki_watermark (id, last_frame_id, last_compiled_at, pages_compiled)
VALUES (1, ?, datetime('now'), ?)
ON CONFLICT(id) DO UPDATE SET
last_frame_id = excluded.last_frame_id,
last_compiled_at = excluded.last_compiled_at,
pages_compiled = excluded.pages_compiled
`).run(lastFrameId, pagesCompiled);
}
// ── Page Records ──────────────────────────────────────────────
getPage(slug: string): PageRecord | undefined {
return this.db.getDatabase().prepare(
'SELECT slug, page_type as pageType, name, content_hash as contentHash, markdown, frame_ids as frameIds, compiled_at as compiledAt, source_count as sourceCount FROM wiki_pages WHERE slug = ?',
).get(slug) as PageRecord | undefined;
}
getAllPages(): PageRecord[] {
return this.db.getDatabase().prepare(
'SELECT slug, page_type as pageType, name, content_hash as contentHash, markdown, frame_ids as frameIds, compiled_at as compiledAt, source_count as sourceCount FROM wiki_pages ORDER BY name',
).all() as PageRecord[];
}
getPagesByType(pageType: WikiPageType): PageRecord[] {
return this.db.getDatabase().prepare(
'SELECT slug, page_type as pageType, name, content_hash as contentHash, markdown, frame_ids as frameIds, compiled_at as compiledAt, source_count as sourceCount FROM wiki_pages WHERE page_type = ? ORDER BY name',
).all(pageType) as PageRecord[];
}
upsertPage(
slug: string,
pageType: WikiPageType,
name: string,
hash: string,
frameIds: number[],
sourceCount: number,
markdown = '',
): { action: 'created' | 'updated' | 'unchanged' } {
const raw = this.db.getDatabase();
const existing = this.getPage(slug);
if (existing && existing.contentHash === hash) {
return { action: 'unchanged' };
}
raw.prepare(`
INSERT INTO wiki_pages (slug, page_type, name, content_hash, markdown, frame_ids, compiled_at, source_count)
VALUES (?, ?, ?, ?, ?, ?, datetime('now'), ?)
ON CONFLICT(slug) DO UPDATE SET
content_hash = excluded.content_hash,
markdown = excluded.markdown,
frame_ids = excluded.frame_ids,
compiled_at = excluded.compiled_at,
source_count = excluded.source_count
`).run(slug, pageType, name, hash, markdown, JSON.stringify(frameIds), sourceCount);
return { action: existing ? 'updated' : 'created' };
}
deletePage(slug: string): boolean {
const result = this.db.getDatabase().prepare(
'DELETE FROM wiki_pages WHERE slug = ?',
).run(slug);
return result.changes > 0;
}
/** Get the highest frame ID in the database. */
getMaxFrameId(): number {
const row = this.db.getDatabase().prepare(
'SELECT COALESCE(MAX(id), 0) as max_id FROM memory_frames',
).get() as { max_id: number };
return row.max_id;
}
/** Get frames newer than a given ID. */
getFramesSince(frameId: number, limit = 500): { id: number; content: string; importance: string; source: string; created_at: string }[] {
return this.db.getDatabase().prepare(
'SELECT id, content, importance, source, created_at FROM memory_frames WHERE id > ? ORDER BY id ASC LIMIT ?',
).all(frameId, limit) as { id: number; content: string; importance: string; source: string; created_at: string }[];
}
}

View File

@@ -0,0 +1,156 @@
/**
* LLM Synthesizer — resolves the best available LLM for wiki page synthesis.
*
* Priority chain:
* 1. Anthropic Haiku (cheapest, fastest, best for synthesis)
* 2. Ollama (free, local)
* 3. Echo (no LLM, returns structured stub)
*
* All synthesizers implement LLMSynthesizeFn: (prompt: string) => Promise<string>
*/
import type { LLMSynthesizeFn } from './types.js';
export interface SynthesizerConfig {
/** Anthropic API key. Checked from env if not provided. */
anthropicApiKey?: string;
/** Ollama base URL. Checked from env if not provided. */
ollamaUrl?: string;
/** Ollama model name (default: llama3.2) */
ollamaModel?: string;
/** Max tokens for synthesis output (default: 1500) */
maxTokens?: number;
}
// ── Anthropic (Haiku) ───────────────────────────────────────────
function createAnthropicSynthesizer(apiKey: string, maxTokens: number): LLMSynthesizeFn {
return async (prompt: string): Promise<string> => {
// Dynamic import — @anthropic-ai/sdk is optional
const { default: Anthropic } = await import('@anthropic-ai/sdk');
const client = new Anthropic({ apiKey });
const response = await client.messages.create({
model: 'claude-haiku-4-5-20251001',
max_tokens: maxTokens,
messages: [{ role: 'user', content: prompt }],
});
const textBlock = response.content.find(b => b.type === 'text');
return textBlock?.text ?? '';
};
}
// ── Ollama ──────────────────────────────────────────────────────
function createOllamaSynthesizer(baseUrl: string, model: string, maxTokens: number): LLMSynthesizeFn {
return async (prompt: string): Promise<string> => {
const url = `${baseUrl.replace(/\/$/, '')}/api/generate`;
const response = await fetch(url, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model,
prompt,
stream: false,
options: { num_predict: maxTokens },
}),
signal: AbortSignal.timeout(60_000),
});
if (!response.ok) {
throw new Error(`Ollama error: ${response.status} ${response.statusText}`);
}
const data = await response.json() as { response: string };
return data.response ?? '';
};
}
// ── Echo (fallback) ─────────────────────────────────────────────
function createEchoSynthesizer(): LLMSynthesizeFn {
return async (prompt: string): Promise<string> => {
const frameMatch = prompt.match(/\((\d+) total\)/);
const frameCount = frameMatch ? frameMatch[1] : '?';
const nameMatch = prompt.match(/about "([^"]+)"/) ?? prompt.match(/concept "([^"]+)"/);
const name = nameMatch ? nameMatch[1] : 'this topic';
// Extract actual frame content for a basic summary
const frameLines = (prompt.match(/\[Frame #\d+.*?\]: .+/g) || []);
const facts = frameLines.slice(0, 8).map(line => {
const m = line.match(/\[Frame (#\d+).*?\]: (.+)/);
return m ? `- ${m[2].slice(0, 200)} *(${m[1]})*` : null;
}).filter(Boolean);
return `## Summary\nCompiled from ${frameCount} source frames about ${name}.\n\n` +
(facts.length > 0 ? `## Key Facts\n${facts.join('\n')}\n\n` : '') +
`> *Compiled with echo synthesizer. Connect an LLM for richer synthesis.*\n` +
`> Set ANTHROPIC_API_KEY or WAGGLE_OLLAMA_URL in your environment.`;
};
}
// ── Resolver ────────────────────────────────────────────────────
export interface ResolvedSynthesizer {
synthesize: LLMSynthesizeFn;
provider: 'anthropic' | 'ollama' | 'echo';
model: string;
}
/**
* Resolve the best available LLM synthesizer.
* Checks env vars and config, returns the first working option.
*/
export async function resolveSynthesizer(config?: SynthesizerConfig): Promise<ResolvedSynthesizer> {
const maxTokens = config?.maxTokens ?? 1500;
// 1. Try Anthropic
const anthropicKey = config?.anthropicApiKey
?? process.env.ANTHROPIC_API_KEY
?? process.env.WAGGLE_ANTHROPIC_API_KEY;
if (anthropicKey) {
try {
// Verify the SDK is importable
await import('@anthropic-ai/sdk');
return {
synthesize: createAnthropicSynthesizer(anthropicKey, maxTokens),
provider: 'anthropic',
model: 'claude-haiku-4-5-20251001',
};
} catch {
// SDK not available — fall through
}
}
// 2. Try Ollama
const ollamaUrl = config?.ollamaUrl ?? process.env.WAGGLE_OLLAMA_URL;
const ollamaModel = config?.ollamaModel ?? process.env.WAGGLE_OLLAMA_MODEL ?? 'llama3.2';
if (ollamaUrl) {
try {
// Quick health check
const health = await fetch(`${ollamaUrl.replace(/\/$/, '')}/api/tags`, {
signal: AbortSignal.timeout(3_000),
});
if (health.ok) {
return {
synthesize: createOllamaSynthesizer(ollamaUrl, ollamaModel, maxTokens),
provider: 'ollama',
model: ollamaModel,
};
}
} catch {
// Ollama not reachable — fall through
}
}
// 3. Echo fallback
return {
synthesize: createEchoSynthesizer(),
provider: 'echo',
model: 'echo',
};
}

View File

@@ -0,0 +1,119 @@
/**
* Wiki Compiler Types — page definitions, compilation config, and state tracking.
*/
// ── Page Types ──────────────────────────────────────────────────
export type WikiPageType =
| 'entity'
| 'concept'
| 'synthesis'
| 'index'
| 'health';
export interface WikiPageFrontmatter {
type: WikiPageType;
name: string;
entity_type?: string;
confidence: number;
sources: number;
last_compiled: string;
frame_ids: number[];
related_entities: string[];
}
export interface WikiPage {
/** URL-safe slug, e.g. "project-alpha" */
slug: string;
/** Page frontmatter */
frontmatter: WikiPageFrontmatter;
/** Full markdown content (including frontmatter as YAML) */
markdown: string;
/** SHA-256 hash of content for change detection */
contentHash: string;
}
// ── Compilation State ───────────────────────────────────────────
export interface CompilationWatermark {
/** Highest frame ID processed in last compilation */
lastFrameId: number;
/** ISO timestamp of last compilation */
lastCompiledAt: string;
/** Number of pages generated/updated */
pagesCompiled: number;
}
export interface PageRecord {
slug: string;
pageType: WikiPageType;
name: string;
contentHash: string;
markdown: string;
frameIds: string; // JSON array
compiledAt: string;
sourceCount: number;
}
// ── Compiler Configuration ──────────────────────────────────────
export type LLMSynthesizeFn = (prompt: string) => Promise<string>;
export interface CompilerConfig {
/** Function to call the LLM for synthesis */
synthesize: LLMSynthesizeFn;
/** Output directory for wiki pages (default: wiki/) */
outputDir?: string;
/** Minimum frames to justify a page (default: 2) */
minFramesPerPage?: number;
/** Maximum frames to send as context per LLM call (default: 30) */
maxFramesPerCall?: number;
/** Minimum confidence for entity pages (default: 0.3) */
minConfidence?: number;
}
// ── Compilation Result ──────────────────────────────────────────
export interface CompilationResult {
pagesCreated: number;
pagesUpdated: number;
pagesUnchanged: number;
entityPages: string[];
conceptPages: string[];
synthesisPages: string[];
healthIssues: number;
watermark: CompilationWatermark;
durationMs: number;
}
// ── Health Report ───────────────────────────────────────────────
export type HealthIssueType =
| 'contradiction'
| 'gap'
| 'orphan_entity'
| 'weak_confidence'
| 'stale_page'
| 'missing_page';
export interface HealthIssue {
type: HealthIssueType;
severity: 'high' | 'medium' | 'low';
description: string;
entity?: string;
frameIds?: number[];
suggestion?: string;
}
export interface HealthReport {
totalEntities: number;
totalFrames: number;
totalPages: number;
/** M-14: entity pages / compilable entities, 0..1. UI renders as %. */
coverage: number;
/** M-14: count of pages flagged with `stale_page` issue. */
stalePageCount: number;
issues: HealthIssue[];
dataQualityScore: number; // 0-100
compiledAt: string;
}

View File

@@ -0,0 +1,169 @@
/**
* Notion adapter unit tests (M-13)
*
* Pure-function coverage for the markdown→blocks converter, rich-text
* builder, frontmatter stripper, and page-id extractor. Network paths
* (`createNotionPage`, `writeToNotionWorkspace`) are not unit-tested —
* they need a mocked fetch and belong in an integration test.
*/
import { describe, it, expect } from 'vitest';
import {
markdownToBlocks,
toRichText,
stripFrontmatter,
extractNotionPageId,
type NotionBlock,
type NotionBlockPayload,
} from '../src/adapters/notion.js';
/**
* Read the rich-text payload a block carries under its dynamic `block.type`
* key. The block stores it as `unknown` (open-ended Notion shape), so narrow
* it to the known {@link NotionBlockPayload} at the test boundary.
*/
function payloadOf(block: NotionBlock): NotionBlockPayload {
return block[block.type] as NotionBlockPayload;
}
describe('stripFrontmatter', () => {
it('removes a leading YAML block and extracts name', () => {
const input = `---
type: entity
name: Project Alpha
confidence: 0.9
---
# Body starts here`;
const out = stripFrontmatter(input);
expect(out.title).toBe('Project Alpha');
expect(out.body.trimStart()).toBe('# Body starts here');
});
it('returns full markdown and no title when frontmatter is absent', () => {
const input = `# Just a header\n\nNo frontmatter.`;
const out = stripFrontmatter(input);
expect(out.title).toBeUndefined();
expect(out.body).toBe(input);
});
it('handles frontmatter without a name field', () => {
const input = `---\ntype: concept\n---\n\nBody`;
const out = stripFrontmatter(input);
expect(out.title).toBeUndefined();
expect(out.body.trimStart()).toBe('Body');
});
});
describe('toRichText', () => {
it('returns empty array for empty input', () => {
expect(toRichText('')).toEqual([]);
});
it('wraps plain text in a single rich_text item', () => {
const out = toRichText('Hello world');
expect(out).toEqual([{ type: 'text', text: { content: 'Hello world' } }]);
});
it('converts [text](url) into a linked rich_text item', () => {
const out = toRichText('See [Notion](https://notion.so) docs.');
expect(out).toHaveLength(3);
expect(out[0]).toEqual({ type: 'text', text: { content: 'See ' } });
expect(out[1]).toEqual({
type: 'text',
text: { content: 'Notion', link: { url: 'https://notion.so' } },
});
expect(out[2]).toEqual({ type: 'text', text: { content: ' docs.' } });
});
it('applies bold, italic, and code annotations', () => {
const out = toRichText('Plain **bold** *italic* `code` end');
// split produces: "Plain ", "**bold**", " ", "*italic*", " ", "`code`", " end"
const bold = out.find(t => t.text.content === 'bold');
const italic = out.find(t => t.text.content === 'italic');
const code = out.find(t => t.text.content === 'code');
expect(bold?.annotations?.bold).toBe(true);
expect(italic?.annotations?.italic).toBe(true);
expect(code?.annotations?.code).toBe(true);
});
});
describe('markdownToBlocks', () => {
it('maps H1/H2/H3 to heading_1/heading_2/heading_3 blocks', () => {
const md = `# H1 Heading\n\n## H2 Heading\n\n### H3 Heading`;
const blocks = markdownToBlocks(md);
expect(blocks).toHaveLength(3);
expect(blocks[0].type).toBe('heading_1');
expect(blocks[1].type).toBe('heading_2');
expect(blocks[2].type).toBe('heading_3');
expect(payloadOf(blocks[0]).rich_text[0].text.content).toBe('H1 Heading');
});
it('maps "- item" and "* item" bullets to bulleted_list_item', () => {
const md = `- First\n- Second\n* Third`;
const blocks = markdownToBlocks(md);
expect(blocks).toHaveLength(3);
for (const b of blocks) expect(b.type).toBe('bulleted_list_item');
expect(payloadOf(blocks[0]).rich_text[0].text.content).toBe('First');
});
it('maps "> quote" to quote block', () => {
const md = `> A quote line`;
const blocks = markdownToBlocks(md);
expect(blocks).toHaveLength(1);
expect(blocks[0].type).toBe('quote');
expect(payloadOf(blocks[0]).rich_text[0].text.content).toBe('A quote line');
});
it('consolidates adjacent non-special lines into a single paragraph', () => {
const md = `First line.\nSecond line continues.\n\nNew paragraph here.`;
const blocks = markdownToBlocks(md);
expect(blocks).toHaveLength(2);
expect(blocks[0].type).toBe('paragraph');
expect(blocks[1].type).toBe('paragraph');
expect(payloadOf(blocks[0]).rich_text[0].text.content).toBe('First line. Second line continues.');
});
it('flushes paragraphs when a heading interrupts the block', () => {
const md = `A paragraph line.\n# Heading\nNext paragraph.`;
const blocks = markdownToBlocks(md);
expect(blocks.map(b => b.type)).toEqual(['paragraph', 'heading_1', 'paragraph']);
});
it('preserves inline links inside paragraphs', () => {
const md = `See [Notion](https://notion.so) for more.`;
const blocks = markdownToBlocks(md);
expect(blocks[0].type).toBe('paragraph');
const richText = payloadOf(blocks[0]).rich_text;
const linkItem = richText.find((t) => t.text.link);
expect(linkItem?.text.content).toBe('Notion');
expect(linkItem?.text.link?.url).toBe('https://notion.so');
});
it('returns empty array for empty input', () => {
expect(markdownToBlocks('')).toEqual([]);
expect(markdownToBlocks(' \n\n ')).toEqual([]);
});
});
describe('extractNotionPageId', () => {
it('accepts a dashed UUID', () => {
const id = '12345678-1234-1234-1234-123456789abc';
expect(extractNotionPageId(id)).toBe(id);
});
it('accepts an undashed 32-hex id', () => {
const id = '123456781234123412341234567890ab';
expect(extractNotionPageId(id)).toBe(id);
});
it('extracts the id from a notion URL', () => {
const url = 'https://www.notion.so/Workspace/Some-Page-Title-123456781234123412341234567890ab';
expect(extractNotionPageId(url)).toBe('123456781234123412341234567890ab');
});
it('returns null for a non-hex input', () => {
expect(extractNotionPageId('not-a-notion-page')).toBeNull();
expect(extractNotionPageId('')).toBeNull();
});
});

View File

@@ -0,0 +1,210 @@
/**
* Obsidian adapter unit tests (M-12).
*
* Covers: filesystem layout, wikilink alias transform, index generation,
* idempotent re-run (files overwritten cleanly), skip of index/health pages.
*/
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as os from 'node:os';
import { writeToObsidianVault } from '../src/adapters/obsidian.js';
import type { PageRecord } from '../src/types.js';
function seedPages(): PageRecord[] {
return [
{
slug: 'project-alpha',
pageType: 'entity',
name: 'Project Alpha',
contentHash: 'h1',
markdown: `---
type: entity
name: Project Alpha
confidence: 0.9
---
# Project Alpha
Project Alpha is led by [[Marko]] and involves [[Egzakta Advisory]].
See also [[Strategy Consulting]] for broader context.
`,
frameIds: '[1,2,3]',
compiledAt: '2026-04-20T00:00:00.000Z',
sourceCount: 3,
},
{
slug: 'marko',
pageType: 'entity',
name: 'Marko',
contentHash: 'h2',
markdown: `---
type: entity
name: Marko
---
# Marko
The lead consultant.`,
frameIds: '[1]',
compiledAt: '2026-04-20T00:00:00.000Z',
sourceCount: 1,
},
{
slug: 'egzakta-advisory',
pageType: 'entity',
name: 'Egzakta Advisory',
contentHash: 'h3',
markdown: `---
type: entity
name: Egzakta Advisory
---
# Egzakta Advisory
Strategy consulting firm.`,
frameIds: '[2]',
compiledAt: '2026-04-20T00:00:00.000Z',
sourceCount: 1,
},
{
slug: 'strategy-consulting',
pageType: 'concept',
name: 'Strategy Consulting',
contentHash: 'h4',
markdown: `---
type: concept
---
# Strategy Consulting
Services offered.`,
frameIds: '[3]',
compiledAt: '2026-04-20T00:00:00.000Z',
sourceCount: 1,
},
{
// Virtual index page — should be SKIPPED by the writer.
slug: 'index',
pageType: 'index',
name: 'Wiki Index',
contentHash: 'h5',
markdown: 'should not be written',
frameIds: '[]',
compiledAt: '2026-04-20T00:00:00.000Z',
sourceCount: 0,
},
];
}
describe('writeToObsidianVault', () => {
let tmpDir: string;
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-obsidian-test-'));
});
afterEach(() => {
fs.rmSync(tmpDir, { recursive: true, force: true });
});
it('writes one file per non-virtual page in a per-type subdirectory', () => {
const result = writeToObsidianVault(seedPages(), tmpDir);
expect(result.outDir).toBe(tmpDir);
// 3 entities + 1 concept + 1 _index.md = 5 total (index/health skipped)
expect(result.filesWritten).toBe(5);
expect(result.byType.entity).toBe(3);
expect(result.byType.concept).toBe(1);
expect(result.byType.index).toBeUndefined();
expect(fs.existsSync(path.join(tmpDir, 'entity', 'project-alpha.md'))).toBe(true);
expect(fs.existsSync(path.join(tmpDir, 'entity', 'marko.md'))).toBe(true);
expect(fs.existsSync(path.join(tmpDir, 'entity', 'egzakta-advisory.md'))).toBe(true);
expect(fs.existsSync(path.join(tmpDir, 'concept', 'strategy-consulting.md'))).toBe(true);
expect(fs.existsSync(path.join(tmpDir, '_index.md'))).toBe(true);
});
it('transforms [[Display Name]] wikilinks to [[slug|Display Name]]', () => {
writeToObsidianVault(seedPages(), tmpDir);
const content = fs.readFileSync(path.join(tmpDir, 'entity', 'project-alpha.md'), 'utf-8');
// Linked-by-name entities get the alias form.
expect(content).toContain('[[marko|Marko]]');
expect(content).toContain('[[egzakta-advisory|Egzakta Advisory]]');
expect(content).toContain('[[strategy-consulting|Strategy Consulting]]');
// Raw display-name wikilinks should NOT remain.
expect(content).not.toContain('[[Marko]]');
expect(content).not.toContain('[[Egzakta Advisory]]');
});
it('leaves already-slug wikilinks alone', () => {
const pages: PageRecord[] = [{
slug: 'main',
pageType: 'entity',
name: 'Main',
contentHash: 'h',
markdown: `# Main\n\nLinks to [[marko]] which is already a slug.`,
frameIds: '[]',
compiledAt: '2026-04-20T00:00:00.000Z',
sourceCount: 1,
}, {
slug: 'marko',
pageType: 'entity',
name: 'Marko',
contentHash: 'h2',
markdown: '# Marko',
frameIds: '[]',
compiledAt: '2026-04-20T00:00:00.000Z',
sourceCount: 1,
}];
writeToObsidianVault(pages, tmpDir);
const content = fs.readFileSync(path.join(tmpDir, 'entity', 'main.md'), 'utf-8');
expect(content).toContain('[[marko]]');
expect(content).not.toContain('[[marko|marko]]');
});
it('writes _index.md with pages grouped by type', () => {
writeToObsidianVault(seedPages(), tmpDir);
const indexContent = fs.readFileSync(path.join(tmpDir, '_index.md'), 'utf-8');
expect(indexContent).toContain('# Waggle Wiki Index');
expect(indexContent).toContain('## Entities (3)');
expect(indexContent).toContain('## Concepts (1)');
expect(indexContent).toContain('[[project-alpha|Project Alpha]]');
expect(indexContent).toContain('[[strategy-consulting|Strategy Consulting]]');
// The virtual index page should NOT appear in the index itself.
expect(indexContent).not.toContain('[[index|');
});
it('preserves YAML frontmatter as-is', () => {
writeToObsidianVault(seedPages(), tmpDir);
const content = fs.readFileSync(path.join(tmpDir, 'entity', 'project-alpha.md'), 'utf-8');
expect(content).toMatch(/^---\n/);
expect(content).toMatch(/type: entity\nname: Project Alpha\nconfidence: 0\.9\n---/);
});
it('is idempotent — re-running overwrites files cleanly', () => {
writeToObsidianVault(seedPages(), tmpDir);
const first = fs.readFileSync(path.join(tmpDir, 'entity', 'project-alpha.md'), 'utf-8');
// Mutate one page's markdown and rerun.
const pages = seedPages();
pages[0].markdown = pages[0].markdown.replace('Strategy Consulting', 'Ops Consulting');
writeToObsidianVault(pages, tmpDir);
const second = fs.readFileSync(path.join(tmpDir, 'entity', 'project-alpha.md'), 'utf-8');
expect(second).not.toBe(first);
expect(second).toContain('Ops Consulting');
});
it('creates outDir if it does not exist', () => {
const nested = path.join(tmpDir, 'does', 'not', 'exist', 'yet');
expect(fs.existsSync(nested)).toBe(false);
const result = writeToObsidianVault(seedPages(), nested);
expect(fs.existsSync(nested)).toBe(true);
expect(result.filesWritten).toBeGreaterThan(0);
});
});

View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "ESNext",
"moduleResolution": "bundler",
"lib": ["ES2022"],
"outDir": "dist",
"rootDir": "src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true,
"resolveJsonModule": true,
"declaration": true,
"declarationMap": true,
"sourceMap": true,
"composite": true
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "dist", "tests"]
}