/** * Waggle Marketplace — Database Access Layer * * SQLite database interface for the marketplace catalog. * Uses better-sqlite3 (same driver Waggle core uses for .mind files). */ import Database from 'better-sqlite3'; import { join } from 'path'; import { homedir } from 'os'; import { seedMcpServers } from './mcp-registry.js'; import type { MarketplacePackage, MarketplaceSource, MarketplacePack, Installation, InstalledPackageRow, PackageUpsertInput, SearchOptions, SearchResult, SearchSort, } from './types.js'; const DEFAULT_DB_PATH = join(homedir(), '.waggle', 'marketplace.db'); export class MarketplaceDB { private db: Database.Database; constructor(dbPath: string = DEFAULT_DB_PATH) { this.db = new Database(dbPath, { readonly: false }); this.db.pragma('journal_mode = WAL'); this.db.pragma('foreign_keys = ON'); // Auto-migrate: ensure is_custom column exists on sources table this.migrateSchema(); // Seed MCP servers with proper install_manifest only on production DBs // (skip for fresh/empty test DBs to avoid interfering with test expectations) try { const count = (this.db.prepare('SELECT COUNT(*) as cnt FROM packages').get() as { cnt: number })?.cnt ?? 0; if (count > 0) seedMcpServers(this); } catch { /* packages table may not exist in empty DBs */ } } /** * Internal escape hatch for sibling marketplace modules (sync, seed, * mcp-registry) that need to run raw SQL not covered by the typed API. * Returns the underlying better-sqlite3 handle. Not part of the public * marketplace surface — prefer the typed methods above where they exist. */ getRawDb(): Database.Database { return this.db; } /** * Apply any necessary schema migrations. * Safe to call on all database versions. */ private migrateSchema(): void { // Migration: add is_custom column to sources try { this.db.prepare("SELECT is_custom FROM sources LIMIT 0").run(); } catch { try { this.db.prepare("ALTER TABLE sources ADD COLUMN is_custom BOOLEAN DEFAULT 0").run(); } catch { // Table might not exist yet (empty DB) — skip migration } } // Migration: add sync_state column to sources (JSON blob for resumable sync) try { this.db.prepare("SELECT sync_state FROM sources LIMIT 0").run(); } catch { try { this.db.prepare("ALTER TABLE sources ADD COLUMN sync_state TEXT DEFAULT NULL").run(); } catch { // Table might not exist yet (empty DB) — skip migration } } } // ─── Package Queries ────────────────────────────────────────────── /** * Search packages using FTS5 full-text search + faceted filtering. */ search(options: SearchOptions = {}): SearchResult { const { query, type, category, pack, source, sort, limit = 50, offset = 0 } = options; const conditions: string[] = []; const params: Record = {}; // Full-text search via FTS5. The raw caller string is NEVER passed // straight into MATCH — it may be a verbose natural-language `need` // (from the agent's acquire_capability path) or contain FTS5 operators // / paths like `D:\X` that implicit-AND to zero matches or raise a // syntax error. toFtsMatchQuery() relaxes it into a safe OR-of-prefix // expression; null means "no usable terms" → fall back to an // unfiltered listing instead of throwing. let baseQuery: string; const ftsExpr = query ? this.toFtsMatchQuery(query) : null; if (ftsExpr) { baseQuery = ` SELECT p.* FROM packages p INNER JOIN packages_fts fts ON p.id = fts.rowid WHERE packages_fts MATCH @query `; params.query = ftsExpr; } else { baseQuery = `SELECT p.* FROM packages p WHERE 1=1`; } // Faceted filters if (type) { conditions.push(`p.waggle_install_type = @type`); params.type = type; } if (category) { conditions.push(`p.category = @category`); params.category = category; } if (pack) { conditions.push(`EXISTS ( SELECT 1 FROM pack_packages pp INNER JOIN packs pk ON pk.id = pp.pack_id WHERE pp.package_id = p.id AND pk.slug = @pack )`); params.pack = pack; } if (source) { conditions.push(`EXISTS ( SELECT 1 FROM sources s WHERE s.id = p.source_id AND s.name = @source )`); params.source = source; } const whereClause = conditions.length > 0 ? ` AND ${conditions.join(' AND ')}` : ''; // Get total count const countQuery = baseQuery.replace('SELECT p.*', 'SELECT COUNT(*) as total') + whereClause; const total = (this.db.prepare(countQuery).get(params) as { total: number }).total; // Determine sort order // Order by FTS rank only when we actually took the FTS branch — `rank` // is an FTS5-only column and is absent on the unfiltered fallback. const orderClause = this.buildOrderClause(sort, !!ftsExpr); // Get packages const fullQuery = baseQuery + whereClause + ` ${orderClause} LIMIT @limit OFFSET @offset`; params.limit = limit; params.offset = offset; const packages = this.db.prepare(fullQuery).all(params) as MarketplacePackage[]; // Parse JSON fields const parsed = packages.map(p => this.parsePackageJson(p)); // Build facets (on unfiltered result set) const facets = this.buildFacets(baseQuery + whereClause, params); // Get installed count const installedCount = this.getInstalledCount(); return { packages: parsed, total, facets, installedCount }; } /** * Get a single package by ID. */ getPackage(id: number): MarketplacePackage | null { const row = this.db.prepare('SELECT * FROM packages WHERE id = ?').get(id) as MarketplacePackage | undefined; return row ? this.parsePackageJson(row) : null; } /** * Get a single package by name. */ getPackageByName(name: string): MarketplacePackage | null { const row = this.db.prepare('SELECT * FROM packages WHERE name = ?').get(name) as MarketplacePackage | undefined; return row ? this.parsePackageJson(row) : null; } /** * Get all packages in a given pack. */ getPacksBySlug(slug: string): { pack: MarketplacePack; packages: MarketplacePackage[] } | null { const pack = this.db.prepare('SELECT * FROM packs WHERE slug = ?').get(slug) as MarketplacePack | undefined; if (!pack) return null; const packages = this.db.prepare(` SELECT p.*, pp.is_core FROM packages p INNER JOIN pack_packages pp ON pp.package_id = p.id WHERE pp.pack_id = ? ORDER BY pp.is_core DESC, p.downloads DESC `).all(pack.id) as (MarketplacePackage & { is_core: boolean })[]; return { pack: { ...pack, connectors_needed: JSON.parse(pack.connectors_needed as unknown as string || '[]') }, packages: packages.map(p => this.parsePackageJson(p)), }; } /** * List all available packs. */ listPacks(): MarketplacePack[] { const rows = this.db.prepare('SELECT * FROM packs ORDER BY priority, display_name').all() as MarketplacePack[]; return rows.map(p => ({ ...p, connectors_needed: JSON.parse(p.connectors_needed as unknown as string || '[]'), })); } /** * List all sources. */ listSources(): MarketplaceSource[] { return this.db.prepare(` SELECT *, COALESCE(is_custom, 0) as is_custom FROM sources ORDER BY total_packages DESC `).all() as MarketplaceSource[]; } /** * List all sources with package counts derived from the packages table. */ listSourcesWithCounts(): (MarketplaceSource & { package_count: number })[] { return this.db.prepare(` SELECT s.*, COALESCE(s.is_custom, 0) as is_custom, COUNT(p.id) as package_count FROM sources s LEFT JOIN packages p ON p.source_id = s.id GROUP BY s.id ORDER BY package_count DESC, s.display_name ASC `).all() as (MarketplaceSource & { package_count: number })[]; } /** * Get a single source by ID. */ getSource(id: number): MarketplaceSource | null { const row = this.db.prepare( 'SELECT *, COALESCE(is_custom, 0) as is_custom FROM sources WHERE id = ?' ).get(id) as MarketplaceSource | undefined; return row ?? null; } /** * Get a single source by name. */ getSourceByName(name: string): MarketplaceSource | null { const row = this.db.prepare( 'SELECT *, COALESCE(is_custom, 0) as is_custom FROM sources WHERE name = ?' ).get(name) as MarketplaceSource | undefined; return row ?? null; } /** * Add a new user-defined source. * Returns the source ID. */ addSource(source: { name: string; display_name: string; url: string; source_type: string; platform?: string; install_method?: string; api_endpoint?: string | null; description?: string; }): number { const stmt = this.db.prepare(` INSERT INTO sources (name, display_name, url, source_type, platform, total_packages, install_method, api_endpoint, description, is_custom) VALUES (@name, @display_name, @url, @source_type, @platform, 0, @install_method, @api_endpoint, @description, 1) `); const result = stmt.run({ name: source.name, display_name: source.display_name, url: source.url, source_type: source.source_type, platform: source.platform || 'waggle', install_method: source.install_method || 'git_clone', api_endpoint: source.api_endpoint ?? null, description: source.description || '', }); return result.lastInsertRowid as number; } /** * Delete a user-defined source and all its packages. * Returns true if deleted, false if source not found or is built-in. */ deleteSource(id: number): boolean { const source = this.getSource(id); if (!source) return false; if (!source.is_custom) return false; // Delete packages belonging to this source first this.db.prepare('DELETE FROM packages WHERE source_id = ?').run(id); // Delete the source const result = this.db.prepare('DELETE FROM sources WHERE id = ? AND is_custom = 1').run(id); return result.changes > 0; } /** * Ensure the sources table has the is_custom column. * Safe to call on existing databases — no-ops if column already exists. */ ensureIsCustomColumn(): void { try { this.db.prepare("SELECT is_custom FROM sources LIMIT 1").get(); } catch { // Column doesn't exist — add it this.db.prepare("ALTER TABLE sources ADD COLUMN is_custom BOOLEAN DEFAULT 0").run(); } } // ─── Sync State (resumable sync for rate-limited sources) ──────── /** * Get the sync state for a source (used for resumable pagination). * Returns null if no state is saved. */ getSyncState(sourceId: number): Record | null { const row = this.db.prepare( 'SELECT sync_state FROM sources WHERE id = ?' ).get(sourceId) as { sync_state: string | null } | undefined; if (!row || !row.sync_state) return null; try { return JSON.parse(row.sync_state); } catch { return null; } } /** * Save sync state for a source (used for resumable pagination). * Pass null to clear the state (e.g., when a full sync completes). */ setSyncState(sourceId: number, state: Record | null): void { this.db.prepare( 'UPDATE sources SET sync_state = ? WHERE id = ?' ).run(state ? JSON.stringify(state) : null, sourceId); } // ─── Installation Tracking ──────────────────────────────────────── /** * Record a successful installation. */ recordInstallation( packageId: number, version: string, installPath: string, config: Record = {}, ): Installation { const stmt = this.db.prepare(` INSERT INTO installations (package_id, installed_version, installed_at, install_path, status, config) VALUES (?, ?, datetime('now'), ?, 'installed', ?) `); const result = stmt.run(packageId, version, installPath, JSON.stringify(config)); return this.getInstallation(result.lastInsertRowid as number)!; } /** * Get installation by ID. */ getInstallation(id: number): Installation | null { const row = this.db.prepare('SELECT * FROM installations WHERE id = ?').get(id) as Installation | undefined; if (!row) return null; return { ...row, config: JSON.parse(row.config as unknown as string || '{}') }; } /** * Get all active installations. */ listInstallations(): InstalledPackageRow[] { return this.db.prepare(` SELECT i.*, p.name as pkg_name, p.display_name as pkg_display_name, p.waggle_install_type, p.category FROM installations i INNER JOIN packages p ON p.id = i.package_id WHERE i.status = 'installed' ORDER BY i.installed_at DESC `).all() as InstalledPackageRow[]; } /** * Check if a package is already installed. */ isInstalled(packageId: number): boolean { const row = this.db.prepare( `SELECT 1 FROM installations WHERE package_id = ? AND status = 'installed'` ).get(packageId); return !!row; } /** * Mark an installation as uninstalled. */ markUninstalled(packageId: number): void { this.db.prepare( `UPDATE installations SET status = 'uninstalled' WHERE package_id = ? AND status = 'installed'` ).run(packageId); } // ─── Upsert (for sync) ─────────────────────────────────────────── /** * Insert or update a package (used by sync scripts). */ upsertPackage(pkg: PackageUpsertInput): number { const existing = this.db.prepare('SELECT id FROM packages WHERE name = ? AND source_id = ?') .get(pkg.name, pkg.source_id) as { id: number } | undefined; if (existing) { const sets: string[] = []; const params: Record = { id: existing.id }; for (const [key, value] of Object.entries(pkg)) { if (key === 'id' || key === 'name' || key === 'source_id') continue; sets.push(`${key} = @${key}`); params[key] = typeof value === 'object' ? JSON.stringify(value) : value; } sets.push(`updated_at = datetime('now')`); this.db.prepare(`UPDATE packages SET ${sets.join(', ')} WHERE id = @id`).run(params); return existing.id; } else { const cols = Object.keys(pkg); const vals = cols.map(c => `@${c}`); const params: Record = {}; for (const [key, value] of Object.entries(pkg)) { params[key] = typeof value === 'object' ? JSON.stringify(value) : value; } const result = this.db.prepare( `INSERT INTO packages (${cols.join(', ')}) VALUES (${vals.join(', ')})` ).run(params); return result.lastInsertRowid as number; } } // ─── Helpers ────────────────────────────────────────────────────── private parsePackageJson(pkg: MarketplacePackage): MarketplacePackage { return { ...pkg, install_manifest: typeof pkg.install_manifest === 'string' ? JSON.parse(pkg.install_manifest) : pkg.install_manifest, platforms: typeof pkg.platforms === 'string' ? JSON.parse(pkg.platforms) : pkg.platforms || [], dependencies: typeof pkg.dependencies === 'string' ? JSON.parse(pkg.dependencies) : pkg.dependencies || [], packs: typeof pkg.packs === 'string' ? JSON.parse(pkg.packs) : pkg.packs || [], }; } /** * Relax arbitrary caller text into a safe FTS5 MATCH expression. * * Extracts alphanumeric tokens, drops <2-char noise, dedupes, caps the * term count, then OR-joins as prefix terms (`token*`). OR (not the FTS5 * implicit AND) so a verbose `need` still matches on any salient word, * with BM25 `rank` floating the best package up. Returns null when no * usable token remains so the caller degrades to an unfiltered listing * instead of throwing an FTS5 syntax error. */ private toFtsMatchQuery(raw: string): string | null { const tokens = raw .toLowerCase() .split(/[^a-z0-9]+/) .filter(t => t.length >= 2); if (tokens.length === 0) return null; const unique = [...new Set(tokens)].slice(0, 24); // cap term explosion return unique.map(t => `${t}*`).join(' OR '); } private buildOrderClause(sort?: SearchSort, hasQuery?: boolean): string { switch (sort) { case 'popular': return 'ORDER BY p.downloads DESC, p.stars DESC'; case 'recent': return 'ORDER BY p.updated_at DESC, p.created_at DESC'; case 'name': return 'ORDER BY p.display_name ASC'; case 'relevance': // When using FTS, the default rank is relevance; fallback to downloads return hasQuery ? 'ORDER BY rank, p.downloads DESC' : 'ORDER BY p.downloads DESC, p.stars DESC'; default: // Default: if query present use relevance, else popularity return hasQuery ? 'ORDER BY rank, p.downloads DESC' : 'ORDER BY p.downloads DESC, p.stars DESC'; } } /** * Count the total number of installed packages. */ getInstalledCount(): number { const row = this.db.prepare( `SELECT COUNT(*) as cnt FROM installations WHERE status = 'installed'` ).get() as { cnt: number }; return row.cnt; } private buildFacets(baseQuery: string, params: Record) { const facetQuery = baseQuery.replace( /SELECT p\.\*/, `SELECT p.waggle_install_type, p.category, s.name as source_name` ).replace( /FROM packages p/, `FROM packages p LEFT JOIN sources s ON s.id = p.source_id` ); const rows = this.db.prepare(facetQuery).all(params) as { waggle_install_type: string; category: string; source_name: string; }[]; const types: Record = {}; const categories: Record = {}; const sources: Record = {}; for (const row of rows) { types[row.waggle_install_type] = (types[row.waggle_install_type] || 0) + 1; categories[row.category] = (categories[row.category] || 0) + 1; if (row.source_name) sources[row.source_name] = (sources[row.source_name] || 0) + 1; } return { types, categories, sources }; } /** * Close the database connection. */ close(): void { this.db.close(); } }