542 lines
18 KiB
TypeScript
542 lines
18 KiB
TypeScript
/**
|
|
* Waggle Marketplace — Database Access Layer
|
|
*
|
|
* SQLite database interface for the marketplace catalog.
|
|
* Uses better-sqlite3 (same driver Waggle core uses for .mind files).
|
|
*/
|
|
|
|
import Database from 'better-sqlite3';
|
|
import { join } from 'path';
|
|
import { homedir } from 'os';
|
|
import { seedMcpServers } from './mcp-registry.js';
|
|
import type {
|
|
MarketplacePackage,
|
|
MarketplaceSource,
|
|
MarketplacePack,
|
|
Installation,
|
|
InstalledPackageRow,
|
|
PackageUpsertInput,
|
|
SearchOptions,
|
|
SearchResult,
|
|
SearchSort,
|
|
} from './types.js';
|
|
|
|
const DEFAULT_DB_PATH = join(homedir(), '.waggle', 'marketplace.db');
|
|
|
|
export class MarketplaceDB {
|
|
private db: Database.Database;
|
|
|
|
constructor(dbPath: string = DEFAULT_DB_PATH) {
|
|
this.db = new Database(dbPath, { readonly: false });
|
|
this.db.pragma('journal_mode = WAL');
|
|
this.db.pragma('foreign_keys = ON');
|
|
// Auto-migrate: ensure is_custom column exists on sources table
|
|
this.migrateSchema();
|
|
// Seed MCP servers with proper install_manifest only on production DBs
|
|
// (skip for fresh/empty test DBs to avoid interfering with test expectations)
|
|
try {
|
|
const count = (this.db.prepare('SELECT COUNT(*) as cnt FROM packages').get() as { cnt: number })?.cnt ?? 0;
|
|
if (count > 0) seedMcpServers(this);
|
|
} catch { /* packages table may not exist in empty DBs */ }
|
|
}
|
|
|
|
/**
|
|
* Internal escape hatch for sibling marketplace modules (sync, seed,
|
|
* mcp-registry) that need to run raw SQL not covered by the typed API.
|
|
* Returns the underlying better-sqlite3 handle. Not part of the public
|
|
* marketplace surface — prefer the typed methods above where they exist.
|
|
*/
|
|
getRawDb(): Database.Database {
|
|
return this.db;
|
|
}
|
|
|
|
/**
|
|
* Apply any necessary schema migrations.
|
|
* Safe to call on all database versions.
|
|
*/
|
|
private migrateSchema(): void {
|
|
// Migration: add is_custom column to sources
|
|
try {
|
|
this.db.prepare("SELECT is_custom FROM sources LIMIT 0").run();
|
|
} catch {
|
|
try {
|
|
this.db.prepare("ALTER TABLE sources ADD COLUMN is_custom BOOLEAN DEFAULT 0").run();
|
|
} catch {
|
|
// Table might not exist yet (empty DB) — skip migration
|
|
}
|
|
}
|
|
|
|
// Migration: add sync_state column to sources (JSON blob for resumable sync)
|
|
try {
|
|
this.db.prepare("SELECT sync_state FROM sources LIMIT 0").run();
|
|
} catch {
|
|
try {
|
|
this.db.prepare("ALTER TABLE sources ADD COLUMN sync_state TEXT DEFAULT NULL").run();
|
|
} catch {
|
|
// Table might not exist yet (empty DB) — skip migration
|
|
}
|
|
}
|
|
}
|
|
|
|
// ─── Package Queries ──────────────────────────────────────────────
|
|
|
|
/**
|
|
* Search packages using FTS5 full-text search + faceted filtering.
|
|
*/
|
|
search(options: SearchOptions = {}): SearchResult {
|
|
const { query, type, category, pack, source, sort, limit = 50, offset = 0 } = options;
|
|
const conditions: string[] = [];
|
|
const params: Record<string, unknown> = {};
|
|
|
|
// Full-text search via FTS5. The raw caller string is NEVER passed
|
|
// straight into MATCH — it may be a verbose natural-language `need`
|
|
// (from the agent's acquire_capability path) or contain FTS5 operators
|
|
// / paths like `D:\X` that implicit-AND to zero matches or raise a
|
|
// syntax error. toFtsMatchQuery() relaxes it into a safe OR-of-prefix
|
|
// expression; null means "no usable terms" → fall back to an
|
|
// unfiltered listing instead of throwing.
|
|
let baseQuery: string;
|
|
const ftsExpr = query ? this.toFtsMatchQuery(query) : null;
|
|
if (ftsExpr) {
|
|
baseQuery = `
|
|
SELECT p.* FROM packages p
|
|
INNER JOIN packages_fts fts ON p.id = fts.rowid
|
|
WHERE packages_fts MATCH @query
|
|
`;
|
|
params.query = ftsExpr;
|
|
} else {
|
|
baseQuery = `SELECT p.* FROM packages p WHERE 1=1`;
|
|
}
|
|
|
|
// Faceted filters
|
|
if (type) {
|
|
conditions.push(`p.waggle_install_type = @type`);
|
|
params.type = type;
|
|
}
|
|
if (category) {
|
|
conditions.push(`p.category = @category`);
|
|
params.category = category;
|
|
}
|
|
if (pack) {
|
|
conditions.push(`EXISTS (
|
|
SELECT 1 FROM pack_packages pp
|
|
INNER JOIN packs pk ON pk.id = pp.pack_id
|
|
WHERE pp.package_id = p.id AND pk.slug = @pack
|
|
)`);
|
|
params.pack = pack;
|
|
}
|
|
if (source) {
|
|
conditions.push(`EXISTS (
|
|
SELECT 1 FROM sources s
|
|
WHERE s.id = p.source_id AND s.name = @source
|
|
)`);
|
|
params.source = source;
|
|
}
|
|
|
|
const whereClause = conditions.length > 0
|
|
? ` AND ${conditions.join(' AND ')}`
|
|
: '';
|
|
|
|
// Get total count
|
|
const countQuery = baseQuery.replace('SELECT p.*', 'SELECT COUNT(*) as total') + whereClause;
|
|
const total = (this.db.prepare(countQuery).get(params) as { total: number }).total;
|
|
|
|
// Determine sort order
|
|
// Order by FTS rank only when we actually took the FTS branch — `rank`
|
|
// is an FTS5-only column and is absent on the unfiltered fallback.
|
|
const orderClause = this.buildOrderClause(sort, !!ftsExpr);
|
|
|
|
// Get packages
|
|
const fullQuery = baseQuery + whereClause + ` ${orderClause} LIMIT @limit OFFSET @offset`;
|
|
params.limit = limit;
|
|
params.offset = offset;
|
|
const packages = this.db.prepare(fullQuery).all(params) as MarketplacePackage[];
|
|
|
|
// Parse JSON fields
|
|
const parsed = packages.map(p => this.parsePackageJson(p));
|
|
|
|
// Build facets (on unfiltered result set)
|
|
const facets = this.buildFacets(baseQuery + whereClause, params);
|
|
|
|
// Get installed count
|
|
const installedCount = this.getInstalledCount();
|
|
|
|
return { packages: parsed, total, facets, installedCount };
|
|
}
|
|
|
|
/**
|
|
* Get a single package by ID.
|
|
*/
|
|
getPackage(id: number): MarketplacePackage | null {
|
|
const row = this.db.prepare('SELECT * FROM packages WHERE id = ?').get(id) as MarketplacePackage | undefined;
|
|
return row ? this.parsePackageJson(row) : null;
|
|
}
|
|
|
|
/**
|
|
* Get a single package by name.
|
|
*/
|
|
getPackageByName(name: string): MarketplacePackage | null {
|
|
const row = this.db.prepare('SELECT * FROM packages WHERE name = ?').get(name) as MarketplacePackage | undefined;
|
|
return row ? this.parsePackageJson(row) : null;
|
|
}
|
|
|
|
/**
|
|
* Get all packages in a given pack.
|
|
*/
|
|
getPacksBySlug(slug: string): { pack: MarketplacePack; packages: MarketplacePackage[] } | null {
|
|
const pack = this.db.prepare('SELECT * FROM packs WHERE slug = ?').get(slug) as MarketplacePack | undefined;
|
|
if (!pack) return null;
|
|
|
|
const packages = this.db.prepare(`
|
|
SELECT p.*, pp.is_core FROM packages p
|
|
INNER JOIN pack_packages pp ON pp.package_id = p.id
|
|
WHERE pp.pack_id = ?
|
|
ORDER BY pp.is_core DESC, p.downloads DESC
|
|
`).all(pack.id) as (MarketplacePackage & { is_core: boolean })[];
|
|
|
|
return {
|
|
pack: { ...pack, connectors_needed: JSON.parse(pack.connectors_needed as unknown as string || '[]') },
|
|
packages: packages.map(p => this.parsePackageJson(p)),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* List all available packs.
|
|
*/
|
|
listPacks(): MarketplacePack[] {
|
|
const rows = this.db.prepare('SELECT * FROM packs ORDER BY priority, display_name').all() as MarketplacePack[];
|
|
return rows.map(p => ({
|
|
...p,
|
|
connectors_needed: JSON.parse(p.connectors_needed as unknown as string || '[]'),
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* List all sources.
|
|
*/
|
|
listSources(): MarketplaceSource[] {
|
|
return this.db.prepare(`
|
|
SELECT *, COALESCE(is_custom, 0) as is_custom FROM sources ORDER BY total_packages DESC
|
|
`).all() as MarketplaceSource[];
|
|
}
|
|
|
|
/**
|
|
* List all sources with package counts derived from the packages table.
|
|
*/
|
|
listSourcesWithCounts(): (MarketplaceSource & { package_count: number })[] {
|
|
return this.db.prepare(`
|
|
SELECT s.*, COALESCE(s.is_custom, 0) as is_custom,
|
|
COUNT(p.id) as package_count
|
|
FROM sources s
|
|
LEFT JOIN packages p ON p.source_id = s.id
|
|
GROUP BY s.id
|
|
ORDER BY package_count DESC, s.display_name ASC
|
|
`).all() as (MarketplaceSource & { package_count: number })[];
|
|
}
|
|
|
|
/**
|
|
* Get a single source by ID.
|
|
*/
|
|
getSource(id: number): MarketplaceSource | null {
|
|
const row = this.db.prepare(
|
|
'SELECT *, COALESCE(is_custom, 0) as is_custom FROM sources WHERE id = ?'
|
|
).get(id) as MarketplaceSource | undefined;
|
|
return row ?? null;
|
|
}
|
|
|
|
/**
|
|
* Get a single source by name.
|
|
*/
|
|
getSourceByName(name: string): MarketplaceSource | null {
|
|
const row = this.db.prepare(
|
|
'SELECT *, COALESCE(is_custom, 0) as is_custom FROM sources WHERE name = ?'
|
|
).get(name) as MarketplaceSource | undefined;
|
|
return row ?? null;
|
|
}
|
|
|
|
/**
|
|
* Add a new user-defined source.
|
|
* Returns the source ID.
|
|
*/
|
|
addSource(source: {
|
|
name: string;
|
|
display_name: string;
|
|
url: string;
|
|
source_type: string;
|
|
platform?: string;
|
|
install_method?: string;
|
|
api_endpoint?: string | null;
|
|
description?: string;
|
|
}): number {
|
|
const stmt = this.db.prepare(`
|
|
INSERT INTO sources (name, display_name, url, source_type, platform, total_packages, install_method, api_endpoint, description, is_custom)
|
|
VALUES (@name, @display_name, @url, @source_type, @platform, 0, @install_method, @api_endpoint, @description, 1)
|
|
`);
|
|
const result = stmt.run({
|
|
name: source.name,
|
|
display_name: source.display_name,
|
|
url: source.url,
|
|
source_type: source.source_type,
|
|
platform: source.platform || 'waggle',
|
|
install_method: source.install_method || 'git_clone',
|
|
api_endpoint: source.api_endpoint ?? null,
|
|
description: source.description || '',
|
|
});
|
|
return result.lastInsertRowid as number;
|
|
}
|
|
|
|
/**
|
|
* Delete a user-defined source and all its packages.
|
|
* Returns true if deleted, false if source not found or is built-in.
|
|
*/
|
|
deleteSource(id: number): boolean {
|
|
const source = this.getSource(id);
|
|
if (!source) return false;
|
|
if (!source.is_custom) return false;
|
|
|
|
// Delete packages belonging to this source first
|
|
this.db.prepare('DELETE FROM packages WHERE source_id = ?').run(id);
|
|
// Delete the source
|
|
const result = this.db.prepare('DELETE FROM sources WHERE id = ? AND is_custom = 1').run(id);
|
|
return result.changes > 0;
|
|
}
|
|
|
|
/**
|
|
* Ensure the sources table has the is_custom column.
|
|
* Safe to call on existing databases — no-ops if column already exists.
|
|
*/
|
|
ensureIsCustomColumn(): void {
|
|
try {
|
|
this.db.prepare("SELECT is_custom FROM sources LIMIT 1").get();
|
|
} catch {
|
|
// Column doesn't exist — add it
|
|
this.db.prepare("ALTER TABLE sources ADD COLUMN is_custom BOOLEAN DEFAULT 0").run();
|
|
}
|
|
}
|
|
|
|
// ─── Sync State (resumable sync for rate-limited sources) ────────
|
|
|
|
/**
|
|
* Get the sync state for a source (used for resumable pagination).
|
|
* Returns null if no state is saved.
|
|
*/
|
|
getSyncState(sourceId: number): Record<string, unknown> | null {
|
|
const row = this.db.prepare(
|
|
'SELECT sync_state FROM sources WHERE id = ?'
|
|
).get(sourceId) as { sync_state: string | null } | undefined;
|
|
if (!row || !row.sync_state) return null;
|
|
try {
|
|
return JSON.parse(row.sync_state);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Save sync state for a source (used for resumable pagination).
|
|
* Pass null to clear the state (e.g., when a full sync completes).
|
|
*/
|
|
setSyncState(sourceId: number, state: Record<string, unknown> | null): void {
|
|
this.db.prepare(
|
|
'UPDATE sources SET sync_state = ? WHERE id = ?'
|
|
).run(state ? JSON.stringify(state) : null, sourceId);
|
|
}
|
|
|
|
// ─── Installation Tracking ────────────────────────────────────────
|
|
|
|
/**
|
|
* Record a successful installation.
|
|
*/
|
|
recordInstallation(
|
|
packageId: number,
|
|
version: string,
|
|
installPath: string,
|
|
config: Record<string, unknown> = {},
|
|
): Installation {
|
|
const stmt = this.db.prepare(`
|
|
INSERT INTO installations (package_id, installed_version, installed_at, install_path, status, config)
|
|
VALUES (?, ?, datetime('now'), ?, 'installed', ?)
|
|
`);
|
|
const result = stmt.run(packageId, version, installPath, JSON.stringify(config));
|
|
return this.getInstallation(result.lastInsertRowid as number)!;
|
|
}
|
|
|
|
/**
|
|
* Get installation by ID.
|
|
*/
|
|
getInstallation(id: number): Installation | null {
|
|
const row = this.db.prepare('SELECT * FROM installations WHERE id = ?').get(id) as Installation | undefined;
|
|
if (!row) return null;
|
|
return { ...row, config: JSON.parse(row.config as unknown as string || '{}') };
|
|
}
|
|
|
|
/**
|
|
* Get all active installations.
|
|
*/
|
|
listInstallations(): InstalledPackageRow[] {
|
|
return this.db.prepare(`
|
|
SELECT i.*, p.name as pkg_name, p.display_name as pkg_display_name,
|
|
p.waggle_install_type, p.category
|
|
FROM installations i
|
|
INNER JOIN packages p ON p.id = i.package_id
|
|
WHERE i.status = 'installed'
|
|
ORDER BY i.installed_at DESC
|
|
`).all() as InstalledPackageRow[];
|
|
}
|
|
|
|
/**
|
|
* Check if a package is already installed.
|
|
*/
|
|
isInstalled(packageId: number): boolean {
|
|
const row = this.db.prepare(
|
|
`SELECT 1 FROM installations WHERE package_id = ? AND status = 'installed'`
|
|
).get(packageId);
|
|
return !!row;
|
|
}
|
|
|
|
/**
|
|
* Mark an installation as uninstalled.
|
|
*/
|
|
markUninstalled(packageId: number): void {
|
|
this.db.prepare(
|
|
`UPDATE installations SET status = 'uninstalled' WHERE package_id = ? AND status = 'installed'`
|
|
).run(packageId);
|
|
}
|
|
|
|
// ─── Upsert (for sync) ───────────────────────────────────────────
|
|
|
|
/**
|
|
* Insert or update a package (used by sync scripts).
|
|
*/
|
|
upsertPackage(pkg: PackageUpsertInput): number {
|
|
const existing = this.db.prepare('SELECT id FROM packages WHERE name = ? AND source_id = ?')
|
|
.get(pkg.name, pkg.source_id) as { id: number } | undefined;
|
|
|
|
if (existing) {
|
|
const sets: string[] = [];
|
|
const params: Record<string, unknown> = { id: existing.id };
|
|
for (const [key, value] of Object.entries(pkg)) {
|
|
if (key === 'id' || key === 'name' || key === 'source_id') continue;
|
|
sets.push(`${key} = @${key}`);
|
|
params[key] = typeof value === 'object' ? JSON.stringify(value) : value;
|
|
}
|
|
sets.push(`updated_at = datetime('now')`);
|
|
this.db.prepare(`UPDATE packages SET ${sets.join(', ')} WHERE id = @id`).run(params);
|
|
return existing.id;
|
|
} else {
|
|
const cols = Object.keys(pkg);
|
|
const vals = cols.map(c => `@${c}`);
|
|
const params: Record<string, unknown> = {};
|
|
for (const [key, value] of Object.entries(pkg)) {
|
|
params[key] = typeof value === 'object' ? JSON.stringify(value) : value;
|
|
}
|
|
const result = this.db.prepare(
|
|
`INSERT INTO packages (${cols.join(', ')}) VALUES (${vals.join(', ')})`
|
|
).run(params);
|
|
return result.lastInsertRowid as number;
|
|
}
|
|
}
|
|
|
|
// ─── Helpers ──────────────────────────────────────────────────────
|
|
|
|
private parsePackageJson(pkg: MarketplacePackage): MarketplacePackage {
|
|
return {
|
|
...pkg,
|
|
install_manifest: typeof pkg.install_manifest === 'string'
|
|
? JSON.parse(pkg.install_manifest)
|
|
: pkg.install_manifest,
|
|
platforms: typeof pkg.platforms === 'string'
|
|
? JSON.parse(pkg.platforms)
|
|
: pkg.platforms || [],
|
|
dependencies: typeof pkg.dependencies === 'string'
|
|
? JSON.parse(pkg.dependencies)
|
|
: pkg.dependencies || [],
|
|
packs: typeof pkg.packs === 'string'
|
|
? JSON.parse(pkg.packs)
|
|
: pkg.packs || [],
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Relax arbitrary caller text into a safe FTS5 MATCH expression.
|
|
*
|
|
* Extracts alphanumeric tokens, drops <2-char noise, dedupes, caps the
|
|
* term count, then OR-joins as prefix terms (`token*`). OR (not the FTS5
|
|
* implicit AND) so a verbose `need` still matches on any salient word,
|
|
* with BM25 `rank` floating the best package up. Returns null when no
|
|
* usable token remains so the caller degrades to an unfiltered listing
|
|
* instead of throwing an FTS5 syntax error.
|
|
*/
|
|
private toFtsMatchQuery(raw: string): string | null {
|
|
const tokens = raw
|
|
.toLowerCase()
|
|
.split(/[^a-z0-9]+/)
|
|
.filter(t => t.length >= 2);
|
|
if (tokens.length === 0) return null;
|
|
const unique = [...new Set(tokens)].slice(0, 24); // cap term explosion
|
|
return unique.map(t => `${t}*`).join(' OR ');
|
|
}
|
|
|
|
private buildOrderClause(sort?: SearchSort, hasQuery?: boolean): string {
|
|
switch (sort) {
|
|
case 'popular':
|
|
return 'ORDER BY p.downloads DESC, p.stars DESC';
|
|
case 'recent':
|
|
return 'ORDER BY p.updated_at DESC, p.created_at DESC';
|
|
case 'name':
|
|
return 'ORDER BY p.display_name ASC';
|
|
case 'relevance':
|
|
// When using FTS, the default rank is relevance; fallback to downloads
|
|
return hasQuery ? 'ORDER BY rank, p.downloads DESC' : 'ORDER BY p.downloads DESC, p.stars DESC';
|
|
default:
|
|
// Default: if query present use relevance, else popularity
|
|
return hasQuery ? 'ORDER BY rank, p.downloads DESC' : 'ORDER BY p.downloads DESC, p.stars DESC';
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Count the total number of installed packages.
|
|
*/
|
|
getInstalledCount(): number {
|
|
const row = this.db.prepare(
|
|
`SELECT COUNT(*) as cnt FROM installations WHERE status = 'installed'`
|
|
).get() as { cnt: number };
|
|
return row.cnt;
|
|
}
|
|
|
|
private buildFacets(baseQuery: string, params: Record<string, unknown>) {
|
|
const facetQuery = baseQuery.replace(
|
|
/SELECT p\.\*/,
|
|
`SELECT p.waggle_install_type, p.category, s.name as source_name`
|
|
).replace(
|
|
/FROM packages p/,
|
|
`FROM packages p LEFT JOIN sources s ON s.id = p.source_id`
|
|
);
|
|
|
|
const rows = this.db.prepare(facetQuery).all(params) as {
|
|
waggle_install_type: string;
|
|
category: string;
|
|
source_name: string;
|
|
}[];
|
|
|
|
const types: Record<string, number> = {};
|
|
const categories: Record<string, number> = {};
|
|
const sources: Record<string, number> = {};
|
|
|
|
for (const row of rows) {
|
|
types[row.waggle_install_type] = (types[row.waggle_install_type] || 0) + 1;
|
|
categories[row.category] = (categories[row.category] || 0) + 1;
|
|
if (row.source_name) sources[row.source_name] = (sources[row.source_name] || 0) + 1;
|
|
}
|
|
|
|
return { types, categories, sources };
|
|
}
|
|
|
|
/**
|
|
* Close the database connection.
|
|
*/
|
|
close(): void {
|
|
this.db.close();
|
|
}
|
|
}
|