This commit is contained in:
Oleg Maslov
456
packages/marketplace/tests/mcp-registry.test.ts
Normal file
456
packages/marketplace/tests/mcp-registry.test.ts
Normal file
@@ -0,0 +1,456 @@
|
||||
/**
|
||||
* MCP Server Registry — Tests
|
||||
*
|
||||
* Validates:
|
||||
* - MCP_SERVERS has at least 15 entries
|
||||
* - Each entry has required fields (name, display_name, description, install_manifest)
|
||||
* - Each install_manifest has mcp_config with command and args
|
||||
* - seedMcpServers inserts into a temp DB correctly
|
||||
* - Duplicate seeding does not create duplicates
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import Database from 'better-sqlite3';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import { MCP_SERVERS, seedMcpServers, type McpServerEntry } from '../src/mcp-registry';
|
||||
import { MarketplaceDB } from '../src/db';
|
||||
|
||||
// ── Schema: Create a temp marketplace DB with the real schema ────────
|
||||
|
||||
const SCHEMA_SQL = `
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sources (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT UNIQUE NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
url TEXT,
|
||||
source_type TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
total_packages INTEGER DEFAULT 0,
|
||||
install_method TEXT,
|
||||
api_endpoint TEXT,
|
||||
description TEXT,
|
||||
last_synced_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS packages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
source_id INTEGER REFERENCES sources(id),
|
||||
name TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
author TEXT,
|
||||
package_type TEXT NOT NULL,
|
||||
waggle_install_type TEXT NOT NULL,
|
||||
waggle_install_path TEXT,
|
||||
version TEXT DEFAULT '1.0.0',
|
||||
license TEXT,
|
||||
repository_url TEXT,
|
||||
homepage_url TEXT,
|
||||
downloads INTEGER DEFAULT 0,
|
||||
stars INTEGER DEFAULT 0,
|
||||
rating REAL DEFAULT 0,
|
||||
rating_count INTEGER DEFAULT 0,
|
||||
category TEXT,
|
||||
subcategory TEXT,
|
||||
install_manifest JSON,
|
||||
platforms JSON DEFAULT '[]',
|
||||
min_waggle_version TEXT,
|
||||
dependencies JSON DEFAULT '[]',
|
||||
packs JSON DEFAULT '[]',
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now')),
|
||||
security_status TEXT DEFAULT 'unscanned',
|
||||
security_score INTEGER DEFAULT -1,
|
||||
last_scanned_at TEXT,
|
||||
content_hash TEXT,
|
||||
scan_engines JSON,
|
||||
scan_findings JSON,
|
||||
scan_blocked BOOLEAN DEFAULT 0,
|
||||
UNIQUE(source_id, name)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS packs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
slug TEXT UNIQUE NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
target_roles TEXT,
|
||||
icon TEXT,
|
||||
priority TEXT DEFAULT 'MEDIUM',
|
||||
connectors_needed JSON DEFAULT '[]',
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pack_packages (
|
||||
pack_id INTEGER REFERENCES packs(id),
|
||||
package_id INTEGER REFERENCES packages(id),
|
||||
is_core BOOLEAN DEFAULT 0,
|
||||
PRIMARY KEY (pack_id, package_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS installations (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
package_id INTEGER REFERENCES packages(id),
|
||||
installed_version TEXT NOT NULL,
|
||||
installed_at TEXT DEFAULT (datetime('now')),
|
||||
install_path TEXT NOT NULL,
|
||||
status TEXT DEFAULT 'active',
|
||||
config JSON DEFAULT '{}'
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS packages_fts USING fts5(
|
||||
name, display_name, description, author, category,
|
||||
content='packages',
|
||||
content_rowid='id'
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scan_history (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
package_id INTEGER REFERENCES packages(id),
|
||||
scanned_at TEXT DEFAULT (datetime('now')),
|
||||
overall_severity TEXT NOT NULL,
|
||||
security_score INTEGER NOT NULL,
|
||||
content_hash TEXT,
|
||||
engines_used JSON,
|
||||
findings JSON,
|
||||
blocked BOOLEAN DEFAULT 0,
|
||||
scan_duration_ms INTEGER,
|
||||
triggered_by TEXT DEFAULT 'manual'
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS security_config (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
`;
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
let tempDbPath: string;
|
||||
let db: MarketplaceDB;
|
||||
|
||||
function createTempDb(): string {
|
||||
const tmpDir = os.tmpdir();
|
||||
const dbPath = path.join(tmpDir, `waggle-mcp-test-${Date.now()}-${Math.random().toString(36).slice(2)}.db`);
|
||||
const rawDb = new Database(dbPath);
|
||||
rawDb.pragma('journal_mode = WAL');
|
||||
rawDb.pragma('foreign_keys = ON');
|
||||
rawDb.exec(SCHEMA_SQL);
|
||||
rawDb.close();
|
||||
return dbPath;
|
||||
}
|
||||
|
||||
// ── Static Data Validation ──────────────────────────────────────────
|
||||
|
||||
describe('MCP_SERVERS definitions', () => {
|
||||
it('has at least 15 MCP server entries', () => {
|
||||
expect(MCP_SERVERS.length).toBeGreaterThanOrEqual(15);
|
||||
});
|
||||
|
||||
it('has at most 25 entries (reasonable catalog size)', () => {
|
||||
expect(MCP_SERVERS.length).toBeLessThanOrEqual(25);
|
||||
});
|
||||
|
||||
it('every entry has name, display_name, description', () => {
|
||||
for (const server of MCP_SERVERS) {
|
||||
expect(typeof server.name).toBe('string');
|
||||
expect(server.name.length).toBeGreaterThan(0);
|
||||
|
||||
expect(typeof server.display_name).toBe('string');
|
||||
expect(server.display_name.length).toBeGreaterThan(0);
|
||||
|
||||
expect(typeof server.description).toBe('string');
|
||||
expect(server.description.length).toBeGreaterThan(10);
|
||||
}
|
||||
});
|
||||
|
||||
it('every entry has install_manifest with mcp_config', () => {
|
||||
for (const server of MCP_SERVERS) {
|
||||
expect(server.install_manifest).toBeDefined();
|
||||
expect(server.install_manifest!.mcp_config).toBeDefined();
|
||||
|
||||
const mcp = server.install_manifest!.mcp_config!;
|
||||
expect(typeof mcp.name).toBe('string');
|
||||
expect(mcp.name.length).toBeGreaterThan(0);
|
||||
|
||||
expect(typeof mcp.command).toBe('string');
|
||||
expect(mcp.command.length).toBeGreaterThan(0);
|
||||
|
||||
expect(Array.isArray(mcp.args)).toBe(true);
|
||||
expect(mcp.args.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('every install_manifest has npm_package', () => {
|
||||
for (const server of MCP_SERVERS) {
|
||||
expect(typeof server.install_manifest!.npm_package).toBe('string');
|
||||
expect(server.install_manifest!.npm_package!.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('all names are unique', () => {
|
||||
const names = MCP_SERVERS.map(s => s.name);
|
||||
expect(new Set(names).size).toBe(names.length);
|
||||
});
|
||||
|
||||
it('all names are kebab-case (no spaces or uppercase)', () => {
|
||||
for (const server of MCP_SERVERS) {
|
||||
expect(server.name).toMatch(/^[a-z0-9]+(-[a-z0-9]+)*$/);
|
||||
}
|
||||
});
|
||||
|
||||
it('every entry has waggle_install_type = mcp', () => {
|
||||
for (const server of MCP_SERVERS) {
|
||||
expect(server.waggle_install_type).toBe('mcp');
|
||||
}
|
||||
});
|
||||
|
||||
it('every entry has package_type = mcp_server', () => {
|
||||
for (const server of MCP_SERVERS) {
|
||||
expect(server.package_type).toBe('mcp_server');
|
||||
}
|
||||
});
|
||||
|
||||
it('every entry has a category', () => {
|
||||
for (const server of MCP_SERVERS) {
|
||||
expect(typeof server.category).toBe('string');
|
||||
expect(server.category!.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('covers expected categories', () => {
|
||||
const categories = new Set(MCP_SERVERS.map(s => s.category));
|
||||
expect(categories.has('developer-tools')).toBe(true);
|
||||
expect(categories.has('web')).toBe(true);
|
||||
expect(categories.has('productivity')).toBe(true);
|
||||
expect(categories.has('knowledge')).toBe(true);
|
||||
expect(categories.has('data')).toBe(true);
|
||||
});
|
||||
|
||||
it('includes key well-known servers', () => {
|
||||
const names = MCP_SERVERS.map(s => s.name);
|
||||
expect(names).toContain('filesystem');
|
||||
expect(names).toContain('github');
|
||||
expect(names).toContain('brave-search');
|
||||
expect(names).toContain('memory');
|
||||
expect(names).toContain('sequential-thinking');
|
||||
expect(names).toContain('puppeteer');
|
||||
expect(names).toContain('slack');
|
||||
});
|
||||
|
||||
it('mcp_config command is npx or uvx', () => {
|
||||
for (const server of MCP_SERVERS) {
|
||||
const cmd = server.install_manifest!.mcp_config!.command;
|
||||
expect(['npx', 'uvx', 'node']).toContain(cmd);
|
||||
}
|
||||
});
|
||||
|
||||
it('entries with env vars have string values (possibly empty for user input)', () => {
|
||||
for (const server of MCP_SERVERS) {
|
||||
const env = server.install_manifest!.mcp_config!.env;
|
||||
if (env) {
|
||||
for (const [key, value] of Object.entries(env)) {
|
||||
expect(typeof key).toBe('string');
|
||||
expect(typeof value).toBe('string');
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── Database Seeding ────────────────────────────────────────────────
|
||||
|
||||
describe('seedMcpServers', () => {
|
||||
beforeEach(() => {
|
||||
tempDbPath = createTempDb();
|
||||
db = new MarketplaceDB(tempDbPath);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
try { db.close(); } catch { /* ignore */ }
|
||||
try { fs.unlinkSync(tempDbPath); } catch { /* ignore */ }
|
||||
// Clean up WAL/SHM files
|
||||
try { fs.unlinkSync(tempDbPath + '-wal'); } catch { /* ignore */ }
|
||||
try { fs.unlinkSync(tempDbPath + '-shm'); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
it('inserts all MCP servers into an empty database', () => {
|
||||
const added = seedMcpServers(db);
|
||||
expect(added).toBe(MCP_SERVERS.length);
|
||||
});
|
||||
|
||||
it('creates the mcp_registry source', () => {
|
||||
seedMcpServers(db);
|
||||
const sources = db.listSources();
|
||||
const mcpSource = sources.find(s => s.name === 'mcp_registry');
|
||||
expect(mcpSource).toBeDefined();
|
||||
expect(mcpSource!.display_name).toBe('MCP Server Registry');
|
||||
expect(mcpSource!.source_type).toBe('registry');
|
||||
});
|
||||
|
||||
it('all seeded packages are retrievable by name', () => {
|
||||
seedMcpServers(db);
|
||||
for (const server of MCP_SERVERS) {
|
||||
const pkg = db.getPackageByName(server.name);
|
||||
expect(pkg).not.toBeNull();
|
||||
expect(pkg!.display_name).toBe(server.display_name);
|
||||
expect(pkg!.waggle_install_type).toBe('mcp');
|
||||
expect(pkg!.package_type).toBe('mcp_server');
|
||||
}
|
||||
});
|
||||
|
||||
it('seeded packages have install_manifest with mcp_config', () => {
|
||||
seedMcpServers(db);
|
||||
for (const server of MCP_SERVERS) {
|
||||
const pkg = db.getPackageByName(server.name);
|
||||
expect(pkg).not.toBeNull();
|
||||
expect(pkg!.install_manifest).toBeDefined();
|
||||
const manifest = pkg!.install_manifest;
|
||||
expect(manifest?.mcp_config).toBeDefined();
|
||||
expect(manifest?.mcp_config?.command).toBeTruthy();
|
||||
expect(Array.isArray(manifest?.mcp_config?.args)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('seeded packages appear in search results', () => {
|
||||
seedMcpServers(db);
|
||||
const results = db.search({ type: 'mcp', limit: 50 });
|
||||
expect(results.total).toBe(MCP_SERVERS.length);
|
||||
expect(results.packages.length).toBe(MCP_SERVERS.length);
|
||||
});
|
||||
|
||||
it('duplicate seeding does not create duplicates', () => {
|
||||
const first = seedMcpServers(db);
|
||||
expect(first).toBe(MCP_SERVERS.length);
|
||||
|
||||
const second = seedMcpServers(db);
|
||||
expect(second).toBe(0);
|
||||
|
||||
// Verify total count unchanged
|
||||
const results = db.search({ type: 'mcp', limit: 100 });
|
||||
expect(results.total).toBe(MCP_SERVERS.length);
|
||||
});
|
||||
|
||||
it('partial seeding skips existing entries', () => {
|
||||
// First seed
|
||||
seedMcpServers(db);
|
||||
|
||||
// Manually delete a few entries and re-seed
|
||||
const rawDb = db.getRawDb();
|
||||
rawDb.prepare("DELETE FROM packages WHERE name = 'filesystem'").run();
|
||||
rawDb.prepare("DELETE FROM packages WHERE name = 'github'").run();
|
||||
|
||||
// Re-seed should only add the 2 deleted ones back
|
||||
const added = seedMcpServers(db);
|
||||
expect(added).toBe(2);
|
||||
|
||||
// Total should still be the full count
|
||||
const results = db.search({ type: 'mcp', limit: 100 });
|
||||
expect(results.total).toBe(MCP_SERVERS.length);
|
||||
});
|
||||
|
||||
it('updates source total_packages count', () => {
|
||||
seedMcpServers(db);
|
||||
const sources = db.listSources();
|
||||
const mcpSource = sources.find(s => s.name === 'mcp_registry');
|
||||
expect(mcpSource).toBeDefined();
|
||||
expect(mcpSource!.total_packages).toBe(MCP_SERVERS.length);
|
||||
});
|
||||
|
||||
it('search by category returns correct results', () => {
|
||||
seedMcpServers(db);
|
||||
|
||||
const devTools = db.search({ type: 'mcp', category: 'developer-tools', limit: 50 });
|
||||
expect(devTools.total).toBeGreaterThanOrEqual(3); // filesystem, git, github, sqlite, postgres
|
||||
|
||||
const web = db.search({ type: 'mcp', category: 'web', limit: 50 });
|
||||
expect(web.total).toBeGreaterThanOrEqual(2); // brave-search, fetch, puppeteer
|
||||
|
||||
const productivity = db.search({ type: 'mcp', category: 'productivity', limit: 50 });
|
||||
expect(productivity.total).toBeGreaterThanOrEqual(3); // google-drive, slack, notion, gmail
|
||||
});
|
||||
|
||||
it('facets include mcp type', () => {
|
||||
seedMcpServers(db);
|
||||
const results = db.search({ limit: 50 });
|
||||
expect(results.facets.types).toHaveProperty('mcp');
|
||||
expect(results.facets.types.mcp).toBe(MCP_SERVERS.length);
|
||||
});
|
||||
});
|
||||
|
||||
// ── FTS5 query relaxation (P0: acquire_capability verbose-need regression) ──
|
||||
//
|
||||
// Root cause: db.search() passed the raw caller string straight into FTS5
|
||||
// `MATCH @query`. FTS5 implicit-ANDs every term, so a verbose natural-language
|
||||
// `need` (always the case when acquire_capability calls searchMarketplace)
|
||||
// matches zero packages, and special chars (':' '\\' '"') in paths like
|
||||
// `D:\Projects\X` raise an FTS5 syntax error that searchMarketplace swallows
|
||||
// to []. Net: the inline capability-install feature never surfaces a
|
||||
// candidate for real agent queries. These tests reproduce that and lock the
|
||||
// relaxation behaviour in.
|
||||
|
||||
describe('db.search — FTS5 query relaxation', () => {
|
||||
let ftsDbPath: string;
|
||||
let ftsDb: MarketplaceDB;
|
||||
|
||||
beforeEach(() => {
|
||||
ftsDbPath = createTempDb();
|
||||
ftsDb = new MarketplaceDB(ftsDbPath);
|
||||
seedMcpServers(ftsDb); // seeds the 'filesystem' MCP server
|
||||
// The bare test schema declares packages_fts as external-content FTS5
|
||||
// with no sync triggers (production ships them in the seed DB). Rebuild
|
||||
// the index from the content table so search() exercises real FTS —
|
||||
// these tests target query *relaxation*, not FTS population. (Uses the
|
||||
// better-sqlite3 statement API, not child_process.)
|
||||
(ftsDb as unknown as { db: import('better-sqlite3').Database }).db
|
||||
.prepare("INSERT INTO packages_fts(packages_fts) VALUES('rebuild')")
|
||||
.run();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
try { ftsDb.close(); } catch { /* ignore */ }
|
||||
try { fs.unlinkSync(ftsDbPath); } catch { /* ignore */ }
|
||||
try { fs.unlinkSync(ftsDbPath + '-wal'); } catch { /* ignore */ }
|
||||
try { fs.unlinkSync(ftsDbPath + '-shm'); } catch { /* ignore */ }
|
||||
});
|
||||
|
||||
const hasFilesystem = (r: { packages: Array<{ name: string; description: string }> }) =>
|
||||
r.packages.some(p => p.name === 'filesystem' || /filesystem/i.test(p.description));
|
||||
|
||||
it('baseline: a single tight keyword finds the filesystem MCP server', () => {
|
||||
const r = ftsDb.search({ query: 'filesystem', limit: 10 });
|
||||
expect(r.total).toBeGreaterThan(0);
|
||||
expect(hasFilesystem(r)).toBe(true);
|
||||
});
|
||||
|
||||
it('REGRESSION: a verbose natural-language need still surfaces the filesystem server', () => {
|
||||
// Exact shape acquire_capability feeds into searchMarketplace(need).
|
||||
const need =
|
||||
'Access and read files from an external local filesystem path outside my managed workspace directory looking for an MCP filesystem connector or similar capability';
|
||||
const r = ftsDb.search({ query: need, limit: 10 });
|
||||
expect(r.total).toBeGreaterThan(0);
|
||||
expect(hasFilesystem(r)).toBe(true);
|
||||
});
|
||||
|
||||
it('ROBUSTNESS: a need with FTS-special chars (path with : and \\ and quotes) does not throw and still matches', () => {
|
||||
const need =
|
||||
'read files at D:\\Projects\\PM-Waggle-OS — need a "filesystem" connector, not workspace-only access';
|
||||
expect(() => ftsDb.search({ query: need, limit: 10 })).not.toThrow();
|
||||
const r = ftsDb.search({ query: need, limit: 10 });
|
||||
expect(r.total).toBeGreaterThan(0);
|
||||
expect(hasFilesystem(r)).toBe(true);
|
||||
});
|
||||
|
||||
it('EMPTY/garbage query degrades gracefully (no throw, no crash)', () => {
|
||||
expect(() => ftsDb.search({ query: ' ', limit: 10 })).not.toThrow();
|
||||
expect(() => ftsDb.search({ query: '!!! "" \\ : * ^', limit: 10 })).not.toThrow();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user