/** * Waggle Marketplace — Source Sync Engine * * Fetches live package data from marketplace sources and updates * the local SQLite database. Supports incremental and full syncs. * * Supported sources: * - ClawHub API (https://clawhub.ai/api/v1/skills) * - SkillsMP API (https://skillsmp.dev/api/v1/skills) * - GitHub Organizations (anthropics, modelcontextprotocol, lobehub, cursor, etc.) * - LobeHub Plugin Registry (https://chat-plugins.lobehub.com/index.json) * - AITMPL Skills (https://www.aitmpl.com/api/skills) * - Awesome-List repos (awesome-mcp-servers, awesome-claude-skills, etc.) * - GitHub Repo Content (repos with SKILL.md files in subdirectories) * - Web Registries (skills.sh, mcpmarket.com, skillsdirectory.com, etc.) * - NPM Registry (mcp-server keyword search) * * Rate limits are respected per source. Default: 500 req/day for SkillsMP, * 100 req/min for GitHub API, unlimited for ClawHub. */ import { MarketplaceDB } from './db.js'; import { type FetchFn, defaultFetch } from './fetcher.js'; import type { MarketplacePackage, MarketplaceSource, SyncOptions, SyncResult, InstallManifest, } from './types.js'; // ─── Helpers ─────────────────────────────────────────────────────── /** Truncate error messages to a reasonable length */ function safeErrorText(err: unknown, maxLen = 200): string { const msg = err instanceof Error ? err.message : String(err); return msg.length > maxLen ? msg.slice(0, maxLen) + '...' : msg; } /** Check if a URL is an awesome-list repo (has 'awesome' anywhere in the repo name) */ function isAwesomeListUrl(url: string): boolean { return /github\.com\/[^/]+\/[^/]*awesome[^/]*/i.test(url); } /** Build GitHub API headers */ function githubHeaders(): Record { const headers: Record = { 'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'waggle-marketplace-sync', }; if (process.env.GITHUB_TOKEN) { headers['Authorization'] = `Bearer ${process.env.GITHUB_TOKEN}`; } return headers; } /** Detect package type from name/URL */ function detectPackageType(name: string, url?: string): 'skill' | 'plugin' | 'mcp' { const combined = `${name} ${url || ''}`.toLowerCase(); if (combined.includes('mcp') || combined.includes('model-context-protocol')) return 'mcp'; if (combined.includes('plugin')) return 'plugin'; return 'skill'; } /** Make a URL-friendly slug from a name */ function slugify(name: string): string { return name .toLowerCase() .replace(/[^a-z0-9]+/g, '-') .replace(/^-+|-+$/g, '') .slice(0, 80); } /** Make a display name from a slug */ function displayNameFromSlug(slug: string): string { return slug .replace(/[-_]/g, ' ') .replace(/\b\w/g, (c: string) => c.toUpperCase()); } // ─── Interface ───────────────────────────────────────────────────── /** * Optional function to look up API keys from the vault. * Accepts a vault key name and returns the secret value or null. */ export type VaultLookupFn = (key: string) => string | null; interface SyncAdapter { name: string; canSync(source: MarketplaceSource): boolean; // `doFetch` is the SSRF-guarded fetcher injected by the server (or plain // global fetch for standalone/CLI). Every outbound registry fetch below must // use it rather than the global `fetch` so an attacker-controlled source URL // cannot reach an internal/link-local host. sync( source: MarketplaceSource, db: MarketplaceDB, vaultLookup: VaultLookupFn | undefined, doFetch: FetchFn, ): Promise; } // ─── External JSON Boundary Shapes ───────────────────────────────── // Marketplace sources return loosely-structured JSON. These interfaces model // the fields the adapters actually read — every field is optional because // upstream payloads vary by provider and version. Treat them as a typed view // over untrusted data: never assume a field is present. /** A single skill/plugin/MCP entry as returned by a remote registry. */ interface RemoteCatalogItem { id?: string | number; slug?: string; name?: string; title?: string; description?: string; summary?: string; author?: string; creator?: string; owner?: string; version?: string; license?: string; type?: string; category?: string; tags?: string[]; url?: string; homepage?: string; repository_url?: string; repo_url?: string; raw_url?: string; content_url?: string; download_url?: string; skillUrl?: string; githubUrl?: string; npm_package?: string; downloads?: number; installs?: number; download_count?: number; stars?: number; likes?: number; rating?: number; rating_count?: number; platforms?: string[]; dependencies?: string[]; identifier?: string; api?: unknown; } /** Generic paginated/wrapped registry response (web-registry + generic adapters). */ interface RemoteCatalogResponse { skills?: RemoteCatalogItem[]; data?: RemoteCatalogItem[] | { skills?: RemoteCatalogItem[]; pagination?: { hasNext?: boolean } }; items?: RemoteCatalogItem[]; tools?: RemoteCatalogItem[]; plugins?: RemoteCatalogItem[]; results?: RemoteCatalogItem[]; } /** GitHub git-tree API response. */ interface GitHubTreeResponse { tree?: Array<{ type?: string; path?: string }>; } /** GitHub repos list entry. */ interface GitHubRepo { name: string; full_name: string; description?: string | null; topics?: string[]; html_url: string; homepage?: string | null; clone_url: string; stargazers_count?: number; license?: { spdx_id?: string | null } | null; } /** npm search API response. */ interface NpmSearchResponse { objects?: NpmSearchObject[]; results?: NpmSearchObject[]; } interface NpmPackageInfo { name?: string; description?: string; version?: string; keywords?: string[]; publisher?: { username?: string }; author?: { name?: string }; maintainers?: Array<{ username?: string }>; links?: { repository?: string; homepage?: string; npm?: string }; repository?: { url?: string }; } /** * A search hit from the npm registry. npm's `/-/v1/search` nests the package * under `.package`, but some mirrors return the package fields inline — so this * extends {@link NpmPackageInfo} to cover both shapes (`obj.package || obj`). */ interface NpmSearchObject extends NpmPackageInfo { package?: NpmPackageInfo; } /** ClawHub paginated skills response. */ interface ClawHubResponse { skills?: RemoteCatalogItem[]; data?: RemoteCatalogItem[]; } /** SkillsMP search response (nested data envelope). */ interface SkillsMpResponse { skills?: RemoteCatalogItem[]; data?: { skills?: RemoteCatalogItem[]; pagination?: { hasNext?: boolean }; } | RemoteCatalogItem[]; } /** LobeHub plugin index response. */ interface LobeHubResponse { plugins?: LobeHubPlugin[]; } interface LobeHubPlugin { identifier?: string; name?: string; description?: string; version?: string; author?: string; homepage?: string; category?: string; installs?: number; api?: unknown; } // ─── Awesome-List Parser Adapter ─────────────────────────────────── // Handles GitHub "awesome-*" repos that contain markdown lists of links. /** * Parse a markdown README for links, supporting: * - Bullet lists: `- [name](url) - description` * - Table rows: `| [name](url) | description |` * - Heading-level links: `### [name](url)` */ export function parseAwesomeListMarkdown(markdown: string): Array<{ name: string; url: string; description: string; }> { const results: Array<{ name: string; url: string; description: string }> = []; const seen = new Set(); // Process line by line to prevent cross-line greedy matching const lines = markdown.split('\n'); const linkPattern = /\[([^\]]+)\]\((https?:\/\/[^)]+)\)(?:[ \t]*[-\u2013|:][ \t]*(.*))?/; for (const line of lines) { const match = linkPattern.exec(line); if (!match) continue; const name = match[1].trim(); const url = match[2].trim(); const description = (match[3] || '').trim(); // Skip badges, images, ToC links, and anchors if (url.includes('shields.io') || url.includes('badge') || url.startsWith('#')) continue; if (/\.(png|jpg|jpeg|gif|svg)$/i.test(url)) continue; if (name.length < 2 || name.length > 120) continue; // Dedup by URL if (seen.has(url)) continue; seen.add(url); results.push({ name, url, description }); } return results; } const awesomeListAdapter: SyncAdapter = { name: 'awesome-list', canSync(source) { // Match community_repo or aggregator sources with awesome-* in URL if (source.source_type !== 'community_repo' && source.source_type !== 'aggregator') return false; return isAwesomeListUrl(source.url); }, async sync(source, db, _vaultLookup, doFetch) { const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] }; // No cap — we store metadata only, actual content fetched on install try { // Extract owner/repo from URL const match = source.url.match(/github\.com\/([^/]+)\/([^/]+)/); if (!match) throw new Error(`Cannot parse GitHub URL: ${source.url}`); const [, owner, repo] = match; // Fetch README.md via raw.githubusercontent.com const readmeUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/README.md`; const response = await doFetch(readmeUrl, { headers: { 'User-Agent': 'waggle-marketplace-sync' }, }); let markdown: string; if (!response.ok) { // Try master branch as fallback const fallbackUrl = `https://raw.githubusercontent.com/${owner}/${repo}/master/README.md`; const fallbackResp = await doFetch(fallbackUrl, { headers: { 'User-Agent': 'waggle-marketplace-sync' }, }); if (!fallbackResp.ok) throw new Error(`README fetch failed: ${response.status} / ${fallbackResp.status}`); markdown = await fallbackResp.text(); } else { markdown = await response.text(); } const links = parseAwesomeListMarkdown(markdown); for (const link of links) { const installType = detectPackageType(link.name, link.url); const slug = slugify(link.name); if (!slug) continue; const manifest: InstallManifest = installType === 'mcp' ? { npm_package: slug, mcp_config: { name: slug, command: 'npx', args: ['-y', slug] } } : { skill_url: link.url }; try { db.upsertPackage({ source_id: source.id, name: `${source.name}-${slug}`, display_name: link.name, description: link.description || `${installType} from ${source.display_name}`, author: owner, package_type: installType === 'mcp' ? 'mcp_server' : installType, waggle_install_type: installType, waggle_install_path: installType === 'skill' ? `skills/${slug}.md` : installType === 'plugin' ? `plugins/${slug}/` : `.mcp.json`, version: '1.0.0', license: null, repository_url: link.url.includes('github.com') ? link.url : null, homepage_url: link.url, downloads: 0, stars: 0, category: installType === 'mcp' ? 'integration' : 'general', platforms: JSON.stringify(['claude_code', 'waggle']), dependencies: JSON.stringify([]), packs: JSON.stringify([]), install_manifest: JSON.stringify(manifest), }); result.added++; } catch (itemErr) { result.errors.push(safeErrorText(itemErr)); } } } catch (err) { result.errors.push(safeErrorText(err)); } return result; }, }; // ─── GitHub Repo Content Adapter ─────────────────────────────────── // Handles single repos with multiple skills in subdirectories. const githubRepoContentAdapter: SyncAdapter = { name: 'github-repo-content', canSync(source) { if (source.source_type !== 'community_repo') return false; // Must be a github.com URL but NOT an awesome-list if (!source.url.includes('github.com')) return false; if (isAwesomeListUrl(source.url)) return false; // Must have a repo path (owner/repo), not just an org const match = source.url.match(/github\.com\/([^/]+)\/([^/]+)/); return !!match; }, async sync(source, db, _vaultLookup, doFetch) { const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] }; // No cap — metadata only, content fetched on install try { const match = source.url.match(/github\.com\/([^/]+)\/([^/]+)/); if (!match) throw new Error(`Cannot parse GitHub repo URL: ${source.url}`); const [, owner, repo] = match; // Use GitHub API to get repo tree const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/main?recursive=1`; const response = await doFetch(treeUrl, { headers: githubHeaders() }); let treeData: GitHubTreeResponse; if (!response.ok) { // Try master branch const fallbackUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/master?recursive=1`; const fallbackResp = await doFetch(fallbackUrl, { headers: githubHeaders() }); if (!fallbackResp.ok) throw new Error(`GitHub tree API: ${response.status} / ${fallbackResp.status}`); treeData = await fallbackResp.json() as GitHubTreeResponse; } else { treeData = await response.json() as GitHubTreeResponse; } const tree = treeData.tree || []; // Look for SKILL.md, skill.md, *.skill.md files const skillFiles = tree.filter((item) => item.type === 'blob' && !!item.path && /(?:^|\/)(?:SKILL\.md|skill\.md|[^/]+\.skill\.md)$/i.test(item.path) ); for (const file of skillFiles) { // Derive skill name from file path (path presence guaranteed by the filter above) const pathParts = file.path!.split('/'); const dirName = pathParts.length > 1 ? pathParts[pathParts.length - 2] : pathParts[0]; const slug = slugify(dirName.replace(/\.skill\.md$/i, '').replace(/\.md$/i, '')); if (!slug) continue; const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/${file.path}`; try { db.upsertPackage({ source_id: source.id, name: `${source.name}-${slug}`, display_name: displayNameFromSlug(slug), description: `Skill from ${owner}/${repo}`, author: owner, package_type: 'skill', waggle_install_type: 'skill', waggle_install_path: `skills/${slug}.md`, version: '1.0.0', license: null, repository_url: `https://github.com/${owner}/${repo}/tree/main/${pathParts.slice(0, -1).join('/')}`, homepage_url: `https://github.com/${owner}/${repo}`, downloads: 0, stars: 0, category: 'general', platforms: JSON.stringify(['claude_code', 'waggle']), dependencies: JSON.stringify([]), packs: JSON.stringify([]), install_manifest: JSON.stringify({ skill_url: rawUrl }), }); result.added++; } catch (itemErr) { result.errors.push(safeErrorText(itemErr)); } } } catch (err) { result.errors.push(safeErrorText(err)); } return result; }, }; // ─── Web Registry Adapter ────────────────────────────────────────── // Handles web-based skill/MCP registries. const webRegistryAdapter: SyncAdapter = { name: 'web-registry', canSync(source) { // Match aggregator sources with api_endpoint, but NOT github URLs if (source.source_type === 'aggregator' && source.api_endpoint && !source.url.includes('github.com')) return true; return false; }, async sync(source, db, vaultLookup, doFetch) { const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] }; try { // Build headers — check vault for API keys (premium sources like skillsdirectory.com) const headers: Record = { 'User-Agent': 'waggle-marketplace-sync', 'Accept': 'application/json', }; if (vaultLookup) { const apiKey = vaultLookup(`marketplace:source:${source.name}:api_key`); if (apiKey) { headers['Authorization'] = `Bearer ${apiKey}`; headers['X-Api-Key'] = apiKey; } } // Resume from saved sync state (for rate-limited sources) const syncState = db.getSyncState(source.id) || {}; let offset = (syncState.lastOffset as number) || 0; const totalSyncedPrior = (syncState.totalSynced as number) || 0; const pageSize = 100; let hasMore = true; while (hasMore) { // Append pagination params to the endpoint URL const baseUrl = source.api_endpoint!; const sep = baseUrl.includes('?') ? '&' : '?'; const pageUrl = `${baseUrl}${sep}limit=${pageSize}&offset=${offset}`; const response = await doFetch(pageUrl, { headers }); if (response.status === 401 || response.status === 403) { result.errors.push( `${source.display_name} API requires authentication (${response.status}). ` + `Configure an API key in the vault as "marketplace:source:${source.name}:api_key".`, ); return result; } // Rate limited — save progress and return gracefully if (response.status === 429) { db.setSyncState(source.id, { lastOffset: offset, totalSynced: totalSyncedPrior + result.added, lastSyncedAt: new Date().toISOString(), }); result.errors.push( `${source.display_name} rate limit reached at offset ${offset}. ` + `Progress saved — next sync will resume from offset ${offset}.`, ); return result; } if (!response.ok) throw new Error(`API ${response.status}: ${response.statusText}`); const data = await response.json() as RemoteCatalogItem[] | RemoteCatalogResponse; // Try multiple JSON response formats let items: RemoteCatalogItem[] = []; if (Array.isArray(data)) { items = data; } else if (data.skills && Array.isArray(data.skills)) { items = data.skills; } else if (data.data && Array.isArray(data.data)) { items = data.data; } else if (data.items && Array.isArray(data.items)) { items = data.items; } else if (data.tools && Array.isArray(data.tools)) { items = data.tools; } else if (data.plugins && Array.isArray(data.plugins)) { items = data.plugins; } else if (data.results && Array.isArray(data.results)) { items = data.results; } if (items.length === 0) { if (offset === 0) { result.errors.push(`No structured data found in response from ${source.api_endpoint}`); } // Full sync complete — reset sync state db.setSyncState(source.id, null); hasMore = false; break; } for (const item of items) { const name = item.slug || item.name || item.title || item.id; if (!name) continue; const slug = slugify(String(name)); if (!slug) continue; const installType = detectPackageType( String(name), item.url || item.repository_url || item.homepage, ); const npmPkg = item.npm_package || slug; const manifest: InstallManifest = installType === 'mcp' ? { npm_package: npmPkg, mcp_config: { name: slug, command: 'npx', args: ['-y', npmPkg] } } : { skill_url: item.raw_url || item.content_url || item.download_url || item.url }; try { db.upsertPackage({ source_id: source.id, name: `${source.name}-${slug}`, display_name: item.name || item.title || displayNameFromSlug(slug), description: item.description || item.summary || '', author: item.author || item.creator || item.owner || source.name, package_type: installType === 'mcp' ? 'mcp_server' : installType, waggle_install_type: installType, waggle_install_path: installType === 'skill' ? `skills/${slug}.md` : installType === 'plugin' ? `plugins/${slug}/` : `.mcp.json`, version: item.version || '1.0.0', license: item.license || null, repository_url: item.repository_url || item.repo_url || null, homepage_url: item.url || item.homepage || null, downloads: item.downloads || item.installs || item.download_count || 0, stars: item.stars || item.likes || item.rating || 0, category: item.category || item.tags?.[0] || 'general', platforms: JSON.stringify(item.platforms || ['claude_code', 'waggle']), dependencies: JSON.stringify(item.dependencies || []), packs: JSON.stringify([]), install_manifest: JSON.stringify(manifest), }); result.added++; } catch (itemErr) { result.errors.push(safeErrorText(itemErr)); } } // Save progress after each page offset += items.length; db.setSyncState(source.id, { lastOffset: offset, totalSynced: totalSyncedPrior + result.added, lastSyncedAt: new Date().toISOString(), }); // Check if we should fetch the next page if (items.length < pageSize) { // Full sync complete — reset sync state db.setSyncState(source.id, null); hasMore = false; } else { // Brief delay to respect rate limits await new Promise(r => setTimeout(r, 200)); } } // end while (hasMore) } catch (err) { result.errors.push(safeErrorText(err)); } return result; }, }; // ─── NPM Search Adapter ─────────────────────────────────────────── // Searches npm registry for MCP server packages. /** A normalized npm package entry produced by {@link parseNpmSearchResults}. */ export interface NpmPackageEntry { name: string; description: string; version: string; author: string; npm_package: string; repository_url: string | null; homepage_url: string | null; keywords: string[]; } /** Parse npm search API response into package entries */ export function parseNpmSearchResults(data: NpmSearchResponse): NpmPackageEntry[] { const objects = data.objects || data.results || []; if (!Array.isArray(objects)) return []; return objects.map((obj): NpmPackageEntry => { const pkg: NpmPackageInfo = obj.package || obj; return { name: pkg.name || '', description: pkg.description || '', version: pkg.version || '1.0.0', author: pkg.publisher?.username || pkg.author?.name || pkg.maintainers?.[0]?.username || 'community', npm_package: pkg.name || '', repository_url: pkg.links?.repository || pkg.repository?.url || null, homepage_url: pkg.links?.homepage || pkg.links?.npm || null, keywords: pkg.keywords || [], }; }).filter((pkg) => pkg.name); } const npmSearchAdapter: SyncAdapter = { name: 'npm-search', canSync(source) { return source.source_type === 'npm_registry'; }, async sync(source, db, _vaultLookup, doFetch) { const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] }; try { const apiUrl = source.api_endpoint || 'https://registry.npmjs.org/-/v1/search?text=keywords:mcp-server&size=250'; const response = await doFetch(apiUrl, { headers: { 'User-Agent': 'waggle-marketplace-sync' }, }); if (!response.ok) throw new Error(`npm API ${response.status}: ${response.statusText}`); const data = await response.json() as NpmSearchResponse; const packages = parseNpmSearchResults(data); for (const pkg of packages) { const slug = slugify(pkg.name); if (!slug) continue; const manifest: InstallManifest = { npm_package: pkg.npm_package, mcp_config: { name: slug, command: 'npx', args: ['-y', pkg.npm_package], }, }; try { db.upsertPackage({ source_id: source.id, name: `${source.name}-${slug}`, display_name: displayNameFromSlug(slug), description: pkg.description || `MCP server from npm: ${pkg.npm_package}`, author: pkg.author, package_type: 'mcp_server', waggle_install_type: 'mcp', waggle_install_path: '.mcp.json', version: pkg.version, license: null, repository_url: pkg.repository_url, homepage_url: pkg.homepage_url || `https://www.npmjs.com/package/${pkg.npm_package}`, downloads: 0, stars: 0, category: 'integration', platforms: JSON.stringify(['claude_code', 'waggle', 'cursor']), dependencies: JSON.stringify([]), packs: JSON.stringify([]), install_manifest: JSON.stringify(manifest), }); result.added++; } catch (itemErr) { result.errors.push(safeErrorText(itemErr)); } } } catch (err) { result.errors.push(safeErrorText(err)); } return result; }, }; // ─── GitHub Org Adapter ──────────────────────────────────────────── // Handles GitHub organization repos (org-level, no specific repo path). const githubAdapter: SyncAdapter = { name: 'github', canSync(source) { if (source.source_type === 'github_org') return true; // GitHub URLs that are org-level (no specific repo) if (!source.url.includes('github.com')) return false; const match = source.url.match(/github\.com\/([^/]+)(\/([^/]+))?/); // Only match if no repo path (org-level) and not npm_registry if (match && !match[3] && source.source_type !== 'npm_registry') return true; // For sources with repo path, only match official_marketplace types // (not community_repo - those go to content/awesome adapters) if (match && match[3] && source.source_type === 'official_marketplace') return true; return false; }, async sync(source, db, _vaultLookup, doFetch) { const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] }; try { // Extract org/user from URL const match = source.url.match(/github\.com\/([^/]+)/); if (!match) throw new Error(`Cannot parse GitHub URL: ${source.url}`); const owner = match[1]; // Fetch repos with topic filters const apiUrl = source.api_endpoint || `https://api.github.com/orgs/${owner}/repos?per_page=100&sort=updated`; const response = await doFetch(apiUrl, { headers: githubHeaders() }); if (!response.ok) throw new Error(`GitHub API ${response.status}: ${response.statusText}`); const repos = await response.json() as GitHubRepo[]; for (const repo of repos) { // Filter for skill/MCP/plugin repos const topics = repo.topics || []; const isSkill = topics.some((t) => ['skill', 'agent-skill', 'claude-code', 'mcp', 'mcp-server', 'plugin'].includes(t) ) || repo.name.includes('skill') || repo.name.includes('mcp'); if (!isSkill && repos.length > 20) continue; // Skip non-skill repos in large orgs const installType = repo.name.includes('mcp') ? 'mcp' : topics.includes('plugin') ? 'plugin' : 'skill'; const manifest: InstallManifest = installType === 'skill' ? { skill_url: `https://raw.githubusercontent.com/${owner}/${repo.name}/main/SKILL.md` } : installType === 'mcp' ? { npm_package: repo.full_name, mcp_config: { name: repo.name, command: 'npx', args: ['-y', repo.full_name] } } : { git_url: repo.clone_url }; db.upsertPackage({ source_id: source.id, name: repo.name, display_name: repo.name.replace(/[-_]/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase()), description: repo.description || `${installType} from ${owner}`, author: owner, package_type: installType === 'mcp' ? 'mcp_server' : installType, waggle_install_type: installType as 'skill' | 'plugin' | 'mcp', waggle_install_path: installType === 'skill' ? `skills/${repo.name}.md` : installType === 'plugin' ? `plugins/${repo.name}/` : `.mcp.json`, version: '1.0.0', license: repo.license?.spdx_id || null, repository_url: repo.html_url, homepage_url: repo.homepage || null, downloads: 0, stars: repo.stargazers_count || 0, category: installType === 'mcp' ? 'integration' : 'development', platforms: JSON.stringify(['claude_code', 'waggle']), dependencies: JSON.stringify([]), packs: JSON.stringify([]), install_manifest: JSON.stringify(manifest), }); result.added++; } } catch (err) { result.errors.push(safeErrorText(err)); } return result; }, }; // ─── ClawHub Adapter ──────────────────────────────────────────────── const clawhubAdapter: SyncAdapter = { name: 'clawhub', canSync(source) { return source.name === 'clawhub' || source.url.includes('clawhub.ai'); }, async sync(source, db, _vaultLookup, doFetch) { const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] }; try { const apiBase = source.api_endpoint || 'https://clawhub.ai/api/v1'; const perPage = 50; let hasMore = true; // Resume from saved sync state const syncState = db.getSyncState(source.id) || {}; let page = (syncState.lastPage as number) || 1; const totalSyncedPrior = (syncState.totalSynced as number) || 0; const maxPage = 10; // Cap at 500 skills per full sync cycle while (hasMore && page <= maxPage) { const url = `${apiBase}/skills?page=${page}&per_page=${perPage}&sort=downloads`; const response = await doFetch(url, { headers: { 'User-Agent': 'waggle-marketplace-sync' }, }); // Rate limited — save progress and return gracefully if (response.status === 429) { db.setSyncState(source.id, { lastPage: page, totalSynced: totalSyncedPrior + result.added, lastSyncedAt: new Date().toISOString(), }); result.errors.push( `ClawHub rate limit reached at page ${page}. ` + `Progress saved — next sync will resume from page ${page}.`, ); return result; } if (!response.ok) { result.errors.push(`ClawHub API page ${page}: ${response.status}`); break; } const data = await response.json() as ClawHubResponse | RemoteCatalogItem[]; const skills = Array.isArray(data) ? data : (data.skills || data.data); if (!Array.isArray(skills) || skills.length === 0) { // Full sync complete — reset sync state db.setSyncState(source.id, null); hasMore = false; break; } for (const skill of skills) { const skillName = skill.slug || skill.name?.toLowerCase().replace(/\s+/g, '-'); if (!skillName) continue; const manifest: InstallManifest = { skill_url: skill.raw_url || skill.download_url || `${apiBase}/skills/${skill.id || skill.slug}/raw`, }; db.upsertPackage({ source_id: source.id, name: skillName, display_name: skill.name || skill.title || skillName, description: skill.description || '', author: skill.author || skill.creator || 'community', package_type: 'skill', waggle_install_type: 'skill', waggle_install_path: `skills/${skillName}.md`, version: skill.version || '1.0.0', license: skill.license || 'MIT', repository_url: skill.repository_url || null, homepage_url: skill.homepage || `https://clawhub.ai/skills/${skill.slug || skill.id}`, downloads: skill.downloads || skill.download_count || 0, stars: skill.stars || skill.likes || 0, rating: skill.rating || 0, rating_count: skill.rating_count || 0, category: skill.category || 'general', platforms: JSON.stringify(skill.platforms || ['claude_code', 'waggle']), dependencies: JSON.stringify(skill.dependencies || []), packs: JSON.stringify([]), install_manifest: JSON.stringify(manifest), }); result.added++; } hasMore = skills.length === perPage; page++; // Save progress after each page if (hasMore) { db.setSyncState(source.id, { lastPage: page, totalSynced: totalSyncedPrior + result.added, lastSyncedAt: new Date().toISOString(), }); } else { // Full sync complete (or reached last page) — reset sync state db.setSyncState(source.id, null); } } // If we hit the page cap, save state for next run if (page > maxPage && hasMore) { db.setSyncState(source.id, { lastPage: page, totalSynced: totalSyncedPrior + result.added, lastSyncedAt: new Date().toISOString(), }); } } catch (err) { result.errors.push(safeErrorText(err)); } return result; }, }; // ─── SkillsMP Adapter ─────────────────────────────────────────────── const skillsmpAdapter: SyncAdapter = { name: 'skillsmp', canSync(source) { return source.name === 'skillsmp' || source.url.includes('skillsmp.dev') || source.url.includes('skillsmp.com'); }, async sync(source, db, vaultLookup, doFetch) { const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] }; try { const apiBase = source.api_endpoint || 'https://skillsmp.com/api/v1'; const perPage = 50; // SkillsMP caps at 50 per page // Build auth headers from vault const headers: Record = { 'User-Agent': 'waggle-marketplace-sync', 'Accept': 'application/json' }; if (vaultLookup) { const apiKey = vaultLookup(`marketplace:source:${source.name}:api_key`); if (apiKey) { headers['Authorization'] = `Bearer ${apiKey}`; } } // SkillsMP uses a search endpoint (no list-all). Query by broad categories to cover all skills. const queries = [ 'code', 'python', 'javascript', 'typescript', 'react', 'api', 'data', 'test', 'deploy', 'docker', 'git', 'write', 'research', 'design', 'security', 'database', 'marketing', 'web', 'ai', 'agent', 'claude', 'devops', 'aws', 'linux', 'css', 'node', 'rust', 'go', 'java', 'swift', 'mobile', 'review', 'debug', 'build', 'format', 'lint', 'document', 'email', 'sql', 'graphql', 'frontend', 'backend', ]; const seenIds = new Set(); // Resume from saved sync state const syncState = db.getSyncState(source.id) || {}; const completedQueries = new Set((syncState.completedQueries as string[]) || []); const startQueryIndex = (syncState.lastQueryIndex as number) || 0; const startPage = (syncState.lastPage as number) || 1; const totalSyncedPrior = (syncState.totalSynced as number) || 0; for (let qi = startQueryIndex; qi < queries.length; qi++) { const query = queries[qi]; // Skip queries already fully synced in a previous run if (completedQueries.has(query)) continue; // For the resumed query, start from lastPage; for subsequent queries, start from 1 let page = (qi === startQueryIndex) ? startPage : 1; let hasMore = true; while (hasMore) { const url = `${apiBase}/skills/search?q=${encodeURIComponent(query)}&limit=${perPage}&page=${page}`; const response = await doFetch(url, { headers }); if (!response.ok) { if (response.status === 429) { // Rate limited — save progress and return gracefully db.setSyncState(source.id, { lastQueryIndex: qi, lastPage: page, completedQueries: Array.from(completedQueries), totalSynced: totalSyncedPrior + result.added, lastSyncedAt: new Date().toISOString(), }); result.errors.push( `SkillsMP rate limit reached at query "${query}" page ${page}. ` + `Progress saved (${completedQueries.size}/${queries.length} queries complete). ` + `Next sync will resume from here.`, ); return result; } result.errors.push(`SkillsMP API q=${query} page ${page}: ${response.status}`); break; } const data = await response.json() as SkillsMpResponse; const nestedData = data?.data; const skills: RemoteCatalogItem[] = (!Array.isArray(nestedData) ? nestedData?.skills : undefined) || data?.skills || (Array.isArray(nestedData) ? nestedData : undefined) || []; if (!Array.isArray(skills) || skills.length === 0) { hasMore = false; break; } for (const skill of skills) { // Dedup across queries by skill ID const skillId = String(skill.id || skill.slug || skill.name || ''); if (!skillId || seenIds.has(skillId)) continue; seenIds.add(skillId); const manifest: InstallManifest = { skill_url: skill.githubUrl ? `https://raw.githubusercontent.com/${skill.githubUrl.replace('https://github.com/', '').replace('/tree/main/', '/main/')}/SKILL.md` : skill.skillUrl || `https://skillsmp.com/skills/${skillId}`, }; db.upsertPackage({ source_id: source.id, name: `skillsmp-${slugify(String(skill.name || skillId))}`, display_name: skill.name || skill.title || skillId, description: skill.description || '', author: skill.author || 'community', package_type: 'skill', waggle_install_type: 'skill', waggle_install_path: `skills/${slugify(String(skill.name || skillId))}.md`, version: '1.0.0', license: 'MIT', repository_url: skill.githubUrl || null, homepage_url: skill.skillUrl || `https://skillsmp.com/skills/${skillId}`, downloads: 0, stars: skill.stars || 0, category: skill.category || 'general', platforms: JSON.stringify(['claude_code', 'codex', 'cursor', 'waggle']), dependencies: JSON.stringify([]), packs: JSON.stringify([]), install_manifest: JSON.stringify(manifest), }); result.added++; } const pagination = (nestedData && !Array.isArray(nestedData)) ? nestedData.pagination : undefined; hasMore = pagination?.hasNext === true && skills.length > 0; page++; // Respect rate limits await new Promise(r => setTimeout(r, 250)); } // end while pages // This query is fully synced completedQueries.add(query); } // end for queries // All queries complete — reset sync state (ready for next full cycle) db.setSyncState(source.id, null); } catch (err) { result.errors.push(safeErrorText(err)); } return result; }, }; // ─── LobeHub Adapter ──────────────────────────────────────────────── const lobehubAdapter: SyncAdapter = { name: 'lobehub', canSync(source) { return source.name === 'lobehub_plugins' || source.url.includes('lobehub'); }, async sync(source, db, _vaultLookup, doFetch) { const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] }; try { const indexUrl = source.api_endpoint || 'https://chat-plugins.lobehub.com/index.json'; const response = await doFetch(indexUrl); if (!response.ok) throw new Error(`LobeHub index: ${response.status}`); const data = await response.json() as LobeHubResponse | LobeHubPlugin[]; const plugins: LobeHubPlugin[] = Array.isArray(data) ? data : (data.plugins || []); if (!Array.isArray(plugins)) throw new Error('Unexpected LobeHub response format'); for (const plugin of plugins) { const identifier = plugin.identifier || plugin.name?.toLowerCase().replace(/\s+/g, '-'); if (!identifier) continue; const manifest: InstallManifest = { plugin_manifest: { name: identifier, version: plugin.version || '1.0.0', description: plugin.description || '', skills: [], mcpServers: plugin.api ? [{ name: identifier, command: 'npx', args: ['-y', `@lobehub/${identifier}`], }] : [], }, }; db.upsertPackage({ source_id: source.id, name: identifier, display_name: plugin.name || identifier, description: plugin.description || '', author: plugin.author || 'lobehub', package_type: 'plugin', waggle_install_type: 'plugin', waggle_install_path: `plugins/${identifier}/`, version: plugin.version || '1.0.0', license: 'MIT', repository_url: `https://github.com/lobehub/lobe-chat-plugins/tree/main/plugins/${identifier}`, homepage_url: plugin.homepage || `https://lobehub.com/plugins/${identifier}`, downloads: plugin.installs || 0, stars: 0, category: plugin.category || 'integration', platforms: JSON.stringify(['lobehub', 'waggle']), dependencies: JSON.stringify([]), packs: JSON.stringify([]), install_manifest: JSON.stringify(manifest), }); result.added++; } } catch (err) { result.errors.push(safeErrorText(err)); } return result; }, }; // ─── Generic JSON Adapter ─────────────────────────────────────────── const genericAdapter: SyncAdapter = { name: 'generic', canSync(source) { return !!source.api_endpoint; }, async sync(source, db, _vaultLookup, doFetch) { const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] }; try { const response = await doFetch(source.api_endpoint!); if (!response.ok) throw new Error(`API ${response.status}`); const data = await response.json() as RemoteCatalogItem[] | RemoteCatalogResponse; const items: RemoteCatalogItem[] = Array.isArray(data) ? data : (data.skills || data.plugins || (Array.isArray(data.data) ? data.data : undefined) || []); const validPackageTypes: ReadonlyArray = ['skill', 'plugin', 'mcp_server', 'template', 'pack']; for (const item of items) { const packageType: MarketplacePackage['package_type'] = item.type && (validPackageTypes as readonly string[]).includes(item.type) ? item.type as MarketplacePackage['package_type'] : 'skill'; db.upsertPackage({ source_id: source.id, name: item.slug || item.name?.toLowerCase().replace(/\s+/g, '-') || `${source.name}-${result.added}`, display_name: item.name || item.title || item.slug, description: item.description || '', author: item.author || source.name, package_type: packageType, waggle_install_type: item.type === 'mcp' ? 'mcp' : item.type === 'plugin' ? 'plugin' : 'skill', waggle_install_path: `skills/${item.slug || item.name?.toLowerCase().replace(/\s+/g, '-')}.md`, version: item.version || '1.0.0', repository_url: item.repository_url || null, homepage_url: item.url || null, downloads: item.downloads || item.installs || 0, stars: item.stars || item.likes || 0, category: item.category || 'general', platforms: JSON.stringify(['waggle']), dependencies: JSON.stringify([]), packs: JSON.stringify([]), install_manifest: JSON.stringify({ skill_url: item.raw_url || item.content_url, }), }); result.added++; } } catch (err) { result.errors.push(safeErrorText(err)); } return result; }, }; // ─── Deduplication ───────────────────────────────────────────────── /** * Normalize a package name for dedup comparison. * Strips source-name prefixes, lowercases, removes hyphens/underscores. */ export function normalizeName(name: string): string { // Remove common source prefixes (e.g., "awesome-mcp-servers-" prefix, "npm-mcp-servers-") const n = name.replace( /^(?:awesome-mcp-servers|awesome-claude-skills-travisvn|awesome-agent-skills-[a-z]+|awesome-cc-plugins|awesome-claude-plugins-[a-z]+|npm-mcp-servers|npm-mcp-protocol|skills-sh|agent-skills-cc|mcpmarket|skillsdirectory|mcpservers-org|awesome-skills-app|claude-skills-collection|plugins-plus-skills|antigravity-awesome-skills|skillmatic-awesome-skills|alirezarezvani-claude-skills|microsoft-skills|muratcankoylan-context-engineering|hoodini-ai-agents-skills|hashicorp-agent-skills|supabase-agent-skills|callstack-agent-skills|ckanner-agent-skills)-/i, '', ); return n.toLowerCase().replace(/[-_]/g, ''); } /** * Remove duplicate packages from the database. * Keeps the entry with the highest (stars + downloads) for each normalized name. * * @returns Count of duplicates removed */ export function deduplicatePackages(db: MarketplaceDB): number { const rawDb = db.getRawDb(); // Get all packages with their scores const allPackages = rawDb.prepare( `SELECT id, name, stars, downloads FROM packages ORDER BY (stars + downloads) DESC` ).all() as Array<{ id: number; name: string; stars: number; downloads: number }>; // Group by normalized name const groups = new Map(); for (const pkg of allPackages) { const norm = normalizeName(pkg.name); if (!norm) continue; const group = groups.get(norm) || []; group.push(pkg); groups.set(norm, group); } // Find duplicates (groups with > 1 entry) const toDelete: number[] = []; for (const [, group] of groups) { if (group.length <= 1) continue; // Already sorted by score desc -- keep the first, delete the rest for (let i = 1; i < group.length; i++) { toDelete.push(group[i].id); } } if (toDelete.length === 0) return 0; // Delete in batches (SQLite has a limit on bound parameters) const batchSize = 100; for (let i = 0; i < toDelete.length; i += batchSize) { const batch = toDelete.slice(i, i + batchSize); const placeholders = batch.map(() => '?').join(','); // Delete from related tables first rawDb.prepare(`DELETE FROM pack_packages WHERE package_id IN (${placeholders})`).run(...batch); rawDb.prepare(`DELETE FROM package_tags WHERE package_id IN (${placeholders})`).run(...batch); // Delete from packages (FTS content-sync table is external-content; // stale FTS entries for deleted rows are harmless and cleaned on rebuild) rawDb.prepare(`DELETE FROM packages WHERE id IN (${placeholders})`).run(...batch); } return toDelete.length; } // ─── Sync Engine ──────────────────────────────────────────────────── /** * Adapter priority order matters! More specific adapters must come before * generic ones. The first adapter whose canSync() returns true is used. */ const ADAPTERS: SyncAdapter[] = [ clawhubAdapter, skillsmpAdapter, lobehubAdapter, awesomeListAdapter, // Before github -- matches awesome-* community repos githubRepoContentAdapter, // Before github -- matches community repos with specific repo path npmSearchAdapter, // NPM registry sources webRegistryAdapter, // Before generic -- handles aggregators with api_endpoint githubAdapter, // GitHub orgs and official repos genericAdapter, // Fallback for anything with an api_endpoint ]; export class MarketplaceSync { private db: MarketplaceDB; private vaultLookup?: VaultLookupFn; /** SSRF-guarded fetch injected by the server; plain global fetch otherwise. */ private doFetch: FetchFn; constructor(db: MarketplaceDB, vaultLookup?: VaultLookupFn, fetchImpl?: FetchFn) { this.db = db; this.vaultLookup = vaultLookup; this.doFetch = fetchImpl ?? defaultFetch; } /** * Sync packages from all (or specified) sources. */ async syncAll(options: SyncOptions = {}): Promise { const sources = this.db.listSources(); const filteredSources = options.sources ? sources.filter(s => options.sources!.includes(s.name)) : sources; const results: SyncResult[] = []; for (const source of filteredSources) { console.log(`[sync] Syncing from ${source.display_name}...`); const adapter = ADAPTERS.find(a => a.canSync(source)); if (!adapter) { results.push({ source: source.name, added: 0, updated: 0, removed: 0, errors: [`No sync adapter for source type: ${source.source_type}`], }); continue; } try { const result = await adapter.sync(source, this.db, this.vaultLookup, this.doFetch); results.push(result); console.log(`[sync] ${source.display_name}: +${result.added} added, ${result.errors.length} errors`); } catch (err) { results.push({ source: source.name, added: 0, updated: 0, removed: 0, errors: [safeErrorText(err)], }); } } // Run dedup after syncing all sources if (!options.sources) { const removed = deduplicatePackages(this.db); if (removed > 0) { console.log(`[sync] Dedup: removed ${removed} duplicate packages`); } } return results; } }