Files
waggle-os/packages/marketplace/src/sync.ts
Oleg Maslov 0c3e2ead3b
Some checks failed
Installer Smoke / installer-smoke (push) Has been cancelled
moving
2026-09-02 10:10:29 +02:00

1387 lines
51 KiB
TypeScript

/**
* Waggle Marketplace — Source Sync Engine
*
* Fetches live package data from marketplace sources and updates
* the local SQLite database. Supports incremental and full syncs.
*
* Supported sources:
* - ClawHub API (https://clawhub.ai/api/v1/skills)
* - SkillsMP API (https://skillsmp.dev/api/v1/skills)
* - GitHub Organizations (anthropics, modelcontextprotocol, lobehub, cursor, etc.)
* - LobeHub Plugin Registry (https://chat-plugins.lobehub.com/index.json)
* - AITMPL Skills (https://www.aitmpl.com/api/skills)
* - Awesome-List repos (awesome-mcp-servers, awesome-claude-skills, etc.)
* - GitHub Repo Content (repos with SKILL.md files in subdirectories)
* - Web Registries (skills.sh, mcpmarket.com, skillsdirectory.com, etc.)
* - NPM Registry (mcp-server keyword search)
*
* Rate limits are respected per source. Default: 500 req/day for SkillsMP,
* 100 req/min for GitHub API, unlimited for ClawHub.
*/
import { MarketplaceDB } from './db.js';
import { type FetchFn, defaultFetch } from './fetcher.js';
import type {
MarketplacePackage,
MarketplaceSource,
SyncOptions,
SyncResult,
InstallManifest,
} from './types.js';
// ─── Helpers ───────────────────────────────────────────────────────
/** Truncate error messages to a reasonable length */
function safeErrorText(err: unknown, maxLen = 200): string {
const msg = err instanceof Error ? err.message : String(err);
return msg.length > maxLen ? msg.slice(0, maxLen) + '...' : msg;
}
/** Check if a URL is an awesome-list repo (has 'awesome' anywhere in the repo name) */
function isAwesomeListUrl(url: string): boolean {
return /github\.com\/[^/]+\/[^/]*awesome[^/]*/i.test(url);
}
/** Build GitHub API headers */
function githubHeaders(): Record<string, string> {
const headers: Record<string, string> = {
'Accept': 'application/vnd.github.v3+json',
'User-Agent': 'waggle-marketplace-sync',
};
if (process.env.GITHUB_TOKEN) {
headers['Authorization'] = `Bearer ${process.env.GITHUB_TOKEN}`;
}
return headers;
}
/** Detect package type from name/URL */
function detectPackageType(name: string, url?: string): 'skill' | 'plugin' | 'mcp' {
const combined = `${name} ${url || ''}`.toLowerCase();
if (combined.includes('mcp') || combined.includes('model-context-protocol')) return 'mcp';
if (combined.includes('plugin')) return 'plugin';
return 'skill';
}
/** Make a URL-friendly slug from a name */
function slugify(name: string): string {
return name
.toLowerCase()
.replace(/[^a-z0-9]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 80);
}
/** Make a display name from a slug */
function displayNameFromSlug(slug: string): string {
return slug
.replace(/[-_]/g, ' ')
.replace(/\b\w/g, (c: string) => c.toUpperCase());
}
// ─── Interface ─────────────────────────────────────────────────────
/**
* Optional function to look up API keys from the vault.
* Accepts a vault key name and returns the secret value or null.
*/
export type VaultLookupFn = (key: string) => string | null;
interface SyncAdapter {
name: string;
canSync(source: MarketplaceSource): boolean;
// `doFetch` is the SSRF-guarded fetcher injected by the server (or plain
// global fetch for standalone/CLI). Every outbound registry fetch below must
// use it rather than the global `fetch` so an attacker-controlled source URL
// cannot reach an internal/link-local host.
sync(
source: MarketplaceSource,
db: MarketplaceDB,
vaultLookup: VaultLookupFn | undefined,
doFetch: FetchFn,
): Promise<SyncResult>;
}
// ─── External JSON Boundary Shapes ─────────────────────────────────
// Marketplace sources return loosely-structured JSON. These interfaces model
// the fields the adapters actually read — every field is optional because
// upstream payloads vary by provider and version. Treat them as a typed view
// over untrusted data: never assume a field is present.
/** A single skill/plugin/MCP entry as returned by a remote registry. */
interface RemoteCatalogItem {
id?: string | number;
slug?: string;
name?: string;
title?: string;
description?: string;
summary?: string;
author?: string;
creator?: string;
owner?: string;
version?: string;
license?: string;
type?: string;
category?: string;
tags?: string[];
url?: string;
homepage?: string;
repository_url?: string;
repo_url?: string;
raw_url?: string;
content_url?: string;
download_url?: string;
skillUrl?: string;
githubUrl?: string;
npm_package?: string;
downloads?: number;
installs?: number;
download_count?: number;
stars?: number;
likes?: number;
rating?: number;
rating_count?: number;
platforms?: string[];
dependencies?: string[];
identifier?: string;
api?: unknown;
}
/** Generic paginated/wrapped registry response (web-registry + generic adapters). */
interface RemoteCatalogResponse {
skills?: RemoteCatalogItem[];
data?: RemoteCatalogItem[] | { skills?: RemoteCatalogItem[]; pagination?: { hasNext?: boolean } };
items?: RemoteCatalogItem[];
tools?: RemoteCatalogItem[];
plugins?: RemoteCatalogItem[];
results?: RemoteCatalogItem[];
}
/** GitHub git-tree API response. */
interface GitHubTreeResponse {
tree?: Array<{ type?: string; path?: string }>;
}
/** GitHub repos list entry. */
interface GitHubRepo {
name: string;
full_name: string;
description?: string | null;
topics?: string[];
html_url: string;
homepage?: string | null;
clone_url: string;
stargazers_count?: number;
license?: { spdx_id?: string | null } | null;
}
/** npm search API response. */
interface NpmSearchResponse {
objects?: NpmSearchObject[];
results?: NpmSearchObject[];
}
interface NpmPackageInfo {
name?: string;
description?: string;
version?: string;
keywords?: string[];
publisher?: { username?: string };
author?: { name?: string };
maintainers?: Array<{ username?: string }>;
links?: { repository?: string; homepage?: string; npm?: string };
repository?: { url?: string };
}
/**
* A search hit from the npm registry. npm's `/-/v1/search` nests the package
* under `.package`, but some mirrors return the package fields inline — so this
* extends {@link NpmPackageInfo} to cover both shapes (`obj.package || obj`).
*/
interface NpmSearchObject extends NpmPackageInfo {
package?: NpmPackageInfo;
}
/** ClawHub paginated skills response. */
interface ClawHubResponse {
skills?: RemoteCatalogItem[];
data?: RemoteCatalogItem[];
}
/** SkillsMP search response (nested data envelope). */
interface SkillsMpResponse {
skills?: RemoteCatalogItem[];
data?: {
skills?: RemoteCatalogItem[];
pagination?: { hasNext?: boolean };
} | RemoteCatalogItem[];
}
/** LobeHub plugin index response. */
interface LobeHubResponse {
plugins?: LobeHubPlugin[];
}
interface LobeHubPlugin {
identifier?: string;
name?: string;
description?: string;
version?: string;
author?: string;
homepage?: string;
category?: string;
installs?: number;
api?: unknown;
}
// ─── Awesome-List Parser Adapter ───────────────────────────────────
// Handles GitHub "awesome-*" repos that contain markdown lists of links.
/**
* Parse a markdown README for links, supporting:
* - Bullet lists: `- [name](url) - description`
* - Table rows: `| [name](url) | description |`
* - Heading-level links: `### [name](url)`
*/
export function parseAwesomeListMarkdown(markdown: string): Array<{
name: string;
url: string;
description: string;
}> {
const results: Array<{ name: string; url: string; description: string }> = [];
const seen = new Set<string>();
// Process line by line to prevent cross-line greedy matching
const lines = markdown.split('\n');
const linkPattern = /\[([^\]]+)\]\((https?:\/\/[^)]+)\)(?:[ \t]*[-\u2013|:][ \t]*(.*))?/;
for (const line of lines) {
const match = linkPattern.exec(line);
if (!match) continue;
const name = match[1].trim();
const url = match[2].trim();
const description = (match[3] || '').trim();
// Skip badges, images, ToC links, and anchors
if (url.includes('shields.io') || url.includes('badge') || url.startsWith('#')) continue;
if (/\.(png|jpg|jpeg|gif|svg)$/i.test(url)) continue;
if (name.length < 2 || name.length > 120) continue;
// Dedup by URL
if (seen.has(url)) continue;
seen.add(url);
results.push({ name, url, description });
}
return results;
}
const awesomeListAdapter: SyncAdapter = {
name: 'awesome-list',
canSync(source) {
// Match community_repo or aggregator sources with awesome-* in URL
if (source.source_type !== 'community_repo' && source.source_type !== 'aggregator') return false;
return isAwesomeListUrl(source.url);
},
async sync(source, db, _vaultLookup, doFetch) {
const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] };
// No cap — we store metadata only, actual content fetched on install
try {
// Extract owner/repo from URL
const match = source.url.match(/github\.com\/([^/]+)\/([^/]+)/);
if (!match) throw new Error(`Cannot parse GitHub URL: ${source.url}`);
const [, owner, repo] = match;
// Fetch README.md via raw.githubusercontent.com
const readmeUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/README.md`;
const response = await doFetch(readmeUrl, {
headers: { 'User-Agent': 'waggle-marketplace-sync' },
});
let markdown: string;
if (!response.ok) {
// Try master branch as fallback
const fallbackUrl = `https://raw.githubusercontent.com/${owner}/${repo}/master/README.md`;
const fallbackResp = await doFetch(fallbackUrl, {
headers: { 'User-Agent': 'waggle-marketplace-sync' },
});
if (!fallbackResp.ok) throw new Error(`README fetch failed: ${response.status} / ${fallbackResp.status}`);
markdown = await fallbackResp.text();
} else {
markdown = await response.text();
}
const links = parseAwesomeListMarkdown(markdown);
for (const link of links) {
const installType = detectPackageType(link.name, link.url);
const slug = slugify(link.name);
if (!slug) continue;
const manifest: InstallManifest = installType === 'mcp'
? { npm_package: slug, mcp_config: { name: slug, command: 'npx', args: ['-y', slug] } }
: { skill_url: link.url };
try {
db.upsertPackage({
source_id: source.id,
name: `${source.name}-${slug}`,
display_name: link.name,
description: link.description || `${installType} from ${source.display_name}`,
author: owner,
package_type: installType === 'mcp' ? 'mcp_server' : installType,
waggle_install_type: installType,
waggle_install_path: installType === 'skill'
? `skills/${slug}.md`
: installType === 'plugin'
? `plugins/${slug}/`
: `.mcp.json`,
version: '1.0.0',
license: null,
repository_url: link.url.includes('github.com') ? link.url : null,
homepage_url: link.url,
downloads: 0,
stars: 0,
category: installType === 'mcp' ? 'integration' : 'general',
platforms: JSON.stringify(['claude_code', 'waggle']),
dependencies: JSON.stringify([]),
packs: JSON.stringify([]),
install_manifest: JSON.stringify(manifest),
});
result.added++;
} catch (itemErr) {
result.errors.push(safeErrorText(itemErr));
}
}
} catch (err) {
result.errors.push(safeErrorText(err));
}
return result;
},
};
// ─── GitHub Repo Content Adapter ───────────────────────────────────
// Handles single repos with multiple skills in subdirectories.
const githubRepoContentAdapter: SyncAdapter = {
name: 'github-repo-content',
canSync(source) {
if (source.source_type !== 'community_repo') return false;
// Must be a github.com URL but NOT an awesome-list
if (!source.url.includes('github.com')) return false;
if (isAwesomeListUrl(source.url)) return false;
// Must have a repo path (owner/repo), not just an org
const match = source.url.match(/github\.com\/([^/]+)\/([^/]+)/);
return !!match;
},
async sync(source, db, _vaultLookup, doFetch) {
const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] };
// No cap — metadata only, content fetched on install
try {
const match = source.url.match(/github\.com\/([^/]+)\/([^/]+)/);
if (!match) throw new Error(`Cannot parse GitHub repo URL: ${source.url}`);
const [, owner, repo] = match;
// Use GitHub API to get repo tree
const treeUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/main?recursive=1`;
const response = await doFetch(treeUrl, { headers: githubHeaders() });
let treeData: GitHubTreeResponse;
if (!response.ok) {
// Try master branch
const fallbackUrl = `https://api.github.com/repos/${owner}/${repo}/git/trees/master?recursive=1`;
const fallbackResp = await doFetch(fallbackUrl, { headers: githubHeaders() });
if (!fallbackResp.ok) throw new Error(`GitHub tree API: ${response.status} / ${fallbackResp.status}`);
treeData = await fallbackResp.json() as GitHubTreeResponse;
} else {
treeData = await response.json() as GitHubTreeResponse;
}
const tree = treeData.tree || [];
// Look for SKILL.md, skill.md, *.skill.md files
const skillFiles = tree.filter((item) =>
item.type === 'blob' &&
!!item.path &&
/(?:^|\/)(?:SKILL\.md|skill\.md|[^/]+\.skill\.md)$/i.test(item.path)
);
for (const file of skillFiles) {
// Derive skill name from file path (path presence guaranteed by the filter above)
const pathParts = file.path!.split('/');
const dirName = pathParts.length > 1 ? pathParts[pathParts.length - 2] : pathParts[0];
const slug = slugify(dirName.replace(/\.skill\.md$/i, '').replace(/\.md$/i, ''));
if (!slug) continue;
const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/${file.path}`;
try {
db.upsertPackage({
source_id: source.id,
name: `${source.name}-${slug}`,
display_name: displayNameFromSlug(slug),
description: `Skill from ${owner}/${repo}`,
author: owner,
package_type: 'skill',
waggle_install_type: 'skill',
waggle_install_path: `skills/${slug}.md`,
version: '1.0.0',
license: null,
repository_url: `https://github.com/${owner}/${repo}/tree/main/${pathParts.slice(0, -1).join('/')}`,
homepage_url: `https://github.com/${owner}/${repo}`,
downloads: 0,
stars: 0,
category: 'general',
platforms: JSON.stringify(['claude_code', 'waggle']),
dependencies: JSON.stringify([]),
packs: JSON.stringify([]),
install_manifest: JSON.stringify({ skill_url: rawUrl }),
});
result.added++;
} catch (itemErr) {
result.errors.push(safeErrorText(itemErr));
}
}
} catch (err) {
result.errors.push(safeErrorText(err));
}
return result;
},
};
// ─── Web Registry Adapter ──────────────────────────────────────────
// Handles web-based skill/MCP registries.
const webRegistryAdapter: SyncAdapter = {
name: 'web-registry',
canSync(source) {
// Match aggregator sources with api_endpoint, but NOT github URLs
if (source.source_type === 'aggregator' && source.api_endpoint && !source.url.includes('github.com')) return true;
return false;
},
async sync(source, db, vaultLookup, doFetch) {
const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] };
try {
// Build headers — check vault for API keys (premium sources like skillsdirectory.com)
const headers: Record<string, string> = {
'User-Agent': 'waggle-marketplace-sync',
'Accept': 'application/json',
};
if (vaultLookup) {
const apiKey = vaultLookup(`marketplace:source:${source.name}:api_key`);
if (apiKey) {
headers['Authorization'] = `Bearer ${apiKey}`;
headers['X-Api-Key'] = apiKey;
}
}
// Resume from saved sync state (for rate-limited sources)
const syncState = db.getSyncState(source.id) || {};
let offset = (syncState.lastOffset as number) || 0;
const totalSyncedPrior = (syncState.totalSynced as number) || 0;
const pageSize = 100;
let hasMore = true;
while (hasMore) {
// Append pagination params to the endpoint URL
const baseUrl = source.api_endpoint!;
const sep = baseUrl.includes('?') ? '&' : '?';
const pageUrl = `${baseUrl}${sep}limit=${pageSize}&offset=${offset}`;
const response = await doFetch(pageUrl, { headers });
if (response.status === 401 || response.status === 403) {
result.errors.push(
`${source.display_name} API requires authentication (${response.status}). ` +
`Configure an API key in the vault as "marketplace:source:${source.name}:api_key".`,
);
return result;
}
// Rate limited — save progress and return gracefully
if (response.status === 429) {
db.setSyncState(source.id, {
lastOffset: offset,
totalSynced: totalSyncedPrior + result.added,
lastSyncedAt: new Date().toISOString(),
});
result.errors.push(
`${source.display_name} rate limit reached at offset ${offset}. ` +
`Progress saved — next sync will resume from offset ${offset}.`,
);
return result;
}
if (!response.ok) throw new Error(`API ${response.status}: ${response.statusText}`);
const data = await response.json() as RemoteCatalogItem[] | RemoteCatalogResponse;
// Try multiple JSON response formats
let items: RemoteCatalogItem[] = [];
if (Array.isArray(data)) {
items = data;
} else if (data.skills && Array.isArray(data.skills)) {
items = data.skills;
} else if (data.data && Array.isArray(data.data)) {
items = data.data;
} else if (data.items && Array.isArray(data.items)) {
items = data.items;
} else if (data.tools && Array.isArray(data.tools)) {
items = data.tools;
} else if (data.plugins && Array.isArray(data.plugins)) {
items = data.plugins;
} else if (data.results && Array.isArray(data.results)) {
items = data.results;
}
if (items.length === 0) {
if (offset === 0) {
result.errors.push(`No structured data found in response from ${source.api_endpoint}`);
}
// Full sync complete — reset sync state
db.setSyncState(source.id, null);
hasMore = false;
break;
}
for (const item of items) {
const name = item.slug || item.name || item.title || item.id;
if (!name) continue;
const slug = slugify(String(name));
if (!slug) continue;
const installType = detectPackageType(
String(name),
item.url || item.repository_url || item.homepage,
);
const npmPkg = item.npm_package || slug;
const manifest: InstallManifest = installType === 'mcp'
? { npm_package: npmPkg, mcp_config: { name: slug, command: 'npx', args: ['-y', npmPkg] } }
: { skill_url: item.raw_url || item.content_url || item.download_url || item.url };
try {
db.upsertPackage({
source_id: source.id,
name: `${source.name}-${slug}`,
display_name: item.name || item.title || displayNameFromSlug(slug),
description: item.description || item.summary || '',
author: item.author || item.creator || item.owner || source.name,
package_type: installType === 'mcp' ? 'mcp_server' : installType,
waggle_install_type: installType,
waggle_install_path: installType === 'skill'
? `skills/${slug}.md`
: installType === 'plugin'
? `plugins/${slug}/`
: `.mcp.json`,
version: item.version || '1.0.0',
license: item.license || null,
repository_url: item.repository_url || item.repo_url || null,
homepage_url: item.url || item.homepage || null,
downloads: item.downloads || item.installs || item.download_count || 0,
stars: item.stars || item.likes || item.rating || 0,
category: item.category || item.tags?.[0] || 'general',
platforms: JSON.stringify(item.platforms || ['claude_code', 'waggle']),
dependencies: JSON.stringify(item.dependencies || []),
packs: JSON.stringify([]),
install_manifest: JSON.stringify(manifest),
});
result.added++;
} catch (itemErr) {
result.errors.push(safeErrorText(itemErr));
}
}
// Save progress after each page
offset += items.length;
db.setSyncState(source.id, {
lastOffset: offset,
totalSynced: totalSyncedPrior + result.added,
lastSyncedAt: new Date().toISOString(),
});
// Check if we should fetch the next page
if (items.length < pageSize) {
// Full sync complete — reset sync state
db.setSyncState(source.id, null);
hasMore = false;
} else {
// Brief delay to respect rate limits
await new Promise(r => setTimeout(r, 200));
}
} // end while (hasMore)
} catch (err) {
result.errors.push(safeErrorText(err));
}
return result;
},
};
// ─── NPM Search Adapter ───────────────────────────────────────────
// Searches npm registry for MCP server packages.
/** A normalized npm package entry produced by {@link parseNpmSearchResults}. */
export interface NpmPackageEntry {
name: string;
description: string;
version: string;
author: string;
npm_package: string;
repository_url: string | null;
homepage_url: string | null;
keywords: string[];
}
/** Parse npm search API response into package entries */
export function parseNpmSearchResults(data: NpmSearchResponse): NpmPackageEntry[] {
const objects = data.objects || data.results || [];
if (!Array.isArray(objects)) return [];
return objects.map((obj): NpmPackageEntry => {
const pkg: NpmPackageInfo = obj.package || obj;
return {
name: pkg.name || '',
description: pkg.description || '',
version: pkg.version || '1.0.0',
author: pkg.publisher?.username || pkg.author?.name || pkg.maintainers?.[0]?.username || 'community',
npm_package: pkg.name || '',
repository_url: pkg.links?.repository || pkg.repository?.url || null,
homepage_url: pkg.links?.homepage || pkg.links?.npm || null,
keywords: pkg.keywords || [],
};
}).filter((pkg) => pkg.name);
}
const npmSearchAdapter: SyncAdapter = {
name: 'npm-search',
canSync(source) {
return source.source_type === 'npm_registry';
},
async sync(source, db, _vaultLookup, doFetch) {
const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] };
try {
const apiUrl = source.api_endpoint || 'https://registry.npmjs.org/-/v1/search?text=keywords:mcp-server&size=250';
const response = await doFetch(apiUrl, {
headers: { 'User-Agent': 'waggle-marketplace-sync' },
});
if (!response.ok) throw new Error(`npm API ${response.status}: ${response.statusText}`);
const data = await response.json() as NpmSearchResponse;
const packages = parseNpmSearchResults(data);
for (const pkg of packages) {
const slug = slugify(pkg.name);
if (!slug) continue;
const manifest: InstallManifest = {
npm_package: pkg.npm_package,
mcp_config: {
name: slug,
command: 'npx',
args: ['-y', pkg.npm_package],
},
};
try {
db.upsertPackage({
source_id: source.id,
name: `${source.name}-${slug}`,
display_name: displayNameFromSlug(slug),
description: pkg.description || `MCP server from npm: ${pkg.npm_package}`,
author: pkg.author,
package_type: 'mcp_server',
waggle_install_type: 'mcp',
waggle_install_path: '.mcp.json',
version: pkg.version,
license: null,
repository_url: pkg.repository_url,
homepage_url: pkg.homepage_url || `https://www.npmjs.com/package/${pkg.npm_package}`,
downloads: 0,
stars: 0,
category: 'integration',
platforms: JSON.stringify(['claude_code', 'waggle', 'cursor']),
dependencies: JSON.stringify([]),
packs: JSON.stringify([]),
install_manifest: JSON.stringify(manifest),
});
result.added++;
} catch (itemErr) {
result.errors.push(safeErrorText(itemErr));
}
}
} catch (err) {
result.errors.push(safeErrorText(err));
}
return result;
},
};
// ─── GitHub Org Adapter ────────────────────────────────────────────
// Handles GitHub organization repos (org-level, no specific repo path).
const githubAdapter: SyncAdapter = {
name: 'github',
canSync(source) {
if (source.source_type === 'github_org') return true;
// GitHub URLs that are org-level (no specific repo)
if (!source.url.includes('github.com')) return false;
const match = source.url.match(/github\.com\/([^/]+)(\/([^/]+))?/);
// Only match if no repo path (org-level) and not npm_registry
if (match && !match[3] && source.source_type !== 'npm_registry') return true;
// For sources with repo path, only match official_marketplace types
// (not community_repo - those go to content/awesome adapters)
if (match && match[3] && source.source_type === 'official_marketplace') return true;
return false;
},
async sync(source, db, _vaultLookup, doFetch) {
const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] };
try {
// Extract org/user from URL
const match = source.url.match(/github\.com\/([^/]+)/);
if (!match) throw new Error(`Cannot parse GitHub URL: ${source.url}`);
const owner = match[1];
// Fetch repos with topic filters
const apiUrl = source.api_endpoint || `https://api.github.com/orgs/${owner}/repos?per_page=100&sort=updated`;
const response = await doFetch(apiUrl, { headers: githubHeaders() });
if (!response.ok) throw new Error(`GitHub API ${response.status}: ${response.statusText}`);
const repos = await response.json() as GitHubRepo[];
for (const repo of repos) {
// Filter for skill/MCP/plugin repos
const topics = repo.topics || [];
const isSkill = topics.some((t) =>
['skill', 'agent-skill', 'claude-code', 'mcp', 'mcp-server', 'plugin'].includes(t)
) || repo.name.includes('skill') || repo.name.includes('mcp');
if (!isSkill && repos.length > 20) continue; // Skip non-skill repos in large orgs
const installType = repo.name.includes('mcp') ? 'mcp' :
topics.includes('plugin') ? 'plugin' : 'skill';
const manifest: InstallManifest = installType === 'skill'
? { skill_url: `https://raw.githubusercontent.com/${owner}/${repo.name}/main/SKILL.md` }
: installType === 'mcp'
? { npm_package: repo.full_name, mcp_config: { name: repo.name, command: 'npx', args: ['-y', repo.full_name] } }
: { git_url: repo.clone_url };
db.upsertPackage({
source_id: source.id,
name: repo.name,
display_name: repo.name.replace(/[-_]/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase()),
description: repo.description || `${installType} from ${owner}`,
author: owner,
package_type: installType === 'mcp' ? 'mcp_server' : installType,
waggle_install_type: installType as 'skill' | 'plugin' | 'mcp',
waggle_install_path: installType === 'skill'
? `skills/${repo.name}.md`
: installType === 'plugin'
? `plugins/${repo.name}/`
: `.mcp.json`,
version: '1.0.0',
license: repo.license?.spdx_id || null,
repository_url: repo.html_url,
homepage_url: repo.homepage || null,
downloads: 0,
stars: repo.stargazers_count || 0,
category: installType === 'mcp' ? 'integration' : 'development',
platforms: JSON.stringify(['claude_code', 'waggle']),
dependencies: JSON.stringify([]),
packs: JSON.stringify([]),
install_manifest: JSON.stringify(manifest),
});
result.added++;
}
} catch (err) {
result.errors.push(safeErrorText(err));
}
return result;
},
};
// ─── ClawHub Adapter ────────────────────────────────────────────────
const clawhubAdapter: SyncAdapter = {
name: 'clawhub',
canSync(source) {
return source.name === 'clawhub' || source.url.includes('clawhub.ai');
},
async sync(source, db, _vaultLookup, doFetch) {
const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] };
try {
const apiBase = source.api_endpoint || 'https://clawhub.ai/api/v1';
const perPage = 50;
let hasMore = true;
// Resume from saved sync state
const syncState = db.getSyncState(source.id) || {};
let page = (syncState.lastPage as number) || 1;
const totalSyncedPrior = (syncState.totalSynced as number) || 0;
const maxPage = 10; // Cap at 500 skills per full sync cycle
while (hasMore && page <= maxPage) {
const url = `${apiBase}/skills?page=${page}&per_page=${perPage}&sort=downloads`;
const response = await doFetch(url, {
headers: { 'User-Agent': 'waggle-marketplace-sync' },
});
// Rate limited — save progress and return gracefully
if (response.status === 429) {
db.setSyncState(source.id, {
lastPage: page,
totalSynced: totalSyncedPrior + result.added,
lastSyncedAt: new Date().toISOString(),
});
result.errors.push(
`ClawHub rate limit reached at page ${page}. ` +
`Progress saved — next sync will resume from page ${page}.`,
);
return result;
}
if (!response.ok) {
result.errors.push(`ClawHub API page ${page}: ${response.status}`);
break;
}
const data = await response.json() as ClawHubResponse | RemoteCatalogItem[];
const skills = Array.isArray(data) ? data : (data.skills || data.data);
if (!Array.isArray(skills) || skills.length === 0) {
// Full sync complete — reset sync state
db.setSyncState(source.id, null);
hasMore = false;
break;
}
for (const skill of skills) {
const skillName = skill.slug || skill.name?.toLowerCase().replace(/\s+/g, '-');
if (!skillName) continue;
const manifest: InstallManifest = {
skill_url: skill.raw_url || skill.download_url || `${apiBase}/skills/${skill.id || skill.slug}/raw`,
};
db.upsertPackage({
source_id: source.id,
name: skillName,
display_name: skill.name || skill.title || skillName,
description: skill.description || '',
author: skill.author || skill.creator || 'community',
package_type: 'skill',
waggle_install_type: 'skill',
waggle_install_path: `skills/${skillName}.md`,
version: skill.version || '1.0.0',
license: skill.license || 'MIT',
repository_url: skill.repository_url || null,
homepage_url: skill.homepage || `https://clawhub.ai/skills/${skill.slug || skill.id}`,
downloads: skill.downloads || skill.download_count || 0,
stars: skill.stars || skill.likes || 0,
rating: skill.rating || 0,
rating_count: skill.rating_count || 0,
category: skill.category || 'general',
platforms: JSON.stringify(skill.platforms || ['claude_code', 'waggle']),
dependencies: JSON.stringify(skill.dependencies || []),
packs: JSON.stringify([]),
install_manifest: JSON.stringify(manifest),
});
result.added++;
}
hasMore = skills.length === perPage;
page++;
// Save progress after each page
if (hasMore) {
db.setSyncState(source.id, {
lastPage: page,
totalSynced: totalSyncedPrior + result.added,
lastSyncedAt: new Date().toISOString(),
});
} else {
// Full sync complete (or reached last page) — reset sync state
db.setSyncState(source.id, null);
}
}
// If we hit the page cap, save state for next run
if (page > maxPage && hasMore) {
db.setSyncState(source.id, {
lastPage: page,
totalSynced: totalSyncedPrior + result.added,
lastSyncedAt: new Date().toISOString(),
});
}
} catch (err) {
result.errors.push(safeErrorText(err));
}
return result;
},
};
// ─── SkillsMP Adapter ───────────────────────────────────────────────
const skillsmpAdapter: SyncAdapter = {
name: 'skillsmp',
canSync(source) {
return source.name === 'skillsmp' || source.url.includes('skillsmp.dev') || source.url.includes('skillsmp.com');
},
async sync(source, db, vaultLookup, doFetch) {
const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] };
try {
const apiBase = source.api_endpoint || 'https://skillsmp.com/api/v1';
const perPage = 50; // SkillsMP caps at 50 per page
// Build auth headers from vault
const headers: Record<string, string> = { 'User-Agent': 'waggle-marketplace-sync', 'Accept': 'application/json' };
if (vaultLookup) {
const apiKey = vaultLookup(`marketplace:source:${source.name}:api_key`);
if (apiKey) {
headers['Authorization'] = `Bearer ${apiKey}`;
}
}
// SkillsMP uses a search endpoint (no list-all). Query by broad categories to cover all skills.
const queries = [
'code', 'python', 'javascript', 'typescript', 'react', 'api', 'data', 'test',
'deploy', 'docker', 'git', 'write', 'research', 'design', 'security', 'database',
'marketing', 'web', 'ai', 'agent', 'claude', 'devops', 'aws', 'linux', 'css',
'node', 'rust', 'go', 'java', 'swift', 'mobile', 'review', 'debug', 'build',
'format', 'lint', 'document', 'email', 'sql', 'graphql', 'frontend', 'backend',
];
const seenIds = new Set<string>();
// Resume from saved sync state
const syncState = db.getSyncState(source.id) || {};
const completedQueries = new Set<string>((syncState.completedQueries as string[]) || []);
const startQueryIndex = (syncState.lastQueryIndex as number) || 0;
const startPage = (syncState.lastPage as number) || 1;
const totalSyncedPrior = (syncState.totalSynced as number) || 0;
for (let qi = startQueryIndex; qi < queries.length; qi++) {
const query = queries[qi];
// Skip queries already fully synced in a previous run
if (completedQueries.has(query)) continue;
// For the resumed query, start from lastPage; for subsequent queries, start from 1
let page = (qi === startQueryIndex) ? startPage : 1;
let hasMore = true;
while (hasMore) {
const url = `${apiBase}/skills/search?q=${encodeURIComponent(query)}&limit=${perPage}&page=${page}`;
const response = await doFetch(url, { headers });
if (!response.ok) {
if (response.status === 429) {
// Rate limited — save progress and return gracefully
db.setSyncState(source.id, {
lastQueryIndex: qi,
lastPage: page,
completedQueries: Array.from(completedQueries),
totalSynced: totalSyncedPrior + result.added,
lastSyncedAt: new Date().toISOString(),
});
result.errors.push(
`SkillsMP rate limit reached at query "${query}" page ${page}. ` +
`Progress saved (${completedQueries.size}/${queries.length} queries complete). ` +
`Next sync will resume from here.`,
);
return result;
}
result.errors.push(`SkillsMP API q=${query} page ${page}: ${response.status}`);
break;
}
const data = await response.json() as SkillsMpResponse;
const nestedData = data?.data;
const skills: RemoteCatalogItem[] =
(!Array.isArray(nestedData) ? nestedData?.skills : undefined)
|| data?.skills
|| (Array.isArray(nestedData) ? nestedData : undefined)
|| [];
if (!Array.isArray(skills) || skills.length === 0) {
hasMore = false;
break;
}
for (const skill of skills) {
// Dedup across queries by skill ID
const skillId = String(skill.id || skill.slug || skill.name || '');
if (!skillId || seenIds.has(skillId)) continue;
seenIds.add(skillId);
const manifest: InstallManifest = {
skill_url: skill.githubUrl
? `https://raw.githubusercontent.com/${skill.githubUrl.replace('https://github.com/', '').replace('/tree/main/', '/main/')}/SKILL.md`
: skill.skillUrl || `https://skillsmp.com/skills/${skillId}`,
};
db.upsertPackage({
source_id: source.id,
name: `skillsmp-${slugify(String(skill.name || skillId))}`,
display_name: skill.name || skill.title || skillId,
description: skill.description || '',
author: skill.author || 'community',
package_type: 'skill',
waggle_install_type: 'skill',
waggle_install_path: `skills/${slugify(String(skill.name || skillId))}.md`,
version: '1.0.0',
license: 'MIT',
repository_url: skill.githubUrl || null,
homepage_url: skill.skillUrl || `https://skillsmp.com/skills/${skillId}`,
downloads: 0,
stars: skill.stars || 0,
category: skill.category || 'general',
platforms: JSON.stringify(['claude_code', 'codex', 'cursor', 'waggle']),
dependencies: JSON.stringify([]),
packs: JSON.stringify([]),
install_manifest: JSON.stringify(manifest),
});
result.added++;
}
const pagination = (nestedData && !Array.isArray(nestedData)) ? nestedData.pagination : undefined;
hasMore = pagination?.hasNext === true && skills.length > 0;
page++;
// Respect rate limits
await new Promise(r => setTimeout(r, 250));
} // end while pages
// This query is fully synced
completedQueries.add(query);
} // end for queries
// All queries complete — reset sync state (ready for next full cycle)
db.setSyncState(source.id, null);
} catch (err) {
result.errors.push(safeErrorText(err));
}
return result;
},
};
// ─── LobeHub Adapter ────────────────────────────────────────────────
const lobehubAdapter: SyncAdapter = {
name: 'lobehub',
canSync(source) {
return source.name === 'lobehub_plugins' || source.url.includes('lobehub');
},
async sync(source, db, _vaultLookup, doFetch) {
const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] };
try {
const indexUrl = source.api_endpoint || 'https://chat-plugins.lobehub.com/index.json';
const response = await doFetch(indexUrl);
if (!response.ok) throw new Error(`LobeHub index: ${response.status}`);
const data = await response.json() as LobeHubResponse | LobeHubPlugin[];
const plugins: LobeHubPlugin[] = Array.isArray(data) ? data : (data.plugins || []);
if (!Array.isArray(plugins)) throw new Error('Unexpected LobeHub response format');
for (const plugin of plugins) {
const identifier = plugin.identifier || plugin.name?.toLowerCase().replace(/\s+/g, '-');
if (!identifier) continue;
const manifest: InstallManifest = {
plugin_manifest: {
name: identifier,
version: plugin.version || '1.0.0',
description: plugin.description || '',
skills: [],
mcpServers: plugin.api ? [{
name: identifier,
command: 'npx',
args: ['-y', `@lobehub/${identifier}`],
}] : [],
},
};
db.upsertPackage({
source_id: source.id,
name: identifier,
display_name: plugin.name || identifier,
description: plugin.description || '',
author: plugin.author || 'lobehub',
package_type: 'plugin',
waggle_install_type: 'plugin',
waggle_install_path: `plugins/${identifier}/`,
version: plugin.version || '1.0.0',
license: 'MIT',
repository_url: `https://github.com/lobehub/lobe-chat-plugins/tree/main/plugins/${identifier}`,
homepage_url: plugin.homepage || `https://lobehub.com/plugins/${identifier}`,
downloads: plugin.installs || 0,
stars: 0,
category: plugin.category || 'integration',
platforms: JSON.stringify(['lobehub', 'waggle']),
dependencies: JSON.stringify([]),
packs: JSON.stringify([]),
install_manifest: JSON.stringify(manifest),
});
result.added++;
}
} catch (err) {
result.errors.push(safeErrorText(err));
}
return result;
},
};
// ─── Generic JSON Adapter ───────────────────────────────────────────
const genericAdapter: SyncAdapter = {
name: 'generic',
canSync(source) {
return !!source.api_endpoint;
},
async sync(source, db, _vaultLookup, doFetch) {
const result: SyncResult = { source: source.name, added: 0, updated: 0, removed: 0, errors: [] };
try {
const response = await doFetch(source.api_endpoint!);
if (!response.ok) throw new Error(`API ${response.status}`);
const data = await response.json() as RemoteCatalogItem[] | RemoteCatalogResponse;
const items: RemoteCatalogItem[] = Array.isArray(data)
? data
: (data.skills || data.plugins || (Array.isArray(data.data) ? data.data : undefined) || []);
const validPackageTypes: ReadonlyArray<MarketplacePackage['package_type']> =
['skill', 'plugin', 'mcp_server', 'template', 'pack'];
for (const item of items) {
const packageType: MarketplacePackage['package_type'] =
item.type && (validPackageTypes as readonly string[]).includes(item.type)
? item.type as MarketplacePackage['package_type']
: 'skill';
db.upsertPackage({
source_id: source.id,
name: item.slug || item.name?.toLowerCase().replace(/\s+/g, '-') || `${source.name}-${result.added}`,
display_name: item.name || item.title || item.slug,
description: item.description || '',
author: item.author || source.name,
package_type: packageType,
waggle_install_type: item.type === 'mcp' ? 'mcp' : item.type === 'plugin' ? 'plugin' : 'skill',
waggle_install_path: `skills/${item.slug || item.name?.toLowerCase().replace(/\s+/g, '-')}.md`,
version: item.version || '1.0.0',
repository_url: item.repository_url || null,
homepage_url: item.url || null,
downloads: item.downloads || item.installs || 0,
stars: item.stars || item.likes || 0,
category: item.category || 'general',
platforms: JSON.stringify(['waggle']),
dependencies: JSON.stringify([]),
packs: JSON.stringify([]),
install_manifest: JSON.stringify({
skill_url: item.raw_url || item.content_url,
}),
});
result.added++;
}
} catch (err) {
result.errors.push(safeErrorText(err));
}
return result;
},
};
// ─── Deduplication ─────────────────────────────────────────────────
/**
* Normalize a package name for dedup comparison.
* Strips source-name prefixes, lowercases, removes hyphens/underscores.
*/
export function normalizeName(name: string): string {
// Remove common source prefixes (e.g., "awesome-mcp-servers-" prefix, "npm-mcp-servers-")
const n = name.replace(
/^(?:awesome-mcp-servers|awesome-claude-skills-travisvn|awesome-agent-skills-[a-z]+|awesome-cc-plugins|awesome-claude-plugins-[a-z]+|npm-mcp-servers|npm-mcp-protocol|skills-sh|agent-skills-cc|mcpmarket|skillsdirectory|mcpservers-org|awesome-skills-app|claude-skills-collection|plugins-plus-skills|antigravity-awesome-skills|skillmatic-awesome-skills|alirezarezvani-claude-skills|microsoft-skills|muratcankoylan-context-engineering|hoodini-ai-agents-skills|hashicorp-agent-skills|supabase-agent-skills|callstack-agent-skills|ckanner-agent-skills)-/i,
'',
);
return n.toLowerCase().replace(/[-_]/g, '');
}
/**
* Remove duplicate packages from the database.
* Keeps the entry with the highest (stars + downloads) for each normalized name.
*
* @returns Count of duplicates removed
*/
export function deduplicatePackages(db: MarketplaceDB): number {
const rawDb = db.getRawDb();
// Get all packages with their scores
const allPackages = rawDb.prepare(
`SELECT id, name, stars, downloads FROM packages ORDER BY (stars + downloads) DESC`
).all() as Array<{ id: number; name: string; stars: number; downloads: number }>;
// Group by normalized name
const groups = new Map<string, typeof allPackages>();
for (const pkg of allPackages) {
const norm = normalizeName(pkg.name);
if (!norm) continue;
const group = groups.get(norm) || [];
group.push(pkg);
groups.set(norm, group);
}
// Find duplicates (groups with > 1 entry)
const toDelete: number[] = [];
for (const [, group] of groups) {
if (group.length <= 1) continue;
// Already sorted by score desc -- keep the first, delete the rest
for (let i = 1; i < group.length; i++) {
toDelete.push(group[i].id);
}
}
if (toDelete.length === 0) return 0;
// Delete in batches (SQLite has a limit on bound parameters)
const batchSize = 100;
for (let i = 0; i < toDelete.length; i += batchSize) {
const batch = toDelete.slice(i, i + batchSize);
const placeholders = batch.map(() => '?').join(',');
// Delete from related tables first
rawDb.prepare(`DELETE FROM pack_packages WHERE package_id IN (${placeholders})`).run(...batch);
rawDb.prepare(`DELETE FROM package_tags WHERE package_id IN (${placeholders})`).run(...batch);
// Delete from packages (FTS content-sync table is external-content;
// stale FTS entries for deleted rows are harmless and cleaned on rebuild)
rawDb.prepare(`DELETE FROM packages WHERE id IN (${placeholders})`).run(...batch);
}
return toDelete.length;
}
// ─── Sync Engine ────────────────────────────────────────────────────
/**
* Adapter priority order matters! More specific adapters must come before
* generic ones. The first adapter whose canSync() returns true is used.
*/
const ADAPTERS: SyncAdapter[] = [
clawhubAdapter,
skillsmpAdapter,
lobehubAdapter,
awesomeListAdapter, // Before github -- matches awesome-* community repos
githubRepoContentAdapter, // Before github -- matches community repos with specific repo path
npmSearchAdapter, // NPM registry sources
webRegistryAdapter, // Before generic -- handles aggregators with api_endpoint
githubAdapter, // GitHub orgs and official repos
genericAdapter, // Fallback for anything with an api_endpoint
];
export class MarketplaceSync {
private db: MarketplaceDB;
private vaultLookup?: VaultLookupFn;
/** SSRF-guarded fetch injected by the server; plain global fetch otherwise. */
private doFetch: FetchFn;
constructor(db: MarketplaceDB, vaultLookup?: VaultLookupFn, fetchImpl?: FetchFn) {
this.db = db;
this.vaultLookup = vaultLookup;
this.doFetch = fetchImpl ?? defaultFetch;
}
/**
* Sync packages from all (or specified) sources.
*/
async syncAll(options: SyncOptions = {}): Promise<SyncResult[]> {
const sources = this.db.listSources();
const filteredSources = options.sources
? sources.filter(s => options.sources!.includes(s.name))
: sources;
const results: SyncResult[] = [];
for (const source of filteredSources) {
console.log(`[sync] Syncing from ${source.display_name}...`);
const adapter = ADAPTERS.find(a => a.canSync(source));
if (!adapter) {
results.push({
source: source.name,
added: 0,
updated: 0,
removed: 0,
errors: [`No sync adapter for source type: ${source.source_type}`],
});
continue;
}
try {
const result = await adapter.sync(source, this.db, this.vaultLookup, this.doFetch);
results.push(result);
console.log(`[sync] ${source.display_name}: +${result.added} added, ${result.errors.length} errors`);
} catch (err) {
results.push({
source: source.name,
added: 0,
updated: 0,
removed: 0,
errors: [safeErrorText(err)],
});
}
}
// Run dedup after syncing all sources
if (!options.sources) {
const removed = deduplicatePackages(this.db);
if (removed > 0) {
console.log(`[sync] Dedup: removed ${removed} duplicate packages`);
}
}
return results;
}
}