Files
waggle-os/packages/server/src/local/routes/marketplace.ts
shumeiko.it b734c1d3d2 refactor: remove KVARK, enterprise packs and the compliance subsystem (P1.4, P1.6)
KVARK is the commercial enterprise product and is not part of the community
edition. The agent's KVARK tools were never wired into the server, so
removing them changes no runtime behaviour beyond the routes listed here.

- agent: delete kvark-tools; CombinedRetrieval and formatCombinedResult now
  merge workspace and personal memory only (no KVARK lane, no source
  conflict note); connector substrate is always 'waggle'
- agent: promote_skill stops at workspace scope; team and enterprise scopes
  answer "not available in this edition" instead of an upgrade prompt
- server: delete src/kvark and GET /api/marketplace/enterprise-packs
- marketplace: delete ENTERPRISE_PACKS
- compliance (EU AI Act dashboard, its templates and the KVARK template
  seeded on every boot): delete core/src/compliance, the /api/compliance/*
  routes, agent compliance-pdf and the Cockpit compliance panel. The
  ai_interactions table lives in the memory substrate and goes with P1.5.
2026-10-20 00:27:16 +02:00

1103 lines
40 KiB
TypeScript

/**
* Marketplace Production Routes
*
* Always-available API endpoints for the marketplace — search catalog,
* list/install/uninstall packages, query packs, and run security scans.
*
* These replace the dev-only /_dev/marketplace/ routes with stable
* /api/marketplace/ contracts.
*/
import type { FastifyInstance, FastifyReply } from 'fastify';
import { existsSync, readFileSync } from 'fs';
import { join } from 'path';
import { homedir } from 'os';
import { MarketplaceDB, MarketplaceInstaller, MarketplaceSync, SecurityGate, PACKAGE_CATEGORIES, recategorizeAll, isCiscoScannerAvailable, resolveSkillSource, SkillSourceError, createMarketplaceMcpProvenance } from '@waggle/marketplace';
import type { InstallationType, SearchSort, ScanResult, MarketplacePackage, MarketplaceMcpProvenance, MarketplaceApprovalIdentity, FetchFn } from '@waggle/marketplace';
import { validateSkillMd } from '@waggle/sdk';
import { safeFetch, assertUrlAllowed, scanForInjection } from '@waggle/agent';
import { emitNotification } from './notifications.js';
import { loadMcpConfig, removeMcpServerEntry } from '../mcp-config.js';
import { enqueueHeldAction } from '../held-action-executor.js';
import { isMarketplaceBackgroundSyncDisabled } from '../marketplace-background-sync.js';
/**
* SSRF-guarded outbound fetch injected into every marketplace component that
* pulls external content (installer skill_url, sync registry adapters, the
* multi-source resolver). An attacker-influenced URL that resolves to a
* private / link-local / loopback address is refused before any request.
*/
const guardedFetch: FetchFn = (url, init) => safeFetch(url, init);
/** Fetch used by /install-url — injectable for hermetic route tests (mirrors
* cisco-scanner's setExecFile test-injection precedent). */
let installUrlFetch: FetchFn = guardedFetch;
export function setInstallUrlFetchForTests(fn: FetchFn | null): void {
installUrlFetch = fn ?? guardedFetch;
}
/** Installed skill names must be safe path segments (mirrors writeSkill's rule). */
const SAFE_SKILL_NAME = /^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$/;
function sanitizeSkillName(name: string): string {
const cleaned = name.trim().replace(/[^a-zA-Z0-9_-]/g, '-').replace(/^[-_]+/, '').slice(0, 64);
return SAFE_SKILL_NAME.test(cleaned) ? cleaned : '';
}
type ScanStatus = 'passed' | 'failed' | 'not_scanned' | 'unavailable';
/**
* The `packages` table carries security-scan columns (populated by the
* security gate) that are not part of the public `MarketplacePackage` shape,
* since `search()` does `SELECT p.*`. Narrow to this when reading them.
*/
type PackageWithSecurity = MarketplacePackage & {
security_status?: string;
security_score?: number;
};
export async function marketplaceRoutes(fastify: FastifyInstance) {
// ── Helpers ──────────────────────────────────────────────────────────
function getDb(): MarketplaceDB | null {
return fastify.marketplace ?? null;
}
function requireDb(reply: FastifyReply): MarketplaceDB | null {
const db = getDb();
if (!db) {
reply.code(503).send({
error: 'Marketplace not available',
hint: 'marketplace.db was not found or failed to load',
});
return null;
}
return db;
}
// ── GET /api/marketplace/search ─────────────────────────────────────
// Search the package catalog with FTS5 + faceted filters.
// Legacy clients used "plugins" before the marketplace catalog generalized
// packages behind /search. Return a machine-readable hint instead of a 404.
fastify.get('/api/marketplace/plugins', async (_request, reply) => {
return reply
.code(301)
.header('Location', '/api/marketplace/search')
.send({
redirect: '/api/marketplace/search',
hint: 'Use /api/marketplace/search; /api/marketplace/plugins is a legacy alias.',
});
});
fastify.get('/api/marketplace/search', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const { query, type, category, pack, source, sort, limit, offset } = request.query as {
query?: string;
type?: string;
category?: string;
pack?: string;
source?: string;
sort?: string;
limit?: string;
offset?: string;
};
const validSorts: SearchSort[] = ['relevance', 'popular', 'recent', 'name'];
const sortParam = sort && validSorts.includes(sort as SearchSort)
? sort as SearchSort
: undefined;
const results = db.search({
query: query || '',
type: type as InstallationType | undefined,
category,
pack,
source,
sort: sortParam,
limit: limit ? parseInt(limit, 10) : 20,
offset: offset ? parseInt(offset, 10) : 0,
});
// Annotate each package with installation status and scan status
const annotated = results.packages.map(pkg => {
const rawPkg = pkg as PackageWithSecurity;
const secStatus = rawPkg.security_status;
const secScore = rawPkg.security_score;
let scanStatus: ScanStatus = 'not_scanned';
if (secStatus && secStatus !== 'unscanned') {
if (secStatus === 'clean' || secStatus === 'low') {
scanStatus = 'passed';
} else if (secStatus === 'critical' || secStatus === 'high') {
scanStatus = 'failed';
} else if (secStatus === 'medium') {
scanStatus = 'passed'; // medium = passed with warnings
}
}
return {
...pkg,
installed: db.isInstalled(pkg.id),
scanStatus,
scanScore: (secScore != null && secScore >= 0) ? secScore : undefined,
};
});
return {
...results,
packages: annotated,
categories: PACKAGE_CATEGORIES,
};
});
// ── GET /api/marketplace/packs ──────────────────────────────────────
// List all capability packs.
fastify.get('/api/marketplace/packs', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const packs = db.listPacks();
return { packs, total: packs.length };
});
// ── GET /api/marketplace/packs/:slug ────────────────────────────────
// Get pack detail with its packages.
fastify.get('/api/marketplace/packs/:slug', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const { slug } = request.params as { slug: string };
const result = db.getPacksBySlug(slug);
if (!result) {
return reply.code(404).send({ error: `Pack "${slug}" not found` });
}
return result;
});
// ── POST /api/marketplace/install ───────────────────────────────────
// Install a package from the marketplace with SecurityGate pre-scan.
// Severity-based gating:
// CRITICAL → 403 (always blocked)
// HIGH → 403 unless force=true (override logged to audit)
// MEDIUM → install proceeds, warnings in response
// LOW → install proceeds, logged to audit trail
// CLEAN → install proceeds immediately
fastify.post('/api/marketplace/install', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const body = request.body as {
packageId: number;
installPath?: string;
settings?: Record<string, string>;
force?: boolean;
forceInsecure?: boolean;
expectedInstallType?: InstallationType;
expectedMcpProvenance?: MarketplaceMcpProvenance;
expectedApprovalIdentity?: MarketplaceApprovalIdentity;
};
if (!body.packageId) {
return reply.code(400).send({ error: 'packageId is required' });
}
// ── SecurityGate pre-scan (heuristics-only for now) ──────────────
const pkg = db.getPackage(body.packageId);
if (!pkg) {
return reply.code(404).send({ error: `Package ID ${body.packageId} not found` });
}
if (
body.expectedMcpProvenance
&& body.expectedInstallType
&& pkg.waggle_install_type !== body.expectedInstallType
) {
const error = 'Marketplace MCP package changed during installation; retry from the refreshed catalog';
return reply.code(409).send({
success: false,
error,
message: error,
errorCode: 'PACKAGE_IDENTITY_CHANGED',
});
}
if (body.expectedMcpProvenance) {
let actualMcpProvenance: MarketplaceMcpProvenance | undefined;
try {
const mcpConfig = pkg.install_manifest?.mcp_config;
if (!mcpConfig) throw new Error('Marketplace MCP snapshot has no configuration.');
actualMcpProvenance = createMarketplaceMcpProvenance(
db.getSource(pkg.source_id),
{ name: pkg.name, version: pkg.version },
mcpConfig,
);
} catch {
// A delegated install treats an invalid fresh snapshot as the same
// retryable identity conflict as a valid-but-different snapshot.
}
if (
!actualMcpProvenance
|| !MarketplaceInstaller.mcpProvenanceMatches(
actualMcpProvenance,
body.expectedMcpProvenance,
)
) {
const error = 'Marketplace MCP package changed during installation; retry from the refreshed catalog';
return reply.code(409).send({
success: false,
error,
message: error,
errorCode: 'PACKAGE_IDENTITY_CHANGED',
});
}
}
let scanResult: ScanResult | undefined;
if (!body.expectedMcpProvenance) {
const gate = new SecurityGate({
enable_gen_trust_hub: false,
enable_cisco_scanner: false,
enable_mcp_guardian: false,
enable_heuristics: true,
});
try {
scanResult = await gate.scan(pkg);
} catch {
// Scan failure should not block installation — proceed with warning
}
}
if (scanResult) {
const severity = scanResult.overall_severity;
const score = scanResult.security_score;
// CRITICAL (score 0): Always blocked — return 403
if (severity === 'CRITICAL') {
// Log to audit store if available
try {
fastify.auditStore?.record({
capabilityName: pkg.name,
capabilityType: pkg.waggle_install_type,
source: 'marketplace',
riskLevel: 'critical',
trustSource: 'security-gate',
approvalClass: 'blocked',
action: 'blocked',
initiator: 'system',
detail: `SecurityGate blocked: ${scanResult.findings.length} finding(s), severity=${severity}, score=${score}`,
});
} catch (err) {
// M2 (Phase 4): this write was silently rejected for months because
// the install_audit risk_level CHECK lacked 'critical' — never
// swallow it invisibly again. Still non-blocking.
fastify.log.warn({ err, pkg: pkg.name }, 'CRITICAL-block install audit write failed');
}
return reply.code(403).send({
blocked: true,
severity,
score,
findings: scanResult.findings,
message: `Installation blocked: ${scanResult.findings.length} CRITICAL security finding(s) detected.`,
});
}
// HIGH (score 25): Blocked unless force=true
if (severity === 'HIGH' && !body.force) {
try {
fastify.auditStore?.record({
capabilityName: pkg.name,
capabilityType: pkg.waggle_install_type,
source: 'marketplace',
riskLevel: 'high',
trustSource: 'security-gate',
approvalClass: 'blocked',
action: 'blocked',
initiator: 'system',
detail: `SecurityGate blocked (HIGH): ${scanResult.findings.length} finding(s). Use force=true to override.`,
});
} catch { /* audit failure is non-blocking */ }
return reply.code(403).send({
blocked: true,
severity,
score,
findings: scanResult.findings,
message: `Installation blocked: ${scanResult.findings.length} HIGH severity finding(s). Send force=true to override.`,
});
}
// HIGH with force=true: Log the override to audit trail
if (severity === 'HIGH' && body.force) {
try {
fastify.auditStore?.record({
capabilityName: pkg.name,
capabilityType: pkg.waggle_install_type,
source: 'marketplace',
riskLevel: 'high',
trustSource: 'security-gate',
approvalClass: 'elevated',
action: 'approved',
initiator: 'user',
detail: `User force-override install despite HIGH severity findings: ${scanResult.findings.map(f => f.title).join('; ')}`,
});
} catch { /* audit failure is non-blocking */ }
}
// MEDIUM: Log warning but proceed
if (severity === 'MEDIUM') {
try {
fastify.auditStore?.record({
capabilityName: pkg.name,
capabilityType: pkg.waggle_install_type,
source: 'marketplace',
riskLevel: 'medium',
trustSource: 'security-gate',
approvalClass: 'standard',
action: 'approved',
initiator: 'system',
detail: `SecurityGate passed with warnings: ${scanResult.findings.length} MEDIUM finding(s)`,
});
} catch { /* audit failure is non-blocking */ }
}
// LOW: Log to audit trail only
if (severity === 'LOW') {
try {
fastify.auditStore?.record({
capabilityName: pkg.name,
capabilityType: pkg.waggle_install_type,
source: 'marketplace',
riskLevel: 'low',
trustSource: 'security-gate',
approvalClass: 'standard',
action: 'approved',
initiator: 'system',
detail: `SecurityGate passed: ${scanResult.findings.length} LOW finding(s)`,
});
} catch { /* audit failure is non-blocking */ }
}
}
// ── Proceed with installation ────────────────────────────────────
const installer = new MarketplaceInstaller(db, {
enable_gen_trust_hub: false,
enable_cisco_scanner: false,
enable_mcp_guardian: false,
enable_heuristics: true,
}, guardedFetch);
const result = await installer.install({
packageId: body.packageId,
installPath: body.installPath,
settings: body.settings,
force: body.force,
forceInsecure: body.forceInsecure,
expectedInstallType: body.expectedInstallType,
expectedMcpProvenance: body.expectedMcpProvenance,
expectedApprovalIdentity: body.expectedApprovalIdentity,
});
// Update security status in DB after successful install. Prefer the
// installer-level scan (it had the package CONTENT; the route pre-scan is
// content-less and returns CLEAN in practice) — otherwise a forced install
// would clobber the recorded CRITICAL status with a clean one.
const effectiveScan = result.scanResult ?? scanResult;
if (result.success && effectiveScan) {
try {
const rawDb = db.getRawDb();
if (rawDb?.prepare) {
rawDb.prepare(`
UPDATE packages SET
security_status = ?,
security_score = ?
WHERE id = ?
`).run(
effectiveScan.overall_severity.toLowerCase(),
effectiveScan.security_score,
body.packageId,
);
}
} catch { /* non-blocking */ }
}
// M2 follow-through (Phase 4): the pre-scan above runs WITHOUT package
// content (the heuristics layer needs content), so in practice it returns
// CLEAN and blocking happens inside installer.install() — a path that
// previously left NO audit trail. Record blocked installs here so the
// §22.1 "install with audit trail" exit criterion holds for every caller
// (direct marketplace installs AND the /api/mcps/install delegation).
if (!result.success && result.scanResult?.blocked) {
const sev = result.scanResult.overall_severity;
try {
fastify.auditStore?.record({
capabilityName: pkg.name,
capabilityType: pkg.waggle_install_type,
source: 'marketplace',
riskLevel: sev === 'CRITICAL' ? 'critical' : sev === 'HIGH' ? 'high' : 'medium',
trustSource: 'security-gate',
approvalClass: 'blocked',
action: 'blocked',
initiator: 'system',
detail: `Installer blocked: ${result.scanResult.findings.length} finding(s), severity=${sev}`,
});
} catch (err) {
fastify.log.warn({ err, pkg: pkg.name }, 'blocked-install audit write failed');
}
}
// forceInsecure override: success WITH a blocked scan is the one path
// where the user deliberately overrode a HIGH/CRITICAL block at the
// installer level. Without this row, the single most dangerous action in
// the surface would leave a cleaner audit trail than a clean install
// (mirrors the route-level HIGH+force override audit above).
if (result.success && result.scanResult?.blocked) {
const sev = result.scanResult.overall_severity;
try {
fastify.auditStore?.record({
capabilityName: pkg.name,
capabilityType: pkg.waggle_install_type,
source: 'marketplace',
riskLevel: sev === 'CRITICAL' ? 'critical' : sev === 'HIGH' ? 'high' : 'medium',
trustSource: 'security-gate',
approvalClass: 'elevated',
action: 'approved',
initiator: 'user',
detail: `User forceInsecure override — installed despite blocked scan (severity=${sev}): `
+ result.scanResult.findings.map(f => f.title).join('; '),
});
} catch (err) {
fastify.log.warn({ err, pkg: pkg.name }, 'forceInsecure-override audit write failed');
}
}
// Attach security scan info to the response
const response: Record<string, unknown> = { ...result };
if (result.errorCode === 'PACKAGE_IDENTITY_CHANGED') {
response.error = result.message;
}
if (scanResult) {
response.security = {
severity: scanResult.overall_severity,
score: scanResult.security_score,
findingsCount: scanResult.findings.length,
findings: scanResult.findings,
warnings: scanResult.overall_severity === 'MEDIUM'
? scanResult.findings.map(f => `[${f.severity}] ${f.title}`)
: undefined,
};
}
return reply.code(result.success ? 200 : result.errorCode === 'PACKAGE_IDENTITY_CHANGED' ? 409 : 422).send(response);
});
// ── POST /api/marketplace/uninstall ─────────────────────────────────
// Uninstall a previously installed package.
fastify.post('/api/marketplace/uninstall', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const body = request.body as { packageId: number };
if (!body.packageId) {
return reply.code(400).send({ error: 'packageId is required' });
}
const pkg = db.getPackage(body.packageId);
if (pkg?.waggle_install_type === 'mcp') {
const manifest = pkg.install_manifest as { mcp_config?: { name?: string } } | null;
const serverName = manifest?.mcp_config?.name || pkg.name;
const dataDir = fastify.localConfig?.dataDir ?? '';
const runtime = (fastify.agentState as {
mcpRuntime?: { getServer(n: string): unknown; removeServer(n: string): Promise<void> };
} | undefined)?.mcpRuntime;
try {
// Remove the boot source before touching the live process: a failed
// shutdown must never leave a server able to resurrect after restart.
removeMcpServerEntry(dataDir, serverName);
if (loadMcpConfig(dataDir).mcpServers[serverName]) {
throw new Error('Canonical MCP configuration still contains the server');
}
if (runtime?.getServer(serverName)) await runtime.removeServer(serverName);
if (runtime?.getServer(serverName)) {
throw new Error('MCP runtime still contains the server');
}
} catch (err) {
const residualState = {
installed: db.isInstalled(body.packageId),
runtimeRegistered: Boolean(runtime?.getServer(serverName)),
bootConfigured: Boolean(loadMcpConfig(dataDir).mcpServers[serverName]),
};
fastify.log.warn({ err, packageId: body.packageId, residualState }, 'MCP revocation incomplete');
return reply.code(503).send({
success: false,
packageId: pkg.id,
packageName: pkg.name,
installType: 'mcp',
installPath: pkg.waggle_install_path,
message: `MCP revocation incomplete: ${(err as Error).message}`,
errors: [(err as Error).message],
errorCode: 'MCP_REVOCATION_INCOMPLETE',
residualState,
});
}
}
const installer = new MarketplaceInstaller(db, undefined, guardedFetch);
const result = await installer.uninstall(body.packageId);
return reply.code(result.success ? 200 : 422).send(result);
});
// ── GET /api/marketplace/installed ──────────────────────────────────
// List all currently installed packages.
fastify.get('/api/marketplace/installed', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const installations = db.listInstallations();
return { installations, total: installations.length };
});
// ── POST /api/marketplace/security-check ────────────────────────────
// Run a security scan on a package (by ID) without installing it.
fastify.post('/api/marketplace/security-check', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const body = request.body as { packageId: number };
if (!body.packageId) {
return reply.code(400).send({ error: 'packageId is required' });
}
const installer = new MarketplaceInstaller(db, {
enable_gen_trust_hub: false,
enable_cisco_scanner: false,
enable_mcp_guardian: false,
enable_heuristics: true,
}, guardedFetch);
const scanResult = await installer.scanOnly(body.packageId);
if (!scanResult) {
return reply.code(404).send({ error: `Package ID ${body.packageId} not found` });
}
return {
packageId: body.packageId,
severity: scanResult.overall_severity,
score: scanResult.security_score,
blocked: scanResult.blocked,
enginesUsed: scanResult.engines_used,
findingsCount: scanResult.findings.length,
findings: scanResult.findings,
durationMs: scanResult.scan_duration_ms,
contentHash: scanResult.content_hash,
};
});
// ── GET /api/marketplace/sources ────────────────────────────────────
// List all marketplace sources with package counts.
fastify.get('/api/marketplace/sources', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const sources = db.listSourcesWithCounts();
return { sources, total: sources.length };
});
// ── POST /api/marketplace/install-url ───────────────────────────────
// Multi-source skill install (steal #11 — SKILLS ONLY, never plugins/MCP).
// Resolves a source string (SKILL.md URL / GitHub URL / owner-repo shorthand
// / .zip URL) to SKILL.md bytes through the SSRF-guarded fetcher, runs the
// security pipeline (SecurityGate heuristics → injection scan → frontmatter
// contract), and lands the result as a HELD create_skill approval — nothing
// touches disk until a human approves it in Approvals (exact bytes shown).
fastify.post('/api/marketplace/install-url', async (request, reply) => {
const body = request.body as { source?: string; sha256?: string; workspaceId?: string };
if (!body.source || typeof body.source !== 'string') {
return reply.code(400).send({ error: 'source is required' });
}
if (body.sha256 !== undefined && !/^[a-fA-F0-9]{64}$/.test(body.sha256)) {
return reply.code(400).send({ error: 'sha256 must be a 64-character hex digest' });
}
let resolved;
try {
resolved = await resolveSkillSource(body.source, {
sha256: body.sha256,
fetchImpl: installUrlFetch,
});
} catch (err) {
const msg = err instanceof SkillSourceError
? err.message
: `Could not resolve skill source: ${err instanceof Error ? err.message : String(err)}`;
return reply.code(400).send({ error: msg });
}
// Frontmatter contract — Claude-Code compatible (name + description required).
const validation = validateSkillMd(resolved.content);
if (!validation.valid || !validation.metadata) {
return reply.code(422).send({ error: 'Invalid SKILL.md', details: validation.errors });
}
const name = sanitizeSkillName(validation.metadata.name);
if (!name) {
return reply.code(422).send({
error: `Skill name "${validation.metadata.name}" cannot be sanitized to a safe name`,
});
}
// Injection scan on the full external content (CLAUDE.md §7.2).
if (!scanForInjection(resolved.content, 'tool_output').safe) {
return reply.code(422).send({ error: 'SKILL.md content failed the injection scan' });
}
// SecurityGate heuristics, content-aware. CRITICAL/HIGH blocks outright
// (matching /install's gate); lower severities proceed — the held approval
// (human reviews exact bytes) is the final gate either way.
const gate = new SecurityGate({
enable_gen_trust_hub: false,
enable_cisco_scanner: false,
enable_mcp_guardian: false,
enable_heuristics: true,
});
const syntheticPkg = {
id: -1, source_id: -1, name, display_name: name,
description: validation.metadata.description, author: validation.metadata.author ?? 'unknown',
package_type: 'skill', waggle_install_type: 'skill', waggle_install_path: '',
version: validation.metadata.version ?? '0.0.0', license: null,
repository_url: null, homepage_url: null, downloads: 0, stars: 0, rating: 0,
rating_count: 0, category: 'community', subcategory: null, install_manifest: null,
platforms: [], min_waggle_version: null, dependencies: [], packs: [],
created_at: '', updated_at: '',
} as MarketplacePackage;
let scan: ScanResult | undefined;
try {
scan = await gate.scan(syntheticPkg, resolved.content);
} catch {
// Heuristics failure never blocks — the held approval still gates.
}
if (scan && (scan.overall_severity === 'CRITICAL' || scan.overall_severity === 'HIGH')) {
return reply.code(403).send({
error: `Security scan blocked install (${scan.overall_severity})`,
findings: scan.findings.slice(0, 5),
});
}
// Held approval — never a direct write. On approve, executeHeldAction runs
// the real create_skill tool → sanctioned writeSkill seam (backup + audit).
const enq = enqueueHeldAction(fastify, {
workspaceId: body.workspaceId ?? null,
source: 'marketplace:install-url',
tool: 'create_skill',
args: { name, content: resolved.content },
summary: `Install skill "${name}" from ${resolved.resolvedUrl}`,
});
if ('refused' in enq) {
return reply.code(422).send({ error: `Skill proposal refused (${enq.refused})` });
}
return reply.code(202).send({
held: true,
id: enq.id,
name,
sourceType: resolved.sourceType,
resolvedUrl: resolved.resolvedUrl,
scanSeverity: scan?.overall_severity ?? 'UNSCANNED',
warnings: validation.warnings,
});
});
// ── POST /api/marketplace/sources ───────────────────────────────────
// Add a user-defined source and trigger an immediate sync for it.
fastify.post('/api/marketplace/sources', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const body = request.body as {
name?: string;
url?: string;
displayName?: string;
};
if (!body.name || !body.url) {
return reply.code(400).send({ error: 'name and url are required' });
}
// Validate URL format
try {
new URL(body.url);
} catch {
return reply.code(400).send({ error: 'Invalid URL format' });
}
// SSRF guard: a user-supplied source URL is fetched by the sync engine —
// refuse anything that resolves to a private/loopback/link-local address
// BEFORE it is persisted (a stored bad source would re-fire on every sync).
try {
await assertUrlAllowed(body.url);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
return reply.code(400).send({ error: `Source URL refused: ${msg}` });
}
// Check for duplicate name
const existing = db.getSourceByName(body.name);
if (existing) {
return reply.code(409).send({ error: `Source "${body.name}" already exists` });
}
// Auto-detect source_type from URL
const sourceType = body.url.includes('github.com') ? 'community_repo' : 'aggregator';
// Ensure the is_custom column exists (safe migration)
db.ensureIsCustomColumn();
// Insert the source
const sourceId = db.addSource({
name: body.name,
display_name: body.displayName || body.name.replace(/[-_]/g, ' ').replace(/\b\w/g, c => c.toUpperCase()),
url: body.url,
source_type: sourceType,
platform: 'waggle',
install_method: sourceType === 'community_repo' ? 'git_clone' : 'api_fetch',
});
const source = db.getSource(sourceId);
// Trigger immediate sync for the new source
const vaultLookup = fastify.vault ? (key: string) => fastify.vault!.get(key)?.value ?? null : undefined;
const sync = new MarketplaceSync(db, vaultLookup, guardedFetch);
let syncResult = null;
try {
const results = await sync.syncAll({ sources: [body.name] });
syncResult = results[0] ?? null;
// Re-categorize any new packages
if (syncResult && syncResult.added > 0) {
recategorizeAll(db);
}
} catch (err) {
syncResult = {
source: body.name,
added: 0,
updated: 0,
removed: 0,
errors: [(err as Error).message],
};
}
return reply.code(201).send({
source,
syncResult,
});
});
// ── DELETE /api/marketplace/sources/:id ─────────────────────────────
// Remove a user-added source (cannot delete built-in sources).
fastify.delete('/api/marketplace/sources/:id', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const { id } = request.params as { id: string };
const sourceId = parseInt(id, 10);
if (isNaN(sourceId)) {
return reply.code(400).send({ error: 'Invalid source ID' });
}
const source = db.getSource(sourceId);
if (!source) {
return reply.code(404).send({ error: `Source ID ${sourceId} not found` });
}
if (!source.is_custom) {
return reply.code(403).send({
error: 'Cannot delete built-in source',
hint: 'Only user-added sources can be removed',
});
}
const deleted = db.deleteSource(sourceId);
if (!deleted) {
return reply.code(500).send({ error: 'Failed to delete source' });
}
return { deleted: true, sourceId, name: source.name };
});
// ── GET /api/marketplace/categories ─────────────────────────────────
// Return the category taxonomy for UI rendering.
fastify.get('/api/marketplace/categories', async (_request, reply) => {
return { categories: PACKAGE_CATEGORIES, total: PACKAGE_CATEGORIES.length };
});
// ── POST /api/marketplace/sync ──────────────────────────────────────
// Trigger a manual sync from all configured marketplace sources.
// In dev/local mode, external sources won't be reachable — errors are
// captured per-source and returned gracefully.
fastify.post('/api/marketplace/sync', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const body = (request.body ?? {}) as { sources?: string[] };
if (isMarketplaceBackgroundSyncDisabled(process.env)) {
return {
skipped: true,
reason: 'Marketplace sync is disabled by environment.',
sourcesChecked: 0,
packagesAdded: 0,
packagesUpdated: 0,
errors: [],
details: [],
};
}
const vaultLookup = fastify.vault ? (key: string) => fastify.vault!.get(key)?.value ?? null : undefined;
const sync = new MarketplaceSync(db, vaultLookup, guardedFetch);
try {
const results = await sync.syncAll(body.sources ? { sources: body.sources } : {});
const sourcesChecked = results.length;
const packagesAdded = results.reduce((sum, r) => sum + r.added, 0);
const packagesUpdated = results.reduce((sum, r) => sum + r.updated, 0);
const errors = results.flatMap(r => r.errors.map(e => `[${r.source}] ${e}`));
// Re-categorize packages after sync
if (packagesAdded > 0 || packagesUpdated > 0) {
try { recategorizeAll(db); } catch { /* non-blocking */ }
}
// Emit notification if new packages were discovered
if (packagesAdded > 0) {
emitNotification(fastify, {
title: 'Marketplace sync complete',
body: `${packagesAdded} new capability${packagesAdded === 1 ? '' : 's'} discovered`,
category: 'agent',
actionUrl: '/skills',
});
}
return {
sourcesChecked,
packagesAdded,
packagesUpdated,
errors,
details: results,
};
} catch (err) {
return reply.code(500).send({
error: 'Sync failed',
message: (err as Error).message,
sourcesChecked: 0,
packagesAdded: 0,
packagesUpdated: 0,
errors: [(err as Error).message],
});
}
});
// ── GET /api/marketplace/security-status ──────────────────────────
// Returns overall security scanning status: Cisco scanner availability,
// JS gate version, and aggregate scan counts across the catalog.
// Used by the Capabilities view to show an install hint when the
// Cisco scanner is not installed.
fastify.get('/api/marketplace/security-status', async (request, reply) => {
const db = getDb();
let ciscoScannerAvailable = false;
try {
ciscoScannerAvailable = await isCiscoScannerAvailable();
} catch {
// Check failed — report as unavailable
}
let totalScanned = 0;
let totalPassed = 0;
let totalFailed = 0;
if (db) {
try {
const rawDb = db.getRawDb();
if (rawDb?.prepare) {
const row = rawDb.prepare(`
SELECT
COUNT(CASE WHEN security_status != 'unscanned' AND security_status IS NOT NULL THEN 1 END) as scanned,
COUNT(CASE WHEN security_status IN ('clean', 'low', 'medium') THEN 1 END) as passed,
COUNT(CASE WHEN security_status IN ('critical', 'high') THEN 1 END) as failed
FROM packages
`).get() as { scanned: number; passed: number; failed: number } | undefined;
if (row) {
totalScanned = row.scanned;
totalPassed = row.passed;
totalFailed = row.failed;
}
}
} catch { /* non-blocking */ }
}
return {
ciscoScannerAvailable,
jsSecurityGateVersion: '1.0',
totalScanned,
totalPassed,
totalFailed,
hint: ciscoScannerAvailable
? undefined
: 'For enhanced security scanning, install the Cisco skill-scanner: pip install cisco-ai-skill-scanner',
};
});
// ── POST /api/marketplace/publish ─────────────────────────────────
// Publish a local skill to the marketplace catalog.
// Reads the skill from ~/.waggle/skills/, validates frontmatter,
// runs SecurityGate scan, then upserts into the marketplace DB.
fastify.post('/api/marketplace/publish', async (request, reply) => {
const db = requireDb(reply);
if (!db) return;
const body = request.body as { skillName?: string };
if (!body.skillName) {
return reply.code(400).send({ error: 'skillName is required' });
}
const skillName = body.skillName;
// Prevent path traversal
if (skillName.includes('..') || skillName.includes('/') || skillName.includes('\\')) {
return reply.code(400).send({ error: 'Invalid skill name' });
}
// Read the skill file from ~/.waggle/skills/
const waggleHome = fastify.localConfig?.dataDir || join(homedir(), '.waggle');
const skillPath = join(waggleHome, 'skills', `${skillName}.md`);
if (!existsSync(skillPath)) {
return reply.code(404).send({
error: `Skill "${skillName}" not found`,
hint: `Expected file at ${skillPath}`,
});
}
const content = readFileSync(skillPath, 'utf-8');
// Validate YAML frontmatter
const validation = validateSkillMd(content);
if (!validation.valid) {
return reply.code(422).send({
error: 'Skill validation failed',
details: validation.errors,
hint: 'Skill file must have YAML frontmatter with name and description fields wrapped in --- delimiters',
});
}
const metadata = validation.metadata!;
// Run SecurityGate scan
const gate = new SecurityGate({
enable_gen_trust_hub: false,
enable_cisco_scanner: false,
enable_mcp_guardian: false,
enable_heuristics: true,
});
const fakePkg = {
id: 0,
name: skillName,
display_name: metadata.name,
description: metadata.description,
waggle_install_type: 'skill',
} as MarketplacePackage;
const scanResult = await gate.scan(fakePkg, content);
if (scanResult.blocked) {
return reply.code(403).send({
error: 'Skill blocked by security scan',
severity: scanResult.overall_severity,
score: scanResult.security_score,
findings: scanResult.findings,
});
}
// Ensure a "user-published" source exists
let source = db.getSourceByName('user-published');
if (!source) {
db.addSource({
name: 'user-published',
display_name: 'User Published',
url: 'local://user-published',
source_type: 'community_repo',
platform: 'waggle',
install_method: 'manual',
description: 'Skills published by the local user',
});
source = db.getSourceByName('user-published')!;
}
// Upsert into marketplace DB
const packageId = db.upsertPackage({
name: skillName,
source_id: source.id,
display_name: metadata.name,
description: metadata.description,
author: metadata.author || 'local-user',
package_type: 'skill',
waggle_install_type: 'skill',
waggle_install_path: skillPath,
version: metadata.version || '1.0.0',
category: 'custom',
downloads: 0,
stars: 0,
rating: 0,
rating_count: 0,
platforms: JSON.stringify(['all']),
dependencies: JSON.stringify([]),
packs: JSON.stringify([]),
install_manifest: JSON.stringify({
skill_content: content,
}),
});
return reply.code(201).send({
success: true,
packageId,
skillName,
metadata: {
name: metadata.name,
description: metadata.description,
version: metadata.version,
author: metadata.author,
},
security: {
severity: scanResult.overall_severity,
score: scanResult.security_score,
findingsCount: scanResult.findings.length,
},
});
});
}