/** * Integration tests for the §D2 skill-audit routes (POST/GET /api/skills/audit). * * Gate order under test: vault key → @ax-llm/ax availability → run. The audit is * free (Solo) — PRO removed, so there is no tier gate; it still requires the * user's own vault Anthropic key (zero Waggle proxy cost). * The run path is stubbed via the `__waggleSkillAuditLlmFactory` global hook so * the loop's synth/run/judge calls never touch the network — mirroring the * evolution-run-route test's factory-override pattern. */ import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'vitest'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { MindDB, VaultStore } from '@waggle/core'; import { buildLocalServer } from '../src/local/index.js'; import { injectWithAuth } from './test-utils.js'; import type { FastifyInstance } from 'fastify'; import type { EvolutionLLM } from '@waggle/agent'; import type { Tier } from '@waggle/shared'; type LLMFactory = (apiKey: string) => EvolutionLLM | Promise; function installLLMFactory(f: LLMFactory): void { (globalThis as unknown as { __waggleSkillAuditLlmFactory?: LLMFactory }).__waggleSkillAuditLlmFactory = f; } function clearLLMFactory(): void { delete (globalThis as unknown as { __waggleSkillAuditLlmFactory?: LLMFactory }).__waggleSkillAuditLlmFactory; } const PASS_JUDGE = '{"correctness":9,"procedure":9,"conciseness":8,"feedback":"good"}'; const FAIL_JUDGE = '{"correctness":2,"procedure":2,"conciseness":2,"feedback":"wrong format"}'; /** EvolutionLLM stub: dispatches scripted replies by the unique rubric marker in each prompt. */ function stubLLM(judge: string): EvolutionLLM { return { async complete(prompt: string): Promise { if (prompt.includes('designing ONE concrete test')) return '{"task":"do the thing","expected":"thing done"}'; if (prompt.includes('strict, fair evaluator')) return judge; if (prompt.includes('repairing an AI agent SKILL document')) return '# Repaired\n\nFollow these specific steps: read the input, apply the rule, return the exact result.'; return 'a concrete answer'; }, } as EvolutionLLM; } function writeTier(dataDir: string, tier: Tier, trialStartedAt: string | null = null): void { const configPath = path.join(dataDir, 'config.json'); let existing: Record = {}; try { existing = JSON.parse(fs.readFileSync(configPath, 'utf-8')); } catch { /* fresh */ } fs.writeFileSync(configPath, JSON.stringify({ ...existing, tier, trialStartedAt })); } describe('skill-audit routes (§D2)', () => { let server: FastifyInstance; let tmpDir: string; const KEY = 'sk-ant-stub-for-tests'; beforeAll(async () => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-skill-audit-')); new MindDB(path.join(tmpDir, 'personal.mind')).close(); new VaultStore(tmpDir).set('anthropic', KEY, { models: ['claude-haiku-4-5-20251001'] }); // One installed skill to audit. const skillsDir = path.join(tmpDir, 'skills'); fs.mkdirSync(skillsDir, { recursive: true }); fs.writeFileSync(path.join(skillsDir, 'verify-me.md'), '# Verify Me\n\nDo a specific deterministic thing with the input and return the exact result.', 'utf-8'); server = await buildLocalServer({ dataDir: tmpDir }); }); afterAll(async () => { await server.close(); clearLLMFactory(); try { fs.rmSync(tmpDir, { recursive: true, force: true }); } catch { /* ignore */ } }); beforeEach(() => { clearLLMFactory(); if (!server.vault?.get('anthropic')) server.vault?.set('anthropic', KEY, { models: ['claude-haiku-4-5-20251001'] }); writeTier(tmpDir, 'FREE'); try { fs.rmSync(path.join(tmpDir, 'skill-audit.json'), { force: true }); } catch { /* ignore */ } }); const post = (payload?: unknown) => injectWithAuth(server, { method: 'POST', url: '/api/skills/audit', payload: payload ?? {} }); it('C1: FREE (Solo) is allowed past the (removed) tier gate — stops at the key check → 422', async () => { // PRO removed: skill-audit is free/Solo. A FREE tier with no key must NOT // 403 — it reaches the vault-key gate and 422s like any other tier. writeTier(tmpDir, 'FREE'); server.vault?.delete('anthropic'); const res = await post(); expect(res.statusCode).toBe(422); expect(JSON.parse(res.body).error).toMatch(/Anthropic API key/i); }); it('C4: Solo with no Anthropic key → 422', async () => { server.vault?.delete('anthropic'); const res = await post(); expect(res.statusCode).toBe(422); expect(JSON.parse(res.body).error).toMatch(/Anthropic API key/i); }); it('C5: Solo + key but the LLM factory yields nothing → 503', async () => { installLLMFactory(() => null as unknown as EvolutionLLM); // simulate @ax-llm/ax unavailable const res = await post(); expect(res.statusCode).toBe(503); expect(JSON.parse(res.body).error).toMatch(/ax-llm|not available/i); }); it('C6: Solo + key + stub factory + dryRun → report, writes nothing', async () => { installLLMFactory(() => stubLLM(FAIL_JUDGE)); const res = await post({ dryRun: true }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.ok).toBe(true); expect(body.report.dryRun).toBe(true); expect(fs.existsSync(path.join(tmpDir, 'skill-audit.json'))).toBe(false); }); it('C7: Solo + key + passing stub → verified badge persisted + GET surfaces it', async () => { installLLMFactory(() => stubLLM(PASS_JUDGE)); const res = await post({}); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.report.verified).toContain('verify-me'); const get = await injectWithAuth(server, { method: 'GET', url: '/api/skills/audit' }); expect(get.statusCode).toBe(200); const badges = JSON.parse(get.body).audit; expect(badges['verify-me']).toMatchObject({ verified: true }); expect(badges['verify-me'].confidence).toBeGreaterThanOrEqual(0.7); }); it('C8: GET /api/skills surfaces the verified badge for the audited skill', async () => { installLLMFactory(() => stubLLM(PASS_JUDGE)); await post({}); const get = await injectWithAuth(server, { method: 'GET', url: '/api/skills' }); const row = JSON.parse(get.body).skills.find((s: { name: string }) => s.name === 'verify-me'); expect(row.verified).toBe(true); expect(row.confidence).toBeGreaterThanOrEqual(0.7); }); it('C9: editing a skill clears its verified badge (staleness on read)', async () => { installLLMFactory(() => stubLLM(PASS_JUDGE)); await post({}); // Edit the skill via the standard PUT — must invalidate the badge. const put = await injectWithAuth(server, { method: 'PUT', url: '/api/skills/verify-me', payload: { content: '# Verify Me\n\nA materially different body that changes the content hash entirely.' }, }); expect(put.statusCode).toBe(200); const get = await injectWithAuth(server, { method: 'GET', url: '/api/skills' }); const row = JSON.parse(get.body).skills.find((s: { name: string }) => s.name === 'verify-me'); expect(row.verified).toBe(false); // badge cleared + hash no longer matches }); });