409 lines
16 KiB
TypeScript
409 lines
16 KiB
TypeScript
/**
|
|
* Provider API Tests — GET /api/providers
|
|
*
|
|
* Tests the single source of truth endpoint for LLM providers,
|
|
* models, and search tools with vault key status.
|
|
*/
|
|
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
|
|
import fs from 'node:fs';
|
|
import path from 'node:path';
|
|
import os from 'node:os';
|
|
import { MindDB, SessionStore, FrameStore } from '@waggle/core';
|
|
import { buildLocalServer } from '../../src/local/index.js';
|
|
import type { FastifyInstance } from 'fastify';
|
|
import { injectWithAuth } from '../test-utils.js';
|
|
import { PROVIDER_ENV_NAMES } from '../../src/local/provider-env.js';
|
|
|
|
/** Shape of a model entry in the GET /api/providers response (test-asserted fields). */
|
|
interface ProviderModelResponse {
|
|
id: string;
|
|
name: string;
|
|
cost: string;
|
|
speed: string;
|
|
source?: string;
|
|
}
|
|
|
|
/** Shape of a provider entry in the GET /api/providers response (test-asserted fields). */
|
|
interface ProviderResponse {
|
|
id: string;
|
|
name: string;
|
|
hasKey: boolean;
|
|
requiresKey: boolean;
|
|
badge: string | null;
|
|
models: ProviderModelResponse[];
|
|
modelsSource?: string;
|
|
}
|
|
|
|
function mockProviderCatalogFetch() {
|
|
const realFetch = globalThis.fetch;
|
|
return vi.spyOn(globalThis, 'fetch').mockImplementation((input, init) => {
|
|
const url = String(input);
|
|
if (url.includes('/api/tags')) return realFetch(input, init);
|
|
if (url.includes('/models')) {
|
|
return Promise.resolve(new Response(JSON.stringify({
|
|
data: [{ id: 'provider-model-added-at-runtime', name: 'Provider Model Added At Runtime' }],
|
|
}), { status: 200, headers: { 'content-type': 'application/json' } }));
|
|
}
|
|
return realFetch(input, init);
|
|
});
|
|
}
|
|
|
|
/** Shape of a search-provider entry in the GET /api/providers response (test-asserted fields). */
|
|
interface SearchProviderResponse {
|
|
id: string;
|
|
hasKey: boolean;
|
|
requiresKey: boolean;
|
|
priority: number;
|
|
}
|
|
|
|
describe('Provider API', () => {
|
|
let server: FastifyInstance;
|
|
let tmpDir: string;
|
|
let prevOllamaHost: string | undefined;
|
|
const originalProviderEnv = new Map<string, string | undefined>();
|
|
|
|
beforeAll(async () => {
|
|
for (const envName of new Set(Object.values(PROVIDER_ENV_NAMES).flat())) {
|
|
originalProviderEnv.set(envName, process.env[envName]);
|
|
delete process.env[envName];
|
|
}
|
|
// Pin Ollama to a dead port so reachability is deterministic everywhere:
|
|
// Windows dev boxes often run a local daemon (:11434 → reachable), CI does
|
|
// not. The route reports hasKey = live reachability for ollama.
|
|
prevOllamaHost = process.env.OLLAMA_HOST;
|
|
process.env.OLLAMA_HOST = 'http://127.0.0.1:1';
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-providers-'));
|
|
const personalPath = path.join(tmpDir, 'personal.mind');
|
|
const mind = new MindDB(personalPath);
|
|
const sessions = new SessionStore(mind);
|
|
const frames = new FrameStore(mind);
|
|
const s1 = sessions.create('providers-test');
|
|
frames.createIFrame(s1.gop_id, 'Provider test', 'normal');
|
|
mind.close();
|
|
server = await buildLocalServer({ dataDir: tmpDir });
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await server.close();
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
if (prevOllamaHost === undefined) delete process.env.OLLAMA_HOST;
|
|
else process.env.OLLAMA_HOST = prevOllamaHost;
|
|
for (const [envName, value] of originalProviderEnv) {
|
|
if (value === undefined) delete process.env[envName];
|
|
else process.env[envName] = value;
|
|
}
|
|
});
|
|
|
|
describe('GET /api/providers', () => {
|
|
it('returns providers array', async () => {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
expect(res.statusCode).toBe(200);
|
|
const body = res.json();
|
|
expect(body.providers).toBeDefined();
|
|
expect(Array.isArray(body.providers)).toBe(true);
|
|
expect(body.providers.length).toBeGreaterThanOrEqual(10);
|
|
});
|
|
|
|
it('each provider has required fields', async () => {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
|
|
for (const p of providers) {
|
|
expect(p.id).toBeDefined();
|
|
expect(p.name).toBeDefined();
|
|
expect(typeof p.hasKey).toBe('boolean');
|
|
expect(typeof p.requiresKey).toBe('boolean');
|
|
expect(Array.isArray(p.models)).toBe(true);
|
|
}
|
|
});
|
|
|
|
it('includes all expected providers', async () => {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
const ids = providers.map((p: ProviderResponse) => p.id);
|
|
|
|
expect(ids).toContain('anthropic');
|
|
expect(ids).toContain('openai');
|
|
expect(ids).toContain('google');
|
|
expect(ids).toContain('deepseek');
|
|
expect(ids).toContain('xai');
|
|
expect(ids).toContain('mistral');
|
|
expect(ids).toContain('alibaba');
|
|
expect(ids).toContain('minimax');
|
|
expect(ids).toContain('zhipu');
|
|
expect(ids).toContain('moonshot');
|
|
expect(ids).toContain('perplexity');
|
|
expect(ids).toContain('openrouter');
|
|
expect(ids).toContain('ollama');
|
|
});
|
|
|
|
it('ollama does not require a key', async () => {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
const ollama = providers.find((p: ProviderResponse) => p.id === 'ollama');
|
|
expect(ollama.requiresKey).toBe(false);
|
|
// hasKey mirrors live daemon reachability for ollama; pinned to a dead
|
|
// port in beforeAll → deterministically false on every platform/CI.
|
|
expect(ollama.hasKey).toBe(false);
|
|
});
|
|
|
|
it('providers without vault keys show hasKey=false', async () => {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
// Fresh vault — no keys configured
|
|
const openai = providers.find((p: ProviderResponse) => p.id === 'openai');
|
|
expect(openai.hasKey).toBe(false);
|
|
});
|
|
|
|
it('providers with vault keys show hasKey=true', async () => {
|
|
// Add a key to vault
|
|
server.vault!.set('anthropic', 'sk-ant-test-key');
|
|
const fetchSpy = mockProviderCatalogFetch();
|
|
|
|
try {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
const anthropic = providers.find((p: ProviderResponse) => p.id === 'anthropic');
|
|
expect(anthropic.hasKey).toBe(true);
|
|
} finally {
|
|
fetchSpy.mockRestore();
|
|
server.vault!.delete('anthropic');
|
|
}
|
|
});
|
|
|
|
it('environment-configured providers expose the same live catalog as Vault keys', async () => {
|
|
process.env.OPENAI_API_KEY = 'openai-env-catalog-key';
|
|
const fetchSpy = mockProviderCatalogFetch();
|
|
|
|
try {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
const openai = providers.find((provider: ProviderResponse) => provider.id === 'openai');
|
|
|
|
expect(openai.hasKey).toBe(true);
|
|
expect(openai.modelsSource).toBe('provider-api');
|
|
expect(openai.models.map((model) => model.id)).toContain('openai/provider-model-added-at-runtime');
|
|
} finally {
|
|
fetchSpy.mockRestore();
|
|
delete process.env.OPENAI_API_KEY;
|
|
}
|
|
});
|
|
|
|
it('returns live provider models with id, name, cost, and speed metadata', async () => {
|
|
server.vault!.set('anthropic', 'sk-ant-catalog-test-key');
|
|
const fetchSpy = mockProviderCatalogFetch();
|
|
|
|
try {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
const anthropic = providers.find((p: ProviderResponse) => p.id === 'anthropic');
|
|
|
|
expect(anthropic.modelsSource).toBe('provider-api');
|
|
expect(anthropic.models.length).toBeGreaterThan(0);
|
|
expect(anthropic.models.map((model) => model.id)).toContain('anthropic/provider-model-added-at-runtime');
|
|
for (const m of anthropic.models) {
|
|
expect(m.id).toBeDefined();
|
|
expect(m.name).toBeDefined();
|
|
expect(['$', '$$', '$$$']).toContain(m.cost);
|
|
expect(['fast', 'medium', 'slow']).toContain(m.speed);
|
|
}
|
|
} finally {
|
|
fetchSpy.mockRestore();
|
|
server.vault!.delete('anthropic');
|
|
}
|
|
});
|
|
|
|
it('does not require a code change when an Alibaba model appears in its API catalog', async () => {
|
|
server.vault!.set('alibaba', 'alibaba-catalog-test-key');
|
|
const fetchSpy = mockProviderCatalogFetch();
|
|
|
|
try {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
const alibaba = providers.find((p: ProviderResponse) => p.id === 'alibaba');
|
|
|
|
expect(alibaba.modelsSource).toBe('provider-api');
|
|
expect(alibaba.models.map((model) => model.id)).toContain('alibaba/provider-model-added-at-runtime');
|
|
} finally {
|
|
fetchSpy.mockRestore();
|
|
server.vault!.delete('alibaba');
|
|
}
|
|
});
|
|
|
|
it('returns search providers with priority', async () => {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { search, activeSearch } = res.json();
|
|
|
|
expect(Array.isArray(search)).toBe(true);
|
|
expect(search.length).toBeGreaterThanOrEqual(4);
|
|
|
|
const ids = search.map((s: SearchProviderResponse) => s.id);
|
|
expect(ids).toContain('perplexity');
|
|
expect(ids).toContain('tavily');
|
|
expect(ids).toContain('brave');
|
|
expect(ids).toContain('duckduckgo');
|
|
|
|
// DuckDuckGo should always have hasKey=true (free)
|
|
const ddg = search.find((s: SearchProviderResponse) => s.id === 'duckduckgo');
|
|
expect(ddg.hasKey).toBe(true);
|
|
expect(ddg.requiresKey).toBe(false);
|
|
|
|
// activeSearch should be defined
|
|
expect(activeSearch).toBeDefined();
|
|
});
|
|
|
|
it('activeSearch reflects vault key status', async () => {
|
|
// No premium keys → DuckDuckGo should be active
|
|
let res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
expect(res.json().activeSearch).toBe('duckduckgo');
|
|
|
|
// Add Tavily key → Tavily should be active
|
|
server.vault!.set('TAVILY_API_KEY', 'tvly-test');
|
|
res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
expect(res.json().activeSearch).toBe('tavily');
|
|
|
|
// Add Perplexity key → Perplexity should be active (higher priority)
|
|
server.vault!.set('perplexity', 'pplx-test');
|
|
res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
expect(res.json().activeSearch).toBe('perplexity');
|
|
|
|
// Cleanup
|
|
server.vault!.delete('TAVILY_API_KEY');
|
|
server.vault!.delete('perplexity');
|
|
});
|
|
|
|
it('search priorities are in correct order', async () => {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { search } = res.json();
|
|
|
|
const sorted = [...search].sort((a: SearchProviderResponse, b: SearchProviderResponse) => a.priority - b.priority);
|
|
expect(sorted[0].id).toBe('perplexity');
|
|
expect(sorted[1].id).toBe('tavily');
|
|
expect(sorted[2].id).toBe('brave');
|
|
expect(sorted[3].id).toBe('duckduckgo');
|
|
});
|
|
|
|
it('perplexity has badge "Search + LLM"', async () => {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
const perplexity = providers.find((p: ProviderResponse) => p.id === 'perplexity');
|
|
expect(perplexity.badge).toBe('Search + LLM');
|
|
});
|
|
|
|
it('openrouter identifies its live provider catalog', async () => {
|
|
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
|
const { providers } = res.json();
|
|
const openrouter = providers.find((p: ProviderResponse) => p.id === 'openrouter');
|
|
expect(openrouter.badge).toBe('Provider catalog');
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('Perplexity Search Tool', () => {
|
|
it('perplexity_search tool exists in createSearchTools output', async () => {
|
|
const { createSearchTools } = await import('../../src/../../../packages/agent/src/search-tools.js');
|
|
const tools = createSearchTools(async () => null);
|
|
const names = tools.map((t) => t.name);
|
|
expect(names).toContain('perplexity_search');
|
|
expect(names).toContain('tavily_search');
|
|
expect(names).toContain('brave_search');
|
|
});
|
|
|
|
it('perplexity_search returns "not configured" when no key', async () => {
|
|
const { createSearchTools } = await import('../../src/../../../packages/agent/src/search-tools.js');
|
|
const tools = createSearchTools(async () => null);
|
|
const perplexity = tools.find((t) => t.name === 'perplexity_search');
|
|
expect(perplexity).toBeDefined();
|
|
const result = await perplexity!.execute({ query: 'test' });
|
|
expect(result).toContain('not configured');
|
|
});
|
|
});
|
|
|
|
describe('Legacy provider key migration', () => {
|
|
it('moves legacy plaintext keys into Vault and scrubs config.json', async () => {
|
|
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-key-migration-'));
|
|
const key = 'sk-ant-legacy-key-1234567890';
|
|
fs.writeFileSync(
|
|
path.join(dataDir, 'config.json'),
|
|
JSON.stringify({ defaultModel: 'test/model', providers: { anthropic: { apiKey: key, models: ['claude-sonnet-4-6'] } } }),
|
|
'utf-8',
|
|
);
|
|
|
|
const migratedServer = await buildLocalServer({ dataDir, port: 0 });
|
|
try {
|
|
const config = JSON.parse(fs.readFileSync(path.join(dataDir, 'config.json'), 'utf-8')) as {
|
|
providers?: Record<string, { apiKey?: string; models?: string[] }>;
|
|
};
|
|
expect(config.providers?.anthropic).toMatchObject({ apiKey: '', models: ['claude-sonnet-4-6'] });
|
|
expect(migratedServer.vault?.get('anthropic')?.value).toBe(key);
|
|
} finally {
|
|
await migratedServer.close();
|
|
fs.rmSync(dataDir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|
|
|
|
describe('Model Validation', () => {
|
|
let server: FastifyInstance;
|
|
let tmpDir: string;
|
|
|
|
beforeAll(async () => {
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-model-val-'));
|
|
const personalPath = path.join(tmpDir, 'personal.mind');
|
|
const mind = new MindDB(personalPath);
|
|
const sessions = new SessionStore(mind);
|
|
const frames = new FrameStore(mind);
|
|
const s1 = sessions.create('model-val-test');
|
|
frames.createIFrame(s1.gop_id, 'Model validation test', 'normal');
|
|
mind.close();
|
|
// Set TRIAL tier so we're not capped at the FREE limit (5 workspaces).
|
|
// Without this, ensureDefault() + 4 test workspaces = 5, making the next
|
|
// POST hit the tier limit (403) before reaching model validation (400).
|
|
fs.writeFileSync(path.join(tmpDir, 'config.json'), JSON.stringify({ tier: 'TRIAL' }));
|
|
server = await buildLocalServer({ dataDir: tmpDir });
|
|
});
|
|
|
|
afterAll(async () => {
|
|
await server.close();
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it('accepts any valid model name when creating workspace', async () => {
|
|
// Standard model
|
|
let res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/workspaces',
|
|
payload: { name: 'Test WS 1', group: 'Test', model: 'claude-sonnet-4-6' },
|
|
});
|
|
expect([200, 201]).toContain(res.statusCode);
|
|
|
|
// Provider-prefixed model
|
|
res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/workspaces',
|
|
payload: { name: 'Test WS 2', group: 'Test', model: 'anthropic/claude-sonnet-4.6' },
|
|
});
|
|
expect([200, 201]).toContain(res.statusCode);
|
|
|
|
// Newer model not in old hardcoded list
|
|
res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/workspaces',
|
|
payload: { name: 'Test WS 3', group: 'Test', model: 'qwen-max' },
|
|
});
|
|
expect([200, 201]).toContain(res.statusCode);
|
|
|
|
// Custom model
|
|
res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/workspaces',
|
|
payload: { name: 'Test WS 4', group: 'Test', model: 'my-custom-ollama-model' },
|
|
});
|
|
expect([200, 201]).toContain(res.statusCode);
|
|
});
|
|
|
|
it('rejects invalid model names', async () => {
|
|
const res = await injectWithAuth(server, {
|
|
method: 'POST', url: '/api/workspaces',
|
|
payload: { name: 'Test WS Bad', group: 'Test', model: 'x' }, // too short
|
|
});
|
|
expect(res.statusCode).toBe(400);
|
|
});
|
|
});
|