This commit is contained in:
Oleg Maslov
408
packages/server/tests/local/providers.test.ts
Normal file
408
packages/server/tests/local/providers.test.ts
Normal file
@@ -0,0 +1,408 @@
|
||||
/**
|
||||
* Provider API Tests — GET /api/providers
|
||||
*
|
||||
* Tests the single source of truth endpoint for LLM providers,
|
||||
* models, and search tools with vault key status.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import os from 'node:os';
|
||||
import { MindDB, SessionStore, FrameStore } from '@waggle/core';
|
||||
import { buildLocalServer } from '../../src/local/index.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import { injectWithAuth } from '../test-utils.js';
|
||||
import { PROVIDER_ENV_NAMES } from '../../src/local/provider-env.js';
|
||||
|
||||
/** Shape of a model entry in the GET /api/providers response (test-asserted fields). */
|
||||
interface ProviderModelResponse {
|
||||
id: string;
|
||||
name: string;
|
||||
cost: string;
|
||||
speed: string;
|
||||
source?: string;
|
||||
}
|
||||
|
||||
/** Shape of a provider entry in the GET /api/providers response (test-asserted fields). */
|
||||
interface ProviderResponse {
|
||||
id: string;
|
||||
name: string;
|
||||
hasKey: boolean;
|
||||
requiresKey: boolean;
|
||||
badge: string | null;
|
||||
models: ProviderModelResponse[];
|
||||
modelsSource?: string;
|
||||
}
|
||||
|
||||
function mockProviderCatalogFetch() {
|
||||
const realFetch = globalThis.fetch;
|
||||
return vi.spyOn(globalThis, 'fetch').mockImplementation((input, init) => {
|
||||
const url = String(input);
|
||||
if (url.includes('/api/tags')) return realFetch(input, init);
|
||||
if (url.includes('/models')) {
|
||||
return Promise.resolve(new Response(JSON.stringify({
|
||||
data: [{ id: 'provider-model-added-at-runtime', name: 'Provider Model Added At Runtime' }],
|
||||
}), { status: 200, headers: { 'content-type': 'application/json' } }));
|
||||
}
|
||||
return realFetch(input, init);
|
||||
});
|
||||
}
|
||||
|
||||
/** Shape of a search-provider entry in the GET /api/providers response (test-asserted fields). */
|
||||
interface SearchProviderResponse {
|
||||
id: string;
|
||||
hasKey: boolean;
|
||||
requiresKey: boolean;
|
||||
priority: number;
|
||||
}
|
||||
|
||||
describe('Provider API', () => {
|
||||
let server: FastifyInstance;
|
||||
let tmpDir: string;
|
||||
let prevOllamaHost: string | undefined;
|
||||
const originalProviderEnv = new Map<string, string | undefined>();
|
||||
|
||||
beforeAll(async () => {
|
||||
for (const envName of new Set(Object.values(PROVIDER_ENV_NAMES).flat())) {
|
||||
originalProviderEnv.set(envName, process.env[envName]);
|
||||
delete process.env[envName];
|
||||
}
|
||||
// Pin Ollama to a dead port so reachability is deterministic everywhere:
|
||||
// Windows dev boxes often run a local daemon (:11434 → reachable), CI does
|
||||
// not. The route reports hasKey = live reachability for ollama.
|
||||
prevOllamaHost = process.env.OLLAMA_HOST;
|
||||
process.env.OLLAMA_HOST = 'http://127.0.0.1:1';
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-providers-'));
|
||||
const personalPath = path.join(tmpDir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
const sessions = new SessionStore(mind);
|
||||
const frames = new FrameStore(mind);
|
||||
const s1 = sessions.create('providers-test');
|
||||
frames.createIFrame(s1.gop_id, 'Provider test', 'normal');
|
||||
mind.close();
|
||||
server = await buildLocalServer({ dataDir: tmpDir });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
if (prevOllamaHost === undefined) delete process.env.OLLAMA_HOST;
|
||||
else process.env.OLLAMA_HOST = prevOllamaHost;
|
||||
for (const [envName, value] of originalProviderEnv) {
|
||||
if (value === undefined) delete process.env[envName];
|
||||
else process.env[envName] = value;
|
||||
}
|
||||
});
|
||||
|
||||
describe('GET /api/providers', () => {
|
||||
it('returns providers array', async () => {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
expect(res.statusCode).toBe(200);
|
||||
const body = res.json();
|
||||
expect(body.providers).toBeDefined();
|
||||
expect(Array.isArray(body.providers)).toBe(true);
|
||||
expect(body.providers.length).toBeGreaterThanOrEqual(10);
|
||||
});
|
||||
|
||||
it('each provider has required fields', async () => {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
|
||||
for (const p of providers) {
|
||||
expect(p.id).toBeDefined();
|
||||
expect(p.name).toBeDefined();
|
||||
expect(typeof p.hasKey).toBe('boolean');
|
||||
expect(typeof p.requiresKey).toBe('boolean');
|
||||
expect(Array.isArray(p.models)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it('includes all expected providers', async () => {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
const ids = providers.map((p: ProviderResponse) => p.id);
|
||||
|
||||
expect(ids).toContain('anthropic');
|
||||
expect(ids).toContain('openai');
|
||||
expect(ids).toContain('google');
|
||||
expect(ids).toContain('deepseek');
|
||||
expect(ids).toContain('xai');
|
||||
expect(ids).toContain('mistral');
|
||||
expect(ids).toContain('alibaba');
|
||||
expect(ids).toContain('minimax');
|
||||
expect(ids).toContain('zhipu');
|
||||
expect(ids).toContain('moonshot');
|
||||
expect(ids).toContain('perplexity');
|
||||
expect(ids).toContain('openrouter');
|
||||
expect(ids).toContain('ollama');
|
||||
});
|
||||
|
||||
it('ollama does not require a key', async () => {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
const ollama = providers.find((p: ProviderResponse) => p.id === 'ollama');
|
||||
expect(ollama.requiresKey).toBe(false);
|
||||
// hasKey mirrors live daemon reachability for ollama; pinned to a dead
|
||||
// port in beforeAll → deterministically false on every platform/CI.
|
||||
expect(ollama.hasKey).toBe(false);
|
||||
});
|
||||
|
||||
it('providers without vault keys show hasKey=false', async () => {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
// Fresh vault — no keys configured
|
||||
const openai = providers.find((p: ProviderResponse) => p.id === 'openai');
|
||||
expect(openai.hasKey).toBe(false);
|
||||
});
|
||||
|
||||
it('providers with vault keys show hasKey=true', async () => {
|
||||
// Add a key to vault
|
||||
server.vault!.set('anthropic', 'sk-ant-test-key');
|
||||
const fetchSpy = mockProviderCatalogFetch();
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
const anthropic = providers.find((p: ProviderResponse) => p.id === 'anthropic');
|
||||
expect(anthropic.hasKey).toBe(true);
|
||||
} finally {
|
||||
fetchSpy.mockRestore();
|
||||
server.vault!.delete('anthropic');
|
||||
}
|
||||
});
|
||||
|
||||
it('environment-configured providers expose the same live catalog as Vault keys', async () => {
|
||||
process.env.OPENAI_API_KEY = 'openai-env-catalog-key';
|
||||
const fetchSpy = mockProviderCatalogFetch();
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
const openai = providers.find((provider: ProviderResponse) => provider.id === 'openai');
|
||||
|
||||
expect(openai.hasKey).toBe(true);
|
||||
expect(openai.modelsSource).toBe('provider-api');
|
||||
expect(openai.models.map((model) => model.id)).toContain('openai/provider-model-added-at-runtime');
|
||||
} finally {
|
||||
fetchSpy.mockRestore();
|
||||
delete process.env.OPENAI_API_KEY;
|
||||
}
|
||||
});
|
||||
|
||||
it('returns live provider models with id, name, cost, and speed metadata', async () => {
|
||||
server.vault!.set('anthropic', 'sk-ant-catalog-test-key');
|
||||
const fetchSpy = mockProviderCatalogFetch();
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
const anthropic = providers.find((p: ProviderResponse) => p.id === 'anthropic');
|
||||
|
||||
expect(anthropic.modelsSource).toBe('provider-api');
|
||||
expect(anthropic.models.length).toBeGreaterThan(0);
|
||||
expect(anthropic.models.map((model) => model.id)).toContain('anthropic/provider-model-added-at-runtime');
|
||||
for (const m of anthropic.models) {
|
||||
expect(m.id).toBeDefined();
|
||||
expect(m.name).toBeDefined();
|
||||
expect(['$', '$$', '$$$']).toContain(m.cost);
|
||||
expect(['fast', 'medium', 'slow']).toContain(m.speed);
|
||||
}
|
||||
} finally {
|
||||
fetchSpy.mockRestore();
|
||||
server.vault!.delete('anthropic');
|
||||
}
|
||||
});
|
||||
|
||||
it('does not require a code change when an Alibaba model appears in its API catalog', async () => {
|
||||
server.vault!.set('alibaba', 'alibaba-catalog-test-key');
|
||||
const fetchSpy = mockProviderCatalogFetch();
|
||||
|
||||
try {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
const alibaba = providers.find((p: ProviderResponse) => p.id === 'alibaba');
|
||||
|
||||
expect(alibaba.modelsSource).toBe('provider-api');
|
||||
expect(alibaba.models.map((model) => model.id)).toContain('alibaba/provider-model-added-at-runtime');
|
||||
} finally {
|
||||
fetchSpy.mockRestore();
|
||||
server.vault!.delete('alibaba');
|
||||
}
|
||||
});
|
||||
|
||||
it('returns search providers with priority', async () => {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { search, activeSearch } = res.json();
|
||||
|
||||
expect(Array.isArray(search)).toBe(true);
|
||||
expect(search.length).toBeGreaterThanOrEqual(4);
|
||||
|
||||
const ids = search.map((s: SearchProviderResponse) => s.id);
|
||||
expect(ids).toContain('perplexity');
|
||||
expect(ids).toContain('tavily');
|
||||
expect(ids).toContain('brave');
|
||||
expect(ids).toContain('duckduckgo');
|
||||
|
||||
// DuckDuckGo should always have hasKey=true (free)
|
||||
const ddg = search.find((s: SearchProviderResponse) => s.id === 'duckduckgo');
|
||||
expect(ddg.hasKey).toBe(true);
|
||||
expect(ddg.requiresKey).toBe(false);
|
||||
|
||||
// activeSearch should be defined
|
||||
expect(activeSearch).toBeDefined();
|
||||
});
|
||||
|
||||
it('activeSearch reflects vault key status', async () => {
|
||||
// No premium keys → DuckDuckGo should be active
|
||||
let res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
expect(res.json().activeSearch).toBe('duckduckgo');
|
||||
|
||||
// Add Tavily key → Tavily should be active
|
||||
server.vault!.set('TAVILY_API_KEY', 'tvly-test');
|
||||
res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
expect(res.json().activeSearch).toBe('tavily');
|
||||
|
||||
// Add Perplexity key → Perplexity should be active (higher priority)
|
||||
server.vault!.set('perplexity', 'pplx-test');
|
||||
res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
expect(res.json().activeSearch).toBe('perplexity');
|
||||
|
||||
// Cleanup
|
||||
server.vault!.delete('TAVILY_API_KEY');
|
||||
server.vault!.delete('perplexity');
|
||||
});
|
||||
|
||||
it('search priorities are in correct order', async () => {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { search } = res.json();
|
||||
|
||||
const sorted = [...search].sort((a: SearchProviderResponse, b: SearchProviderResponse) => a.priority - b.priority);
|
||||
expect(sorted[0].id).toBe('perplexity');
|
||||
expect(sorted[1].id).toBe('tavily');
|
||||
expect(sorted[2].id).toBe('brave');
|
||||
expect(sorted[3].id).toBe('duckduckgo');
|
||||
});
|
||||
|
||||
it('perplexity has badge "Search + LLM"', async () => {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
const perplexity = providers.find((p: ProviderResponse) => p.id === 'perplexity');
|
||||
expect(perplexity.badge).toBe('Search + LLM');
|
||||
});
|
||||
|
||||
it('openrouter identifies its live provider catalog', async () => {
|
||||
const res = await injectWithAuth(server, { method: 'GET', url: '/api/providers' });
|
||||
const { providers } = res.json();
|
||||
const openrouter = providers.find((p: ProviderResponse) => p.id === 'openrouter');
|
||||
expect(openrouter.badge).toBe('Provider catalog');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('Perplexity Search Tool', () => {
|
||||
it('perplexity_search tool exists in createSearchTools output', async () => {
|
||||
const { createSearchTools } = await import('../../src/../../../packages/agent/src/search-tools.js');
|
||||
const tools = createSearchTools(async () => null);
|
||||
const names = tools.map((t) => t.name);
|
||||
expect(names).toContain('perplexity_search');
|
||||
expect(names).toContain('tavily_search');
|
||||
expect(names).toContain('brave_search');
|
||||
});
|
||||
|
||||
it('perplexity_search returns "not configured" when no key', async () => {
|
||||
const { createSearchTools } = await import('../../src/../../../packages/agent/src/search-tools.js');
|
||||
const tools = createSearchTools(async () => null);
|
||||
const perplexity = tools.find((t) => t.name === 'perplexity_search');
|
||||
expect(perplexity).toBeDefined();
|
||||
const result = await perplexity!.execute({ query: 'test' });
|
||||
expect(result).toContain('not configured');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Legacy provider key migration', () => {
|
||||
it('moves legacy plaintext keys into Vault and scrubs config.json', async () => {
|
||||
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-key-migration-'));
|
||||
const key = 'sk-ant-legacy-key-1234567890';
|
||||
fs.writeFileSync(
|
||||
path.join(dataDir, 'config.json'),
|
||||
JSON.stringify({ defaultModel: 'test/model', providers: { anthropic: { apiKey: key, models: ['claude-sonnet-4-6'] } } }),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const migratedServer = await buildLocalServer({ dataDir, port: 0 });
|
||||
try {
|
||||
const config = JSON.parse(fs.readFileSync(path.join(dataDir, 'config.json'), 'utf-8')) as {
|
||||
providers?: Record<string, { apiKey?: string; models?: string[] }>;
|
||||
};
|
||||
expect(config.providers?.anthropic).toMatchObject({ apiKey: '', models: ['claude-sonnet-4-6'] });
|
||||
expect(migratedServer.vault?.get('anthropic')?.value).toBe(key);
|
||||
} finally {
|
||||
await migratedServer.close();
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Model Validation', () => {
|
||||
let server: FastifyInstance;
|
||||
let tmpDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-model-val-'));
|
||||
const personalPath = path.join(tmpDir, 'personal.mind');
|
||||
const mind = new MindDB(personalPath);
|
||||
const sessions = new SessionStore(mind);
|
||||
const frames = new FrameStore(mind);
|
||||
const s1 = sessions.create('model-val-test');
|
||||
frames.createIFrame(s1.gop_id, 'Model validation test', 'normal');
|
||||
mind.close();
|
||||
// Set TRIAL tier so we're not capped at the FREE limit (5 workspaces).
|
||||
// Without this, ensureDefault() + 4 test workspaces = 5, making the next
|
||||
// POST hit the tier limit (403) before reaching model validation (400).
|
||||
fs.writeFileSync(path.join(tmpDir, 'config.json'), JSON.stringify({ tier: 'TRIAL' }));
|
||||
server = await buildLocalServer({ dataDir: tmpDir });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await server.close();
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('accepts any valid model name when creating workspace', async () => {
|
||||
// Standard model
|
||||
let res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/workspaces',
|
||||
payload: { name: 'Test WS 1', group: 'Test', model: 'claude-sonnet-4-6' },
|
||||
});
|
||||
expect([200, 201]).toContain(res.statusCode);
|
||||
|
||||
// Provider-prefixed model
|
||||
res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/workspaces',
|
||||
payload: { name: 'Test WS 2', group: 'Test', model: 'anthropic/claude-sonnet-4.6' },
|
||||
});
|
||||
expect([200, 201]).toContain(res.statusCode);
|
||||
|
||||
// Newer model not in old hardcoded list
|
||||
res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/workspaces',
|
||||
payload: { name: 'Test WS 3', group: 'Test', model: 'qwen-max' },
|
||||
});
|
||||
expect([200, 201]).toContain(res.statusCode);
|
||||
|
||||
// Custom model
|
||||
res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/workspaces',
|
||||
payload: { name: 'Test WS 4', group: 'Test', model: 'my-custom-ollama-model' },
|
||||
});
|
||||
expect([200, 201]).toContain(res.statusCode);
|
||||
});
|
||||
|
||||
it('rejects invalid model names', async () => {
|
||||
const res = await injectWithAuth(server, {
|
||||
method: 'POST', url: '/api/workspaces',
|
||||
payload: { name: 'Test WS Bad', group: 'Test', model: 'x' }, // too short
|
||||
});
|
||||
expect(res.statusCode).toBe(400);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user