import { describe, it, expect, beforeAll, afterAll, vi, beforeEach, afterEach } from 'vitest'; import fs from 'node:fs'; import path from 'node:path'; import os from 'node:os'; import type { FastifyInstance } from 'fastify'; // Mock the lifecycle module before importing anything that uses it vi.mock('../src/local/lifecycle.js', () => ({ getLiteLLMStatus: vi.fn(), startLiteLLM: vi.fn(), stopLiteLLM: vi.fn(), })); import { buildLocalServer } from '../src/local/index.js'; import { getLiteLLMStatus, startLiteLLM, stopLiteLLM } from '../src/local/lifecycle.js'; import { listOllamaChatModelIds, resolveExplicitRoutableModel, resolveUsableModel, } from '../src/local/model-availability.js'; import { PROVIDER_ENV_NAMES } from '../src/local/provider-env.js'; import { startService } from '../src/local/service.js'; import { injectWithAuth } from './test-utils.js'; const mockGetStatus = getLiteLLMStatus as ReturnType; const mockStart = startLiteLLM as ReturnType; const mockStop = stopLiteLLM as ReturnType; describe('LiteLLM Management API', () => { let server: FastifyInstance; let dataDir: string; const originalProviderEnv = new Map(); beforeAll(async () => { for (const envName of new Set(Object.values(PROVIDER_ENV_NAMES).flat())) { originalProviderEnv.set(envName, process.env[envName]); delete process.env[envName]; } dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-litellm-api-')); // Write minimal config.json fs.writeFileSync( path.join(dataDir, 'config.json'), JSON.stringify({ defaultModel: 'test/model', providers: {} }), 'utf-8' ); server = await buildLocalServer({ dataDir, port: 0, manageLiteLLM: true, managedLiteLLMPort: 4000, }); }); afterAll(async () => { await server.close(); fs.rmSync(dataDir, { recursive: true, force: true }); for (const [envName, value] of originalProviderEnv) { if (value === undefined) delete process.env[envName]; else process.env[envName] = value; } }); beforeEach(() => { vi.clearAllMocks(); for (const envName of originalProviderEnv.keys()) delete process.env[envName]; }); afterEach(() => { vi.restoreAllMocks(); server.vault.delete('openai'); server.vault.delete('google'); server.vault.delete('openrouter'); }); // --- GET /api/litellm/status --- it('GET /api/litellm/status returns running status', async () => { mockGetStatus.mockResolvedValue({ status: 'running', port: 4000 }); const res = await injectWithAuth(server, { method: 'GET', url: '/api/litellm/status', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.running).toBe(true); expect(body.port).toBe(4000); }); it('GET /api/litellm/status returns not running when error', async () => { mockGetStatus.mockResolvedValue({ status: 'error', port: 4000, error: 'LiteLLM is not running', }); const res = await injectWithAuth(server, { method: 'GET', url: '/api/litellm/status', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.running).toBe(false); expect(body.port).toBe(4000); }); // --- POST /api/litellm/restart --- function configureDynamicCatalog(model = 'provider-model-added-today'): void { server.vault.set('openai', 'openai-router-test-key'); vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ data: [{ id: model }], }), { status: 200 })); } it('POST /api/litellm/restart calls stop then start, returns new status', async () => { configureDynamicCatalog(); mockStop.mockResolvedValue(undefined); mockStart.mockResolvedValue({ status: 'started', port: 4000 }); const res = await injectWithAuth(server, { method: 'POST', url: '/api/litellm/restart', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.running).toBe(true); expect(body.port).toBe(4000); expect(body.models).toEqual(['openai/provider-model-added-today']); // Verify stop was called before start expect(mockStop).toHaveBeenCalledTimes(1); expect(mockStart).toHaveBeenCalledTimes(1); const stopOrder = mockStop.mock.invocationCallOrder[0]; const startOrder = mockStart.mock.invocationCallOrder[0]; expect(stopOrder).toBeLessThan(startOrder); expect(mockStart).toHaveBeenCalledWith(4000, path.join(dataDir, 'litellm.runtime.json')); server.vault.delete('openai'); }); it('POST /api/litellm/restart returns error on start failure', async () => { configureDynamicCatalog(); mockStop.mockResolvedValue(undefined); mockStart.mockResolvedValue({ status: 'error', port: 4000, error: 'Failed to spawn LiteLLM', }); const res = await injectWithAuth(server, { method: 'POST', url: '/api/litellm/restart', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.running).toBe(false); expect(body.error).toBe('Failed to spawn LiteLLM'); server.vault.delete('openai'); }); it('POST /api/litellm/restart proceeds to start even if stop throws', async () => { configureDynamicCatalog(); mockStop.mockRejectedValue(new Error('kill ESRCH')); mockStart.mockResolvedValue({ status: 'started', port: 4000 }); const res = await injectWithAuth(server, { method: 'POST', url: '/api/litellm/restart', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.running).toBe(true); expect(body.port).toBe(4000); expect(mockStart).toHaveBeenCalledTimes(1); server.vault.delete('openai'); }); it('POST /api/litellm/restart returns fallback error on timeout status', async () => { configureDynamicCatalog(); mockStop.mockResolvedValue(undefined); mockStart.mockResolvedValue({ status: 'timeout', port: 4000 }); const res = await injectWithAuth(server, { method: 'POST', url: '/api/litellm/restart', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.running).toBe(false); expect(body.error).toBe('LiteLLM did not start in time'); server.vault.delete('openai'); }); it('GET /api/litellm/pricing uses router metadata instead of a static model list', async () => { vi.spyOn(globalThis, 'fetch').mockResolvedValue(new Response(JSON.stringify({ data: [{ model_name: 'openai/model-added-after-release', model_info: { input_cost_per_token: 0.000002, output_cost_per_token: 0.000006 }, }], }), { status: 200 })); const res = await injectWithAuth(server, { method: 'GET', url: '/api/litellm/pricing' }); expect(res.statusCode).toBe(200); expect(JSON.parse(res.body)).toEqual([{ model: 'openai/model-added-after-release', inputPer1k: 0.002, outputPer1k: 0.006, provider: 'openai', }]); }); // --- GET /api/litellm/models --- it('GET /api/litellm/models returns model list', async () => { vi.spyOn(globalThis, 'fetch').mockResolvedValue({ ok: true, json: async () => ({ data: [ { id: 'gpt-4o' }, { id: 'claude-sonnet-4-20250514' }, { id: 'gemini-pro' }, ], }), } as Response); const res = await injectWithAuth(server, { method: 'GET', url: '/api/litellm/models', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.models).toEqual(['gpt-4o', 'claude-sonnet-4-20250514', 'gemini-pro']); }); it('GET /api/litellm/models merges newly discovered models from configured providers', async () => { server.vault!.set('openai', 'openai-catalog-key'); vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url === 'https://api.openai.com/v1/models') { return new Response(JSON.stringify({ data: [{ id: 'new-model-v9' }] }), { status: 200 }); } if (url.endsWith('/api/tags')) return { ok: false, status: 503 } as Response; if (url.endsWith('/models')) return new Response(JSON.stringify({ data: [] }), { status: 200 }); throw new Error(`unexpected fetch ${url}`); }); try { const res = await injectWithAuth(server, { method: 'GET', url: '/api/litellm/models', }); expect(res.statusCode).toBe(200); expect(JSON.parse(res.body).models).toContain('openai/new-model-v9'); } finally { server.vault!.delete('openai'); } }); it('GET /api/litellm/models returns empty array on fetch failure', async () => { vi.spyOn(globalThis, 'fetch').mockRejectedValue(new Error('Connection refused')); const res = await injectWithAuth(server, { method: 'GET', url: '/api/litellm/models', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.models).toEqual([]); }); it('GET /api/litellm/models returns empty array when LiteLLM returns non-ok response', async () => { vi.spyOn(globalThis, 'fetch').mockResolvedValue({ ok: false, status: 503, } as Response); const res = await injectWithAuth(server, { method: 'GET', url: '/api/litellm/models', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.models).toEqual([]); }); it('GET /api/litellm/models falls back to local Ollama chat models', async () => { vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url.endsWith('/models')) { return { ok: false, status: 503 } as Response; } if (url.endsWith('/api/tags')) { return { ok: true, json: async () => ({ models: [ { name: 'nomic-embed-text:latest', size: 262_000_000 }, { name: 'llama3.2:latest', size: 2_000_000_000 }, ], }), } as Response; } throw new Error(`unexpected fetch ${url}`); }); const res = await injectWithAuth(server, { method: 'GET', url: '/api/litellm/models', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.models).toEqual(['ollama/llama3.2:latest']); }); it('local inference status separates remote aliases from installed models', async () => { vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url.endsWith('/api/tags')) { return new Response(JSON.stringify({ models: [ { name: 'minimax-m2.7:cloud', remote_host: 'https://ollama.com:443' }, { name: 'gemma4:31b' }, ], }), { status: 200 }); } if (url.endsWith('/api/version')) { return new Response(JSON.stringify({ version: '0.12.0' }), { status: 200 }); } return new Response('', { status: 503 }); }); const res = await injectWithAuth(server, { method: 'GET', url: '/api/local-inference/status' }); expect(res.statusCode).toBe(200); const body = res.json(); expect(body.offlineReady).toBe(true); expect(body.setupRequired).toBe(false); expect(body.totalLocalModels).toBe(1); expect(body.primaryServer.models).toEqual(['gemma4:31b']); expect(body.primaryServer.cloudModels).toEqual(['minimax-m2.7:cloud']); }); it('local inference status reports setup required for cloud-only Ollama', async () => { vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url.endsWith('/api/tags')) { return new Response(JSON.stringify({ models: [{ name: 'minimax-m2.7:cloud' }] }), { status: 200, }); } if (url.endsWith('/api/version')) { return new Response(JSON.stringify({ version: '0.12.0' }), { status: 200 }); } return new Response('', { status: 503 }); }); const res = await injectWithAuth(server, { method: 'GET', url: '/api/local-inference/status' }); expect(res.statusCode).toBe(200); const body = res.json(); expect(body.ollamaInstalled).toBe(true); expect(body.offlineReady).toBe(false); expect(body.setupRequired).toBe(true); expect(body.totalLocalModels).toBe(0); expect(body.primaryServer).toBeNull(); expect(body.servers[0].cloudModels).toEqual(['minimax-m2.7:cloud']); expect(body.setupMessage).toMatch(/install|pull/i); }); it('desktop startup stays degraded when Ollama exposes only a cloud alias', async () => { const soloDir = fs.mkdtempSync(path.join(os.tmpdir(), 'waggle-cloud-only-startup-')); vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url.endsWith('/api/tags')) { return new Response(JSON.stringify({ models: [{ name: 'minimax-m2.7:cloud', remote_host: 'https://ollama.com:443', }], }), { status: 200 }); } return new Response('', { status: 503 }); }); const { server: soloServer } = await startService({ dataDir: soloDir, port: 0, litellmPort: 49_999, skipLiteLLM: true, }); try { const body = (await soloServer.inject({ method: 'GET', url: '/health' })).json(); expect(body.status).toBe('degraded'); expect(body.llm).toMatchObject({ provider: 'anthropic-proxy', health: 'degraded' }); expect(soloServer.agentState.llmProvider.detail).toContain('no API key'); expect(soloServer.agentState.currentModel).not.toBe('ollama/minimax-m2.7:cloud'); } finally { await soloServer.close(); fs.rmSync(soloDir, { recursive: true, force: true }); } }); it('GET /api/agent/model resolves a cloud default to a local chat model when no provider key exists', async () => { vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url.endsWith('/api/tags')) { return { ok: true, json: async () => ({ models: [ { name: 'nomic-embed-text:latest', size: 262_000_000 }, { name: 'llama3.2:latest', size: 2_000_000_000 }, ], }), } as Response; } return { ok: false, status: 503 } as Response; }); await injectWithAuth(server, { method: 'PUT', url: '/api/agent/model', payload: { model: 'claude-sonnet-4-6' }, }); const res = await injectWithAuth(server, { method: 'GET', url: '/api/agent/model', }); expect(res.statusCode).toBe(200); const body = JSON.parse(res.body); expect(body.model).toBe('ollama/llama3.2:latest'); }); it('prefers a deterministic credentialed cloud fallback over an unrelated host Ollama model', async () => { const runtimePath = path.join(dataDir, 'litellm.runtime.json'); const priorRuntime = fs.existsSync(runtimePath) ? fs.readFileSync(runtimePath, 'utf-8') : null; const priorCurrentModel = server.agentState.currentModel; server.agentState.currentModel = 'claude-sonnet-4-6'; server.vault.set('google', 'google-cloud-fallback-test-key'); server.vault.set('openrouter', 'openrouter-cloud-fallback-test-key'); fs.writeFileSync(runtimePath, JSON.stringify({ model_list: [ { model_name: 'google/gemini-2.5-flash' }, { model_name: 'openrouter/openai/gpt-5.3-codex' }, ], }), 'utf-8'); const discoveredProviders: string[] = []; let ollamaRequests = 0; vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url.startsWith('https://generativelanguage.googleapis.com/')) { discoveredProviders.push('google'); return new Response(JSON.stringify({ models: [ { name: 'models/text-embedding-004' }, { name: 'models/gemini-2.5-flash' }, ], }), { status: 200 }); } if (url.startsWith('https://openrouter.ai/')) { discoveredProviders.push('openrouter'); return new Response(JSON.stringify({ data: [{ id: 'openai/gpt-5.3-codex' }], }), { status: 200 }); } if (url.endsWith('/api/tags')) { ollamaRequests += 1; return new Response(JSON.stringify({ models: [{ name: 'minicpm5-fable:1b' }], }), { status: 200 }); } return new Response('', { status: 503 }); }); try { await expect(resolveUsableModel(server, 'claude-sonnet-4-6')) .resolves.toBe('google/gemini-2.5-flash'); expect(discoveredProviders.sort()).toEqual(['google', 'openrouter']); expect(ollamaRequests).toBe(0); } finally { server.agentState.currentModel = priorCurrentModel; if (priorRuntime === null) fs.rmSync(runtimePath, { force: true }); else fs.writeFileSync(runtimePath, priorRuntime, 'utf-8'); } }); it('quality-ranks an OpenRouter fallback instead of selecting the first catalog item', async () => { const priorCurrentModel = server.agentState.currentModel; const priorProvider = { ...server.agentState.llmProvider }; server.agentState.currentModel = 'claude-sonnet-4-6'; server.agentState.llmProvider = { provider: 'anthropic-proxy', health: 'degraded', detail: 'Built-in provider proxy (verification pending)', checkedAt: new Date().toISOString(), }; server.vault.set('openrouter', 'openrouter-quality-fallback-test-key'); vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url.startsWith('https://openrouter.ai/')) { return new Response(JSON.stringify({ data: [ { id: 'thinkingmachines/inkling' }, { id: 'anthropic/claude-sonnet-5' }, { id: 'openai/gpt-5.6-sol' }, ], }), { status: 200 }); } if (url.endsWith('/api/tags')) { return new Response(JSON.stringify({ models: [] }), { status: 200 }); } return new Response('', { status: 503 }); }); try { await expect(resolveUsableModel(server, 'claude-sonnet-4-6')) .resolves.toBe('openrouter/anthropic/claude-sonnet-5'); } finally { server.agentState.currentModel = priorCurrentModel; server.agentState.llmProvider = priorProvider; } }); it('routes a preferred Claude alias through the built-in OpenRouter proxy without catalog discovery', async () => { const priorCurrentModel = server.agentState.currentModel; const priorProvider = { ...server.agentState.llmProvider }; server.agentState.currentModel = 'claude-sonnet-4-6'; server.agentState.llmProvider = { provider: 'anthropic-proxy', health: 'degraded', detail: 'Built-in provider proxy (verification pending)', checkedAt: new Date().toISOString(), }; server.vault.set('openrouter', 'openrouter-direct-fallback-test-key'); let catalogRequests = 0; vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url.startsWith('https://')) catalogRequests += 1; if (url.endsWith('/api/tags')) { return new Response(JSON.stringify({ models: [{ name: 'minicpm5-fable:1b' }] }), { status: 200 }); } return new Response('', { status: 503 }); }); try { await expect(resolveUsableModel(server, 'claude-sonnet-4-6')) .resolves.toBe('openrouter/anthropic/claude-sonnet-5'); expect(catalogRequests).toBe(0); } finally { server.agentState.currentModel = priorCurrentModel; server.agentState.llmProvider = priorProvider; } }); it('keeps an explicit built-in proxy model exact when managed LiteLLM catalog state is stale', async () => { const requestedModel = 'openrouter/openai/gpt-5.3-codex'; const fallbackModel = 'google/gemini-2.5-flash'; const runtimePath = path.join(dataDir, 'litellm.runtime.json'); const priorRuntime = fs.existsSync(runtimePath) ? fs.readFileSync(runtimePath, 'utf-8') : null; const priorCurrentModel = server.agentState.currentModel; const priorProvider = { ...server.agentState.llmProvider }; server.agentState.currentModel = fallbackModel; server.agentState.llmProvider = { provider: 'anthropic-proxy', health: 'degraded', detail: 'Built-in provider proxy (verification pending)', checkedAt: new Date().toISOString(), }; server.vault.set('google', 'google-model-lock-test-key'); server.vault.set('openrouter', 'openrouter-model-lock-test-key'); fs.writeFileSync(runtimePath, JSON.stringify({ model_list: [{ model_name: fallbackModel }], }), 'utf-8'); const probedModels: string[] = []; vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { const url = String(input); if (url.includes('/v1/chat/completions')) { const body = JSON.parse(String(init?.body)) as { model: string }; probedModels.push(body.model); return new Response('{}', { status: 200 }); } if (url.startsWith('https://generativelanguage.googleapis.com/')) { return new Response(JSON.stringify({ models: [{ name: 'models/gemini-2.5-flash' }], }), { status: 200 }); } if (url.startsWith('https://openrouter.ai/')) { return new Response('', { status: 503 }); } if (url.endsWith('/api/tags')) { return new Response(JSON.stringify({ models: [] }), { status: 200 }); } return new Response('', { status: 503 }); }); try { await expect(resolveUsableModel(server, requestedModel)).resolves.toBe(requestedModel); await expect(resolveExplicitRoutableModel(server, requestedModel)).resolves.toBe(requestedModel); const response = await injectWithAuth(server, { method: 'POST', url: '/api/settings/probe-model', payload: { model: requestedModel }, }); expect(response.statusCode).toBe(200); expect(response.json()).toMatchObject({ model: requestedModel, configured: true, verified: true, }); expect(probedModels).toEqual([requestedModel]); } finally { server.agentState.currentModel = priorCurrentModel; server.agentState.llmProvider = priorProvider; if (priorRuntime === null) fs.rmSync(runtimePath, { force: true }); else fs.writeFileSync(runtimePath, priorRuntime, 'utf-8'); } }); it('never exposes or selects a remote Ollama cloud alias as a local model', async () => { vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { const url = String(input); if (url.endsWith('/api/tags')) { return { ok: true, json: async () => ({ models: [ { name: 'nomic-embed-text:latest' }, { name: 'minimax-m2.7:cloud', remote_host: 'https://ollama.com:443' }, { name: 'gemma4:31b' }, ], }), } as Response; } return { ok: false, status: 503 } as Response; }); await expect(listOllamaChatModelIds()).resolves.toEqual(['ollama/gemma4:31b']); const selected = await injectWithAuth(server, { method: 'PUT', url: '/api/agent/model', payload: { model: 'ollama/minimax-m2.7:cloud' }, }); expect(selected.statusCode).toBe(409); expect(selected.json()).toMatchObject({ code: 'OLLAMA_MODEL_NOT_LOCAL' }); await expect(resolveUsableModel(server, 'ollama/minimax-m2.7:cloud')) .rejects.toMatchObject({ code: 'OLLAMA_MODEL_NOT_LOCAL', statusCode: 409 }); }); it('rejects an exact Ollama tag that is not installed instead of choosing another tag', async () => { vi.spyOn(globalThis, 'fetch').mockImplementation(async (input) => { if (String(input).endsWith('/api/tags')) { return { ok: true, json: async () => ({ models: [{ name: 'gemma4:31b' }] }), } as Response; } return { ok: false, status: 503 } as Response; }); await expect(resolveUsableModel(server, 'ollama/llama3.2:latest')) .rejects.toMatchObject({ code: 'OLLAMA_MODEL_NOT_LOCAL', statusCode: 409 }); }); });