From a70bc14c63b64771771bc1c6159097947e8bca5c Mon Sep 17 00:00:00 2001 From: "shumeiko.it" Date: Sun, 1 Nov 2026 11:15:37 +0100 Subject: [PATCH] fix(cli): talk to local Ollama models without a proxy The CLI sent every request to a LiteLLM proxy on :4000, which the community edition no longer starts (it needs Python), so the CLI could not answer with any model out of the box. ollama/ models now go to Ollama's OpenAI-compatible API directly; other models still use the configured proxy. --- packages/cli/src/model-endpoint.ts | 37 +++++++++++++++++++++++ packages/cli/src/repl.ts | 14 ++++----- packages/cli/tests/model-endpoint.test.ts | 25 +++++++++++++++ 3 files changed, 69 insertions(+), 7 deletions(-) create mode 100644 packages/cli/src/model-endpoint.ts create mode 100644 packages/cli/tests/model-endpoint.test.ts diff --git a/packages/cli/src/model-endpoint.ts b/packages/cli/src/model-endpoint.ts new file mode 100644 index 0000000..b002520 --- /dev/null +++ b/packages/cli/src/model-endpoint.ts @@ -0,0 +1,37 @@ +/** + * Where the CLI sends a model request. + * + * Local models (`ollama/`) go straight to Ollama's OpenAI-compatible API, + * so the CLI answers with a local model without any proxy running. Every other + * model goes through the LiteLLM-compatible proxy named in the workspace config + * (default http://localhost:4000/v1), which the community edition does not + * start by default. + */ +export interface ModelEndpoint { + /** Base URL; the agent loop appends /chat/completions. */ + url: string; + apiKey: string; + /** The model name the endpoint expects. */ + model: string; +} + +const OLLAMA_PREFIX = 'ollama/'; +const DEFAULT_OLLAMA_HOST = 'http://127.0.0.1:11434'; +const DEFAULT_PROXY_URL = 'http://localhost:4000/v1'; + +export function modelEndpoint( + model: string, + options: { litellmUrl?: string; env: NodeJS.ProcessEnv }, +): ModelEndpoint { + if (model.startsWith(OLLAMA_PREFIX)) { + const host = (options.env.OLLAMA_HOST || DEFAULT_OLLAMA_HOST).replace(/\/+$/, ''); + const base = /^https?:\/\//.test(host) ? host : `http://${host}`; + // Ollama ignores the key, but the request needs an Authorization header. + return { url: `${base}/v1`, apiKey: 'ollama', model: model.slice(OLLAMA_PREFIX.length) }; + } + return { + url: options.litellmUrl ?? DEFAULT_PROXY_URL, + apiKey: options.env.LITELLM_API_KEY ?? options.env.LITELLM_MASTER_KEY ?? 'sk-waggle-dev', + model, + }; +} diff --git a/packages/cli/src/repl.ts b/packages/cli/src/repl.ts index 393249a..a7951f2 100644 --- a/packages/cli/src/repl.ts +++ b/packages/cli/src/repl.ts @@ -15,6 +15,7 @@ import { PERSONAL_SCOPE } from '@waggle/memory-contract'; import { createMem0MemoryStore } from '@waggle/memory-mem0'; import { Orchestrator, ModelRouter, runAgentLoop, createSystemTools, createPlanTools, createGitTools, Workspace, ensureIdentity, loadSystemPromptWithOverrides, loadSkills, CostTracker, HookRegistry, loadHooksFromConfig, needsConfirmation } from '@waggle/agent'; import { parseCommand, COMMANDS } from './commands.js'; +import { modelEndpoint } from './model-endpoint.js'; import { renderMarkdown } from './renderer.js'; export interface ReplOptions { @@ -53,11 +54,8 @@ export async function startRepl(options: ReplOptions = {}): Promise { const hookRegistry = new HookRegistry(); await loadHooksFromConfig(path.join(waggleHome, 'hooks.json'), hookRegistry); - // Get LiteLLM config from workspace + // The proxy for non-local models; local ollama/* models skip it (model-endpoint.ts). const wsConfig = workspace.getConfig(); - const litellmUrl = wsConfig.litellmUrl ?? 'http://localhost:4000/v1'; - // LiteLLM master key for proxy auth (NOT the provider API key — LiteLLM reads that from its own env) - const litellmApiKey = process.env.LITELLM_API_KEY ?? process.env.LITELLM_MASTER_KEY ?? 'sk-waggle-dev'; // Memory: the same store the desktop app uses (ADR-0007), in ~/.waggle/memory. const memory = await createMem0MemoryStore({ @@ -298,10 +296,12 @@ When asked about current events, products, releases, docs, or anything you're no conversationHistory.push({ role: 'user', content: input }); + const endpoint = modelEndpoint(currentModel, { litellmUrl: wsConfig.litellmUrl, env: process.env }); const result = await runAgentLoop({ - litellmUrl, - litellmApiKey, - model: currentModel, + litellmUrl: endpoint.url, + litellmApiKey: endpoint.apiKey, + model: endpoint.model, + billingModel: currentModel, systemPrompt, tools, messages: conversationHistory, diff --git a/packages/cli/tests/model-endpoint.test.ts b/packages/cli/tests/model-endpoint.test.ts new file mode 100644 index 0000000..2b3ae32 --- /dev/null +++ b/packages/cli/tests/model-endpoint.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest'; +import { modelEndpoint } from '../src/model-endpoint.js'; + +describe('modelEndpoint', () => { + it('sends local models straight to Ollama under their own tag', () => { + expect(modelEndpoint('ollama/llama3.1:8b', { env: {} })).toEqual({ + url: 'http://127.0.0.1:11434/v1', + apiKey: 'ollama', + model: 'llama3.1:8b', + }); + expect(modelEndpoint('ollama/qwen3:8b', { env: { OLLAMA_HOST: '10.0.0.5:11500/' } }).url) + .toBe('http://10.0.0.5:11500/v1'); + }); + + it('sends other models through the configured proxy', () => { + expect(modelEndpoint('claude-sonnet-4-6', { env: {} })).toEqual({ + url: 'http://localhost:4000/v1', + apiKey: 'sk-waggle-dev', + model: 'claude-sonnet-4-6', + }); + expect( + modelEndpoint('gpt-4o', { litellmUrl: 'http://proxy:9000/v1', env: { LITELLM_MASTER_KEY: 'master' } }), + ).toEqual({ url: 'http://proxy:9000/v1', apiKey: 'master', model: 'gpt-4o' }); + }); +});