fix(cli): talk to local Ollama models without a proxy
The CLI sent every request to a LiteLLM proxy on :4000, which the community edition no longer starts (it needs Python), so the CLI could not answer with any model out of the box. ollama/<tag> models now go to Ollama's OpenAI-compatible API directly; other models still use the configured proxy.
This commit is contained in:
37
packages/cli/src/model-endpoint.ts
Normal file
37
packages/cli/src/model-endpoint.ts
Normal file
@@ -0,0 +1,37 @@
|
|||||||
|
/**
|
||||||
|
* Where the CLI sends a model request.
|
||||||
|
*
|
||||||
|
* Local models (`ollama/<tag>`) go straight to Ollama's OpenAI-compatible API,
|
||||||
|
* so the CLI answers with a local model without any proxy running. Every other
|
||||||
|
* model goes through the LiteLLM-compatible proxy named in the workspace config
|
||||||
|
* (default http://localhost:4000/v1), which the community edition does not
|
||||||
|
* start by default.
|
||||||
|
*/
|
||||||
|
export interface ModelEndpoint {
|
||||||
|
/** Base URL; the agent loop appends /chat/completions. */
|
||||||
|
url: string;
|
||||||
|
apiKey: string;
|
||||||
|
/** The model name the endpoint expects. */
|
||||||
|
model: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const OLLAMA_PREFIX = 'ollama/';
|
||||||
|
const DEFAULT_OLLAMA_HOST = 'http://127.0.0.1:11434';
|
||||||
|
const DEFAULT_PROXY_URL = 'http://localhost:4000/v1';
|
||||||
|
|
||||||
|
export function modelEndpoint(
|
||||||
|
model: string,
|
||||||
|
options: { litellmUrl?: string; env: NodeJS.ProcessEnv },
|
||||||
|
): ModelEndpoint {
|
||||||
|
if (model.startsWith(OLLAMA_PREFIX)) {
|
||||||
|
const host = (options.env.OLLAMA_HOST || DEFAULT_OLLAMA_HOST).replace(/\/+$/, '');
|
||||||
|
const base = /^https?:\/\//.test(host) ? host : `http://${host}`;
|
||||||
|
// Ollama ignores the key, but the request needs an Authorization header.
|
||||||
|
return { url: `${base}/v1`, apiKey: 'ollama', model: model.slice(OLLAMA_PREFIX.length) };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
url: options.litellmUrl ?? DEFAULT_PROXY_URL,
|
||||||
|
apiKey: options.env.LITELLM_API_KEY ?? options.env.LITELLM_MASTER_KEY ?? 'sk-waggle-dev',
|
||||||
|
model,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -15,6 +15,7 @@ import { PERSONAL_SCOPE } from '@waggle/memory-contract';
|
|||||||
import { createMem0MemoryStore } from '@waggle/memory-mem0';
|
import { createMem0MemoryStore } from '@waggle/memory-mem0';
|
||||||
import { Orchestrator, ModelRouter, runAgentLoop, createSystemTools, createPlanTools, createGitTools, Workspace, ensureIdentity, loadSystemPromptWithOverrides, loadSkills, CostTracker, HookRegistry, loadHooksFromConfig, needsConfirmation } from '@waggle/agent';
|
import { Orchestrator, ModelRouter, runAgentLoop, createSystemTools, createPlanTools, createGitTools, Workspace, ensureIdentity, loadSystemPromptWithOverrides, loadSkills, CostTracker, HookRegistry, loadHooksFromConfig, needsConfirmation } from '@waggle/agent';
|
||||||
import { parseCommand, COMMANDS } from './commands.js';
|
import { parseCommand, COMMANDS } from './commands.js';
|
||||||
|
import { modelEndpoint } from './model-endpoint.js';
|
||||||
import { renderMarkdown } from './renderer.js';
|
import { renderMarkdown } from './renderer.js';
|
||||||
|
|
||||||
export interface ReplOptions {
|
export interface ReplOptions {
|
||||||
@@ -53,11 +54,8 @@ export async function startRepl(options: ReplOptions = {}): Promise<void> {
|
|||||||
const hookRegistry = new HookRegistry();
|
const hookRegistry = new HookRegistry();
|
||||||
await loadHooksFromConfig(path.join(waggleHome, 'hooks.json'), hookRegistry);
|
await loadHooksFromConfig(path.join(waggleHome, 'hooks.json'), hookRegistry);
|
||||||
|
|
||||||
// Get LiteLLM config from workspace
|
// The proxy for non-local models; local ollama/* models skip it (model-endpoint.ts).
|
||||||
const wsConfig = workspace.getConfig();
|
const wsConfig = workspace.getConfig();
|
||||||
const litellmUrl = wsConfig.litellmUrl ?? 'http://localhost:4000/v1';
|
|
||||||
// LiteLLM master key for proxy auth (NOT the provider API key — LiteLLM reads that from its own env)
|
|
||||||
const litellmApiKey = process.env.LITELLM_API_KEY ?? process.env.LITELLM_MASTER_KEY ?? 'sk-waggle-dev';
|
|
||||||
|
|
||||||
// Memory: the same store the desktop app uses (ADR-0007), in ~/.waggle/memory.
|
// Memory: the same store the desktop app uses (ADR-0007), in ~/.waggle/memory.
|
||||||
const memory = await createMem0MemoryStore({
|
const memory = await createMem0MemoryStore({
|
||||||
@@ -298,10 +296,12 @@ When asked about current events, products, releases, docs, or anything you're no
|
|||||||
|
|
||||||
conversationHistory.push({ role: 'user', content: input });
|
conversationHistory.push({ role: 'user', content: input });
|
||||||
|
|
||||||
|
const endpoint = modelEndpoint(currentModel, { litellmUrl: wsConfig.litellmUrl, env: process.env });
|
||||||
const result = await runAgentLoop({
|
const result = await runAgentLoop({
|
||||||
litellmUrl,
|
litellmUrl: endpoint.url,
|
||||||
litellmApiKey,
|
litellmApiKey: endpoint.apiKey,
|
||||||
model: currentModel,
|
model: endpoint.model,
|
||||||
|
billingModel: currentModel,
|
||||||
systemPrompt,
|
systemPrompt,
|
||||||
tools,
|
tools,
|
||||||
messages: conversationHistory,
|
messages: conversationHistory,
|
||||||
|
|||||||
25
packages/cli/tests/model-endpoint.test.ts
Normal file
25
packages/cli/tests/model-endpoint.test.ts
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
import { describe, expect, it } from 'vitest';
|
||||||
|
import { modelEndpoint } from '../src/model-endpoint.js';
|
||||||
|
|
||||||
|
describe('modelEndpoint', () => {
|
||||||
|
it('sends local models straight to Ollama under their own tag', () => {
|
||||||
|
expect(modelEndpoint('ollama/llama3.1:8b', { env: {} })).toEqual({
|
||||||
|
url: 'http://127.0.0.1:11434/v1',
|
||||||
|
apiKey: 'ollama',
|
||||||
|
model: 'llama3.1:8b',
|
||||||
|
});
|
||||||
|
expect(modelEndpoint('ollama/qwen3:8b', { env: { OLLAMA_HOST: '10.0.0.5:11500/' } }).url)
|
||||||
|
.toBe('http://10.0.0.5:11500/v1');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('sends other models through the configured proxy', () => {
|
||||||
|
expect(modelEndpoint('claude-sonnet-4-6', { env: {} })).toEqual({
|
||||||
|
url: 'http://localhost:4000/v1',
|
||||||
|
apiKey: 'sk-waggle-dev',
|
||||||
|
model: 'claude-sonnet-4-6',
|
||||||
|
});
|
||||||
|
expect(
|
||||||
|
modelEndpoint('gpt-4o', { litellmUrl: 'http://proxy:9000/v1', env: { LITELLM_MASTER_KEY: 'master' } }),
|
||||||
|
).toEqual({ url: 'http://proxy:9000/v1', apiKey: 'master', model: 'gpt-4o' });
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user