The CLI sent every request to a LiteLLM proxy on :4000, which the community edition no longer starts (it needs Python), so the CLI could not answer with any model out of the box. ollama/<tag> models now go to Ollama's OpenAI-compatible API directly; other models still use the configured proxy.
38 lines
1.4 KiB
TypeScript
38 lines
1.4 KiB
TypeScript
/**
|
|
* Where the CLI sends a model request.
|
|
*
|
|
* Local models (`ollama/<tag>`) go straight to Ollama's OpenAI-compatible API,
|
|
* so the CLI answers with a local model without any proxy running. Every other
|
|
* model goes through the LiteLLM-compatible proxy named in the workspace config
|
|
* (default http://localhost:4000/v1), which the community edition does not
|
|
* start by default.
|
|
*/
|
|
export interface ModelEndpoint {
|
|
/** Base URL; the agent loop appends /chat/completions. */
|
|
url: string;
|
|
apiKey: string;
|
|
/** The model name the endpoint expects. */
|
|
model: string;
|
|
}
|
|
|
|
const OLLAMA_PREFIX = 'ollama/';
|
|
const DEFAULT_OLLAMA_HOST = 'http://127.0.0.1:11434';
|
|
const DEFAULT_PROXY_URL = 'http://localhost:4000/v1';
|
|
|
|
export function modelEndpoint(
|
|
model: string,
|
|
options: { litellmUrl?: string; env: NodeJS.ProcessEnv },
|
|
): ModelEndpoint {
|
|
if (model.startsWith(OLLAMA_PREFIX)) {
|
|
const host = (options.env.OLLAMA_HOST || DEFAULT_OLLAMA_HOST).replace(/\/+$/, '');
|
|
const base = /^https?:\/\//.test(host) ? host : `http://${host}`;
|
|
// Ollama ignores the key, but the request needs an Authorization header.
|
|
return { url: `${base}/v1`, apiKey: 'ollama', model: model.slice(OLLAMA_PREFIX.length) };
|
|
}
|
|
return {
|
|
url: options.litellmUrl ?? DEFAULT_PROXY_URL,
|
|
apiKey: options.env.LITELLM_API_KEY ?? options.env.LITELLM_MASTER_KEY ?? 'sk-waggle-dev',
|
|
model,
|
|
};
|
|
}
|