/** * OpenAI-compatible chat completions adapter. * * Works with any provider that exposes the standard * POST /v1/chat/completions endpoint (OpenAI, Groq, Together, * Mistral, LiteLLM proxy, etc.). */ import type { ResolvedModel } from '../model-router.js'; export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string; } export interface ChatResponse { content: string; model: string; usage: { input_tokens: number; output_tokens: number }; } export interface CompletionUsage { inputTokens: number; outputTokens: number; totalCostUsd: number; } export interface ParsedOpenAiTextCompletion { content: string; model: string; usage: CompletionUsage; } export type IncompleteCompletionError = Error & { code: 'INCOMPLETE_COMPLETION'; usage: CompletionUsage; }; function incompleteCompletionError( reason: string, usage: IncompleteCompletionError['usage'], ): IncompleteCompletionError { const error = new Error( `OpenAI-compatible completion was not complete (${reason}); partial content was rejected.`, ) as IncompleteCompletionError; error.name = 'IncompleteCompletionError'; error.code = 'INCOMPLETE_COMPLETION'; error.usage = usage; return error; } export function isIncompleteCompletionError(error: unknown): error is IncompleteCompletionError { return typeof error === 'object' && error !== null && (error as { code?: unknown }).code === 'INCOMPLETE_COMPLETION'; } function usageNumber(value: unknown): number { return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : 0; } /** * Validate a non-streaming OpenAI-compatible text completion. * * HTTP 200 is not sufficient evidence of a complete answer: only an explicit * `finish_reason: "stop"` with non-blank text and no tool calls is accepted. * Reported usage is attached to integrity failures so callers can account for * paid partial responses without replaying them. */ export function parseOpenAiTextCompletion(rawData: unknown): ParsedOpenAiTextCompletion { if (typeof rawData !== 'object' || rawData === null) { throw incompleteCompletionError('invalid response body', { inputTokens: 0, outputTokens: 0, totalCostUsd: 0, }); } const data = rawData as { error?: { message?: unknown } | string; choices?: Array<{ finish_reason?: string | null; message?: { content?: string | null; refusal?: string | null; tool_calls?: unknown; }; }>; model?: unknown; usage?: { prompt_tokens?: unknown; completion_tokens?: unknown; total_cost?: unknown; }; }; const usage: CompletionUsage = { inputTokens: usageNumber(data.usage?.prompt_tokens), outputTokens: usageNumber(data.usage?.completion_tokens), totalCostUsd: usageNumber(data.usage?.total_cost), }; if (data.error) { const detail = typeof data.error === 'string' ? data.error : typeof data.error.message === 'string' ? data.error.message : 'upstream error payload'; throw incompleteCompletionError(detail, usage); } const choice = data.choices?.[0]; if (!choice) { throw incompleteCompletionError('missing completion choice', usage); } if (choice.finish_reason !== 'stop') { const reason = choice.finish_reason ?? 'missing'; throw incompleteCompletionError(`finish_reason=${reason}`, usage); } const toolCalls = choice.message?.tool_calls; if (toolCalls !== undefined && toolCalls !== null && (!Array.isArray(toolCalls) || toolCalls.length > 0)) { throw incompleteCompletionError('finish_reason=stop with tool_calls', usage); } if (typeof choice.message?.refusal === 'string' && choice.message.refusal.trim().length > 0) { throw incompleteCompletionError('assistant refusal', usage); } const content = choice.message?.content; if (typeof content !== 'string' || content.trim().length === 0) { throw incompleteCompletionError('missing assistant text', usage); } return { content, model: typeof data.model === 'string' ? data.model : '', usage, }; } /** Per-request wall-clock timeout before the request is aborted. */ const DEFAULT_TIMEOUT_MS = 60_000; /** Additional attempts after the first on a transient failure. */ const DEFAULT_MAX_RETRIES = 3; /** Cap on exponential backoff between attempts. */ const BACKOFF_CAP_MS = 30_000; /** Statuses worth retrying: rate limit + gateway/transient server errors. */ function isRetryableStatus(status: number): boolean { return status === 429 || status === 500 || status === 502 || status === 503 || status === 504; } /** Exponential backoff (1s, 2s, 4s, …) capped at BACKOFF_CAP_MS. */ function backoffMs(attempt: number): number { return Math.min(1000 * 2 ** (attempt - 1), BACKOFF_CAP_MS); } function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } export interface OpenAiChatOptions { /** Per-request timeout in ms (default 60s). Aborts a hung upstream. */ timeoutMs?: number; /** Extra retry attempts on 5xx/429/network failure (default 3). */ maxRetries?: number; /** Injectable fetch (tests). Defaults to globalThis.fetch. */ fetchImpl?: typeof globalThis.fetch; /** Injectable backoff sleep (tests) — lets a test skip real delays. */ sleepImpl?: (ms: number) => Promise; } /** * Send a chat completion request to an OpenAI-compatible API. * * Wraps the request in an AbortSignal timeout (a hung upstream no longer wedges * the caller forever) and retries transient failures (429 / 5xx / network) with * exponential backoff, mirroring the agent-loop retry policy. Non-transient * errors (4xx other than 429, malformed body) throw immediately. * * @param resolved - Provider details from ModelRouter.resolve() * @param messages - Conversation history * @param systemPrompt - Optional system prompt (prepended as a system message) * @param options - Timeout / retry / injection overrides */ export async function openaiChat( resolved: ResolvedModel, messages: ChatMessage[], systemPrompt?: string, options: OpenAiChatOptions = {}, ): Promise { const baseUrl = resolved.baseUrl ?? 'https://api.openai.com/v1'; const url = `${baseUrl.replace(/\/+$/, '')}/chat/completions`; const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS; const maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES; const fetchImpl = options.fetchImpl ?? globalThis.fetch; const doSleep = options.sleepImpl ?? sleep; const allMessages: ChatMessage[] = []; if (systemPrompt) { allMessages.push({ role: 'system', content: systemPrompt }); } allMessages.push(...messages); const requestInit: RequestInit = { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${resolved.apiKey}`, }, body: JSON.stringify({ model: resolved.model, messages: allMessages, }), }; let attempt = 0; // Loop bound: 1 initial attempt + maxRetries retries. for (;;) { let res: Response; try { res = await fetchImpl(url, { ...requestInit, signal: AbortSignal.timeout(timeoutMs) }); } catch (err: unknown) { // Network-level failure or timeout abort — retry with backoff. if (attempt < maxRetries) { attempt++; await doSleep(backoffMs(attempt)); continue; } const detail = err instanceof Error ? err.message : String(err); throw new Error( `OpenAI-compatible request failed after ${maxRetries + 1} attempts: ${detail}`, ); } if (!res.ok) { if (isRetryableStatus(res.status) && attempt < maxRetries) { attempt++; await res.text().catch(() => ''); // drain body to free the socket await doSleep(backoffMs(attempt)); continue; } const body = await res.text().catch(() => ''); throw new Error( `OpenAI-compatible API error ${res.status}: ${res.statusText}${body ? ` — ${body}` : ''}`, ); } let rawData: unknown; try { rawData = await res.json(); } catch { throw incompleteCompletionError('invalid JSON response body', { inputTokens: 0, outputTokens: 0, totalCostUsd: 0, }); } const parsed = parseOpenAiTextCompletion(rawData); return { content: parsed.content, model: parsed.model || resolved.model, usage: { input_tokens: parsed.usage.inputTokens, output_tokens: parsed.usage.outputTokens, }, }; } }