From 3e1a188c0324ebf8700c2a132596c8c098681cb6 Mon Sep 17 00:00:00 2001 From: "shumeiko.it" Date: Thu, 15 Oct 2026 18:13:34 +0200 Subject: [PATCH] fix(server): stream Ollama model pulls so long downloads finish A non-streaming /api/pull holds the response headers until the download ends, and Node fetch gives up on headers after five minutes, cancelling every larger model pull. Request stream:true and read the NDJSON progress, failing on an in-stream error or a missing success status (UF-008). --- .../src/local/routes/local-inference.ts | 45 ++++++++++++- .../tests/local/local-inference-route.test.ts | 64 +++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/packages/server/src/local/routes/local-inference.ts b/packages/server/src/local/routes/local-inference.ts index 6930e32..071d4c7 100644 --- a/packages/server/src/local/routes/local-inference.ts +++ b/packages/server/src/local/routes/local-inference.ts @@ -70,6 +70,38 @@ function isValidOllamaModelRef(model: string): boolean { return repository.split('/').every((segment) => segment.length > 0 && segment !== '.' && segment !== '..'); } +class OllamaPullError extends Error {} + +/** + * Reads Ollama's NDJSON pull progress to the end and returns the final status. + * Throws OllamaPullError when Ollama reports an error line or the stream ends + * without "success". + */ +async function readOllamaPullStream(res: Response): Promise> { + if (!res.body) throw new OllamaPullError('Ollama returned an empty pull response'); + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffered = ''; + let last: Record | null = null; + for (;;) { + const chunk = await reader.read(); + buffered += chunk.done ? decoder.decode() : decoder.decode(chunk.value, { stream: true }); + const lines = buffered.split('\n'); + buffered = chunk.done ? '' : (lines.pop() ?? ''); + for (const line of lines) { + if (!line.trim()) continue; + const status = JSON.parse(line) as Record; + if (typeof status.error === 'string') throw new OllamaPullError(status.error); + last = status; + } + if (chunk.done) break; + } + if (!last || last.status !== 'success') { + throw new OllamaPullError('the pull stream ended before Ollama reported success'); + } + return last; +} + export interface LocalInferenceRuntimeController { getStatus(): ManagedOllamaStatus; ensureReady(): Promise; @@ -327,14 +359,23 @@ export async function localInferenceRoutes( const res = await fetch(`${OLLAMA_URL}/api/pull`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ name: model, stream: false }), + // Stream progress. With stream:false Ollama sends no headers until the + // download completes, and Node's fetch aborts after 300 s without + // headers, which cancelled every pull longer than five minutes. + body: JSON.stringify({ name: model, stream: true }), signal: AbortSignal.timeout(45 * 60_000), }); if (!res.ok) { const text = await res.text(); return reply.code(502).send({ error: `Ollama pull failed: ${text}` }); } - const pullStatus = await res.json(); + let pullStatus: Record; + try { + pullStatus = await readOllamaPullStream(res); + } catch (error) { + if (!(error instanceof OllamaPullError)) throw error; + return reply.code(502).send({ error: `Ollama pull failed: ${error.message}` }); + } const installed = await probeOllama(OLLAMA_URL); const installedModel = installed.models.find((name) => name === model || name === `${model}:latest`); if (!installedModel) { diff --git a/packages/server/tests/local/local-inference-route.test.ts b/packages/server/tests/local/local-inference-route.test.ts index 24499b0..d4b8ccb 100644 --- a/packages/server/tests/local/local-inference-route.test.ts +++ b/packages/server/tests/local/local-inference-route.test.ts @@ -732,6 +732,70 @@ describe('local-inference route — verified model installation', () => { } }); + it('streams the pull so downloads longer than the fetch header timeout survive', async () => { + // With stream:false Ollama sends no response headers until the download + // finishes, and Node's fetch gives up after 300 s without headers, which + // cancelled every pull longer than five minutes. + const fetchMock = vi.fn(async (input: string | URL | Request) => { + const url = String(input); + if (url.endsWith('/api/pull')) { + return new Response([ + JSON.stringify({ status: 'pulling manifest' }), + JSON.stringify({ status: 'pulling sha256:abc', total: 100, completed: 50 }), + JSON.stringify({ status: 'success' }), + '', + ].join('\n'), { status: 200, headers: { 'content-type': 'application/x-ndjson' } }); + } + return new Response(JSON.stringify({ response: 'OK', done: true }), { status: 200 }); + }); + vi.stubGlobal('fetch', fetchMock); + const server = await buildPullServer(['qwen3:1.7b']); + try { + const response = await server.inject({ + method: 'POST', + url: '/api/local-inference/pull', + payload: { model: 'qwen3:1.7b' }, + }); + expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toMatchObject({ + name: 'qwen3:1.7b', + stream: true, + }); + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + ok: true, + model: 'qwen3:1.7b', + status: { status: 'success' }, + verifiedGeneration: true, + }); + } finally { + await server.close(); + vi.unstubAllGlobals(); + } + }); + + it('fails the pull when Ollama reports an error inside the progress stream', async () => { + const fetchMock = vi.fn().mockResolvedValueOnce(new Response([ + JSON.stringify({ status: 'pulling manifest' }), + JSON.stringify({ error: 'pull model manifest: file does not exist' }), + '', + ].join('\n'), { status: 200, headers: { 'content-type': 'application/x-ndjson' } })); + vi.stubGlobal('fetch', fetchMock); + const server = await buildPullServer([]); + try { + const response = await server.inject({ + method: 'POST', + url: '/api/local-inference/pull', + payload: { model: 'qwen3:1.7b' }, + }); + expect(response.statusCode).toBe(502); + expect(response.json().error).toContain('pull model manifest: file does not exist'); + expect(fetchMock).toHaveBeenCalledTimes(1); + } finally { + await server.close(); + vi.unstubAllGlobals(); + } + }); + it('canonicalizes the bare digest advertised by the live Ollama tag API', async () => { const wireDigest = 'B'.repeat(64); const digest = `sha256:${'b'.repeat(64)}`;