fix(server): stream Ollama model pulls so long downloads finish
A non-streaming /api/pull holds the response headers until the download ends, and Node fetch gives up on headers after five minutes, cancelling every larger model pull. Request stream:true and read the NDJSON progress, failing on an in-stream error or a missing success status (UF-008).
This commit is contained in:
@@ -70,6 +70,38 @@ function isValidOllamaModelRef(model: string): boolean {
|
|||||||
return repository.split('/').every((segment) => segment.length > 0 && segment !== '.' && segment !== '..');
|
return repository.split('/').every((segment) => segment.length > 0 && segment !== '.' && segment !== '..');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
class OllamaPullError extends Error {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reads Ollama's NDJSON pull progress to the end and returns the final status.
|
||||||
|
* Throws OllamaPullError when Ollama reports an error line or the stream ends
|
||||||
|
* without "success".
|
||||||
|
*/
|
||||||
|
async function readOllamaPullStream(res: Response): Promise<Record<string, unknown>> {
|
||||||
|
if (!res.body) throw new OllamaPullError('Ollama returned an empty pull response');
|
||||||
|
const reader = res.body.getReader();
|
||||||
|
const decoder = new TextDecoder();
|
||||||
|
let buffered = '';
|
||||||
|
let last: Record<string, unknown> | null = null;
|
||||||
|
for (;;) {
|
||||||
|
const chunk = await reader.read();
|
||||||
|
buffered += chunk.done ? decoder.decode() : decoder.decode(chunk.value, { stream: true });
|
||||||
|
const lines = buffered.split('\n');
|
||||||
|
buffered = chunk.done ? '' : (lines.pop() ?? '');
|
||||||
|
for (const line of lines) {
|
||||||
|
if (!line.trim()) continue;
|
||||||
|
const status = JSON.parse(line) as Record<string, unknown>;
|
||||||
|
if (typeof status.error === 'string') throw new OllamaPullError(status.error);
|
||||||
|
last = status;
|
||||||
|
}
|
||||||
|
if (chunk.done) break;
|
||||||
|
}
|
||||||
|
if (!last || last.status !== 'success') {
|
||||||
|
throw new OllamaPullError('the pull stream ended before Ollama reported success');
|
||||||
|
}
|
||||||
|
return last;
|
||||||
|
}
|
||||||
|
|
||||||
export interface LocalInferenceRuntimeController {
|
export interface LocalInferenceRuntimeController {
|
||||||
getStatus(): ManagedOllamaStatus;
|
getStatus(): ManagedOllamaStatus;
|
||||||
ensureReady(): Promise<ManagedOllamaReadyResult>;
|
ensureReady(): Promise<ManagedOllamaReadyResult>;
|
||||||
@@ -327,14 +359,23 @@ export async function localInferenceRoutes(
|
|||||||
const res = await fetch(`${OLLAMA_URL}/api/pull`, {
|
const res = await fetch(`${OLLAMA_URL}/api/pull`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { 'Content-Type': 'application/json' },
|
headers: { 'Content-Type': 'application/json' },
|
||||||
body: JSON.stringify({ name: model, stream: false }),
|
// Stream progress. With stream:false Ollama sends no headers until the
|
||||||
|
// download completes, and Node's fetch aborts after 300 s without
|
||||||
|
// headers, which cancelled every pull longer than five minutes.
|
||||||
|
body: JSON.stringify({ name: model, stream: true }),
|
||||||
signal: AbortSignal.timeout(45 * 60_000),
|
signal: AbortSignal.timeout(45 * 60_000),
|
||||||
});
|
});
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const text = await res.text();
|
const text = await res.text();
|
||||||
return reply.code(502).send({ error: `Ollama pull failed: ${text}` });
|
return reply.code(502).send({ error: `Ollama pull failed: ${text}` });
|
||||||
}
|
}
|
||||||
const pullStatus = await res.json();
|
let pullStatus: Record<string, unknown>;
|
||||||
|
try {
|
||||||
|
pullStatus = await readOllamaPullStream(res);
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof OllamaPullError)) throw error;
|
||||||
|
return reply.code(502).send({ error: `Ollama pull failed: ${error.message}` });
|
||||||
|
}
|
||||||
const installed = await probeOllama(OLLAMA_URL);
|
const installed = await probeOllama(OLLAMA_URL);
|
||||||
const installedModel = installed.models.find((name) => name === model || name === `${model}:latest`);
|
const installedModel = installed.models.find((name) => name === model || name === `${model}:latest`);
|
||||||
if (!installedModel) {
|
if (!installedModel) {
|
||||||
|
|||||||
@@ -732,6 +732,70 @@ describe('local-inference route — verified model installation', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('streams the pull so downloads longer than the fetch header timeout survive', async () => {
|
||||||
|
// With stream:false Ollama sends no response headers until the download
|
||||||
|
// finishes, and Node's fetch gives up after 300 s without headers, which
|
||||||
|
// cancelled every pull longer than five minutes.
|
||||||
|
const fetchMock = vi.fn(async (input: string | URL | Request) => {
|
||||||
|
const url = String(input);
|
||||||
|
if (url.endsWith('/api/pull')) {
|
||||||
|
return new Response([
|
||||||
|
JSON.stringify({ status: 'pulling manifest' }),
|
||||||
|
JSON.stringify({ status: 'pulling sha256:abc', total: 100, completed: 50 }),
|
||||||
|
JSON.stringify({ status: 'success' }),
|
||||||
|
'',
|
||||||
|
].join('\n'), { status: 200, headers: { 'content-type': 'application/x-ndjson' } });
|
||||||
|
}
|
||||||
|
return new Response(JSON.stringify({ response: 'OK', done: true }), { status: 200 });
|
||||||
|
});
|
||||||
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
|
const server = await buildPullServer(['qwen3:1.7b']);
|
||||||
|
try {
|
||||||
|
const response = await server.inject({
|
||||||
|
method: 'POST',
|
||||||
|
url: '/api/local-inference/pull',
|
||||||
|
payload: { model: 'qwen3:1.7b' },
|
||||||
|
});
|
||||||
|
expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toMatchObject({
|
||||||
|
name: 'qwen3:1.7b',
|
||||||
|
stream: true,
|
||||||
|
});
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
expect(response.json()).toMatchObject({
|
||||||
|
ok: true,
|
||||||
|
model: 'qwen3:1.7b',
|
||||||
|
status: { status: 'success' },
|
||||||
|
verifiedGeneration: true,
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
await server.close();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it('fails the pull when Ollama reports an error inside the progress stream', async () => {
|
||||||
|
const fetchMock = vi.fn().mockResolvedValueOnce(new Response([
|
||||||
|
JSON.stringify({ status: 'pulling manifest' }),
|
||||||
|
JSON.stringify({ error: 'pull model manifest: file does not exist' }),
|
||||||
|
'',
|
||||||
|
].join('\n'), { status: 200, headers: { 'content-type': 'application/x-ndjson' } }));
|
||||||
|
vi.stubGlobal('fetch', fetchMock);
|
||||||
|
const server = await buildPullServer([]);
|
||||||
|
try {
|
||||||
|
const response = await server.inject({
|
||||||
|
method: 'POST',
|
||||||
|
url: '/api/local-inference/pull',
|
||||||
|
payload: { model: 'qwen3:1.7b' },
|
||||||
|
});
|
||||||
|
expect(response.statusCode).toBe(502);
|
||||||
|
expect(response.json().error).toContain('pull model manifest: file does not exist');
|
||||||
|
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||||
|
} finally {
|
||||||
|
await server.close();
|
||||||
|
vi.unstubAllGlobals();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
it('canonicalizes the bare digest advertised by the live Ollama tag API', async () => {
|
it('canonicalizes the bare digest advertised by the live Ollama tag API', async () => {
|
||||||
const wireDigest = 'B'.repeat(64);
|
const wireDigest = 'B'.repeat(64);
|
||||||
const digest = `sha256:${'b'.repeat(64)}`;
|
const digest = `sha256:${'b'.repeat(64)}`;
|
||||||
|
|||||||
Reference in New Issue
Block a user