Files
waggle-os/packages/server/src/local/managed-ollama-runtime.ts
shumeiko.it 55f42a1077 feat(server): run managed Ollama on Linux and skip CUDA downloads on install (P1.3b)
- Pinned Ollama 0.32.3 (and rollback 0.32.0) Linux x64/arm64 artifacts with
  their published SHA-256 and sizes; .tar.zst archives are decompressed by
  Node 22's zstd and unpacked by the system tar.
- install.sh sets ONNXRUNTIME_NODE_INSTALL_CUDA=skip: onnxruntime-node
  otherwise downloads CUDA libraries on every Linux x64 install.
- Drop the root sqlite-vec-windows-x64 dependency; the sqlite-vec package
  already carries per-platform binaries.
2026-10-24 12:04:18 +02:00

1812 lines
69 KiB
TypeScript

import Database from 'better-sqlite3';
import type { Database as DatabaseType } from 'better-sqlite3';
import { createHash, randomUUID } from 'node:crypto';
import { spawn, execFile, type ChildProcess, type SpawnOptions } from 'node:child_process';
import {
chmodSync,
closeSync,
createReadStream,
createWriteStream,
existsSync,
fsyncSync,
openSync,
readFileSync,
realpathSync,
statSync,
writeFileSync,
} from 'node:fs';
import { mkdir, readdir, rename, rm } from 'node:fs/promises';
import { createServer } from 'node:net';
import path from 'node:path';
import { Readable, Transform } from 'node:stream';
import { pipeline } from 'node:stream/promises';
import { promisify } from 'node:util';
import zlib from 'node:zlib';
import { createLogger } from './logger.js';
const execFileAsync = promisify(execFile);
const log = createLogger('managed-ollama');
const OLLAMA_TARGET_VERSION = '0.32.3';
const OLLAMA_ROLLBACK_VERSION = '0.32.0';
const ollamaReleaseRoot = (version: string): string =>
`https://github.com/ollama/ollama/releases/download/v${version}`;
const DOWNLOAD_TIMEOUT_MS = 45 * 60_000;
const DOWNLOAD_MAX_ATTEMPTS = 3;
const DOWNLOAD_RETRY_DELAYS_MS = [250, 1_000] as const;
const START_TIMEOUT_MS = 45_000;
const WINDOWS_START_TIMEOUT_MS = 120_000;
const START_DIAGNOSTIC_RAW_TAIL_BYTES = 64 * 1024;
const START_DIAGNOSTIC_TAIL_BYTES = 4 * 1024;
const START_DIAGNOSTIC_DRAIN_MS = 250;
const STOP_TIMEOUT_MS = 5_000;
const WATCHDOG_EXIT_TERMINATION_CONFIRMED = 0;
const WATCHDOG_EXIT_CLEANUP_UNCONFIRMED = 1;
const WATCHDOG_EXIT_OWNED_DAEMON_EXITED = 2;
const INSTALL_LOCK_WAIT_MS = DOWNLOAD_TIMEOUT_MS + 15 * 60_000;
const INSTALL_LOCK_BUSY_TIMEOUT_MS = 0;
const INSTALL_LOCK_RETRY_MIN_MS = 200;
const INSTALL_LOCK_RETRY_MAX_MS = 1_000;
const TRANSIENT_DOWNLOAD_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
class PermanentManagedRuntimeDownloadError extends Error {}
/**
* The desktop shell terminates the Node sidecar directly on exit, which bypasses
* Fastify's onClose hooks on Windows. Keep the model daemon behind a tiny Node
* supervisor connected to the sidecar over IPC: loss of that IPC handle means
* the parent died, so the supervisor terminates the entire Ollama process tree.
*/
export const MANAGED_OLLAMA_WATCHDOG_SOURCE = String.raw`
const { spawn } = require('node:child_process');
const executable = process.argv[1];
const args = JSON.parse(process.argv[2] || '[]');
const child = spawn(executable, args, {
env: process.env,
windowsHide: true,
shell: false,
stdio: ['ignore', 'pipe', 'pipe'],
detached: process.platform !== 'win32',
});
process.stdout.on('error', () => {});
process.stderr.on('error', () => {});
child.stdout?.on('error', () => {});
child.stderr?.on('error', () => {});
child.stdout?.pipe(process.stdout);
child.stderr?.pipe(process.stderr);
let stopping = false;
let forceTimer;
let drainTimer;
let outputClosed = false;
let pendingFinishCode;
const finish = (code) => {
if (forceTimer) clearTimeout(forceTimer);
if (drainTimer) clearTimeout(drainTimer);
process.exit(code);
};
const finishAfterOutput = (code) => {
if (outputClosed) return finish(code);
if (pendingFinishCode === undefined) pendingFinishCode = code;
if (!drainTimer) {
drainTimer = setTimeout(() => finish(pendingFinishCode), ${START_DIAGNOSTIC_DRAIN_MS});
drainTimer.unref();
}
};
const stopTree = () => {
if (stopping) return;
stopping = true;
if (!child.pid) return finish(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED});
forceTimer = setTimeout(() => finish(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}), 4000);
if (process.platform === 'win32') {
const killer = spawn('taskkill', ['/PID', String(child.pid), '/T', '/F'], {
windowsHide: true,
shell: false,
stdio: 'ignore',
});
killer.once('error', () => finishAfterOutput(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}));
killer.once('exit', (code) => finishAfterOutput(code === 0
? ${WATCHDOG_EXIT_TERMINATION_CONFIRMED}
: ${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}));
} else {
try { process.kill(-child.pid, 'SIGTERM'); } catch { try { child.kill(); } catch {} }
setTimeout(() => {
try { process.kill(-child.pid, 'SIGKILL'); } catch {}
setTimeout(() => {
try {
process.kill(-child.pid, 0);
finish(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED});
} catch (error) {
finish(error && error.code === 'ESRCH'
? ${WATCHDOG_EXIT_TERMINATION_CONFIRMED}
: ${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED});
}
}, 100).unref();
}, 1500).unref();
}
};
child.once('spawn', () => {
if (process.send) process.send({ type: 'spawned', pid: child.pid });
});
child.once('error', () => finishAfterOutput(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}));
child.once('exit', () => {
if (!stopping) finishAfterOutput(${WATCHDOG_EXIT_OWNED_DAEMON_EXITED});
});
child.once('close', () => {
outputClosed = true;
if (pendingFinishCode !== undefined) finish(pendingFinishCode);
});
process.once('disconnect', stopTree);
process.on('message', (message) => { if (message === 'shutdown') stopTree(); });
process.once('SIGTERM', stopTree);
process.once('SIGINT', stopTree);
`;
export interface OllamaRuntimeArtifact {
version: string;
platform: NodeJS.Platform;
arch: string;
filename: string;
url: string;
sha256: string;
sizeBytes: number;
executableName: string;
}
/**
* Pinned official standalone artifacts. Updating this table is an intentional
* supply-chain change: version, byte size, and GitHub-published SHA-256 must move
* together. The runtime is downloaded only from this hardcoded release URL.
*/
export const OLLAMA_RUNTIME_ARTIFACTS: ReadonlyArray<OllamaRuntimeArtifact> = [
{
version: OLLAMA_ROLLBACK_VERSION,
platform: 'win32',
arch: 'x64',
filename: 'ollama-windows-amd64.zip',
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-windows-amd64.zip`,
sha256: '56561a8f0a904483303c610e61af61c5a7b6f5496ce3707e207d25d4ff67b89e',
sizeBytes: 1_503_047_573,
executableName: 'ollama.exe',
},
{
version: OLLAMA_ROLLBACK_VERSION,
platform: 'win32',
arch: 'arm64',
filename: 'ollama-windows-arm64.zip',
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-windows-arm64.zip`,
sha256: '82b7d36b63e62a44d3f9853c2f8edb829cf871eaf722ce20070e09e96922c0cc',
sizeBytes: 16_346_970,
executableName: 'ollama.exe',
},
{
version: OLLAMA_ROLLBACK_VERSION,
platform: 'darwin',
arch: 'x64',
filename: 'ollama-darwin.tgz',
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-darwin.tgz`,
sha256: '3b12a49c6c4cbafd7ffba5ccba60cbf80274cdc22eea3ead79c646aba888174c',
sizeBytes: 145_356_966,
executableName: 'ollama',
},
{
version: OLLAMA_ROLLBACK_VERSION,
platform: 'darwin',
arch: 'arm64',
filename: 'ollama-darwin.tgz',
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-darwin.tgz`,
sha256: '3b12a49c6c4cbafd7ffba5ccba60cbf80274cdc22eea3ead79c646aba888174c',
sizeBytes: 145_356_966,
executableName: 'ollama',
},
{
version: OLLAMA_ROLLBACK_VERSION,
platform: 'linux',
arch: 'x64',
filename: 'ollama-linux-amd64.tar.zst',
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-linux-amd64.tar.zst`,
sha256: '56362d7609dfa9e35aaebb7c9cab25605d8f0528ec3d5d585dc83d6642002bab',
sizeBytes: 1_436_128_693,
executableName: 'ollama',
},
{
version: OLLAMA_ROLLBACK_VERSION,
platform: 'linux',
arch: 'arm64',
filename: 'ollama-linux-arm64.tar.zst',
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-linux-arm64.tar.zst`,
sha256: '2c1fbc47a5351c74f5400d7e4b1104bb470291af5d2f425c37c151487a477ad6',
sizeBytes: 1_556_234_636,
executableName: 'ollama',
},
{
version: OLLAMA_TARGET_VERSION,
platform: 'win32',
arch: 'x64',
filename: 'ollama-windows-amd64.zip',
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-windows-amd64.zip`,
sha256: 'c66dd7dde4d5ec4822eaa57dd421d51aa7c633a3ff36a974040837df73a5969e',
sizeBytes: 1_457_806_156,
executableName: 'ollama.exe',
},
{
version: OLLAMA_TARGET_VERSION,
platform: 'win32',
arch: 'arm64',
filename: 'ollama-windows-arm64.zip',
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-windows-arm64.zip`,
sha256: '8431fc4ccf7e86a273dbcc52e81585fc92133e4d6f9c1abdd3663e2568f2fe90',
sizeBytes: 209_405_705,
executableName: 'ollama.exe',
},
{
version: OLLAMA_TARGET_VERSION,
platform: 'darwin',
arch: 'x64',
filename: 'ollama-darwin.tgz',
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-darwin.tgz`,
sha256: '14462bd438815eb2c1d4c61224744637131ab744e858a2e2562e7fc7fc2c4f7d',
sizeBytes: 145_790_989,
executableName: 'ollama',
},
{
version: OLLAMA_TARGET_VERSION,
platform: 'darwin',
arch: 'arm64',
filename: 'ollama-darwin.tgz',
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-darwin.tgz`,
sha256: '14462bd438815eb2c1d4c61224744637131ab744e858a2e2562e7fc7fc2c4f7d',
sizeBytes: 145_790_989,
executableName: 'ollama',
},
{
version: OLLAMA_TARGET_VERSION,
platform: 'linux',
arch: 'x64',
filename: 'ollama-linux-amd64.tar.zst',
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-linux-amd64.tar.zst`,
sha256: '2597d74fbe654ef6a37db56f771cf37d4a85c6bde4018127874e3927d3113800',
sizeBytes: 1_421_853_873,
executableName: 'ollama',
},
{
version: OLLAMA_TARGET_VERSION,
platform: 'linux',
arch: 'arm64',
filename: 'ollama-linux-arm64.tar.zst',
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-linux-arm64.tar.zst`,
sha256: '61af0def977ffc9e76cc961f9505d621805653c63f444796de2f77b4eaaa7047',
sizeBytes: 1_542_500_407,
executableName: 'ollama',
}
];
interface InstallMetadata {
version: string;
sha256: string;
executable: string;
installedAt: string;
}
interface RuntimeReceipt {
version: string;
sha256: string;
executable: string;
installedAt: string;
}
interface RuntimeActivationState {
schemaVersion: 1;
active: RuntimeReceipt;
previous: RuntimeReceipt | null;
lastRollback: ManagedRuntimeRollback | null;
}
type RuntimeStateLoad =
| { kind: 'missing' }
| {
kind: 'invalid';
reason: string;
recoveryArtifact?: OllamaRuntimeArtifact;
recoveryReceipt?: RuntimeReceipt;
lastRollback?: ManagedRuntimeRollback | null;
}
| {
kind: 'valid';
state: RuntimeActivationState;
activeArtifact: OllamaRuntimeArtifact;
previousArtifact: OllamaRuntimeArtifact | null;
};
interface InstallLockClaim {
database: DatabaseType;
}
export interface ManagedOllamaStatus {
source: 'waggle-managed';
supported: boolean;
installed: boolean;
running: boolean;
targetVersion: string | null;
version: string | null;
targetInstalled: boolean;
activeVersion: string | null;
previousVersion: string | null;
fallbackActive: boolean;
rollback: {
available: boolean;
active: boolean;
lastAttempt: ManagedRuntimeRollback | null;
};
artifactSizeBytes: number | null;
downloadRequired: boolean;
dockerRequired: false;
reason?: string;
}
export interface ManagedRuntimeRollback {
failedVersion: string;
restoredVersion: string;
occurredAt: string;
reason: string;
}
export class ManagedRuntimeRollbackError extends Error {
readonly rollback: ManagedRuntimeRollback;
readonly failedVersion: string;
readonly restoredVersion: string;
constructor(rollback: ManagedRuntimeRollback, cause: unknown) {
super(
`Managed Ollama ${rollback.failedVersion} failed; restored verified runtime ${rollback.restoredVersion}: ${rollback.reason}`,
{ cause },
);
this.name = 'ManagedRuntimeRollbackError';
this.rollback = rollback;
this.failedVersion = rollback.failedVersion;
this.restoredVersion = rollback.restoredVersion;
}
}
class ManagedRuntimeTerminationError extends Error {
constructor(reason = 'watchdog did not confirm process-tree termination') {
super(`Managed Ollama ${reason}; refusing to start a replacement runtime`);
this.name = 'ManagedRuntimeTerminationError';
}
}
export interface ManagedOllamaReadyResult {
installedNow: boolean;
startedNow: boolean;
endpoint: string;
status: ManagedOllamaStatus;
}
interface RuntimeDependencies {
fetchImpl?: typeof fetch;
extractArchive?: (
archivePath: string,
destination: string,
artifact: OllamaRuntimeArtifact,
) => Promise<void>;
spawnImpl?: (file: string, args: readonly string[], options: SpawnOptions) => ChildProcess;
probe?: (baseUrl: string) => Promise<boolean>;
endpointQuiescent?: (baseUrl: string) => Promise<boolean>;
renameImpl?: typeof rename;
stateRenameImpl?: typeof rename;
rmImpl?: typeof rm;
warn?: (message: string, error: unknown) => void;
artifact?: OllamaRuntimeArtifact | null;
artifactCatalog?: ReadonlyArray<OllamaRuntimeArtifact>;
platform?: NodeJS.Platform;
arch?: string;
}
export function resolveOllamaRuntimeArtifact(
platform: NodeJS.Platform = process.platform,
arch: string = process.arch,
): OllamaRuntimeArtifact | null {
const targets = OLLAMA_RUNTIME_ARTIFACTS.filter(
(entry) => entry.platform === platform
&& entry.arch === arch
&& entry.version === OLLAMA_TARGET_VERSION,
);
return targets.length === 1 ? targets[0] : null;
}
function isLoopbackEndpoint(baseUrl: string): boolean {
try {
const url = new URL(baseUrl);
return url.protocol === 'http:'
&& (url.hostname === '127.0.0.1' || url.hostname === 'localhost' || url.hostname === '::1')
&& (url.pathname === '/' || url.pathname === '');
} catch {
return false;
}
}
export function buildManagedOllamaEnv(
baseUrl: string,
modelsDir: string,
source: NodeJS.ProcessEnv = process.env,
managedProfileDir?: string,
): NodeJS.ProcessEnv {
if (!isLoopbackEndpoint(baseUrl)) {
throw new Error('Waggle-managed Ollama must bind to an HTTP loopback endpoint');
}
const endpoint = new URL(baseUrl);
return {
...source,
...(managedProfileDir
? { HOME: managedProfileDir, USERPROFILE: managedProfileDir }
: {}),
OLLAMA_HOST: endpoint.host,
OLLAMA_MODELS: modelsDir,
OLLAMA_NOHISTORY: '1',
// Ollama otherwise defaults to 4K on common 8 GB GPUs. The qualified
// 8K-token tool request plus Waggle's 14K synthesis ceiling needs a
// 32K window to retain the full input and leave the promised output room.
OLLAMA_CONTEXT_LENGTH: '32768',
};
}
async function defaultProbe(baseUrl: string): Promise<boolean> {
try {
const response = await fetch(`${baseUrl.replace(/\/+$/, '')}/api/tags`, {
signal: AbortSignal.timeout(3_000),
});
if (!response.ok) return false;
const payload = await response.json() as { models?: unknown };
return Array.isArray(payload.models);
} catch {
return false;
}
}
async function defaultExtract(
archivePath: string,
destination: string,
artifact: OllamaRuntimeArtifact,
): Promise<void> {
if (artifact.filename.endsWith('.tar.zst')) {
await extractTarZst(archivePath, destination);
return;
}
const executable = artifact.platform === 'win32' ? 'tar.exe' : 'tar';
await execFileAsync(executable, ['-xf', archivePath, '-C', destination], {
windowsHide: true,
timeout: 10 * 60_000,
});
}
/**
* Linux releases are zstd-compressed tarballs. Node 22 decompresses zstd itself
* and the system tar only unpacks; on older Node, tar needs zstd support.
*/
export async function extractTarZst(archivePath: string, destination: string): Promise<void> {
const createZstdDecompress = (zlib as { createZstdDecompress?: () => Transform }).createZstdDecompress;
if (!createZstdDecompress) {
await execFileAsync('tar', ['--zstd', '-xf', archivePath, '-C', destination], { timeout: 10 * 60_000 });
return;
}
const tar = spawn('tar', ['-xf', '-', '-C', destination], { stdio: ['pipe', 'ignore', 'pipe'] });
let stderr = '';
tar.stderr?.on('data', (chunk: Buffer) => { stderr += chunk.toString(); });
const exited = new Promise<void>((resolve, reject) => {
tar.once('error', reject);
tar.once('close', (code) => (code === 0 ? resolve() : reject(new Error(`tar exited with ${code}: ${stderr.trim()}`))));
});
await Promise.all([pipeline(createReadStream(archivePath), createZstdDecompress(), tar.stdin!), exited]);
}
function inside(root: string, candidate: string): boolean {
const relative = path.relative(path.resolve(root), path.resolve(candidate));
return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
}
const TRANSIENT_FILESYSTEM_CODES = new Set(['EPERM', 'EACCES', 'EBUSY']);
const FILESYSTEM_ATTEMPTS = 4;
const PROCESS_INSTALLS = new Map<string, Promise<{ executable: string; installedNow: boolean }>>();
function isTransientFilesystemError(error: unknown): boolean {
return TRANSIENT_FILESYSTEM_CODES.has((error as NodeJS.ErrnoException)?.code ?? '');
}
function redactManagedRuntimeDiagnostic(value: string): string {
const redacted = value
.replace(/(authorization\s*[:=]\s*)[^\r\n]*/gi, '$1[REDACTED]')
.replace(/(https?:\/\/)[^\s/@:]+:[^\s/@]+@/gi, '$1[REDACTED]@')
.replace(
/((?:api[_-]?key|token|password|secret|credential|private[_-]?key)\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s,;\]}]+)/gi,
'$1[REDACTED]',
);
return Array.from(redacted, (character) => {
const code = character.charCodeAt(0);
const disallowedControl = (code < 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d)
|| code === 0x7f;
return disallowedControl ? '?' : character;
}).join('');
}
function appendRawDiagnosticTail(current: string, chunk: string | Buffer): string {
const next = Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk);
const combined = Buffer.from(current + next, 'utf8');
if (combined.length <= START_DIAGNOSTIC_RAW_TAIL_BYTES) return combined.toString('utf8');
return combined
.subarray(combined.length - START_DIAGNOSTIC_RAW_TAIL_BYTES)
.toString('utf8');
}
function formatDiagnosticTail(raw: string): string {
const combined = Buffer.from(redactManagedRuntimeDiagnostic(raw), 'utf8');
if (combined.length <= START_DIAGNOSTIC_TAIL_BYTES) return combined.toString('utf8');
return combined
.subarray(combined.length - START_DIAGNOSTIC_TAIL_BYTES)
.toString('utf8');
}
async function waitForDiagnosticStreams(child: ChildProcess): Promise<void> {
const streams = [child.stdout, child.stderr]
.filter((stream): stream is Readable => stream != null);
const cleanups: Array<() => void> = [];
const waits = streams.map((stream) => new Promise<void>((resolve) => {
if (stream.readableEnded || stream.destroyed) {
resolve();
return;
}
const finish = () => {
cleanup();
resolve();
};
const cleanup = () => {
stream.off('end', finish);
stream.off('close', finish);
stream.off('error', finish);
};
stream.once('end', finish);
stream.once('close', finish);
stream.once('error', finish);
cleanups.push(cleanup);
}));
if (waits.length === 0) return;
let timeout: NodeJS.Timeout | undefined;
try {
await Promise.race([
Promise.all(waits),
new Promise<void>((resolve) => {
timeout = setTimeout(resolve, START_DIAGNOSTIC_DRAIN_MS);
}),
]);
} finally {
if (timeout) clearTimeout(timeout);
for (const cleanup of cleanups) cleanup();
}
}
async function waitForFilesystemRetry(attempt: number): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, 25 * attempt));
}
async function removeWithin(root: string, target: string, rmImpl: typeof rm = rm): Promise<void> {
if (!inside(root, target) || path.resolve(root) === path.resolve(target)) {
throw new Error(`Refusing to remove path outside managed runtime root: ${target}`);
}
for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) {
try {
await rmImpl(target, { recursive: true, force: true });
return;
} catch (error) {
if (!isTransientFilesystemError(error) || attempt === FILESYSTEM_ATTEMPTS) throw error;
await waitForFilesystemRetry(attempt);
}
}
}
function isSqliteBusyError(error: unknown): boolean {
const code = (error as { code?: unknown } | null)?.code;
return code === 'SQLITE_BUSY' || code === 'SQLITE_BUSY_SNAPSHOT' || code === 'SQLITE_LOCKED';
}
async function findExecutable(root: string, executableName: string): Promise<string> {
const queue: Array<{ directory: string; depth: number }> = [{ directory: root, depth: 0 }];
let visited = 0;
while (queue.length > 0 && visited < 10_000) {
const current = queue.shift()!;
const entries = await readdir(current.directory, { withFileTypes: true });
for (const entry of entries) {
visited += 1;
const candidate = path.join(current.directory, entry.name);
if (entry.name.toLowerCase() === executableName.toLowerCase()) {
const resolved = realpathSync(candidate);
if (!inside(root, resolved) || !statSync(resolved).isFile()) continue;
return resolved;
}
if (entry.isDirectory() && current.depth < 6) {
queue.push({ directory: candidate, depth: current.depth + 1 });
}
}
}
throw new Error(`Official runtime archive did not contain ${executableName}`);
}
export class ManagedOllamaRuntime {
private readonly root: string;
private readonly modelsDir: string;
private readonly profileDir: string;
private readonly artifact: OllamaRuntimeArtifact | null;
private readonly artifactCatalog: ReadonlyArray<OllamaRuntimeArtifact>;
private readonly platform: NodeJS.Platform;
private readonly arch: string;
private readonly fetchImpl: typeof fetch;
private readonly extractArchive: NonNullable<RuntimeDependencies['extractArchive']>;
private readonly spawnImpl: NonNullable<RuntimeDependencies['spawnImpl']>;
private readonly probe: NonNullable<RuntimeDependencies['probe']>;
private readonly endpointQuiescent: NonNullable<RuntimeDependencies['endpointQuiescent']>;
private readonly renameImpl: NonNullable<RuntimeDependencies['renameImpl']>;
private readonly stateRenameImpl: NonNullable<RuntimeDependencies['stateRenameImpl']>;
private readonly rmImpl: NonNullable<RuntimeDependencies['rmImpl']>;
private readonly warn: NonNullable<RuntimeDependencies['warn']>;
private installPromise: Promise<{ executable: string; installedNow: boolean }> | null = null;
private startPromise: Promise<boolean> | null = null;
private activationPromise: Promise<unknown> | null = null;
private child: ChildProcess | null = null;
private runningArtifact: OllamaRuntimeArtifact | null = null;
private terminationFailure: ManagedRuntimeTerminationError | null = null;
private pendingCrashRecovery: OllamaRuntimeArtifact | null = null;
private crashRecoveryConsumed = false;
private stopInProgress = false;
private readonly stopRequestedChildren = new WeakSet<ChildProcess>();
constructor(
dataDir: string,
private readonly baseUrl = 'http://127.0.0.1:11434',
dependencies: RuntimeDependencies = {},
) {
const platform = dependencies.platform ?? dependencies.artifact?.platform ?? process.platform;
const arch = dependencies.arch ?? dependencies.artifact?.arch ?? process.arch;
this.platform = platform;
this.arch = arch;
this.root = path.resolve(dataDir, 'runtimes', 'ollama');
this.modelsDir = path.resolve(dataDir, 'models', 'ollama');
this.profileDir = path.resolve(this.root, 'profile');
this.artifact = dependencies.artifact === undefined
? resolveOllamaRuntimeArtifact(platform, arch)
: dependencies.artifact;
this.artifactCatalog = dependencies.artifactCatalog
?? (dependencies.artifact === undefined
? OLLAMA_RUNTIME_ARTIFACTS
: (dependencies.artifact ? [dependencies.artifact] : []));
this.fetchImpl = dependencies.fetchImpl ?? fetch;
this.extractArchive = dependencies.extractArchive ?? defaultExtract;
this.spawnImpl = dependencies.spawnImpl ?? ((file, args, options) => spawn(file, args, options));
this.probe = dependencies.probe ?? defaultProbe;
this.endpointQuiescent = dependencies.endpointQuiescent ?? defaultEndpointQuiescent;
this.renameImpl = dependencies.renameImpl ?? rename;
this.stateRenameImpl = dependencies.stateRenameImpl ?? rename;
this.rmImpl = dependencies.rmImpl ?? rm;
this.warn = dependencies.warn ?? ((message, error) => log.warn(message, {
error: error instanceof Error ? error.message : String(error),
code: (error as { code?: unknown } | null)?.code ?? null,
}));
}
getStatus(): ManagedOllamaStatus {
const executable = this.getInstalledExecutable();
const targetInstalled = executable !== null;
const loadedState = this.loadRuntimeState();
const running = this.child !== null && this.child.exitCode === null;
const managedRunningVersion = running ? this.runningArtifact?.version ?? null : null;
const activeVersion = managedRunningVersion
?? (loadedState.kind === 'valid' ? loadedState.state.active.version : null);
const previousVersion = loadedState.kind === 'valid'
? loadedState.state.previous?.version ?? null
: loadedState.kind === 'invalid'
? loadedState.recoveryArtifact?.version ?? null
: null;
const persistedRollbackActive = loadedState.kind === 'valid'
&& this.artifact !== null
&& !this.sameArtifact(loadedState.activeArtifact, this.artifact)
&& loadedState.state.lastRollback?.failedVersion === this.artifact.version
&& loadedState.state.lastRollback.restoredVersion === loadedState.activeArtifact.version;
const fallbackActive = persistedRollbackActive || (running
&& this.runningArtifact !== null
&& this.artifact !== null
&& !this.sameArtifact(this.runningArtifact, this.artifact));
const rollbackAvailable = loadedState.kind === 'valid'
&& this.selectFallbackArtifact(loadedState, this.artifact) !== null;
const supported = this.artifact !== null && isLoopbackEndpoint(this.baseUrl);
return {
source: 'waggle-managed',
supported,
installed: targetInstalled
|| loadedState.kind === 'valid'
|| (loadedState.kind === 'invalid' && loadedState.recoveryArtifact !== undefined)
|| this.runningArtifact !== null,
running,
targetVersion: this.artifact?.version ?? null,
version: activeVersion ?? (targetInstalled ? this.artifact?.version ?? null : null),
targetInstalled,
activeVersion,
previousVersion,
fallbackActive,
rollback: {
available: rollbackAvailable || fallbackActive,
active: fallbackActive,
lastAttempt: loadedState.kind === 'valid'
? loadedState.state.lastRollback
: loadedState.kind === 'invalid'
? loadedState.lastRollback ?? null
: null,
},
artifactSizeBytes: this.artifact?.sizeBytes ?? null,
downloadRequired: !targetInstalled,
dockerRequired: false,
...(!supported
? { reason: this.artifact === null
? `No managed Ollama artifact for ${process.platform}/${process.arch}`
: 'Managed Ollama requires a loopback OLLAMA_HOST' }
: loadedState.kind === 'invalid'
? { reason: loadedState.reason }
: {}),
};
}
private versionDir(artifact = this.artifact): string | null {
if (!artifact) return null;
const candidate = path.join(this.root, artifact.version);
return inside(this.root, candidate) && path.resolve(candidate) !== this.root ? candidate : null;
}
private installVersionKey(): string {
return createHash('sha256').update(this.artifact!.version).digest('hex');
}
private installLockDatabasePath(): string {
return path.join(this.root, `.install-lock-${this.installVersionKey()}.sqlite`);
}
private getVerifiedInstall(artifact = this.artifact): { executable: string; receipt: RuntimeReceipt } | null {
const versionDir = this.versionDir(artifact);
if (!versionDir) return null;
const metadataPath = path.join(versionDir, 'install.json');
try {
const metadata = JSON.parse(readFileSync(metadataPath, 'utf8')) as InstallMetadata;
if (metadata.version !== artifact?.version || metadata.sha256 !== artifact.sha256) return null;
if (path.isAbsolute(metadata.executable) || metadata.executable.split(/[\\/]/).includes('..')) return null;
if (path.basename(metadata.executable).toLowerCase() !== artifact.executableName.toLowerCase()) return null;
if (typeof metadata.installedAt !== 'string' || !Number.isFinite(Date.parse(metadata.installedAt))) return null;
const executable = realpathSync(path.join(versionDir, metadata.executable));
if (!inside(versionDir, executable) || !statSync(executable).isFile()) return null;
return {
executable,
receipt: {
version: metadata.version,
sha256: metadata.sha256,
executable: metadata.executable,
installedAt: metadata.installedAt,
},
};
} catch {
return null;
}
}
private getInstalledExecutable(artifact = this.artifact): string | null {
return this.getVerifiedInstall(artifact)?.executable ?? null;
}
private statePath(): string {
return path.join(this.root, 'runtime-state.json');
}
private receiptFor(artifact: OllamaRuntimeArtifact): RuntimeReceipt {
const installation = this.getVerifiedInstall(artifact);
if (!installation) throw new Error(`Managed Ollama ${artifact.version} install receipt failed validation`);
return installation.receipt;
}
private sameReceipt(left: RuntimeReceipt, right: RuntimeReceipt): boolean {
return left.version === right.version
&& left.sha256 === right.sha256
&& left.executable === right.executable
&& left.installedAt === right.installedAt;
}
private sameArtifact(left: OllamaRuntimeArtifact, right: OllamaRuntimeArtifact): boolean {
return left.version === right.version && left.sha256 === right.sha256;
}
private sameCatalogArtifact(left: OllamaRuntimeArtifact, right: OllamaRuntimeArtifact): boolean {
return this.sameArtifact(left, right)
&& left.platform === right.platform
&& left.arch === right.arch
&& left.filename === right.filename
&& left.url === right.url
&& left.sizeBytes === right.sizeBytes
&& left.executableName === right.executableName;
}
private assertTrustedTarget(): OllamaRuntimeArtifact {
if (!this.artifact) throw new Error(`Managed Ollama is unsupported on ${process.platform}/${process.arch}`);
const trusted = this.artifactCatalog.find((candidate) => this.sameCatalogArtifact(candidate, this.artifact!));
if (!trusted) throw new Error(`Managed Ollama ${this.artifact.version} is not present in the trusted artifact catalog`);
return trusted;
}
private catalogArtifact(receipt: RuntimeReceipt): OllamaRuntimeArtifact | null {
return this.artifactCatalog.find((candidate) => (
candidate.platform === this.platform
&& candidate.arch === this.arch
&& candidate.version === receipt.version
&& candidate.sha256 === receipt.sha256
)) ?? null;
}
private validReceipt(value: unknown): RuntimeReceipt | null {
if (!value || typeof value !== 'object') return null;
const candidate = value as Partial<RuntimeReceipt>;
if (typeof candidate.version !== 'string' || candidate.version.length === 0 || candidate.version.length > 128) return null;
if (typeof candidate.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(candidate.sha256)) return null;
if (typeof candidate.executable !== 'string' || candidate.executable.length === 0 || candidate.executable.length > 1_024) return null;
if (path.isAbsolute(candidate.executable) || candidate.executable.split(/[\\/]/).includes('..')) return null;
if (typeof candidate.installedAt !== 'string' || !Number.isFinite(Date.parse(candidate.installedAt))) return null;
return {
version: candidate.version,
sha256: candidate.sha256,
executable: candidate.executable,
installedAt: candidate.installedAt,
};
}
private validRollback(value: unknown): ManagedRuntimeRollback | null | undefined {
if (value === null) return null;
if (!value || typeof value !== 'object') return undefined;
const candidate = value as Partial<ManagedRuntimeRollback>;
if (typeof candidate.failedVersion !== 'string' || candidate.failedVersion.length === 0 || candidate.failedVersion.length > 128) return undefined;
if (typeof candidate.restoredVersion !== 'string' || candidate.restoredVersion.length === 0 || candidate.restoredVersion.length > 128) return undefined;
if (typeof candidate.occurredAt !== 'string' || !Number.isFinite(Date.parse(candidate.occurredAt))) return undefined;
if (typeof candidate.reason !== 'string' || candidate.reason.length === 0 || candidate.reason.length > 2_048) return undefined;
return {
failedVersion: candidate.failedVersion,
restoredVersion: candidate.restoredVersion,
occurredAt: candidate.occurredAt,
reason: candidate.reason,
};
}
private loadRuntimeState(): RuntimeStateLoad {
let parsed: unknown;
try {
parsed = JSON.parse(readFileSync(this.statePath(), 'utf8'));
} catch (error) {
if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') return { kind: 'missing' };
return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation' };
}
if (!parsed || typeof parsed !== 'object') {
return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation' };
}
const candidate = parsed as Partial<RuntimeActivationState>;
if (candidate.schemaVersion !== 1) {
return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation' };
}
const parsedRollback = this.validRollback(candidate.lastRollback);
const lastRollback = parsedRollback === undefined ? null : parsedRollback;
const previous = candidate.previous === null ? null : this.validReceipt(candidate.previous);
const previousArtifact = previous ? this.catalogArtifact(previous) : null;
const previousInstall = previousArtifact ? this.getVerifiedInstall(previousArtifact) : null;
const trustedPrevious = previous && previousArtifact && previousInstall
&& this.sameReceipt(previous, previousInstall.receipt)
? { receipt: previous, artifact: previousArtifact }
: null;
const recovery = trustedPrevious
? {
recoveryArtifact: trustedPrevious.artifact,
recoveryReceipt: trustedPrevious.receipt,
lastRollback,
}
: { lastRollback };
const active = this.validReceipt(candidate.active);
if (!active) {
return {
kind: 'invalid',
reason: 'Managed Ollama activation state failed validation',
...recovery,
};
}
const activeArtifact = this.catalogArtifact(active);
const activeInstall = activeArtifact ? this.getVerifiedInstall(activeArtifact) : null;
if (!activeArtifact || !activeInstall || !this.sameReceipt(active, activeInstall.receipt)) {
return {
kind: 'invalid',
reason: 'Managed Ollama activation state references an untrusted or invalid runtime',
...recovery,
};
}
const usablePrevious = trustedPrevious
&& !this.sameArtifact(activeArtifact, trustedPrevious.artifact)
? trustedPrevious
: null;
return {
kind: 'valid',
state: {
schemaVersion: 1,
active,
previous: usablePrevious?.receipt ?? null,
lastRollback,
},
activeArtifact,
previousArtifact: usablePrevious?.artifact ?? null,
};
}
private async writeRuntimeState(state: RuntimeActivationState): Promise<void> {
await mkdir(this.root, { recursive: true });
const temporary = path.join(this.root, `.runtime-state.${process.pid}-${randomUUID()}.tmp`);
let descriptor: number | null = null;
try {
descriptor = openSync(temporary, 'wx');
writeFileSync(descriptor, `${JSON.stringify(state, null, 2)}\n`);
fsyncSync(descriptor);
closeSync(descriptor);
descriptor = null;
for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) {
try {
await this.stateRenameImpl(temporary, this.statePath());
break;
} catch (error) {
if (!isTransientFilesystemError(error) || attempt === FILESYSTEM_ATTEMPTS) throw error;
await waitForFilesystemRetry(attempt);
}
}
} finally {
if (descriptor !== null) closeSync(descriptor);
if (existsSync(temporary)) await removeWithin(this.root, temporary, this.rmImpl);
}
}
private async promoteActive(artifact: OllamaRuntimeArtifact): Promise<void> {
const loaded = this.loadRuntimeState();
const receipt = this.receiptFor(artifact);
const current = loaded.kind === 'valid' ? loaded.state : null;
await this.writeRuntimeState({
schemaVersion: 1,
active: receipt,
previous: current && !this.sameReceipt(current.active, receipt) ? current.active : current?.previous ?? null,
lastRollback: current?.lastRollback ?? null,
});
}
private selectFallbackArtifact(
loaded: Extract<RuntimeStateLoad, { kind: 'valid' }>,
failedArtifact: OllamaRuntimeArtifact | null,
): OllamaRuntimeArtifact | null {
if (!failedArtifact || !this.sameArtifact(loaded.activeArtifact, failedArtifact)) return loaded.activeArtifact;
return loaded.previousArtifact && !this.sameArtifact(loaded.previousArtifact, failedArtifact)
? loaded.previousArtifact
: null;
}
private verifiedCatalogInstalls(): Array<{ artifact: OllamaRuntimeArtifact; executable: string }> {
const seen = new Set<string>();
const installations: Array<{ artifact: OllamaRuntimeArtifact; executable: string }> = [];
for (const artifact of this.artifactCatalog) {
if (artifact.platform !== this.platform || artifact.arch !== this.arch) continue;
const key = `${artifact.version}:${artifact.sha256}`;
if (seen.has(key)) continue;
seen.add(key);
const executable = this.getInstalledExecutable(artifact);
if (executable) installations.push({ artifact, executable });
}
return installations;
}
private legacyStartCandidate(): { artifact: OllamaRuntimeArtifact; executable: string } | null {
const installations = this.verifiedCatalogInstalls();
return installations.length === 1 ? installations[0]! : null;
}
private rollbackReason(error: unknown): string {
const reason = error instanceof Error ? error.message : String(error);
return (reason || 'target runtime failed').slice(0, 2_048);
}
private async persistRollback(
failedArtifact: OllamaRuntimeArtifact,
restoredArtifact: OllamaRuntimeArtifact,
rollback: ManagedRuntimeRollback,
): Promise<void> {
const loaded = this.loadRuntimeState();
const restored = this.receiptFor(restoredArtifact);
const current = loaded.kind === 'valid' ? loaded.state : null;
const failedWasActive = current !== null
&& current.active.version === failedArtifact.version
&& current.active.sha256 === failedArtifact.sha256;
await this.writeRuntimeState({
schemaVersion: 1,
active: restored,
previous: failedWasActive ? current.active : current?.previous ?? null,
lastRollback: rollback,
});
}
private activationLockDatabasePath(): string {
return path.join(this.root, '.activation-lock.sqlite');
}
private async acquireActivationLock(): Promise<InstallLockClaim> {
await mkdir(this.root, { recursive: true });
const database = new Database(this.activationLockDatabasePath(), { timeout: INSTALL_LOCK_BUSY_TIMEOUT_MS });
const deadline = Date.now() + INSTALL_LOCK_WAIT_MS;
let retryDelay = INSTALL_LOCK_RETRY_MIN_MS;
while (Date.now() < deadline) {
try {
database.exec('BEGIN EXCLUSIVE');
return { database };
} catch (error) {
if (!isSqliteBusyError(error)) {
try { database.close(); } catch { /* primary error wins */ }
throw error;
}
}
await new Promise((resolve) => setTimeout(resolve, retryDelay));
retryDelay = Math.min(retryDelay * 2, INSTALL_LOCK_RETRY_MAX_MS);
}
try { database.close(); } catch { /* timeout error wins */ }
throw new Error('Timed out waiting for the managed Ollama activation coordinator');
}
private withActivationLock<T>(operation: () => Promise<T>): Promise<T> {
const prior = this.activationPromise;
const activation = (async () => {
if (prior) {
try {
await prior;
} catch {
// The next caller must observe the resulting state, including a healthy rollback.
}
}
const lock = await this.acquireActivationLock();
try {
return await operation();
} finally {
await this.releaseInstallLock(lock);
}
})();
this.activationPromise = activation;
return activation.finally(() => {
if (this.activationPromise === activation) this.activationPromise = null;
});
}
async install(): Promise<{ executable: string; installedNow: boolean }> {
const existing = this.getInstalledExecutable();
if (existing) return { executable: existing, installedNow: false };
if (this.installPromise) return this.installPromise;
const coordinationKey = this.installLockDatabasePath();
const processInstall = PROCESS_INSTALLS.get(coordinationKey);
if (processInstall) {
const result = await processInstall;
return { executable: result.executable, installedNow: false };
}
const installation = this.installInternal();
PROCESS_INSTALLS.set(coordinationKey, installation);
this.installPromise = installation.finally(() => {
if (PROCESS_INSTALLS.get(coordinationKey) === installation) PROCESS_INSTALLS.delete(coordinationKey);
this.installPromise = null;
});
return this.installPromise;
}
private reportWarning(message: string, error: unknown): void {
try {
this.warn(message, error);
} catch {
// Diagnostics must never change runtime availability.
}
}
private async acquireInstallLock(): Promise<InstallLockClaim> {
const database = new Database(this.installLockDatabasePath(), { timeout: INSTALL_LOCK_BUSY_TIMEOUT_MS });
const deadline = Date.now() + INSTALL_LOCK_WAIT_MS;
let retryDelay = INSTALL_LOCK_RETRY_MIN_MS;
while (Date.now() < deadline) {
try {
database.exec('BEGIN EXCLUSIVE');
return { database };
} catch (error) {
if (!isSqliteBusyError(error)) {
try {
database.close();
} catch (closeError) {
this.reportWarning('Could not close the managed Ollama install coordinator', closeError);
}
throw error;
}
}
const jitter = Math.floor(Math.random() * Math.min(100, retryDelay / 4));
await new Promise((resolve) => setTimeout(resolve, retryDelay + jitter));
retryDelay = Math.min(retryDelay * 2, INSTALL_LOCK_RETRY_MAX_MS);
}
try {
database.close();
} catch (error) {
this.reportWarning('Could not close the timed-out managed Ollama install coordinator', error);
}
throw new Error(`Timed out waiting for the managed Ollama ${this.artifact!.version} installation coordinator`);
}
private async releaseInstallLock(claim: InstallLockClaim): Promise<void> {
try {
if (claim.database.inTransaction) claim.database.exec('ROLLBACK');
} catch (error) {
this.reportWarning('Could not roll back the managed Ollama install coordinator', error);
} finally {
try {
claim.database.close();
} catch (error) {
this.reportWarning('Could not close the managed Ollama install coordinator', error);
}
}
}
private async cleanupInstallAttempt(targets: readonly string[]): Promise<void> {
for (const target of targets) {
try {
await removeWithin(this.root, target, this.rmImpl);
} catch (error) {
this.reportWarning(`Could not remove managed Ollama install artifact ${path.basename(target)}`, error);
}
}
}
private async removeOrphanedInstallAttempts(): Promise<void> {
const versionKey = this.installVersionKey();
const stagingPrefix = `.install-attempt-${versionKey}.`;
const archivePrefix = `.${this.artifact!.filename}.${versionKey}.`;
const entries = await readdir(this.root, { withFileTypes: true });
for (const entry of entries) {
const isStaging = entry.name.startsWith(stagingPrefix);
const isArchive = entry.name.startsWith(archivePrefix) && entry.name.endsWith('.part');
if (!isStaging && !isArchive) continue;
await removeWithin(this.root, path.join(this.root, entry.name), this.rmImpl);
}
}
private async downloadArtifact(archive: string): Promise<void> {
const artifact = this.artifact!;
const deadline = Date.now() + DOWNLOAD_TIMEOUT_MS;
for (let attempt = 1; attempt <= DOWNLOAD_MAX_ATTEMPTS; attempt += 1) {
let offset = existsSync(archive) ? statSync(archive).size : 0;
if (offset > artifact.sizeBytes) {
throw new PermanentManagedRuntimeDownloadError(
`Official Ollama runtime exceeded the expected ${artifact.sizeBytes} bytes`,
);
}
try {
if (offset < artifact.sizeBytes) {
const remainingTime = deadline - Date.now();
if (remainingTime <= 0) throw new Error('Official Ollama runtime download timed out');
const response = await this.fetchImpl(artifact.url, {
redirect: 'follow',
...(offset > 0 ? { headers: { Range: `bytes=${offset}-` } } : {}),
signal: AbortSignal.timeout(remainingTime),
});
if (!response.ok || !response.body) {
const message = `Official Ollama runtime download failed with HTTP ${response.status}`;
if (!TRANSIENT_DOWNLOAD_STATUS.has(response.status)) {
throw new PermanentManagedRuntimeDownloadError(message);
}
throw new Error(message);
}
if (offset > 0 && response.status === 200) {
offset = 0;
} else if (offset > 0) {
const expectedRange = `bytes ${offset}-${artifact.sizeBytes - 1}/${artifact.sizeBytes}`;
if (response.status !== 206 || response.headers.get('content-range') !== expectedRange) {
throw new PermanentManagedRuntimeDownloadError(
'Official Ollama runtime returned an invalid resume range',
);
}
} else if (response.status !== 200) {
throw new PermanentManagedRuntimeDownloadError(
`Official Ollama runtime returned unexpected HTTP ${response.status}`,
);
}
const expectedResponseBytes = artifact.sizeBytes - offset;
const declaredSize = Number(response.headers.get('content-length'));
if (Number.isFinite(declaredSize) && declaredSize !== expectedResponseBytes) {
throw new PermanentManagedRuntimeDownloadError(
`Official Ollama runtime size mismatch: expected ${expectedResponseBytes}, got ${declaredSize}`,
);
}
let downloaded = offset;
const verifier = new Transform({
transform(chunk: Buffer, _encoding, callback) {
downloaded += chunk.length;
if (downloaded > artifact.sizeBytes) {
callback(new PermanentManagedRuntimeDownloadError(
`Official Ollama runtime exceeded the expected ${artifact.sizeBytes} bytes`,
));
return;
}
callback(null, chunk);
},
});
const body = Readable.fromWeb(response.body as import('node:stream/web').ReadableStream);
await pipeline(
body,
verifier,
createWriteStream(archive, { flags: offset > 0 ? 'a' : 'w' }),
);
if (downloaded !== artifact.sizeBytes || statSync(archive).size !== artifact.sizeBytes) {
throw new Error(
`Official Ollama runtime size mismatch: expected ${artifact.sizeBytes}, got ${statSync(archive).size}`,
);
}
}
const digest = createHash('sha256');
for await (const chunk of createReadStream(archive)) digest.update(chunk as Buffer);
if (digest.digest('hex') !== artifact.sha256) {
throw new PermanentManagedRuntimeDownloadError(
'Official Ollama runtime checksum verification failed',
);
}
return;
} catch (error) {
if (error instanceof PermanentManagedRuntimeDownloadError
|| attempt === DOWNLOAD_MAX_ATTEMPTS) throw error;
await new Promise((resolve) => setTimeout(
resolve,
DOWNLOAD_RETRY_DELAYS_MS[attempt - 1] ?? DOWNLOAD_RETRY_DELAYS_MS.at(-1)!,
));
}
}
}
private async installInternal(): Promise<{ executable: string; installedNow: boolean }> {
this.assertTrustedTarget();
if (!this.artifact) throw new Error(`Managed Ollama is unsupported on ${process.platform}/${process.arch}`);
if (!isLoopbackEndpoint(this.baseUrl)) throw new Error('Managed Ollama requires a loopback OLLAMA_HOST');
await mkdir(this.root, { recursive: true });
const lock = await this.acquireInstallLock();
try {
const concurrentlyInstalled = this.getInstalledExecutable();
if (concurrentlyInstalled) return { executable: concurrentlyInstalled, installedNow: false };
await this.removeOrphanedInstallAttempts();
const finalDir = this.versionDir()!;
if (existsSync(finalDir)) await removeWithin(this.root, finalDir, this.rmImpl);
const attemptId = `${process.pid}-${randomUUID()}`;
const versionKey = this.installVersionKey();
const staging = path.join(this.root, `.install-attempt-${versionKey}.${attemptId}`);
const archive = path.join(this.root, `.${this.artifact.filename}.${versionKey}.${attemptId}.part`);
await mkdir(staging, { recursive: true });
try {
await this.downloadArtifact(archive);
await this.extractArchive(archive, staging, this.artifact);
const executable = await findExecutable(staging, this.artifact.executableName);
if (this.artifact.platform !== 'win32') chmodSync(executable, 0o755);
const relativeExecutable = path.relative(staging, executable);
const metadata: InstallMetadata = {
version: this.artifact.version,
sha256: this.artifact.sha256,
executable: relativeExecutable,
installedAt: new Date().toISOString(),
};
writeFileSync(path.join(staging, 'install.json'), `${JSON.stringify(metadata, null, 2)}\n`, { flag: 'wx' });
return await this.publishInstall(staging, finalDir);
} finally {
await this.cleanupInstallAttempt([archive, staging]);
}
} finally {
await this.releaseInstallLock(lock);
}
}
private async waitForInstalledExecutable(): Promise<string | null> {
for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) {
const executable = this.getInstalledExecutable();
if (executable) return executable;
if (attempt < FILESYSTEM_ATTEMPTS) await waitForFilesystemRetry(attempt);
}
return null;
}
private async publishInstall(
staging: string,
finalDir: string,
): Promise<{ executable: string; installedNow: boolean }> {
for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) {
try {
await this.renameImpl(staging, finalDir);
} catch (error) {
const winner = existsSync(finalDir) ? await this.waitForInstalledExecutable() : null;
if (winner) return { executable: winner, installedNow: false };
if (!isTransientFilesystemError(error) || attempt === FILESYSTEM_ATTEMPTS) throw error;
await waitForFilesystemRetry(attempt);
continue;
}
const executable = await this.waitForInstalledExecutable();
if (!executable) throw new Error('Published Ollama runtime failed install metadata validation');
return { executable, installedNow: true };
}
throw new Error('Managed Ollama runtime publication exhausted all retry attempts');
}
private fallbackCandidate(
loaded: RuntimeStateLoad,
failedArtifact: OllamaRuntimeArtifact,
): { artifact: OllamaRuntimeArtifact; executable: string } | null {
const artifact = loaded.kind === 'valid'
? this.selectFallbackArtifact(loaded, failedArtifact)
: loaded.kind === 'invalid' && loaded.recoveryArtifact
? loaded.recoveryArtifact
: this.legacyStartCandidate()?.artifact ?? null;
if (!artifact || this.sameArtifact(artifact, failedArtifact)) return null;
const executable = this.getInstalledExecutable(artifact);
return executable ? { artifact, executable } : null;
}
private async rollbackAfterFailure(
failedArtifact: OllamaRuntimeArtifact,
fallback: { artifact: OllamaRuntimeArtifact; executable: string } | null,
cause: unknown,
): Promise<never> {
if (cause instanceof ManagedRuntimeTerminationError) throw cause;
await this.stopInternal();
if (!fallback) throw cause;
try {
const started = await this.start(fallback.executable, fallback.artifact);
if (!started || !this.runningArtifact || !this.sameArtifact(this.runningArtifact, fallback.artifact)) {
throw new Error(`Verified fallback runtime ${fallback.artifact.version} was not started by Waggle`);
}
} catch (rollbackError) {
throw new Error(
`Managed Ollama ${failedArtifact.version} failed and verified fallback ${fallback.artifact.version} could not start: ${this.rollbackReason(rollbackError)}`,
{ cause },
);
}
const rollback: ManagedRuntimeRollback = {
failedVersion: failedArtifact.version,
restoredVersion: fallback.artifact.version,
occurredAt: new Date().toISOString(),
reason: this.rollbackReason(cause),
};
try {
await this.persistRollback(failedArtifact, fallback.artifact, rollback);
} catch (stateError) {
this.reportWarning('Could not persist the managed Ollama rollback receipt', stateError);
}
throw new ManagedRuntimeRollbackError(rollback, cause);
}
private async recoverCrashedRuntime(): Promise<ManagedOllamaReadyResult | null> {
const artifact = this.pendingCrashRecovery;
if (!artifact) return null;
if (this.crashRecoveryConsumed) {
throw this.markTerminationUnconfirmed('managed Ollama crash-recovery budget is exhausted');
}
const executable = this.getInstalledExecutable(artifact);
if (!executable) {
throw this.markTerminationUnconfirmed('verified managed Ollama runtime disappeared before crash recovery');
}
if (!await this.endpointQuiescent(this.baseUrl)) {
throw this.markTerminationUnconfirmed(
'owned Ollama daemon exited but its loopback endpoint remains occupied',
);
}
this.assertTerminationConfirmed();
this.crashRecoveryConsumed = true;
this.pendingCrashRecovery = null;
try {
const started = await this.start(executable, artifact, true);
if (!started || !this.runningArtifact || !this.sameArtifact(this.runningArtifact, artifact)) {
throw new Error(`verified runtime ${artifact.version} was not restarted by Waggle`);
}
} catch (error) {
throw this.markTerminationUnconfirmed(
`managed Ollama crash recovery failed: ${this.rollbackReason(error)}`,
);
}
return {
installedNow: false,
startedNow: true,
endpoint: this.baseUrl,
status: this.getStatus(),
};
}
private queueCrashRecovery(artifact: OllamaRuntimeArtifact): void {
this.pendingCrashRecovery = artifact;
const recovery = this.withActivationLock(async () => {
this.assertTerminationConfirmed();
await this.recoverCrashedRuntime();
});
void recovery.catch((error: unknown) => {
if (!this.terminationFailure) {
this.markTerminationUnconfirmed(
`managed Ollama crash recovery failed: ${this.rollbackReason(error)}`,
);
}
});
}
async startInstalled(): Promise<ManagedOllamaReadyResult> {
return this.withActivationLock(async () => {
this.assertTerminationConfirmed();
let recovered = await this.recoverCrashedRuntime();
if (recovered) return recovered;
const endpointReady = await this.probe(this.baseUrl);
this.assertTerminationConfirmed();
recovered = await this.recoverCrashedRuntime();
if (recovered) return recovered;
if (endpointReady) {
return { installedNow: false, startedNow: false, endpoint: this.baseUrl, status: this.getStatus() };
}
const loaded = this.loadRuntimeState();
const candidate = loaded.kind === 'valid'
? {
artifact: loaded.activeArtifact,
executable: this.getInstalledExecutable(loaded.activeArtifact),
}
: loaded.kind === 'invalid' && loaded.recoveryArtifact
? {
artifact: loaded.recoveryArtifact,
executable: this.getInstalledExecutable(loaded.recoveryArtifact),
}
: this.legacyStartCandidate();
if (!candidate?.executable) {
throw new Error('Verified Waggle-managed Ollama runtime is not installed');
}
const fallback = this.fallbackCandidate(loaded, candidate.artifact);
let startedNow: boolean;
try {
startedNow = await this.start(candidate.executable, candidate.artifact);
if (startedNow && loaded.kind !== 'valid') await this.promoteActive(candidate.artifact);
} catch (error) {
return this.rollbackAfterFailure(candidate.artifact, fallback, error);
}
return {
installedNow: false,
startedNow,
endpoint: this.baseUrl,
status: this.getStatus(),
};
});
}
async ensureReady(): Promise<ManagedOllamaReadyResult> {
return this.withActivationLock(async () => {
this.assertTerminationConfirmed();
let recovered = await this.recoverCrashedRuntime();
if (recovered) return recovered;
const endpointReady = await this.probe(this.baseUrl);
this.assertTerminationConfirmed();
recovered = await this.recoverCrashedRuntime();
if (recovered) return recovered;
if (endpointReady) {
return { installedNow: false, startedNow: false, endpoint: this.baseUrl, status: this.getStatus() };
}
const target = this.assertTrustedTarget();
const loadedBefore = this.loadRuntimeState();
const fallback = this.fallbackCandidate(loadedBefore, target);
let installation: { executable: string; installedNow: boolean };
try {
installation = await this.install();
} catch (error) {
return this.rollbackAfterFailure(target, fallback, error);
}
let startedNow: boolean;
try {
startedNow = await this.start(installation.executable, target);
if (startedNow) await this.promoteActive(target);
} catch (error) {
return this.rollbackAfterFailure(target, fallback, error);
}
return {
installedNow: installation.installedNow,
startedNow,
endpoint: this.baseUrl,
status: this.getStatus(),
};
});
}
private async start(
executable: string,
artifact: OllamaRuntimeArtifact,
requireQuiescentEndpoint = false,
): Promise<boolean> {
if (this.startPromise) return this.startPromise;
this.assertTerminationConfirmed();
const endpointReady = await this.probe(this.baseUrl);
this.assertTerminationConfirmed();
const recovered = await this.recoverCrashedRuntime();
if (recovered) return false;
if (endpointReady) {
if (requireQuiescentEndpoint) {
throw new ManagedRuntimeTerminationError('managed Ollama loopback endpoint remains occupied');
}
return false;
}
if (requireQuiescentEndpoint && !await this.endpointQuiescent(this.baseUrl)) {
throw new ManagedRuntimeTerminationError('managed Ollama loopback endpoint remains occupied');
}
if (this.child) {
if (this.child.exitCode === null) await this.stopInternal();
else if (this.child.exitCode === WATCHDOG_EXIT_OWNED_DAEMON_EXITED) {
if (!await this.endpointQuiescent(this.baseUrl)) {
throw this.markTerminationUnconfirmed('owned Ollama daemon exited but its loopback endpoint remains occupied');
}
this.child = null;
this.runningArtifact = null;
} else if (this.child.exitCode !== WATCHDOG_EXIT_TERMINATION_CONFIRMED) {
throw this.markTerminationUnconfirmed(`watchdog exited with code ${this.child.exitCode}`);
} else {
this.child = null;
this.runningArtifact = null;
}
}
this.assertTerminationConfirmed();
this.startPromise = this.startInternal(executable, artifact).finally(() => { this.startPromise = null; });
return this.startPromise;
}
private async startInternal(executable: string, artifact: OllamaRuntimeArtifact): Promise<boolean> {
await mkdir(this.modelsDir, { recursive: true });
await mkdir(this.profileDir, { recursive: true });
const startTimeoutMs = this.platform === 'win32'
? WINDOWS_START_TIMEOUT_MS
: START_TIMEOUT_MS;
const startedAt = Date.now();
let stdoutRawTail = '';
let stderrRawTail = '';
const child = this.spawnImpl(process.execPath, [
'-e',
MANAGED_OLLAMA_WATCHDOG_SOURCE,
executable,
JSON.stringify(['serve']),
], {
env: buildManagedOllamaEnv(this.baseUrl, this.modelsDir, process.env, this.profileDir),
windowsHide: true,
shell: false,
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
});
const captureStdout = (chunk: string | Buffer) => {
stdoutRawTail = appendRawDiagnosticTail(stdoutRawTail, chunk);
};
const captureStderr = (chunk: string | Buffer) => {
stderrRawTail = appendRawDiagnosticTail(stderrRawTail, chunk);
};
const ignoreDiagnosticStreamError = () => {};
child.stdout?.on('error', ignoreDiagnosticStreamError);
child.stderr?.on('error', ignoreDiagnosticStreamError);
child.stdout?.on('data', captureStdout);
child.stderr?.on('data', captureStderr);
const releaseDiagnosticCapture = () => {
child.stdout?.off('data', captureStdout);
child.stderr?.off('data', captureStderr);
child.stdout?.resume();
child.stderr?.resume();
};
const reportStartupDiagnostics = (reason: string) => {
const elapsedMs = Math.max(0, Date.now() - startedAt);
const stdoutTail = formatDiagnosticTail(stdoutRawTail);
const stderrTail = formatDiagnosticTail(stderrRawTail);
const watchdogState = child.exitCode === null
? 'running'
: `exited (${child.exitCode})`;
this.reportWarning(
`Managed Ollama ${artifact.version} startup failed`,
new Error([
`reason=${reason}`,
`endpoint=${this.baseUrl.replace(/\/+$/, '')}/api/tags`,
`elapsedMs=${elapsedMs}`,
`watchdog=${watchdogState}`,
`stdoutTail=${stdoutTail || '<empty>'}`,
`stderrTail=${stderrTail || '<empty>'}`,
].join('\n')),
);
};
this.child = child;
let startupError: Error | null = null;
child.once('error', (error) => {
startupError = error;
if (this.child === child) {
this.child = null;
this.runningArtifact = null;
}
});
child.once('exit', (code) => {
if (this.child === child) {
const readyArtifact = this.runningArtifact && this.sameArtifact(this.runningArtifact, artifact)
? this.runningArtifact
: null;
const stopWasRequested = this.stopRequestedChildren.has(child);
if (code === WATCHDOG_EXIT_TERMINATION_CONFIRMED) {
this.child = null;
this.terminationFailure = null;
this.pendingCrashRecovery = null;
} else if (code === WATCHDOG_EXIT_OWNED_DAEMON_EXITED) {
this.child = null;
if (readyArtifact && !stopWasRequested && !this.stopInProgress && !this.terminationFailure) {
if (this.crashRecoveryConsumed) {
this.markTerminationUnconfirmed('managed Ollama replacement exited after its one recovery attempt');
} else {
this.queueCrashRecovery(readyArtifact);
}
}
} else {
this.markTerminationUnconfirmed(
code === null ? 'watchdog exited without a success code' : `watchdog exited with code ${code}`,
);
}
this.runningArtifact = null;
}
});
try {
const deadline = Date.now() + startTimeoutMs;
while (Date.now() < deadline) {
if (startupError) throw startupError;
if (child.exitCode !== null) break;
const ready = await this.probe(this.baseUrl);
if (startupError) throw startupError;
if (child.exitCode !== null || this.child !== child) break;
if (ready) {
this.runningArtifact = artifact;
releaseDiagnosticCapture();
return true;
}
await new Promise((resolve) => setTimeout(resolve, 500));
}
} catch (error) {
if (startupError && child.pid === undefined) {
reportStartupDiagnostics('watchdog spawn error');
releaseDiagnosticCapture();
if (this.child === child) this.child = null;
this.runningArtifact = null;
throw error;
}
try {
await this.stopChild(child);
} catch (stopError) {
await waitForDiagnosticStreams(child);
reportStartupDiagnostics('startup error with unconfirmed cleanup');
releaseDiagnosticCapture();
throw stopError;
}
await waitForDiagnosticStreams(child);
reportStartupDiagnostics('startup error');
releaseDiagnosticCapture();
if (this.child === child) this.child = null;
if (this.runningArtifact === artifact) this.runningArtifact = null;
throw error;
}
const watchdogWasRunning = child.exitCode === null;
try {
await this.stopChild(child);
} catch (stopError) {
await waitForDiagnosticStreams(child);
reportStartupDiagnostics('readiness timeout with unconfirmed cleanup');
releaseDiagnosticCapture();
throw stopError;
}
await waitForDiagnosticStreams(child);
reportStartupDiagnostics('readiness timeout');
releaseDiagnosticCapture();
if (this.child === child) this.child = null;
if (this.runningArtifact === artifact) this.runningArtifact = null;
throw new Error(
`Waggle-managed Ollama ${artifact.version} did not make `
+ `${this.baseUrl.replace(/\/+$/, '')}/api/tags ready within ${startTimeoutMs}ms `
+ `(watchdog ${watchdogWasRunning ? 'remained live until cleanup' : 'exited before readiness'})`,
);
}
private requestStop(child: ChildProcess): void {
try {
if (child.connected) child.send('shutdown');
} catch { /* a disconnected watchdog must confirm shutdown by exiting cleanly */ }
}
private markTerminationUnconfirmed(reason: string): ManagedRuntimeTerminationError {
this.pendingCrashRecovery = null;
this.terminationFailure ??= new ManagedRuntimeTerminationError(reason);
return this.terminationFailure;
}
private assertTerminationConfirmed(): void {
if (this.terminationFailure) throw this.terminationFailure;
}
private async stopChild(child: ChildProcess): Promise<void> {
this.stopRequestedChildren.add(child);
if (child.exitCode !== null) {
return this.confirmWatchdogExit(child.exitCode);
}
const exitPromise = new Promise<number | null | undefined>((resolve) => {
const onExit = (code: number | null) => {
clearTimeout(timeout);
resolve(code);
};
const timeout = setTimeout(() => {
child.off('exit', onExit);
resolve(undefined);
}, STOP_TIMEOUT_MS);
child.once('exit', onExit);
});
this.requestStop(child);
const exitCode = await exitPromise;
if (exitCode === undefined) {
throw this.markTerminationUnconfirmed('watchdog did not confirm process-tree termination');
}
return this.confirmWatchdogExit(exitCode);
}
private async confirmWatchdogExit(exitCode: number | null): Promise<void> {
if (exitCode === WATCHDOG_EXIT_TERMINATION_CONFIRMED) {
this.terminationFailure = null;
return;
}
if (exitCode === WATCHDOG_EXIT_OWNED_DAEMON_EXITED) {
if (await this.endpointQuiescent(this.baseUrl)) return;
throw this.markTerminationUnconfirmed('owned Ollama daemon exited but its loopback endpoint remains occupied');
}
throw this.markTerminationUnconfirmed(
exitCode === null ? 'watchdog exited without a success code' : `watchdog exited with code ${exitCode}`,
);
}
private async stopInternal(): Promise<void> {
this.stopInProgress = true;
this.pendingCrashRecovery = null;
try {
const child = this.child;
if (!child) return;
this.stopRequestedChildren.add(child);
if (child.exitCode !== null) {
await this.confirmWatchdogExit(child.exitCode);
if (this.child === child) this.child = null;
this.runningArtifact = null;
return;
}
await this.stopChild(child);
if (this.child === child) this.child = null;
this.runningArtifact = null;
} finally {
this.pendingCrashRecovery = null;
this.stopInProgress = false;
}
}
async stop(): Promise<void> {
return this.withActivationLock(() => this.stopInternal());
}
}
async function defaultEndpointQuiescent(baseUrl: string): Promise<boolean> {
const endpoint = new URL(baseUrl);
const host = endpoint.hostname.replace(/^\[|\]$/g, '');
const port = Number.parseInt(endpoint.port || '80', 10);
return new Promise((resolve) => {
const server = createServer();
server.unref();
server.once('error', () => resolve(false));
server.listen({ host, port, exclusive: true }, () => {
server.close((error) => resolve(error === undefined));
});
});
}