import Database from 'better-sqlite3'; import type { Database as DatabaseType } from 'better-sqlite3'; import { createHash, randomUUID } from 'node:crypto'; import { spawn, execFile, type ChildProcess, type SpawnOptions } from 'node:child_process'; import { chmodSync, closeSync, createReadStream, createWriteStream, existsSync, fsyncSync, openSync, readFileSync, realpathSync, statSync, writeFileSync, } from 'node:fs'; import { mkdir, readdir, rename, rm } from 'node:fs/promises'; import { createServer } from 'node:net'; import path from 'node:path'; import { Readable, Transform } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { promisify } from 'node:util'; import zlib from 'node:zlib'; import { createLogger } from './logger.js'; const execFileAsync = promisify(execFile); const log = createLogger('managed-ollama'); const OLLAMA_TARGET_VERSION = '0.32.3'; const OLLAMA_ROLLBACK_VERSION = '0.32.0'; const ollamaReleaseRoot = (version: string): string => `https://github.com/ollama/ollama/releases/download/v${version}`; const DOWNLOAD_TIMEOUT_MS = 45 * 60_000; const DOWNLOAD_MAX_ATTEMPTS = 3; const DOWNLOAD_RETRY_DELAYS_MS = [250, 1_000] as const; const START_TIMEOUT_MS = 45_000; const WINDOWS_START_TIMEOUT_MS = 120_000; const START_DIAGNOSTIC_RAW_TAIL_BYTES = 64 * 1024; const START_DIAGNOSTIC_TAIL_BYTES = 4 * 1024; const START_DIAGNOSTIC_DRAIN_MS = 250; const STOP_TIMEOUT_MS = 5_000; const WATCHDOG_EXIT_TERMINATION_CONFIRMED = 0; const WATCHDOG_EXIT_CLEANUP_UNCONFIRMED = 1; const WATCHDOG_EXIT_OWNED_DAEMON_EXITED = 2; const INSTALL_LOCK_WAIT_MS = DOWNLOAD_TIMEOUT_MS + 15 * 60_000; const INSTALL_LOCK_BUSY_TIMEOUT_MS = 0; const INSTALL_LOCK_RETRY_MIN_MS = 200; const INSTALL_LOCK_RETRY_MAX_MS = 1_000; const TRANSIENT_DOWNLOAD_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]); class PermanentManagedRuntimeDownloadError extends Error {} /** * The desktop shell terminates the Node sidecar directly on exit, which bypasses * Fastify's onClose hooks on Windows. Keep the model daemon behind a tiny Node * supervisor connected to the sidecar over IPC: loss of that IPC handle means * the parent died, so the supervisor terminates the entire Ollama process tree. */ export const MANAGED_OLLAMA_WATCHDOG_SOURCE = String.raw` const { spawn } = require('node:child_process'); const executable = process.argv[1]; const args = JSON.parse(process.argv[2] || '[]'); const child = spawn(executable, args, { env: process.env, windowsHide: true, shell: false, stdio: ['ignore', 'pipe', 'pipe'], detached: process.platform !== 'win32', }); process.stdout.on('error', () => {}); process.stderr.on('error', () => {}); child.stdout?.on('error', () => {}); child.stderr?.on('error', () => {}); child.stdout?.pipe(process.stdout); child.stderr?.pipe(process.stderr); let stopping = false; let forceTimer; let drainTimer; let outputClosed = false; let pendingFinishCode; const finish = (code) => { if (forceTimer) clearTimeout(forceTimer); if (drainTimer) clearTimeout(drainTimer); process.exit(code); }; const finishAfterOutput = (code) => { if (outputClosed) return finish(code); if (pendingFinishCode === undefined) pendingFinishCode = code; if (!drainTimer) { drainTimer = setTimeout(() => finish(pendingFinishCode), ${START_DIAGNOSTIC_DRAIN_MS}); drainTimer.unref(); } }; const stopTree = () => { if (stopping) return; stopping = true; if (!child.pid) return finish(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}); forceTimer = setTimeout(() => finish(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}), 4000); if (process.platform === 'win32') { const killer = spawn('taskkill', ['/PID', String(child.pid), '/T', '/F'], { windowsHide: true, shell: false, stdio: 'ignore', }); killer.once('error', () => finishAfterOutput(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED})); killer.once('exit', (code) => finishAfterOutput(code === 0 ? ${WATCHDOG_EXIT_TERMINATION_CONFIRMED} : ${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED})); } else { try { process.kill(-child.pid, 'SIGTERM'); } catch { try { child.kill(); } catch {} } setTimeout(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} setTimeout(() => { try { process.kill(-child.pid, 0); finish(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}); } catch (error) { finish(error && error.code === 'ESRCH' ? ${WATCHDOG_EXIT_TERMINATION_CONFIRMED} : ${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}); } }, 100).unref(); }, 1500).unref(); } }; child.once('spawn', () => { if (process.send) process.send({ type: 'spawned', pid: child.pid }); }); child.once('error', () => finishAfterOutput(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED})); child.once('exit', () => { if (!stopping) finishAfterOutput(${WATCHDOG_EXIT_OWNED_DAEMON_EXITED}); }); child.once('close', () => { outputClosed = true; if (pendingFinishCode !== undefined) finish(pendingFinishCode); }); process.once('disconnect', stopTree); process.on('message', (message) => { if (message === 'shutdown') stopTree(); }); process.once('SIGTERM', stopTree); process.once('SIGINT', stopTree); `; export interface OllamaRuntimeArtifact { version: string; platform: NodeJS.Platform; arch: string; filename: string; url: string; sha256: string; sizeBytes: number; executableName: string; } /** * Pinned official standalone artifacts. Updating this table is an intentional * supply-chain change: version, byte size, and GitHub-published SHA-256 must move * together. The runtime is downloaded only from this hardcoded release URL. */ export const OLLAMA_RUNTIME_ARTIFACTS: ReadonlyArray = [ { version: OLLAMA_ROLLBACK_VERSION, platform: 'win32', arch: 'x64', filename: 'ollama-windows-amd64.zip', url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-windows-amd64.zip`, sha256: '56561a8f0a904483303c610e61af61c5a7b6f5496ce3707e207d25d4ff67b89e', sizeBytes: 1_503_047_573, executableName: 'ollama.exe', }, { version: OLLAMA_ROLLBACK_VERSION, platform: 'win32', arch: 'arm64', filename: 'ollama-windows-arm64.zip', url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-windows-arm64.zip`, sha256: '82b7d36b63e62a44d3f9853c2f8edb829cf871eaf722ce20070e09e96922c0cc', sizeBytes: 16_346_970, executableName: 'ollama.exe', }, { version: OLLAMA_ROLLBACK_VERSION, platform: 'darwin', arch: 'x64', filename: 'ollama-darwin.tgz', url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-darwin.tgz`, sha256: '3b12a49c6c4cbafd7ffba5ccba60cbf80274cdc22eea3ead79c646aba888174c', sizeBytes: 145_356_966, executableName: 'ollama', }, { version: OLLAMA_ROLLBACK_VERSION, platform: 'darwin', arch: 'arm64', filename: 'ollama-darwin.tgz', url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-darwin.tgz`, sha256: '3b12a49c6c4cbafd7ffba5ccba60cbf80274cdc22eea3ead79c646aba888174c', sizeBytes: 145_356_966, executableName: 'ollama', }, { version: OLLAMA_ROLLBACK_VERSION, platform: 'linux', arch: 'x64', filename: 'ollama-linux-amd64.tar.zst', url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-linux-amd64.tar.zst`, sha256: '56362d7609dfa9e35aaebb7c9cab25605d8f0528ec3d5d585dc83d6642002bab', sizeBytes: 1_436_128_693, executableName: 'ollama', }, { version: OLLAMA_ROLLBACK_VERSION, platform: 'linux', arch: 'arm64', filename: 'ollama-linux-arm64.tar.zst', url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-linux-arm64.tar.zst`, sha256: '2c1fbc47a5351c74f5400d7e4b1104bb470291af5d2f425c37c151487a477ad6', sizeBytes: 1_556_234_636, executableName: 'ollama', }, { version: OLLAMA_TARGET_VERSION, platform: 'win32', arch: 'x64', filename: 'ollama-windows-amd64.zip', url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-windows-amd64.zip`, sha256: 'c66dd7dde4d5ec4822eaa57dd421d51aa7c633a3ff36a974040837df73a5969e', sizeBytes: 1_457_806_156, executableName: 'ollama.exe', }, { version: OLLAMA_TARGET_VERSION, platform: 'win32', arch: 'arm64', filename: 'ollama-windows-arm64.zip', url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-windows-arm64.zip`, sha256: '8431fc4ccf7e86a273dbcc52e81585fc92133e4d6f9c1abdd3663e2568f2fe90', sizeBytes: 209_405_705, executableName: 'ollama.exe', }, { version: OLLAMA_TARGET_VERSION, platform: 'darwin', arch: 'x64', filename: 'ollama-darwin.tgz', url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-darwin.tgz`, sha256: '14462bd438815eb2c1d4c61224744637131ab744e858a2e2562e7fc7fc2c4f7d', sizeBytes: 145_790_989, executableName: 'ollama', }, { version: OLLAMA_TARGET_VERSION, platform: 'darwin', arch: 'arm64', filename: 'ollama-darwin.tgz', url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-darwin.tgz`, sha256: '14462bd438815eb2c1d4c61224744637131ab744e858a2e2562e7fc7fc2c4f7d', sizeBytes: 145_790_989, executableName: 'ollama', }, { version: OLLAMA_TARGET_VERSION, platform: 'linux', arch: 'x64', filename: 'ollama-linux-amd64.tar.zst', url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-linux-amd64.tar.zst`, sha256: '2597d74fbe654ef6a37db56f771cf37d4a85c6bde4018127874e3927d3113800', sizeBytes: 1_421_853_873, executableName: 'ollama', }, { version: OLLAMA_TARGET_VERSION, platform: 'linux', arch: 'arm64', filename: 'ollama-linux-arm64.tar.zst', url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-linux-arm64.tar.zst`, sha256: '61af0def977ffc9e76cc961f9505d621805653c63f444796de2f77b4eaaa7047', sizeBytes: 1_542_500_407, executableName: 'ollama', } ]; interface InstallMetadata { version: string; sha256: string; executable: string; installedAt: string; } interface RuntimeReceipt { version: string; sha256: string; executable: string; installedAt: string; } interface RuntimeActivationState { schemaVersion: 1; active: RuntimeReceipt; previous: RuntimeReceipt | null; lastRollback: ManagedRuntimeRollback | null; } type RuntimeStateLoad = | { kind: 'missing' } | { kind: 'invalid'; reason: string; recoveryArtifact?: OllamaRuntimeArtifact; recoveryReceipt?: RuntimeReceipt; lastRollback?: ManagedRuntimeRollback | null; } | { kind: 'valid'; state: RuntimeActivationState; activeArtifact: OllamaRuntimeArtifact; previousArtifact: OllamaRuntimeArtifact | null; }; interface InstallLockClaim { database: DatabaseType; } export interface ManagedOllamaStatus { source: 'waggle-managed'; supported: boolean; installed: boolean; running: boolean; targetVersion: string | null; version: string | null; targetInstalled: boolean; activeVersion: string | null; previousVersion: string | null; fallbackActive: boolean; rollback: { available: boolean; active: boolean; lastAttempt: ManagedRuntimeRollback | null; }; artifactSizeBytes: number | null; downloadRequired: boolean; dockerRequired: false; reason?: string; } export interface ManagedRuntimeRollback { failedVersion: string; restoredVersion: string; occurredAt: string; reason: string; } export class ManagedRuntimeRollbackError extends Error { readonly rollback: ManagedRuntimeRollback; readonly failedVersion: string; readonly restoredVersion: string; constructor(rollback: ManagedRuntimeRollback, cause: unknown) { super( `Managed Ollama ${rollback.failedVersion} failed; restored verified runtime ${rollback.restoredVersion}: ${rollback.reason}`, { cause }, ); this.name = 'ManagedRuntimeRollbackError'; this.rollback = rollback; this.failedVersion = rollback.failedVersion; this.restoredVersion = rollback.restoredVersion; } } class ManagedRuntimeTerminationError extends Error { constructor(reason = 'watchdog did not confirm process-tree termination') { super(`Managed Ollama ${reason}; refusing to start a replacement runtime`); this.name = 'ManagedRuntimeTerminationError'; } } export interface ManagedOllamaReadyResult { installedNow: boolean; startedNow: boolean; endpoint: string; status: ManagedOllamaStatus; } interface RuntimeDependencies { fetchImpl?: typeof fetch; extractArchive?: ( archivePath: string, destination: string, artifact: OllamaRuntimeArtifact, ) => Promise; spawnImpl?: (file: string, args: readonly string[], options: SpawnOptions) => ChildProcess; probe?: (baseUrl: string) => Promise; endpointQuiescent?: (baseUrl: string) => Promise; renameImpl?: typeof rename; stateRenameImpl?: typeof rename; rmImpl?: typeof rm; warn?: (message: string, error: unknown) => void; artifact?: OllamaRuntimeArtifact | null; artifactCatalog?: ReadonlyArray; platform?: NodeJS.Platform; arch?: string; } export function resolveOllamaRuntimeArtifact( platform: NodeJS.Platform = process.platform, arch: string = process.arch, ): OllamaRuntimeArtifact | null { const targets = OLLAMA_RUNTIME_ARTIFACTS.filter( (entry) => entry.platform === platform && entry.arch === arch && entry.version === OLLAMA_TARGET_VERSION, ); return targets.length === 1 ? targets[0] : null; } function isLoopbackEndpoint(baseUrl: string): boolean { try { const url = new URL(baseUrl); return url.protocol === 'http:' && (url.hostname === '127.0.0.1' || url.hostname === 'localhost' || url.hostname === '::1') && (url.pathname === '/' || url.pathname === ''); } catch { return false; } } export function buildManagedOllamaEnv( baseUrl: string, modelsDir: string, source: NodeJS.ProcessEnv = process.env, managedProfileDir?: string, ): NodeJS.ProcessEnv { if (!isLoopbackEndpoint(baseUrl)) { throw new Error('Waggle-managed Ollama must bind to an HTTP loopback endpoint'); } const endpoint = new URL(baseUrl); return { ...source, ...(managedProfileDir ? { HOME: managedProfileDir, USERPROFILE: managedProfileDir } : {}), OLLAMA_HOST: endpoint.host, OLLAMA_MODELS: modelsDir, OLLAMA_NOHISTORY: '1', // Ollama otherwise defaults to 4K on common 8 GB GPUs. The qualified // 8K-token tool request plus Waggle's 14K synthesis ceiling needs a // 32K window to retain the full input and leave the promised output room. OLLAMA_CONTEXT_LENGTH: '32768', }; } async function defaultProbe(baseUrl: string): Promise { try { const response = await fetch(`${baseUrl.replace(/\/+$/, '')}/api/tags`, { signal: AbortSignal.timeout(3_000), }); if (!response.ok) return false; const payload = await response.json() as { models?: unknown }; return Array.isArray(payload.models); } catch { return false; } } async function defaultExtract( archivePath: string, destination: string, artifact: OllamaRuntimeArtifact, ): Promise { if (artifact.filename.endsWith('.tar.zst')) { await extractTarZst(archivePath, destination); return; } const executable = artifact.platform === 'win32' ? 'tar.exe' : 'tar'; await execFileAsync(executable, ['-xf', archivePath, '-C', destination], { windowsHide: true, timeout: 10 * 60_000, }); } /** * Linux releases are zstd-compressed tarballs. Node 22 decompresses zstd itself * and the system tar only unpacks; on older Node, tar needs zstd support. */ export async function extractTarZst(archivePath: string, destination: string): Promise { const createZstdDecompress = (zlib as { createZstdDecompress?: () => Transform }).createZstdDecompress; if (!createZstdDecompress) { await execFileAsync('tar', ['--zstd', '-xf', archivePath, '-C', destination], { timeout: 10 * 60_000 }); return; } const tar = spawn('tar', ['-xf', '-', '-C', destination], { stdio: ['pipe', 'ignore', 'pipe'] }); let stderr = ''; tar.stderr?.on('data', (chunk: Buffer) => { stderr += chunk.toString(); }); const exited = new Promise((resolve, reject) => { tar.once('error', reject); tar.once('close', (code) => (code === 0 ? resolve() : reject(new Error(`tar exited with ${code}: ${stderr.trim()}`)))); }); await Promise.all([pipeline(createReadStream(archivePath), createZstdDecompress(), tar.stdin!), exited]); } function inside(root: string, candidate: string): boolean { const relative = path.relative(path.resolve(root), path.resolve(candidate)); return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative)); } const TRANSIENT_FILESYSTEM_CODES = new Set(['EPERM', 'EACCES', 'EBUSY']); const FILESYSTEM_ATTEMPTS = 4; const PROCESS_INSTALLS = new Map>(); function isTransientFilesystemError(error: unknown): boolean { return TRANSIENT_FILESYSTEM_CODES.has((error as NodeJS.ErrnoException)?.code ?? ''); } function redactManagedRuntimeDiagnostic(value: string): string { const redacted = value .replace(/(authorization\s*[:=]\s*)[^\r\n]*/gi, '$1[REDACTED]') .replace(/(https?:\/\/)[^\s/@:]+:[^\s/@]+@/gi, '$1[REDACTED]@') .replace( /((?:api[_-]?key|token|password|secret|credential|private[_-]?key)\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s,;\]}]+)/gi, '$1[REDACTED]', ); return Array.from(redacted, (character) => { const code = character.charCodeAt(0); const disallowedControl = (code < 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d) || code === 0x7f; return disallowedControl ? '?' : character; }).join(''); } function appendRawDiagnosticTail(current: string, chunk: string | Buffer): string { const next = Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk); const combined = Buffer.from(current + next, 'utf8'); if (combined.length <= START_DIAGNOSTIC_RAW_TAIL_BYTES) return combined.toString('utf8'); return combined .subarray(combined.length - START_DIAGNOSTIC_RAW_TAIL_BYTES) .toString('utf8'); } function formatDiagnosticTail(raw: string): string { const combined = Buffer.from(redactManagedRuntimeDiagnostic(raw), 'utf8'); if (combined.length <= START_DIAGNOSTIC_TAIL_BYTES) return combined.toString('utf8'); return combined .subarray(combined.length - START_DIAGNOSTIC_TAIL_BYTES) .toString('utf8'); } async function waitForDiagnosticStreams(child: ChildProcess): Promise { const streams = [child.stdout, child.stderr] .filter((stream): stream is Readable => stream != null); const cleanups: Array<() => void> = []; const waits = streams.map((stream) => new Promise((resolve) => { if (stream.readableEnded || stream.destroyed) { resolve(); return; } const finish = () => { cleanup(); resolve(); }; const cleanup = () => { stream.off('end', finish); stream.off('close', finish); stream.off('error', finish); }; stream.once('end', finish); stream.once('close', finish); stream.once('error', finish); cleanups.push(cleanup); })); if (waits.length === 0) return; let timeout: NodeJS.Timeout | undefined; try { await Promise.race([ Promise.all(waits), new Promise((resolve) => { timeout = setTimeout(resolve, START_DIAGNOSTIC_DRAIN_MS); }), ]); } finally { if (timeout) clearTimeout(timeout); for (const cleanup of cleanups) cleanup(); } } async function waitForFilesystemRetry(attempt: number): Promise { await new Promise((resolve) => setTimeout(resolve, 25 * attempt)); } async function removeWithin(root: string, target: string, rmImpl: typeof rm = rm): Promise { if (!inside(root, target) || path.resolve(root) === path.resolve(target)) { throw new Error(`Refusing to remove path outside managed runtime root: ${target}`); } for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) { try { await rmImpl(target, { recursive: true, force: true }); return; } catch (error) { if (!isTransientFilesystemError(error) || attempt === FILESYSTEM_ATTEMPTS) throw error; await waitForFilesystemRetry(attempt); } } } function isSqliteBusyError(error: unknown): boolean { const code = (error as { code?: unknown } | null)?.code; return code === 'SQLITE_BUSY' || code === 'SQLITE_BUSY_SNAPSHOT' || code === 'SQLITE_LOCKED'; } async function findExecutable(root: string, executableName: string): Promise { const queue: Array<{ directory: string; depth: number }> = [{ directory: root, depth: 0 }]; let visited = 0; while (queue.length > 0 && visited < 10_000) { const current = queue.shift()!; const entries = await readdir(current.directory, { withFileTypes: true }); for (const entry of entries) { visited += 1; const candidate = path.join(current.directory, entry.name); if (entry.name.toLowerCase() === executableName.toLowerCase()) { const resolved = realpathSync(candidate); if (!inside(root, resolved) || !statSync(resolved).isFile()) continue; return resolved; } if (entry.isDirectory() && current.depth < 6) { queue.push({ directory: candidate, depth: current.depth + 1 }); } } } throw new Error(`Official runtime archive did not contain ${executableName}`); } export class ManagedOllamaRuntime { private readonly root: string; private readonly modelsDir: string; private readonly profileDir: string; private readonly artifact: OllamaRuntimeArtifact | null; private readonly artifactCatalog: ReadonlyArray; private readonly platform: NodeJS.Platform; private readonly arch: string; private readonly fetchImpl: typeof fetch; private readonly extractArchive: NonNullable; private readonly spawnImpl: NonNullable; private readonly probe: NonNullable; private readonly endpointQuiescent: NonNullable; private readonly renameImpl: NonNullable; private readonly stateRenameImpl: NonNullable; private readonly rmImpl: NonNullable; private readonly warn: NonNullable; private installPromise: Promise<{ executable: string; installedNow: boolean }> | null = null; private startPromise: Promise | null = null; private activationPromise: Promise | null = null; private child: ChildProcess | null = null; private runningArtifact: OllamaRuntimeArtifact | null = null; private terminationFailure: ManagedRuntimeTerminationError | null = null; private pendingCrashRecovery: OllamaRuntimeArtifact | null = null; private crashRecoveryConsumed = false; private stopInProgress = false; private readonly stopRequestedChildren = new WeakSet(); constructor( dataDir: string, private readonly baseUrl = 'http://127.0.0.1:11434', dependencies: RuntimeDependencies = {}, ) { const platform = dependencies.platform ?? dependencies.artifact?.platform ?? process.platform; const arch = dependencies.arch ?? dependencies.artifact?.arch ?? process.arch; this.platform = platform; this.arch = arch; this.root = path.resolve(dataDir, 'runtimes', 'ollama'); this.modelsDir = path.resolve(dataDir, 'models', 'ollama'); this.profileDir = path.resolve(this.root, 'profile'); this.artifact = dependencies.artifact === undefined ? resolveOllamaRuntimeArtifact(platform, arch) : dependencies.artifact; this.artifactCatalog = dependencies.artifactCatalog ?? (dependencies.artifact === undefined ? OLLAMA_RUNTIME_ARTIFACTS : (dependencies.artifact ? [dependencies.artifact] : [])); this.fetchImpl = dependencies.fetchImpl ?? fetch; this.extractArchive = dependencies.extractArchive ?? defaultExtract; this.spawnImpl = dependencies.spawnImpl ?? ((file, args, options) => spawn(file, args, options)); this.probe = dependencies.probe ?? defaultProbe; this.endpointQuiescent = dependencies.endpointQuiescent ?? defaultEndpointQuiescent; this.renameImpl = dependencies.renameImpl ?? rename; this.stateRenameImpl = dependencies.stateRenameImpl ?? rename; this.rmImpl = dependencies.rmImpl ?? rm; this.warn = dependencies.warn ?? ((message, error) => log.warn(message, { error: error instanceof Error ? error.message : String(error), code: (error as { code?: unknown } | null)?.code ?? null, })); } getStatus(): ManagedOllamaStatus { const executable = this.getInstalledExecutable(); const targetInstalled = executable !== null; const loadedState = this.loadRuntimeState(); const running = this.child !== null && this.child.exitCode === null; const managedRunningVersion = running ? this.runningArtifact?.version ?? null : null; const activeVersion = managedRunningVersion ?? (loadedState.kind === 'valid' ? loadedState.state.active.version : null); const previousVersion = loadedState.kind === 'valid' ? loadedState.state.previous?.version ?? null : loadedState.kind === 'invalid' ? loadedState.recoveryArtifact?.version ?? null : null; const persistedRollbackActive = loadedState.kind === 'valid' && this.artifact !== null && !this.sameArtifact(loadedState.activeArtifact, this.artifact) && loadedState.state.lastRollback?.failedVersion === this.artifact.version && loadedState.state.lastRollback.restoredVersion === loadedState.activeArtifact.version; const fallbackActive = persistedRollbackActive || (running && this.runningArtifact !== null && this.artifact !== null && !this.sameArtifact(this.runningArtifact, this.artifact)); const rollbackAvailable = loadedState.kind === 'valid' && this.selectFallbackArtifact(loadedState, this.artifact) !== null; const supported = this.artifact !== null && isLoopbackEndpoint(this.baseUrl); return { source: 'waggle-managed', supported, installed: targetInstalled || loadedState.kind === 'valid' || (loadedState.kind === 'invalid' && loadedState.recoveryArtifact !== undefined) || this.runningArtifact !== null, running, targetVersion: this.artifact?.version ?? null, version: activeVersion ?? (targetInstalled ? this.artifact?.version ?? null : null), targetInstalled, activeVersion, previousVersion, fallbackActive, rollback: { available: rollbackAvailable || fallbackActive, active: fallbackActive, lastAttempt: loadedState.kind === 'valid' ? loadedState.state.lastRollback : loadedState.kind === 'invalid' ? loadedState.lastRollback ?? null : null, }, artifactSizeBytes: this.artifact?.sizeBytes ?? null, downloadRequired: !targetInstalled, dockerRequired: false, ...(!supported ? { reason: this.artifact === null ? `No managed Ollama artifact for ${process.platform}/${process.arch}` : 'Managed Ollama requires a loopback OLLAMA_HOST' } : loadedState.kind === 'invalid' ? { reason: loadedState.reason } : {}), }; } private versionDir(artifact = this.artifact): string | null { if (!artifact) return null; const candidate = path.join(this.root, artifact.version); return inside(this.root, candidate) && path.resolve(candidate) !== this.root ? candidate : null; } private installVersionKey(): string { return createHash('sha256').update(this.artifact!.version).digest('hex'); } private installLockDatabasePath(): string { return path.join(this.root, `.install-lock-${this.installVersionKey()}.sqlite`); } private getVerifiedInstall(artifact = this.artifact): { executable: string; receipt: RuntimeReceipt } | null { const versionDir = this.versionDir(artifact); if (!versionDir) return null; const metadataPath = path.join(versionDir, 'install.json'); try { const metadata = JSON.parse(readFileSync(metadataPath, 'utf8')) as InstallMetadata; if (metadata.version !== artifact?.version || metadata.sha256 !== artifact.sha256) return null; if (path.isAbsolute(metadata.executable) || metadata.executable.split(/[\\/]/).includes('..')) return null; if (path.basename(metadata.executable).toLowerCase() !== artifact.executableName.toLowerCase()) return null; if (typeof metadata.installedAt !== 'string' || !Number.isFinite(Date.parse(metadata.installedAt))) return null; const executable = realpathSync(path.join(versionDir, metadata.executable)); if (!inside(versionDir, executable) || !statSync(executable).isFile()) return null; return { executable, receipt: { version: metadata.version, sha256: metadata.sha256, executable: metadata.executable, installedAt: metadata.installedAt, }, }; } catch { return null; } } private getInstalledExecutable(artifact = this.artifact): string | null { return this.getVerifiedInstall(artifact)?.executable ?? null; } private statePath(): string { return path.join(this.root, 'runtime-state.json'); } private receiptFor(artifact: OllamaRuntimeArtifact): RuntimeReceipt { const installation = this.getVerifiedInstall(artifact); if (!installation) throw new Error(`Managed Ollama ${artifact.version} install receipt failed validation`); return installation.receipt; } private sameReceipt(left: RuntimeReceipt, right: RuntimeReceipt): boolean { return left.version === right.version && left.sha256 === right.sha256 && left.executable === right.executable && left.installedAt === right.installedAt; } private sameArtifact(left: OllamaRuntimeArtifact, right: OllamaRuntimeArtifact): boolean { return left.version === right.version && left.sha256 === right.sha256; } private sameCatalogArtifact(left: OllamaRuntimeArtifact, right: OllamaRuntimeArtifact): boolean { return this.sameArtifact(left, right) && left.platform === right.platform && left.arch === right.arch && left.filename === right.filename && left.url === right.url && left.sizeBytes === right.sizeBytes && left.executableName === right.executableName; } private assertTrustedTarget(): OllamaRuntimeArtifact { if (!this.artifact) throw new Error(`Managed Ollama is unsupported on ${process.platform}/${process.arch}`); const trusted = this.artifactCatalog.find((candidate) => this.sameCatalogArtifact(candidate, this.artifact!)); if (!trusted) throw new Error(`Managed Ollama ${this.artifact.version} is not present in the trusted artifact catalog`); return trusted; } private catalogArtifact(receipt: RuntimeReceipt): OllamaRuntimeArtifact | null { return this.artifactCatalog.find((candidate) => ( candidate.platform === this.platform && candidate.arch === this.arch && candidate.version === receipt.version && candidate.sha256 === receipt.sha256 )) ?? null; } private validReceipt(value: unknown): RuntimeReceipt | null { if (!value || typeof value !== 'object') return null; const candidate = value as Partial; if (typeof candidate.version !== 'string' || candidate.version.length === 0 || candidate.version.length > 128) return null; if (typeof candidate.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(candidate.sha256)) return null; if (typeof candidate.executable !== 'string' || candidate.executable.length === 0 || candidate.executable.length > 1_024) return null; if (path.isAbsolute(candidate.executable) || candidate.executable.split(/[\\/]/).includes('..')) return null; if (typeof candidate.installedAt !== 'string' || !Number.isFinite(Date.parse(candidate.installedAt))) return null; return { version: candidate.version, sha256: candidate.sha256, executable: candidate.executable, installedAt: candidate.installedAt, }; } private validRollback(value: unknown): ManagedRuntimeRollback | null | undefined { if (value === null) return null; if (!value || typeof value !== 'object') return undefined; const candidate = value as Partial; if (typeof candidate.failedVersion !== 'string' || candidate.failedVersion.length === 0 || candidate.failedVersion.length > 128) return undefined; if (typeof candidate.restoredVersion !== 'string' || candidate.restoredVersion.length === 0 || candidate.restoredVersion.length > 128) return undefined; if (typeof candidate.occurredAt !== 'string' || !Number.isFinite(Date.parse(candidate.occurredAt))) return undefined; if (typeof candidate.reason !== 'string' || candidate.reason.length === 0 || candidate.reason.length > 2_048) return undefined; return { failedVersion: candidate.failedVersion, restoredVersion: candidate.restoredVersion, occurredAt: candidate.occurredAt, reason: candidate.reason, }; } private loadRuntimeState(): RuntimeStateLoad { let parsed: unknown; try { parsed = JSON.parse(readFileSync(this.statePath(), 'utf8')); } catch (error) { if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') return { kind: 'missing' }; return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation' }; } if (!parsed || typeof parsed !== 'object') { return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation' }; } const candidate = parsed as Partial; if (candidate.schemaVersion !== 1) { return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation' }; } const parsedRollback = this.validRollback(candidate.lastRollback); const lastRollback = parsedRollback === undefined ? null : parsedRollback; const previous = candidate.previous === null ? null : this.validReceipt(candidate.previous); const previousArtifact = previous ? this.catalogArtifact(previous) : null; const previousInstall = previousArtifact ? this.getVerifiedInstall(previousArtifact) : null; const trustedPrevious = previous && previousArtifact && previousInstall && this.sameReceipt(previous, previousInstall.receipt) ? { receipt: previous, artifact: previousArtifact } : null; const recovery = trustedPrevious ? { recoveryArtifact: trustedPrevious.artifact, recoveryReceipt: trustedPrevious.receipt, lastRollback, } : { lastRollback }; const active = this.validReceipt(candidate.active); if (!active) { return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation', ...recovery, }; } const activeArtifact = this.catalogArtifact(active); const activeInstall = activeArtifact ? this.getVerifiedInstall(activeArtifact) : null; if (!activeArtifact || !activeInstall || !this.sameReceipt(active, activeInstall.receipt)) { return { kind: 'invalid', reason: 'Managed Ollama activation state references an untrusted or invalid runtime', ...recovery, }; } const usablePrevious = trustedPrevious && !this.sameArtifact(activeArtifact, trustedPrevious.artifact) ? trustedPrevious : null; return { kind: 'valid', state: { schemaVersion: 1, active, previous: usablePrevious?.receipt ?? null, lastRollback, }, activeArtifact, previousArtifact: usablePrevious?.artifact ?? null, }; } private async writeRuntimeState(state: RuntimeActivationState): Promise { await mkdir(this.root, { recursive: true }); const temporary = path.join(this.root, `.runtime-state.${process.pid}-${randomUUID()}.tmp`); let descriptor: number | null = null; try { descriptor = openSync(temporary, 'wx'); writeFileSync(descriptor, `${JSON.stringify(state, null, 2)}\n`); fsyncSync(descriptor); closeSync(descriptor); descriptor = null; for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) { try { await this.stateRenameImpl(temporary, this.statePath()); break; } catch (error) { if (!isTransientFilesystemError(error) || attempt === FILESYSTEM_ATTEMPTS) throw error; await waitForFilesystemRetry(attempt); } } } finally { if (descriptor !== null) closeSync(descriptor); if (existsSync(temporary)) await removeWithin(this.root, temporary, this.rmImpl); } } private async promoteActive(artifact: OllamaRuntimeArtifact): Promise { const loaded = this.loadRuntimeState(); const receipt = this.receiptFor(artifact); const current = loaded.kind === 'valid' ? loaded.state : null; await this.writeRuntimeState({ schemaVersion: 1, active: receipt, previous: current && !this.sameReceipt(current.active, receipt) ? current.active : current?.previous ?? null, lastRollback: current?.lastRollback ?? null, }); } private selectFallbackArtifact( loaded: Extract, failedArtifact: OllamaRuntimeArtifact | null, ): OllamaRuntimeArtifact | null { if (!failedArtifact || !this.sameArtifact(loaded.activeArtifact, failedArtifact)) return loaded.activeArtifact; return loaded.previousArtifact && !this.sameArtifact(loaded.previousArtifact, failedArtifact) ? loaded.previousArtifact : null; } private verifiedCatalogInstalls(): Array<{ artifact: OllamaRuntimeArtifact; executable: string }> { const seen = new Set(); const installations: Array<{ artifact: OllamaRuntimeArtifact; executable: string }> = []; for (const artifact of this.artifactCatalog) { if (artifact.platform !== this.platform || artifact.arch !== this.arch) continue; const key = `${artifact.version}:${artifact.sha256}`; if (seen.has(key)) continue; seen.add(key); const executable = this.getInstalledExecutable(artifact); if (executable) installations.push({ artifact, executable }); } return installations; } private legacyStartCandidate(): { artifact: OllamaRuntimeArtifact; executable: string } | null { const installations = this.verifiedCatalogInstalls(); return installations.length === 1 ? installations[0]! : null; } private rollbackReason(error: unknown): string { const reason = error instanceof Error ? error.message : String(error); return (reason || 'target runtime failed').slice(0, 2_048); } private async persistRollback( failedArtifact: OllamaRuntimeArtifact, restoredArtifact: OllamaRuntimeArtifact, rollback: ManagedRuntimeRollback, ): Promise { const loaded = this.loadRuntimeState(); const restored = this.receiptFor(restoredArtifact); const current = loaded.kind === 'valid' ? loaded.state : null; const failedWasActive = current !== null && current.active.version === failedArtifact.version && current.active.sha256 === failedArtifact.sha256; await this.writeRuntimeState({ schemaVersion: 1, active: restored, previous: failedWasActive ? current.active : current?.previous ?? null, lastRollback: rollback, }); } private activationLockDatabasePath(): string { return path.join(this.root, '.activation-lock.sqlite'); } private async acquireActivationLock(): Promise { await mkdir(this.root, { recursive: true }); const database = new Database(this.activationLockDatabasePath(), { timeout: INSTALL_LOCK_BUSY_TIMEOUT_MS }); const deadline = Date.now() + INSTALL_LOCK_WAIT_MS; let retryDelay = INSTALL_LOCK_RETRY_MIN_MS; while (Date.now() < deadline) { try { database.exec('BEGIN EXCLUSIVE'); return { database }; } catch (error) { if (!isSqliteBusyError(error)) { try { database.close(); } catch { /* primary error wins */ } throw error; } } await new Promise((resolve) => setTimeout(resolve, retryDelay)); retryDelay = Math.min(retryDelay * 2, INSTALL_LOCK_RETRY_MAX_MS); } try { database.close(); } catch { /* timeout error wins */ } throw new Error('Timed out waiting for the managed Ollama activation coordinator'); } private withActivationLock(operation: () => Promise): Promise { const prior = this.activationPromise; const activation = (async () => { if (prior) { try { await prior; } catch { // The next caller must observe the resulting state, including a healthy rollback. } } const lock = await this.acquireActivationLock(); try { return await operation(); } finally { await this.releaseInstallLock(lock); } })(); this.activationPromise = activation; return activation.finally(() => { if (this.activationPromise === activation) this.activationPromise = null; }); } async install(): Promise<{ executable: string; installedNow: boolean }> { const existing = this.getInstalledExecutable(); if (existing) return { executable: existing, installedNow: false }; if (this.installPromise) return this.installPromise; const coordinationKey = this.installLockDatabasePath(); const processInstall = PROCESS_INSTALLS.get(coordinationKey); if (processInstall) { const result = await processInstall; return { executable: result.executable, installedNow: false }; } const installation = this.installInternal(); PROCESS_INSTALLS.set(coordinationKey, installation); this.installPromise = installation.finally(() => { if (PROCESS_INSTALLS.get(coordinationKey) === installation) PROCESS_INSTALLS.delete(coordinationKey); this.installPromise = null; }); return this.installPromise; } private reportWarning(message: string, error: unknown): void { try { this.warn(message, error); } catch { // Diagnostics must never change runtime availability. } } private async acquireInstallLock(): Promise { const database = new Database(this.installLockDatabasePath(), { timeout: INSTALL_LOCK_BUSY_TIMEOUT_MS }); const deadline = Date.now() + INSTALL_LOCK_WAIT_MS; let retryDelay = INSTALL_LOCK_RETRY_MIN_MS; while (Date.now() < deadline) { try { database.exec('BEGIN EXCLUSIVE'); return { database }; } catch (error) { if (!isSqliteBusyError(error)) { try { database.close(); } catch (closeError) { this.reportWarning('Could not close the managed Ollama install coordinator', closeError); } throw error; } } const jitter = Math.floor(Math.random() * Math.min(100, retryDelay / 4)); await new Promise((resolve) => setTimeout(resolve, retryDelay + jitter)); retryDelay = Math.min(retryDelay * 2, INSTALL_LOCK_RETRY_MAX_MS); } try { database.close(); } catch (error) { this.reportWarning('Could not close the timed-out managed Ollama install coordinator', error); } throw new Error(`Timed out waiting for the managed Ollama ${this.artifact!.version} installation coordinator`); } private async releaseInstallLock(claim: InstallLockClaim): Promise { try { if (claim.database.inTransaction) claim.database.exec('ROLLBACK'); } catch (error) { this.reportWarning('Could not roll back the managed Ollama install coordinator', error); } finally { try { claim.database.close(); } catch (error) { this.reportWarning('Could not close the managed Ollama install coordinator', error); } } } private async cleanupInstallAttempt(targets: readonly string[]): Promise { for (const target of targets) { try { await removeWithin(this.root, target, this.rmImpl); } catch (error) { this.reportWarning(`Could not remove managed Ollama install artifact ${path.basename(target)}`, error); } } } private async removeOrphanedInstallAttempts(): Promise { const versionKey = this.installVersionKey(); const stagingPrefix = `.install-attempt-${versionKey}.`; const archivePrefix = `.${this.artifact!.filename}.${versionKey}.`; const entries = await readdir(this.root, { withFileTypes: true }); for (const entry of entries) { const isStaging = entry.name.startsWith(stagingPrefix); const isArchive = entry.name.startsWith(archivePrefix) && entry.name.endsWith('.part'); if (!isStaging && !isArchive) continue; await removeWithin(this.root, path.join(this.root, entry.name), this.rmImpl); } } private async downloadArtifact(archive: string): Promise { const artifact = this.artifact!; const deadline = Date.now() + DOWNLOAD_TIMEOUT_MS; for (let attempt = 1; attempt <= DOWNLOAD_MAX_ATTEMPTS; attempt += 1) { let offset = existsSync(archive) ? statSync(archive).size : 0; if (offset > artifact.sizeBytes) { throw new PermanentManagedRuntimeDownloadError( `Official Ollama runtime exceeded the expected ${artifact.sizeBytes} bytes`, ); } try { if (offset < artifact.sizeBytes) { const remainingTime = deadline - Date.now(); if (remainingTime <= 0) throw new Error('Official Ollama runtime download timed out'); const response = await this.fetchImpl(artifact.url, { redirect: 'follow', ...(offset > 0 ? { headers: { Range: `bytes=${offset}-` } } : {}), signal: AbortSignal.timeout(remainingTime), }); if (!response.ok || !response.body) { const message = `Official Ollama runtime download failed with HTTP ${response.status}`; if (!TRANSIENT_DOWNLOAD_STATUS.has(response.status)) { throw new PermanentManagedRuntimeDownloadError(message); } throw new Error(message); } if (offset > 0 && response.status === 200) { offset = 0; } else if (offset > 0) { const expectedRange = `bytes ${offset}-${artifact.sizeBytes - 1}/${artifact.sizeBytes}`; if (response.status !== 206 || response.headers.get('content-range') !== expectedRange) { throw new PermanentManagedRuntimeDownloadError( 'Official Ollama runtime returned an invalid resume range', ); } } else if (response.status !== 200) { throw new PermanentManagedRuntimeDownloadError( `Official Ollama runtime returned unexpected HTTP ${response.status}`, ); } const expectedResponseBytes = artifact.sizeBytes - offset; const declaredSize = Number(response.headers.get('content-length')); if (Number.isFinite(declaredSize) && declaredSize !== expectedResponseBytes) { throw new PermanentManagedRuntimeDownloadError( `Official Ollama runtime size mismatch: expected ${expectedResponseBytes}, got ${declaredSize}`, ); } let downloaded = offset; const verifier = new Transform({ transform(chunk: Buffer, _encoding, callback) { downloaded += chunk.length; if (downloaded > artifact.sizeBytes) { callback(new PermanentManagedRuntimeDownloadError( `Official Ollama runtime exceeded the expected ${artifact.sizeBytes} bytes`, )); return; } callback(null, chunk); }, }); const body = Readable.fromWeb(response.body as import('node:stream/web').ReadableStream); await pipeline( body, verifier, createWriteStream(archive, { flags: offset > 0 ? 'a' : 'w' }), ); if (downloaded !== artifact.sizeBytes || statSync(archive).size !== artifact.sizeBytes) { throw new Error( `Official Ollama runtime size mismatch: expected ${artifact.sizeBytes}, got ${statSync(archive).size}`, ); } } const digest = createHash('sha256'); for await (const chunk of createReadStream(archive)) digest.update(chunk as Buffer); if (digest.digest('hex') !== artifact.sha256) { throw new PermanentManagedRuntimeDownloadError( 'Official Ollama runtime checksum verification failed', ); } return; } catch (error) { if (error instanceof PermanentManagedRuntimeDownloadError || attempt === DOWNLOAD_MAX_ATTEMPTS) throw error; await new Promise((resolve) => setTimeout( resolve, DOWNLOAD_RETRY_DELAYS_MS[attempt - 1] ?? DOWNLOAD_RETRY_DELAYS_MS.at(-1)!, )); } } } private async installInternal(): Promise<{ executable: string; installedNow: boolean }> { this.assertTrustedTarget(); if (!this.artifact) throw new Error(`Managed Ollama is unsupported on ${process.platform}/${process.arch}`); if (!isLoopbackEndpoint(this.baseUrl)) throw new Error('Managed Ollama requires a loopback OLLAMA_HOST'); await mkdir(this.root, { recursive: true }); const lock = await this.acquireInstallLock(); try { const concurrentlyInstalled = this.getInstalledExecutable(); if (concurrentlyInstalled) return { executable: concurrentlyInstalled, installedNow: false }; await this.removeOrphanedInstallAttempts(); const finalDir = this.versionDir()!; if (existsSync(finalDir)) await removeWithin(this.root, finalDir, this.rmImpl); const attemptId = `${process.pid}-${randomUUID()}`; const versionKey = this.installVersionKey(); const staging = path.join(this.root, `.install-attempt-${versionKey}.${attemptId}`); const archive = path.join(this.root, `.${this.artifact.filename}.${versionKey}.${attemptId}.part`); await mkdir(staging, { recursive: true }); try { await this.downloadArtifact(archive); await this.extractArchive(archive, staging, this.artifact); const executable = await findExecutable(staging, this.artifact.executableName); if (this.artifact.platform !== 'win32') chmodSync(executable, 0o755); const relativeExecutable = path.relative(staging, executable); const metadata: InstallMetadata = { version: this.artifact.version, sha256: this.artifact.sha256, executable: relativeExecutable, installedAt: new Date().toISOString(), }; writeFileSync(path.join(staging, 'install.json'), `${JSON.stringify(metadata, null, 2)}\n`, { flag: 'wx' }); return await this.publishInstall(staging, finalDir); } finally { await this.cleanupInstallAttempt([archive, staging]); } } finally { await this.releaseInstallLock(lock); } } private async waitForInstalledExecutable(): Promise { for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) { const executable = this.getInstalledExecutable(); if (executable) return executable; if (attempt < FILESYSTEM_ATTEMPTS) await waitForFilesystemRetry(attempt); } return null; } private async publishInstall( staging: string, finalDir: string, ): Promise<{ executable: string; installedNow: boolean }> { for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) { try { await this.renameImpl(staging, finalDir); } catch (error) { const winner = existsSync(finalDir) ? await this.waitForInstalledExecutable() : null; if (winner) return { executable: winner, installedNow: false }; if (!isTransientFilesystemError(error) || attempt === FILESYSTEM_ATTEMPTS) throw error; await waitForFilesystemRetry(attempt); continue; } const executable = await this.waitForInstalledExecutable(); if (!executable) throw new Error('Published Ollama runtime failed install metadata validation'); return { executable, installedNow: true }; } throw new Error('Managed Ollama runtime publication exhausted all retry attempts'); } private fallbackCandidate( loaded: RuntimeStateLoad, failedArtifact: OllamaRuntimeArtifact, ): { artifact: OllamaRuntimeArtifact; executable: string } | null { const artifact = loaded.kind === 'valid' ? this.selectFallbackArtifact(loaded, failedArtifact) : loaded.kind === 'invalid' && loaded.recoveryArtifact ? loaded.recoveryArtifact : this.legacyStartCandidate()?.artifact ?? null; if (!artifact || this.sameArtifact(artifact, failedArtifact)) return null; const executable = this.getInstalledExecutable(artifact); return executable ? { artifact, executable } : null; } private async rollbackAfterFailure( failedArtifact: OllamaRuntimeArtifact, fallback: { artifact: OllamaRuntimeArtifact; executable: string } | null, cause: unknown, ): Promise { if (cause instanceof ManagedRuntimeTerminationError) throw cause; await this.stopInternal(); if (!fallback) throw cause; try { const started = await this.start(fallback.executable, fallback.artifact); if (!started || !this.runningArtifact || !this.sameArtifact(this.runningArtifact, fallback.artifact)) { throw new Error(`Verified fallback runtime ${fallback.artifact.version} was not started by Waggle`); } } catch (rollbackError) { throw new Error( `Managed Ollama ${failedArtifact.version} failed and verified fallback ${fallback.artifact.version} could not start: ${this.rollbackReason(rollbackError)}`, { cause }, ); } const rollback: ManagedRuntimeRollback = { failedVersion: failedArtifact.version, restoredVersion: fallback.artifact.version, occurredAt: new Date().toISOString(), reason: this.rollbackReason(cause), }; try { await this.persistRollback(failedArtifact, fallback.artifact, rollback); } catch (stateError) { this.reportWarning('Could not persist the managed Ollama rollback receipt', stateError); } throw new ManagedRuntimeRollbackError(rollback, cause); } private async recoverCrashedRuntime(): Promise { const artifact = this.pendingCrashRecovery; if (!artifact) return null; if (this.crashRecoveryConsumed) { throw this.markTerminationUnconfirmed('managed Ollama crash-recovery budget is exhausted'); } const executable = this.getInstalledExecutable(artifact); if (!executable) { throw this.markTerminationUnconfirmed('verified managed Ollama runtime disappeared before crash recovery'); } if (!await this.endpointQuiescent(this.baseUrl)) { throw this.markTerminationUnconfirmed( 'owned Ollama daemon exited but its loopback endpoint remains occupied', ); } this.assertTerminationConfirmed(); this.crashRecoveryConsumed = true; this.pendingCrashRecovery = null; try { const started = await this.start(executable, artifact, true); if (!started || !this.runningArtifact || !this.sameArtifact(this.runningArtifact, artifact)) { throw new Error(`verified runtime ${artifact.version} was not restarted by Waggle`); } } catch (error) { throw this.markTerminationUnconfirmed( `managed Ollama crash recovery failed: ${this.rollbackReason(error)}`, ); } return { installedNow: false, startedNow: true, endpoint: this.baseUrl, status: this.getStatus(), }; } private queueCrashRecovery(artifact: OllamaRuntimeArtifact): void { this.pendingCrashRecovery = artifact; const recovery = this.withActivationLock(async () => { this.assertTerminationConfirmed(); await this.recoverCrashedRuntime(); }); void recovery.catch((error: unknown) => { if (!this.terminationFailure) { this.markTerminationUnconfirmed( `managed Ollama crash recovery failed: ${this.rollbackReason(error)}`, ); } }); } async startInstalled(): Promise { return this.withActivationLock(async () => { this.assertTerminationConfirmed(); let recovered = await this.recoverCrashedRuntime(); if (recovered) return recovered; const endpointReady = await this.probe(this.baseUrl); this.assertTerminationConfirmed(); recovered = await this.recoverCrashedRuntime(); if (recovered) return recovered; if (endpointReady) { return { installedNow: false, startedNow: false, endpoint: this.baseUrl, status: this.getStatus() }; } const loaded = this.loadRuntimeState(); const candidate = loaded.kind === 'valid' ? { artifact: loaded.activeArtifact, executable: this.getInstalledExecutable(loaded.activeArtifact), } : loaded.kind === 'invalid' && loaded.recoveryArtifact ? { artifact: loaded.recoveryArtifact, executable: this.getInstalledExecutable(loaded.recoveryArtifact), } : this.legacyStartCandidate(); if (!candidate?.executable) { throw new Error('Verified Waggle-managed Ollama runtime is not installed'); } const fallback = this.fallbackCandidate(loaded, candidate.artifact); let startedNow: boolean; try { startedNow = await this.start(candidate.executable, candidate.artifact); if (startedNow && loaded.kind !== 'valid') await this.promoteActive(candidate.artifact); } catch (error) { return this.rollbackAfterFailure(candidate.artifact, fallback, error); } return { installedNow: false, startedNow, endpoint: this.baseUrl, status: this.getStatus(), }; }); } async ensureReady(): Promise { return this.withActivationLock(async () => { this.assertTerminationConfirmed(); let recovered = await this.recoverCrashedRuntime(); if (recovered) return recovered; const endpointReady = await this.probe(this.baseUrl); this.assertTerminationConfirmed(); recovered = await this.recoverCrashedRuntime(); if (recovered) return recovered; if (endpointReady) { return { installedNow: false, startedNow: false, endpoint: this.baseUrl, status: this.getStatus() }; } const target = this.assertTrustedTarget(); const loadedBefore = this.loadRuntimeState(); const fallback = this.fallbackCandidate(loadedBefore, target); let installation: { executable: string; installedNow: boolean }; try { installation = await this.install(); } catch (error) { return this.rollbackAfterFailure(target, fallback, error); } let startedNow: boolean; try { startedNow = await this.start(installation.executable, target); if (startedNow) await this.promoteActive(target); } catch (error) { return this.rollbackAfterFailure(target, fallback, error); } return { installedNow: installation.installedNow, startedNow, endpoint: this.baseUrl, status: this.getStatus(), }; }); } private async start( executable: string, artifact: OllamaRuntimeArtifact, requireQuiescentEndpoint = false, ): Promise { if (this.startPromise) return this.startPromise; this.assertTerminationConfirmed(); const endpointReady = await this.probe(this.baseUrl); this.assertTerminationConfirmed(); const recovered = await this.recoverCrashedRuntime(); if (recovered) return false; if (endpointReady) { if (requireQuiescentEndpoint) { throw new ManagedRuntimeTerminationError('managed Ollama loopback endpoint remains occupied'); } return false; } if (requireQuiescentEndpoint && !await this.endpointQuiescent(this.baseUrl)) { throw new ManagedRuntimeTerminationError('managed Ollama loopback endpoint remains occupied'); } if (this.child) { if (this.child.exitCode === null) await this.stopInternal(); else if (this.child.exitCode === WATCHDOG_EXIT_OWNED_DAEMON_EXITED) { if (!await this.endpointQuiescent(this.baseUrl)) { throw this.markTerminationUnconfirmed('owned Ollama daemon exited but its loopback endpoint remains occupied'); } this.child = null; this.runningArtifact = null; } else if (this.child.exitCode !== WATCHDOG_EXIT_TERMINATION_CONFIRMED) { throw this.markTerminationUnconfirmed(`watchdog exited with code ${this.child.exitCode}`); } else { this.child = null; this.runningArtifact = null; } } this.assertTerminationConfirmed(); this.startPromise = this.startInternal(executable, artifact).finally(() => { this.startPromise = null; }); return this.startPromise; } private async startInternal(executable: string, artifact: OllamaRuntimeArtifact): Promise { await mkdir(this.modelsDir, { recursive: true }); await mkdir(this.profileDir, { recursive: true }); const startTimeoutMs = this.platform === 'win32' ? WINDOWS_START_TIMEOUT_MS : START_TIMEOUT_MS; const startedAt = Date.now(); let stdoutRawTail = ''; let stderrRawTail = ''; const child = this.spawnImpl(process.execPath, [ '-e', MANAGED_OLLAMA_WATCHDOG_SOURCE, executable, JSON.stringify(['serve']), ], { env: buildManagedOllamaEnv(this.baseUrl, this.modelsDir, process.env, this.profileDir), windowsHide: true, shell: false, stdio: ['ignore', 'pipe', 'pipe', 'ipc'], }); const captureStdout = (chunk: string | Buffer) => { stdoutRawTail = appendRawDiagnosticTail(stdoutRawTail, chunk); }; const captureStderr = (chunk: string | Buffer) => { stderrRawTail = appendRawDiagnosticTail(stderrRawTail, chunk); }; const ignoreDiagnosticStreamError = () => {}; child.stdout?.on('error', ignoreDiagnosticStreamError); child.stderr?.on('error', ignoreDiagnosticStreamError); child.stdout?.on('data', captureStdout); child.stderr?.on('data', captureStderr); const releaseDiagnosticCapture = () => { child.stdout?.off('data', captureStdout); child.stderr?.off('data', captureStderr); child.stdout?.resume(); child.stderr?.resume(); }; const reportStartupDiagnostics = (reason: string) => { const elapsedMs = Math.max(0, Date.now() - startedAt); const stdoutTail = formatDiagnosticTail(stdoutRawTail); const stderrTail = formatDiagnosticTail(stderrRawTail); const watchdogState = child.exitCode === null ? 'running' : `exited (${child.exitCode})`; this.reportWarning( `Managed Ollama ${artifact.version} startup failed`, new Error([ `reason=${reason}`, `endpoint=${this.baseUrl.replace(/\/+$/, '')}/api/tags`, `elapsedMs=${elapsedMs}`, `watchdog=${watchdogState}`, `stdoutTail=${stdoutTail || ''}`, `stderrTail=${stderrTail || ''}`, ].join('\n')), ); }; this.child = child; let startupError: Error | null = null; child.once('error', (error) => { startupError = error; if (this.child === child) { this.child = null; this.runningArtifact = null; } }); child.once('exit', (code) => { if (this.child === child) { const readyArtifact = this.runningArtifact && this.sameArtifact(this.runningArtifact, artifact) ? this.runningArtifact : null; const stopWasRequested = this.stopRequestedChildren.has(child); if (code === WATCHDOG_EXIT_TERMINATION_CONFIRMED) { this.child = null; this.terminationFailure = null; this.pendingCrashRecovery = null; } else if (code === WATCHDOG_EXIT_OWNED_DAEMON_EXITED) { this.child = null; if (readyArtifact && !stopWasRequested && !this.stopInProgress && !this.terminationFailure) { if (this.crashRecoveryConsumed) { this.markTerminationUnconfirmed('managed Ollama replacement exited after its one recovery attempt'); } else { this.queueCrashRecovery(readyArtifact); } } } else { this.markTerminationUnconfirmed( code === null ? 'watchdog exited without a success code' : `watchdog exited with code ${code}`, ); } this.runningArtifact = null; } }); try { const deadline = Date.now() + startTimeoutMs; while (Date.now() < deadline) { if (startupError) throw startupError; if (child.exitCode !== null) break; const ready = await this.probe(this.baseUrl); if (startupError) throw startupError; if (child.exitCode !== null || this.child !== child) break; if (ready) { this.runningArtifact = artifact; releaseDiagnosticCapture(); return true; } await new Promise((resolve) => setTimeout(resolve, 500)); } } catch (error) { if (startupError && child.pid === undefined) { reportStartupDiagnostics('watchdog spawn error'); releaseDiagnosticCapture(); if (this.child === child) this.child = null; this.runningArtifact = null; throw error; } try { await this.stopChild(child); } catch (stopError) { await waitForDiagnosticStreams(child); reportStartupDiagnostics('startup error with unconfirmed cleanup'); releaseDiagnosticCapture(); throw stopError; } await waitForDiagnosticStreams(child); reportStartupDiagnostics('startup error'); releaseDiagnosticCapture(); if (this.child === child) this.child = null; if (this.runningArtifact === artifact) this.runningArtifact = null; throw error; } const watchdogWasRunning = child.exitCode === null; try { await this.stopChild(child); } catch (stopError) { await waitForDiagnosticStreams(child); reportStartupDiagnostics('readiness timeout with unconfirmed cleanup'); releaseDiagnosticCapture(); throw stopError; } await waitForDiagnosticStreams(child); reportStartupDiagnostics('readiness timeout'); releaseDiagnosticCapture(); if (this.child === child) this.child = null; if (this.runningArtifact === artifact) this.runningArtifact = null; throw new Error( `Waggle-managed Ollama ${artifact.version} did not make ` + `${this.baseUrl.replace(/\/+$/, '')}/api/tags ready within ${startTimeoutMs}ms ` + `(watchdog ${watchdogWasRunning ? 'remained live until cleanup' : 'exited before readiness'})`, ); } private requestStop(child: ChildProcess): void { try { if (child.connected) child.send('shutdown'); } catch { /* a disconnected watchdog must confirm shutdown by exiting cleanly */ } } private markTerminationUnconfirmed(reason: string): ManagedRuntimeTerminationError { this.pendingCrashRecovery = null; this.terminationFailure ??= new ManagedRuntimeTerminationError(reason); return this.terminationFailure; } private assertTerminationConfirmed(): void { if (this.terminationFailure) throw this.terminationFailure; } private async stopChild(child: ChildProcess): Promise { this.stopRequestedChildren.add(child); if (child.exitCode !== null) { return this.confirmWatchdogExit(child.exitCode); } const exitPromise = new Promise((resolve) => { const onExit = (code: number | null) => { clearTimeout(timeout); resolve(code); }; const timeout = setTimeout(() => { child.off('exit', onExit); resolve(undefined); }, STOP_TIMEOUT_MS); child.once('exit', onExit); }); this.requestStop(child); const exitCode = await exitPromise; if (exitCode === undefined) { throw this.markTerminationUnconfirmed('watchdog did not confirm process-tree termination'); } return this.confirmWatchdogExit(exitCode); } private async confirmWatchdogExit(exitCode: number | null): Promise { if (exitCode === WATCHDOG_EXIT_TERMINATION_CONFIRMED) { this.terminationFailure = null; return; } if (exitCode === WATCHDOG_EXIT_OWNED_DAEMON_EXITED) { if (await this.endpointQuiescent(this.baseUrl)) return; throw this.markTerminationUnconfirmed('owned Ollama daemon exited but its loopback endpoint remains occupied'); } throw this.markTerminationUnconfirmed( exitCode === null ? 'watchdog exited without a success code' : `watchdog exited with code ${exitCode}`, ); } private async stopInternal(): Promise { this.stopInProgress = true; this.pendingCrashRecovery = null; try { const child = this.child; if (!child) return; this.stopRequestedChildren.add(child); if (child.exitCode !== null) { await this.confirmWatchdogExit(child.exitCode); if (this.child === child) this.child = null; this.runningArtifact = null; return; } await this.stopChild(child); if (this.child === child) this.child = null; this.runningArtifact = null; } finally { this.pendingCrashRecovery = null; this.stopInProgress = false; } } async stop(): Promise { return this.withActivationLock(() => this.stopInternal()); } } async function defaultEndpointQuiescent(baseUrl: string): Promise { const endpoint = new URL(baseUrl); const host = endpoint.hostname.replace(/^\[|\]$/g, ''); const port = Number.parseInt(endpoint.port || '80', 10); return new Promise((resolve) => { const server = createServer(); server.unref(); server.once('error', () => resolve(false)); server.listen({ host, port, exclusive: true }, () => { server.close((error) => resolve(error === undefined)); }); }); }