- Pinned Ollama 0.32.3 (and rollback 0.32.0) Linux x64/arm64 artifacts with their published SHA-256 and sizes; .tar.zst archives are decompressed by Node 22's zstd and unpacked by the system tar. - install.sh sets ONNXRUNTIME_NODE_INSTALL_CUDA=skip: onnxruntime-node otherwise downloads CUDA libraries on every Linux x64 install. - Drop the root sqlite-vec-windows-x64 dependency; the sqlite-vec package already carries per-platform binaries.
1812 lines
69 KiB
TypeScript
1812 lines
69 KiB
TypeScript
import Database from 'better-sqlite3';
|
|
import type { Database as DatabaseType } from 'better-sqlite3';
|
|
import { createHash, randomUUID } from 'node:crypto';
|
|
import { spawn, execFile, type ChildProcess, type SpawnOptions } from 'node:child_process';
|
|
import {
|
|
chmodSync,
|
|
closeSync,
|
|
createReadStream,
|
|
createWriteStream,
|
|
existsSync,
|
|
fsyncSync,
|
|
openSync,
|
|
readFileSync,
|
|
realpathSync,
|
|
statSync,
|
|
writeFileSync,
|
|
} from 'node:fs';
|
|
import { mkdir, readdir, rename, rm } from 'node:fs/promises';
|
|
import { createServer } from 'node:net';
|
|
import path from 'node:path';
|
|
import { Readable, Transform } from 'node:stream';
|
|
import { pipeline } from 'node:stream/promises';
|
|
import { promisify } from 'node:util';
|
|
import zlib from 'node:zlib';
|
|
import { createLogger } from './logger.js';
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
const log = createLogger('managed-ollama');
|
|
const OLLAMA_TARGET_VERSION = '0.32.3';
|
|
const OLLAMA_ROLLBACK_VERSION = '0.32.0';
|
|
const ollamaReleaseRoot = (version: string): string =>
|
|
`https://github.com/ollama/ollama/releases/download/v${version}`;
|
|
const DOWNLOAD_TIMEOUT_MS = 45 * 60_000;
|
|
const DOWNLOAD_MAX_ATTEMPTS = 3;
|
|
const DOWNLOAD_RETRY_DELAYS_MS = [250, 1_000] as const;
|
|
const START_TIMEOUT_MS = 45_000;
|
|
const WINDOWS_START_TIMEOUT_MS = 120_000;
|
|
const START_DIAGNOSTIC_RAW_TAIL_BYTES = 64 * 1024;
|
|
const START_DIAGNOSTIC_TAIL_BYTES = 4 * 1024;
|
|
const START_DIAGNOSTIC_DRAIN_MS = 250;
|
|
const STOP_TIMEOUT_MS = 5_000;
|
|
const WATCHDOG_EXIT_TERMINATION_CONFIRMED = 0;
|
|
const WATCHDOG_EXIT_CLEANUP_UNCONFIRMED = 1;
|
|
const WATCHDOG_EXIT_OWNED_DAEMON_EXITED = 2;
|
|
const INSTALL_LOCK_WAIT_MS = DOWNLOAD_TIMEOUT_MS + 15 * 60_000;
|
|
const INSTALL_LOCK_BUSY_TIMEOUT_MS = 0;
|
|
const INSTALL_LOCK_RETRY_MIN_MS = 200;
|
|
const INSTALL_LOCK_RETRY_MAX_MS = 1_000;
|
|
const TRANSIENT_DOWNLOAD_STATUS = new Set([408, 425, 429, 500, 502, 503, 504]);
|
|
|
|
class PermanentManagedRuntimeDownloadError extends Error {}
|
|
|
|
/**
|
|
* The desktop shell terminates the Node sidecar directly on exit, which bypasses
|
|
* Fastify's onClose hooks on Windows. Keep the model daemon behind a tiny Node
|
|
* supervisor connected to the sidecar over IPC: loss of that IPC handle means
|
|
* the parent died, so the supervisor terminates the entire Ollama process tree.
|
|
*/
|
|
export const MANAGED_OLLAMA_WATCHDOG_SOURCE = String.raw`
|
|
const { spawn } = require('node:child_process');
|
|
const executable = process.argv[1];
|
|
const args = JSON.parse(process.argv[2] || '[]');
|
|
const child = spawn(executable, args, {
|
|
env: process.env,
|
|
windowsHide: true,
|
|
shell: false,
|
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
detached: process.platform !== 'win32',
|
|
});
|
|
process.stdout.on('error', () => {});
|
|
process.stderr.on('error', () => {});
|
|
child.stdout?.on('error', () => {});
|
|
child.stderr?.on('error', () => {});
|
|
child.stdout?.pipe(process.stdout);
|
|
child.stderr?.pipe(process.stderr);
|
|
let stopping = false;
|
|
let forceTimer;
|
|
let drainTimer;
|
|
let outputClosed = false;
|
|
let pendingFinishCode;
|
|
const finish = (code) => {
|
|
if (forceTimer) clearTimeout(forceTimer);
|
|
if (drainTimer) clearTimeout(drainTimer);
|
|
process.exit(code);
|
|
};
|
|
const finishAfterOutput = (code) => {
|
|
if (outputClosed) return finish(code);
|
|
if (pendingFinishCode === undefined) pendingFinishCode = code;
|
|
if (!drainTimer) {
|
|
drainTimer = setTimeout(() => finish(pendingFinishCode), ${START_DIAGNOSTIC_DRAIN_MS});
|
|
drainTimer.unref();
|
|
}
|
|
};
|
|
const stopTree = () => {
|
|
if (stopping) return;
|
|
stopping = true;
|
|
if (!child.pid) return finish(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED});
|
|
forceTimer = setTimeout(() => finish(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}), 4000);
|
|
if (process.platform === 'win32') {
|
|
const killer = spawn('taskkill', ['/PID', String(child.pid), '/T', '/F'], {
|
|
windowsHide: true,
|
|
shell: false,
|
|
stdio: 'ignore',
|
|
});
|
|
killer.once('error', () => finishAfterOutput(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}));
|
|
killer.once('exit', (code) => finishAfterOutput(code === 0
|
|
? ${WATCHDOG_EXIT_TERMINATION_CONFIRMED}
|
|
: ${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}));
|
|
} else {
|
|
try { process.kill(-child.pid, 'SIGTERM'); } catch { try { child.kill(); } catch {} }
|
|
setTimeout(() => {
|
|
try { process.kill(-child.pid, 'SIGKILL'); } catch {}
|
|
setTimeout(() => {
|
|
try {
|
|
process.kill(-child.pid, 0);
|
|
finish(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED});
|
|
} catch (error) {
|
|
finish(error && error.code === 'ESRCH'
|
|
? ${WATCHDOG_EXIT_TERMINATION_CONFIRMED}
|
|
: ${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED});
|
|
}
|
|
}, 100).unref();
|
|
}, 1500).unref();
|
|
}
|
|
};
|
|
child.once('spawn', () => {
|
|
if (process.send) process.send({ type: 'spawned', pid: child.pid });
|
|
});
|
|
child.once('error', () => finishAfterOutput(${WATCHDOG_EXIT_CLEANUP_UNCONFIRMED}));
|
|
child.once('exit', () => {
|
|
if (!stopping) finishAfterOutput(${WATCHDOG_EXIT_OWNED_DAEMON_EXITED});
|
|
});
|
|
child.once('close', () => {
|
|
outputClosed = true;
|
|
if (pendingFinishCode !== undefined) finish(pendingFinishCode);
|
|
});
|
|
process.once('disconnect', stopTree);
|
|
process.on('message', (message) => { if (message === 'shutdown') stopTree(); });
|
|
process.once('SIGTERM', stopTree);
|
|
process.once('SIGINT', stopTree);
|
|
`;
|
|
|
|
export interface OllamaRuntimeArtifact {
|
|
version: string;
|
|
platform: NodeJS.Platform;
|
|
arch: string;
|
|
filename: string;
|
|
url: string;
|
|
sha256: string;
|
|
sizeBytes: number;
|
|
executableName: string;
|
|
}
|
|
|
|
/**
|
|
* Pinned official standalone artifacts. Updating this table is an intentional
|
|
* supply-chain change: version, byte size, and GitHub-published SHA-256 must move
|
|
* together. The runtime is downloaded only from this hardcoded release URL.
|
|
*/
|
|
export const OLLAMA_RUNTIME_ARTIFACTS: ReadonlyArray<OllamaRuntimeArtifact> = [
|
|
{
|
|
version: OLLAMA_ROLLBACK_VERSION,
|
|
platform: 'win32',
|
|
arch: 'x64',
|
|
filename: 'ollama-windows-amd64.zip',
|
|
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-windows-amd64.zip`,
|
|
sha256: '56561a8f0a904483303c610e61af61c5a7b6f5496ce3707e207d25d4ff67b89e',
|
|
sizeBytes: 1_503_047_573,
|
|
executableName: 'ollama.exe',
|
|
},
|
|
{
|
|
version: OLLAMA_ROLLBACK_VERSION,
|
|
platform: 'win32',
|
|
arch: 'arm64',
|
|
filename: 'ollama-windows-arm64.zip',
|
|
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-windows-arm64.zip`,
|
|
sha256: '82b7d36b63e62a44d3f9853c2f8edb829cf871eaf722ce20070e09e96922c0cc',
|
|
sizeBytes: 16_346_970,
|
|
executableName: 'ollama.exe',
|
|
},
|
|
{
|
|
version: OLLAMA_ROLLBACK_VERSION,
|
|
platform: 'darwin',
|
|
arch: 'x64',
|
|
filename: 'ollama-darwin.tgz',
|
|
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-darwin.tgz`,
|
|
sha256: '3b12a49c6c4cbafd7ffba5ccba60cbf80274cdc22eea3ead79c646aba888174c',
|
|
sizeBytes: 145_356_966,
|
|
executableName: 'ollama',
|
|
},
|
|
{
|
|
version: OLLAMA_ROLLBACK_VERSION,
|
|
platform: 'darwin',
|
|
arch: 'arm64',
|
|
filename: 'ollama-darwin.tgz',
|
|
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-darwin.tgz`,
|
|
sha256: '3b12a49c6c4cbafd7ffba5ccba60cbf80274cdc22eea3ead79c646aba888174c',
|
|
sizeBytes: 145_356_966,
|
|
executableName: 'ollama',
|
|
},
|
|
{
|
|
version: OLLAMA_ROLLBACK_VERSION,
|
|
platform: 'linux',
|
|
arch: 'x64',
|
|
filename: 'ollama-linux-amd64.tar.zst',
|
|
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-linux-amd64.tar.zst`,
|
|
sha256: '56362d7609dfa9e35aaebb7c9cab25605d8f0528ec3d5d585dc83d6642002bab',
|
|
sizeBytes: 1_436_128_693,
|
|
executableName: 'ollama',
|
|
},
|
|
{
|
|
version: OLLAMA_ROLLBACK_VERSION,
|
|
platform: 'linux',
|
|
arch: 'arm64',
|
|
filename: 'ollama-linux-arm64.tar.zst',
|
|
url: `${ollamaReleaseRoot(OLLAMA_ROLLBACK_VERSION)}/ollama-linux-arm64.tar.zst`,
|
|
sha256: '2c1fbc47a5351c74f5400d7e4b1104bb470291af5d2f425c37c151487a477ad6',
|
|
sizeBytes: 1_556_234_636,
|
|
executableName: 'ollama',
|
|
},
|
|
{
|
|
version: OLLAMA_TARGET_VERSION,
|
|
platform: 'win32',
|
|
arch: 'x64',
|
|
filename: 'ollama-windows-amd64.zip',
|
|
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-windows-amd64.zip`,
|
|
sha256: 'c66dd7dde4d5ec4822eaa57dd421d51aa7c633a3ff36a974040837df73a5969e',
|
|
sizeBytes: 1_457_806_156,
|
|
executableName: 'ollama.exe',
|
|
},
|
|
{
|
|
version: OLLAMA_TARGET_VERSION,
|
|
platform: 'win32',
|
|
arch: 'arm64',
|
|
filename: 'ollama-windows-arm64.zip',
|
|
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-windows-arm64.zip`,
|
|
sha256: '8431fc4ccf7e86a273dbcc52e81585fc92133e4d6f9c1abdd3663e2568f2fe90',
|
|
sizeBytes: 209_405_705,
|
|
executableName: 'ollama.exe',
|
|
},
|
|
{
|
|
version: OLLAMA_TARGET_VERSION,
|
|
platform: 'darwin',
|
|
arch: 'x64',
|
|
filename: 'ollama-darwin.tgz',
|
|
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-darwin.tgz`,
|
|
sha256: '14462bd438815eb2c1d4c61224744637131ab744e858a2e2562e7fc7fc2c4f7d',
|
|
sizeBytes: 145_790_989,
|
|
executableName: 'ollama',
|
|
},
|
|
{
|
|
version: OLLAMA_TARGET_VERSION,
|
|
platform: 'darwin',
|
|
arch: 'arm64',
|
|
filename: 'ollama-darwin.tgz',
|
|
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-darwin.tgz`,
|
|
sha256: '14462bd438815eb2c1d4c61224744637131ab744e858a2e2562e7fc7fc2c4f7d',
|
|
sizeBytes: 145_790_989,
|
|
executableName: 'ollama',
|
|
},
|
|
{
|
|
version: OLLAMA_TARGET_VERSION,
|
|
platform: 'linux',
|
|
arch: 'x64',
|
|
filename: 'ollama-linux-amd64.tar.zst',
|
|
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-linux-amd64.tar.zst`,
|
|
sha256: '2597d74fbe654ef6a37db56f771cf37d4a85c6bde4018127874e3927d3113800',
|
|
sizeBytes: 1_421_853_873,
|
|
executableName: 'ollama',
|
|
},
|
|
{
|
|
version: OLLAMA_TARGET_VERSION,
|
|
platform: 'linux',
|
|
arch: 'arm64',
|
|
filename: 'ollama-linux-arm64.tar.zst',
|
|
url: `${ollamaReleaseRoot(OLLAMA_TARGET_VERSION)}/ollama-linux-arm64.tar.zst`,
|
|
sha256: '61af0def977ffc9e76cc961f9505d621805653c63f444796de2f77b4eaaa7047',
|
|
sizeBytes: 1_542_500_407,
|
|
executableName: 'ollama',
|
|
}
|
|
];
|
|
|
|
interface InstallMetadata {
|
|
version: string;
|
|
sha256: string;
|
|
executable: string;
|
|
installedAt: string;
|
|
}
|
|
|
|
interface RuntimeReceipt {
|
|
version: string;
|
|
sha256: string;
|
|
executable: string;
|
|
installedAt: string;
|
|
}
|
|
|
|
interface RuntimeActivationState {
|
|
schemaVersion: 1;
|
|
active: RuntimeReceipt;
|
|
previous: RuntimeReceipt | null;
|
|
lastRollback: ManagedRuntimeRollback | null;
|
|
}
|
|
|
|
type RuntimeStateLoad =
|
|
| { kind: 'missing' }
|
|
| {
|
|
kind: 'invalid';
|
|
reason: string;
|
|
recoveryArtifact?: OllamaRuntimeArtifact;
|
|
recoveryReceipt?: RuntimeReceipt;
|
|
lastRollback?: ManagedRuntimeRollback | null;
|
|
}
|
|
| {
|
|
kind: 'valid';
|
|
state: RuntimeActivationState;
|
|
activeArtifact: OllamaRuntimeArtifact;
|
|
previousArtifact: OllamaRuntimeArtifact | null;
|
|
};
|
|
|
|
interface InstallLockClaim {
|
|
database: DatabaseType;
|
|
}
|
|
|
|
export interface ManagedOllamaStatus {
|
|
source: 'waggle-managed';
|
|
supported: boolean;
|
|
installed: boolean;
|
|
running: boolean;
|
|
targetVersion: string | null;
|
|
version: string | null;
|
|
targetInstalled: boolean;
|
|
activeVersion: string | null;
|
|
previousVersion: string | null;
|
|
fallbackActive: boolean;
|
|
rollback: {
|
|
available: boolean;
|
|
active: boolean;
|
|
lastAttempt: ManagedRuntimeRollback | null;
|
|
};
|
|
artifactSizeBytes: number | null;
|
|
downloadRequired: boolean;
|
|
dockerRequired: false;
|
|
reason?: string;
|
|
}
|
|
|
|
export interface ManagedRuntimeRollback {
|
|
failedVersion: string;
|
|
restoredVersion: string;
|
|
occurredAt: string;
|
|
reason: string;
|
|
}
|
|
|
|
export class ManagedRuntimeRollbackError extends Error {
|
|
readonly rollback: ManagedRuntimeRollback;
|
|
readonly failedVersion: string;
|
|
readonly restoredVersion: string;
|
|
|
|
constructor(rollback: ManagedRuntimeRollback, cause: unknown) {
|
|
super(
|
|
`Managed Ollama ${rollback.failedVersion} failed; restored verified runtime ${rollback.restoredVersion}: ${rollback.reason}`,
|
|
{ cause },
|
|
);
|
|
this.name = 'ManagedRuntimeRollbackError';
|
|
this.rollback = rollback;
|
|
this.failedVersion = rollback.failedVersion;
|
|
this.restoredVersion = rollback.restoredVersion;
|
|
}
|
|
}
|
|
|
|
class ManagedRuntimeTerminationError extends Error {
|
|
constructor(reason = 'watchdog did not confirm process-tree termination') {
|
|
super(`Managed Ollama ${reason}; refusing to start a replacement runtime`);
|
|
this.name = 'ManagedRuntimeTerminationError';
|
|
}
|
|
}
|
|
|
|
export interface ManagedOllamaReadyResult {
|
|
installedNow: boolean;
|
|
startedNow: boolean;
|
|
endpoint: string;
|
|
status: ManagedOllamaStatus;
|
|
}
|
|
|
|
interface RuntimeDependencies {
|
|
fetchImpl?: typeof fetch;
|
|
extractArchive?: (
|
|
archivePath: string,
|
|
destination: string,
|
|
artifact: OllamaRuntimeArtifact,
|
|
) => Promise<void>;
|
|
spawnImpl?: (file: string, args: readonly string[], options: SpawnOptions) => ChildProcess;
|
|
probe?: (baseUrl: string) => Promise<boolean>;
|
|
endpointQuiescent?: (baseUrl: string) => Promise<boolean>;
|
|
renameImpl?: typeof rename;
|
|
stateRenameImpl?: typeof rename;
|
|
rmImpl?: typeof rm;
|
|
warn?: (message: string, error: unknown) => void;
|
|
artifact?: OllamaRuntimeArtifact | null;
|
|
artifactCatalog?: ReadonlyArray<OllamaRuntimeArtifact>;
|
|
platform?: NodeJS.Platform;
|
|
arch?: string;
|
|
}
|
|
|
|
export function resolveOllamaRuntimeArtifact(
|
|
platform: NodeJS.Platform = process.platform,
|
|
arch: string = process.arch,
|
|
): OllamaRuntimeArtifact | null {
|
|
const targets = OLLAMA_RUNTIME_ARTIFACTS.filter(
|
|
(entry) => entry.platform === platform
|
|
&& entry.arch === arch
|
|
&& entry.version === OLLAMA_TARGET_VERSION,
|
|
);
|
|
return targets.length === 1 ? targets[0] : null;
|
|
}
|
|
|
|
function isLoopbackEndpoint(baseUrl: string): boolean {
|
|
try {
|
|
const url = new URL(baseUrl);
|
|
return url.protocol === 'http:'
|
|
&& (url.hostname === '127.0.0.1' || url.hostname === 'localhost' || url.hostname === '::1')
|
|
&& (url.pathname === '/' || url.pathname === '');
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
export function buildManagedOllamaEnv(
|
|
baseUrl: string,
|
|
modelsDir: string,
|
|
source: NodeJS.ProcessEnv = process.env,
|
|
managedProfileDir?: string,
|
|
): NodeJS.ProcessEnv {
|
|
if (!isLoopbackEndpoint(baseUrl)) {
|
|
throw new Error('Waggle-managed Ollama must bind to an HTTP loopback endpoint');
|
|
}
|
|
const endpoint = new URL(baseUrl);
|
|
return {
|
|
...source,
|
|
...(managedProfileDir
|
|
? { HOME: managedProfileDir, USERPROFILE: managedProfileDir }
|
|
: {}),
|
|
OLLAMA_HOST: endpoint.host,
|
|
OLLAMA_MODELS: modelsDir,
|
|
OLLAMA_NOHISTORY: '1',
|
|
// Ollama otherwise defaults to 4K on common 8 GB GPUs. The qualified
|
|
// 8K-token tool request plus Waggle's 14K synthesis ceiling needs a
|
|
// 32K window to retain the full input and leave the promised output room.
|
|
OLLAMA_CONTEXT_LENGTH: '32768',
|
|
};
|
|
}
|
|
|
|
async function defaultProbe(baseUrl: string): Promise<boolean> {
|
|
try {
|
|
const response = await fetch(`${baseUrl.replace(/\/+$/, '')}/api/tags`, {
|
|
signal: AbortSignal.timeout(3_000),
|
|
});
|
|
if (!response.ok) return false;
|
|
const payload = await response.json() as { models?: unknown };
|
|
return Array.isArray(payload.models);
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
async function defaultExtract(
|
|
archivePath: string,
|
|
destination: string,
|
|
artifact: OllamaRuntimeArtifact,
|
|
): Promise<void> {
|
|
if (artifact.filename.endsWith('.tar.zst')) {
|
|
await extractTarZst(archivePath, destination);
|
|
return;
|
|
}
|
|
const executable = artifact.platform === 'win32' ? 'tar.exe' : 'tar';
|
|
await execFileAsync(executable, ['-xf', archivePath, '-C', destination], {
|
|
windowsHide: true,
|
|
timeout: 10 * 60_000,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Linux releases are zstd-compressed tarballs. Node 22 decompresses zstd itself
|
|
* and the system tar only unpacks; on older Node, tar needs zstd support.
|
|
*/
|
|
export async function extractTarZst(archivePath: string, destination: string): Promise<void> {
|
|
const createZstdDecompress = (zlib as { createZstdDecompress?: () => Transform }).createZstdDecompress;
|
|
if (!createZstdDecompress) {
|
|
await execFileAsync('tar', ['--zstd', '-xf', archivePath, '-C', destination], { timeout: 10 * 60_000 });
|
|
return;
|
|
}
|
|
const tar = spawn('tar', ['-xf', '-', '-C', destination], { stdio: ['pipe', 'ignore', 'pipe'] });
|
|
let stderr = '';
|
|
tar.stderr?.on('data', (chunk: Buffer) => { stderr += chunk.toString(); });
|
|
const exited = new Promise<void>((resolve, reject) => {
|
|
tar.once('error', reject);
|
|
tar.once('close', (code) => (code === 0 ? resolve() : reject(new Error(`tar exited with ${code}: ${stderr.trim()}`))));
|
|
});
|
|
await Promise.all([pipeline(createReadStream(archivePath), createZstdDecompress(), tar.stdin!), exited]);
|
|
}
|
|
|
|
function inside(root: string, candidate: string): boolean {
|
|
const relative = path.relative(path.resolve(root), path.resolve(candidate));
|
|
return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
|
|
}
|
|
|
|
const TRANSIENT_FILESYSTEM_CODES = new Set(['EPERM', 'EACCES', 'EBUSY']);
|
|
const FILESYSTEM_ATTEMPTS = 4;
|
|
const PROCESS_INSTALLS = new Map<string, Promise<{ executable: string; installedNow: boolean }>>();
|
|
|
|
function isTransientFilesystemError(error: unknown): boolean {
|
|
return TRANSIENT_FILESYSTEM_CODES.has((error as NodeJS.ErrnoException)?.code ?? '');
|
|
}
|
|
|
|
function redactManagedRuntimeDiagnostic(value: string): string {
|
|
const redacted = value
|
|
.replace(/(authorization\s*[:=]\s*)[^\r\n]*/gi, '$1[REDACTED]')
|
|
.replace(/(https?:\/\/)[^\s/@:]+:[^\s/@]+@/gi, '$1[REDACTED]@')
|
|
.replace(
|
|
/((?:api[_-]?key|token|password|secret|credential|private[_-]?key)\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s,;\]}]+)/gi,
|
|
'$1[REDACTED]',
|
|
);
|
|
return Array.from(redacted, (character) => {
|
|
const code = character.charCodeAt(0);
|
|
const disallowedControl = (code < 0x20 && code !== 0x09 && code !== 0x0a && code !== 0x0d)
|
|
|| code === 0x7f;
|
|
return disallowedControl ? '?' : character;
|
|
}).join('');
|
|
}
|
|
|
|
function appendRawDiagnosticTail(current: string, chunk: string | Buffer): string {
|
|
const next = Buffer.isBuffer(chunk) ? chunk.toString('utf8') : String(chunk);
|
|
const combined = Buffer.from(current + next, 'utf8');
|
|
if (combined.length <= START_DIAGNOSTIC_RAW_TAIL_BYTES) return combined.toString('utf8');
|
|
return combined
|
|
.subarray(combined.length - START_DIAGNOSTIC_RAW_TAIL_BYTES)
|
|
.toString('utf8');
|
|
}
|
|
|
|
function formatDiagnosticTail(raw: string): string {
|
|
const combined = Buffer.from(redactManagedRuntimeDiagnostic(raw), 'utf8');
|
|
if (combined.length <= START_DIAGNOSTIC_TAIL_BYTES) return combined.toString('utf8');
|
|
return combined
|
|
.subarray(combined.length - START_DIAGNOSTIC_TAIL_BYTES)
|
|
.toString('utf8');
|
|
}
|
|
|
|
async function waitForDiagnosticStreams(child: ChildProcess): Promise<void> {
|
|
const streams = [child.stdout, child.stderr]
|
|
.filter((stream): stream is Readable => stream != null);
|
|
const cleanups: Array<() => void> = [];
|
|
const waits = streams.map((stream) => new Promise<void>((resolve) => {
|
|
if (stream.readableEnded || stream.destroyed) {
|
|
resolve();
|
|
return;
|
|
}
|
|
const finish = () => {
|
|
cleanup();
|
|
resolve();
|
|
};
|
|
const cleanup = () => {
|
|
stream.off('end', finish);
|
|
stream.off('close', finish);
|
|
stream.off('error', finish);
|
|
};
|
|
stream.once('end', finish);
|
|
stream.once('close', finish);
|
|
stream.once('error', finish);
|
|
cleanups.push(cleanup);
|
|
}));
|
|
if (waits.length === 0) return;
|
|
|
|
let timeout: NodeJS.Timeout | undefined;
|
|
try {
|
|
await Promise.race([
|
|
Promise.all(waits),
|
|
new Promise<void>((resolve) => {
|
|
timeout = setTimeout(resolve, START_DIAGNOSTIC_DRAIN_MS);
|
|
}),
|
|
]);
|
|
} finally {
|
|
if (timeout) clearTimeout(timeout);
|
|
for (const cleanup of cleanups) cleanup();
|
|
}
|
|
}
|
|
|
|
async function waitForFilesystemRetry(attempt: number): Promise<void> {
|
|
await new Promise((resolve) => setTimeout(resolve, 25 * attempt));
|
|
}
|
|
|
|
async function removeWithin(root: string, target: string, rmImpl: typeof rm = rm): Promise<void> {
|
|
if (!inside(root, target) || path.resolve(root) === path.resolve(target)) {
|
|
throw new Error(`Refusing to remove path outside managed runtime root: ${target}`);
|
|
}
|
|
for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) {
|
|
try {
|
|
await rmImpl(target, { recursive: true, force: true });
|
|
return;
|
|
} catch (error) {
|
|
if (!isTransientFilesystemError(error) || attempt === FILESYSTEM_ATTEMPTS) throw error;
|
|
await waitForFilesystemRetry(attempt);
|
|
}
|
|
}
|
|
}
|
|
|
|
function isSqliteBusyError(error: unknown): boolean {
|
|
const code = (error as { code?: unknown } | null)?.code;
|
|
return code === 'SQLITE_BUSY' || code === 'SQLITE_BUSY_SNAPSHOT' || code === 'SQLITE_LOCKED';
|
|
}
|
|
|
|
async function findExecutable(root: string, executableName: string): Promise<string> {
|
|
const queue: Array<{ directory: string; depth: number }> = [{ directory: root, depth: 0 }];
|
|
let visited = 0;
|
|
while (queue.length > 0 && visited < 10_000) {
|
|
const current = queue.shift()!;
|
|
const entries = await readdir(current.directory, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
visited += 1;
|
|
const candidate = path.join(current.directory, entry.name);
|
|
if (entry.name.toLowerCase() === executableName.toLowerCase()) {
|
|
const resolved = realpathSync(candidate);
|
|
if (!inside(root, resolved) || !statSync(resolved).isFile()) continue;
|
|
return resolved;
|
|
}
|
|
if (entry.isDirectory() && current.depth < 6) {
|
|
queue.push({ directory: candidate, depth: current.depth + 1 });
|
|
}
|
|
}
|
|
}
|
|
throw new Error(`Official runtime archive did not contain ${executableName}`);
|
|
}
|
|
|
|
export class ManagedOllamaRuntime {
|
|
private readonly root: string;
|
|
private readonly modelsDir: string;
|
|
private readonly profileDir: string;
|
|
private readonly artifact: OllamaRuntimeArtifact | null;
|
|
private readonly artifactCatalog: ReadonlyArray<OllamaRuntimeArtifact>;
|
|
private readonly platform: NodeJS.Platform;
|
|
private readonly arch: string;
|
|
private readonly fetchImpl: typeof fetch;
|
|
private readonly extractArchive: NonNullable<RuntimeDependencies['extractArchive']>;
|
|
private readonly spawnImpl: NonNullable<RuntimeDependencies['spawnImpl']>;
|
|
private readonly probe: NonNullable<RuntimeDependencies['probe']>;
|
|
private readonly endpointQuiescent: NonNullable<RuntimeDependencies['endpointQuiescent']>;
|
|
private readonly renameImpl: NonNullable<RuntimeDependencies['renameImpl']>;
|
|
private readonly stateRenameImpl: NonNullable<RuntimeDependencies['stateRenameImpl']>;
|
|
private readonly rmImpl: NonNullable<RuntimeDependencies['rmImpl']>;
|
|
private readonly warn: NonNullable<RuntimeDependencies['warn']>;
|
|
private installPromise: Promise<{ executable: string; installedNow: boolean }> | null = null;
|
|
private startPromise: Promise<boolean> | null = null;
|
|
private activationPromise: Promise<unknown> | null = null;
|
|
private child: ChildProcess | null = null;
|
|
private runningArtifact: OllamaRuntimeArtifact | null = null;
|
|
private terminationFailure: ManagedRuntimeTerminationError | null = null;
|
|
private pendingCrashRecovery: OllamaRuntimeArtifact | null = null;
|
|
private crashRecoveryConsumed = false;
|
|
private stopInProgress = false;
|
|
private readonly stopRequestedChildren = new WeakSet<ChildProcess>();
|
|
|
|
constructor(
|
|
dataDir: string,
|
|
private readonly baseUrl = 'http://127.0.0.1:11434',
|
|
dependencies: RuntimeDependencies = {},
|
|
) {
|
|
const platform = dependencies.platform ?? dependencies.artifact?.platform ?? process.platform;
|
|
const arch = dependencies.arch ?? dependencies.artifact?.arch ?? process.arch;
|
|
this.platform = platform;
|
|
this.arch = arch;
|
|
this.root = path.resolve(dataDir, 'runtimes', 'ollama');
|
|
this.modelsDir = path.resolve(dataDir, 'models', 'ollama');
|
|
this.profileDir = path.resolve(this.root, 'profile');
|
|
this.artifact = dependencies.artifact === undefined
|
|
? resolveOllamaRuntimeArtifact(platform, arch)
|
|
: dependencies.artifact;
|
|
this.artifactCatalog = dependencies.artifactCatalog
|
|
?? (dependencies.artifact === undefined
|
|
? OLLAMA_RUNTIME_ARTIFACTS
|
|
: (dependencies.artifact ? [dependencies.artifact] : []));
|
|
this.fetchImpl = dependencies.fetchImpl ?? fetch;
|
|
this.extractArchive = dependencies.extractArchive ?? defaultExtract;
|
|
this.spawnImpl = dependencies.spawnImpl ?? ((file, args, options) => spawn(file, args, options));
|
|
this.probe = dependencies.probe ?? defaultProbe;
|
|
this.endpointQuiescent = dependencies.endpointQuiescent ?? defaultEndpointQuiescent;
|
|
this.renameImpl = dependencies.renameImpl ?? rename;
|
|
this.stateRenameImpl = dependencies.stateRenameImpl ?? rename;
|
|
this.rmImpl = dependencies.rmImpl ?? rm;
|
|
this.warn = dependencies.warn ?? ((message, error) => log.warn(message, {
|
|
error: error instanceof Error ? error.message : String(error),
|
|
code: (error as { code?: unknown } | null)?.code ?? null,
|
|
}));
|
|
}
|
|
|
|
getStatus(): ManagedOllamaStatus {
|
|
const executable = this.getInstalledExecutable();
|
|
const targetInstalled = executable !== null;
|
|
const loadedState = this.loadRuntimeState();
|
|
const running = this.child !== null && this.child.exitCode === null;
|
|
const managedRunningVersion = running ? this.runningArtifact?.version ?? null : null;
|
|
const activeVersion = managedRunningVersion
|
|
?? (loadedState.kind === 'valid' ? loadedState.state.active.version : null);
|
|
const previousVersion = loadedState.kind === 'valid'
|
|
? loadedState.state.previous?.version ?? null
|
|
: loadedState.kind === 'invalid'
|
|
? loadedState.recoveryArtifact?.version ?? null
|
|
: null;
|
|
const persistedRollbackActive = loadedState.kind === 'valid'
|
|
&& this.artifact !== null
|
|
&& !this.sameArtifact(loadedState.activeArtifact, this.artifact)
|
|
&& loadedState.state.lastRollback?.failedVersion === this.artifact.version
|
|
&& loadedState.state.lastRollback.restoredVersion === loadedState.activeArtifact.version;
|
|
const fallbackActive = persistedRollbackActive || (running
|
|
&& this.runningArtifact !== null
|
|
&& this.artifact !== null
|
|
&& !this.sameArtifact(this.runningArtifact, this.artifact));
|
|
const rollbackAvailable = loadedState.kind === 'valid'
|
|
&& this.selectFallbackArtifact(loadedState, this.artifact) !== null;
|
|
const supported = this.artifact !== null && isLoopbackEndpoint(this.baseUrl);
|
|
return {
|
|
source: 'waggle-managed',
|
|
supported,
|
|
installed: targetInstalled
|
|
|| loadedState.kind === 'valid'
|
|
|| (loadedState.kind === 'invalid' && loadedState.recoveryArtifact !== undefined)
|
|
|| this.runningArtifact !== null,
|
|
running,
|
|
targetVersion: this.artifact?.version ?? null,
|
|
version: activeVersion ?? (targetInstalled ? this.artifact?.version ?? null : null),
|
|
targetInstalled,
|
|
activeVersion,
|
|
previousVersion,
|
|
fallbackActive,
|
|
rollback: {
|
|
available: rollbackAvailable || fallbackActive,
|
|
active: fallbackActive,
|
|
lastAttempt: loadedState.kind === 'valid'
|
|
? loadedState.state.lastRollback
|
|
: loadedState.kind === 'invalid'
|
|
? loadedState.lastRollback ?? null
|
|
: null,
|
|
},
|
|
artifactSizeBytes: this.artifact?.sizeBytes ?? null,
|
|
downloadRequired: !targetInstalled,
|
|
dockerRequired: false,
|
|
...(!supported
|
|
? { reason: this.artifact === null
|
|
? `No managed Ollama artifact for ${process.platform}/${process.arch}`
|
|
: 'Managed Ollama requires a loopback OLLAMA_HOST' }
|
|
: loadedState.kind === 'invalid'
|
|
? { reason: loadedState.reason }
|
|
: {}),
|
|
};
|
|
}
|
|
|
|
private versionDir(artifact = this.artifact): string | null {
|
|
if (!artifact) return null;
|
|
const candidate = path.join(this.root, artifact.version);
|
|
return inside(this.root, candidate) && path.resolve(candidate) !== this.root ? candidate : null;
|
|
}
|
|
|
|
private installVersionKey(): string {
|
|
return createHash('sha256').update(this.artifact!.version).digest('hex');
|
|
}
|
|
|
|
private installLockDatabasePath(): string {
|
|
return path.join(this.root, `.install-lock-${this.installVersionKey()}.sqlite`);
|
|
}
|
|
|
|
private getVerifiedInstall(artifact = this.artifact): { executable: string; receipt: RuntimeReceipt } | null {
|
|
const versionDir = this.versionDir(artifact);
|
|
if (!versionDir) return null;
|
|
const metadataPath = path.join(versionDir, 'install.json');
|
|
try {
|
|
const metadata = JSON.parse(readFileSync(metadataPath, 'utf8')) as InstallMetadata;
|
|
if (metadata.version !== artifact?.version || metadata.sha256 !== artifact.sha256) return null;
|
|
if (path.isAbsolute(metadata.executable) || metadata.executable.split(/[\\/]/).includes('..')) return null;
|
|
if (path.basename(metadata.executable).toLowerCase() !== artifact.executableName.toLowerCase()) return null;
|
|
if (typeof metadata.installedAt !== 'string' || !Number.isFinite(Date.parse(metadata.installedAt))) return null;
|
|
const executable = realpathSync(path.join(versionDir, metadata.executable));
|
|
if (!inside(versionDir, executable) || !statSync(executable).isFile()) return null;
|
|
return {
|
|
executable,
|
|
receipt: {
|
|
version: metadata.version,
|
|
sha256: metadata.sha256,
|
|
executable: metadata.executable,
|
|
installedAt: metadata.installedAt,
|
|
},
|
|
};
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
private getInstalledExecutable(artifact = this.artifact): string | null {
|
|
return this.getVerifiedInstall(artifact)?.executable ?? null;
|
|
}
|
|
|
|
private statePath(): string {
|
|
return path.join(this.root, 'runtime-state.json');
|
|
}
|
|
|
|
private receiptFor(artifact: OllamaRuntimeArtifact): RuntimeReceipt {
|
|
const installation = this.getVerifiedInstall(artifact);
|
|
if (!installation) throw new Error(`Managed Ollama ${artifact.version} install receipt failed validation`);
|
|
return installation.receipt;
|
|
}
|
|
|
|
private sameReceipt(left: RuntimeReceipt, right: RuntimeReceipt): boolean {
|
|
return left.version === right.version
|
|
&& left.sha256 === right.sha256
|
|
&& left.executable === right.executable
|
|
&& left.installedAt === right.installedAt;
|
|
}
|
|
|
|
private sameArtifact(left: OllamaRuntimeArtifact, right: OllamaRuntimeArtifact): boolean {
|
|
return left.version === right.version && left.sha256 === right.sha256;
|
|
}
|
|
|
|
private sameCatalogArtifact(left: OllamaRuntimeArtifact, right: OllamaRuntimeArtifact): boolean {
|
|
return this.sameArtifact(left, right)
|
|
&& left.platform === right.platform
|
|
&& left.arch === right.arch
|
|
&& left.filename === right.filename
|
|
&& left.url === right.url
|
|
&& left.sizeBytes === right.sizeBytes
|
|
&& left.executableName === right.executableName;
|
|
}
|
|
|
|
private assertTrustedTarget(): OllamaRuntimeArtifact {
|
|
if (!this.artifact) throw new Error(`Managed Ollama is unsupported on ${process.platform}/${process.arch}`);
|
|
const trusted = this.artifactCatalog.find((candidate) => this.sameCatalogArtifact(candidate, this.artifact!));
|
|
if (!trusted) throw new Error(`Managed Ollama ${this.artifact.version} is not present in the trusted artifact catalog`);
|
|
return trusted;
|
|
}
|
|
|
|
private catalogArtifact(receipt: RuntimeReceipt): OllamaRuntimeArtifact | null {
|
|
return this.artifactCatalog.find((candidate) => (
|
|
candidate.platform === this.platform
|
|
&& candidate.arch === this.arch
|
|
&& candidate.version === receipt.version
|
|
&& candidate.sha256 === receipt.sha256
|
|
)) ?? null;
|
|
}
|
|
|
|
private validReceipt(value: unknown): RuntimeReceipt | null {
|
|
if (!value || typeof value !== 'object') return null;
|
|
const candidate = value as Partial<RuntimeReceipt>;
|
|
if (typeof candidate.version !== 'string' || candidate.version.length === 0 || candidate.version.length > 128) return null;
|
|
if (typeof candidate.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(candidate.sha256)) return null;
|
|
if (typeof candidate.executable !== 'string' || candidate.executable.length === 0 || candidate.executable.length > 1_024) return null;
|
|
if (path.isAbsolute(candidate.executable) || candidate.executable.split(/[\\/]/).includes('..')) return null;
|
|
if (typeof candidate.installedAt !== 'string' || !Number.isFinite(Date.parse(candidate.installedAt))) return null;
|
|
return {
|
|
version: candidate.version,
|
|
sha256: candidate.sha256,
|
|
executable: candidate.executable,
|
|
installedAt: candidate.installedAt,
|
|
};
|
|
}
|
|
|
|
private validRollback(value: unknown): ManagedRuntimeRollback | null | undefined {
|
|
if (value === null) return null;
|
|
if (!value || typeof value !== 'object') return undefined;
|
|
const candidate = value as Partial<ManagedRuntimeRollback>;
|
|
if (typeof candidate.failedVersion !== 'string' || candidate.failedVersion.length === 0 || candidate.failedVersion.length > 128) return undefined;
|
|
if (typeof candidate.restoredVersion !== 'string' || candidate.restoredVersion.length === 0 || candidate.restoredVersion.length > 128) return undefined;
|
|
if (typeof candidate.occurredAt !== 'string' || !Number.isFinite(Date.parse(candidate.occurredAt))) return undefined;
|
|
if (typeof candidate.reason !== 'string' || candidate.reason.length === 0 || candidate.reason.length > 2_048) return undefined;
|
|
return {
|
|
failedVersion: candidate.failedVersion,
|
|
restoredVersion: candidate.restoredVersion,
|
|
occurredAt: candidate.occurredAt,
|
|
reason: candidate.reason,
|
|
};
|
|
}
|
|
|
|
private loadRuntimeState(): RuntimeStateLoad {
|
|
let parsed: unknown;
|
|
try {
|
|
parsed = JSON.parse(readFileSync(this.statePath(), 'utf8'));
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException)?.code === 'ENOENT') return { kind: 'missing' };
|
|
return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation' };
|
|
}
|
|
if (!parsed || typeof parsed !== 'object') {
|
|
return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation' };
|
|
}
|
|
const candidate = parsed as Partial<RuntimeActivationState>;
|
|
if (candidate.schemaVersion !== 1) {
|
|
return { kind: 'invalid', reason: 'Managed Ollama activation state failed validation' };
|
|
}
|
|
const parsedRollback = this.validRollback(candidate.lastRollback);
|
|
const lastRollback = parsedRollback === undefined ? null : parsedRollback;
|
|
const previous = candidate.previous === null ? null : this.validReceipt(candidate.previous);
|
|
const previousArtifact = previous ? this.catalogArtifact(previous) : null;
|
|
const previousInstall = previousArtifact ? this.getVerifiedInstall(previousArtifact) : null;
|
|
const trustedPrevious = previous && previousArtifact && previousInstall
|
|
&& this.sameReceipt(previous, previousInstall.receipt)
|
|
? { receipt: previous, artifact: previousArtifact }
|
|
: null;
|
|
const recovery = trustedPrevious
|
|
? {
|
|
recoveryArtifact: trustedPrevious.artifact,
|
|
recoveryReceipt: trustedPrevious.receipt,
|
|
lastRollback,
|
|
}
|
|
: { lastRollback };
|
|
const active = this.validReceipt(candidate.active);
|
|
if (!active) {
|
|
return {
|
|
kind: 'invalid',
|
|
reason: 'Managed Ollama activation state failed validation',
|
|
...recovery,
|
|
};
|
|
}
|
|
const activeArtifact = this.catalogArtifact(active);
|
|
const activeInstall = activeArtifact ? this.getVerifiedInstall(activeArtifact) : null;
|
|
if (!activeArtifact || !activeInstall || !this.sameReceipt(active, activeInstall.receipt)) {
|
|
return {
|
|
kind: 'invalid',
|
|
reason: 'Managed Ollama activation state references an untrusted or invalid runtime',
|
|
...recovery,
|
|
};
|
|
}
|
|
const usablePrevious = trustedPrevious
|
|
&& !this.sameArtifact(activeArtifact, trustedPrevious.artifact)
|
|
? trustedPrevious
|
|
: null;
|
|
return {
|
|
kind: 'valid',
|
|
state: {
|
|
schemaVersion: 1,
|
|
active,
|
|
previous: usablePrevious?.receipt ?? null,
|
|
lastRollback,
|
|
},
|
|
activeArtifact,
|
|
previousArtifact: usablePrevious?.artifact ?? null,
|
|
};
|
|
}
|
|
|
|
private async writeRuntimeState(state: RuntimeActivationState): Promise<void> {
|
|
await mkdir(this.root, { recursive: true });
|
|
const temporary = path.join(this.root, `.runtime-state.${process.pid}-${randomUUID()}.tmp`);
|
|
let descriptor: number | null = null;
|
|
try {
|
|
descriptor = openSync(temporary, 'wx');
|
|
writeFileSync(descriptor, `${JSON.stringify(state, null, 2)}\n`);
|
|
fsyncSync(descriptor);
|
|
closeSync(descriptor);
|
|
descriptor = null;
|
|
for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) {
|
|
try {
|
|
await this.stateRenameImpl(temporary, this.statePath());
|
|
break;
|
|
} catch (error) {
|
|
if (!isTransientFilesystemError(error) || attempt === FILESYSTEM_ATTEMPTS) throw error;
|
|
await waitForFilesystemRetry(attempt);
|
|
}
|
|
}
|
|
} finally {
|
|
if (descriptor !== null) closeSync(descriptor);
|
|
if (existsSync(temporary)) await removeWithin(this.root, temporary, this.rmImpl);
|
|
}
|
|
}
|
|
|
|
private async promoteActive(artifact: OllamaRuntimeArtifact): Promise<void> {
|
|
const loaded = this.loadRuntimeState();
|
|
const receipt = this.receiptFor(artifact);
|
|
const current = loaded.kind === 'valid' ? loaded.state : null;
|
|
await this.writeRuntimeState({
|
|
schemaVersion: 1,
|
|
active: receipt,
|
|
previous: current && !this.sameReceipt(current.active, receipt) ? current.active : current?.previous ?? null,
|
|
lastRollback: current?.lastRollback ?? null,
|
|
});
|
|
}
|
|
|
|
private selectFallbackArtifact(
|
|
loaded: Extract<RuntimeStateLoad, { kind: 'valid' }>,
|
|
failedArtifact: OllamaRuntimeArtifact | null,
|
|
): OllamaRuntimeArtifact | null {
|
|
if (!failedArtifact || !this.sameArtifact(loaded.activeArtifact, failedArtifact)) return loaded.activeArtifact;
|
|
return loaded.previousArtifact && !this.sameArtifact(loaded.previousArtifact, failedArtifact)
|
|
? loaded.previousArtifact
|
|
: null;
|
|
}
|
|
|
|
private verifiedCatalogInstalls(): Array<{ artifact: OllamaRuntimeArtifact; executable: string }> {
|
|
const seen = new Set<string>();
|
|
const installations: Array<{ artifact: OllamaRuntimeArtifact; executable: string }> = [];
|
|
for (const artifact of this.artifactCatalog) {
|
|
if (artifact.platform !== this.platform || artifact.arch !== this.arch) continue;
|
|
const key = `${artifact.version}:${artifact.sha256}`;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
const executable = this.getInstalledExecutable(artifact);
|
|
if (executable) installations.push({ artifact, executable });
|
|
}
|
|
return installations;
|
|
}
|
|
|
|
private legacyStartCandidate(): { artifact: OllamaRuntimeArtifact; executable: string } | null {
|
|
const installations = this.verifiedCatalogInstalls();
|
|
return installations.length === 1 ? installations[0]! : null;
|
|
}
|
|
|
|
private rollbackReason(error: unknown): string {
|
|
const reason = error instanceof Error ? error.message : String(error);
|
|
return (reason || 'target runtime failed').slice(0, 2_048);
|
|
}
|
|
|
|
private async persistRollback(
|
|
failedArtifact: OllamaRuntimeArtifact,
|
|
restoredArtifact: OllamaRuntimeArtifact,
|
|
rollback: ManagedRuntimeRollback,
|
|
): Promise<void> {
|
|
const loaded = this.loadRuntimeState();
|
|
const restored = this.receiptFor(restoredArtifact);
|
|
const current = loaded.kind === 'valid' ? loaded.state : null;
|
|
const failedWasActive = current !== null
|
|
&& current.active.version === failedArtifact.version
|
|
&& current.active.sha256 === failedArtifact.sha256;
|
|
await this.writeRuntimeState({
|
|
schemaVersion: 1,
|
|
active: restored,
|
|
previous: failedWasActive ? current.active : current?.previous ?? null,
|
|
lastRollback: rollback,
|
|
});
|
|
}
|
|
|
|
private activationLockDatabasePath(): string {
|
|
return path.join(this.root, '.activation-lock.sqlite');
|
|
}
|
|
|
|
private async acquireActivationLock(): Promise<InstallLockClaim> {
|
|
await mkdir(this.root, { recursive: true });
|
|
const database = new Database(this.activationLockDatabasePath(), { timeout: INSTALL_LOCK_BUSY_TIMEOUT_MS });
|
|
const deadline = Date.now() + INSTALL_LOCK_WAIT_MS;
|
|
let retryDelay = INSTALL_LOCK_RETRY_MIN_MS;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
database.exec('BEGIN EXCLUSIVE');
|
|
return { database };
|
|
} catch (error) {
|
|
if (!isSqliteBusyError(error)) {
|
|
try { database.close(); } catch { /* primary error wins */ }
|
|
throw error;
|
|
}
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
|
retryDelay = Math.min(retryDelay * 2, INSTALL_LOCK_RETRY_MAX_MS);
|
|
}
|
|
try { database.close(); } catch { /* timeout error wins */ }
|
|
throw new Error('Timed out waiting for the managed Ollama activation coordinator');
|
|
}
|
|
|
|
private withActivationLock<T>(operation: () => Promise<T>): Promise<T> {
|
|
const prior = this.activationPromise;
|
|
const activation = (async () => {
|
|
if (prior) {
|
|
try {
|
|
await prior;
|
|
} catch {
|
|
// The next caller must observe the resulting state, including a healthy rollback.
|
|
}
|
|
}
|
|
const lock = await this.acquireActivationLock();
|
|
try {
|
|
return await operation();
|
|
} finally {
|
|
await this.releaseInstallLock(lock);
|
|
}
|
|
})();
|
|
this.activationPromise = activation;
|
|
return activation.finally(() => {
|
|
if (this.activationPromise === activation) this.activationPromise = null;
|
|
});
|
|
}
|
|
|
|
async install(): Promise<{ executable: string; installedNow: boolean }> {
|
|
const existing = this.getInstalledExecutable();
|
|
if (existing) return { executable: existing, installedNow: false };
|
|
if (this.installPromise) return this.installPromise;
|
|
const coordinationKey = this.installLockDatabasePath();
|
|
const processInstall = PROCESS_INSTALLS.get(coordinationKey);
|
|
if (processInstall) {
|
|
const result = await processInstall;
|
|
return { executable: result.executable, installedNow: false };
|
|
}
|
|
const installation = this.installInternal();
|
|
PROCESS_INSTALLS.set(coordinationKey, installation);
|
|
this.installPromise = installation.finally(() => {
|
|
if (PROCESS_INSTALLS.get(coordinationKey) === installation) PROCESS_INSTALLS.delete(coordinationKey);
|
|
this.installPromise = null;
|
|
});
|
|
return this.installPromise;
|
|
}
|
|
|
|
private reportWarning(message: string, error: unknown): void {
|
|
try {
|
|
this.warn(message, error);
|
|
} catch {
|
|
// Diagnostics must never change runtime availability.
|
|
}
|
|
}
|
|
|
|
private async acquireInstallLock(): Promise<InstallLockClaim> {
|
|
const database = new Database(this.installLockDatabasePath(), { timeout: INSTALL_LOCK_BUSY_TIMEOUT_MS });
|
|
const deadline = Date.now() + INSTALL_LOCK_WAIT_MS;
|
|
let retryDelay = INSTALL_LOCK_RETRY_MIN_MS;
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
database.exec('BEGIN EXCLUSIVE');
|
|
return { database };
|
|
} catch (error) {
|
|
if (!isSqliteBusyError(error)) {
|
|
try {
|
|
database.close();
|
|
} catch (closeError) {
|
|
this.reportWarning('Could not close the managed Ollama install coordinator', closeError);
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
const jitter = Math.floor(Math.random() * Math.min(100, retryDelay / 4));
|
|
await new Promise((resolve) => setTimeout(resolve, retryDelay + jitter));
|
|
retryDelay = Math.min(retryDelay * 2, INSTALL_LOCK_RETRY_MAX_MS);
|
|
}
|
|
try {
|
|
database.close();
|
|
} catch (error) {
|
|
this.reportWarning('Could not close the timed-out managed Ollama install coordinator', error);
|
|
}
|
|
throw new Error(`Timed out waiting for the managed Ollama ${this.artifact!.version} installation coordinator`);
|
|
}
|
|
|
|
private async releaseInstallLock(claim: InstallLockClaim): Promise<void> {
|
|
try {
|
|
if (claim.database.inTransaction) claim.database.exec('ROLLBACK');
|
|
} catch (error) {
|
|
this.reportWarning('Could not roll back the managed Ollama install coordinator', error);
|
|
} finally {
|
|
try {
|
|
claim.database.close();
|
|
} catch (error) {
|
|
this.reportWarning('Could not close the managed Ollama install coordinator', error);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async cleanupInstallAttempt(targets: readonly string[]): Promise<void> {
|
|
for (const target of targets) {
|
|
try {
|
|
await removeWithin(this.root, target, this.rmImpl);
|
|
} catch (error) {
|
|
this.reportWarning(`Could not remove managed Ollama install artifact ${path.basename(target)}`, error);
|
|
}
|
|
}
|
|
}
|
|
|
|
private async removeOrphanedInstallAttempts(): Promise<void> {
|
|
const versionKey = this.installVersionKey();
|
|
const stagingPrefix = `.install-attempt-${versionKey}.`;
|
|
const archivePrefix = `.${this.artifact!.filename}.${versionKey}.`;
|
|
const entries = await readdir(this.root, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const isStaging = entry.name.startsWith(stagingPrefix);
|
|
const isArchive = entry.name.startsWith(archivePrefix) && entry.name.endsWith('.part');
|
|
if (!isStaging && !isArchive) continue;
|
|
await removeWithin(this.root, path.join(this.root, entry.name), this.rmImpl);
|
|
}
|
|
}
|
|
|
|
private async downloadArtifact(archive: string): Promise<void> {
|
|
const artifact = this.artifact!;
|
|
const deadline = Date.now() + DOWNLOAD_TIMEOUT_MS;
|
|
|
|
for (let attempt = 1; attempt <= DOWNLOAD_MAX_ATTEMPTS; attempt += 1) {
|
|
let offset = existsSync(archive) ? statSync(archive).size : 0;
|
|
if (offset > artifact.sizeBytes) {
|
|
throw new PermanentManagedRuntimeDownloadError(
|
|
`Official Ollama runtime exceeded the expected ${artifact.sizeBytes} bytes`,
|
|
);
|
|
}
|
|
|
|
try {
|
|
if (offset < artifact.sizeBytes) {
|
|
const remainingTime = deadline - Date.now();
|
|
if (remainingTime <= 0) throw new Error('Official Ollama runtime download timed out');
|
|
const response = await this.fetchImpl(artifact.url, {
|
|
redirect: 'follow',
|
|
...(offset > 0 ? { headers: { Range: `bytes=${offset}-` } } : {}),
|
|
signal: AbortSignal.timeout(remainingTime),
|
|
});
|
|
if (!response.ok || !response.body) {
|
|
const message = `Official Ollama runtime download failed with HTTP ${response.status}`;
|
|
if (!TRANSIENT_DOWNLOAD_STATUS.has(response.status)) {
|
|
throw new PermanentManagedRuntimeDownloadError(message);
|
|
}
|
|
throw new Error(message);
|
|
}
|
|
|
|
if (offset > 0 && response.status === 200) {
|
|
offset = 0;
|
|
} else if (offset > 0) {
|
|
const expectedRange = `bytes ${offset}-${artifact.sizeBytes - 1}/${artifact.sizeBytes}`;
|
|
if (response.status !== 206 || response.headers.get('content-range') !== expectedRange) {
|
|
throw new PermanentManagedRuntimeDownloadError(
|
|
'Official Ollama runtime returned an invalid resume range',
|
|
);
|
|
}
|
|
} else if (response.status !== 200) {
|
|
throw new PermanentManagedRuntimeDownloadError(
|
|
`Official Ollama runtime returned unexpected HTTP ${response.status}`,
|
|
);
|
|
}
|
|
|
|
const expectedResponseBytes = artifact.sizeBytes - offset;
|
|
const declaredSize = Number(response.headers.get('content-length'));
|
|
if (Number.isFinite(declaredSize) && declaredSize !== expectedResponseBytes) {
|
|
throw new PermanentManagedRuntimeDownloadError(
|
|
`Official Ollama runtime size mismatch: expected ${expectedResponseBytes}, got ${declaredSize}`,
|
|
);
|
|
}
|
|
|
|
let downloaded = offset;
|
|
const verifier = new Transform({
|
|
transform(chunk: Buffer, _encoding, callback) {
|
|
downloaded += chunk.length;
|
|
if (downloaded > artifact.sizeBytes) {
|
|
callback(new PermanentManagedRuntimeDownloadError(
|
|
`Official Ollama runtime exceeded the expected ${artifact.sizeBytes} bytes`,
|
|
));
|
|
return;
|
|
}
|
|
callback(null, chunk);
|
|
},
|
|
});
|
|
const body = Readable.fromWeb(response.body as import('node:stream/web').ReadableStream);
|
|
await pipeline(
|
|
body,
|
|
verifier,
|
|
createWriteStream(archive, { flags: offset > 0 ? 'a' : 'w' }),
|
|
);
|
|
if (downloaded !== artifact.sizeBytes || statSync(archive).size !== artifact.sizeBytes) {
|
|
throw new Error(
|
|
`Official Ollama runtime size mismatch: expected ${artifact.sizeBytes}, got ${statSync(archive).size}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const digest = createHash('sha256');
|
|
for await (const chunk of createReadStream(archive)) digest.update(chunk as Buffer);
|
|
if (digest.digest('hex') !== artifact.sha256) {
|
|
throw new PermanentManagedRuntimeDownloadError(
|
|
'Official Ollama runtime checksum verification failed',
|
|
);
|
|
}
|
|
return;
|
|
} catch (error) {
|
|
if (error instanceof PermanentManagedRuntimeDownloadError
|
|
|| attempt === DOWNLOAD_MAX_ATTEMPTS) throw error;
|
|
await new Promise((resolve) => setTimeout(
|
|
resolve,
|
|
DOWNLOAD_RETRY_DELAYS_MS[attempt - 1] ?? DOWNLOAD_RETRY_DELAYS_MS.at(-1)!,
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
private async installInternal(): Promise<{ executable: string; installedNow: boolean }> {
|
|
this.assertTrustedTarget();
|
|
if (!this.artifact) throw new Error(`Managed Ollama is unsupported on ${process.platform}/${process.arch}`);
|
|
if (!isLoopbackEndpoint(this.baseUrl)) throw new Error('Managed Ollama requires a loopback OLLAMA_HOST');
|
|
|
|
await mkdir(this.root, { recursive: true });
|
|
const lock = await this.acquireInstallLock();
|
|
try {
|
|
const concurrentlyInstalled = this.getInstalledExecutable();
|
|
if (concurrentlyInstalled) return { executable: concurrentlyInstalled, installedNow: false };
|
|
await this.removeOrphanedInstallAttempts();
|
|
const finalDir = this.versionDir()!;
|
|
if (existsSync(finalDir)) await removeWithin(this.root, finalDir, this.rmImpl);
|
|
const attemptId = `${process.pid}-${randomUUID()}`;
|
|
const versionKey = this.installVersionKey();
|
|
const staging = path.join(this.root, `.install-attempt-${versionKey}.${attemptId}`);
|
|
const archive = path.join(this.root, `.${this.artifact.filename}.${versionKey}.${attemptId}.part`);
|
|
await mkdir(staging, { recursive: true });
|
|
|
|
try {
|
|
await this.downloadArtifact(archive);
|
|
|
|
await this.extractArchive(archive, staging, this.artifact);
|
|
const executable = await findExecutable(staging, this.artifact.executableName);
|
|
if (this.artifact.platform !== 'win32') chmodSync(executable, 0o755);
|
|
const relativeExecutable = path.relative(staging, executable);
|
|
const metadata: InstallMetadata = {
|
|
version: this.artifact.version,
|
|
sha256: this.artifact.sha256,
|
|
executable: relativeExecutable,
|
|
installedAt: new Date().toISOString(),
|
|
};
|
|
writeFileSync(path.join(staging, 'install.json'), `${JSON.stringify(metadata, null, 2)}\n`, { flag: 'wx' });
|
|
return await this.publishInstall(staging, finalDir);
|
|
} finally {
|
|
await this.cleanupInstallAttempt([archive, staging]);
|
|
}
|
|
} finally {
|
|
await this.releaseInstallLock(lock);
|
|
}
|
|
}
|
|
|
|
private async waitForInstalledExecutable(): Promise<string | null> {
|
|
for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) {
|
|
const executable = this.getInstalledExecutable();
|
|
if (executable) return executable;
|
|
if (attempt < FILESYSTEM_ATTEMPTS) await waitForFilesystemRetry(attempt);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
private async publishInstall(
|
|
staging: string,
|
|
finalDir: string,
|
|
): Promise<{ executable: string; installedNow: boolean }> {
|
|
for (let attempt = 1; attempt <= FILESYSTEM_ATTEMPTS; attempt += 1) {
|
|
try {
|
|
await this.renameImpl(staging, finalDir);
|
|
} catch (error) {
|
|
const winner = existsSync(finalDir) ? await this.waitForInstalledExecutable() : null;
|
|
if (winner) return { executable: winner, installedNow: false };
|
|
if (!isTransientFilesystemError(error) || attempt === FILESYSTEM_ATTEMPTS) throw error;
|
|
await waitForFilesystemRetry(attempt);
|
|
continue;
|
|
}
|
|
const executable = await this.waitForInstalledExecutable();
|
|
if (!executable) throw new Error('Published Ollama runtime failed install metadata validation');
|
|
return { executable, installedNow: true };
|
|
}
|
|
throw new Error('Managed Ollama runtime publication exhausted all retry attempts');
|
|
}
|
|
|
|
private fallbackCandidate(
|
|
loaded: RuntimeStateLoad,
|
|
failedArtifact: OllamaRuntimeArtifact,
|
|
): { artifact: OllamaRuntimeArtifact; executable: string } | null {
|
|
const artifact = loaded.kind === 'valid'
|
|
? this.selectFallbackArtifact(loaded, failedArtifact)
|
|
: loaded.kind === 'invalid' && loaded.recoveryArtifact
|
|
? loaded.recoveryArtifact
|
|
: this.legacyStartCandidate()?.artifact ?? null;
|
|
if (!artifact || this.sameArtifact(artifact, failedArtifact)) return null;
|
|
const executable = this.getInstalledExecutable(artifact);
|
|
return executable ? { artifact, executable } : null;
|
|
}
|
|
|
|
private async rollbackAfterFailure(
|
|
failedArtifact: OllamaRuntimeArtifact,
|
|
fallback: { artifact: OllamaRuntimeArtifact; executable: string } | null,
|
|
cause: unknown,
|
|
): Promise<never> {
|
|
if (cause instanceof ManagedRuntimeTerminationError) throw cause;
|
|
await this.stopInternal();
|
|
if (!fallback) throw cause;
|
|
|
|
try {
|
|
const started = await this.start(fallback.executable, fallback.artifact);
|
|
if (!started || !this.runningArtifact || !this.sameArtifact(this.runningArtifact, fallback.artifact)) {
|
|
throw new Error(`Verified fallback runtime ${fallback.artifact.version} was not started by Waggle`);
|
|
}
|
|
} catch (rollbackError) {
|
|
throw new Error(
|
|
`Managed Ollama ${failedArtifact.version} failed and verified fallback ${fallback.artifact.version} could not start: ${this.rollbackReason(rollbackError)}`,
|
|
{ cause },
|
|
);
|
|
}
|
|
|
|
const rollback: ManagedRuntimeRollback = {
|
|
failedVersion: failedArtifact.version,
|
|
restoredVersion: fallback.artifact.version,
|
|
occurredAt: new Date().toISOString(),
|
|
reason: this.rollbackReason(cause),
|
|
};
|
|
try {
|
|
await this.persistRollback(failedArtifact, fallback.artifact, rollback);
|
|
} catch (stateError) {
|
|
this.reportWarning('Could not persist the managed Ollama rollback receipt', stateError);
|
|
}
|
|
throw new ManagedRuntimeRollbackError(rollback, cause);
|
|
}
|
|
|
|
private async recoverCrashedRuntime(): Promise<ManagedOllamaReadyResult | null> {
|
|
const artifact = this.pendingCrashRecovery;
|
|
if (!artifact) return null;
|
|
if (this.crashRecoveryConsumed) {
|
|
throw this.markTerminationUnconfirmed('managed Ollama crash-recovery budget is exhausted');
|
|
}
|
|
const executable = this.getInstalledExecutable(artifact);
|
|
if (!executable) {
|
|
throw this.markTerminationUnconfirmed('verified managed Ollama runtime disappeared before crash recovery');
|
|
}
|
|
if (!await this.endpointQuiescent(this.baseUrl)) {
|
|
throw this.markTerminationUnconfirmed(
|
|
'owned Ollama daemon exited but its loopback endpoint remains occupied',
|
|
);
|
|
}
|
|
this.assertTerminationConfirmed();
|
|
|
|
this.crashRecoveryConsumed = true;
|
|
this.pendingCrashRecovery = null;
|
|
try {
|
|
const started = await this.start(executable, artifact, true);
|
|
if (!started || !this.runningArtifact || !this.sameArtifact(this.runningArtifact, artifact)) {
|
|
throw new Error(`verified runtime ${artifact.version} was not restarted by Waggle`);
|
|
}
|
|
} catch (error) {
|
|
throw this.markTerminationUnconfirmed(
|
|
`managed Ollama crash recovery failed: ${this.rollbackReason(error)}`,
|
|
);
|
|
}
|
|
return {
|
|
installedNow: false,
|
|
startedNow: true,
|
|
endpoint: this.baseUrl,
|
|
status: this.getStatus(),
|
|
};
|
|
}
|
|
|
|
private queueCrashRecovery(artifact: OllamaRuntimeArtifact): void {
|
|
this.pendingCrashRecovery = artifact;
|
|
const recovery = this.withActivationLock(async () => {
|
|
this.assertTerminationConfirmed();
|
|
await this.recoverCrashedRuntime();
|
|
});
|
|
void recovery.catch((error: unknown) => {
|
|
if (!this.terminationFailure) {
|
|
this.markTerminationUnconfirmed(
|
|
`managed Ollama crash recovery failed: ${this.rollbackReason(error)}`,
|
|
);
|
|
}
|
|
});
|
|
}
|
|
|
|
async startInstalled(): Promise<ManagedOllamaReadyResult> {
|
|
return this.withActivationLock(async () => {
|
|
this.assertTerminationConfirmed();
|
|
let recovered = await this.recoverCrashedRuntime();
|
|
if (recovered) return recovered;
|
|
const endpointReady = await this.probe(this.baseUrl);
|
|
this.assertTerminationConfirmed();
|
|
recovered = await this.recoverCrashedRuntime();
|
|
if (recovered) return recovered;
|
|
if (endpointReady) {
|
|
return { installedNow: false, startedNow: false, endpoint: this.baseUrl, status: this.getStatus() };
|
|
}
|
|
const loaded = this.loadRuntimeState();
|
|
const candidate = loaded.kind === 'valid'
|
|
? {
|
|
artifact: loaded.activeArtifact,
|
|
executable: this.getInstalledExecutable(loaded.activeArtifact),
|
|
}
|
|
: loaded.kind === 'invalid' && loaded.recoveryArtifact
|
|
? {
|
|
artifact: loaded.recoveryArtifact,
|
|
executable: this.getInstalledExecutable(loaded.recoveryArtifact),
|
|
}
|
|
: this.legacyStartCandidate();
|
|
if (!candidate?.executable) {
|
|
throw new Error('Verified Waggle-managed Ollama runtime is not installed');
|
|
}
|
|
const fallback = this.fallbackCandidate(loaded, candidate.artifact);
|
|
let startedNow: boolean;
|
|
try {
|
|
startedNow = await this.start(candidate.executable, candidate.artifact);
|
|
if (startedNow && loaded.kind !== 'valid') await this.promoteActive(candidate.artifact);
|
|
} catch (error) {
|
|
return this.rollbackAfterFailure(candidate.artifact, fallback, error);
|
|
}
|
|
return {
|
|
installedNow: false,
|
|
startedNow,
|
|
endpoint: this.baseUrl,
|
|
status: this.getStatus(),
|
|
};
|
|
});
|
|
}
|
|
|
|
async ensureReady(): Promise<ManagedOllamaReadyResult> {
|
|
return this.withActivationLock(async () => {
|
|
this.assertTerminationConfirmed();
|
|
let recovered = await this.recoverCrashedRuntime();
|
|
if (recovered) return recovered;
|
|
const endpointReady = await this.probe(this.baseUrl);
|
|
this.assertTerminationConfirmed();
|
|
recovered = await this.recoverCrashedRuntime();
|
|
if (recovered) return recovered;
|
|
if (endpointReady) {
|
|
return { installedNow: false, startedNow: false, endpoint: this.baseUrl, status: this.getStatus() };
|
|
}
|
|
const target = this.assertTrustedTarget();
|
|
const loadedBefore = this.loadRuntimeState();
|
|
const fallback = this.fallbackCandidate(loadedBefore, target);
|
|
let installation: { executable: string; installedNow: boolean };
|
|
try {
|
|
installation = await this.install();
|
|
} catch (error) {
|
|
return this.rollbackAfterFailure(target, fallback, error);
|
|
}
|
|
|
|
let startedNow: boolean;
|
|
try {
|
|
startedNow = await this.start(installation.executable, target);
|
|
if (startedNow) await this.promoteActive(target);
|
|
} catch (error) {
|
|
return this.rollbackAfterFailure(target, fallback, error);
|
|
}
|
|
return {
|
|
installedNow: installation.installedNow,
|
|
startedNow,
|
|
endpoint: this.baseUrl,
|
|
status: this.getStatus(),
|
|
};
|
|
});
|
|
}
|
|
|
|
private async start(
|
|
executable: string,
|
|
artifact: OllamaRuntimeArtifact,
|
|
requireQuiescentEndpoint = false,
|
|
): Promise<boolean> {
|
|
if (this.startPromise) return this.startPromise;
|
|
this.assertTerminationConfirmed();
|
|
const endpointReady = await this.probe(this.baseUrl);
|
|
this.assertTerminationConfirmed();
|
|
const recovered = await this.recoverCrashedRuntime();
|
|
if (recovered) return false;
|
|
if (endpointReady) {
|
|
if (requireQuiescentEndpoint) {
|
|
throw new ManagedRuntimeTerminationError('managed Ollama loopback endpoint remains occupied');
|
|
}
|
|
return false;
|
|
}
|
|
if (requireQuiescentEndpoint && !await this.endpointQuiescent(this.baseUrl)) {
|
|
throw new ManagedRuntimeTerminationError('managed Ollama loopback endpoint remains occupied');
|
|
}
|
|
if (this.child) {
|
|
if (this.child.exitCode === null) await this.stopInternal();
|
|
else if (this.child.exitCode === WATCHDOG_EXIT_OWNED_DAEMON_EXITED) {
|
|
if (!await this.endpointQuiescent(this.baseUrl)) {
|
|
throw this.markTerminationUnconfirmed('owned Ollama daemon exited but its loopback endpoint remains occupied');
|
|
}
|
|
this.child = null;
|
|
this.runningArtifact = null;
|
|
} else if (this.child.exitCode !== WATCHDOG_EXIT_TERMINATION_CONFIRMED) {
|
|
throw this.markTerminationUnconfirmed(`watchdog exited with code ${this.child.exitCode}`);
|
|
} else {
|
|
this.child = null;
|
|
this.runningArtifact = null;
|
|
}
|
|
}
|
|
this.assertTerminationConfirmed();
|
|
this.startPromise = this.startInternal(executable, artifact).finally(() => { this.startPromise = null; });
|
|
return this.startPromise;
|
|
}
|
|
|
|
private async startInternal(executable: string, artifact: OllamaRuntimeArtifact): Promise<boolean> {
|
|
await mkdir(this.modelsDir, { recursive: true });
|
|
await mkdir(this.profileDir, { recursive: true });
|
|
const startTimeoutMs = this.platform === 'win32'
|
|
? WINDOWS_START_TIMEOUT_MS
|
|
: START_TIMEOUT_MS;
|
|
const startedAt = Date.now();
|
|
let stdoutRawTail = '';
|
|
let stderrRawTail = '';
|
|
const child = this.spawnImpl(process.execPath, [
|
|
'-e',
|
|
MANAGED_OLLAMA_WATCHDOG_SOURCE,
|
|
executable,
|
|
JSON.stringify(['serve']),
|
|
], {
|
|
env: buildManagedOllamaEnv(this.baseUrl, this.modelsDir, process.env, this.profileDir),
|
|
windowsHide: true,
|
|
shell: false,
|
|
stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
|
|
});
|
|
const captureStdout = (chunk: string | Buffer) => {
|
|
stdoutRawTail = appendRawDiagnosticTail(stdoutRawTail, chunk);
|
|
};
|
|
const captureStderr = (chunk: string | Buffer) => {
|
|
stderrRawTail = appendRawDiagnosticTail(stderrRawTail, chunk);
|
|
};
|
|
const ignoreDiagnosticStreamError = () => {};
|
|
child.stdout?.on('error', ignoreDiagnosticStreamError);
|
|
child.stderr?.on('error', ignoreDiagnosticStreamError);
|
|
child.stdout?.on('data', captureStdout);
|
|
child.stderr?.on('data', captureStderr);
|
|
const releaseDiagnosticCapture = () => {
|
|
child.stdout?.off('data', captureStdout);
|
|
child.stderr?.off('data', captureStderr);
|
|
child.stdout?.resume();
|
|
child.stderr?.resume();
|
|
};
|
|
const reportStartupDiagnostics = (reason: string) => {
|
|
const elapsedMs = Math.max(0, Date.now() - startedAt);
|
|
const stdoutTail = formatDiagnosticTail(stdoutRawTail);
|
|
const stderrTail = formatDiagnosticTail(stderrRawTail);
|
|
const watchdogState = child.exitCode === null
|
|
? 'running'
|
|
: `exited (${child.exitCode})`;
|
|
this.reportWarning(
|
|
`Managed Ollama ${artifact.version} startup failed`,
|
|
new Error([
|
|
`reason=${reason}`,
|
|
`endpoint=${this.baseUrl.replace(/\/+$/, '')}/api/tags`,
|
|
`elapsedMs=${elapsedMs}`,
|
|
`watchdog=${watchdogState}`,
|
|
`stdoutTail=${stdoutTail || '<empty>'}`,
|
|
`stderrTail=${stderrTail || '<empty>'}`,
|
|
].join('\n')),
|
|
);
|
|
};
|
|
this.child = child;
|
|
let startupError: Error | null = null;
|
|
child.once('error', (error) => {
|
|
startupError = error;
|
|
if (this.child === child) {
|
|
this.child = null;
|
|
this.runningArtifact = null;
|
|
}
|
|
});
|
|
child.once('exit', (code) => {
|
|
if (this.child === child) {
|
|
const readyArtifact = this.runningArtifact && this.sameArtifact(this.runningArtifact, artifact)
|
|
? this.runningArtifact
|
|
: null;
|
|
const stopWasRequested = this.stopRequestedChildren.has(child);
|
|
if (code === WATCHDOG_EXIT_TERMINATION_CONFIRMED) {
|
|
this.child = null;
|
|
this.terminationFailure = null;
|
|
this.pendingCrashRecovery = null;
|
|
} else if (code === WATCHDOG_EXIT_OWNED_DAEMON_EXITED) {
|
|
this.child = null;
|
|
if (readyArtifact && !stopWasRequested && !this.stopInProgress && !this.terminationFailure) {
|
|
if (this.crashRecoveryConsumed) {
|
|
this.markTerminationUnconfirmed('managed Ollama replacement exited after its one recovery attempt');
|
|
} else {
|
|
this.queueCrashRecovery(readyArtifact);
|
|
}
|
|
}
|
|
} else {
|
|
this.markTerminationUnconfirmed(
|
|
code === null ? 'watchdog exited without a success code' : `watchdog exited with code ${code}`,
|
|
);
|
|
}
|
|
this.runningArtifact = null;
|
|
}
|
|
});
|
|
|
|
try {
|
|
const deadline = Date.now() + startTimeoutMs;
|
|
while (Date.now() < deadline) {
|
|
if (startupError) throw startupError;
|
|
if (child.exitCode !== null) break;
|
|
const ready = await this.probe(this.baseUrl);
|
|
if (startupError) throw startupError;
|
|
if (child.exitCode !== null || this.child !== child) break;
|
|
if (ready) {
|
|
this.runningArtifact = artifact;
|
|
releaseDiagnosticCapture();
|
|
return true;
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 500));
|
|
}
|
|
} catch (error) {
|
|
if (startupError && child.pid === undefined) {
|
|
reportStartupDiagnostics('watchdog spawn error');
|
|
releaseDiagnosticCapture();
|
|
if (this.child === child) this.child = null;
|
|
this.runningArtifact = null;
|
|
throw error;
|
|
}
|
|
try {
|
|
await this.stopChild(child);
|
|
} catch (stopError) {
|
|
await waitForDiagnosticStreams(child);
|
|
reportStartupDiagnostics('startup error with unconfirmed cleanup');
|
|
releaseDiagnosticCapture();
|
|
throw stopError;
|
|
}
|
|
await waitForDiagnosticStreams(child);
|
|
reportStartupDiagnostics('startup error');
|
|
releaseDiagnosticCapture();
|
|
if (this.child === child) this.child = null;
|
|
if (this.runningArtifact === artifact) this.runningArtifact = null;
|
|
throw error;
|
|
}
|
|
const watchdogWasRunning = child.exitCode === null;
|
|
try {
|
|
await this.stopChild(child);
|
|
} catch (stopError) {
|
|
await waitForDiagnosticStreams(child);
|
|
reportStartupDiagnostics('readiness timeout with unconfirmed cleanup');
|
|
releaseDiagnosticCapture();
|
|
throw stopError;
|
|
}
|
|
await waitForDiagnosticStreams(child);
|
|
reportStartupDiagnostics('readiness timeout');
|
|
releaseDiagnosticCapture();
|
|
if (this.child === child) this.child = null;
|
|
if (this.runningArtifact === artifact) this.runningArtifact = null;
|
|
throw new Error(
|
|
`Waggle-managed Ollama ${artifact.version} did not make `
|
|
+ `${this.baseUrl.replace(/\/+$/, '')}/api/tags ready within ${startTimeoutMs}ms `
|
|
+ `(watchdog ${watchdogWasRunning ? 'remained live until cleanup' : 'exited before readiness'})`,
|
|
);
|
|
}
|
|
|
|
private requestStop(child: ChildProcess): void {
|
|
try {
|
|
if (child.connected) child.send('shutdown');
|
|
} catch { /* a disconnected watchdog must confirm shutdown by exiting cleanly */ }
|
|
}
|
|
|
|
private markTerminationUnconfirmed(reason: string): ManagedRuntimeTerminationError {
|
|
this.pendingCrashRecovery = null;
|
|
this.terminationFailure ??= new ManagedRuntimeTerminationError(reason);
|
|
return this.terminationFailure;
|
|
}
|
|
|
|
private assertTerminationConfirmed(): void {
|
|
if (this.terminationFailure) throw this.terminationFailure;
|
|
}
|
|
|
|
private async stopChild(child: ChildProcess): Promise<void> {
|
|
this.stopRequestedChildren.add(child);
|
|
if (child.exitCode !== null) {
|
|
return this.confirmWatchdogExit(child.exitCode);
|
|
}
|
|
const exitPromise = new Promise<number | null | undefined>((resolve) => {
|
|
const onExit = (code: number | null) => {
|
|
clearTimeout(timeout);
|
|
resolve(code);
|
|
};
|
|
const timeout = setTimeout(() => {
|
|
child.off('exit', onExit);
|
|
resolve(undefined);
|
|
}, STOP_TIMEOUT_MS);
|
|
child.once('exit', onExit);
|
|
});
|
|
this.requestStop(child);
|
|
const exitCode = await exitPromise;
|
|
if (exitCode === undefined) {
|
|
throw this.markTerminationUnconfirmed('watchdog did not confirm process-tree termination');
|
|
}
|
|
return this.confirmWatchdogExit(exitCode);
|
|
}
|
|
|
|
private async confirmWatchdogExit(exitCode: number | null): Promise<void> {
|
|
if (exitCode === WATCHDOG_EXIT_TERMINATION_CONFIRMED) {
|
|
this.terminationFailure = null;
|
|
return;
|
|
}
|
|
if (exitCode === WATCHDOG_EXIT_OWNED_DAEMON_EXITED) {
|
|
if (await this.endpointQuiescent(this.baseUrl)) return;
|
|
throw this.markTerminationUnconfirmed('owned Ollama daemon exited but its loopback endpoint remains occupied');
|
|
}
|
|
throw this.markTerminationUnconfirmed(
|
|
exitCode === null ? 'watchdog exited without a success code' : `watchdog exited with code ${exitCode}`,
|
|
);
|
|
}
|
|
|
|
private async stopInternal(): Promise<void> {
|
|
this.stopInProgress = true;
|
|
this.pendingCrashRecovery = null;
|
|
try {
|
|
const child = this.child;
|
|
if (!child) return;
|
|
this.stopRequestedChildren.add(child);
|
|
if (child.exitCode !== null) {
|
|
await this.confirmWatchdogExit(child.exitCode);
|
|
if (this.child === child) this.child = null;
|
|
this.runningArtifact = null;
|
|
return;
|
|
}
|
|
await this.stopChild(child);
|
|
if (this.child === child) this.child = null;
|
|
this.runningArtifact = null;
|
|
} finally {
|
|
this.pendingCrashRecovery = null;
|
|
this.stopInProgress = false;
|
|
}
|
|
}
|
|
|
|
async stop(): Promise<void> {
|
|
return this.withActivationLock(() => this.stopInternal());
|
|
}
|
|
}
|
|
|
|
async function defaultEndpointQuiescent(baseUrl: string): Promise<boolean> {
|
|
const endpoint = new URL(baseUrl);
|
|
const host = endpoint.hostname.replace(/^\[|\]$/g, '');
|
|
const port = Number.parseInt(endpoint.port || '80', 10);
|
|
return new Promise((resolve) => {
|
|
const server = createServer();
|
|
server.unref();
|
|
server.once('error', () => resolve(false));
|
|
server.listen({ host, port, exclusive: true }, () => {
|
|
server.close((error) => resolve(error === undefined));
|
|
});
|
|
});
|
|
}
|