diff --git a/scripts/release-smoke.mjs b/scripts/release-smoke.mjs new file mode 100644 index 0000000..b2c5fdd --- /dev/null +++ b/scripts/release-smoke.mjs @@ -0,0 +1,336 @@ +#!/usr/bin/env node +/** + * Smoke tests the release workflow runs on every target after building. + * + * node scripts/release-smoke.mjs installer [--archive FILE] [--timeout SECONDS] + * + * Installs the server archive with the built installer + * (packages/launcher/dist/cli.js) into a temporary directory, runs + * `doctor`, starts `run` on a free port with a temporary data directory, + * waits for /health, stops the server and uninstalls. The default archive + * is the one in artifacts/ for this system. + * + * node scripts/release-smoke.mjs desktop --app PATH [--port 3333] [--timeout SECONDS] + * + * Starts the built desktop app (its executable, or a macOS .app bundle) + * with a temporary data directory, waits until its sidecar answers + * /health on 127.0.0.1:, then stops the app and its sidecar and + * checks that the port is released. On Linux run it under xvfb-run. + */ +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import net from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const isWindows = process.platform === 'win32'; + +// Keep in step with releaseTarget() in packages/launcher/src/platform.ts. +const TARGETS = { + 'win32-x64': 'windows-x64', + 'darwin-arm64': 'macos-arm64', + 'darwin-x64': 'macos-x64', + 'linux-x64': 'linux-x64', +}; + +class SmokeError extends Error {} + +function log(message) { + console.log(`[release-smoke] ${message}`); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function parseArgs(argv) { + const [command, ...rest] = argv; + if (command !== 'installer' && command !== 'desktop') { + throw new SmokeError('Usage: release-smoke.mjs installer|desktop [options]'); + } + const options = { port: 3333, timeout: 300 }; + for (let i = 0; i < rest.length; i += 2) { + const [flag, value] = [rest[i], rest[i + 1]]; + if (value === undefined) throw new SmokeError(`Missing value for ${flag}`); + if (flag === '--archive') options.archive = path.resolve(value); + else if (flag === '--app') options.app = path.resolve(value); + else if (flag === '--port' || flag === '--timeout') { + const number = Number(value); + if (!Number.isInteger(number) || number < 1) throw new SmokeError(`Invalid ${flag}: ${value}`); + options[flag.slice(2)] = number; + } else { + throw new SmokeError(`Unknown option: ${flag}`); + } + } + return { command, options }; +} + +function tempRoot(name) { + return fs.mkdtempSync(path.join(process.env.RUNNER_TEMP || os.tmpdir(), `waggle-smoke-${name}-`)); +} + +function removeTree(dir) { + fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 1000 }); +} + +function portIsFree(port) { + return new Promise((resolve) => { + const server = net.createServer(); + server.once('error', () => resolve(false)); + server.listen(port, '127.0.0.1', () => server.close(() => resolve(true))); + }); +} + +function freePort() { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const { port } = server.address(); + server.close(() => resolve(port)); + }); + }); +} + +/** The /health body of a Waggle server on this port, or null. */ +async function health(port) { + try { + const response = await fetch(`http://127.0.0.1:${port}/health`, { signal: AbortSignal.timeout(2000) }); + if (!response.ok) return null; + const body = await response.json(); + return body && body.mode === 'local' ? body : null; + } catch { + return null; + } +} + +/** + * Starts a long-running process. On macOS and Linux it leads its own process + * group, so stopping the group also stops the sidecar it starts. + */ +function startBackground(label, command, args, env) { + const child = spawn(command, args, { + env, + stdio: ['ignore', 'inherit', 'inherit'], + detached: !isWindows, + windowsHide: true, + }); + const state = { label, child, error: null, exited: false }; + child.once('error', (error) => { + state.error = error; + }); + child.once('exit', () => { + state.exited = true; + }); + return state; +} + +function waitForExit(state, ms) { + if (state.exited || state.error) return Promise.resolve(true); + return new Promise((resolve) => { + const timer = setTimeout(() => resolve(false), ms); + state.child.once('exit', () => { + clearTimeout(timer); + resolve(true); + }); + }); +} + +async function waitForHealth(state, port, timeoutSeconds) { + const deadline = Date.now() + timeoutSeconds * 1000; + while (Date.now() < deadline) { + if (state.error) throw new SmokeError(`${state.label} did not start: ${state.error.message}`); + if (state.exited) { + const { exitCode, signalCode } = state.child; + throw new SmokeError(`${state.label} exited (code ${exitCode}, signal ${signalCode}) before /health answered`); + } + const body = await health(port); + if (body) return body; + await sleep(1000); + } + throw new SmokeError(`${state.label} did not answer /health on 127.0.0.1:${port} within ${timeoutSeconds} s`); +} + +function signalGroup(pid, signal) { + try { + process.kill(-pid, signal); + } catch (error) { + if (error.code !== 'ESRCH') throw error; + } +} + +async function waitForPortFree(port, ms) { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + if (await portIsFree(port)) return true; + await sleep(500); + } + return false; +} + +/** Stops the process and everything it started, then checks the port is released. */ +async function stopTree(state, port) { + if (state.child.pid === undefined) return; + if (isWindows) { + spawnSync('taskkill', ['/PID', String(state.child.pid), '/T', '/F'], { stdio: 'inherit', windowsHide: true }); + await waitForExit(state, 10_000); + } else { + signalGroup(state.child.pid, 'SIGTERM'); + if (!(await waitForExit(state, 15_000))) signalGroup(state.child.pid, 'SIGKILL'); + await waitForExit(state, 5_000); + } + if (await waitForPortFree(port, 20_000)) return; + if (!isWindows) { + signalGroup(state.child.pid, 'SIGKILL'); + if (await waitForPortFree(port, 10_000)) return; + } + throw new SmokeError(`Port ${port} is still in use after stopping ${state.label}: its server outlived it`); +} + +/** + * Waits for /health from a started process, then stops it. A failure to stop + * is reported after, not instead of, a failed health check. + */ +async function checkHealthThenStop(state, port, timeoutSeconds, onFailure = () => {}) { + let failure = null; + try { + await waitForHealth(state, port, timeoutSeconds); + log(`${state.label} answers /health on 127.0.0.1:${port}`); + } catch (error) { + failure = error; + onFailure(); + } + try { + await stopTree(state, port); + } catch (error) { + if (failure) console.error(`[release-smoke] ${error.message}`); + else failure = error; + } + if (failure) throw failure; +} + +// ── installer ──────────────────────────────────────────────────────────────── + +function defaultArchive() { + const target = TARGETS[`${process.platform}-${process.arch}`]; + if (!target) throw new SmokeError(`No release target for ${process.platform}-${process.arch}`); + const dir = path.join(root, 'artifacts'); + const pattern = new RegExp(`^waggle-server-.+-${target}\\.tar\\.gz$`); + const matches = fs.existsSync(dir) ? fs.readdirSync(dir).filter((name) => pattern.test(name)) : []; + if (matches.length !== 1) { + throw new SmokeError(`Expected one ${target} server archive in artifacts/, found ${matches.length}. Pass --archive.`); + } + return path.join(dir, matches[0]); +} + +function runLauncher(launcher, args) { + log(`waggle-server ${args.join(' ')}`); + const result = spawnSync(process.execPath, [launcher, ...args], { encoding: 'utf8', windowsHide: true }); + process.stdout.write(result.stdout ?? ''); + process.stderr.write(result.stderr ?? ''); + if (result.error) throw result.error; + return result; +} + +/** doctor must pass; the only warning allowed is that the command's directory is not on PATH. */ +function checkDoctor(result) { + if (result.status !== 0) throw new SmokeError(`doctor failed with exit code ${result.status}`); + const warnings = result.stdout.split(/\r?\n/).filter((line) => /^\s*warn\s/.test(line)); + const unexpected = warnings.filter((line) => !/^\s*warn\s+Command: .+ is not on your PATH$/.test(line)); + if (unexpected.length > 0) throw new SmokeError(`doctor warned:\n${unexpected.join('\n')}`); + if (!/^\s*ok\s+Native modules: SQLite loads$/m.test(result.stdout)) { + throw new SmokeError('doctor did not report that SQLite loads with the bundled runtime'); + } +} + +async function installerSmoke(options) { + const launcher = path.join(root, 'packages', 'launcher', 'dist', 'cli.js'); + if (!fs.existsSync(launcher)) { + throw new SmokeError('packages/launcher/dist/cli.js is missing. Run: npm run build --workspace @waggle-ai/waggle'); + } + const archive = options.archive ?? defaultArchive(); + log(`Archive: ${archive}`); + const temp = tempRoot('installer'); + const installDir = path.join(temp, 'install'); + const dataDir = path.join(temp, 'data'); + const location = ['--dir', installDir, '--bin-dir', path.join(temp, 'bin')]; + try { + const install = runLauncher(launcher, ['install', '--from', archive, ...location]); + if (install.status !== 0) throw new SmokeError(`install failed with exit code ${install.status}`); + + const port = await freePort(); + checkDoctor(runLauncher(launcher, ['doctor', ...location, '--data-dir', dataDir, '--port', String(port)])); + + log(`waggle-server run --no-open --port ${port}`); + const server = startBackground( + 'The installed server', + process.execPath, + [launcher, 'run', '--no-open', '--port', String(port), '--dir', installDir, '--data-dir', dataDir], + process.env, + ); + await checkHealthThenStop(server, port, options.timeout); + + // Windows can hold the stopped runtime's files for a moment. + let uninstall; + for (let attempt = 1; attempt <= 5; attempt++) { + uninstall = runLauncher(launcher, ['uninstall', ...location, '--data-dir', dataDir]); + if (uninstall.status === 0) break; + await sleep(2000); + } + if (uninstall.status !== 0) throw new SmokeError(`uninstall failed with exit code ${uninstall.status}`); + if (fs.existsSync(installDir)) throw new SmokeError(`uninstall left ${installDir}`); + if (!fs.existsSync(dataDir)) throw new SmokeError('uninstall removed the data directory'); + log('Installer smoke test passed'); + } finally { + removeTree(temp); + } +} + +// ── desktop ────────────────────────────────────────────────────────────────── + +/** The executable of a macOS .app bundle, or the given executable. */ +function appExecutable(app) { + if (!fs.existsSync(app)) throw new SmokeError(`${app} does not exist`); + if (!app.endsWith('.app') || !fs.statSync(app).isDirectory()) return app; + const plist = fs.readFileSync(path.join(app, 'Contents', 'Info.plist'), 'utf8'); + const name = /CFBundleExecutable<\/key>\s*([^<]+)<\/string>/.exec(plist)?.[1]; + if (!name) throw new SmokeError(`${app}/Contents/Info.plist names no CFBundleExecutable`); + return path.join(app, 'Contents', 'MacOS', name); +} + +function printServiceLog(dataDir) { + const file = path.join(dataDir, 'logs', 'service.log'); + if (!fs.existsSync(file)) return; + const lines = fs.readFileSync(file, 'utf8').split(/\r?\n/); + log(`Last lines of ${file}:`); + console.log(lines.slice(-200).join('\n')); +} + +async function desktopSmoke(options) { + if (!options.app) throw new SmokeError('desktop needs --app '); + const executable = appExecutable(options.app); + if (!(await portIsFree(options.port))) { + throw new SmokeError(`Port ${options.port} is in use; the desktop smoke test needs it free`); + } + const temp = tempRoot('desktop'); + const dataDir = path.join(temp, 'data'); + fs.mkdirSync(dataDir, { recursive: true }); + try { + log(`Starting ${executable}`); + const app = startBackground('The desktop app', executable, [], { ...process.env, WAGGLE_DATA_DIR: dataDir }); + await checkHealthThenStop(app, options.port, options.timeout, () => printServiceLog(dataDir)); + log('Desktop smoke test passed'); + } finally { + removeTree(temp); + } +} + +try { + const { command, options } = parseArgs(process.argv.slice(2)); + await (command === 'installer' ? installerSmoke(options) : desktopSmoke(options)); +} catch (error) { + console.error(`[release-smoke] FAILED: ${error instanceof Error ? error.message : String(error)}`); + process.exit(1); +} diff --git a/scripts/release-smoke.test.ts b/scripts/release-smoke.test.ts new file mode 100644 index 0000000..2d31387 --- /dev/null +++ b/scripts/release-smoke.test.ts @@ -0,0 +1,146 @@ +/** + * scripts/release-smoke.mjs against stand-in payloads. The installer smoke + * test installs, checks, runs and removes a packed archive whose service is a + * small HTTP server; the desktop smoke test starts a stand-in .app that starts + * a stand-in sidecar, and must stop both. + */ +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import net from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +const ROOT = path.resolve(import.meta.dirname, '..'); +const SMOKE = path.join(ROOT, 'scripts', 'release-smoke.mjs'); +const LAUNCHER = path.join(ROOT, 'packages', 'launcher', 'dist', 'cli.js'); +const TARGETS: Record = { + 'win32-x64': 'windows-x64', + 'darwin-arm64': 'macos-arm64', + 'darwin-x64': 'macos-x64', + 'linux-x64': 'linux-x64', +}; +const target = TARGETS[`${process.platform}-${process.arch}`]; + +// Answers /health the way the sidecar does. +const healthServer = (port: string) => ` +const http = require('node:http'); +const port = Number(${port}); +http.createServer((request, response) => { + response.setHeader('content-type', 'application/json'); + response.end(JSON.stringify(request.url === '/health' ? { status: 'ok', mode: 'local', port } : {})); +}).listen(port, '127.0.0.1'); +`; + +const temps: string[] = []; +function tempDir(prefix: string): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temps.push(dir); + return dir; +} +afterEach(() => { + for (const dir of temps.splice(0)) fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 500 }); +}); + +function write(file: string, content: string, mode?: number): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, content, { mode }); +} + +function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + const address = server.address() as net.AddressInfo; + server.close(() => resolve(address.port)); + }); + }); +} + +function runSmoke(args: string[]) { + return spawnSync(process.execPath, [SMOKE, ...args], { encoding: 'utf8', timeout: 120_000, windowsHide: true }); +} + +describe('release smoke tests', () => { + it.skipIf(!target || !fs.existsSync(LAUNCHER))( + 'installs, checks, runs and uninstalls a server archive', + () => { + const work = tempDir('waggle-smoke-archive-'); + const resources = path.join(work, 'resources'); + const runtime = path.join(resources, process.platform === 'win32' ? 'node.exe' : 'node'); + fs.mkdirSync(resources, { recursive: true }); + // The payload runs on its own Node; this test's Node stands in for it. + if (process.platform === 'win32') fs.copyFileSync(process.execPath, runtime); + else fs.symlinkSync(process.execPath, runtime); + write(path.join(resources, 'service.js'), healthServer('process.env.WAGGLE_PORT')); + write( + path.join(resources, 'node_modules', 'better-sqlite3', 'index.js'), + 'module.exports = class { prepare() { return { get: () => ({ 1: 1 }) }; } };\n', + ); + write(path.join(work, 'web', 'index.html'), ''); + const out = path.join(work, 'out'); + const pack = spawnSync( + process.execPath, + [ + path.join(ROOT, 'scripts', 'build-server-archive.mjs'), + '--resources', resources, '--web', path.join(work, 'web'), '--out', out, + ], + { encoding: 'utf8' }, + ); + expect(pack.status, pack.stderr).toBe(0); + const archive = path.join(out, fs.readdirSync(out).find((name) => name.endsWith('.tar.gz'))!); + + const result = runSmoke(['installer', '--archive', archive, '--timeout', '60']); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain('The installed server answers /health'); + expect(result.stdout).toContain('Installer smoke test passed'); + }, + 150_000, + ); + + it.skipIf(process.platform === 'win32')( + 'starts a desktop app bundle and stops it together with its sidecar', + async () => { + const port = await freePort(); + const work = tempDir('waggle-smoke-desktop-'); + const contents = path.join(work, 'Stand-in.app', 'Contents'); + write( + path.join(contents, 'Info.plist'), + 'CFBundleExecutable\n stand-in\n', + ); + write(path.join(work, 'sidecar.js'), healthServer(String(port))); + // The app stays alive and owns a child process that serves /health. + write( + path.join(work, 'app.js'), + `require('node:child_process').spawn(process.execPath, [${JSON.stringify(path.join(work, 'sidecar.js'))}], { stdio: 'inherit' });\n` + + 'setInterval(() => {}, 1000);\n', + ); + write( + path.join(contents, 'MacOS', 'stand-in'), + `#!/bin/sh\nexec ${JSON.stringify(process.execPath)} ${JSON.stringify(path.join(work, 'app.js'))}\n`, + 0o755, + ); + + const result = runSmoke(['desktop', '--app', path.join(work, 'Stand-in.app'), '--port', String(port), '--timeout', '30']); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + expect(result.stdout).toContain(`answers /health on 127.0.0.1:${port}`); + expect(result.stdout).toContain('Desktop smoke test passed'); + + const probe = net.connect(port, '127.0.0.1'); + const connected = await new Promise((resolve) => { + probe.once('connect', () => resolve(true)); + probe.once('error', () => resolve(false)); + }); + probe.destroy(); + expect(connected).toBe(false); + }, + 90_000, + ); + + it('rejects a desktop run without an app and unknown options', () => { + expect(runSmoke(['desktop']).stderr).toContain('desktop needs --app'); + expect(runSmoke(['installer', '--fast', '1']).stderr).toContain('Unknown option: --fast'); + expect(runSmoke(['publish']).status).toBe(1); + }); +});