2029 lines
71 KiB
Rust
2029 lines
71 KiB
Rust
use serde::{Deserialize, Serialize};
|
|
use std::ffi::OsStr;
|
|
#[cfg(windows)]
|
|
use std::fs::{File, OpenOptions};
|
|
#[cfg(windows)]
|
|
use std::io::{Read, Write};
|
|
use std::path::{Path, PathBuf};
|
|
#[cfg(windows)]
|
|
use std::process::Stdio;
|
|
use std::process::{Child, Command};
|
|
#[cfg(windows)]
|
|
use std::sync::Arc;
|
|
use std::sync::{Mutex, OnceLock};
|
|
use std::time::{Duration, Instant};
|
|
use tauri::{AppHandle, Emitter, Manager, State};
|
|
use uuid::Uuid;
|
|
|
|
const WATCHDOG_FAILURE_THRESHOLD: u32 = 3;
|
|
const WATCHDOG_INITIAL_STARTUP_GRACE: Duration = Duration::from_secs(600);
|
|
/// How long a stopping sidecar gets to shut down on its own before SIGKILL.
|
|
const SERVICE_STOP_GRACE: Duration = Duration::from_secs(5);
|
|
const READY_RECORD_MAX_BYTES: u64 = 16 * 1024;
|
|
#[cfg(windows)]
|
|
const CREATE_NO_WINDOW: u32 = 0x08000000;
|
|
#[cfg(windows)]
|
|
const MAX_SERVICE_LOG_BYTES: u64 = 5 * 1024 * 1024;
|
|
#[cfg(windows)]
|
|
const SERVICE_LOG_DIR_ENV: &str = "WAGGLE_DESKTOP_SERVICE_LOG_DIR";
|
|
|
|
#[cfg(windows)]
|
|
#[derive(Debug)]
|
|
struct KillOnCloseJob(windows_sys::Win32::Foundation::HANDLE);
|
|
|
|
#[cfg(windows)]
|
|
unsafe impl Send for KillOnCloseJob {}
|
|
|
|
#[cfg(windows)]
|
|
impl Drop for KillOnCloseJob {
|
|
fn drop(&mut self) {
|
|
unsafe {
|
|
windows_sys::Win32::Foundation::CloseHandle(self.0);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
fn create_kill_on_close_job() -> Result<KillOnCloseJob, String> {
|
|
use windows_sys::Win32::System::JobObjects::{
|
|
CreateJobObjectW, JobObjectExtendedLimitInformation, SetInformationJobObject,
|
|
JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE,
|
|
JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK,
|
|
};
|
|
|
|
let job = unsafe { CreateJobObjectW(std::ptr::null(), std::ptr::null()) };
|
|
if job.is_null() {
|
|
return Err(format!(
|
|
"Unable to create the managed service job: {}",
|
|
std::io::Error::last_os_error()
|
|
));
|
|
}
|
|
let job = KillOnCloseJob(job);
|
|
let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default();
|
|
limits.BasicLimitInformation.LimitFlags =
|
|
JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK;
|
|
let configured = unsafe {
|
|
SetInformationJobObject(
|
|
job.0,
|
|
JobObjectExtendedLimitInformation,
|
|
std::ptr::from_ref(&limits).cast(),
|
|
std::mem::size_of::<JOBOBJECT_EXTENDED_LIMIT_INFORMATION>() as u32,
|
|
)
|
|
};
|
|
if configured == 0 {
|
|
return Err(format!(
|
|
"Unable to configure the managed service job: {}",
|
|
std::io::Error::last_os_error()
|
|
));
|
|
}
|
|
Ok(job)
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
fn assign_child_to_job(job: &KillOnCloseJob, child: &Child) -> Result<(), String> {
|
|
use std::os::windows::io::AsRawHandle;
|
|
use windows_sys::Win32::Foundation::HANDLE;
|
|
use windows_sys::Win32::System::JobObjects::AssignProcessToJobObject;
|
|
|
|
let assigned = unsafe { AssignProcessToJobObject(job.0, child.as_raw_handle() as HANDLE) };
|
|
if assigned == 0 {
|
|
return Err(format!(
|
|
"Unable to bind the service lifetime to Waggle: {}",
|
|
std::io::Error::last_os_error()
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub struct ServiceEndpoint {
|
|
pub port: u16,
|
|
pub instance_id: String,
|
|
pub bootstrap_token: String,
|
|
}
|
|
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
struct ManagedLaunchConfig {
|
|
instance_id: String,
|
|
bootstrap_token: String,
|
|
ready_path: PathBuf,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
|
struct DesktopReadyRecord {
|
|
schema_version: u8,
|
|
instance_id: String,
|
|
pid: u32,
|
|
host: String,
|
|
preferred_port: u16,
|
|
port: u16,
|
|
started_at: String,
|
|
}
|
|
|
|
#[derive(Debug, Deserialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
struct HealthIdentity {
|
|
instance_id: Option<String>,
|
|
port: Option<u16>,
|
|
}
|
|
|
|
fn managed_launch_config(instance_id: &str) -> Result<ManagedLaunchConfig, String> {
|
|
let instance_id = instance_id.trim();
|
|
if instance_id.is_empty() {
|
|
return Err("Managed service instance ID cannot be empty".to_string());
|
|
}
|
|
let ready_path = std::env::temp_dir()
|
|
.join("waggle-sidecar-ready")
|
|
.join(format!("sidecar-{}-{instance_id}.json", std::process::id()));
|
|
if !ready_path.is_absolute() {
|
|
return Err("Managed service ready path must be absolute".to_string());
|
|
}
|
|
Ok(ManagedLaunchConfig {
|
|
instance_id: instance_id.to_string(),
|
|
bootstrap_token: Uuid::new_v4().to_string(),
|
|
ready_path,
|
|
})
|
|
}
|
|
|
|
fn new_managed_launch_config() -> Result<ManagedLaunchConfig, String> {
|
|
managed_launch_config(&Uuid::new_v4().to_string())
|
|
}
|
|
|
|
fn apply_managed_launch_environment(
|
|
command: &mut Command,
|
|
preferred_port: u16,
|
|
config: &ManagedLaunchConfig,
|
|
) {
|
|
command.env("WAGGLE_PORT", preferred_port.to_string());
|
|
command.env("WAGGLE_DESKTOP_PORT_FALLBACK", "1");
|
|
command.env("WAGGLE_INSTANCE_ID", &config.instance_id);
|
|
command.env("WAGGLE_DESKTOP_BOOTSTRAP_TOKEN", &config.bootstrap_token);
|
|
command.env("WAGGLE_READY_FILE", &config.ready_path);
|
|
}
|
|
|
|
fn parse_ready_record(
|
|
bytes: &[u8],
|
|
expected_instance_id: &str,
|
|
expected_pid: u32,
|
|
expected_preferred_port: u16,
|
|
bootstrap_token: &str,
|
|
) -> Result<ServiceEndpoint, String> {
|
|
if bytes.len() as u64 > READY_RECORD_MAX_BYTES {
|
|
return Err("Managed service ready record is too large".to_string());
|
|
}
|
|
let record: DesktopReadyRecord = serde_json::from_slice(bytes)
|
|
.map_err(|error| format!("Invalid managed service ready record: {error}"))?;
|
|
if record.schema_version != 1
|
|
|| record.instance_id != expected_instance_id
|
|
|| record.pid != expected_pid
|
|
|| record.host != "127.0.0.1"
|
|
|| record.preferred_port != expected_preferred_port
|
|
|| record.port == 0
|
|
|| record.started_at.trim().is_empty()
|
|
{
|
|
return Err("Managed service ready record does not match the owned launch".to_string());
|
|
}
|
|
Ok(ServiceEndpoint {
|
|
port: record.port,
|
|
instance_id: record.instance_id,
|
|
bootstrap_token: bootstrap_token.to_string(),
|
|
})
|
|
}
|
|
|
|
fn health_payload_matches(bytes: &[u8], endpoint: &ServiceEndpoint) -> bool {
|
|
serde_json::from_slice::<HealthIdentity>(bytes).is_ok_and(|health| {
|
|
health.instance_id.as_deref() == Some(endpoint.instance_id.as_str())
|
|
&& health.port == Some(endpoint.port)
|
|
})
|
|
}
|
|
|
|
fn netstat_has_owned_listener(bytes: &[u8], port: u16, pid: u32) -> bool {
|
|
String::from_utf8_lossy(bytes).lines().any(|line| {
|
|
let columns = line.split_whitespace().collect::<Vec<_>>();
|
|
if columns.len() < 5
|
|
|| !columns[0].eq_ignore_ascii_case("TCP")
|
|
|| !columns[3].eq_ignore_ascii_case("LISTENING")
|
|
|| columns[4].parse::<u32>().ok() != Some(pid)
|
|
{
|
|
return false;
|
|
}
|
|
let Some((host, local_port)) = columns[1].rsplit_once(':') else {
|
|
return false;
|
|
};
|
|
host == "127.0.0.1" && local_port.parse::<u16>().ok() == Some(port)
|
|
})
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
fn listener_is_owned_by_pid(port: u16, pid: u32) -> Result<bool, String> {
|
|
let system_root = std::env::var_os("SystemRoot")
|
|
.or_else(|| std::env::var_os("WINDIR"))
|
|
.map(PathBuf::from)
|
|
.ok_or_else(|| "Unable to resolve the Windows system directory".to_string())?;
|
|
let netstat = system_root.join("System32").join("netstat.exe");
|
|
if !netstat.is_file() {
|
|
return Err("Unable to locate the Windows TCP ownership verifier".to_string());
|
|
}
|
|
let mut command = Command::new(netstat);
|
|
command.args(["-ano", "-p", "tcp"]);
|
|
hide_windows_console(&mut command);
|
|
let output = command
|
|
.output()
|
|
.map_err(|error| format!("Unable to inspect Windows TCP ownership: {error}"))?;
|
|
if !output.status.success() {
|
|
return Err("Windows TCP ownership verification failed".to_string());
|
|
}
|
|
Ok(netstat_has_owned_listener(&output.stdout, port, pid))
|
|
}
|
|
|
|
#[cfg(not(windows))]
|
|
fn listener_is_owned_by_pid(_port: u16, _pid: u32) -> Result<bool, String> {
|
|
// The active launch gate is Windows-first. macOS retains the existing
|
|
// instance/child-handle validation until its platform-specific owner
|
|
// query is implemented in the deferred macOS certification phase.
|
|
Ok(true)
|
|
}
|
|
|
|
struct ManagedLaunch {
|
|
generation: u64,
|
|
child: Child,
|
|
#[cfg(windows)]
|
|
_job: KillOnCloseJob,
|
|
config: ManagedLaunchConfig,
|
|
endpoint: Option<ServiceEndpoint>,
|
|
started_at: Instant,
|
|
}
|
|
|
|
struct ServiceRuntime {
|
|
next_generation: u64,
|
|
active: Option<ManagedLaunch>,
|
|
desired_running: bool,
|
|
}
|
|
|
|
pub struct ServiceState {
|
|
preferred_port: u16,
|
|
runtime: Mutex<ServiceRuntime>,
|
|
}
|
|
|
|
impl ServiceState {
|
|
pub fn new(preferred_port: u16) -> Self {
|
|
Self {
|
|
preferred_port,
|
|
runtime: Mutex::new(ServiceRuntime {
|
|
next_generation: 1,
|
|
active: None,
|
|
desired_running: true,
|
|
}),
|
|
}
|
|
}
|
|
|
|
pub(crate) fn verified_port(&self) -> Result<u16, String> {
|
|
current_launch_token(self)?
|
|
.and_then(|token| token.endpoint.map(|endpoint| endpoint.port))
|
|
.ok_or_else(|| "Managed service endpoint has not been verified".to_string())
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
struct LaunchToken {
|
|
generation: u64,
|
|
instance_id: String,
|
|
bootstrap_token: String,
|
|
ready_path: PathBuf,
|
|
pid: u32,
|
|
preferred_port: u16,
|
|
endpoint: Option<ServiceEndpoint>,
|
|
started_at: Instant,
|
|
}
|
|
|
|
fn valid_node_override(custom: Option<&OsStr>) -> Option<PathBuf> {
|
|
let custom_path = PathBuf::from(custom?);
|
|
custom_path.is_file().then_some(custom_path)
|
|
}
|
|
|
|
/// Tauri's resource directory for this install, set once at startup: next to
|
|
/// the executable on Windows, `Contents/Resources` in a macOS app bundle,
|
|
/// `/usr/lib/<app>` for a deb package and `$APPDIR/usr/lib/<app>` in an AppImage.
|
|
static TAURI_RESOURCE_DIR: OnceLock<PathBuf> = OnceLock::new();
|
|
|
|
pub fn set_resource_dir(dir: PathBuf) {
|
|
let _ = TAURI_RESOURCE_DIR.set(dir);
|
|
}
|
|
|
|
/// Directory of the bundled sidecar payload (service.js, node, native,
|
|
/// node_modules).
|
|
fn bundled_resources_dir(exe_dir: &Path) -> PathBuf {
|
|
bundled_resources_dir_from(TAURI_RESOURCE_DIR.get().map(PathBuf::as_path), exe_dir)
|
|
}
|
|
|
|
/// Prefers Tauri's resource directory. The paths derived from the executable
|
|
/// cover a launch before setup has recorded it: Windows keeps the payload next
|
|
/// to the executable, a macOS app bundle in `Contents/Resources` (UF-004).
|
|
fn bundled_resources_dir_from(resource_dir: Option<&Path>, exe_dir: &Path) -> PathBuf {
|
|
let next_to_exe = exe_dir.join("resources");
|
|
let app_bundle = exe_dir
|
|
.parent()
|
|
.map(|contents| contents.join("Resources").join("resources"));
|
|
resource_dir
|
|
.map(|dir| dir.join("resources"))
|
|
.into_iter()
|
|
.chain(std::iter::once(next_to_exe.clone()))
|
|
.chain(app_bundle)
|
|
.find(|dir| dir.join("service.js").exists())
|
|
.unwrap_or(next_to_exe)
|
|
}
|
|
|
|
/// Resolve the Node.js binary path.
|
|
/// Priority: WAGGLE_NODE_PATH env → bundled resources/node[.exe] → system PATH "node"
|
|
fn resolve_node_path() -> String {
|
|
// 1. Explicit env override (development/advanced users)
|
|
let custom = std::env::var_os("WAGGLE_NODE_PATH");
|
|
if let Some(custom_path) = valid_node_override(custom.as_deref()) {
|
|
return custom_path.to_string_lossy().to_string();
|
|
}
|
|
if custom.as_deref().is_some_and(|value| !value.is_empty()) {
|
|
eprintln!("[waggle] Ignoring WAGGLE_NODE_PATH because it is not a runtime file");
|
|
}
|
|
|
|
// 2. Bundled Node.js in the bundled resources directory
|
|
let exe_dir = std::env::current_exe()
|
|
.ok()
|
|
.and_then(|p| p.parent().map(|d| d.to_path_buf()));
|
|
|
|
if let Some(ref dir) = exe_dir {
|
|
let resources = bundled_resources_dir(dir);
|
|
let bundled = if cfg!(windows) {
|
|
resources.join("node.exe")
|
|
} else {
|
|
resources.join("node")
|
|
};
|
|
if bundled.exists() {
|
|
return bundled.to_string_lossy().to_string();
|
|
}
|
|
}
|
|
|
|
// 3. Development fallback: relative to src-tauri working dir
|
|
if cfg!(debug_assertions) {
|
|
let dev_resources = if cfg!(windows) {
|
|
"resources/node.exe"
|
|
} else {
|
|
"resources/node"
|
|
};
|
|
if std::path::Path::new(dev_resources).exists() {
|
|
return dev_resources.to_string();
|
|
}
|
|
}
|
|
|
|
// 4. System PATH fallback
|
|
"node".to_string()
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
enum ServiceScriptKind {
|
|
Bundled,
|
|
DevSource,
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
struct ServiceScript {
|
|
path: PathBuf,
|
|
kind: ServiceScriptKind,
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
struct BundledNpmEnvironment {
|
|
data_dir: PathBuf,
|
|
path: std::ffi::OsString,
|
|
npm_exec_path: PathBuf,
|
|
npm_prefix: PathBuf,
|
|
npm_cache: PathBuf,
|
|
}
|
|
|
|
fn bundled_npm_environment(
|
|
resources_dir: &Path,
|
|
configured_data_dir: Option<&OsStr>,
|
|
home_dir: Option<&OsStr>,
|
|
ambient_path: Option<&OsStr>,
|
|
) -> Result<BundledNpmEnvironment, String> {
|
|
let data_dir = configured_data_dir
|
|
.filter(|value| !value.is_empty())
|
|
.map(PathBuf::from)
|
|
.or_else(|| {
|
|
home_dir
|
|
.filter(|value| !value.is_empty())
|
|
.map(PathBuf::from)
|
|
.map(|home| home.join(".waggle"))
|
|
})
|
|
.ok_or_else(|| "Unable to resolve writable npm data directory".to_string())?;
|
|
let runtime_dir = resources_dir
|
|
.join("node_modules")
|
|
.join("waggle-node-runtime");
|
|
let npm_root = data_dir.join("npm");
|
|
let separator = if cfg!(windows) { ";" } else { ":" };
|
|
let mut path = std::ffi::OsString::from(runtime_dir.join("bin"));
|
|
path.push(separator);
|
|
path.push(resources_dir);
|
|
if let Some(existing_path) = ambient_path.filter(|value| !value.is_empty()) {
|
|
path.push(separator);
|
|
path.push(existing_path);
|
|
}
|
|
|
|
Ok(BundledNpmEnvironment {
|
|
data_dir,
|
|
path,
|
|
npm_exec_path: runtime_dir
|
|
.join("node_modules")
|
|
.join("npm")
|
|
.join("bin")
|
|
.join("npm-cli.js"),
|
|
npm_prefix: npm_root.join("prefix"),
|
|
npm_cache: npm_root.join("cache"),
|
|
})
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
fn hide_windows_console(command: &mut Command) {
|
|
use std::os::windows::process::CommandExt;
|
|
command.creation_flags(CREATE_NO_WINDOW);
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
fn rotate_service_log(log_path: &Path) -> Result<(), String> {
|
|
let metadata = match std::fs::symlink_metadata(log_path) {
|
|
Ok(metadata) => metadata,
|
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
|
Err(error) => return Err(format!("Unable to inspect service log: {error}")),
|
|
};
|
|
if !metadata.file_type().is_file() || metadata.file_type().is_symlink() {
|
|
return Err("Service log path is not a regular file".to_string());
|
|
}
|
|
if metadata.len() < MAX_SERVICE_LOG_BYTES {
|
|
return Ok(());
|
|
}
|
|
|
|
let previous_path = log_path.with_file_name("service.log.1");
|
|
if previous_path.exists() {
|
|
std::fs::remove_file(&previous_path)
|
|
.map_err(|error| format!("Unable to replace previous service log: {error}"))?;
|
|
}
|
|
std::fs::rename(log_path, previous_path)
|
|
.map_err(|error| format!("Unable to rotate service log: {error}"))
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
struct ServiceLogWriter {
|
|
file: Option<File>,
|
|
length: u64,
|
|
path: PathBuf,
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
impl ServiceLogWriter {
|
|
fn open(data_dir: &Path) -> Result<Self, String> {
|
|
let log_dir = data_dir.join("logs");
|
|
std::fs::create_dir_all(&log_dir)
|
|
.map_err(|error| format!("Unable to create service log directory: {error}"))?;
|
|
let path = log_dir.join("service.log");
|
|
rotate_service_log(&path)?;
|
|
let file = OpenOptions::new()
|
|
.create(true)
|
|
.append(true)
|
|
.open(&path)
|
|
.map_err(|error| format!("Unable to open service log: {error}"))?;
|
|
let length = file
|
|
.metadata()
|
|
.map_err(|error| format!("Unable to inspect service log: {error}"))?
|
|
.len();
|
|
Ok(Self {
|
|
file: Some(file),
|
|
length,
|
|
path,
|
|
})
|
|
}
|
|
|
|
fn rotate(&mut self) -> Result<(), String> {
|
|
if let Some(mut file) = self.file.take() {
|
|
file.flush()
|
|
.map_err(|error| format!("Unable to flush service log: {error}"))?;
|
|
}
|
|
rotate_service_log(&self.path)?;
|
|
self.file = Some(
|
|
OpenOptions::new()
|
|
.create(true)
|
|
.append(true)
|
|
.open(&self.path)
|
|
.map_err(|error| format!("Unable to reopen service log: {error}"))?,
|
|
);
|
|
self.length = 0;
|
|
Ok(())
|
|
}
|
|
|
|
fn write_capped(&mut self, mut bytes: &[u8]) -> Result<(), String> {
|
|
while !bytes.is_empty() {
|
|
if self.length >= MAX_SERVICE_LOG_BYTES {
|
|
self.rotate()?;
|
|
}
|
|
let available = (MAX_SERVICE_LOG_BYTES - self.length) as usize;
|
|
let count = available.min(bytes.len());
|
|
let file = self
|
|
.file
|
|
.as_mut()
|
|
.ok_or_else(|| "Service log is unavailable".to_string())?;
|
|
file.write_all(&bytes[..count])
|
|
.map_err(|error| format!("Unable to write service log: {error}"))?;
|
|
file.flush()
|
|
.map_err(|error| format!("Unable to flush service log: {error}"))?;
|
|
self.length += count as u64;
|
|
bytes = &bytes[count..];
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
fn pump_service_log<R: Read + Send + 'static>(mut input: R, output: Arc<Mutex<ServiceLogWriter>>) {
|
|
std::thread::spawn(move || {
|
|
let mut buffer = [0u8; 8192];
|
|
loop {
|
|
let count = match input.read(&mut buffer) {
|
|
Ok(0) | Err(_) => break,
|
|
Ok(count) => count,
|
|
};
|
|
let Ok(mut writer) = output.lock() else {
|
|
break;
|
|
};
|
|
if writer.write_capped(&buffer[..count]).is_err() {
|
|
break;
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
fn start_service_log_capture(child: &mut Child, data_dir: &Path) -> Result<(), String> {
|
|
let stdout = child
|
|
.stdout
|
|
.take()
|
|
.ok_or_else(|| "Managed service stdout pipe is unavailable".to_string())?;
|
|
let stderr = child
|
|
.stderr
|
|
.take()
|
|
.ok_or_else(|| "Managed service stderr pipe is unavailable".to_string())?;
|
|
let writer = Arc::new(Mutex::new(ServiceLogWriter::open(data_dir)?));
|
|
pump_service_log(stdout, Arc::clone(&writer));
|
|
pump_service_log(stderr, writer);
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
fn configured_service_log_dir(command: &Command) -> Option<PathBuf> {
|
|
command
|
|
.get_envs()
|
|
.find(|(name, _)| *name == OsStr::new(SERVICE_LOG_DIR_ENV))
|
|
.and_then(|(_, value)| value)
|
|
.map(PathBuf::from)
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
fn configure_bundled_windows_process(command: &mut Command, data_dir: &Path) -> Result<(), String> {
|
|
ServiceLogWriter::open(data_dir)?;
|
|
hide_windows_console(command);
|
|
command
|
|
.env(SERVICE_LOG_DIR_ENV, data_dir)
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::piped())
|
|
.stderr(Stdio::piped());
|
|
Ok(())
|
|
}
|
|
|
|
fn find_dev_service_script(current_dir: &Path) -> Option<PathBuf> {
|
|
for dir in current_dir.ancestors() {
|
|
let candidate = dir
|
|
.join("packages")
|
|
.join("server")
|
|
.join("src")
|
|
.join("local")
|
|
.join("service.ts");
|
|
if candidate.exists() {
|
|
return Some(candidate);
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
fn resolve_service_script(
|
|
exe_dir: Option<&Path>,
|
|
current_dir: &Path,
|
|
) -> Result<ServiceScript, String> {
|
|
if let Some(dir) = exe_dir {
|
|
let bundled = bundled_resources_dir(dir).join("service.js");
|
|
if bundled.exists() {
|
|
return Ok(ServiceScript {
|
|
path: bundled,
|
|
kind: ServiceScriptKind::Bundled,
|
|
});
|
|
}
|
|
}
|
|
|
|
if cfg!(debug_assertions) {
|
|
if let Some(script) = find_dev_service_script(current_dir) {
|
|
return Ok(ServiceScript {
|
|
path: script,
|
|
kind: ServiceScriptKind::DevSource,
|
|
});
|
|
}
|
|
}
|
|
|
|
Err("Unable to locate sidecar service.js resource".to_string())
|
|
}
|
|
|
|
/// Build the Command used to spawn the sidecar process.
|
|
/// Shared by both the sync auto-start path and the async `ensure_service` tauri command.
|
|
fn build_service_command(
|
|
preferred_port: u16,
|
|
launch_config: &ManagedLaunchConfig,
|
|
) -> Result<Command, String> {
|
|
let node_path = resolve_node_path();
|
|
|
|
let exe_dir = std::env::current_exe()
|
|
.ok()
|
|
.and_then(|p| p.parent().map(|d| d.to_path_buf()));
|
|
|
|
let current_dir = std::env::current_dir().map_err(|e| e.to_string())?;
|
|
let service_script = resolve_service_script(exe_dir.as_deref(), ¤t_dir)?;
|
|
|
|
let mut cmd = if service_script.kind == ServiceScriptKind::DevSource {
|
|
let mut c = Command::new(&node_path);
|
|
// Workspace packages resolve to their sources through the `source`
|
|
// export condition, so a dev launch needs no build:packages first.
|
|
c.arg("--conditions=source")
|
|
.arg("--import")
|
|
.arg("tsx")
|
|
.arg(&service_script.path);
|
|
c
|
|
} else {
|
|
let mut c = Command::new(&node_path);
|
|
c.arg(&service_script.path);
|
|
c
|
|
};
|
|
|
|
apply_managed_launch_environment(&mut cmd, preferred_port, launch_config);
|
|
|
|
if service_script.kind == ServiceScriptKind::Bundled {
|
|
cmd.env("WAGGLE_SKIP_LITELLM", "1");
|
|
|
|
if let Some(ref dir) = exe_dir {
|
|
let resources_dir = bundled_resources_dir(dir);
|
|
let native_dir = resources_dir.join("native");
|
|
let node_modules_dir = resources_dir.join("node_modules");
|
|
let configured_data_dir = std::env::var_os("WAGGLE_DATA_DIR");
|
|
let home_dir = if cfg!(windows) {
|
|
std::env::var_os("USERPROFILE")
|
|
.filter(|value| !value.is_empty())
|
|
.or_else(|| std::env::var_os("HOME").filter(|value| !value.is_empty()))
|
|
} else {
|
|
std::env::var_os("HOME")
|
|
.filter(|value| !value.is_empty())
|
|
.or_else(|| std::env::var_os("USERPROFILE").filter(|value| !value.is_empty()))
|
|
};
|
|
let ambient_path = std::env::var_os("PATH");
|
|
let npm_environment = bundled_npm_environment(
|
|
&resources_dir,
|
|
configured_data_dir.as_deref(),
|
|
home_dir.as_deref(),
|
|
ambient_path.as_deref(),
|
|
)?;
|
|
std::fs::create_dir_all(&npm_environment.npm_prefix)
|
|
.map_err(|error| format!("Unable to create bundled npm prefix: {error}"))?;
|
|
std::fs::create_dir_all(&npm_environment.npm_cache)
|
|
.map_err(|error| format!("Unable to create bundled npm cache: {error}"))?;
|
|
cmd.env("PATH", &npm_environment.path);
|
|
cmd.env("NPM_EXECPATH", &npm_environment.npm_exec_path);
|
|
cmd.env("NPM_CONFIG_PREFIX", &npm_environment.npm_prefix);
|
|
cmd.env("NPM_CONFIG_CACHE", &npm_environment.npm_cache);
|
|
#[cfg(windows)]
|
|
configure_bundled_windows_process(&mut cmd, &npm_environment.data_dir)?;
|
|
|
|
// NODE_PATH must include the staged production deps
|
|
// (resources/node_modules — better-sqlite3, @fastify/static,
|
|
// drizzle-orm, @huggingface/transformers, …) so the sidecar's bare
|
|
// require()/import() calls resolve, plus resources/native for any
|
|
// abs-path native consumers. Node accepts multiple entries,
|
|
// ';'-separated on Windows and ':' elsewhere.
|
|
let sep = if cfg!(windows) { ";" } else { ":" };
|
|
let node_path = format!(
|
|
"{}{}{}",
|
|
node_modules_dir.to_string_lossy(),
|
|
sep,
|
|
native_dir.to_string_lossy(),
|
|
);
|
|
cmd.env("NODE_PATH", node_path);
|
|
|
|
let ort_dir = native_dir.join("onnxruntime");
|
|
if ort_dir.exists() {
|
|
cmd.env(
|
|
"ONNXRUNTIME_NODE_BINDING_PATH",
|
|
ort_dir.to_string_lossy().as_ref(),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(cmd)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use std::process::Stdio;
|
|
use std::thread;
|
|
|
|
fn long_lived_command() -> Command {
|
|
let mut command = if cfg!(windows) {
|
|
let mut command = Command::new("cmd");
|
|
command.args(["/C", "ping -n 30 127.0.0.1 >NUL"]);
|
|
command
|
|
} else {
|
|
let mut command = Command::new("sh");
|
|
command.args(["-c", "sleep 30"]);
|
|
command
|
|
};
|
|
command
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::null())
|
|
.stderr(Stdio::null());
|
|
command
|
|
}
|
|
|
|
#[test]
|
|
fn managed_launch_uses_unique_ready_paths_and_exact_child_environment() {
|
|
let first = managed_launch_config("instance-a").expect("first launch config");
|
|
let second = managed_launch_config("instance-b").expect("second launch config");
|
|
assert!(first.ready_path.is_absolute());
|
|
assert_ne!(first.ready_path, second.ready_path);
|
|
assert_ne!(first.bootstrap_token, second.bootstrap_token);
|
|
|
|
let mut command = long_lived_command();
|
|
apply_managed_launch_environment(&mut command, 3333, &first);
|
|
let environment = command
|
|
.get_envs()
|
|
.filter_map(|(name, value)| value.map(|value| (name.to_owned(), value.to_owned())))
|
|
.collect::<std::collections::HashMap<_, _>>();
|
|
|
|
assert_eq!(
|
|
environment.get(OsStr::new("WAGGLE_PORT")),
|
|
Some(&3333.to_string().into())
|
|
);
|
|
assert_eq!(
|
|
environment.get(OsStr::new("WAGGLE_DESKTOP_PORT_FALLBACK")),
|
|
Some(&"1".into())
|
|
);
|
|
assert_eq!(
|
|
environment.get(OsStr::new("WAGGLE_INSTANCE_ID")),
|
|
Some(&"instance-a".into())
|
|
);
|
|
assert_eq!(
|
|
environment.get(OsStr::new("WAGGLE_DESKTOP_BOOTSTRAP_TOKEN")),
|
|
Some(&first.bootstrap_token.clone().into())
|
|
);
|
|
assert_eq!(
|
|
environment.get(OsStr::new("WAGGLE_READY_FILE")),
|
|
Some(&first.ready_path.into_os_string())
|
|
);
|
|
|
|
let generated_first = new_managed_launch_config().expect("generated first launch");
|
|
let generated_second = new_managed_launch_config().expect("generated second launch");
|
|
assert_ne!(generated_first.instance_id, generated_second.instance_id);
|
|
assert_ne!(
|
|
generated_first.bootstrap_token,
|
|
generated_second.bootstrap_token
|
|
);
|
|
assert_ne!(generated_first.ready_path, generated_second.ready_path);
|
|
}
|
|
|
|
#[test]
|
|
fn ready_record_accepts_only_the_owned_complete_launch() {
|
|
let valid = br#"{"schemaVersion":1,"instanceId":"instance-a","pid":42,"host":"127.0.0.1","preferredPort":3333,"port":49152,"startedAt":"2026-08-03T00:00:00.000Z"}"#;
|
|
assert_eq!(
|
|
parse_ready_record(valid, "instance-a", 42, 3333, "bootstrap-a")
|
|
.expect("owned ready record"),
|
|
ServiceEndpoint {
|
|
port: 49152,
|
|
instance_id: "instance-a".to_string(),
|
|
bootstrap_token: "bootstrap-a".to_string()
|
|
}
|
|
);
|
|
assert!(parse_ready_record(valid, "instance-b", 42, 3333, "bootstrap-a").is_err());
|
|
assert!(parse_ready_record(valid, "instance-a", 43, 3333, "bootstrap-a").is_err());
|
|
assert!(parse_ready_record(valid, "instance-a", 42, 3334, "bootstrap-a").is_err());
|
|
assert!(parse_ready_record(
|
|
br#"{"instanceId":"instance-a","port":49152}"#,
|
|
"instance-a",
|
|
42,
|
|
3333,
|
|
"bootstrap-a"
|
|
)
|
|
.is_err());
|
|
assert!(parse_ready_record(br#"{"schemaVersion":1,"instanceId":"instance-a","pid":42,"host":"127.0.0.1","preferredPort":3333,"port":0,"startedAt":"now"}"#, "instance-a", 42, 3333, "bootstrap-a").is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn health_payload_requires_the_exact_owned_endpoint() {
|
|
let endpoint = ServiceEndpoint {
|
|
port: 49152,
|
|
instance_id: "instance-a".to_string(),
|
|
bootstrap_token: "bootstrap-a".to_string(),
|
|
};
|
|
assert!(health_payload_matches(
|
|
br#"{"status":"ok","instanceId":"instance-a","port":49152}"#,
|
|
&endpoint
|
|
));
|
|
assert!(!health_payload_matches(
|
|
br#"{"status":"ok","instanceId":"instance-b","port":49152}"#,
|
|
&endpoint
|
|
));
|
|
assert!(!health_payload_matches(
|
|
br#"{"status":"ok","port":49152}"#,
|
|
&endpoint
|
|
));
|
|
assert!(!health_payload_matches(
|
|
br#"{"status":"ok","instanceId":"instance-a","port":3333}"#,
|
|
&endpoint
|
|
));
|
|
assert!(!health_payload_matches(b"not-json", &endpoint));
|
|
}
|
|
|
|
#[test]
|
|
fn windows_listener_proof_requires_exact_loopback_port_and_child_pid() {
|
|
let output =
|
|
b"Proto Local Address Foreign Address State PID\r\n\
|
|
TCP 127.0.0.1:49152 0.0.0.0:0 LISTENING 4242\r\n\
|
|
TCP 127.0.0.1:49153 0.0.0.0:0 LISTENING 9999\r\n";
|
|
assert!(netstat_has_owned_listener(output, 49152, 4242));
|
|
assert!(!netstat_has_owned_listener(output, 49152, 9999));
|
|
assert!(!netstat_has_owned_listener(output, 49153, 4242));
|
|
assert!(!netstat_has_owned_listener(
|
|
b"TCP 0.0.0.0:49152 0.0.0.0:0 LISTENING 4242",
|
|
49152,
|
|
4242,
|
|
));
|
|
assert!(!netstat_has_owned_listener(
|
|
b"TCP 127.0.0.1:49152 0.0.0.0:0 ESTABLISHED 4242",
|
|
49152,
|
|
4242,
|
|
));
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
#[test]
|
|
fn windows_listener_owner_is_verified_against_the_os_table() {
|
|
let listener =
|
|
std::net::TcpListener::bind(("127.0.0.1", 0)).expect("binds owned loopback listener");
|
|
let port = listener
|
|
.local_addr()
|
|
.expect("reads listener address")
|
|
.port();
|
|
assert!(listener_is_owned_by_pid(port, std::process::id())
|
|
.expect("queries Windows TCP ownership"));
|
|
assert!(
|
|
!listener_is_owned_by_pid(port, std::process::id().saturating_add(1))
|
|
.expect("rejects a different owner")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn explicit_stop_suppresses_watchdog_respawn() {
|
|
let state = ServiceState::new(3333);
|
|
spawn_managed_service_with(&state, |_, _| Ok(long_lived_command()))
|
|
.expect("starts managed child");
|
|
stop_service_sync(&state).expect("stops managed child");
|
|
let build_called = std::cell::Cell::new(false);
|
|
|
|
let recovery = recover_service_with(
|
|
&state,
|
|
true,
|
|
Duration::ZERO,
|
|
WATCHDOG_FAILURE_THRESHOLD,
|
|
true,
|
|
|_, _| {
|
|
build_called.set(true);
|
|
Ok(long_lived_command())
|
|
},
|
|
)
|
|
.expect("suppresses stopped service recovery");
|
|
|
|
assert_eq!(recovery, WatchdogRecovery::Suppressed);
|
|
assert!(!build_called.get());
|
|
}
|
|
|
|
#[test]
|
|
fn stale_generation_cannot_commit_or_delete_the_current_launch() {
|
|
let state = ServiceState::new(3333);
|
|
let first = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command()))
|
|
.expect("starts first generation");
|
|
std::fs::create_dir_all(first.ready_path.parent().expect("ready parent"))
|
|
.expect("creates ready parent");
|
|
std::fs::write(&first.ready_path, b"first").expect("writes first ready file");
|
|
stop_service_sync(&state).expect("stops first generation");
|
|
assert!(!first.ready_path.exists());
|
|
|
|
let second = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command()))
|
|
.expect("starts second generation");
|
|
assert_ne!(first.generation, second.generation);
|
|
assert_ne!(first.instance_id, second.instance_id);
|
|
std::fs::create_dir_all(second.ready_path.parent().expect("second ready parent"))
|
|
.expect("creates second ready parent");
|
|
std::fs::write(&second.ready_path, b"second").expect("writes second ready file");
|
|
assert!(commit_endpoint_if_current(
|
|
&state,
|
|
&second,
|
|
ServiceEndpoint {
|
|
port: 49153,
|
|
instance_id: second.instance_id.clone(),
|
|
bootstrap_token: second.bootstrap_token.clone()
|
|
},
|
|
)
|
|
.expect("commits current endpoint"));
|
|
assert!(!commit_endpoint_if_current(
|
|
&state,
|
|
&first,
|
|
ServiceEndpoint {
|
|
port: 49152,
|
|
instance_id: first.instance_id.clone(),
|
|
bootstrap_token: first.bootstrap_token.clone()
|
|
},
|
|
)
|
|
.expect("rejects stale commit"));
|
|
remove_launch_ready_file(&first.ready_path);
|
|
assert!(second.ready_path.exists());
|
|
assert_eq!(
|
|
state.verified_port().expect("preserves current endpoint"),
|
|
49153
|
|
);
|
|
assert_eq!(
|
|
current_launch_token(&state)
|
|
.expect("reads current launch")
|
|
.expect("active launch")
|
|
.generation,
|
|
second.generation,
|
|
);
|
|
stop_service_sync(&state).expect("cleans up second generation");
|
|
}
|
|
|
|
#[test]
|
|
fn command_port_is_unavailable_until_the_owned_endpoint_is_committed() {
|
|
let state = ServiceState::new(3333);
|
|
let launch = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command()))
|
|
.expect("starts managed child");
|
|
assert!(state.verified_port().is_err());
|
|
|
|
let endpoint = ServiceEndpoint {
|
|
port: 49152,
|
|
instance_id: launch.instance_id.clone(),
|
|
bootstrap_token: launch.bootstrap_token.clone(),
|
|
};
|
|
assert!(
|
|
commit_endpoint_if_current(&state, &launch, endpoint).expect("commits owned endpoint")
|
|
);
|
|
assert_eq!(state.verified_port().expect("reads verified port"), 49152);
|
|
stop_service_sync(&state).expect("stops managed child");
|
|
assert!(state.verified_port().is_err());
|
|
}
|
|
|
|
#[test]
|
|
fn service_script_resolves_from_a_macos_app_bundle() {
|
|
let root = std::env::temp_dir().join(format!("waggle-app-bundle-{}", std::process::id()));
|
|
let macos = root.join("Waggle.app").join("Contents").join("MacOS");
|
|
let resources = root
|
|
.join("Waggle.app")
|
|
.join("Contents")
|
|
.join("Resources")
|
|
.join("resources");
|
|
std::fs::create_dir_all(&macos).expect("creates MacOS dir");
|
|
std::fs::create_dir_all(&resources).expect("creates bundle resources");
|
|
std::fs::write(resources.join("service.js"), "console.log('ok')").expect("writes service");
|
|
|
|
let script = resolve_service_script(Some(&macos), Path::new("/nonexistent"))
|
|
.expect("bundled script resolves from Contents/Resources");
|
|
|
|
assert_eq!(script.kind, ServiceScriptKind::Bundled);
|
|
assert_eq!(script.path, resources.join("service.js"));
|
|
assert_eq!(bundled_resources_dir(&macos), resources);
|
|
|
|
let _ = std::fs::remove_dir_all(root);
|
|
}
|
|
|
|
#[test]
|
|
fn resources_resolve_from_the_tauri_resource_dir_of_a_linux_package() {
|
|
let root = std::env::temp_dir().join(format!("waggle-deb-{}", std::process::id()));
|
|
let bin = root.join("usr").join("bin");
|
|
let resource_dir = root.join("usr").join("lib").join("Waggle");
|
|
let resources = resource_dir.join("resources");
|
|
std::fs::create_dir_all(&bin).expect("creates bin dir");
|
|
std::fs::create_dir_all(&resources).expect("creates package resources");
|
|
std::fs::write(resources.join("service.js"), "console.log('ok')").expect("writes service");
|
|
|
|
assert_eq!(bundled_resources_dir_from(Some(&resource_dir), &bin), resources);
|
|
assert_eq!(bundled_resources_dir_from(None, &bin), bin.join("resources"));
|
|
|
|
let _ = std::fs::remove_dir_all(root);
|
|
}
|
|
|
|
#[test]
|
|
fn service_script_prefers_bundled_resource_when_present() {
|
|
let root =
|
|
std::env::temp_dir().join(format!("waggle-service-script-{}", std::process::id()));
|
|
let resources = root.join("resources");
|
|
std::fs::create_dir_all(&resources).expect("creates temp resources");
|
|
std::fs::write(resources.join("service.js"), "console.log('ok')").expect("writes service");
|
|
|
|
let script = resolve_service_script(Some(&root), Path::new("D:/work/waggle-os"))
|
|
.expect("bundled script resolves");
|
|
|
|
assert_eq!(script.kind, ServiceScriptKind::Bundled);
|
|
assert_eq!(script.path, resources.join("service.js"));
|
|
|
|
let _ = std::fs::remove_dir_all(root);
|
|
}
|
|
|
|
#[test]
|
|
fn dev_service_script_searches_current_dir_ancestors() {
|
|
let root =
|
|
std::env::temp_dir().join(format!("waggle-dev-service-script-{}", std::process::id()));
|
|
let script = root
|
|
.join("packages")
|
|
.join("server")
|
|
.join("src")
|
|
.join("local")
|
|
.join("service.ts");
|
|
std::fs::create_dir_all(script.parent().expect("script parent")).expect("creates dirs");
|
|
std::fs::write(&script, "export {};").expect("writes service");
|
|
|
|
let nested = root.join("app").join("src-tauri");
|
|
std::fs::create_dir_all(&nested).expect("creates nested cwd");
|
|
|
|
assert_eq!(find_dev_service_script(&nested), Some(script));
|
|
|
|
let _ = std::fs::remove_dir_all(root);
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_node_overrides_do_not_shadow_runtime_fallbacks() {
|
|
let directory_override = std::env::temp_dir();
|
|
assert_eq!(valid_node_override(None), None);
|
|
assert_eq!(valid_node_override(Some(OsStr::new(""))), None);
|
|
assert_eq!(
|
|
valid_node_override(Some(directory_override.as_os_str())),
|
|
None
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn existing_node_override_is_accepted() {
|
|
let root =
|
|
std::env::temp_dir().join(format!("waggle-node-override-{}", std::process::id()));
|
|
std::fs::create_dir_all(&root).expect("creates temp runtime dir");
|
|
let runtime = root.join(if cfg!(windows) { "node.exe" } else { "node" });
|
|
std::fs::write(&runtime, b"runtime").expect("writes temp runtime");
|
|
|
|
assert_eq!(
|
|
valid_node_override(Some(runtime.as_os_str())),
|
|
Some(runtime.clone())
|
|
);
|
|
|
|
let _ = std::fs::remove_dir_all(root);
|
|
}
|
|
|
|
#[test]
|
|
fn bundled_npm_environment_uses_configured_data_dir_and_preserves_path() {
|
|
let resources = Path::new("waggle resources");
|
|
let data_dir = Path::new("waggle data");
|
|
let ambient_entries = vec![PathBuf::from("ambient-one"), PathBuf::from("ambient two")];
|
|
let ambient_path = std::env::join_paths(&ambient_entries).expect("joins ambient PATH");
|
|
let environment = bundled_npm_environment(
|
|
resources,
|
|
Some(data_dir.as_os_str()),
|
|
Some(OsStr::new("unused home")),
|
|
Some(ambient_path.as_os_str()),
|
|
)
|
|
.expect("bundled npm environment resolves");
|
|
|
|
assert_eq!(
|
|
environment.npm_exec_path,
|
|
resources
|
|
.join("node_modules")
|
|
.join("waggle-node-runtime")
|
|
.join("node_modules")
|
|
.join("npm")
|
|
.join("bin")
|
|
.join("npm-cli.js")
|
|
);
|
|
assert_eq!(environment.npm_prefix, data_dir.join("npm").join("prefix"));
|
|
assert_eq!(environment.npm_cache, data_dir.join("npm").join("cache"));
|
|
assert_eq!(environment.data_dir, data_dir);
|
|
let mut expected_path_entries = vec![
|
|
resources
|
|
.join("node_modules")
|
|
.join("waggle-node-runtime")
|
|
.join("bin"),
|
|
resources.to_path_buf(),
|
|
];
|
|
expected_path_entries.extend(ambient_entries);
|
|
assert_eq!(
|
|
std::env::split_paths(&environment.path).collect::<Vec<_>>(),
|
|
expected_path_entries
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn bundled_npm_environment_falls_back_to_waggle_home() {
|
|
let resources = Path::new("resources");
|
|
let home = Path::new("home");
|
|
let environment = bundled_npm_environment(
|
|
resources,
|
|
Some(OsStr::new("")),
|
|
Some(home.as_os_str()),
|
|
None,
|
|
)
|
|
.expect("fallback npm environment resolves");
|
|
|
|
assert_eq!(
|
|
environment.npm_prefix,
|
|
home.join(".waggle").join("npm").join("prefix")
|
|
);
|
|
assert_eq!(
|
|
environment.npm_cache,
|
|
home.join(".waggle").join("npm").join("cache")
|
|
);
|
|
assert_eq!(environment.data_dir, home.join(".waggle"));
|
|
}
|
|
|
|
#[test]
|
|
fn bundled_npm_environment_requires_a_writable_home() {
|
|
let result = bundled_npm_environment(Path::new("resources"), None, None, None);
|
|
assert_eq!(
|
|
result,
|
|
Err("Unable to resolve writable npm data directory".to_string())
|
|
);
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
#[test]
|
|
fn oversized_service_log_is_rotated_before_launch() {
|
|
let root = std::env::temp_dir().join(format!(
|
|
"waggle-service-log-{}-{}",
|
|
std::process::id(),
|
|
Uuid::new_v4()
|
|
));
|
|
let log_dir = root.join("logs");
|
|
std::fs::create_dir_all(&log_dir).expect("creates log directory");
|
|
let log_path = log_dir.join("service.log");
|
|
std::fs::write(&log_path, vec![b'x'; MAX_SERVICE_LOG_BYTES as usize])
|
|
.expect("writes oversized service log");
|
|
|
|
rotate_service_log(&log_path).expect("rotates service log");
|
|
|
|
assert!(!log_path.exists());
|
|
assert_eq!(
|
|
std::fs::metadata(log_dir.join("service.log.1"))
|
|
.expect("reads rotated log")
|
|
.len(),
|
|
MAX_SERVICE_LOG_BYTES
|
|
);
|
|
let _ = std::fs::remove_dir_all(root);
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
#[test]
|
|
fn service_log_writer_caps_current_and_retained_logs() {
|
|
let root = std::env::temp_dir().join(format!(
|
|
"waggle-service-log-cap-{}-{}",
|
|
std::process::id(),
|
|
Uuid::new_v4()
|
|
));
|
|
let mut writer = ServiceLogWriter::open(&root).expect("opens service log writer");
|
|
writer
|
|
.write_capped(&vec![b'x'; MAX_SERVICE_LOG_BYTES as usize + 1024])
|
|
.expect("writes oversized diagnostics");
|
|
writer
|
|
.write_capped(&vec![b'y'; MAX_SERVICE_LOG_BYTES as usize])
|
|
.expect("rolls diagnostics during one process lifetime");
|
|
drop(writer);
|
|
|
|
let log_dir = root.join("logs");
|
|
let current = std::fs::metadata(log_dir.join("service.log"))
|
|
.expect("reads current service log")
|
|
.len();
|
|
let retained = std::fs::metadata(log_dir.join("service.log.1"))
|
|
.expect("reads retained service log")
|
|
.len();
|
|
assert!(current <= MAX_SERVICE_LOG_BYTES);
|
|
assert!(retained <= MAX_SERVICE_LOG_BYTES);
|
|
assert!(current + retained <= MAX_SERVICE_LOG_BYTES * 2);
|
|
let _ = std::fs::remove_dir_all(root);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[test]
|
|
fn stopping_the_service_lets_it_exit_on_sigterm() {
|
|
let mut child = Command::new("sh")
|
|
.args(["-c", "trap 'exit 0' TERM; while :; do sleep 0.05; done"])
|
|
.spawn()
|
|
.expect("starts service child");
|
|
// Let the shell install its trap before the signal arrives.
|
|
thread::sleep(Duration::from_millis(200));
|
|
|
|
let status =
|
|
stop_service_child(&mut child, Duration::from_secs(5)).expect("reaps service child");
|
|
|
|
assert!(
|
|
status.success(),
|
|
"expected a clean exit on SIGTERM, got {status:?}"
|
|
);
|
|
}
|
|
|
|
#[cfg(unix)]
|
|
#[test]
|
|
fn stopping_the_service_kills_it_when_sigterm_is_ignored() {
|
|
use std::os::unix::process::ExitStatusExt;
|
|
let mut child = Command::new("sh")
|
|
.args(["-c", "trap '' TERM; while :; do sleep 0.05; done"])
|
|
.spawn()
|
|
.expect("starts service child");
|
|
thread::sleep(Duration::from_millis(200));
|
|
let started = Instant::now();
|
|
|
|
let status = stop_service_child(&mut child, Duration::from_millis(300))
|
|
.expect("reaps service child");
|
|
|
|
assert_eq!(status.signal(), Some(libc::SIGKILL));
|
|
assert!(started.elapsed() < Duration::from_secs(3));
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
#[test]
|
|
fn closing_the_service_job_terminates_its_child() {
|
|
let mut child = long_lived_command().spawn().expect("starts service child");
|
|
let job = create_kill_on_close_job().expect("creates kill-on-close job");
|
|
assign_child_to_job(&job, &child).expect("assigns service child to job");
|
|
|
|
drop(job);
|
|
|
|
let deadline = Instant::now() + Duration::from_secs(5);
|
|
loop {
|
|
if child.try_wait().expect("inspects service child").is_some() {
|
|
break;
|
|
}
|
|
if Instant::now() >= deadline {
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
panic!("service child survived closing the kill-on-close job");
|
|
}
|
|
thread::sleep(Duration::from_millis(25));
|
|
}
|
|
}
|
|
|
|
#[cfg(windows)]
|
|
#[test]
|
|
fn closing_the_service_job_preserves_a_user_detached_descendant() {
|
|
use windows_sys::Win32::Foundation::{CloseHandle, WAIT_TIMEOUT};
|
|
use windows_sys::Win32::System::Threading::{
|
|
OpenProcess, WaitForSingleObject, PROCESS_QUERY_LIMITED_INFORMATION,
|
|
};
|
|
|
|
let root = std::env::temp_dir().join(format!(
|
|
"waggle-detached-job-{}-{}",
|
|
std::process::id(),
|
|
Uuid::new_v4()
|
|
));
|
|
std::fs::create_dir_all(&root).expect("creates detached-job fixture");
|
|
let marker = root.join("pid.txt");
|
|
let escaped_marker = marker.to_string_lossy().replace('\'', "''");
|
|
let script = format!(
|
|
"$child = Start-Process -FilePath (Join-Path $env:SystemRoot 'System32\\ping.exe') -ArgumentList '-t','127.0.0.1' -WindowStyle Hidden -PassThru; Set-Content -LiteralPath '{escaped_marker}' -Value $child.Id -NoNewline; Start-Sleep -Seconds 30"
|
|
);
|
|
let mut command = Command::new("powershell.exe");
|
|
command.args([
|
|
"-NoLogo",
|
|
"-NoProfile",
|
|
"-NonInteractive",
|
|
"-WindowStyle",
|
|
"Hidden",
|
|
"-Command",
|
|
&script,
|
|
]);
|
|
command
|
|
.stdin(Stdio::null())
|
|
.stdout(Stdio::null())
|
|
.stderr(Stdio::null());
|
|
hide_windows_console(&mut command);
|
|
|
|
let mut parent = command.spawn().expect("starts detached-session fixture");
|
|
let job = create_kill_on_close_job().expect("creates service job");
|
|
assign_child_to_job(&job, &parent).expect("assigns service fixture to job");
|
|
let deadline = Instant::now() + Duration::from_secs(10);
|
|
while !marker.exists() {
|
|
if Instant::now() >= deadline {
|
|
let _ = parent.kill();
|
|
let _ = parent.wait();
|
|
panic!("detached-session fixture did not publish its pid");
|
|
}
|
|
thread::sleep(Duration::from_millis(25));
|
|
}
|
|
let detached_pid = std::fs::read_to_string(&marker)
|
|
.expect("reads detached pid")
|
|
.trim()
|
|
.parse::<u32>()
|
|
.expect("parses detached pid");
|
|
|
|
drop(job);
|
|
let _ = parent.wait();
|
|
const SYNCHRONIZE_ACCESS: u32 = 0x00100000;
|
|
let detached = unsafe {
|
|
OpenProcess(
|
|
PROCESS_QUERY_LIMITED_INFORMATION | SYNCHRONIZE_ACCESS,
|
|
0,
|
|
detached_pid,
|
|
)
|
|
};
|
|
let survived = !detached.is_null()
|
|
&& unsafe { WaitForSingleObject(detached, 0) } == WAIT_TIMEOUT;
|
|
if !detached.is_null() {
|
|
unsafe { CloseHandle(detached) };
|
|
}
|
|
let mut cleanup = Command::new("taskkill");
|
|
cleanup
|
|
.args(["/PID", &detached_pid.to_string(), "/T", "/F"])
|
|
.stdout(Stdio::null())
|
|
.stderr(Stdio::null());
|
|
hide_windows_console(&mut cleanup);
|
|
let _ = cleanup.status();
|
|
let _ = std::fs::remove_dir_all(root);
|
|
|
|
assert!(survived, "user-owned detached session died with the service job");
|
|
}
|
|
|
|
#[test]
|
|
fn exited_service_child_does_not_block_restart() {
|
|
let state = ServiceState::new(3333);
|
|
let command = if cfg!(windows) {
|
|
let mut command = Command::new("cmd");
|
|
command.args(["/C", "exit", "0"]);
|
|
command
|
|
} else {
|
|
let mut command = Command::new("sh");
|
|
command.args(["-c", "exit 0"]);
|
|
command
|
|
};
|
|
spawn_managed_service_with(&state, |_, _| Ok(command)).expect("starts short-lived child");
|
|
|
|
for _ in 0..100 {
|
|
if current_launch_token(&state)
|
|
.expect("inspects service child")
|
|
.is_none()
|
|
{
|
|
break;
|
|
}
|
|
thread::sleep(Duration::from_millis(10));
|
|
}
|
|
assert!(current_launch_token(&state)
|
|
.expect("reads launch")
|
|
.is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn running_service_child_remains_registered() {
|
|
let state = ServiceState::new(3333);
|
|
let launch = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command()))
|
|
.expect("starts long-lived child");
|
|
|
|
assert_eq!(
|
|
current_launch_token(&state)
|
|
.expect("inspects service child")
|
|
.expect("preserves running service child")
|
|
.generation,
|
|
launch.generation,
|
|
);
|
|
stop_service_sync(&state).expect("stops long-lived child");
|
|
}
|
|
|
|
#[test]
|
|
fn service_spawn_holds_process_lock_until_child_registered() {
|
|
let state = ServiceState::new(3333);
|
|
spawn_managed_service_with(&state, |_, _| {
|
|
assert!(matches!(
|
|
state.runtime.try_lock(),
|
|
Err(std::sync::TryLockError::WouldBlock)
|
|
));
|
|
Ok(long_lived_command())
|
|
})
|
|
.expect("spawns service while holding process lock");
|
|
stop_service_sync(&state).expect("stops service child");
|
|
}
|
|
|
|
#[test]
|
|
fn watchdog_keeps_a_live_sidecar_during_cold_embedding_startup() {
|
|
assert!(!watchdog_should_restart(
|
|
false,
|
|
true,
|
|
Duration::from_secs(45),
|
|
WATCHDOG_FAILURE_THRESHOLD,
|
|
));
|
|
assert!(!watchdog_should_restart(
|
|
false,
|
|
true,
|
|
WATCHDOG_INITIAL_STARTUP_GRACE - Duration::from_secs(1),
|
|
u32::MAX,
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn watchdog_restarts_an_exited_sidecar_during_cold_startup() {
|
|
assert!(watchdog_should_restart(false, false, Duration::ZERO, 1,));
|
|
}
|
|
|
|
#[test]
|
|
fn watchdog_restarts_after_startup_grace_or_established_health_loss() {
|
|
assert!(watchdog_should_restart(
|
|
false,
|
|
true,
|
|
WATCHDOG_INITIAL_STARTUP_GRACE,
|
|
WATCHDOG_FAILURE_THRESHOLD,
|
|
));
|
|
assert!(!watchdog_should_restart(
|
|
true,
|
|
true,
|
|
Duration::ZERO,
|
|
WATCHDOG_FAILURE_THRESHOLD - 1,
|
|
));
|
|
assert!(watchdog_should_restart(
|
|
true,
|
|
true,
|
|
Duration::ZERO,
|
|
WATCHDOG_FAILURE_THRESHOLD,
|
|
));
|
|
}
|
|
|
|
#[test]
|
|
fn watchdog_does_not_spawn_while_a_cold_sidecar_is_alive() {
|
|
let state = ServiceState::new(3333);
|
|
spawn_managed_service_with(&state, |_, _| Ok(long_lived_command()))
|
|
.expect("starts cold sidecar");
|
|
let build_called = std::cell::Cell::new(false);
|
|
|
|
let recovery = recover_service_with(
|
|
&state,
|
|
false,
|
|
Duration::from_secs(45),
|
|
u32::MAX,
|
|
true,
|
|
|_, _| {
|
|
build_called.set(true);
|
|
Ok(long_lived_command())
|
|
},
|
|
)
|
|
.expect("defers cold sidecar recovery");
|
|
|
|
assert_eq!(recovery, WatchdogRecovery::Deferred);
|
|
assert!(!build_called.get());
|
|
stop_service_sync(&state).expect("stops cold sidecar");
|
|
}
|
|
|
|
#[test]
|
|
fn watchdog_restart_holds_the_process_lock_until_registration() {
|
|
let state = ServiceState::new(3333);
|
|
|
|
let recovery = recover_service_with(&state, false, Duration::ZERO, 1, true, |_, _| {
|
|
assert!(matches!(
|
|
state.runtime.try_lock(),
|
|
Err(std::sync::TryLockError::WouldBlock)
|
|
));
|
|
Ok(long_lived_command())
|
|
})
|
|
.expect("restarts exited sidecar atomically");
|
|
|
|
assert_eq!(recovery, WatchdogRecovery::Restarted);
|
|
assert!(current_launch_token(&state)
|
|
.expect("reads restarted sidecar")
|
|
.is_some());
|
|
stop_service_sync(&state).expect("stops restarted sidecar");
|
|
}
|
|
|
|
#[test]
|
|
fn watchdog_restart_rotates_identity_and_removes_old_ready_file() {
|
|
let state = ServiceState::new(3333);
|
|
let first = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command()))
|
|
.expect("starts first sidecar");
|
|
std::fs::create_dir_all(first.ready_path.parent().expect("ready parent"))
|
|
.expect("creates ready parent");
|
|
std::fs::write(&first.ready_path, b"first").expect("writes old ready file");
|
|
|
|
assert_eq!(
|
|
recover_service_with(
|
|
&state,
|
|
true,
|
|
Duration::ZERO,
|
|
WATCHDOG_FAILURE_THRESHOLD,
|
|
true,
|
|
|_, _| Ok(long_lived_command()),
|
|
)
|
|
.expect("restarts managed child"),
|
|
WatchdogRecovery::Restarted,
|
|
);
|
|
let second = current_launch_token(&state)
|
|
.expect("reads restarted child")
|
|
.expect("registered restarted child");
|
|
assert_ne!(first.generation, second.generation);
|
|
assert_ne!(first.instance_id, second.instance_id);
|
|
assert!(!first.ready_path.exists());
|
|
assert!(second.endpoint.is_none());
|
|
stop_service_sync(&state).expect("stops restarted sidecar");
|
|
}
|
|
}
|
|
|
|
fn remove_launch_ready_file(path: &Path) {
|
|
match std::fs::symlink_metadata(path) {
|
|
Ok(_) => {
|
|
let _ = std::fs::remove_file(path);
|
|
}
|
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
|
Err(_) => {}
|
|
}
|
|
}
|
|
|
|
fn managed_launch_is_running(launch: &mut ManagedLaunch) -> Result<bool, String> {
|
|
match launch.child.try_wait() {
|
|
Ok(None) => Ok(true),
|
|
Ok(Some(_)) => Ok(false),
|
|
Err(error) => Err(format!("Failed to inspect service process: {error}")),
|
|
}
|
|
}
|
|
|
|
fn launch_token(launch: &ManagedLaunch, preferred_port: u16) -> LaunchToken {
|
|
LaunchToken {
|
|
generation: launch.generation,
|
|
instance_id: launch.config.instance_id.clone(),
|
|
bootstrap_token: launch.config.bootstrap_token.clone(),
|
|
ready_path: launch.config.ready_path.clone(),
|
|
pid: launch.child.id(),
|
|
preferred_port,
|
|
endpoint: launch.endpoint.clone(),
|
|
started_at: launch.started_at,
|
|
}
|
|
}
|
|
|
|
fn current_launch_token(state: &ServiceState) -> Result<Option<LaunchToken>, String> {
|
|
let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?;
|
|
let exited = match runtime.active.as_mut() {
|
|
Some(launch) => !managed_launch_is_running(launch)?,
|
|
None => return Ok(None),
|
|
};
|
|
if exited {
|
|
if let Some(launch) = runtime.active.take() {
|
|
remove_launch_ready_file(&launch.config.ready_path);
|
|
}
|
|
return Ok(None);
|
|
}
|
|
Ok(runtime
|
|
.active
|
|
.as_ref()
|
|
.map(|launch| launch_token(launch, state.preferred_port)))
|
|
}
|
|
|
|
fn spawn_locked(
|
|
runtime: &mut ServiceRuntime,
|
|
preferred_port: u16,
|
|
build_command: impl FnOnce(u16, &ManagedLaunchConfig) -> Result<Command, String>,
|
|
) -> Result<LaunchToken, String> {
|
|
let config = new_managed_launch_config()?;
|
|
remove_launch_ready_file(&config.ready_path);
|
|
let generation = runtime.next_generation;
|
|
runtime.next_generation = runtime.next_generation.saturating_add(1);
|
|
let mut command = build_command(preferred_port, &config)?;
|
|
#[cfg(windows)]
|
|
let log_dir = configured_service_log_dir(&command);
|
|
#[cfg(windows)]
|
|
let job = create_kill_on_close_job()?;
|
|
let mut child = command
|
|
.spawn()
|
|
.map_err(|error| format!("Failed to start service: {error}"))?;
|
|
#[cfg(windows)]
|
|
{
|
|
if let Err(error) = assign_child_to_job(&job, &child) {
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
return Err(error);
|
|
}
|
|
if let Some(log_dir) = log_dir {
|
|
if let Err(error) = start_service_log_capture(&mut child, &log_dir) {
|
|
let _ = child.kill();
|
|
let _ = child.wait();
|
|
return Err(error);
|
|
}
|
|
}
|
|
}
|
|
runtime.active = Some(ManagedLaunch {
|
|
generation,
|
|
child,
|
|
#[cfg(windows)]
|
|
_job: job,
|
|
config,
|
|
endpoint: None,
|
|
started_at: Instant::now(),
|
|
});
|
|
Ok(launch_token(
|
|
runtime
|
|
.active
|
|
.as_ref()
|
|
.expect("managed launch was registered"),
|
|
preferred_port,
|
|
))
|
|
}
|
|
|
|
fn spawn_managed_service_with(
|
|
state: &ServiceState,
|
|
build_command: impl FnOnce(u16, &ManagedLaunchConfig) -> Result<Command, String>,
|
|
) -> Result<LaunchToken, String> {
|
|
let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?;
|
|
runtime.desired_running = true;
|
|
let active_is_running = match runtime.active.as_mut() {
|
|
Some(launch) => managed_launch_is_running(launch)?,
|
|
None => false,
|
|
};
|
|
if active_is_running {
|
|
return Ok(launch_token(
|
|
runtime
|
|
.active
|
|
.as_ref()
|
|
.expect("running launch remains registered"),
|
|
state.preferred_port,
|
|
));
|
|
}
|
|
if let Some(launch) = runtime.active.take() {
|
|
remove_launch_ready_file(&launch.config.ready_path);
|
|
}
|
|
spawn_locked(&mut runtime, state.preferred_port, build_command)
|
|
}
|
|
|
|
/// Stops a sidecar process. On Unix it first sends SIGTERM, so the service closes
|
|
/// what it started (the WhatsApp browser, MCP servers, a managed Ollama) before
|
|
/// it exits, and gets SIGKILL only if it is still running after `grace`. Windows
|
|
/// has no SIGTERM; there the kill-on-close job ends the whole process tree.
|
|
fn stop_service_child(child: &mut Child, grace: Duration) -> Option<std::process::ExitStatus> {
|
|
#[cfg(unix)]
|
|
if let (Ok(None), Ok(pid)) = (child.try_wait(), libc::pid_t::try_from(child.id())) {
|
|
// SAFETY: kill(2) on the pid of a child this process has not reaped yet.
|
|
unsafe { libc::kill(pid, libc::SIGTERM) };
|
|
let deadline = Instant::now() + grace;
|
|
while Instant::now() < deadline {
|
|
match child.try_wait() {
|
|
Ok(Some(status)) => return Some(status),
|
|
Ok(None) => std::thread::sleep(Duration::from_millis(50)),
|
|
Err(_) => break,
|
|
}
|
|
}
|
|
}
|
|
#[cfg(not(unix))]
|
|
let _ = grace;
|
|
let _ = child.kill();
|
|
child.wait().ok()
|
|
}
|
|
|
|
pub(crate) fn stop_service_sync(state: &ServiceState) -> Result<(), String> {
|
|
let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?;
|
|
runtime.desired_running = false;
|
|
if let Some(mut launch) = runtime.active.take() {
|
|
let _ = stop_service_child(&mut launch.child, SERVICE_STOP_GRACE);
|
|
remove_launch_ready_file(&launch.config.ready_path);
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn watchdog_should_restart(
|
|
has_ever_been_healthy: bool,
|
|
child_is_running: bool,
|
|
initial_startup_elapsed: Duration,
|
|
consecutive_failures: u32,
|
|
) -> bool {
|
|
if !child_is_running {
|
|
return true;
|
|
}
|
|
|
|
consecutive_failures >= WATCHDOG_FAILURE_THRESHOLD
|
|
&& (has_ever_been_healthy || initial_startup_elapsed >= WATCHDOG_INITIAL_STARTUP_GRACE)
|
|
}
|
|
|
|
#[derive(Debug, PartialEq, Eq)]
|
|
enum WatchdogRecovery {
|
|
Deferred,
|
|
Restarted,
|
|
RestartLimitReached,
|
|
Suppressed,
|
|
}
|
|
|
|
fn recover_service_with(
|
|
state: &ServiceState,
|
|
has_ever_been_healthy: bool,
|
|
initial_startup_elapsed: Duration,
|
|
consecutive_failures: u32,
|
|
restart_allowed: bool,
|
|
build_command: impl FnOnce(u16, &ManagedLaunchConfig) -> Result<Command, String>,
|
|
) -> Result<WatchdogRecovery, String> {
|
|
let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?;
|
|
if !runtime.desired_running {
|
|
return Ok(WatchdogRecovery::Suppressed);
|
|
}
|
|
let child_is_running = match runtime.active.as_mut() {
|
|
Some(launch) => managed_launch_is_running(launch)?,
|
|
None => false,
|
|
};
|
|
if !child_is_running {
|
|
if let Some(launch) = runtime.active.take() {
|
|
remove_launch_ready_file(&launch.config.ready_path);
|
|
}
|
|
}
|
|
if !watchdog_should_restart(
|
|
has_ever_been_healthy,
|
|
child_is_running,
|
|
initial_startup_elapsed,
|
|
consecutive_failures,
|
|
) {
|
|
return Ok(WatchdogRecovery::Deferred);
|
|
}
|
|
if !restart_allowed {
|
|
return Ok(WatchdogRecovery::RestartLimitReached);
|
|
}
|
|
|
|
if let Some(mut launch) = runtime.active.take() {
|
|
let _ = stop_service_child(&mut launch.child, SERVICE_STOP_GRACE);
|
|
remove_launch_ready_file(&launch.config.ready_path);
|
|
}
|
|
|
|
spawn_locked(&mut runtime, state.preferred_port, build_command)
|
|
.map_err(|error| format!("Failed to restart service: {error}"))?;
|
|
Ok(WatchdogRecovery::Restarted)
|
|
}
|
|
|
|
fn read_ready_endpoint(token: &LaunchToken) -> Result<Option<ServiceEndpoint>, String> {
|
|
let metadata = match std::fs::symlink_metadata(&token.ready_path) {
|
|
Ok(metadata) => metadata,
|
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None),
|
|
Err(error) => {
|
|
return Err(format!(
|
|
"Unable to inspect managed service ready record: {error}"
|
|
))
|
|
}
|
|
};
|
|
if !metadata.file_type().is_file() || metadata.file_type().is_symlink() {
|
|
return Err("Managed service ready record is not a regular file".to_string());
|
|
}
|
|
if metadata.len() > READY_RECORD_MAX_BYTES {
|
|
return Err("Managed service ready record is too large".to_string());
|
|
}
|
|
let bytes = std::fs::read(&token.ready_path)
|
|
.map_err(|error| format!("Unable to read managed service ready record: {error}"))?;
|
|
parse_ready_record(
|
|
&bytes,
|
|
&token.instance_id,
|
|
token.pid,
|
|
token.preferred_port,
|
|
&token.bootstrap_token,
|
|
)
|
|
.map(Some)
|
|
}
|
|
|
|
async fn probe_owned_endpoint(endpoint: &ServiceEndpoint) -> Result<bool, String> {
|
|
let client = reqwest::Client::builder()
|
|
.redirect(reqwest::redirect::Policy::none())
|
|
.timeout(Duration::from_secs(2))
|
|
.build()
|
|
.map_err(|error| format!("Unable to build service health client: {error}"))?;
|
|
let url = format!("http://127.0.0.1:{}/health", endpoint.port);
|
|
let response = match client.get(url).send().await {
|
|
Ok(response) => response,
|
|
Err(_) => return Ok(false),
|
|
};
|
|
if !response.status().is_success() {
|
|
return Ok(false);
|
|
}
|
|
if response
|
|
.content_length()
|
|
.is_some_and(|length| length > READY_RECORD_MAX_BYTES)
|
|
{
|
|
return Err("Managed service health response is too large".to_string());
|
|
}
|
|
let bytes = response
|
|
.bytes()
|
|
.await
|
|
.map_err(|error| format!("Unable to read managed service health response: {error}"))?;
|
|
if bytes.len() as u64 > READY_RECORD_MAX_BYTES {
|
|
return Err("Managed service health response is too large".to_string());
|
|
}
|
|
if !health_payload_matches(&bytes, endpoint) {
|
|
return Err("Managed service health identity does not match the owned launch".to_string());
|
|
}
|
|
Ok(true)
|
|
}
|
|
|
|
fn commit_endpoint_if_current(
|
|
state: &ServiceState,
|
|
token: &LaunchToken,
|
|
endpoint: ServiceEndpoint,
|
|
) -> Result<bool, String> {
|
|
if endpoint.port == 0 || endpoint.instance_id != token.instance_id {
|
|
return Ok(false);
|
|
}
|
|
let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?;
|
|
let Some(active) = runtime.active.as_mut() else {
|
|
return Ok(false);
|
|
};
|
|
if active.generation != token.generation
|
|
|| active.child.id() != token.pid
|
|
|| active.config.instance_id != token.instance_id
|
|
|| !managed_launch_is_running(active)?
|
|
{
|
|
return Ok(false);
|
|
}
|
|
active.endpoint = Some(endpoint);
|
|
Ok(true)
|
|
}
|
|
|
|
async fn discover_owned_endpoint(
|
|
state: &ServiceState,
|
|
token: &LaunchToken,
|
|
) -> Result<Option<ServiceEndpoint>, String> {
|
|
let current = current_launch_token(state)?
|
|
.ok_or_else(|| "Managed service process exited before becoming ready".to_string())?;
|
|
if current.generation != token.generation
|
|
|| current.pid != token.pid
|
|
|| current.instance_id != token.instance_id
|
|
{
|
|
return Err("Managed service launch changed during readiness validation".to_string());
|
|
}
|
|
let endpoint_was_verified = current.endpoint.is_some();
|
|
let endpoint = match current.endpoint.clone() {
|
|
Some(endpoint) => endpoint,
|
|
None => match read_ready_endpoint(¤t)? {
|
|
Some(endpoint) => endpoint,
|
|
None => return Ok(None),
|
|
},
|
|
};
|
|
if !probe_owned_endpoint(&endpoint).await? {
|
|
return Ok(None);
|
|
}
|
|
if !endpoint_was_verified && !listener_is_owned_by_pid(endpoint.port, current.pid)? {
|
|
return Err(
|
|
"Managed service endpoint is not owned by the spawned child process".to_string(),
|
|
);
|
|
}
|
|
if !commit_endpoint_if_current(state, ¤t, endpoint.clone())? {
|
|
return Err("Managed service launch changed before endpoint commit".to_string());
|
|
}
|
|
Ok(Some(endpoint))
|
|
}
|
|
|
|
/// Synchronously spawn the sidecar process if not already running. Does not wait
|
|
/// for the owned ready record. Safe to call from Tauri's synchronous `.setup()` callback.
|
|
pub fn spawn_service_sync(state: &ServiceState) -> Result<(), String> {
|
|
spawn_managed_service_with(state, build_service_command).map(|_| ())
|
|
}
|
|
|
|
#[tauri::command]
|
|
pub async fn ensure_service(state: State<'_, ServiceState>) -> Result<ServiceEndpoint, String> {
|
|
let token = spawn_managed_service_with(&state, build_service_command)?;
|
|
for _ in 0..600 {
|
|
if let Some(endpoint) = discover_owned_endpoint(&state, &token).await? {
|
|
return Ok(endpoint);
|
|
}
|
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
}
|
|
Err("Managed service failed to publish an owned endpoint within 600 seconds".to_string())
|
|
}
|
|
|
|
#[tauri::command]
|
|
pub async fn stop_service(state: State<'_, ServiceState>) -> Result<String, String> {
|
|
stop_service_sync(&state)?;
|
|
Ok("Service stopped".to_string())
|
|
}
|
|
|
|
#[tauri::command]
|
|
pub async fn get_service_port(state: State<'_, ServiceState>) -> Result<u16, String> {
|
|
state.verified_port()
|
|
}
|
|
|
|
pub fn start_watchdog(app: AppHandle) {
|
|
tauri::async_runtime::spawn(async move {
|
|
let mut consecutive_failures: u32 = 0;
|
|
let mut has_ever_been_healthy = false;
|
|
let mut initial_startup_started = Instant::now();
|
|
let mut tracked_generation: Option<u64> = None;
|
|
let mut restart_count: u32 = 0;
|
|
let mut restart_window_start = Instant::now();
|
|
const MAX_RESTARTS: u32 = 5;
|
|
const RESTART_WINDOW: Duration = Duration::from_secs(600);
|
|
|
|
// Wait for initial startup
|
|
tokio::time::sleep(Duration::from_secs(15)).await;
|
|
|
|
loop {
|
|
tokio::time::sleep(Duration::from_secs(10)).await;
|
|
|
|
let Some(state) = app.try_state::<ServiceState>() else {
|
|
eprintln!("[waggle] Watchdog: service state is unavailable");
|
|
break;
|
|
};
|
|
let token = match current_launch_token(&state) {
|
|
Ok(token) => token,
|
|
Err(error) => {
|
|
eprintln!("[waggle] Watchdog: process inspection failed: {error}");
|
|
None
|
|
}
|
|
};
|
|
if let Some(ref token) = token {
|
|
if tracked_generation != Some(token.generation) {
|
|
tracked_generation = Some(token.generation);
|
|
consecutive_failures = 0;
|
|
has_ever_been_healthy = false;
|
|
initial_startup_started = token.started_at;
|
|
}
|
|
}
|
|
|
|
let endpoint = match token.as_ref() {
|
|
Some(token) => match discover_owned_endpoint(&state, token).await {
|
|
Ok(endpoint) => endpoint,
|
|
Err(error) => {
|
|
eprintln!("[waggle] Watchdog: ownership validation failed: {error}");
|
|
None
|
|
}
|
|
},
|
|
None => None,
|
|
};
|
|
if let Some(endpoint) = endpoint {
|
|
if !has_ever_been_healthy {
|
|
let _ = app.emit(
|
|
"waggle://service-status",
|
|
serde_json::json!({ "status": "ready", "endpoint": endpoint }),
|
|
);
|
|
}
|
|
has_ever_been_healthy = true;
|
|
consecutive_failures = 0;
|
|
continue;
|
|
}
|
|
|
|
consecutive_failures = consecutive_failures.saturating_add(1);
|
|
if restart_window_start.elapsed() > RESTART_WINDOW {
|
|
restart_count = 0;
|
|
restart_window_start = Instant::now();
|
|
}
|
|
let recovery = recover_service_with(
|
|
&state,
|
|
has_ever_been_healthy,
|
|
initial_startup_started.elapsed(),
|
|
consecutive_failures,
|
|
restart_count < MAX_RESTARTS,
|
|
build_service_command,
|
|
);
|
|
match recovery {
|
|
Ok(WatchdogRecovery::Deferred) => continue,
|
|
Ok(WatchdogRecovery::Suppressed) => {
|
|
consecutive_failures = 0;
|
|
tracked_generation = None;
|
|
continue;
|
|
}
|
|
Ok(WatchdogRecovery::RestartLimitReached) => {
|
|
let _ = app.emit(
|
|
"waggle://service-status",
|
|
serde_json::json!({ "status": "failed" }),
|
|
);
|
|
eprintln!("[waggle] Watchdog: max restarts exceeded, giving up");
|
|
break;
|
|
}
|
|
Ok(WatchdogRecovery::Restarted) => {
|
|
let _ = app.emit(
|
|
"waggle://service-status",
|
|
serde_json::json!({ "status": "restarting" }),
|
|
);
|
|
let _ = app.emit("waggle://service-restart-needed", ());
|
|
restart_count += 1;
|
|
consecutive_failures = 0;
|
|
has_ever_been_healthy = false;
|
|
tracked_generation = None;
|
|
initial_startup_started = Instant::now();
|
|
}
|
|
Err(error) => {
|
|
eprintln!("[waggle] Watchdog: respawn failed: {error}");
|
|
let _ = app.emit("waggle://service-restart-needed", ());
|
|
restart_count += 1;
|
|
consecutive_failures = 0;
|
|
has_ever_been_healthy = false;
|
|
tracked_generation = None;
|
|
initial_startup_started = Instant::now();
|
|
}
|
|
}
|
|
tokio::time::sleep(Duration::from_secs(10)).await;
|
|
}
|
|
});
|
|
}
|