use serde::{Deserialize, Serialize}; use std::ffi::OsStr; #[cfg(windows)] use std::fs::{File, OpenOptions}; #[cfg(windows)] use std::io::{Read, Write}; use std::path::{Path, PathBuf}; #[cfg(windows)] use std::process::Stdio; use std::process::{Child, Command}; #[cfg(windows)] use std::sync::Arc; use std::sync::Mutex; use std::time::{Duration, Instant}; use tauri::{AppHandle, Emitter, Manager, State}; use uuid::Uuid; const WATCHDOG_FAILURE_THRESHOLD: u32 = 3; const WATCHDOG_INITIAL_STARTUP_GRACE: Duration = Duration::from_secs(600); /// How long a stopping sidecar gets to shut down on its own before SIGKILL. const SERVICE_STOP_GRACE: Duration = Duration::from_secs(5); const READY_RECORD_MAX_BYTES: u64 = 16 * 1024; #[cfg(windows)] const CREATE_NO_WINDOW: u32 = 0x08000000; #[cfg(windows)] const MAX_SERVICE_LOG_BYTES: u64 = 5 * 1024 * 1024; #[cfg(windows)] const SERVICE_LOG_DIR_ENV: &str = "WAGGLE_DESKTOP_SERVICE_LOG_DIR"; #[cfg(windows)] #[derive(Debug)] struct KillOnCloseJob(windows_sys::Win32::Foundation::HANDLE); #[cfg(windows)] unsafe impl Send for KillOnCloseJob {} #[cfg(windows)] impl Drop for KillOnCloseJob { fn drop(&mut self) { unsafe { windows_sys::Win32::Foundation::CloseHandle(self.0); } } } #[cfg(windows)] fn create_kill_on_close_job() -> Result { use windows_sys::Win32::System::JobObjects::{ CreateJobObjectW, JobObjectExtendedLimitInformation, SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK, }; let job = unsafe { CreateJobObjectW(std::ptr::null(), std::ptr::null()) }; if job.is_null() { return Err(format!( "Unable to create the managed service job: {}", std::io::Error::last_os_error() )); } let job = KillOnCloseJob(job); let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK; let configured = unsafe { SetInformationJobObject( job.0, JobObjectExtendedLimitInformation, std::ptr::from_ref(&limits).cast(), std::mem::size_of::() as u32, ) }; if configured == 0 { return Err(format!( "Unable to configure the managed service job: {}", std::io::Error::last_os_error() )); } Ok(job) } #[cfg(windows)] fn assign_child_to_job(job: &KillOnCloseJob, child: &Child) -> Result<(), String> { use std::os::windows::io::AsRawHandle; use windows_sys::Win32::Foundation::HANDLE; use windows_sys::Win32::System::JobObjects::AssignProcessToJobObject; let assigned = unsafe { AssignProcessToJobObject(job.0, child.as_raw_handle() as HANDLE) }; if assigned == 0 { return Err(format!( "Unable to bind the service lifetime to Waggle: {}", std::io::Error::last_os_error() )); } Ok(()) } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ServiceEndpoint { pub port: u16, pub instance_id: String, pub bootstrap_token: String, } #[derive(Clone, Debug, PartialEq, Eq)] struct ManagedLaunchConfig { instance_id: String, bootstrap_token: String, ready_path: PathBuf, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct DesktopReadyRecord { schema_version: u8, instance_id: String, pid: u32, host: String, preferred_port: u16, port: u16, started_at: String, } #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct HealthIdentity { instance_id: Option, port: Option, } fn managed_launch_config(instance_id: &str) -> Result { let instance_id = instance_id.trim(); if instance_id.is_empty() { return Err("Managed service instance ID cannot be empty".to_string()); } let ready_path = std::env::temp_dir() .join("waggle-sidecar-ready") .join(format!("sidecar-{}-{instance_id}.json", std::process::id())); if !ready_path.is_absolute() { return Err("Managed service ready path must be absolute".to_string()); } Ok(ManagedLaunchConfig { instance_id: instance_id.to_string(), bootstrap_token: Uuid::new_v4().to_string(), ready_path, }) } fn new_managed_launch_config() -> Result { managed_launch_config(&Uuid::new_v4().to_string()) } fn apply_managed_launch_environment( command: &mut Command, preferred_port: u16, config: &ManagedLaunchConfig, ) { command.env("WAGGLE_PORT", preferred_port.to_string()); command.env("WAGGLE_DESKTOP_PORT_FALLBACK", "1"); command.env("WAGGLE_INSTANCE_ID", &config.instance_id); command.env("WAGGLE_DESKTOP_BOOTSTRAP_TOKEN", &config.bootstrap_token); command.env("WAGGLE_READY_FILE", &config.ready_path); } fn parse_ready_record( bytes: &[u8], expected_instance_id: &str, expected_pid: u32, expected_preferred_port: u16, bootstrap_token: &str, ) -> Result { if bytes.len() as u64 > READY_RECORD_MAX_BYTES { return Err("Managed service ready record is too large".to_string()); } let record: DesktopReadyRecord = serde_json::from_slice(bytes) .map_err(|error| format!("Invalid managed service ready record: {error}"))?; if record.schema_version != 1 || record.instance_id != expected_instance_id || record.pid != expected_pid || record.host != "127.0.0.1" || record.preferred_port != expected_preferred_port || record.port == 0 || record.started_at.trim().is_empty() { return Err("Managed service ready record does not match the owned launch".to_string()); } Ok(ServiceEndpoint { port: record.port, instance_id: record.instance_id, bootstrap_token: bootstrap_token.to_string(), }) } fn health_payload_matches(bytes: &[u8], endpoint: &ServiceEndpoint) -> bool { serde_json::from_slice::(bytes).is_ok_and(|health| { health.instance_id.as_deref() == Some(endpoint.instance_id.as_str()) && health.port == Some(endpoint.port) }) } fn netstat_has_owned_listener(bytes: &[u8], port: u16, pid: u32) -> bool { String::from_utf8_lossy(bytes).lines().any(|line| { let columns = line.split_whitespace().collect::>(); if columns.len() < 5 || !columns[0].eq_ignore_ascii_case("TCP") || !columns[3].eq_ignore_ascii_case("LISTENING") || columns[4].parse::().ok() != Some(pid) { return false; } let Some((host, local_port)) = columns[1].rsplit_once(':') else { return false; }; host == "127.0.0.1" && local_port.parse::().ok() == Some(port) }) } #[cfg(windows)] fn listener_is_owned_by_pid(port: u16, pid: u32) -> Result { let system_root = std::env::var_os("SystemRoot") .or_else(|| std::env::var_os("WINDIR")) .map(PathBuf::from) .ok_or_else(|| "Unable to resolve the Windows system directory".to_string())?; let netstat = system_root.join("System32").join("netstat.exe"); if !netstat.is_file() { return Err("Unable to locate the Windows TCP ownership verifier".to_string()); } let mut command = Command::new(netstat); command.args(["-ano", "-p", "tcp"]); hide_windows_console(&mut command); let output = command .output() .map_err(|error| format!("Unable to inspect Windows TCP ownership: {error}"))?; if !output.status.success() { return Err("Windows TCP ownership verification failed".to_string()); } Ok(netstat_has_owned_listener(&output.stdout, port, pid)) } #[cfg(not(windows))] fn listener_is_owned_by_pid(_port: u16, _pid: u32) -> Result { // The active launch gate is Windows-first. macOS retains the existing // instance/child-handle validation until its platform-specific owner // query is implemented in the deferred macOS certification phase. Ok(true) } struct ManagedLaunch { generation: u64, child: Child, #[cfg(windows)] _job: KillOnCloseJob, config: ManagedLaunchConfig, endpoint: Option, started_at: Instant, } struct ServiceRuntime { next_generation: u64, active: Option, desired_running: bool, } pub struct ServiceState { preferred_port: u16, runtime: Mutex, } impl ServiceState { pub fn new(preferred_port: u16) -> Self { Self { preferred_port, runtime: Mutex::new(ServiceRuntime { next_generation: 1, active: None, desired_running: true, }), } } pub(crate) fn verified_port(&self) -> Result { current_launch_token(self)? .and_then(|token| token.endpoint.map(|endpoint| endpoint.port)) .ok_or_else(|| "Managed service endpoint has not been verified".to_string()) } } #[derive(Clone, Debug)] struct LaunchToken { generation: u64, instance_id: String, bootstrap_token: String, ready_path: PathBuf, pid: u32, preferred_port: u16, endpoint: Option, started_at: Instant, } fn valid_node_override(custom: Option<&OsStr>) -> Option { let custom_path = PathBuf::from(custom?); custom_path.is_file().then_some(custom_path) } /// Directory of the bundled sidecar payload (service.js, node, native, /// node_modules). Windows installs keep it next to the executable; a macOS app /// bundle keeps it in `Contents/Resources` (UF-004). fn bundled_resources_dir(exe_dir: &Path) -> PathBuf { let next_to_exe = exe_dir.join("resources"); let app_bundle = exe_dir .parent() .map(|contents| contents.join("Resources").join("resources")); std::iter::once(next_to_exe.clone()) .chain(app_bundle) .find(|dir| dir.join("service.js").exists()) .unwrap_or(next_to_exe) } /// Resolve the Node.js binary path. /// Priority: WAGGLE_NODE_PATH env → bundled resources/node[.exe] → system PATH "node" fn resolve_node_path() -> String { // 1. Explicit env override (development/advanced users) let custom = std::env::var_os("WAGGLE_NODE_PATH"); if let Some(custom_path) = valid_node_override(custom.as_deref()) { return custom_path.to_string_lossy().to_string(); } if custom.as_deref().is_some_and(|value| !value.is_empty()) { eprintln!("[waggle] Ignoring WAGGLE_NODE_PATH because it is not a runtime file"); } // 2. Bundled Node.js in the bundled resources directory let exe_dir = std::env::current_exe() .ok() .and_then(|p| p.parent().map(|d| d.to_path_buf())); if let Some(ref dir) = exe_dir { let resources = bundled_resources_dir(dir); let bundled = if cfg!(windows) { resources.join("node.exe") } else { resources.join("node") }; if bundled.exists() { return bundled.to_string_lossy().to_string(); } } // 3. Development fallback: relative to src-tauri working dir if cfg!(debug_assertions) { let dev_resources = if cfg!(windows) { "resources/node.exe" } else { "resources/node" }; if std::path::Path::new(dev_resources).exists() { return dev_resources.to_string(); } } // 4. System PATH fallback "node".to_string() } #[derive(Debug, PartialEq, Eq)] enum ServiceScriptKind { Bundled, DevSource, } #[derive(Debug, PartialEq, Eq)] struct ServiceScript { path: PathBuf, kind: ServiceScriptKind, } #[derive(Debug, PartialEq, Eq)] struct BundledNpmEnvironment { data_dir: PathBuf, path: std::ffi::OsString, npm_exec_path: PathBuf, npm_prefix: PathBuf, npm_cache: PathBuf, } fn bundled_npm_environment( resources_dir: &Path, configured_data_dir: Option<&OsStr>, home_dir: Option<&OsStr>, ambient_path: Option<&OsStr>, ) -> Result { let data_dir = configured_data_dir .filter(|value| !value.is_empty()) .map(PathBuf::from) .or_else(|| { home_dir .filter(|value| !value.is_empty()) .map(PathBuf::from) .map(|home| home.join(".waggle")) }) .ok_or_else(|| "Unable to resolve writable npm data directory".to_string())?; let runtime_dir = resources_dir .join("node_modules") .join("waggle-node-runtime"); let npm_root = data_dir.join("npm"); let separator = if cfg!(windows) { ";" } else { ":" }; let mut path = std::ffi::OsString::from(runtime_dir.join("bin")); path.push(separator); path.push(resources_dir); if let Some(existing_path) = ambient_path.filter(|value| !value.is_empty()) { path.push(separator); path.push(existing_path); } Ok(BundledNpmEnvironment { data_dir, path, npm_exec_path: runtime_dir .join("node_modules") .join("npm") .join("bin") .join("npm-cli.js"), npm_prefix: npm_root.join("prefix"), npm_cache: npm_root.join("cache"), }) } #[cfg(windows)] fn hide_windows_console(command: &mut Command) { use std::os::windows::process::CommandExt; command.creation_flags(CREATE_NO_WINDOW); } #[cfg(windows)] fn rotate_service_log(log_path: &Path) -> Result<(), String> { let metadata = match std::fs::symlink_metadata(log_path) { Ok(metadata) => metadata, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), Err(error) => return Err(format!("Unable to inspect service log: {error}")), }; if !metadata.file_type().is_file() || metadata.file_type().is_symlink() { return Err("Service log path is not a regular file".to_string()); } if metadata.len() < MAX_SERVICE_LOG_BYTES { return Ok(()); } let previous_path = log_path.with_file_name("service.log.1"); if previous_path.exists() { std::fs::remove_file(&previous_path) .map_err(|error| format!("Unable to replace previous service log: {error}"))?; } std::fs::rename(log_path, previous_path) .map_err(|error| format!("Unable to rotate service log: {error}")) } #[cfg(windows)] struct ServiceLogWriter { file: Option, length: u64, path: PathBuf, } #[cfg(windows)] impl ServiceLogWriter { fn open(data_dir: &Path) -> Result { let log_dir = data_dir.join("logs"); std::fs::create_dir_all(&log_dir) .map_err(|error| format!("Unable to create service log directory: {error}"))?; let path = log_dir.join("service.log"); rotate_service_log(&path)?; let file = OpenOptions::new() .create(true) .append(true) .open(&path) .map_err(|error| format!("Unable to open service log: {error}"))?; let length = file .metadata() .map_err(|error| format!("Unable to inspect service log: {error}"))? .len(); Ok(Self { file: Some(file), length, path, }) } fn rotate(&mut self) -> Result<(), String> { if let Some(mut file) = self.file.take() { file.flush() .map_err(|error| format!("Unable to flush service log: {error}"))?; } rotate_service_log(&self.path)?; self.file = Some( OpenOptions::new() .create(true) .append(true) .open(&self.path) .map_err(|error| format!("Unable to reopen service log: {error}"))?, ); self.length = 0; Ok(()) } fn write_capped(&mut self, mut bytes: &[u8]) -> Result<(), String> { while !bytes.is_empty() { if self.length >= MAX_SERVICE_LOG_BYTES { self.rotate()?; } let available = (MAX_SERVICE_LOG_BYTES - self.length) as usize; let count = available.min(bytes.len()); let file = self .file .as_mut() .ok_or_else(|| "Service log is unavailable".to_string())?; file.write_all(&bytes[..count]) .map_err(|error| format!("Unable to write service log: {error}"))?; file.flush() .map_err(|error| format!("Unable to flush service log: {error}"))?; self.length += count as u64; bytes = &bytes[count..]; } Ok(()) } } #[cfg(windows)] fn pump_service_log(mut input: R, output: Arc>) { std::thread::spawn(move || { let mut buffer = [0u8; 8192]; loop { let count = match input.read(&mut buffer) { Ok(0) | Err(_) => break, Ok(count) => count, }; let Ok(mut writer) = output.lock() else { break; }; if writer.write_capped(&buffer[..count]).is_err() { break; } } }); } #[cfg(windows)] fn start_service_log_capture(child: &mut Child, data_dir: &Path) -> Result<(), String> { let stdout = child .stdout .take() .ok_or_else(|| "Managed service stdout pipe is unavailable".to_string())?; let stderr = child .stderr .take() .ok_or_else(|| "Managed service stderr pipe is unavailable".to_string())?; let writer = Arc::new(Mutex::new(ServiceLogWriter::open(data_dir)?)); pump_service_log(stdout, Arc::clone(&writer)); pump_service_log(stderr, writer); Ok(()) } #[cfg(windows)] fn configured_service_log_dir(command: &Command) -> Option { command .get_envs() .find(|(name, _)| *name == OsStr::new(SERVICE_LOG_DIR_ENV)) .and_then(|(_, value)| value) .map(PathBuf::from) } #[cfg(windows)] fn configure_bundled_windows_process(command: &mut Command, data_dir: &Path) -> Result<(), String> { ServiceLogWriter::open(data_dir)?; hide_windows_console(command); command .env(SERVICE_LOG_DIR_ENV, data_dir) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); Ok(()) } fn find_dev_service_script(current_dir: &Path) -> Option { for dir in current_dir.ancestors() { let candidate = dir .join("packages") .join("server") .join("src") .join("local") .join("service.ts"); if candidate.exists() { return Some(candidate); } } None } fn resolve_service_script( exe_dir: Option<&Path>, current_dir: &Path, ) -> Result { if let Some(dir) = exe_dir { let bundled = bundled_resources_dir(dir).join("service.js"); if bundled.exists() { return Ok(ServiceScript { path: bundled, kind: ServiceScriptKind::Bundled, }); } } if cfg!(debug_assertions) { if let Some(script) = find_dev_service_script(current_dir) { return Ok(ServiceScript { path: script, kind: ServiceScriptKind::DevSource, }); } } Err("Unable to locate sidecar service.js resource".to_string()) } /// Build the Command used to spawn the sidecar process. /// Shared by both the sync auto-start path and the async `ensure_service` tauri command. fn build_service_command( preferred_port: u16, launch_config: &ManagedLaunchConfig, ) -> Result { let node_path = resolve_node_path(); let exe_dir = std::env::current_exe() .ok() .and_then(|p| p.parent().map(|d| d.to_path_buf())); let current_dir = std::env::current_dir().map_err(|e| e.to_string())?; let service_script = resolve_service_script(exe_dir.as_deref(), ¤t_dir)?; let mut cmd = if service_script.kind == ServiceScriptKind::DevSource { let mut c = Command::new(&node_path); // Workspace packages resolve to their sources through the `source` // export condition, so a dev launch needs no build:packages first. c.arg("--conditions=source") .arg("--import") .arg("tsx") .arg(&service_script.path); c } else { let mut c = Command::new(&node_path); c.arg(&service_script.path); c }; apply_managed_launch_environment(&mut cmd, preferred_port, launch_config); if service_script.kind == ServiceScriptKind::Bundled { cmd.env("WAGGLE_SKIP_LITELLM", "1"); if let Some(ref dir) = exe_dir { let resources_dir = bundled_resources_dir(dir); let native_dir = resources_dir.join("native"); let node_modules_dir = resources_dir.join("node_modules"); let configured_data_dir = std::env::var_os("WAGGLE_DATA_DIR"); let home_dir = if cfg!(windows) { std::env::var_os("USERPROFILE") .filter(|value| !value.is_empty()) .or_else(|| std::env::var_os("HOME").filter(|value| !value.is_empty())) } else { std::env::var_os("HOME") .filter(|value| !value.is_empty()) .or_else(|| std::env::var_os("USERPROFILE").filter(|value| !value.is_empty())) }; let ambient_path = std::env::var_os("PATH"); let npm_environment = bundled_npm_environment( &resources_dir, configured_data_dir.as_deref(), home_dir.as_deref(), ambient_path.as_deref(), )?; std::fs::create_dir_all(&npm_environment.npm_prefix) .map_err(|error| format!("Unable to create bundled npm prefix: {error}"))?; std::fs::create_dir_all(&npm_environment.npm_cache) .map_err(|error| format!("Unable to create bundled npm cache: {error}"))?; cmd.env("PATH", &npm_environment.path); cmd.env("NPM_EXECPATH", &npm_environment.npm_exec_path); cmd.env("NPM_CONFIG_PREFIX", &npm_environment.npm_prefix); cmd.env("NPM_CONFIG_CACHE", &npm_environment.npm_cache); #[cfg(windows)] configure_bundled_windows_process(&mut cmd, &npm_environment.data_dir)?; // NODE_PATH must include the staged production deps // (resources/node_modules — better-sqlite3, @fastify/static, // drizzle-orm, @huggingface/transformers, …) so the sidecar's bare // require()/import() calls resolve, plus resources/native for any // abs-path native consumers. Node accepts multiple entries, // ';'-separated on Windows and ':' elsewhere. let sep = if cfg!(windows) { ";" } else { ":" }; let node_path = format!( "{}{}{}", node_modules_dir.to_string_lossy(), sep, native_dir.to_string_lossy(), ); cmd.env("NODE_PATH", node_path); let ort_dir = native_dir.join("onnxruntime"); if ort_dir.exists() { cmd.env( "ONNXRUNTIME_NODE_BINDING_PATH", ort_dir.to_string_lossy().as_ref(), ); } } } Ok(cmd) } #[cfg(test)] mod tests { use super::*; use std::process::Stdio; use std::thread; fn long_lived_command() -> Command { let mut command = if cfg!(windows) { let mut command = Command::new("cmd"); command.args(["/C", "ping -n 30 127.0.0.1 >NUL"]); command } else { let mut command = Command::new("sh"); command.args(["-c", "sleep 30"]); command }; command .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()); command } #[test] fn managed_launch_uses_unique_ready_paths_and_exact_child_environment() { let first = managed_launch_config("instance-a").expect("first launch config"); let second = managed_launch_config("instance-b").expect("second launch config"); assert!(first.ready_path.is_absolute()); assert_ne!(first.ready_path, second.ready_path); assert_ne!(first.bootstrap_token, second.bootstrap_token); let mut command = long_lived_command(); apply_managed_launch_environment(&mut command, 3333, &first); let environment = command .get_envs() .filter_map(|(name, value)| value.map(|value| (name.to_owned(), value.to_owned()))) .collect::>(); assert_eq!( environment.get(OsStr::new("WAGGLE_PORT")), Some(&3333.to_string().into()) ); assert_eq!( environment.get(OsStr::new("WAGGLE_DESKTOP_PORT_FALLBACK")), Some(&"1".into()) ); assert_eq!( environment.get(OsStr::new("WAGGLE_INSTANCE_ID")), Some(&"instance-a".into()) ); assert_eq!( environment.get(OsStr::new("WAGGLE_DESKTOP_BOOTSTRAP_TOKEN")), Some(&first.bootstrap_token.clone().into()) ); assert_eq!( environment.get(OsStr::new("WAGGLE_READY_FILE")), Some(&first.ready_path.into_os_string()) ); let generated_first = new_managed_launch_config().expect("generated first launch"); let generated_second = new_managed_launch_config().expect("generated second launch"); assert_ne!(generated_first.instance_id, generated_second.instance_id); assert_ne!( generated_first.bootstrap_token, generated_second.bootstrap_token ); assert_ne!(generated_first.ready_path, generated_second.ready_path); } #[test] fn ready_record_accepts_only_the_owned_complete_launch() { let valid = br#"{"schemaVersion":1,"instanceId":"instance-a","pid":42,"host":"127.0.0.1","preferredPort":3333,"port":49152,"startedAt":"2026-08-03T00:00:00.000Z"}"#; assert_eq!( parse_ready_record(valid, "instance-a", 42, 3333, "bootstrap-a") .expect("owned ready record"), ServiceEndpoint { port: 49152, instance_id: "instance-a".to_string(), bootstrap_token: "bootstrap-a".to_string() } ); assert!(parse_ready_record(valid, "instance-b", 42, 3333, "bootstrap-a").is_err()); assert!(parse_ready_record(valid, "instance-a", 43, 3333, "bootstrap-a").is_err()); assert!(parse_ready_record(valid, "instance-a", 42, 3334, "bootstrap-a").is_err()); assert!(parse_ready_record( br#"{"instanceId":"instance-a","port":49152}"#, "instance-a", 42, 3333, "bootstrap-a" ) .is_err()); assert!(parse_ready_record(br#"{"schemaVersion":1,"instanceId":"instance-a","pid":42,"host":"127.0.0.1","preferredPort":3333,"port":0,"startedAt":"now"}"#, "instance-a", 42, 3333, "bootstrap-a").is_err()); } #[test] fn health_payload_requires_the_exact_owned_endpoint() { let endpoint = ServiceEndpoint { port: 49152, instance_id: "instance-a".to_string(), bootstrap_token: "bootstrap-a".to_string(), }; assert!(health_payload_matches( br#"{"status":"ok","instanceId":"instance-a","port":49152}"#, &endpoint )); assert!(!health_payload_matches( br#"{"status":"ok","instanceId":"instance-b","port":49152}"#, &endpoint )); assert!(!health_payload_matches( br#"{"status":"ok","port":49152}"#, &endpoint )); assert!(!health_payload_matches( br#"{"status":"ok","instanceId":"instance-a","port":3333}"#, &endpoint )); assert!(!health_payload_matches(b"not-json", &endpoint)); } #[test] fn windows_listener_proof_requires_exact_loopback_port_and_child_pid() { let output = b"Proto Local Address Foreign Address State PID\r\n\ TCP 127.0.0.1:49152 0.0.0.0:0 LISTENING 4242\r\n\ TCP 127.0.0.1:49153 0.0.0.0:0 LISTENING 9999\r\n"; assert!(netstat_has_owned_listener(output, 49152, 4242)); assert!(!netstat_has_owned_listener(output, 49152, 9999)); assert!(!netstat_has_owned_listener(output, 49153, 4242)); assert!(!netstat_has_owned_listener( b"TCP 0.0.0.0:49152 0.0.0.0:0 LISTENING 4242", 49152, 4242, )); assert!(!netstat_has_owned_listener( b"TCP 127.0.0.1:49152 0.0.0.0:0 ESTABLISHED 4242", 49152, 4242, )); } #[cfg(windows)] #[test] fn windows_listener_owner_is_verified_against_the_os_table() { let listener = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("binds owned loopback listener"); let port = listener .local_addr() .expect("reads listener address") .port(); assert!(listener_is_owned_by_pid(port, std::process::id()) .expect("queries Windows TCP ownership")); assert!( !listener_is_owned_by_pid(port, std::process::id().saturating_add(1)) .expect("rejects a different owner") ); } #[test] fn explicit_stop_suppresses_watchdog_respawn() { let state = ServiceState::new(3333); spawn_managed_service_with(&state, |_, _| Ok(long_lived_command())) .expect("starts managed child"); stop_service_sync(&state).expect("stops managed child"); let build_called = std::cell::Cell::new(false); let recovery = recover_service_with( &state, true, Duration::ZERO, WATCHDOG_FAILURE_THRESHOLD, true, |_, _| { build_called.set(true); Ok(long_lived_command()) }, ) .expect("suppresses stopped service recovery"); assert_eq!(recovery, WatchdogRecovery::Suppressed); assert!(!build_called.get()); } #[test] fn stale_generation_cannot_commit_or_delete_the_current_launch() { let state = ServiceState::new(3333); let first = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command())) .expect("starts first generation"); std::fs::create_dir_all(first.ready_path.parent().expect("ready parent")) .expect("creates ready parent"); std::fs::write(&first.ready_path, b"first").expect("writes first ready file"); stop_service_sync(&state).expect("stops first generation"); assert!(!first.ready_path.exists()); let second = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command())) .expect("starts second generation"); assert_ne!(first.generation, second.generation); assert_ne!(first.instance_id, second.instance_id); std::fs::create_dir_all(second.ready_path.parent().expect("second ready parent")) .expect("creates second ready parent"); std::fs::write(&second.ready_path, b"second").expect("writes second ready file"); assert!(commit_endpoint_if_current( &state, &second, ServiceEndpoint { port: 49153, instance_id: second.instance_id.clone(), bootstrap_token: second.bootstrap_token.clone() }, ) .expect("commits current endpoint")); assert!(!commit_endpoint_if_current( &state, &first, ServiceEndpoint { port: 49152, instance_id: first.instance_id.clone(), bootstrap_token: first.bootstrap_token.clone() }, ) .expect("rejects stale commit")); remove_launch_ready_file(&first.ready_path); assert!(second.ready_path.exists()); assert_eq!( state.verified_port().expect("preserves current endpoint"), 49153 ); assert_eq!( current_launch_token(&state) .expect("reads current launch") .expect("active launch") .generation, second.generation, ); stop_service_sync(&state).expect("cleans up second generation"); } #[test] fn command_port_is_unavailable_until_the_owned_endpoint_is_committed() { let state = ServiceState::new(3333); let launch = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command())) .expect("starts managed child"); assert!(state.verified_port().is_err()); let endpoint = ServiceEndpoint { port: 49152, instance_id: launch.instance_id.clone(), bootstrap_token: launch.bootstrap_token.clone(), }; assert!( commit_endpoint_if_current(&state, &launch, endpoint).expect("commits owned endpoint") ); assert_eq!(state.verified_port().expect("reads verified port"), 49152); stop_service_sync(&state).expect("stops managed child"); assert!(state.verified_port().is_err()); } #[test] fn service_script_resolves_from_a_macos_app_bundle() { let root = std::env::temp_dir().join(format!("waggle-app-bundle-{}", std::process::id())); let macos = root.join("Waggle.app").join("Contents").join("MacOS"); let resources = root .join("Waggle.app") .join("Contents") .join("Resources") .join("resources"); std::fs::create_dir_all(&macos).expect("creates MacOS dir"); std::fs::create_dir_all(&resources).expect("creates bundle resources"); std::fs::write(resources.join("service.js"), "console.log('ok')").expect("writes service"); let script = resolve_service_script(Some(&macos), Path::new("/nonexistent")) .expect("bundled script resolves from Contents/Resources"); assert_eq!(script.kind, ServiceScriptKind::Bundled); assert_eq!(script.path, resources.join("service.js")); assert_eq!(bundled_resources_dir(&macos), resources); let _ = std::fs::remove_dir_all(root); } #[test] fn service_script_prefers_bundled_resource_when_present() { let root = std::env::temp_dir().join(format!("waggle-service-script-{}", std::process::id())); let resources = root.join("resources"); std::fs::create_dir_all(&resources).expect("creates temp resources"); std::fs::write(resources.join("service.js"), "console.log('ok')").expect("writes service"); let script = resolve_service_script(Some(&root), Path::new("D:/Projects/waggle-os")) .expect("bundled script resolves"); assert_eq!(script.kind, ServiceScriptKind::Bundled); assert_eq!(script.path, resources.join("service.js")); let _ = std::fs::remove_dir_all(root); } #[test] fn dev_service_script_searches_current_dir_ancestors() { let root = std::env::temp_dir().join(format!("waggle-dev-service-script-{}", std::process::id())); let script = root .join("packages") .join("server") .join("src") .join("local") .join("service.ts"); std::fs::create_dir_all(script.parent().expect("script parent")).expect("creates dirs"); std::fs::write(&script, "export {};").expect("writes service"); let nested = root.join("app").join("src-tauri"); std::fs::create_dir_all(&nested).expect("creates nested cwd"); assert_eq!(find_dev_service_script(&nested), Some(script)); let _ = std::fs::remove_dir_all(root); } #[test] fn invalid_node_overrides_do_not_shadow_runtime_fallbacks() { let directory_override = std::env::temp_dir(); assert_eq!(valid_node_override(None), None); assert_eq!(valid_node_override(Some(OsStr::new(""))), None); assert_eq!( valid_node_override(Some(directory_override.as_os_str())), None ); } #[test] fn existing_node_override_is_accepted() { let root = std::env::temp_dir().join(format!("waggle-node-override-{}", std::process::id())); std::fs::create_dir_all(&root).expect("creates temp runtime dir"); let runtime = root.join(if cfg!(windows) { "node.exe" } else { "node" }); std::fs::write(&runtime, b"runtime").expect("writes temp runtime"); assert_eq!( valid_node_override(Some(runtime.as_os_str())), Some(runtime.clone()) ); let _ = std::fs::remove_dir_all(root); } #[test] fn bundled_npm_environment_uses_configured_data_dir_and_preserves_path() { let resources = Path::new("waggle resources"); let data_dir = Path::new("waggle data"); let ambient_entries = vec![PathBuf::from("ambient-one"), PathBuf::from("ambient two")]; let ambient_path = std::env::join_paths(&ambient_entries).expect("joins ambient PATH"); let environment = bundled_npm_environment( resources, Some(data_dir.as_os_str()), Some(OsStr::new("unused home")), Some(ambient_path.as_os_str()), ) .expect("bundled npm environment resolves"); assert_eq!( environment.npm_exec_path, resources .join("node_modules") .join("waggle-node-runtime") .join("node_modules") .join("npm") .join("bin") .join("npm-cli.js") ); assert_eq!(environment.npm_prefix, data_dir.join("npm").join("prefix")); assert_eq!(environment.npm_cache, data_dir.join("npm").join("cache")); assert_eq!(environment.data_dir, data_dir); let mut expected_path_entries = vec![ resources .join("node_modules") .join("waggle-node-runtime") .join("bin"), resources.to_path_buf(), ]; expected_path_entries.extend(ambient_entries); assert_eq!( std::env::split_paths(&environment.path).collect::>(), expected_path_entries ); } #[test] fn bundled_npm_environment_falls_back_to_waggle_home() { let resources = Path::new("resources"); let home = Path::new("home"); let environment = bundled_npm_environment( resources, Some(OsStr::new("")), Some(home.as_os_str()), None, ) .expect("fallback npm environment resolves"); assert_eq!( environment.npm_prefix, home.join(".waggle").join("npm").join("prefix") ); assert_eq!( environment.npm_cache, home.join(".waggle").join("npm").join("cache") ); assert_eq!(environment.data_dir, home.join(".waggle")); } #[test] fn bundled_npm_environment_requires_a_writable_home() { let result = bundled_npm_environment(Path::new("resources"), None, None, None); assert_eq!( result, Err("Unable to resolve writable npm data directory".to_string()) ); } #[cfg(windows)] #[test] fn oversized_service_log_is_rotated_before_launch() { let root = std::env::temp_dir().join(format!( "waggle-service-log-{}-{}", std::process::id(), Uuid::new_v4() )); let log_dir = root.join("logs"); std::fs::create_dir_all(&log_dir).expect("creates log directory"); let log_path = log_dir.join("service.log"); std::fs::write(&log_path, vec![b'x'; MAX_SERVICE_LOG_BYTES as usize]) .expect("writes oversized service log"); rotate_service_log(&log_path).expect("rotates service log"); assert!(!log_path.exists()); assert_eq!( std::fs::metadata(log_dir.join("service.log.1")) .expect("reads rotated log") .len(), MAX_SERVICE_LOG_BYTES ); let _ = std::fs::remove_dir_all(root); } #[cfg(windows)] #[test] fn service_log_writer_caps_current_and_retained_logs() { let root = std::env::temp_dir().join(format!( "waggle-service-log-cap-{}-{}", std::process::id(), Uuid::new_v4() )); let mut writer = ServiceLogWriter::open(&root).expect("opens service log writer"); writer .write_capped(&vec![b'x'; MAX_SERVICE_LOG_BYTES as usize + 1024]) .expect("writes oversized diagnostics"); writer .write_capped(&vec![b'y'; MAX_SERVICE_LOG_BYTES as usize]) .expect("rolls diagnostics during one process lifetime"); drop(writer); let log_dir = root.join("logs"); let current = std::fs::metadata(log_dir.join("service.log")) .expect("reads current service log") .len(); let retained = std::fs::metadata(log_dir.join("service.log.1")) .expect("reads retained service log") .len(); assert!(current <= MAX_SERVICE_LOG_BYTES); assert!(retained <= MAX_SERVICE_LOG_BYTES); assert!(current + retained <= MAX_SERVICE_LOG_BYTES * 2); let _ = std::fs::remove_dir_all(root); } #[cfg(unix)] #[test] fn stopping_the_service_lets_it_exit_on_sigterm() { let mut child = Command::new("sh") .args(["-c", "trap 'exit 0' TERM; while :; do sleep 0.05; done"]) .spawn() .expect("starts service child"); // Let the shell install its trap before the signal arrives. thread::sleep(Duration::from_millis(200)); let status = stop_service_child(&mut child, Duration::from_secs(5)).expect("reaps service child"); assert!( status.success(), "expected a clean exit on SIGTERM, got {status:?}" ); } #[cfg(unix)] #[test] fn stopping_the_service_kills_it_when_sigterm_is_ignored() { use std::os::unix::process::ExitStatusExt; let mut child = Command::new("sh") .args(["-c", "trap '' TERM; while :; do sleep 0.05; done"]) .spawn() .expect("starts service child"); thread::sleep(Duration::from_millis(200)); let started = Instant::now(); let status = stop_service_child(&mut child, Duration::from_millis(300)) .expect("reaps service child"); assert_eq!(status.signal(), Some(libc::SIGKILL)); assert!(started.elapsed() < Duration::from_secs(3)); } #[cfg(windows)] #[test] fn closing_the_service_job_terminates_its_child() { let mut child = long_lived_command().spawn().expect("starts service child"); let job = create_kill_on_close_job().expect("creates kill-on-close job"); assign_child_to_job(&job, &child).expect("assigns service child to job"); drop(job); let deadline = Instant::now() + Duration::from_secs(5); loop { if child.try_wait().expect("inspects service child").is_some() { break; } if Instant::now() >= deadline { let _ = child.kill(); let _ = child.wait(); panic!("service child survived closing the kill-on-close job"); } thread::sleep(Duration::from_millis(25)); } } #[cfg(windows)] #[test] fn closing_the_service_job_preserves_a_user_detached_descendant() { use windows_sys::Win32::Foundation::{CloseHandle, WAIT_TIMEOUT}; use windows_sys::Win32::System::Threading::{ OpenProcess, WaitForSingleObject, PROCESS_QUERY_LIMITED_INFORMATION, }; let root = std::env::temp_dir().join(format!( "waggle-detached-job-{}-{}", std::process::id(), Uuid::new_v4() )); std::fs::create_dir_all(&root).expect("creates detached-job fixture"); let marker = root.join("pid.txt"); let escaped_marker = marker.to_string_lossy().replace('\'', "''"); let script = format!( "$child = Start-Process -FilePath (Join-Path $env:SystemRoot 'System32\\ping.exe') -ArgumentList '-t','127.0.0.1' -WindowStyle Hidden -PassThru; Set-Content -LiteralPath '{escaped_marker}' -Value $child.Id -NoNewline; Start-Sleep -Seconds 30" ); let mut command = Command::new("powershell.exe"); command.args([ "-NoLogo", "-NoProfile", "-NonInteractive", "-WindowStyle", "Hidden", "-Command", &script, ]); command .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()); hide_windows_console(&mut command); let mut parent = command.spawn().expect("starts detached-session fixture"); let job = create_kill_on_close_job().expect("creates service job"); assign_child_to_job(&job, &parent).expect("assigns service fixture to job"); let deadline = Instant::now() + Duration::from_secs(10); while !marker.exists() { if Instant::now() >= deadline { let _ = parent.kill(); let _ = parent.wait(); panic!("detached-session fixture did not publish its pid"); } thread::sleep(Duration::from_millis(25)); } let detached_pid = std::fs::read_to_string(&marker) .expect("reads detached pid") .trim() .parse::() .expect("parses detached pid"); drop(job); let _ = parent.wait(); const SYNCHRONIZE_ACCESS: u32 = 0x00100000; let detached = unsafe { OpenProcess( PROCESS_QUERY_LIMITED_INFORMATION | SYNCHRONIZE_ACCESS, 0, detached_pid, ) }; let survived = !detached.is_null() && unsafe { WaitForSingleObject(detached, 0) } == WAIT_TIMEOUT; if !detached.is_null() { unsafe { CloseHandle(detached) }; } let mut cleanup = Command::new("taskkill"); cleanup .args(["/PID", &detached_pid.to_string(), "/T", "/F"]) .stdout(Stdio::null()) .stderr(Stdio::null()); hide_windows_console(&mut cleanup); let _ = cleanup.status(); let _ = std::fs::remove_dir_all(root); assert!(survived, "user-owned detached session died with the service job"); } #[test] fn exited_service_child_does_not_block_restart() { let state = ServiceState::new(3333); let command = if cfg!(windows) { let mut command = Command::new("cmd"); command.args(["/C", "exit", "0"]); command } else { let mut command = Command::new("sh"); command.args(["-c", "exit 0"]); command }; spawn_managed_service_with(&state, |_, _| Ok(command)).expect("starts short-lived child"); for _ in 0..100 { if current_launch_token(&state) .expect("inspects service child") .is_none() { break; } thread::sleep(Duration::from_millis(10)); } assert!(current_launch_token(&state) .expect("reads launch") .is_none()); } #[test] fn running_service_child_remains_registered() { let state = ServiceState::new(3333); let launch = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command())) .expect("starts long-lived child"); assert_eq!( current_launch_token(&state) .expect("inspects service child") .expect("preserves running service child") .generation, launch.generation, ); stop_service_sync(&state).expect("stops long-lived child"); } #[test] fn service_spawn_holds_process_lock_until_child_registered() { let state = ServiceState::new(3333); spawn_managed_service_with(&state, |_, _| { assert!(matches!( state.runtime.try_lock(), Err(std::sync::TryLockError::WouldBlock) )); Ok(long_lived_command()) }) .expect("spawns service while holding process lock"); stop_service_sync(&state).expect("stops service child"); } #[test] fn watchdog_keeps_a_live_sidecar_during_cold_embedding_startup() { assert!(!watchdog_should_restart( false, true, Duration::from_secs(45), WATCHDOG_FAILURE_THRESHOLD, )); assert!(!watchdog_should_restart( false, true, WATCHDOG_INITIAL_STARTUP_GRACE - Duration::from_secs(1), u32::MAX, )); } #[test] fn watchdog_restarts_an_exited_sidecar_during_cold_startup() { assert!(watchdog_should_restart(false, false, Duration::ZERO, 1,)); } #[test] fn watchdog_restarts_after_startup_grace_or_established_health_loss() { assert!(watchdog_should_restart( false, true, WATCHDOG_INITIAL_STARTUP_GRACE, WATCHDOG_FAILURE_THRESHOLD, )); assert!(!watchdog_should_restart( true, true, Duration::ZERO, WATCHDOG_FAILURE_THRESHOLD - 1, )); assert!(watchdog_should_restart( true, true, Duration::ZERO, WATCHDOG_FAILURE_THRESHOLD, )); } #[test] fn watchdog_does_not_spawn_while_a_cold_sidecar_is_alive() { let state = ServiceState::new(3333); spawn_managed_service_with(&state, |_, _| Ok(long_lived_command())) .expect("starts cold sidecar"); let build_called = std::cell::Cell::new(false); let recovery = recover_service_with( &state, false, Duration::from_secs(45), u32::MAX, true, |_, _| { build_called.set(true); Ok(long_lived_command()) }, ) .expect("defers cold sidecar recovery"); assert_eq!(recovery, WatchdogRecovery::Deferred); assert!(!build_called.get()); stop_service_sync(&state).expect("stops cold sidecar"); } #[test] fn watchdog_restart_holds_the_process_lock_until_registration() { let state = ServiceState::new(3333); let recovery = recover_service_with(&state, false, Duration::ZERO, 1, true, |_, _| { assert!(matches!( state.runtime.try_lock(), Err(std::sync::TryLockError::WouldBlock) )); Ok(long_lived_command()) }) .expect("restarts exited sidecar atomically"); assert_eq!(recovery, WatchdogRecovery::Restarted); assert!(current_launch_token(&state) .expect("reads restarted sidecar") .is_some()); stop_service_sync(&state).expect("stops restarted sidecar"); } #[test] fn watchdog_restart_rotates_identity_and_removes_old_ready_file() { let state = ServiceState::new(3333); let first = spawn_managed_service_with(&state, |_, _| Ok(long_lived_command())) .expect("starts first sidecar"); std::fs::create_dir_all(first.ready_path.parent().expect("ready parent")) .expect("creates ready parent"); std::fs::write(&first.ready_path, b"first").expect("writes old ready file"); assert_eq!( recover_service_with( &state, true, Duration::ZERO, WATCHDOG_FAILURE_THRESHOLD, true, |_, _| Ok(long_lived_command()), ) .expect("restarts managed child"), WatchdogRecovery::Restarted, ); let second = current_launch_token(&state) .expect("reads restarted child") .expect("registered restarted child"); assert_ne!(first.generation, second.generation); assert_ne!(first.instance_id, second.instance_id); assert!(!first.ready_path.exists()); assert!(second.endpoint.is_none()); stop_service_sync(&state).expect("stops restarted sidecar"); } } fn remove_launch_ready_file(path: &Path) { match std::fs::symlink_metadata(path) { Ok(_) => { let _ = std::fs::remove_file(path); } Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(_) => {} } } fn managed_launch_is_running(launch: &mut ManagedLaunch) -> Result { match launch.child.try_wait() { Ok(None) => Ok(true), Ok(Some(_)) => Ok(false), Err(error) => Err(format!("Failed to inspect service process: {error}")), } } fn launch_token(launch: &ManagedLaunch, preferred_port: u16) -> LaunchToken { LaunchToken { generation: launch.generation, instance_id: launch.config.instance_id.clone(), bootstrap_token: launch.config.bootstrap_token.clone(), ready_path: launch.config.ready_path.clone(), pid: launch.child.id(), preferred_port, endpoint: launch.endpoint.clone(), started_at: launch.started_at, } } fn current_launch_token(state: &ServiceState) -> Result, String> { let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?; let exited = match runtime.active.as_mut() { Some(launch) => !managed_launch_is_running(launch)?, None => return Ok(None), }; if exited { if let Some(launch) = runtime.active.take() { remove_launch_ready_file(&launch.config.ready_path); } return Ok(None); } Ok(runtime .active .as_ref() .map(|launch| launch_token(launch, state.preferred_port))) } fn spawn_locked( runtime: &mut ServiceRuntime, preferred_port: u16, build_command: impl FnOnce(u16, &ManagedLaunchConfig) -> Result, ) -> Result { let config = new_managed_launch_config()?; remove_launch_ready_file(&config.ready_path); let generation = runtime.next_generation; runtime.next_generation = runtime.next_generation.saturating_add(1); let mut command = build_command(preferred_port, &config)?; #[cfg(windows)] let log_dir = configured_service_log_dir(&command); #[cfg(windows)] let job = create_kill_on_close_job()?; let mut child = command .spawn() .map_err(|error| format!("Failed to start service: {error}"))?; #[cfg(windows)] { if let Err(error) = assign_child_to_job(&job, &child) { let _ = child.kill(); let _ = child.wait(); return Err(error); } if let Some(log_dir) = log_dir { if let Err(error) = start_service_log_capture(&mut child, &log_dir) { let _ = child.kill(); let _ = child.wait(); return Err(error); } } } runtime.active = Some(ManagedLaunch { generation, child, #[cfg(windows)] _job: job, config, endpoint: None, started_at: Instant::now(), }); Ok(launch_token( runtime .active .as_ref() .expect("managed launch was registered"), preferred_port, )) } fn spawn_managed_service_with( state: &ServiceState, build_command: impl FnOnce(u16, &ManagedLaunchConfig) -> Result, ) -> Result { let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?; runtime.desired_running = true; let active_is_running = match runtime.active.as_mut() { Some(launch) => managed_launch_is_running(launch)?, None => false, }; if active_is_running { return Ok(launch_token( runtime .active .as_ref() .expect("running launch remains registered"), state.preferred_port, )); } if let Some(launch) = runtime.active.take() { remove_launch_ready_file(&launch.config.ready_path); } spawn_locked(&mut runtime, state.preferred_port, build_command) } /// Stops a sidecar process. On Unix it first sends SIGTERM, so the service closes /// what it started (the WhatsApp browser, MCP servers, a managed Ollama) before /// it exits, and gets SIGKILL only if it is still running after `grace`. Windows /// has no SIGTERM; there the kill-on-close job ends the whole process tree. fn stop_service_child(child: &mut Child, grace: Duration) -> Option { #[cfg(unix)] if let (Ok(None), Ok(pid)) = (child.try_wait(), libc::pid_t::try_from(child.id())) { // SAFETY: kill(2) on the pid of a child this process has not reaped yet. unsafe { libc::kill(pid, libc::SIGTERM) }; let deadline = Instant::now() + grace; while Instant::now() < deadline { match child.try_wait() { Ok(Some(status)) => return Some(status), Ok(None) => std::thread::sleep(Duration::from_millis(50)), Err(_) => break, } } } #[cfg(not(unix))] let _ = grace; let _ = child.kill(); child.wait().ok() } pub(crate) fn stop_service_sync(state: &ServiceState) -> Result<(), String> { let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?; runtime.desired_running = false; if let Some(mut launch) = runtime.active.take() { let _ = stop_service_child(&mut launch.child, SERVICE_STOP_GRACE); remove_launch_ready_file(&launch.config.ready_path); } Ok(()) } fn watchdog_should_restart( has_ever_been_healthy: bool, child_is_running: bool, initial_startup_elapsed: Duration, consecutive_failures: u32, ) -> bool { if !child_is_running { return true; } consecutive_failures >= WATCHDOG_FAILURE_THRESHOLD && (has_ever_been_healthy || initial_startup_elapsed >= WATCHDOG_INITIAL_STARTUP_GRACE) } #[derive(Debug, PartialEq, Eq)] enum WatchdogRecovery { Deferred, Restarted, RestartLimitReached, Suppressed, } fn recover_service_with( state: &ServiceState, has_ever_been_healthy: bool, initial_startup_elapsed: Duration, consecutive_failures: u32, restart_allowed: bool, build_command: impl FnOnce(u16, &ManagedLaunchConfig) -> Result, ) -> Result { let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?; if !runtime.desired_running { return Ok(WatchdogRecovery::Suppressed); } let child_is_running = match runtime.active.as_mut() { Some(launch) => managed_launch_is_running(launch)?, None => false, }; if !child_is_running { if let Some(launch) = runtime.active.take() { remove_launch_ready_file(&launch.config.ready_path); } } if !watchdog_should_restart( has_ever_been_healthy, child_is_running, initial_startup_elapsed, consecutive_failures, ) { return Ok(WatchdogRecovery::Deferred); } if !restart_allowed { return Ok(WatchdogRecovery::RestartLimitReached); } if let Some(mut launch) = runtime.active.take() { let _ = stop_service_child(&mut launch.child, SERVICE_STOP_GRACE); remove_launch_ready_file(&launch.config.ready_path); } spawn_locked(&mut runtime, state.preferred_port, build_command) .map_err(|error| format!("Failed to restart service: {error}"))?; Ok(WatchdogRecovery::Restarted) } fn read_ready_endpoint(token: &LaunchToken) -> Result, String> { let metadata = match std::fs::symlink_metadata(&token.ready_path) { Ok(metadata) => metadata, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), Err(error) => { return Err(format!( "Unable to inspect managed service ready record: {error}" )) } }; if !metadata.file_type().is_file() || metadata.file_type().is_symlink() { return Err("Managed service ready record is not a regular file".to_string()); } if metadata.len() > READY_RECORD_MAX_BYTES { return Err("Managed service ready record is too large".to_string()); } let bytes = std::fs::read(&token.ready_path) .map_err(|error| format!("Unable to read managed service ready record: {error}"))?; parse_ready_record( &bytes, &token.instance_id, token.pid, token.preferred_port, &token.bootstrap_token, ) .map(Some) } async fn probe_owned_endpoint(endpoint: &ServiceEndpoint) -> Result { let client = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .timeout(Duration::from_secs(2)) .build() .map_err(|error| format!("Unable to build service health client: {error}"))?; let url = format!("http://127.0.0.1:{}/health", endpoint.port); let response = match client.get(url).send().await { Ok(response) => response, Err(_) => return Ok(false), }; if !response.status().is_success() { return Ok(false); } if response .content_length() .is_some_and(|length| length > READY_RECORD_MAX_BYTES) { return Err("Managed service health response is too large".to_string()); } let bytes = response .bytes() .await .map_err(|error| format!("Unable to read managed service health response: {error}"))?; if bytes.len() as u64 > READY_RECORD_MAX_BYTES { return Err("Managed service health response is too large".to_string()); } if !health_payload_matches(&bytes, endpoint) { return Err("Managed service health identity does not match the owned launch".to_string()); } Ok(true) } fn commit_endpoint_if_current( state: &ServiceState, token: &LaunchToken, endpoint: ServiceEndpoint, ) -> Result { if endpoint.port == 0 || endpoint.instance_id != token.instance_id { return Ok(false); } let mut runtime = state.runtime.lock().map_err(|error| error.to_string())?; let Some(active) = runtime.active.as_mut() else { return Ok(false); }; if active.generation != token.generation || active.child.id() != token.pid || active.config.instance_id != token.instance_id || !managed_launch_is_running(active)? { return Ok(false); } active.endpoint = Some(endpoint); Ok(true) } async fn discover_owned_endpoint( state: &ServiceState, token: &LaunchToken, ) -> Result, String> { let current = current_launch_token(state)? .ok_or_else(|| "Managed service process exited before becoming ready".to_string())?; if current.generation != token.generation || current.pid != token.pid || current.instance_id != token.instance_id { return Err("Managed service launch changed during readiness validation".to_string()); } let endpoint_was_verified = current.endpoint.is_some(); let endpoint = match current.endpoint.clone() { Some(endpoint) => endpoint, None => match read_ready_endpoint(¤t)? { Some(endpoint) => endpoint, None => return Ok(None), }, }; if !probe_owned_endpoint(&endpoint).await? { return Ok(None); } if !endpoint_was_verified && !listener_is_owned_by_pid(endpoint.port, current.pid)? { return Err( "Managed service endpoint is not owned by the spawned child process".to_string(), ); } if !commit_endpoint_if_current(state, ¤t, endpoint.clone())? { return Err("Managed service launch changed before endpoint commit".to_string()); } Ok(Some(endpoint)) } /// Synchronously spawn the sidecar process if not already running. Does not wait /// for the owned ready record. Safe to call from Tauri's synchronous `.setup()` callback. pub fn spawn_service_sync(state: &ServiceState) -> Result<(), String> { spawn_managed_service_with(state, build_service_command).map(|_| ()) } #[tauri::command] pub async fn ensure_service(state: State<'_, ServiceState>) -> Result { let token = spawn_managed_service_with(&state, build_service_command)?; for _ in 0..600 { if let Some(endpoint) = discover_owned_endpoint(&state, &token).await? { return Ok(endpoint); } tokio::time::sleep(Duration::from_secs(1)).await; } Err("Managed service failed to publish an owned endpoint within 600 seconds".to_string()) } #[tauri::command] pub async fn stop_service(state: State<'_, ServiceState>) -> Result { stop_service_sync(&state)?; Ok("Service stopped".to_string()) } #[tauri::command] pub async fn get_service_port(state: State<'_, ServiceState>) -> Result { state.verified_port() } pub fn start_watchdog(app: AppHandle) { tauri::async_runtime::spawn(async move { let mut consecutive_failures: u32 = 0; let mut has_ever_been_healthy = false; let mut initial_startup_started = Instant::now(); let mut tracked_generation: Option = None; let mut restart_count: u32 = 0; let mut restart_window_start = Instant::now(); const MAX_RESTARTS: u32 = 5; const RESTART_WINDOW: Duration = Duration::from_secs(600); // Wait for initial startup tokio::time::sleep(Duration::from_secs(15)).await; loop { tokio::time::sleep(Duration::from_secs(10)).await; let Some(state) = app.try_state::() else { eprintln!("[waggle] Watchdog: service state is unavailable"); break; }; let token = match current_launch_token(&state) { Ok(token) => token, Err(error) => { eprintln!("[waggle] Watchdog: process inspection failed: {error}"); None } }; if let Some(ref token) = token { if tracked_generation != Some(token.generation) { tracked_generation = Some(token.generation); consecutive_failures = 0; has_ever_been_healthy = false; initial_startup_started = token.started_at; } } let endpoint = match token.as_ref() { Some(token) => match discover_owned_endpoint(&state, token).await { Ok(endpoint) => endpoint, Err(error) => { eprintln!("[waggle] Watchdog: ownership validation failed: {error}"); None } }, None => None, }; if let Some(endpoint) = endpoint { if !has_ever_been_healthy { let _ = app.emit( "waggle://service-status", serde_json::json!({ "status": "ready", "endpoint": endpoint }), ); } has_ever_been_healthy = true; consecutive_failures = 0; continue; } consecutive_failures = consecutive_failures.saturating_add(1); if restart_window_start.elapsed() > RESTART_WINDOW { restart_count = 0; restart_window_start = Instant::now(); } let recovery = recover_service_with( &state, has_ever_been_healthy, initial_startup_started.elapsed(), consecutive_failures, restart_count < MAX_RESTARTS, build_service_command, ); match recovery { Ok(WatchdogRecovery::Deferred) => continue, Ok(WatchdogRecovery::Suppressed) => { consecutive_failures = 0; tracked_generation = None; continue; } Ok(WatchdogRecovery::RestartLimitReached) => { let _ = app.emit( "waggle://service-status", serde_json::json!({ "status": "failed" }), ); eprintln!("[waggle] Watchdog: max restarts exceeded, giving up"); break; } Ok(WatchdogRecovery::Restarted) => { let _ = app.emit( "waggle://service-status", serde_json::json!({ "status": "restarting" }), ); let _ = app.emit("waggle://service-restart-needed", ()); restart_count += 1; consecutive_failures = 0; has_ever_been_healthy = false; tracked_generation = None; initial_startup_started = Instant::now(); } Err(error) => { eprintln!("[waggle] Watchdog: respawn failed: {error}"); let _ = app.emit("waggle://service-restart-needed", ()); restart_count += 1; consecutive_failures = 0; has_ever_been_healthy = false; tracked_generation = None; initial_startup_started = Instant::now(); } } tokio::time::sleep(Duration::from_secs(10)).await; } }); }