fix(core): prevent startup failures after core updates
Atomically replace core files, recover from macOS SIGKILLs, and capture startup output for diagnostics. Fixes #224
This commit is contained in:
1 parent
08ee8ca8d3
commit
53ef4dc264
4 files changed
+338
-35
No files matched your search
@@ -867,6 +867,10 @@ pub(crate) fn auth_dir_path_for_core(auth_dir: &str, install_dir: &Path) -> Path
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn core_logs_dir_path(auth_dir: &str, install_dir: &Path) -> PathBuf {
|
||||
auth_dir_path_for_core(auth_dir, install_dir).join("logs")
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "macos", test))]
|
||||
fn normalize_path_lexically(path: &Path) -> PathBuf {
|
||||
let mut normalized = PathBuf::new();
|
||||
|
||||
+234
-28
@@ -1201,6 +1201,147 @@ fn core_management_address(listen_host: &str, port: u16) -> Result<SocketAddr, S
|
||||
Ok(SocketAddr::new(ip, port))
|
||||
}
|
||||
|
||||
pub(crate) enum CoreStartupFailure {
|
||||
Exited(std::process::ExitStatus),
|
||||
Spawn(String),
|
||||
StatusCheck(io::Error),
|
||||
TimedOut(u16),
|
||||
}
|
||||
|
||||
impl CoreStartupFailure {
|
||||
fn child_has_exited(&self) -> bool {
|
||||
matches!(self, Self::Exited(_))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn was_killed_by_sigkill(&self) -> bool {
|
||||
use std::os::unix::process::ExitStatusExt;
|
||||
|
||||
matches!(
|
||||
self,
|
||||
Self::Exited(status)
|
||||
if status.code().is_none() && status.signal() == Some(libc::SIGKILL)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for CoreStartupFailure {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Exited(status) => write!(formatter, "CPA 内核启动后立即退出: {status}"),
|
||||
Self::Spawn(error) => formatter.write_str(error),
|
||||
Self::StatusCheck(error) => write!(formatter, "检查 CPA 内核启动状态失败: {error}"),
|
||||
Self::TimedOut(port) => {
|
||||
write!(formatter, "CPA 内核启动超时:10 秒内未监听管理端口 {port}")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn core_start_log_path(install_dir: &Path, auth_dir: &str) -> PathBuf {
|
||||
core_logs_dir_path(auth_dir, install_dir).join("core-start-output.log")
|
||||
}
|
||||
|
||||
pub(crate) fn core_start_stdio(log_path: &Path) -> io::Result<(Stdio, Stdio)> {
|
||||
if let Some(parent) = log_path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
// Keep only the current process run so console output cannot grow without
|
||||
// bound across restarts. Both child handles use append mode to avoid their
|
||||
// independent file cursors overwriting each other's output.
|
||||
let mut header_file = File::options()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(log_path)?;
|
||||
writeln!(header_file, "===== CPA 内核启动 {} =====", unix_now())?;
|
||||
drop(header_file);
|
||||
|
||||
let stdout_file = File::options().append(true).open(log_path)?;
|
||||
let stderr_file = File::options().append(true).create(true).open(log_path)?;
|
||||
|
||||
Ok((Stdio::from(stdout_file), Stdio::from(stderr_file)))
|
||||
}
|
||||
|
||||
fn core_start_error_with_log(error: &str, log_path: &Path, log_error: Option<&str>) -> String {
|
||||
match log_error {
|
||||
Some(log_error) => format!(
|
||||
"{error};无法写入启动日志 {}: {log_error}",
|
||||
path_to_string(log_path)
|
||||
),
|
||||
None => format!("{error};启动日志: {}", path_to_string(log_path)),
|
||||
}
|
||||
}
|
||||
|
||||
struct CoreStartAttemptFailure {
|
||||
failure: CoreStartupFailure,
|
||||
log_error: Option<String>,
|
||||
}
|
||||
|
||||
impl CoreStartAttemptFailure {
|
||||
#[cfg(target_os = "macos")]
|
||||
fn was_killed_by_sigkill(&self) -> bool {
|
||||
self.failure.was_killed_by_sigkill()
|
||||
}
|
||||
|
||||
fn message(&self, log_path: &Path) -> String {
|
||||
self.message_with_detail(log_path, None)
|
||||
}
|
||||
|
||||
fn message_with_detail(&self, log_path: &Path, detail: Option<&str>) -> String {
|
||||
let error = match detail {
|
||||
Some(detail) => format!("{};{detail}", self.failure),
|
||||
None => self.failure.to_string(),
|
||||
};
|
||||
core_start_error_with_log(&error, log_path, self.log_error.as_deref())
|
||||
}
|
||||
}
|
||||
|
||||
fn start_core_process_once(
|
||||
binary_path: &Path,
|
||||
config_path: &str,
|
||||
install_dir: &Path,
|
||||
log_path: &Path,
|
||||
management_address: SocketAddr,
|
||||
) -> Result<Child, CoreStartAttemptFailure> {
|
||||
let mut command = Command::new(binary_path);
|
||||
command
|
||||
.args(["-config", config_path])
|
||||
.current_dir(install_dir)
|
||||
.stdin(Stdio::null());
|
||||
let log_error = match core_start_stdio(log_path) {
|
||||
Ok((stdout, stderr)) => {
|
||||
command.stdout(stdout).stderr(stderr);
|
||||
None
|
||||
}
|
||||
Err(error) => {
|
||||
command.stdout(Stdio::null()).stderr(Stdio::null());
|
||||
Some(error.to_string())
|
||||
}
|
||||
};
|
||||
configure_background_command(&mut command);
|
||||
|
||||
let mut child = match spawn_core_child(command) {
|
||||
Ok(child) => child,
|
||||
Err(error) => {
|
||||
return Err(CoreStartAttemptFailure {
|
||||
failure: CoreStartupFailure::Spawn(error),
|
||||
log_error,
|
||||
});
|
||||
}
|
||||
};
|
||||
match wait_for_core_management_port(&mut child, management_address) {
|
||||
Ok(()) => Ok(child),
|
||||
Err(failure) => {
|
||||
if !failure.child_has_exited() {
|
||||
let _ = terminate_child(&mut child);
|
||||
}
|
||||
Err(CoreStartAttemptFailure { failure, log_error })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn start_core_process_inner(
|
||||
process_state: &CoreProcessState,
|
||||
gui_config: &GuiConfigFile,
|
||||
@@ -1231,47 +1372,61 @@ pub(crate) fn start_core_process_inner(
|
||||
|
||||
let config_path = merge_core_config_for_start(&install_dir, gui_config)?;
|
||||
let config_path = path_to_string(&config_path);
|
||||
let mut command = Command::new(&binary_path);
|
||||
command
|
||||
.args(["-config", &config_path])
|
||||
.current_dir(&install_dir)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null());
|
||||
configure_background_command(&mut command);
|
||||
let log_path = core_start_log_path(&install_dir, &gui_config.auth_dir);
|
||||
let start_once = || {
|
||||
start_core_process_once(
|
||||
&binary_path,
|
||||
&config_path,
|
||||
&install_dir,
|
||||
&log_path,
|
||||
management_address,
|
||||
)
|
||||
};
|
||||
|
||||
let mut child = spawn_core_child(command)?;
|
||||
|
||||
if let Err(error) = wait_for_core_management_port(&mut child, management_address) {
|
||||
let _ = terminate_child(&mut child);
|
||||
return Err(error);
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
let child = match start_once() {
|
||||
Ok(child) => child,
|
||||
Err(failure) if failure.was_killed_by_sigkill() => {
|
||||
if let Err(heal_error) = rematerialize_core_binary(&binary_path) {
|
||||
return Err(failure.message_with_detail(
|
||||
&log_path,
|
||||
Some(&format!("自动修复 CPA 内核文件失败: {heal_error}")),
|
||||
));
|
||||
}
|
||||
match start_once() {
|
||||
Ok(child) => child,
|
||||
Err(failure) if failure.was_killed_by_sigkill() => {
|
||||
return Err(failure.message_with_detail(
|
||||
&log_path,
|
||||
Some("系统再次终止 CPA 内核,请重新安装内核后重试"),
|
||||
));
|
||||
}
|
||||
Err(failure) => return Err(failure.message(&log_path)),
|
||||
}
|
||||
}
|
||||
Err(failure) => return Err(failure.message(&log_path)),
|
||||
};
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
let child = start_once().map_err(|failure| failure.message(&log_path))?;
|
||||
|
||||
process_state.store_child(child)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn wait_for_core_management_port(
|
||||
child: &mut Child,
|
||||
address: SocketAddr,
|
||||
) -> Result<(), String> {
|
||||
) -> Result<(), CoreStartupFailure> {
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
if let Some(status) = child
|
||||
.try_wait()
|
||||
.map_err(|err| format!("检查 CPA 内核启动状态失败: {err}"))?
|
||||
{
|
||||
return Err(format!("CPA 内核启动后立即退出: {status}"));
|
||||
if let Some(status) = child.try_wait().map_err(CoreStartupFailure::StatusCheck)? {
|
||||
return Err(CoreStartupFailure::Exited(status));
|
||||
}
|
||||
if TcpStream::connect_timeout(&address, Duration::from_millis(200)).is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"CPA 内核启动超时:10 秒内未监听管理端口 {}",
|
||||
address.port()
|
||||
));
|
||||
return Err(CoreStartupFailure::TimedOut(address.port()));
|
||||
}
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
@@ -2195,9 +2350,7 @@ fn overlay_directory(source_dir: &Path, target_dir: &Path) -> Result<(), String>
|
||||
path_to_string(&target_path)
|
||||
));
|
||||
}
|
||||
fs::copy(&source_path, &target_path).map_err(|err| {
|
||||
format!("覆盖内核文件失败 {}: {err}", path_to_string(&target_path))
|
||||
})?;
|
||||
copy_core_file_replace(&source_path, &target_path)?;
|
||||
} else {
|
||||
return Err(format!(
|
||||
"内核暂存目录包含不支持的条目: {}",
|
||||
@@ -2209,6 +2362,59 @@ fn overlay_directory(source_dir: &Path, target_dir: &Path) -> Result<(), String>
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Copies through a sibling temporary file and atomically replaces the target.
|
||||
///
|
||||
/// In particular, do not change this back to copying over an existing file:
|
||||
/// macOS caches code-signature validation by vnode, so in-place updates can
|
||||
/// leave an otherwise valid executable permanently rejected with SIGKILL.
|
||||
pub(crate) fn copy_core_file_replace(source_path: &Path, target_path: &Path) -> Result<(), String> {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static SEQUENCE: AtomicU64 = AtomicU64::new(0);
|
||||
let parent = target_path.parent().ok_or_else(|| {
|
||||
format!(
|
||||
"覆盖内核文件失败 {}: 无法确定父目录",
|
||||
path_to_string(target_path)
|
||||
)
|
||||
})?;
|
||||
fs::create_dir_all(parent)
|
||||
.map_err(|error| format!("创建内核文件目录失败 {}: {error}", path_to_string(parent)))?;
|
||||
let file_name = target_path
|
||||
.file_name()
|
||||
.map(|name| name.to_string_lossy())
|
||||
.unwrap_or_else(|| "cpa-core".into());
|
||||
let temporary_path = parent.join(format!(
|
||||
".{file_name}.replace.{}.{}",
|
||||
std::process::id(),
|
||||
SEQUENCE.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
|
||||
let replace_result = (|| -> io::Result<()> {
|
||||
fs::copy(source_path, &temporary_path)?;
|
||||
let temporary_file = File::options().write(true).open(&temporary_path)?;
|
||||
temporary_file.sync_all()?;
|
||||
drop(temporary_file);
|
||||
replace_file_atomically(&temporary_path, target_path)
|
||||
})();
|
||||
|
||||
if let Err(error) = replace_result {
|
||||
let _ = fs::remove_file(&temporary_path);
|
||||
return Err(format!(
|
||||
"覆盖内核文件失败 {}: {error}",
|
||||
path_to_string(target_path)
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "macos", test))]
|
||||
pub(crate) fn rematerialize_core_binary(binary_path: &Path) -> Result<(), String> {
|
||||
// Replacing the path with an identical copy gives it a fresh vnode and
|
||||
// clears the macOS signature-cache state left by older in-place updates.
|
||||
copy_core_file_replace(binary_path, binary_path)
|
||||
}
|
||||
|
||||
pub(crate) fn extract_tar_gz(archive_path: &Path, install_dir: &Path) -> Result<(), String> {
|
||||
let archive_file =
|
||||
File::open(archive_path).map_err(|err| format!("打开 tar.gz 失败: {err}"))?;
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
#[cfg(target_os = "windows")]
|
||||
use super::windows_explorer_executable;
|
||||
use super::{
|
||||
auth_dir_path_for_core, configure_background_command, core_install_dir, core_origin,
|
||||
current_core_tls_settings, is_hashed_management_secret_key, open_oauth_url_inner,
|
||||
auth_dir_path_for_core, configure_background_command, core_install_dir, core_logs_dir_path,
|
||||
core_origin, current_core_tls_settings, is_hashed_management_secret_key, open_oauth_url_inner,
|
||||
path_to_string, truncate_for_error, GuiConfigFile, GuiConfigState,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -10,7 +10,7 @@ use std::{
|
||||
collections::HashMap,
|
||||
error::Error,
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
path::Path,
|
||||
process::{Command, Stdio},
|
||||
sync::LazyLock,
|
||||
time::Duration,
|
||||
@@ -152,10 +152,6 @@ pub(crate) fn open_core_logs_directory(
|
||||
open_directory_in_file_manager(&logs_dir)
|
||||
}
|
||||
|
||||
fn core_logs_dir_path(auth_dir: &str, install_dir: &Path) -> PathBuf {
|
||||
auth_dir_path_for_core(auth_dir, install_dir).join("logs")
|
||||
}
|
||||
|
||||
fn open_directory_in_file_manager(path: &Path) -> Result<(), String> {
|
||||
#[cfg(target_os = "windows")]
|
||||
let mut command = Command::new(windows_explorer_executable());
|
||||
@@ -466,6 +462,7 @@ fn format_management_error(status: u16, body: &str) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
fn core_logs_follow_the_default_auth_directory() {
|
||||
|
||||
@@ -307,6 +307,9 @@ fn replacing_a_core_preserves_only_regular_bundled_assets() {
|
||||
|
||||
#[test]
|
||||
fn overlaying_a_core_updates_packaged_files_and_preserves_plugins() {
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
|
||||
let root = agent_test_home("core-overlay-preserves-plugins");
|
||||
let install_dir = root.join("cpa-core");
|
||||
let staging_dir = root.join("cpa-core.staging");
|
||||
@@ -329,6 +332,10 @@ fn overlaying_a_core_updates_packaged_files_and_preserves_plugins() {
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(staging_dir.join("runtime/default.json"), b"new runtime").unwrap();
|
||||
#[cfg(unix)]
|
||||
let original_binary_inode = fs::metadata(install_dir.join(core_binary_name()))
|
||||
.unwrap()
|
||||
.ino();
|
||||
|
||||
overlay_install_dir(&install_dir, &staging_dir).unwrap();
|
||||
|
||||
@@ -356,6 +363,14 @@ fn overlaying_a_core_updates_packaged_files_and_preserves_plugins() {
|
||||
fs::read(install_dir.join("user-data.json")).unwrap(),
|
||||
b"user data"
|
||||
);
|
||||
#[cfg(unix)]
|
||||
assert_ne!(
|
||||
fs::metadata(install_dir.join(core_binary_name()))
|
||||
.unwrap()
|
||||
.ino(),
|
||||
original_binary_inode,
|
||||
"更新后的内核必须使用新 inode"
|
||||
);
|
||||
assert!(!staging_dir.exists());
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
@@ -537,3 +552,84 @@ fn release_page_assets_parse_download_links_and_sha256() {
|
||||
.browser_download_url
|
||||
.ends_with("/releases/download/v1.2.3/CLIProxyAPI_1.2.3_linux_amd64.tar.gz"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rematerializing_core_binary_preserves_bytes_and_cleans_up_temporary_file() {
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
|
||||
let root = agent_test_home("core-rematerialize");
|
||||
fs::create_dir_all(&root).unwrap();
|
||||
let binary_path = root.join(core_binary_name());
|
||||
fs::write(&binary_path, b"signed core bytes").unwrap();
|
||||
#[cfg(unix)]
|
||||
let original_inode = fs::metadata(&binary_path).unwrap().ino();
|
||||
|
||||
rematerialize_core_binary(&binary_path).unwrap();
|
||||
|
||||
assert_eq!(fs::read(&binary_path).unwrap(), b"signed core bytes");
|
||||
#[cfg(unix)]
|
||||
assert_ne!(fs::metadata(&binary_path).unwrap().ino(), original_inode);
|
||||
assert_eq!(fs::read_dir(&root).unwrap().count(), 1);
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_start_log_path_follows_the_managed_logs_directory() {
|
||||
let base_dir = PathBuf::from("test-base");
|
||||
let install_dir = base_dir.join("cpa-core");
|
||||
|
||||
assert_eq!(
|
||||
core_start_log_path(&install_dir, DEFAULT_AUTH_DIR),
|
||||
base_dir
|
||||
.join("oauth")
|
||||
.join("logs")
|
||||
.join("core-start-output.log")
|
||||
);
|
||||
assert_eq!(
|
||||
core_start_log_path(&install_dir, "custom-auth"),
|
||||
install_dir
|
||||
.join("custom-auth")
|
||||
.join("logs")
|
||||
.join("core-start-output.log")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_start_output_helper() {
|
||||
if env::var_os("EASYCLIPROXYAPI_CORE_OUTPUT_TEST_HELPER").is_none() {
|
||||
return;
|
||||
}
|
||||
println!("core stdout marker");
|
||||
eprintln!("core stderr marker");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn core_start_log_captures_stdout_and_stderr() {
|
||||
let root = agent_test_home("core-start-output");
|
||||
let log_path = root.join("logs").join("core-start-output.log");
|
||||
fs::create_dir_all(log_path.parent().unwrap()).unwrap();
|
||||
fs::write(&log_path, "stale startup output").unwrap();
|
||||
let (stdout, stderr) = core_start_stdio(&log_path).unwrap();
|
||||
let mut command = Command::new(env::current_exe().unwrap());
|
||||
command
|
||||
.args([
|
||||
"--exact",
|
||||
"tests::core_runtime::core_start_output_helper",
|
||||
"--nocapture",
|
||||
])
|
||||
.env("EASYCLIPROXYAPI_CORE_OUTPUT_TEST_HELPER", "1")
|
||||
.stdin(Stdio::null())
|
||||
.stdout(stdout)
|
||||
.stderr(stderr);
|
||||
configure_background_command(&mut command);
|
||||
|
||||
assert!(command.status().unwrap().success());
|
||||
|
||||
let output = fs::read_to_string(&log_path).unwrap();
|
||||
assert!(output.contains("===== CPA 内核启动"));
|
||||
assert!(output.contains("core stdout marker"));
|
||||
assert!(output.contains("core stderr marker"));
|
||||
assert!(!output.contains("stale startup output"));
|
||||
fs::remove_dir_all(root).unwrap();
|
||||
}
|
||||
Reference in new issue
Block a user