From ceda386010fbf864e218defdde0d54f09b5d5978 Mon Sep 17 00:00:00 2001 From: imengying Date: Tue, 18 Aug 2026 00:44:00 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=A1=B9=E7=9B=AE=E9=80=BB?= =?UTF-8?q?=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yml | 43 +- Cross.toml | 2 +- README.md | 8 + agent/src/main.rs | 863 +++++++++++------ build.rs | 1 + frontend/src/App.tsx | 166 +++- frontend/src/api.ts | 3 +- frontend/src/components/AdminPanel.tsx | 6 +- frontend/src/types.ts | 7 + migrations/initial_01.sql | 179 ++-- scripts/build-agent.sh | 5 +- scripts/build-frontend.sh | 7 + scripts/build-key.sh | 5 +- scripts/build-worker.sh | 8 +- scripts/resolve-version.sh | 23 + scripts/smoke-test.sh | 73 +- src/auth.rs | 6 - src/cloudflare.rs | 6 +- src/db.rs | 1184 ++++++++++++++++-------- src/latency.rs | 365 ++++---- src/lib.rs | 116 ++- src/live.rs | 1168 ++++++++++++++++++++--- src/models.rs | 6 +- src/notify.rs | 36 +- src/routes/admin.rs | 9 +- src/routes/agent.rs | 89 +- src/routes/public.rs | 145 +-- src/routes/site.rs | 18 + wrangler.jsonc | 12 +- 29 files changed, 3306 insertions(+), 1253 deletions(-) create mode 100755 scripts/resolve-version.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d58696e..273243e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,8 @@ jobs: needs: metadata runs-on: ubuntu-latest timeout-minutes: 30 + env: + NODEFLARE_VERSION: ${{ needs.metadata.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -73,6 +75,42 @@ jobs: run: cargo test --locked --manifest-path agent/Cargo.toml - name: Build Worker run: NODEFLARE_ADMIN_READY=1 sh scripts/build-worker.sh + - name: Smoke test Worker and D1 + shell: bash + run: | + set -euo pipefail + state_dir="$RUNNER_TEMP/nodeflare-state" + log_file="$RUNNER_TEMP/nodeflare-dev.log" + export XDG_CONFIG_HOME="$RUNNER_TEMP/wrangler-config" + mkdir -p "$XDG_CONFIG_HOME" + + bunx wrangler d1 migrations apply DB --local --persist-to "$state_dir" + NODEFLARE_ADMIN_READY=1 NODEFLARE_WORKER_READY=1 \ + bunx wrangler dev --local --persist-to "$state_dir" --port 8787 \ + --var ADMIN_USERNAME:smoke --var ADMIN_PASSWORD:SmokePassword123 \ + >"$log_file" 2>&1 & + dev_pid=$! + trap 'kill "$dev_pid" 2>/dev/null || true' EXIT + + ready=0 + for _ in $(seq 1 30); do + if curl --fail --silent http://127.0.0.1:8787/api/bootstrap >/dev/null; then + ready=1 + break + fi + sleep 1 + done + if [ "$ready" != 1 ]; then + cat "$log_file" + exit 1 + fi + + if ! MONITOR_BASE_URL=http://127.0.0.1:8787 \ + MONITOR_ADMIN_USERNAME=smoke MONITOR_ADMIN_PASSWORD=SmokePassword123 \ + sh scripts/smoke-test.sh; then + cat "$log_file" + exit 1 + fi agent: name: Agent (${{ matrix.platform }}) @@ -85,7 +123,7 @@ jobs: id-token: write env: CARGO_TARGET_DIR: target/agent - NODEFLARE_AGENT_VERSION: ${{ needs.metadata.outputs.version }} + NODEFLARE_VERSION: ${{ needs.metadata.outputs.version }} strategy: fail-fast: false matrix: @@ -197,6 +235,9 @@ jobs: attestations: write contents: read id-token: write + env: + NODEFLARE_VERSION: ${{ needs.metadata.outputs.version }} + VITE_NODEFLARE_VERSION: ${{ needs.metadata.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/Cross.toml b/Cross.toml index e004cce..5b45f7b 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,5 +1,5 @@ [build.env] -passthrough = ["NODEFLARE_AGENT_VERSION"] +passthrough = ["NODEFLARE_VERSION"] [target.aarch64-unknown-linux-musl] runner = "qemu-user" diff --git a/README.md b/README.md index 502b24e..d849c37 100644 --- a/README.md +++ b/README.md @@ -41,10 +41,17 @@ | `CF_USAGE_ACCOUNT_ID` | 可选 | Cloudflare 用量查询的账户 ID，不设置则不启用用量查询 | | `CF_USAGE_API_TOKEN` | 可选 | 用量查询 Token，勾选“加密”；需要 Account Analytics: Read，并授权对应账户 | +## Agent 卸载 + +```sh +curl -fsSL https://raw.githubusercontent.com/imengying/NodeFlare/main/agent/agent.sh | sudo sh -s -- --uninstall +``` + ## API | 方法 | 路径 | 用途 | | --- | --- | --- | +| `GET` | `/api/bootstrap` | 前台配置、节点、汇率和访问状态 | | `GET` | `/api/config` | 公开站点配置 | | `GET` | `/api/exchange-rates` | D1 中的 CNY 基准汇率快照 | | `GET` | `/api/servers` | 公开节点和最新指标 | @@ -52,6 +59,7 @@ | `GET` | `/api/latency/:id?hours=24` | 节点延迟任务历史 | | `GET` | `/api/ws` | 实时 WebSocket | | `GET` | `/api/agent/live` | Agent 实时指标 WebSocket | +| `GET` | `/api/agent/config` | Agent 配置同步，节点 Bearer Token | | `POST` | `/api/agent/report` | Agent 上报，节点 Bearer Token | | `POST` | `/api/admin/login` | 管理登录 | | `POST` | `/api/admin/logout` | 退出登录（清除会话 Cookie） | diff --git a/agent/src/main.rs b/agent/src/main.rs index fb7b6a5..0dcff3d 100644 --- a/agent/src/main.rs +++ b/agent/src/main.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::env; use std::fs; use std::io::{self, ErrorKind, Read, Write}; @@ -11,6 +11,7 @@ use std::path::Path; use std::process::Command; #[cfg(target_os = "windows")] use std::process::Stdio; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{mpsc, Arc, Condvar, Mutex}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -19,14 +20,14 @@ use clap::Parser; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; #[cfg(any(target_os = "windows", target_os = "macos"))] -use sysinfo::{Disks, Networks, System}; +use sysinfo::{Disks, Networks, ProcessesToUpdate, System}; use tungstenite::client::IntoClientRequest; use tungstenite::{connect, Error as WebSocketError, Message}; #[cfg(not(any(target_os = "linux", target_os = "windows", target_os = "macos")))] compile_error!("nodeflare-agent supports Linux, Windows, and macOS"); -const VERSION: &str = match option_env!("NODEFLARE_AGENT_VERSION") { +const VERSION: &str = match option_env!("NODEFLARE_VERSION") { Some(version) if !version.is_empty() => version, _ => env!("CARGO_PKG_VERSION"), }; @@ -50,6 +51,9 @@ const LIVE_BATCH_CAPACITY: usize = 32; // D1 persistence can take a few hundred milliseconds; allow the ACK to arrive // before continuing with the configured live interval. const LIVE_ACK_READ_TIMEOUT: Duration = Duration::from_millis(1_000); +const LIVE_HINT_READ_TIMEOUT: Duration = Duration::from_millis(10); +const LIVE_REPORT_DIVISOR: u64 = 15; +const BASIC_INFO_REFRESH_INTERVAL: Duration = Duration::from_secs(5 * 60); const CLOCK_CALIBRATION_MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60); const CLOCK_CALIBRATION_MIN_CHANGE_MS: i64 = 20_000; @@ -228,28 +232,40 @@ impl ClockCalibration { type SharedClock = Arc>; struct LiveSender { - pending: Arc<(Mutex>, Condvar)>, + pending: Arc<(Mutex>, Condvar)>, send_interval: Arc>, + healthy: Arc, } impl LiveSender { fn start(config: &RuntimeConfig, clock: SharedClock) -> Result { - let pending = Arc::new((Mutex::new(Vec::new()), Condvar::new())); - let send_interval = Arc::new(Mutex::new(Duration::from_secs( - config.collect_interval.clamp(2, 60), + let pending = Arc::new((Mutex::new(VecDeque::new()), Condvar::new())); + let send_interval = Arc::new(Mutex::new(live_batch_interval( + config.report_interval, + config.collect_interval, ))); + let healthy = Arc::new(AtomicBool::new(false)); let endpoint = live_endpoint(&config.endpoint)?; let token = config.token.clone(); let sender_state = Arc::clone(&pending); let sender_interval = Arc::clone(&send_interval); + let sender_healthy = Arc::clone(&healthy); thread::Builder::new() .name("nodeflare-live".to_string()) .spawn(move || { - live_sender_loop(&endpoint, &token, sender_state, sender_interval, clock) + live_sender_loop( + &endpoint, + &token, + sender_state, + sender_interval, + sender_healthy, + clock, + ) })?; Ok(Self { pending, send_interval, + healthy, }) } @@ -257,19 +273,32 @@ impl LiveSender { let (pending, ready) = &*self.pending; if let Ok(mut pending) = pending.lock() { if pending.len() >= LIVE_QUEUE_CAPACITY { - pending.remove(0); + pending.pop_front(); } - pending.push(report.clone()); + pending.push_back(report.clone()); ready.notify_one(); } } - fn set_send_interval(&self, seconds: u64) { + fn set_send_interval(&self, report_interval: u64, collect_interval: u64) { if let Ok(mut interval) = self.send_interval.lock() { - *interval = Duration::from_secs(seconds.clamp(2, 60)); + *interval = live_batch_interval(report_interval, collect_interval); } self.pending.1.notify_one(); } + + fn is_healthy(&self) -> bool { + self.healthy.load(Ordering::Acquire) + } +} + +fn live_batch_interval(report_interval: u64, collect_interval: u64) -> Duration { + let seconds = report_interval + .clamp(15, 3600) + .div_ceil(LIVE_REPORT_DIVISOR) + .clamp(1, 60) + .max(collect_interval.clamp(1, 60)); + Duration::from_secs(seconds) } #[derive(Debug, Serialize)] @@ -340,6 +369,19 @@ struct Report { latency_results: Vec, } +#[derive(Debug, Default, Clone)] +struct BasicMetrics { + cpu_cores: i64, + cpu_model: String, + os: String, + kernel: String, + arch: String, + virtualization: String, + gpu_usage: f64, + gpu_model: String, + gpus: Vec, +} + #[cfg(target_os = "linux")] #[derive(Debug, Default, Clone, Copy)] struct CpuSample { @@ -1022,220 +1064,357 @@ fn u64_to_i64(value: u64) -> i64 { value.min(i64::MAX as u64) as i64 } +fn per_second(value: u64, elapsed_seconds: f64) -> f64 { + value as f64 / elapsed_seconds.max(0.001) +} + +fn valid_sample_schedule(report_interval: u64, collect_interval: u64) -> bool { + (15..=3600).contains(&report_interval) + && (1..=60).contains(&collect_interval) + && collect_interval <= report_interval + && report_interval.div_ceil(collect_interval) <= LIVE_QUEUE_CAPACITY as u64 +} + +fn advance_deadline(mut deadline: Instant, interval: Duration, now: Instant) -> Instant { + while deadline <= now { + deadline += interval; + } + deadline +} + #[cfg(target_os = "linux")] -fn collect(config: &RuntimeConfig, latency_results: Vec, timestamp: i64) -> Report { - let cpu_before = cpu_sample(); - let io_before = io_sample(&config.network_interface); - thread::sleep(Duration::from_secs(1)); - let cpu_after = cpu_sample(); - let io_after = io_sample(&config.network_interface); - let total = cpu_after.total.saturating_sub(cpu_before.total); - let idle = cpu_after.idle.saturating_sub(cpu_before.idle); - let cpu = if total == 0 { - 0.0 - } else { - (total.saturating_sub(idle)) as f64 * 100.0 / total as f64 - }; +struct Collector { + previous_cpu: CpuSample, + previous_io: IoSample, + previous_at: Instant, + network_interface: String, + basic: BasicMetrics, + basic_at: Instant, +} - let mem = text("/proc/meminfo"); - let mem_total = mem_value(&mem, "MemTotal:"); - let mem_used = mem_total.saturating_sub(mem_value(&mem, "MemAvailable:")); - let swap_total = mem_value(&mem, "SwapTotal:"); - let swap_used = swap_total.saturating_sub(mem_value(&mem, "SwapFree:")); - let loads = text("/proc/loadavg") - .split_whitespace() - .take(3) - .filter_map(|value| value.parse::().ok()) - .collect::>(); - let uptime = text("/proc/uptime") - .split_whitespace() - .next() - .and_then(|value| value.parse::().ok()) - .unwrap_or(0.0) as i64; - let processes = fs::read_dir("/proc") - .map(|items| { - items - .filter_map(|item| item.ok()) - .filter(|item| { - item.file_name() - .to_string_lossy() - .chars() - .all(|ch| ch.is_ascii_digit()) - }) - .count() as i64 - }) - .unwrap_or(0); - let disks = disk_usage(); - let disk_used = disks.iter().map(|disk| disk.used).sum(); - let disk_total = disks.iter().map(|disk| disk.total).sum(); - let gpus = gpu_info(); - let gpu_usage = average_gpu_usage(&gpus); - let gpu_model = gpus - .iter() - .map(|gpu| gpu.model.as_str()) - .collect::>() - .join(" · "); - let read_ops = io_after.read_ops.saturating_sub(io_before.read_ops); - let write_ops = io_after.write_ops.saturating_sub(io_before.write_ops); - let total_ops = read_ops.saturating_add(write_ops); - let io_wait = io_after - .read_millis - .saturating_sub(io_before.read_millis) - .saturating_add(io_after.write_millis.saturating_sub(io_before.write_millis)); +#[cfg(target_os = "linux")] +impl Collector { + fn new(config: &RuntimeConfig) -> Self { + let mut collector = Self { + previous_cpu: cpu_sample(), + previous_io: io_sample(&config.network_interface), + previous_at: Instant::now(), + network_interface: config.network_interface.clone(), + basic: BasicMetrics::default(), + basic_at: Instant::now(), + }; + collector.refresh_basic(); + collector + } - Report { - timestamp, - cpu, - load1: loads.first().copied().unwrap_or(0.0), - load5: loads.get(1).copied().unwrap_or(0.0), - load15: loads.get(2).copied().unwrap_or(0.0), - mem_used, - mem_total, - swap_used, - swap_total, - disk_used, - disk_total, - net_in: io_after.rx.saturating_sub(io_before.rx) as f64, - net_out: io_after.tx.saturating_sub(io_before.tx) as f64, - net_rx_total: io_after.rx.min(i64::MAX as u64) as i64, - net_tx_total: io_after.tx.min(i64::MAX as u64) as i64, - uptime, - processes, - tcp_connections: file_line_count("/proc/net/tcp") + file_line_count("/proc/net/tcp6"), - udp_connections: file_line_count("/proc/net/udp") + file_line_count("/proc/net/udp6"), - cpu_cores: thread::available_parallelism() - .map(|value| value.get() as i64) - .unwrap_or(1), - cpu_model: cpu_model(), - os: os_name(), - kernel: command("uname", &["-r"]), - arch: env::consts::ARCH.to_string(), - virtualization: command("systemd-detect-virt", &[]), - gpu_usage, - gpu_model, - agent_version: VERSION.to_string(), - disk_read_bps: io_after - .read_sectors - .saturating_sub(io_before.read_sectors) - .saturating_mul(512) as f64, - disk_write_bps: io_after - .write_sectors - .saturating_sub(io_before.write_sectors) - .saturating_mul(512) as f64, - disk_read_iops: io_after.read_ops.saturating_sub(io_before.read_ops) as f64, - disk_write_iops: io_after.write_ops.saturating_sub(io_before.write_ops) as f64, - disk_await_ms: if total_ops == 0 { + fn refresh_basic(&mut self) { + let gpus = gpu_info(); + self.basic = BasicMetrics { + cpu_cores: thread::available_parallelism() + .map(|value| value.get() as i64) + .unwrap_or(1), + cpu_model: cpu_model(), + os: os_name(), + kernel: command("uname", &["-r"]), + arch: env::consts::ARCH.to_string(), + virtualization: command("systemd-detect-virt", &[]), + gpu_usage: average_gpu_usage(&gpus), + gpu_model: gpus + .iter() + .map(|gpu| gpu.model.as_str()) + .collect::>() + .join(" · "), + gpus, + }; + self.basic_at = Instant::now(); + } + + fn collect( + &mut self, + config: &RuntimeConfig, + latency_results: Vec, + timestamp: i64, + ) -> Report { + let sampled_at = Instant::now(); + let elapsed = sampled_at + .saturating_duration_since(self.previous_at) + .as_secs_f64() + .max(0.001); + let cpu_now = cpu_sample(); + let io_now = io_sample(&config.network_interface); + let total = cpu_now.total.saturating_sub(self.previous_cpu.total); + let idle = cpu_now.idle.saturating_sub(self.previous_cpu.idle); + let cpu = if total == 0 { 0.0 } else { - io_wait as f64 / total_ops as f64 - }, - disk_utilization: (io_after.io_millis.saturating_sub(io_before.io_millis) as f64 / 10.0) - .clamp(0.0, 100.0), - disks, - gpus, - latency_results, + (total.saturating_sub(idle)) as f64 * 100.0 / total as f64 + }; + let interface_changed = self.network_interface != config.network_interface; + let io_before = if interface_changed { + io_now + } else { + self.previous_io + }; + self.previous_cpu = cpu_now; + self.previous_io = io_now; + self.previous_at = sampled_at; + self.network_interface.clone_from(&config.network_interface); + + if self.basic_at.elapsed() >= BASIC_INFO_REFRESH_INTERVAL { + self.refresh_basic(); + } + + let mem = text("/proc/meminfo"); + let mem_total = mem_value(&mem, "MemTotal:"); + let mem_used = mem_total.saturating_sub(mem_value(&mem, "MemAvailable:")); + let swap_total = mem_value(&mem, "SwapTotal:"); + let swap_used = swap_total.saturating_sub(mem_value(&mem, "SwapFree:")); + let loads = text("/proc/loadavg") + .split_whitespace() + .take(3) + .filter_map(|value| value.parse::().ok()) + .collect::>(); + let uptime = text("/proc/uptime") + .split_whitespace() + .next() + .and_then(|value| value.parse::().ok()) + .unwrap_or(0.0) as i64; + let processes = fs::read_dir("/proc") + .map(|items| { + items + .filter_map(|item| item.ok()) + .filter(|item| { + item.file_name() + .to_string_lossy() + .chars() + .all(|ch| ch.is_ascii_digit()) + }) + .count() as i64 + }) + .unwrap_or(0); + let disks = disk_usage(); + let disk_used = disks.iter().map(|disk| disk.used).sum(); + let disk_total = disks.iter().map(|disk| disk.total).sum(); + let tcp_connections = file_line_count("/proc/net/tcp") + file_line_count("/proc/net/tcp6"); + let udp_connections = file_line_count("/proc/net/udp") + file_line_count("/proc/net/udp6"); + let read_ops = io_now.read_ops.saturating_sub(io_before.read_ops); + let write_ops = io_now.write_ops.saturating_sub(io_before.write_ops); + let total_ops = read_ops.saturating_add(write_ops); + let io_wait = io_now + .read_millis + .saturating_sub(io_before.read_millis) + .saturating_add(io_now.write_millis.saturating_sub(io_before.write_millis)); + let elapsed_ms = elapsed * 1000.0; + + Report { + timestamp, + cpu, + load1: loads.first().copied().unwrap_or(0.0), + load5: loads.get(1).copied().unwrap_or(0.0), + load15: loads.get(2).copied().unwrap_or(0.0), + mem_used, + mem_total, + swap_used, + swap_total, + disk_used, + disk_total, + net_in: per_second(io_now.rx.saturating_sub(io_before.rx), elapsed), + net_out: per_second(io_now.tx.saturating_sub(io_before.tx), elapsed), + net_rx_total: io_now.rx.min(i64::MAX as u64) as i64, + net_tx_total: io_now.tx.min(i64::MAX as u64) as i64, + uptime, + processes, + tcp_connections, + udp_connections, + cpu_cores: self.basic.cpu_cores, + cpu_model: self.basic.cpu_model.clone(), + os: self.basic.os.clone(), + kernel: self.basic.kernel.clone(), + arch: self.basic.arch.clone(), + virtualization: self.basic.virtualization.clone(), + gpu_usage: self.basic.gpu_usage, + gpu_model: self.basic.gpu_model.clone(), + agent_version: VERSION.to_string(), + disk_read_bps: per_second( + io_now + .read_sectors + .saturating_sub(io_before.read_sectors) + .saturating_mul(512), + elapsed, + ), + disk_write_bps: per_second( + io_now + .write_sectors + .saturating_sub(io_before.write_sectors) + .saturating_mul(512), + elapsed, + ), + disk_read_iops: per_second(read_ops, elapsed), + disk_write_iops: per_second(write_ops, elapsed), + disk_await_ms: if total_ops == 0 { + 0.0 + } else { + io_wait as f64 / total_ops as f64 + }, + disk_utilization: if elapsed_ms <= 0.0 { + 0.0 + } else { + (io_now.io_millis.saturating_sub(io_before.io_millis) as f64 * 100.0 / elapsed_ms) + .clamp(0.0, 100.0) + }, + disks, + gpus: self.basic.gpus.clone(), + latency_results, + } } } #[cfg(any(target_os = "windows", target_os = "macos"))] -fn collect(config: &RuntimeConfig, latency_results: Vec, timestamp: i64) -> Report { - let mut system = System::new_all(); - let mut networks = Networks::new_with_refreshed_list(); - thread::sleep(Duration::from_secs(1)); - system.refresh_all(); - networks.refresh(true); +struct Collector { + system: System, + networks: Networks, + previous_at: Instant, + basic: BasicMetrics, + basic_at: Instant, +} - let mut net_in = 0_u64; - let mut net_out = 0_u64; - let mut net_rx_total = 0_u64; - let mut net_tx_total = 0_u64; - for (name, network) in &networks { - if !selected_interface(name, &config.network_interface) { - continue; - } - net_in = net_in.saturating_add(network.received()); - net_out = net_out.saturating_add(network.transmitted()); - net_rx_total = net_rx_total.saturating_add(network.total_received()); - net_tx_total = net_tx_total.saturating_add(network.total_transmitted()); +#[cfg(any(target_os = "windows", target_os = "macos"))] +impl Collector { + fn new(_config: &RuntimeConfig) -> Self { + let mut collector = Self { + system: System::new_all(), + networks: Networks::new_with_refreshed_list(), + previous_at: Instant::now(), + basic: BasicMetrics::default(), + basic_at: Instant::now(), + }; + collector.refresh_basic(); + collector } - let disks = Disks::new_with_refreshed_list(); - #[cfg(target_os = "windows")] - let root = format!( - "{}\\", - env::var("SystemDrive").unwrap_or_else(|_| "C:".to_string()) - ); - #[cfg(target_os = "macos")] - let root = "/".to_string(); - let mut disk_metrics = disks - .iter() - .filter(|disk| !disk.is_removable()) - .map(|disk| { - let total = disk.total_space(); - DiskMetric { - name: disk.name().to_string_lossy().to_string(), - mount_point: disk.mount_point().to_string_lossy().to_string(), - used: u64_to_i64(total.saturating_sub(disk.available_space())), - total: u64_to_i64(total), - ..DiskMetric::default() - } - }) - .collect::>(); - disk_metrics.sort_by_key(|disk| (disk.mount_point != root, disk.mount_point.clone())); - let disk_used = disk_metrics.iter().map(|disk| disk.used).sum(); - let disk_total = disk_metrics.iter().map(|disk| disk.total).sum(); - let load = System::load_average(); - let gpus = gpu_info(); - let gpu_usage = average_gpu_usage(&gpus); - let gpu_model = gpus - .iter() - .map(|gpu| gpu.model.as_str()) - .collect::>() - .join(" · "); - let (tcp_connections, udp_connections) = connection_counts(); + fn refresh_basic(&mut self) { + let gpus = gpu_info(); + self.basic = BasicMetrics { + cpu_cores: self.system.cpus().len().max(1) as i64, + cpu_model: self + .system + .cpus() + .first() + .map(|cpu| cpu.brand().to_string()) + .unwrap_or_default(), + os: System::long_os_version().unwrap_or_else(|| env::consts::OS.to_string()), + kernel: System::kernel_version().unwrap_or_default(), + arch: env::consts::ARCH.to_string(), + virtualization: String::new(), + gpu_usage: average_gpu_usage(&gpus), + gpu_model: gpus + .iter() + .map(|gpu| gpu.model.as_str()) + .collect::>() + .join(" · "), + gpus, + }; + self.basic_at = Instant::now(); + } - Report { - timestamp, - cpu: system.global_cpu_usage() as f64, - load1: load.one, - load5: load.five, - load15: load.fifteen, - mem_used: u64_to_i64(system.used_memory()), - mem_total: u64_to_i64(system.total_memory()), - swap_used: u64_to_i64(system.used_swap()), - swap_total: u64_to_i64(system.total_swap()), - disk_used, - disk_total, - net_in: net_in as f64, - net_out: net_out as f64, - net_rx_total: u64_to_i64(net_rx_total), - net_tx_total: u64_to_i64(net_tx_total), - uptime: u64_to_i64(System::uptime()), - processes: system.processes().len() as i64, - tcp_connections, - udp_connections, - cpu_cores: system.cpus().len().max(1) as i64, - cpu_model: system - .cpus() - .first() - .map(|cpu| cpu.brand().to_string()) - .unwrap_or_default(), - os: System::long_os_version().unwrap_or_else(|| env::consts::OS.to_string()), - kernel: System::kernel_version().unwrap_or_default(), - arch: env::consts::ARCH.to_string(), - virtualization: String::new(), - gpu_usage, - gpu_model, - agent_version: VERSION.to_string(), - disk_read_bps: 0.0, - disk_write_bps: 0.0, - disk_read_iops: 0.0, - disk_write_iops: 0.0, - disk_await_ms: 0.0, - disk_utilization: 0.0, - disks: disk_metrics, - gpus, - latency_results, + fn collect( + &mut self, + config: &RuntimeConfig, + latency_results: Vec, + timestamp: i64, + ) -> Report { + let sampled_at = Instant::now(); + let elapsed = sampled_at + .saturating_duration_since(self.previous_at) + .as_secs_f64() + .max(0.001); + self.system.refresh_cpu_usage(); + self.system.refresh_memory(); + self.system.refresh_processes(ProcessesToUpdate::All, true); + self.networks.refresh(true); + self.previous_at = sampled_at; + if self.basic_at.elapsed() >= BASIC_INFO_REFRESH_INTERVAL { + self.refresh_basic(); + } + + let disks = Disks::new_with_refreshed_list(); + #[cfg(target_os = "windows")] + let root = format!( + "{}\\", + env::var("SystemDrive").unwrap_or_else(|_| "C:".to_string()) + ); + #[cfg(target_os = "macos")] + let root = "/".to_string(); + let mut disk_metrics = disks + .iter() + .filter(|disk| !disk.is_removable()) + .map(|disk| { + let total = disk.total_space(); + DiskMetric { + name: disk.name().to_string_lossy().to_string(), + mount_point: disk.mount_point().to_string_lossy().to_string(), + used: u64_to_i64(total.saturating_sub(disk.available_space())), + total: u64_to_i64(total), + ..DiskMetric::default() + } + }) + .collect::>(); + disk_metrics.sort_by_key(|disk| (disk.mount_point != root, disk.mount_point.clone())); + let (tcp_connections, udp_connections) = connection_counts(); + + let mut net_in = 0_u64; + let mut net_out = 0_u64; + let mut net_rx_total = 0_u64; + let mut net_tx_total = 0_u64; + for (name, network) in &self.networks { + if !selected_interface(name, &config.network_interface) { + continue; + } + net_in = net_in.saturating_add(network.received()); + net_out = net_out.saturating_add(network.transmitted()); + net_rx_total = net_rx_total.saturating_add(network.total_received()); + net_tx_total = net_tx_total.saturating_add(network.total_transmitted()); + } + let load = System::load_average(); + Report { + timestamp, + cpu: self.system.global_cpu_usage() as f64, + load1: load.one, + load5: load.five, + load15: load.fifteen, + mem_used: u64_to_i64(self.system.used_memory()), + mem_total: u64_to_i64(self.system.total_memory()), + swap_used: u64_to_i64(self.system.used_swap()), + swap_total: u64_to_i64(self.system.total_swap()), + disk_used: disk_metrics.iter().map(|disk| disk.used).sum(), + disk_total: disk_metrics.iter().map(|disk| disk.total).sum(), + net_in: per_second(net_in, elapsed), + net_out: per_second(net_out, elapsed), + net_rx_total: u64_to_i64(net_rx_total), + net_tx_total: u64_to_i64(net_tx_total), + uptime: u64_to_i64(System::uptime()), + processes: self.system.processes().len() as i64, + tcp_connections, + udp_connections, + cpu_cores: self.basic.cpu_cores, + cpu_model: self.basic.cpu_model.clone(), + os: self.basic.os.clone(), + kernel: self.basic.kernel.clone(), + arch: self.basic.arch.clone(), + virtualization: self.basic.virtualization.clone(), + gpu_usage: self.basic.gpu_usage, + gpu_model: self.basic.gpu_model.clone(), + agent_version: VERSION.to_string(), + disk_read_bps: 0.0, + disk_write_bps: 0.0, + disk_read_iops: 0.0, + disk_write_iops: 0.0, + disk_await_ms: 0.0, + disk_utilization: 0.0, + disks: disk_metrics, + gpus: self.basic.gpus.clone(), + latency_results, + } } } @@ -1258,7 +1437,7 @@ fn runtime_config(options: &CliOptions) -> Result { token, endpoint: endpoint.trim_end_matches('/').to_string(), report_interval: interval, - collect_interval: 5, + collect_interval: 1, network_interface: String::new(), agent_mirror: String::new(), auto_update: true, @@ -1311,6 +1490,49 @@ fn submit( }) } +fn fetch_remote_config( + agent: &ureq::Agent, + config: &RuntimeConfig, + config_hash: &str, +) -> Result { + let started_ms = unix_timestamp_millis(); + let mut response = agent + .get(&format!("{}/api/agent/config", config.endpoint)) + .header("Authorization", format!("Bearer {}", config.token)) + .header("User-Agent", format!("nodeflare-agent/{VERSION}")) + .header("X-Agent-Config-Sha256", config_hash) + .call()?; + let ended_ms = unix_timestamp_millis(); + let clock_offset_ms = response + .headers() + .get("x-nodeflare-server-time") + .and_then(|value| value.to_str().ok()) + .and_then(|value| clock_offset_from_server_seconds(value, started_ms, ended_ms)) + .or_else(|| { + response + .headers() + .get("date") + .and_then(|value| value.to_str().ok()) + .and_then(|value| clock_offset_from_http_date(value, started_ms, ended_ms)) + }); + let next_hash = response + .headers() + .get("X-Agent-Config-Sha256") + .and_then(|value| value.to_str().ok()) + .unwrap_or(config_hash) + .to_string(); + let remote = if response.status().as_u16() == 204 { + None + } else { + Some(response.body_mut().read_json::()?) + }; + Ok(SubmitResult { + config: remote, + config_hash: next_hash, + clock_offset_ms, + }) +} + fn fetch_clock_offset(agent: &ureq::Agent, endpoint: &str) -> Result> { let started_ms = unix_timestamp_millis(); let response = agent @@ -1383,11 +1605,13 @@ struct LiveAck { next_d1_write_after_ms: u64, #[serde(rename = "nextWssReportAfterMs")] next_wss_report_after_ms: u64, + #[serde(rename = "realtimeHint")] + realtime_hint: bool, } fn ack_interval(ack: &LiveAck) -> Duration { Duration::from_millis(ack.next_wss_report_after_ms) - .clamp(Duration::from_secs(2), Duration::from_secs(60)) + .clamp(Duration::from_secs(1), Duration::from_secs(60)) } fn read_live_ack(socket: &mut LiveSocket) -> Result> { @@ -1399,7 +1623,7 @@ fn read_live_ack(socket: &mut LiveSocket) -> Result> { if ack.message_type != "ack" || ack.ts <= 0 { return Ok(None); } - let _ = (ack.persisted, ack.next_d1_write_after_ms); + let _ = (ack.persisted, ack.next_d1_write_after_ms, ack.realtime_hint); Ok(Some(ack_interval(&ack))) } Ok(Message::Ping(payload)) => { @@ -1424,18 +1648,18 @@ fn live_update_payload(reports: &[Report]) -> Result { }))?) } -fn requeue_reports(pending: &Arc<(Mutex>, Condvar)>, reports: Vec) { +fn requeue_reports(pending: &Arc<(Mutex>, Condvar)>, reports: Vec) { if reports.is_empty() { return; } let (queue, ready) = &**pending; if let Ok(mut queue) = queue.lock() { - let mut merged = reports; - merged.append(&mut queue); - if merged.len() > LIVE_QUEUE_CAPACITY { - merged.truncate(LIVE_QUEUE_CAPACITY); + for report in reports.into_iter().rev() { + queue.push_front(report); + } + while queue.len() > LIVE_QUEUE_CAPACITY { + queue.pop_back(); } - *queue = merged; ready.notify_one(); } } @@ -1443,19 +1667,20 @@ fn requeue_reports(pending: &Arc<(Mutex>, Condvar)>, reports: Vec>, Condvar)>, + pending: Arc<(Mutex>, Condvar)>, configured_interval: Arc>, + healthy: Arc, clock: SharedClock, ) { let mut socket: Option = None; let mut send_interval = configured_interval .lock() .map(|interval| *interval) - .unwrap_or(Duration::from_secs(5)); - let mut next_send_at = Instant::now(); + .unwrap_or(Duration::from_secs(1)); + let mut next_send_at = Instant::now() + send_interval; loop { - let (queue, ready) = &*pending; - let mut queue = match queue.lock() { + let (queue_lock, ready) = &*pending; + let mut queue = match queue_lock.lock() { Ok(queue) => queue, Err(_) => return, }; @@ -1471,13 +1696,44 @@ fn live_sender_loop( Ok((queue, _)) => queue, Err(_) => return, }; + drop(queue); + + let mut drop_socket = false; + if let Some(connected) = socket.as_mut() { + if set_live_read_timeout(connected, Some(LIVE_HINT_READ_TIMEOUT)).is_err() { + drop_socket = true; + } else { + match read_live_ack(connected) { + Ok(Some(Duration::ZERO)) => drop_socket = true, + Ok(Some(interval)) => { + healthy.store(true, Ordering::Release); + send_interval = interval; + next_send_at = Instant::now() + send_interval; + } + Ok(None) => {} + Err(error) => { + eprintln!("live hint read failed: {error}"); + drop_socket = true; + } + } + } + } + if drop_socket { + socket = None; + healthy.store(false, Ordering::Release); + next_send_at = Instant::now(); + } + queue = match queue_lock.lock() { + Ok(queue) => queue, + Err(_) => return, + }; } let count = queue.len().min(LIVE_BATCH_CAPACITY); let batch = queue.drain(..count).collect::>(); drop(queue); if let Ok(interval) = configured_interval.lock() { - send_interval = (*interval).clamp(Duration::from_secs(2), Duration::from_secs(60)); + send_interval = (*interval).clamp(Duration::from_secs(1), Duration::from_secs(60)); } if socket.is_none() { match connect_live(endpoint, token) { @@ -1485,6 +1741,7 @@ fn live_sender_loop( observe_clock(&clock, offset_ms); if set_live_read_timeout(&mut connected, Some(LIVE_ACK_READ_TIMEOUT)).is_err() { socket = None; + healthy.store(false, Ordering::Release); thread::sleep(LIVE_RECONNECT_DELAY); requeue_reports(&pending, batch); continue; @@ -1492,6 +1749,7 @@ fn live_sender_loop( socket = Some(connected); } Err(error) => { + healthy.store(false, Ordering::Release); eprintln!("live connection failed: {error}"); requeue_reports(&pending, batch); thread::sleep(LIVE_RECONNECT_DELAY); @@ -1513,6 +1771,7 @@ fn live_sender_loop( .is_some_and(|socket| socket.send(Message::Text(payload.clone().into())).is_ok()); if !sent { socket = None; + healthy.store(false, Ordering::Release); thread::sleep(LIVE_RECONNECT_DELAY); let mut resent = false; if let Ok((mut connected, offset_ms)) = connect_live(endpoint, token) { @@ -1533,31 +1792,35 @@ fn live_sender_loop( next_send_at = Instant::now() + send_interval; let mut drop_socket = false; if let Some(socket) = socket.as_mut() { - match read_live_ack(socket) { - Ok(Some(Duration::ZERO)) => drop_socket = true, - Ok(Some(interval)) => { - send_interval = interval; - next_send_at = Instant::now() + send_interval; - } - Ok(None) => {} - Err(error) => { - eprintln!("live ACK read failed: {error}"); - drop_socket = true; + if set_live_read_timeout(socket, Some(LIVE_ACK_READ_TIMEOUT)).is_err() { + drop_socket = true; + } else { + match read_live_ack(socket) { + Ok(Some(Duration::ZERO)) => drop_socket = true, + Ok(Some(interval)) => { + healthy.store(true, Ordering::Release); + send_interval = interval; + next_send_at = Instant::now() + send_interval; + } + Ok(None) => {} + Err(error) => { + eprintln!("live ACK read failed: {error}"); + drop_socket = true; + } } } } if drop_socket { socket = None; + healthy.store(false, Ordering::Release); } } } fn apply_remote(config: &mut RuntimeConfig, remote: &RemoteConfig) -> bool { - if (15..=3600).contains(&remote.report_interval) { + if valid_sample_schedule(remote.report_interval, remote.collect_interval) { config.report_interval = remote.report_interval; - } - if (2..=60).contains(&remote.collect_interval) { - config.collect_interval = remote.collect_interval.min(config.report_interval); + config.collect_interval = remote.collect_interval; } config .network_interface @@ -1870,8 +2133,9 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { .timeout_global(Some(Duration::from_secs(20))) .build() .into(); + let mut collector = Collector::new(&config); let mut next_report = Instant::now(); - let mut next_collect = Instant::now(); + let mut next_collect = Instant::now() + Duration::from_secs(config.collect_interval); let mut next_latency: HashMap = HashMap::new(); let mut latency_executor = LatencyExecutor::new()?; let mut pending_results = Vec::new(); @@ -1879,6 +2143,7 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { let mut report_failures = 0_u32; let mut next_update_check = Instant::now(); let mut config_hash = String::new(); + let mut live_was_healthy = false; let clock = Arc::new(Mutex::new(ClockCalibration::default())); if !print_only { match fetch_clock_offset(&agent, &config.endpoint) { @@ -1890,6 +2155,11 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { .then(|| LiveSender::start(&config, Arc::clone(&clock))) .transpose()?; loop { + let live_healthy = live.as_ref().is_some_and(LiveSender::is_healthy); + if live_was_healthy && !live_healthy { + next_report = Instant::now(); + } + live_was_healthy = live_healthy; pending_results.extend(latency_executor.drain()); let current = Instant::now(); let due = config @@ -1920,7 +2190,7 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { result.timestamp = corrected_timestamp(result.timestamp, offset_ms); latest_results.insert(result.task_id.clone(), result); } - let mut report = collect(&config, latest_results.into_values().collect(), 0); + let mut report = collector.collect(&config, latest_results.into_values().collect(), 0); report.timestamp = corrected_timestamp(unix_timestamp(), offset_ms); if print_only { println!("{}", serde_json::to_string_pretty(&report)?); @@ -1934,26 +2204,51 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { &mut pending_samples, corrected_timestamp(unix_timestamp(), offset_ms), ); - if pending_samples.len() > 720 { - let overflow = pending_samples.len() - 720; + if pending_samples.len() > LIVE_QUEUE_CAPACITY { + let overflow = pending_samples.len() - LIVE_QUEUE_CAPACITY; pending_samples.drain(..overflow); } - next_collect = Instant::now() + Duration::from_secs(config.collect_interval); + next_collect = advance_deadline( + next_collect, + Duration::from_secs(config.collect_interval), + Instant::now(), + ); } if Instant::now() >= next_report && !pending_samples.is_empty() { - match submit(&agent, &config, &pending_samples, &config_hash) { - Ok(result) => { + let sync_result = if live_healthy { + fetch_remote_config(&agent, &config, &config_hash).map(|result| (result, false)) + } else { + submit(&agent, &config, &pending_samples, &config_hash).map(|result| (result, true)) + }; + let sync_failed = sync_result.is_err(); + match sync_result { + Ok((result, submitted_metrics)) => { observe_clock(&clock, result.clock_offset_ms); - pending_samples.clear(); + if submitted_metrics { + pending_samples.clear(); + } report_failures = 0; config_hash = result.config_hash; if let Some(remote) = result.config { + let previous_collect_interval = config.collect_interval; + let previous_report_interval = config.report_interval; if apply_remote(&mut config, &remote) { next_latency.clear(); } - if let Some(live) = &live { - live.set_send_interval(config.collect_interval); + if config.collect_interval != previous_collect_interval { + next_collect = + Instant::now() + Duration::from_secs(config.collect_interval); + } + if config.collect_interval != previous_collect_interval + || config.report_interval != previous_report_interval + { + if let Some(live) = &live { + live.set_send_interval( + config.report_interval, + config.collect_interval, + ); + } } } if !once && config.auto_update && Instant::now() >= next_update_check { @@ -1970,14 +2265,22 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { } } Err(error) => { - eprintln!("report failed: {error}"); - report_failures = report_failures.saturating_add(1); + if live_healthy { + eprintln!("config sync failed: {error}"); + } else { + eprintln!("report failed: {error}"); + report_failures = report_failures.saturating_add(1); + } if once { return Err(error); } } } - let retry_delay = report_retry_delay(report_failures, config.report_interval); + let retry_delay = if live_healthy && sync_failed { + REPORT_RETRY_MIN + } else { + report_retry_delay(report_failures, config.report_interval) + }; next_report = Instant::now() + retry_delay; if once { return Ok(()); @@ -2017,14 +2320,15 @@ mod tests { use std::time::{Duration, Instant}; use super::{ - ack_interval, agent_artifact_name, clock_offset_from_http_date, corrected_timestamp, - executable_format_valid, gpu_name_from_uevent, is_public_probe_ip, live_endpoint, - live_update_payload, median, mirrored_download_url, normalized_version, - parse_lspci_gpu_names, parse_probe_target, parse_system_profiler_gpu_names, ping_latency, - prune_report_samples, release_asset_sha256, report_retry_delay, sanitize_latency_tasks, - selected_interface, tcp_latency_probe_address, valid_endpoint, version_triplet, CliOptions, - ClockCalibration, GithubReleaseAsset, LatencyResult, LatencyTask, LiveAck, Report, - CLOCK_CALIBRATION_MAX_AGE, MAX_LATENCY_TASKS, MAX_PENDING_LATENCY_RESULTS, PROBE_ATTEMPTS, + ack_interval, advance_deadline, agent_artifact_name, clock_offset_from_http_date, + corrected_timestamp, executable_format_valid, gpu_name_from_uevent, is_public_probe_ip, + live_batch_interval, live_endpoint, live_update_payload, median, mirrored_download_url, + normalized_version, parse_lspci_gpu_names, parse_probe_target, + parse_system_profiler_gpu_names, ping_latency, prune_report_samples, release_asset_sha256, + report_retry_delay, sanitize_latency_tasks, selected_interface, tcp_latency_probe_address, + valid_endpoint, valid_sample_schedule, version_triplet, CliOptions, ClockCalibration, + GithubReleaseAsset, LatencyResult, LatencyTask, LiveAck, Report, CLOCK_CALIBRATION_MAX_AGE, + MAX_LATENCY_TASKS, MAX_PENDING_LATENCY_RESULTS, PROBE_ATTEMPTS, }; #[cfg(any(target_os = "windows", target_os = "macos"))] @@ -2147,15 +2451,16 @@ mod tests { #[test] fn accepts_server_realtime_ack_interval() { let ack: LiveAck = serde_json::from_str( - r#"{"type":"ack","ts":100,"persisted":false,"nextD1WriteAfterMs":60000,"nextWssReportAfterMs":5000}"#, + r#"{"type":"ack","ts":100,"persisted":false,"nextD1WriteAfterMs":60000,"nextWssReportAfterMs":5000,"realtimeHint":false}"#, ) .unwrap(); assert_eq!(ack_interval(&ack), Duration::from_secs(5)); let slow: LiveAck = serde_json::from_str( - r#"{"type":"ack","ts":100,"persisted":true,"nextD1WriteAfterMs":60000,"nextWssReportAfterMs":1}"#, + r#"{"type":"ack","ts":100,"persisted":true,"nextD1WriteAfterMs":60000,"nextWssReportAfterMs":1,"realtimeHint":true}"#, ) .unwrap(); - assert_eq!(ack_interval(&slow), Duration::from_secs(2)); + assert_eq!(ack_interval(&slow), Duration::from_secs(1)); + assert!(slow.realtime_hint); } #[test] @@ -2280,6 +2585,32 @@ mod tests { assert_eq!(report_retry_delay(20, 60), Duration::from_secs(300)); } + #[test] + fn validates_sample_schedules() { + assert!(valid_sample_schedule(3600, 5)); + assert!(!valid_sample_schedule(3600, 4)); + assert!(!valid_sample_schedule(60, 0)); + assert!(!valid_sample_schedule(10, 1)); + } + + #[test] + fn batches_live_samples_at_one_fifteenth_of_the_history_interval() { + assert_eq!(live_batch_interval(60, 1), Duration::from_secs(4)); + assert_eq!(live_batch_interval(60, 5), Duration::from_secs(5)); + assert_eq!(live_batch_interval(120, 1), Duration::from_secs(8)); + assert_eq!(live_batch_interval(3_600, 1), Duration::from_secs(60)); + } + + #[test] + fn advances_collection_deadlines_without_drift() { + let start = Instant::now(); + let interval = Duration::from_secs(1); + assert_eq!( + advance_deadline(start, interval, start + Duration::from_millis(2500)), + start + Duration::from_secs(3) + ); + } + #[test] fn drops_samples_the_worker_would_reject_as_expired() { let mut samples = vec![ diff --git a/build.rs b/build.rs index 49ba222..44e5e57 100644 --- a/build.rs +++ b/build.rs @@ -9,6 +9,7 @@ fn main() { println!("cargo:rerun-if-changed=frontend/admin-dist/admin.html"); println!("cargo:rerun-if-changed=frontend/admin-dist/admin.js"); println!("cargo:rerun-if-changed=frontend/admin-dist/admin.css"); + println!("cargo:rerun-if-env-changed=NODEFLARE_VERSION"); let files = [ ( diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 3dcf95a..9c132b8 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -12,10 +12,12 @@ import { derivePassword } from "./password"; const defaultConfig: Config = { ...demoConfig, site_description: "", site_name: "" }; const demoMode = import.meta.env.DEV && new URLSearchParams(window.location.search).has("demo"); +const MAX_PLAYBACK_SAMPLES_PER_SERVER = 600; const NodeDetails = lazy(() => import("./components/NodeDetails").then((module) => ({ default: module.NodeDetails }))); interface LiveMetrics { timestamp: number; + displayTimestamp: number; metrics: Partial; latencyResults?: LiveLatencyResult[]; } @@ -69,6 +71,7 @@ export default function App() { const [configReady, setConfigReady] = useState(demoMode); const [servers, setServers] = useState([]); const [liveMetrics, setLiveMetrics] = useState>({}); + const [clockNow, setClockNow] = useState(() => Date.now()); const [exchangeRates, setExchangeRates] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(""); @@ -90,18 +93,28 @@ export default function App() { const [systemDark, setSystemDark] = useState(() => matchMedia("(prefers-color-scheme: dark)").matches); const wsRef = useRef([]); const liveConnectedRef = useRef(false); + const playbackRef = useRef>(new Map()); + const serversRef = useRef([]); const dark = appearance ? appearance === "dark" : config.default_theme === "system" ? systemDark : config.default_theme === "dark"; const background = resolveBackground(config.background_url, dark); const blur = themeToggle(config, "enableBlur"); const liveServers = useMemo(() => servers.map((server) => { const live = liveMetrics[server.id]; - if (!live) return server; - const latency = live.latencyResults?.length ? mergeLiveLatency(server, live.latencyResults) : server.latency; - if (live.timestamp <= (server.timestamp ?? 0)) { - return latency === server.latency ? server : { ...server, latency }; - } - return { ...server, ...live.metrics, latency, timestamp: live.timestamp }; - }), [liveMetrics, servers]); + const merged = !live ? server : (() => { + const latency = live.latencyResults?.length ? mergeLiveLatency(server, live.latencyResults) : server.latency; + if (live.timestamp <= (server.timestamp ?? 0)) { + return latency === server.latency ? server : { ...server, latency }; + } + return { ...server, ...live.metrics, latency, timestamp: live.timestamp }; + })(); + // Keep the card clock moving between Agent samples, while preserving the + // actual sample timestamp for freshness/online checks. + if (!merged.timestamp || !merged.uptime || !Number.isFinite(merged.uptime)) return merged; + const elapsed = Math.max(0, Math.floor(clockNow / 1000 - merged.timestamp)); + return elapsed > 0 && elapsed <= config.offline_threshold_seconds + ? { ...merged, uptime: merged.uptime + elapsed } + : merged; + }), [clockNow, config.offline_threshold_seconds, liveMetrics, servers]); const load = useCallback(async (quiet = false) => { if (!quiet) setLoading(true); @@ -114,29 +127,32 @@ export default function App() { setLoading(false); return; } - let dashboardPublic = true; try { - const nextConfig = await api.config(); - dashboardPublic = nextConfig.public_dashboard; - setConfig(nextConfig); + const result = await api.bootstrap(); + setConfig(result.config); setConfigReady(true); - const [serverResult, ratesResult] = await Promise.all([api.servers(), api.exchangeRates()]); - setServers(serverResult.servers); - setExchangeRates(ratesResult); - setNeedsVerification(false); - setNeedsLogin(false); + setServers(result.servers); + setExchangeRates(result.exchange_rates); + setNeedsVerification(result.access === "turnstile"); + setNeedsLogin(result.access === "login"); + if (result.access !== "ok") { + playbackRef.current.clear(); + setLiveMetrics({}); + } setError(""); } catch (reason) { - if (reason instanceof ApiError && (reason.status === 401 || (!dashboardPublic && reason.status === 403))) { + if (reason instanceof ApiError && reason.status === 401) { setNeedsLogin(true); setNeedsVerification(false); setServers([]); + playbackRef.current.clear(); setLiveMetrics({}); setError(""); } else if (reason instanceof ApiError && reason.status === 403) { setNeedsVerification(true); setNeedsLogin(false); setServers([]); + playbackRef.current.clear(); setLiveMetrics({}); setError(reason.message); } else { @@ -189,14 +205,65 @@ export default function App() { }, [load]); useEffect(() => { - if (needsLogin) return; + serversRef.current = servers; + for (const server of servers) { + const samples = playbackRef.current.get(server.id); + if (!samples || !server.timestamp) continue; + const fresh = samples.filter((sample) => sample.ts > server.timestamp!); + if (fresh.length) playbackRef.current.set(server.id, fresh); + else playbackRef.current.delete(server.id); + } + }, [servers]); + + useEffect(() => { + if (needsLogin || needsVerification) return; let ticks = 0; const timer = window.setInterval(() => { ticks += 1; - if (!liveConnectedRef.current || ticks % 4 === 0) void load(true); + if (!liveConnectedRef.current || ticks % 20 === 0) void load(true); }, 15_000); return () => clearInterval(timer); - }, [load, needsLogin]); + }, [load, needsLogin, needsVerification]); + + useEffect(() => { + let previous = Date.now(); + const timer = window.setInterval(() => { + const current = Date.now(); + const elapsed = Math.max(0, Math.min(5_000, current - previous)); + previous = current; + setClockNow(current); + if (elapsed === 0) return; + setLiveMetrics((currentMetrics) => { + if (!playbackRef.current.size) return currentMetrics; + const next = { ...currentMetrics }; + for (const [serverId, samples] of playbackRef.current) { + const state = next[serverId]; + if (!state || !samples.length) continue; + const displayTimestamp = state.displayTimestamp + elapsed / 1000; + let selected: LiveSample | undefined; + while (samples.length && samples[0].ts <= displayTimestamp) selected = samples.shift(); + if (selected) { + const metrics = { ...selected.data }; + const latencyResults = Array.isArray(metrics.latency_results) ? metrics.latency_results : []; + delete metrics.latency_results; + next[serverId] = { + timestamp: selected.ts, + displayTimestamp, + metrics, + latencyResults: latencyResults.length + ? mergeLiveResults(state.latencyResults, latencyResults) + : state.latencyResults, + }; + } else { + next[serverId] = { ...state, displayTimestamp }; + } + if (!samples.length) playbackRef.current.delete(serverId); + } + return next; + }); + }, 1_000); + return () => clearInterval(timer); + }, []); useEffect(() => { document.documentElement.classList.toggle("dark", dark); @@ -249,7 +316,6 @@ export default function App() { wsRef.current.push(socket); socket.onopen = () => { liveConnectedRef.current = true; - void load(true); }; socket.onclose = () => { wsRef.current = wsRef.current.filter((current) => current !== socket); @@ -270,29 +336,55 @@ export default function App() { return; } if (message.type === "batchUpdate" && Array.isArray(message.updates)) { + const replayCached = message.cached === true; setLiveMetrics((current) => { const next = { ...current }; - for (const update of message.updates as Array<{ serverId?: string; samples?: LiveSample[] }>) { + for (const update of message.updates as Array<{ + serverId?: string; + samples?: LiveSample[]; + reportAgeMs?: number; + }>) { if (!update.serverId || !Array.isArray(update.samples) || !update.samples.length) continue; const samples = update.samples .filter((sample) => Number.isFinite(sample.ts) && sample.data && typeof sample.data === "object") .sort((left, right) => left.ts - right.ts); - const latest = samples.at(-1); - if (!latest) continue; const previous = next[update.serverId]; - if (previous && previous.timestamp >= latest.ts) continue; - const results = samples.flatMap((sample) => Array.isArray(sample.data.latency_results) - ? sample.data.latency_results - : []); - const metrics = { ...latest.data }; - delete metrics.latency_results; - next[update.serverId] = { - timestamp: latest.ts, - metrics, - latencyResults: Array.isArray(results) && results.length - ? mergeLiveResults(previous?.latencyResults, results) - : previous?.latencyResults, - }; + const pending = playbackRef.current.get(update.serverId) ?? []; + const persistedTimestamp = serversRef.current.find((server) => server.id === update.serverId)?.timestamp ?? 0; + const appliedTimestamp = Math.max(previous?.timestamp ?? 0, persistedTimestamp); + const seen = new Set([ + ...pending.map((sample) => sample.ts), + ]); + const incoming = samples.filter((sample) => sample.ts > appliedTimestamp && !seen.has(sample.ts)); + if (!incoming.length) continue; + const reportAgeSeconds = replayCached && Number.isFinite(update.reportAgeMs) + ? Math.max(0, update.reportAgeMs! / 1000) + : 0; + const cursor = replayCached + ? Math.max(previous?.displayTimestamp ?? 0, incoming[incoming.length - 1].ts + reportAgeSeconds) + : previous?.displayTimestamp ?? incoming[0].ts; + const all = [...pending, ...incoming] + .sort((left, right) => left.ts - right.ts) + .slice(-MAX_PLAYBACK_SAMPLES_PER_SERVER); + let selected: LiveSample | undefined; + while (all.length && all[0].ts <= cursor) selected = all.shift(); + if (selected) { + const metrics = { ...selected.data }; + const latencyResults = Array.isArray(metrics.latency_results) ? metrics.latency_results : []; + delete metrics.latency_results; + next[update.serverId] = { + timestamp: selected.ts, + displayTimestamp: cursor, + metrics, + latencyResults: latencyResults.length + ? mergeLiveResults(previous?.latencyResults, latencyResults) + : previous?.latencyResults, + }; + } else if (!previous) { + continue; + } + if (all.length) playbackRef.current.set(update.serverId, all); + else playbackRef.current.delete(update.serverId); } return next; }); diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 3bf9b70..d6234fb 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -1,4 +1,4 @@ -import type { AdminServer, AlertRule, AlertRuleInput, CloudflareUsage, Config, DatabaseStats, ExchangeRates, HistoryPoint, LatencySample, LatencyTask, LatencyTaskInput, LatencyTestPoint, Server, ServerInput, Settings, Theme, ThemeSettingsSchema } from "./types"; +import type { AdminServer, AlertRule, AlertRuleInput, Bootstrap, CloudflareUsage, Config, DatabaseStats, ExchangeRates, HistoryPoint, LatencySample, LatencyTask, LatencyTaskInput, LatencyTestPoint, Server, ServerInput, Settings, Theme, ThemeSettingsSchema } from "./types"; const TOKEN_KEY = "nodeflare-admin-token"; export const ADMIN_UNAUTHORIZED_EVENT = "nodeflare:admin-unauthorized"; @@ -38,6 +38,7 @@ async function request(path: string, init: RequestInit = {}, admin = false): } export const api = { + bootstrap: () => request("/api/bootstrap"), config: () => request("/api/config"), exchangeRates: () => request("/api/exchange-rates"), refreshExchangeRates: () => request("/api/admin/exchange-rates/refresh", { method: "POST" }, true), diff --git a/frontend/src/components/AdminPanel.tsx b/frontend/src/components/AdminPanel.tsx index 8c7e33d..2af4775 100644 --- a/frontend/src/components/AdminPanel.tsx +++ b/frontend/src/components/AdminPanel.tsx @@ -42,7 +42,7 @@ import { AlertRuleManager } from "./AlertRuleManager"; import { Flag } from "./Flag"; import pkg from "../../package.json"; -const { version: VERSION } = pkg; +const VERSION = import.meta.env.VITE_NODEFLARE_VERSION || pkg.version; type AdminTab = "servers" | "latency" | "appearance" | "themes" | "themeSettings" | "alerts" | "security" | "data" | "about"; type AgentPlatform = "linux" | "windows" | "macos"; @@ -81,7 +81,7 @@ const emptyServer: ServerInput = { network_interface: "", reset_day: 1, report_interval: 60, - collect_interval: 5, + collect_interval: 1, rx_correction: 0, tx_correction: 0, agent_mirror: "", @@ -679,7 +679,7 @@ export function AdminPanel({
-
+
updateForm("auto_renewal", value)} /> updateForm("auto_update", value)} /> updateForm("hidden", value)} /> updateForm("offline_notify_disabled", value)} />
diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 8bc078a..71ae026 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -76,6 +76,13 @@ export interface ExchangeRates { stale: boolean; } +export interface Bootstrap { + config: Config; + access: "ok" | "login" | "turnstile"; + servers: Server[]; + exchange_rates: ExchangeRates | null; +} + interface DiskMetric { name: string; mount_point: string; diff --git a/migrations/initial_01.sql b/migrations/initial_01.sql index 8c6a8e0..c205e38 100644 --- a/migrations/initial_01.sql +++ b/migrations/initial_01.sql @@ -1,8 +1,8 @@ PRAGMA foreign_keys = ON; CREATE TABLE settings ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL, + id INTEGER PRIMARY KEY CHECK(id = 1), + value TEXT NOT NULL CHECK(json_valid(value)), updated_at INTEGER NOT NULL ); @@ -25,7 +25,7 @@ CREATE TABLE servers ( network_interface TEXT NOT NULL DEFAULT '', reset_day INTEGER NOT NULL DEFAULT 1, report_interval INTEGER NOT NULL DEFAULT 60, - collect_interval INTEGER NOT NULL DEFAULT 5, + collect_interval INTEGER NOT NULL DEFAULT 1, rx_correction INTEGER NOT NULL DEFAULT 0, tx_correction INTEGER NOT NULL DEFAULT 0, agent_mirror TEXT NOT NULL DEFAULT '', @@ -36,57 +36,6 @@ CREATE TABLE servers ( updated_at INTEGER NOT NULL ); -CREATE TABLE latest_metrics ( - server_id TEXT PRIMARY KEY REFERENCES servers(id) ON DELETE CASCADE, - timestamp INTEGER NOT NULL, - cpu REAL NOT NULL DEFAULT 0, - load1 REAL NOT NULL DEFAULT 0, - load5 REAL NOT NULL DEFAULT 0, - load15 REAL NOT NULL DEFAULT 0, - mem_used INTEGER NOT NULL DEFAULT 0, - mem_total INTEGER NOT NULL DEFAULT 0, - swap_used INTEGER NOT NULL DEFAULT 0, - swap_total INTEGER NOT NULL DEFAULT 0, - disk_used INTEGER NOT NULL DEFAULT 0, - disk_total INTEGER NOT NULL DEFAULT 0, - net_in REAL NOT NULL DEFAULT 0, - net_out REAL NOT NULL DEFAULT 0, - net_rx_total INTEGER NOT NULL DEFAULT 0, - net_tx_total INTEGER NOT NULL DEFAULT 0, - uptime INTEGER NOT NULL DEFAULT 0, - processes INTEGER NOT NULL DEFAULT 0, - tcp_connections INTEGER NOT NULL DEFAULT 0, - udp_connections INTEGER NOT NULL DEFAULT 0, - cpu_cores INTEGER NOT NULL DEFAULT 0, - cpu_model TEXT NOT NULL DEFAULT '', - os TEXT NOT NULL DEFAULT '', - kernel TEXT NOT NULL DEFAULT '', - arch TEXT NOT NULL DEFAULT '', - virtualization TEXT NOT NULL DEFAULT '', - gpu_usage REAL NOT NULL DEFAULT 0, - gpu_model TEXT NOT NULL DEFAULT '', - agent_version TEXT NOT NULL DEFAULT '', - disk_read_bps REAL NOT NULL DEFAULT 0, - disk_write_bps REAL NOT NULL DEFAULT 0, - disk_read_iops REAL NOT NULL DEFAULT 0, - disk_write_iops REAL NOT NULL DEFAULT 0, - disk_await_ms REAL NOT NULL DEFAULT 0, - disk_utilization REAL NOT NULL DEFAULT 0, - disk_info TEXT NOT NULL DEFAULT '[]', - gpu_info TEXT NOT NULL DEFAULT '[]' -); - -CREATE TABLE traffic_cycles ( - server_id TEXT PRIMARY KEY REFERENCES servers(id) ON DELETE CASCADE, - cycle_key INTEGER NOT NULL, - reset_day INTEGER NOT NULL DEFAULT 1, - timestamp INTEGER NOT NULL, - raw_rx INTEGER NOT NULL DEFAULT 0, - raw_tx INTEGER NOT NULL DEFAULT 0, - used_rx INTEGER NOT NULL DEFAULT 0, - used_tx INTEGER NOT NULL DEFAULT 0 -); - CREATE TABLE metric_history ( server_id TEXT NOT NULL REFERENCES servers(id) ON DELETE CASCADE, timestamp INTEGER NOT NULL, @@ -114,8 +63,11 @@ CREATE TABLE metric_history ( disk_write_iops REAL NOT NULL DEFAULT 0, disk_await_ms REAL NOT NULL DEFAULT 0, disk_utilization REAL NOT NULL DEFAULT 0, + latest_timestamp INTEGER NOT NULL, + latest_json TEXT NOT NULL CHECK(json_valid(latest_json)), + latency_json TEXT NOT NULL DEFAULT '[]' CHECK(json_valid(latency_json)), PRIMARY KEY(server_id, timestamp) -); +) WITHOUT ROWID; CREATE TABLE metric_history_hourly ( server_id TEXT NOT NULL REFERENCES servers(id) ON DELETE CASCADE, @@ -144,51 +96,57 @@ CREATE TABLE metric_history_hourly ( disk_write_iops REAL NOT NULL DEFAULT 0, disk_await_ms REAL NOT NULL DEFAULT 0, disk_utilization REAL NOT NULL DEFAULT 0, + latest_timestamp INTEGER NOT NULL, + latest_json TEXT NOT NULL CHECK(json_valid(latest_json)), + latency_json TEXT NOT NULL DEFAULT '[]' CHECK(json_valid(latency_json)), PRIMARY KEY(server_id, timestamp) -); +) WITHOUT ROWID; -CREATE INDEX idx_history_server_time - ON metric_history(server_id, timestamp DESC); -CREATE INDEX idx_history_hourly_server_time - ON metric_history_hourly(server_id, timestamp DESC); CREATE INDEX idx_servers_sort ON servers(sort_order, created_at); -INSERT INTO settings(key, value, updated_at) VALUES - ('site_description', '轻量、实时的服务器运行状态', unixepoch()), - ('site_announcement', '', unixepoch()), - ('favicon_url', '', unixepoch()), - ('locale', 'zh-CN', unixepoch()), - ('public_dashboard', 'true', unixepoch()), - ('history_cache_version', '0', unixepoch()), - ('default_theme', 'system', unixepoch()), - ('active_theme_id', 'builtin-nodeflare-glass', unixepoch()), - ('background_url', '', unixepoch()), - ('theme_options', '{}', unixepoch()), - ('show_search', 'true', unixepoch()), - ('show_groups', 'true', unixepoch()), - ('show_stats', 'true', unixepoch()), - ('show_assets', 'true', unixepoch()), - ('show_traffic', 'true', unixepoch()), - ('show_speed', 'true', unixepoch()), - ('show_price', 'true', unixepoch()), - ('show_expiry', 'true', unixepoch()), - ('show_latency', 'true', unixepoch()), - ('show_uptime', 'true', unixepoch()), - ('admin_username', '', unixepoch()), - ('admin_password_hash', '', unixepoch()), - ('password_client_salt', lower(hex(randomblob(16))), unixepoch()), - ('turnstile_enabled', 'false', unixepoch()), - ('turnstile_login_enabled', 'true', unixepoch()), - ('turnstile_site_key', '', unixepoch()), - ('turnstile_secret_key', '', unixepoch()), - ('notification_enabled', 'false', unixepoch()), - ('notification_endpoint', '', unixepoch()), - ('notification_target', '', unixepoch()), - ('offline_alert_minutes', '5', unixepoch()), - ('expiry_alert_days', '7', unixepoch()), - ('cloudflare_account_id', '', unixepoch()), - ('cloudflare_api_token', '', unixepoch()); +INSERT INTO settings(id, value, updated_at) VALUES ( + 1, + json_patch( + '{ + "site_description": "轻量、实时的服务器运行状态", + "site_announcement": "", + "favicon_url": "", + "locale": "zh-CN", + "public_dashboard": "true", + "history_cache_version": "0", + "default_theme": "system", + "active_theme_id": "builtin-nodeflare-glass", + "background_url": "", + "theme_options": "{}", + "show_search": "true", + "show_groups": "true", + "show_stats": "true", + "show_assets": "true", + "show_traffic": "true", + "show_speed": "true", + "show_price": "true", + "show_expiry": "true", + "show_latency": "true", + "show_uptime": "true", + "admin_username": "", + "admin_password_hash": "", + "turnstile_enabled": "false", + "turnstile_login_enabled": "true", + "turnstile_site_key": "", + "turnstile_secret_key": "", + "notification_enabled": "false", + "notification_endpoint": "", + "notification_target": "", + "offline_alert_minutes": "5", + "expiry_alert_days": "7", + "cloudflare_account_id": "", + "cloudflare_api_token": "" + }', + json_object('password_client_salt', lower(hex(randomblob(16)))) + ), + unixepoch() +); CREATE TABLE exchange_rate_snapshots ( base_currency TEXT PRIMARY KEY, @@ -236,33 +194,12 @@ CREATE TABLE latency_tasks ( CREATE TABLE latency_task_servers ( task_id TEXT NOT NULL REFERENCES latency_tasks(id) ON DELETE CASCADE, server_id TEXT NOT NULL REFERENCES servers(id) ON DELETE CASCADE, + assigned_at INTEGER NOT NULL, PRIMARY KEY(task_id, server_id) -); - -CREATE TABLE latency_latest ( - task_id TEXT NOT NULL REFERENCES latency_tasks(id) ON DELETE CASCADE, - server_id TEXT NOT NULL REFERENCES servers(id) ON DELETE CASCADE, - timestamp INTEGER NOT NULL, - latency_ms REAL NOT NULL, - packet_loss REAL NOT NULL, - PRIMARY KEY(task_id, server_id) -); - -CREATE TABLE latency_history ( - task_id TEXT NOT NULL REFERENCES latency_tasks(id) ON DELETE CASCADE, - server_id TEXT NOT NULL REFERENCES servers(id) ON DELETE CASCADE, - timestamp INTEGER NOT NULL, - latency_ms REAL NOT NULL, - packet_loss REAL NOT NULL, - PRIMARY KEY(task_id, server_id, timestamp) -); +) WITHOUT ROWID; CREATE INDEX idx_latency_task_servers_server ON latency_task_servers(server_id, task_id); -CREATE INDEX idx_latency_history_server_time - ON latency_history(server_id, timestamp DESC); -CREATE INDEX idx_latency_history_task_time - ON latency_history(task_id, timestamp DESC); CREATE TABLE alert_rules ( id TEXT PRIMARY KEY, @@ -280,7 +217,7 @@ CREATE TABLE alert_rule_servers ( rule_id TEXT NOT NULL REFERENCES alert_rules(id) ON DELETE CASCADE, server_id TEXT NOT NULL REFERENCES servers(id) ON DELETE CASCADE, PRIMARY KEY(rule_id, server_id) -); +) WITHOUT ROWID; CREATE TABLE alert_states ( rule_id TEXT NOT NULL REFERENCES alert_rules(id) ON DELETE CASCADE, @@ -288,4 +225,6 @@ CREATE TABLE alert_states ( active INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL, PRIMARY KEY(rule_id, server_id) -); +) WITHOUT ROWID; + +PRAGMA optimize; diff --git a/scripts/build-agent.sh b/scripts/build-agent.sh index 1d56fb4..6fb123a 100644 --- a/scripts/build-agent.sh +++ b/scripts/build-agent.sh @@ -2,8 +2,11 @@ set -eu script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +root_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +NODEFLARE_VERSION=$(sh "$script_dir/resolve-version.sh") NODEFLARE_NATIVE_RUST_ONLY=1 -export NODEFLARE_NATIVE_RUST_ONLY +export NODEFLARE_NATIVE_RUST_ONLY NODEFLARE_VERSION +cd "$root_dir" . "$script_dir/ensure-rust.sh" agent_target_dir=${CARGO_TARGET_DIR:-target/agent} diff --git a/scripts/build-frontend.sh b/scripts/build-frontend.sh index 8ddfd00..a77e2bd 100644 --- a/scripts/build-frontend.sh +++ b/scripts/build-frontend.sh @@ -1,6 +1,13 @@ #!/bin/sh set -eu +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +root_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +NODEFLARE_VERSION=$(sh "$script_dir/resolve-version.sh") +VITE_NODEFLARE_VERSION=$NODEFLARE_VERSION +export NODEFLARE_VERSION VITE_NODEFLARE_VERSION +cd "$root_dir" + bun run --cwd frontend build if build_key=$(sh scripts/build-key.sh frontend 2>/dev/null); then diff --git a/scripts/build-key.sh b/scripts/build-key.sh index 92d6dcf..ff8123e 100644 --- a/scripts/build-key.sh +++ b/scripts/build-key.sh @@ -3,10 +3,10 @@ set -eu case "${1:-}" in frontend) - paths="frontend package.json bun.lock scripts/build-frontend.sh scripts/build-key.sh" + paths="frontend package.json bun.lock scripts/build-frontend.sh scripts/build-key.sh scripts/resolve-version.sh" ;; worker) - paths="src frontend Cargo.toml Cargo.lock build.rs scripts/build-worker.sh scripts/build-key.sh scripts/worker-shim.js" + paths="src frontend Cargo.toml Cargo.lock build.rs scripts/build-worker.sh scripts/build-key.sh scripts/resolve-version.sh scripts/worker-shim.js" ;; *) echo "usage: $0 frontend|worker" >&2 @@ -15,6 +15,7 @@ case "${1:-}" in esac { + sh scripts/resolve-version.sh git rev-parse HEAD git diff --binary -- $paths git ls-files --others --exclude-standard -- $paths | while IFS= read -r file; do diff --git a/scripts/build-worker.sh b/scripts/build-worker.sh index 074e75f..d8b44d5 100644 --- a/scripts/build-worker.sh +++ b/scripts/build-worker.sh @@ -1,6 +1,13 @@ #!/bin/sh set -eu +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +root_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) +NODEFLARE_VERSION=$(sh "$script_dir/resolve-version.sh") +VITE_NODEFLARE_VERSION=$NODEFLARE_VERSION +export NODEFLARE_VERSION VITE_NODEFLARE_VERSION +cd "$root_dir" + current_build_key=$(sh scripts/build-key.sh worker 2>/dev/null || true) if [ "${NODEFLARE_WORKER_READY:-0}" = "1" ] && [ -n "$current_build_key" ]; then built_build_key=$(sed -n '1p' build/.nodeflare-revision 2>/dev/null || true) @@ -12,7 +19,6 @@ if [ "${NODEFLARE_WORKER_READY:-0}" = "1" ] && [ -n "$current_build_key" ]; then fi fi -script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) . "$script_dir/ensure-rust.sh" if [ "${NODEFLARE_ADMIN_READY:-0}" != "1" ]; then diff --git a/scripts/resolve-version.sh b/scripts/resolve-version.sh new file mode 100755 index 0000000..a936c7d --- /dev/null +++ b/scripts/resolve-version.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +root_dir=$(CDPATH= cd -- "$script_dir/.." && pwd) + +raw=${NODEFLARE_VERSION:-} +if [ -z "$raw" ]; then + raw=$(git -C "$root_dir" describe --tags --exact-match --match 'v[0-9]*.[0-9]*.[0-9]*' 2>/dev/null || true) +fi +if [ -z "$raw" ]; then + raw=$(git -C "$root_dir" describe --tags --abbrev=0 --match 'v[0-9]*.[0-9]*.[0-9]*' 2>/dev/null || true) +fi +if [ -z "$raw" ]; then + raw=$(sed -n 's/^[[:space:]]*"version": "\([^"]*\)",*$/\1/p' "$root_dir/package.json" | sed -n '1p') +fi + +version=${raw#v} +if ! printf '%s\n' "$version" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "invalid NodeFlare version: $raw" >&2 + exit 1 +fi +printf '%s\n' "$version" diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 46eacb7..22c521b 100644 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -17,7 +17,9 @@ request() { monitor_curl --fail --location --silent --show-error "$@" } -config_json=$(request "$MONITOR_BASE_URL/api/config") +bootstrap_json=$(request "$MONITOR_BASE_URL/api/bootstrap") +printf '%s' "$bootstrap_json" | jq -e '.access == "ok" and (.servers | type == "array")' >/dev/null +config_json=$(printf '%s' "$bootstrap_json" | jq -c '.config') printf '%s' "$config_json" | jq -e '.site_name | length > 0' >/dev/null password_client_salt=$(printf '%s' "$config_json" | jq -er '.password_client_salt | select(length > 0)') password_derived=$(NODEFLARE_PASSWORD="$MONITOR_ADMIN_PASSWORD" NODEFLARE_SALT="$password_client_salt" bun -e ' @@ -27,7 +29,7 @@ const bits = await crypto.subtle.deriveBits({ name: "PBKDF2", hash: "SHA-256", i console.log(Array.from(new Uint8Array(bits), byte => byte.toString(16).padStart(2, "0")).join("")); ') cross_origin_status=$(monitor_curl --silent --output /dev/null --write-out '%{http_code}' \ - -H 'Origin: https://not-nodeflare.invalid' "$MONITOR_BASE_URL/api/config") + -H 'Origin: https://not-nodeflare.invalid' "$MONITOR_BASE_URL/api/bootstrap") [ "$cross_origin_status" = "403" ] admin_html=$(request "$MONITOR_BASE_URL/admin") @@ -45,8 +47,8 @@ case "$admin_html" in exit 1 ;; esac -request "$MONITOR_BASE_URL/admin-assets/admin.js" >/dev/null -request "$MONITOR_BASE_URL/admin-assets/admin.css" >/dev/null +request "$MONITOR_BASE_URL/admin-assets/admin.js" | grep -q '管理面板' +request "$MONITOR_BASE_URL/admin-assets/admin.css" | grep -q 'admin-shell' login_json=$(request -H 'Content-Type: application/json' \ --data "$(jq -nc --arg username "$MONITOR_ADMIN_USERNAME" --arg password "$MONITOR_ADMIN_PASSWORD" --arg password_derived "$password_derived" --arg turnstile_token "$MONITOR_TURNSTILE_TOKEN" '{username:$username,password:$password,password_derived:$password_derived,turnstile_token:$turnstile_token}')" \ @@ -55,13 +57,16 @@ admin_token=$(printf '%s' "$login_json" | jq -er '.token') # Login protection defaults to enabled, but without a complete Turnstile pair it # remains inactive so the first admin settings save must still work. +settings_payload=$(request -H "Authorization: Bearer $admin_token" \ + "$MONITOR_BASE_URL/api/admin/settings" | \ + jq -c '.site_description = "Smoke settings" | del(.admin_password_configured)') request -H "Authorization: Bearer $admin_token" -H 'Content-Type: application/json' -X PATCH \ - --data '{"site_description":"Smoke settings"}' \ + --data "$settings_payload" \ "$MONITOR_BASE_URL/api/admin/settings" | jq -e '.settings.site_description == "Smoke settings"' >/dev/null request -H "Authorization: Bearer $admin_token" \ - "$MONITOR_BASE_URL/api/exchange-rates" | \ - jq -e '.base == "CNY" and .rates.CNY == 1 and .rates.USD > 0 and .rates.CAD > 0 and (has("cny") | not)' >/dev/null + "$MONITOR_BASE_URL/api/bootstrap" | \ + jq -e '.config.site_description == "Smoke settings" and .exchange_rates.base == "CNY" and .exchange_rates.rates.CNY == 1 and .exchange_rates.rates.USD > 0 and .exchange_rates.rates.CAD > 0 and (.exchange_rates | has("cny") | not)' >/dev/null request -H "Authorization: Bearer $admin_token" \ "$MONITOR_BASE_URL/api/admin/themes" | \ @@ -72,7 +77,11 @@ server_input='{"name":"Smoke Test Node","region":"JP","group_name":"Test","tags" server_id= latency_task_id= alert_rule_id= +stress_report_file= cleanup() { + if [ -n "$stress_report_file" ]; then + rm -f "$stress_report_file" + fi if [ -n "$alert_rule_id" ]; then monitor_curl --silent --show-error -H "Authorization: Bearer $admin_token" \ -X DELETE "$MONITOR_BASE_URL/api/admin/alert-rules/$alert_rule_id" >/dev/null || true @@ -106,7 +115,7 @@ token_without_admin_status=$(monitor_curl --silent --output /dev/null --write-ou request -H "Authorization: Bearer $admin_token" \ "$MONITOR_BASE_URL/api/admin/servers/$server_id/token" | \ jq -e --arg token "$agent_token" '.agent_token == $token' >/dev/null -agent_version=$(sed -n 's/^version = "\([^"]*\)"/\1/p' agent/Cargo.toml | head -1) +agent_version=$(sh scripts/resolve-version.sh) sample=$(jq -nc --arg agent_version "$agent_version" '{ timestamp:(now|floor),cpu:18.5,load1:0.42,load5:0.36,load15:0.31, @@ -131,6 +140,10 @@ report_response=$(request -H "Authorization: Bearer $agent_token" -H 'Content-Ty printf '%s' "$report_response" | jq -e \ --arg task_id "$latency_task_id" \ '.collect_interval == 5 and .agent_mirror == "https://mirror.example.com" and .auto_update == true and (.latency_tasks | any(.id == $task_id and .task_type == "tcp" and .target == "example.com" and .port == 443))' >/dev/null +request -H "Authorization: Bearer $agent_token" \ + "$MONITOR_BASE_URL/api/agent/config" | jq -e \ + --arg task_id "$latency_task_id" \ + '.collect_interval == 5 and .agent_mirror == "https://mirror.example.com" and (.latency_tasks | any(.id == $task_id))' >/dev/null request -H "Authorization: Bearer $admin_token" "$MONITOR_BASE_URL/api/admin/servers" | \ jq -e --arg id "$server_id" '.servers | any(.id == $id and .last_ip == "8.8.8.8")' >/dev/null @@ -138,16 +151,46 @@ latency_report=$(printf '%s' "$report" | jq --arg task_id "$latency_task_id" '.s request -H "Authorization: Bearer $agent_token" -H 'Content-Type: application/json' \ --data "$latency_report" "$MONITOR_BASE_URL/api/agent/report" | jq -e '.collect_interval == 5' >/dev/null -# Traffic counters may reset when a host reboots. Cycle usage must retain the -# bytes already observed before the reset and continue from the new counter. -traffic_report=$(printf '%s' "$report" | jq '.samples[0].timestamp += 2 | .samples[0].net_rx_total=2147483648 | .samples[0].net_tx_total=1073741824') +# A whole Agent batch must collapse into one traffic-cycle write while preserving +# counter resets. Replaying the same batch must not count any bytes twice. +traffic_report=$(printf '%s' "$report" | jq '.samples = [ + (.samples[0] | .timestamp += 2 | .net_rx_total=2147483648 | .net_tx_total=1073741824), + (.samples[0] | .timestamp += 3 | .net_rx_total=268435456 | .net_tx_total=134217728), + (.samples[0] | .timestamp += 4 | .net_rx_total=536870912 | .net_tx_total=268435456) +]') request -H "Authorization: Bearer $agent_token" -H 'Content-Type: application/json' \ --data "$traffic_report" "$MONITOR_BASE_URL/api/agent/report" | jq -e '.collect_interval == 5' >/dev/null -reset_report=$(printf '%s' "$report" | jq '.samples[0].timestamp += 3 | .samples[0].net_rx_total=268435456 | .samples[0].net_tx_total=134217728') request -H "Authorization: Bearer $agent_token" -H 'Content-Type: application/json' \ - --data "$reset_report" "$MONITOR_BASE_URL/api/agent/report" | jq -e '.collect_interval == 5' >/dev/null + --data "$traffic_report" "$MONITOR_BASE_URL/api/agent/report" | jq -e '.collect_interval == 5' >/dev/null -request -H "Authorization: Bearer $admin_token" "$MONITOR_BASE_URL/api/servers" | jq -e --arg id "$server_id" --arg task_id "$latency_task_id" '.servers | any(.id == $id and .cpu == 18.5 and .gpu_usage == 32.5 and .disk_await_ms == 1.4 and (.gpus | length) == 1 and (.disks | length) == 1 and .disk_used == 21474836480 and .traffic_limit == 107374182400 and .net_rx_total == 2415919104 and .net_tx_total == 1207959552 and .price == 9.9 and (has("last_ip") | not) and (.latency | any(.task_id == $task_id and .latency_ms == 28.4 and .packet_loss == 25)))' >/dev/null +request -H "Authorization: Bearer $admin_token" "$MONITOR_BASE_URL/api/bootstrap" | jq -e --arg id "$server_id" --arg task_id "$latency_task_id" '.servers | any(.id == $id and .cpu == 18.5 and .gpu_usage == 32.5 and .disk_await_ms == 1.4 and (.gpus | length) == 1 and (.disks | length) == 1 and .disk_used == 21474836480 and .traffic_limit == 107374182400 and .net_rx_total == 2684354560 and .net_tx_total == 1342177280 and .price == 9.9 and (has("last_ip") | not) and (.latency | any(.task_id == $task_id and .latency_ms == 28.4 and .packet_loss == 25)))' >/dev/null + +# Changing the reset day starts a new traffic cycle. Only growth after the first +# sample in that cycle counts toward usage. +request -H "Authorization: Bearer $admin_token" -H 'Content-Type: application/json' -X PATCH \ + --data "$(printf '%s' "$server_input" | jq '.reset_day=2')" \ + "$MONITOR_BASE_URL/api/admin/servers/$server_id" >/dev/null +cycle_report=$(printf '%s' "$report" | jq '.samples = [ + (.samples[0] | .timestamp += 5 | .net_rx_total=805306368 | .net_tx_total=536870912), + (.samples[0] | .timestamp += 6 | .net_rx_total=1073741824 | .net_tx_total=805306368) +]') +request -H "Authorization: Bearer $agent_token" -H 'Content-Type: application/json' \ + --data "$cycle_report" "$MONITOR_BASE_URL/api/agent/report" | jq -e '.collect_interval == 5' >/dev/null +request -H "Authorization: Bearer $admin_token" "$MONITOR_BASE_URL/api/bootstrap" | \ + jq -e --arg id "$server_id" '.servers | any(.id == $id and .net_rx_total == 268435456 and .net_tx_total == 268435456)' >/dev/null + +# Exercise the protocol maximum so batch aggregation and request-size limits +# stay covered when an Agent flushes a full pending queue. +stress_report_file=$(mktemp "${TMPDIR:-/tmp}/nodeflare-stress.XXXXXX") +printf '%s' "$report" | jq '(now | floor) as $start | .samples = [ + range(0; 720) as $index | + (.samples[0] | + .timestamp=($start + $index + 10) | + .net_rx_total=(1073741824 + (($index + 1) * 4096)) | + .net_tx_total=(805306368 + (($index + 1) * 2048))) +]' >"$stress_report_file" +request -H "Authorization: Bearer $agent_token" -H 'Content-Type: application/json' \ + --data-binary "@$stress_report_file" "$MONITOR_BASE_URL/api/agent/report" | jq -e '.collect_interval == 5' >/dev/null request -H "Authorization: Bearer $admin_token" "$MONITOR_BASE_URL/api/history/$server_id?hours=1" | jq -e '.points | length >= 1 and any(.gpu_usage == 32.5)' >/dev/null history_cache_header=$(monitor_curl --silent --dump-header - --output /dev/null \ -H "Authorization: Bearer $admin_token" "$MONITOR_BASE_URL/api/history/$server_id?hours=1" | \ @@ -174,7 +217,7 @@ request -H "Authorization: Bearer $admin_token" "$MONITOR_BASE_URL/api/admin/ale request -H "Authorization: Bearer $admin_token" -H 'Content-Type: application/json' -X PATCH \ --data "$(printf '%s' "$server_input" | jq '.hidden=true')" \ "$MONITOR_BASE_URL/api/admin/servers/$server_id" >/dev/null -request -H "Authorization: Bearer $admin_token" "$MONITOR_BASE_URL/api/servers" | \ +request -H "Authorization: Bearer $admin_token" "$MONITOR_BASE_URL/api/bootstrap" | \ jq -e --arg id "$server_id" '.servers | all(.id != $id)' >/dev/null hidden_history_status=$(monitor_curl --silent --output /dev/null --write-out '%{http_code}' \ -H "Authorization: Bearer $admin_token" \ diff --git a/src/auth.rs b/src/auth.rs index 30807e4..845e16b 100644 --- a/src/auth.rs +++ b/src/auth.rs @@ -330,12 +330,6 @@ mod tests { issued_at + ADMIN_SESSION_SECONDS + 1 )); assert!(!verify_admin_jwt(&token, "different-secret", issued_at)); - } - - #[test] - fn rejects_tampered_admin_jwt() { - let issued_at = 1_800_000_000; - let token = create_admin_jwt_with_secret("d1-password-hash", issued_at).expect("jwt"); let tampered = format!("{}x", token); assert!(!verify_admin_jwt(&tampered, "d1-password-hash", issued_at)); } diff --git a/src/cloudflare.rs b/src/cloudflare.rs index 8c1565f..31c9e33 100644 --- a/src/cloudflare.rs +++ b/src/cloudflare.rs @@ -368,14 +368,10 @@ mod tests { } #[test] - fn classifies_durable_objects_hibernation_invocations() { + fn calculates_durable_objects_billable_requests() { assert!(is_hibernation_invocation_type("webSocketMessage")); assert!(is_hibernation_invocation_type("hibernationWakeup")); assert!(!is_hibernation_invocation_type("fetch")); - } - - #[test] - fn calculates_durable_objects_billable_requests() { let summary = summarize_durable_objects( vec![ DurableObjectsInvocationGroup { diff --git a/src/db.rs b/src/db.rs index 7e33819..69c8a93 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1,4 +1,5 @@ -use std::collections::{BTreeMap, HashMap}; +use std::cell::RefCell; +use std::collections::HashMap; use serde::{Deserialize, Serialize, Serializer}; use worker::{wasm_bindgen::JsValue, D1Database, Date, Result}; @@ -9,6 +10,15 @@ use crate::models::{ }; const SERVER_SELECT: &str = r#" +WITH latest_state AS ( + SELECT s0.id AS server_id, COALESCE( + (SELECT h.latest_json FROM metric_history h + WHERE h.server_id = s0.id ORDER BY h.timestamp DESC LIMIT 1), + (SELECT h.latest_json FROM metric_history_hourly h + WHERE h.server_id = s0.id ORDER BY h.timestamp DESC LIMIT 1) + ) AS state + FROM servers s0 +) SELECT s.id, s.name, s.region, s.group_name, s.tags, s.hidden, s.expires_at, s.traffic_limit, s.traffic_limit_type, @@ -17,20 +27,74 @@ SELECT s.network_interface, s.reset_day, s.report_interval, s.collect_interval, s.rx_correction, s.tx_correction, s.agent_mirror, s.offline_notify_disabled, s.auto_update, s.created_at, - m.timestamp, m.cpu, m.load1, m.load5, m.load15, m.mem_used, m.mem_total, - m.swap_used, m.swap_total, m.disk_used, m.disk_total, m.net_in, m.net_out, - CASE WHEN m.net_rx_total IS NULL THEN NULL ELSE MAX(0, COALESCE(c.used_rx, m.net_rx_total) + s.rx_correction) END AS net_rx_total, - CASE WHEN m.net_tx_total IS NULL THEN NULL ELSE MAX(0, COALESCE(c.used_tx, m.net_tx_total) + s.tx_correction) END AS net_tx_total, - m.uptime, m.processes, m.tcp_connections, - m.udp_connections, m.cpu_cores, m.cpu_model, m.os, m.kernel, m.arch, - m.virtualization, m.gpu_usage, m.gpu_model, m.agent_version, - m.disk_read_bps, m.disk_write_bps, m.disk_read_iops, m.disk_write_iops, - m.disk_await_ms, m.disk_utilization, m.disk_info, m.gpu_info + CAST(json_extract(m.state, '$.report.timestamp') AS INTEGER) AS timestamp, + CAST(json_extract(m.state, '$.report.cpu') AS REAL) AS cpu, + CAST(json_extract(m.state, '$.report.load1') AS REAL) AS load1, + CAST(json_extract(m.state, '$.report.load5') AS REAL) AS load5, + CAST(json_extract(m.state, '$.report.load15') AS REAL) AS load15, + CAST(json_extract(m.state, '$.report.mem_used') AS INTEGER) AS mem_used, + CAST(json_extract(m.state, '$.report.mem_total') AS INTEGER) AS mem_total, + CAST(json_extract(m.state, '$.report.swap_used') AS INTEGER) AS swap_used, + CAST(json_extract(m.state, '$.report.swap_total') AS INTEGER) AS swap_total, + CAST(json_extract(m.state, '$.report.disk_used') AS INTEGER) AS disk_used, + CAST(json_extract(m.state, '$.report.disk_total') AS INTEGER) AS disk_total, + CAST(json_extract(m.state, '$.report.net_in') AS REAL) AS net_in, + CAST(json_extract(m.state, '$.report.net_out') AS REAL) AS net_out, + CASE WHEN m.state IS NULL THEN NULL ELSE MAX(0, + CAST(json_extract(m.state, '$.report.net_rx_total') AS INTEGER) + s.rx_correction) + END AS net_rx_total, + CASE WHEN m.state IS NULL THEN NULL ELSE MAX(0, + CAST(json_extract(m.state, '$.report.net_tx_total') AS INTEGER) + s.tx_correction) + END AS net_tx_total, + CAST(json_extract(m.state, '$.report.uptime') AS INTEGER) AS uptime, + CAST(json_extract(m.state, '$.report.processes') AS INTEGER) AS processes, + CAST(json_extract(m.state, '$.report.tcp_connections') AS INTEGER) AS tcp_connections, + CAST(json_extract(m.state, '$.report.udp_connections') AS INTEGER) AS udp_connections, + CAST(json_extract(m.state, '$.report.cpu_cores') AS INTEGER) AS cpu_cores, + json_extract(m.state, '$.report.cpu_model') AS cpu_model, + json_extract(m.state, '$.report.os') AS os, + json_extract(m.state, '$.report.kernel') AS kernel, + json_extract(m.state, '$.report.arch') AS arch, + json_extract(m.state, '$.report.virtualization') AS virtualization, + CAST(json_extract(m.state, '$.report.gpu_usage') AS REAL) AS gpu_usage, + json_extract(m.state, '$.report.gpu_model') AS gpu_model, + json_extract(m.state, '$.report.agent_version') AS agent_version, + CAST(json_extract(m.state, '$.report.disk_read_bps') AS REAL) AS disk_read_bps, + CAST(json_extract(m.state, '$.report.disk_write_bps') AS REAL) AS disk_write_bps, + CAST(json_extract(m.state, '$.report.disk_read_iops') AS REAL) AS disk_read_iops, + CAST(json_extract(m.state, '$.report.disk_write_iops') AS REAL) AS disk_write_iops, + CAST(json_extract(m.state, '$.report.disk_await_ms') AS REAL) AS disk_await_ms, + CAST(json_extract(m.state, '$.report.disk_utilization') AS REAL) AS disk_utilization, + json_extract(m.state, '$.report.disks') AS disk_info, + json_extract(m.state, '$.report.gpus') AS gpu_info FROM servers s -LEFT JOIN latest_metrics m ON m.server_id = s.id -LEFT JOIN traffic_cycles c ON c.server_id = s.id +LEFT JOIN latest_state m ON m.server_id = s.id "#; +const SETTINGS_CACHE_TTL_SECONDS: i64 = 30; + +#[derive(Clone, PartialEq)] +struct SettingsCacheKey { + default_name: String, + default_threshold: i64, + default_retention: i64, + default_username: String, +} + +struct SettingsCacheEntry { + key: SettingsCacheKey, + expires_at: i64, + settings: SettingsView, +} + +thread_local! { + static SETTINGS_CACHE: RefCell> = const { RefCell::new(None) }; +} + +fn invalidate_settings_cache() { + SETTINGS_CACHE.with(|cache| *cache.borrow_mut() = None); +} + pub const SECRET_MASK: &str = "********"; fn secret_for_api(value: &str) -> &str { @@ -48,12 +112,6 @@ where serializer.serialize_str(secret_for_api(value)) } -#[derive(Debug, serde::Deserialize)] -struct SettingRow { - key: String, - value: String, -} - #[derive(Debug, Deserialize)] struct ThemeRow { id: String, @@ -78,6 +136,8 @@ pub struct AgentLiveContext { pub collect_interval: i64, pub reset_day: i64, pub cycle_key: i64, + pub traffic_reset_day: i64, + pub traffic_timestamp: i64, pub raw_rx: i64, pub raw_tx: i64, pub used_rx: i64, @@ -87,13 +147,195 @@ pub struct AgentLiveContext { pub last_persisted_at: i64, } -#[derive(Serialize)] -struct TrafficCounterSample { - timestamp: i64, +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct TrafficCounterState { + pub cycle_key: i64, + pub reset_day: i64, + pub timestamp: i64, + pub raw_rx: i64, + pub raw_tx: i64, + pub used_rx: i64, + pub used_tx: i64, +} + +#[derive(Debug, Deserialize)] +struct TrafficCounterContext { + configured_reset_day: i64, cycle_key: i64, - reset_day: i64, + traffic_reset_day: i64, + traffic_timestamp: i64, raw_rx: i64, raw_tx: i64, + used_rx: i64, + used_tx: i64, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct HistoryMetricAggregate { + #[serde(rename = "n")] + count: u64, + #[serde(rename = "t")] + timestamp: i64, + #[serde(rename = "c")] + cpu_sum: f64, + #[serde(rename = "a")] + load1_sum: f64, + #[serde(rename = "b")] + load5_sum: f64, + #[serde(rename = "d")] + load15_sum: f64, + #[serde(rename = "m")] + mem_used_sum: f64, + #[serde(rename = "w")] + swap_used_sum: f64, + #[serde(rename = "mt")] + mem_total: i64, + #[serde(rename = "wt")] + swap_total: i64, + #[serde(rename = "du")] + disk_used: i64, + #[serde(rename = "dt")] + disk_total: i64, + #[serde(rename = "nr")] + net_rx_total: i64, + #[serde(rename = "nt")] + net_tx_total: i64, + #[serde(rename = "g")] + gpu_usage: f64, + #[serde(rename = "ni")] + net_in: f64, + #[serde(rename = "no")] + net_out: f64, + #[serde(rename = "p")] + processes: i64, + #[serde(rename = "tc")] + tcp_connections: i64, + #[serde(rename = "uc")] + udp_connections: i64, + #[serde(rename = "dr")] + disk_read_bps: f64, + #[serde(rename = "dw")] + disk_write_bps: f64, + #[serde(rename = "ri")] + disk_read_iops: f64, + #[serde(rename = "wi")] + disk_write_iops: f64, + #[serde(rename = "da")] + disk_await_ms: f64, + #[serde(rename = "di")] + disk_utilization: f64, +} + +impl HistoryMetricAggregate { + pub fn extend(&mut self, reports: &[AgentReport]) { + for report in reports { + self.count = self.count.saturating_add(1); + self.cpu_sum += report.cpu; + self.load1_sum += report.load1; + self.load5_sum += report.load5; + self.load15_sum += report.load15; + self.mem_used_sum += report.mem_used as f64; + self.swap_used_sum += report.swap_used as f64; + + self.net_in = self.net_in.max(report.net_in); + self.net_out = self.net_out.max(report.net_out); + self.processes = self.processes.max(report.processes); + self.tcp_connections = self.tcp_connections.max(report.tcp_connections); + self.udp_connections = self.udp_connections.max(report.udp_connections); + self.disk_read_bps = self.disk_read_bps.max(report.disk_read_bps); + self.disk_write_bps = self.disk_write_bps.max(report.disk_write_bps); + self.disk_read_iops = self.disk_read_iops.max(report.disk_read_iops); + self.disk_write_iops = self.disk_write_iops.max(report.disk_write_iops); + self.disk_await_ms = self.disk_await_ms.max(report.disk_await_ms); + self.disk_utilization = self.disk_utilization.max(report.disk_utilization); + + if report.timestamp >= self.timestamp { + self.timestamp = report.timestamp; + self.mem_total = report.mem_total; + self.swap_total = report.swap_total; + self.disk_used = report.disk_used; + self.disk_total = report.disk_total; + self.net_rx_total = report.net_rx_total; + self.net_tx_total = report.net_tx_total; + self.gpu_usage = report.gpu_usage; + } + } + } + + pub fn merge(&mut self, other: Self) { + if other.count == 0 { + return; + } + if self.count == 0 { + *self = other; + return; + } + + self.count = self.count.saturating_add(other.count); + self.cpu_sum += other.cpu_sum; + self.load1_sum += other.load1_sum; + self.load5_sum += other.load5_sum; + self.load15_sum += other.load15_sum; + self.mem_used_sum += other.mem_used_sum; + self.swap_used_sum += other.swap_used_sum; + self.net_in = self.net_in.max(other.net_in); + self.net_out = self.net_out.max(other.net_out); + self.processes = self.processes.max(other.processes); + self.tcp_connections = self.tcp_connections.max(other.tcp_connections); + self.udp_connections = self.udp_connections.max(other.udp_connections); + self.disk_read_bps = self.disk_read_bps.max(other.disk_read_bps); + self.disk_write_bps = self.disk_write_bps.max(other.disk_write_bps); + self.disk_read_iops = self.disk_read_iops.max(other.disk_read_iops); + self.disk_write_iops = self.disk_write_iops.max(other.disk_write_iops); + self.disk_await_ms = self.disk_await_ms.max(other.disk_await_ms); + self.disk_utilization = self.disk_utilization.max(other.disk_utilization); + + if other.timestamp > self.timestamp { + self.timestamp = other.timestamp; + self.mem_total = other.mem_total; + self.swap_total = other.swap_total; + self.disk_used = other.disk_used; + self.disk_total = other.disk_total; + self.net_rx_total = other.net_rx_total; + self.net_tx_total = other.net_tx_total; + self.gpu_usage = other.gpu_usage; + } + } + + pub fn point(&self) -> Option { + if self.count == 0 { + return None; + } + let count = self.count as f64; + let rounded_mean = |sum: f64| (sum / count).round() as i64; + Some(HistoryPoint { + timestamp: self.timestamp / 60 * 60, + cpu: self.cpu_sum / count, + load1: self.load1_sum / count, + load5: self.load5_sum / count, + load15: self.load15_sum / count, + mem_used: rounded_mean(self.mem_used_sum), + mem_total: self.mem_total, + swap_used: rounded_mean(self.swap_used_sum), + swap_total: self.swap_total, + disk_used: self.disk_used, + disk_total: self.disk_total, + net_in: self.net_in, + net_out: self.net_out, + net_rx_total: self.net_rx_total, + net_tx_total: self.net_tx_total, + processes: self.processes, + tcp_connections: self.tcp_connections, + udp_connections: self.udp_connections, + gpu_usage: self.gpu_usage, + disk_read_bps: self.disk_read_bps, + disk_write_bps: self.disk_write_bps, + disk_read_iops: self.disk_read_iops, + disk_write_iops: self.disk_write_iops, + disk_await_ms: self.disk_await_ms, + disk_utilization: self.disk_utilization, + }) + } } #[derive(Debug, Clone, Serialize)] @@ -178,10 +420,11 @@ struct AlertRuleRow { #[derive(Debug, Deserialize)] struct AlertServerRow { + rule_id: String, server_id: String, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct AlertMetricRow { pub server_id: String, pub name: String, @@ -245,6 +488,66 @@ pub(crate) fn traffic_cycle_key(timestamp: i64, reset_day: i64) -> i64 { } } +impl TrafficCounterState { + pub fn apply(&mut self, report: &AgentReport, configured_reset_day: i64) { + if report.timestamp <= self.timestamp { + return; + } + let configured_reset_day = configured_reset_day.clamp(1, 31); + let cycle_key = traffic_cycle_key(report.timestamp, configured_reset_day); + if self.timestamp <= 0 { + self.used_rx = report.net_rx_total; + self.used_tx = report.net_tx_total; + } else if cycle_key != self.cycle_key || configured_reset_day != self.reset_day { + self.used_rx = 0; + self.used_tx = 0; + } else { + let rx_delta = if report.net_rx_total >= self.raw_rx { + report.net_rx_total - self.raw_rx + } else { + report.net_rx_total + }; + let tx_delta = if report.net_tx_total >= self.raw_tx { + report.net_tx_total - self.raw_tx + } else { + report.net_tx_total + }; + self.used_rx = self.used_rx.saturating_add(rx_delta); + self.used_tx = self.used_tx.saturating_add(tx_delta); + } + self.cycle_key = cycle_key; + self.reset_day = configured_reset_day; + self.timestamp = report.timestamp; + self.raw_rx = report.net_rx_total; + self.raw_tx = report.net_tx_total; + } + + pub fn extend(&mut self, reports: &[AgentReport], configured_reset_day: i64) { + let mut reports = reports.iter().collect::>(); + reports.sort_by_key(|report| report.timestamp); + for report in reports { + self.apply(report, configured_reset_day); + } + } +} + +impl TrafficCounterContext { + fn into_parts(self) -> (TrafficCounterState, i64) { + ( + TrafficCounterState { + cycle_key: self.cycle_key, + reset_day: self.traffic_reset_day, + timestamp: self.traffic_timestamp, + raw_rx: self.raw_rx, + raw_tx: self.raw_tx, + used_rx: self.used_rx, + used_tx: self.used_tx, + }, + self.configured_reset_day, + ) + } +} + pub async fn list_servers(db: &D1Database, include_hidden: bool) -> Result> { let filter = if include_hidden { "" @@ -279,21 +582,29 @@ pub async fn get_agent_identity(db: &D1Database, token: &str) -> Result Result> { let mut context: Option = db .prepare( - r#"SELECT + r#"WITH latest_state AS ( + SELECT COALESCE( + (SELECT latest_json FROM metric_history + WHERE server_id = ?1 ORDER BY timestamp DESC LIMIT 1), + (SELECT latest_json FROM metric_history_hourly + WHERE server_id = ?1 ORDER BY timestamp DESC LIMIT 1) + ) AS state + ) SELECT s.report_interval, s.collect_interval, s.reset_day, - COALESCE(c.cycle_key, 0) AS cycle_key, - COALESCE(c.raw_rx, m.net_rx_total, 0) AS raw_rx, - COALESCE(c.raw_tx, m.net_tx_total, 0) AS raw_tx, - COALESCE(c.used_rx, m.net_rx_total, 0) AS used_rx, - COALESCE(c.used_tx, m.net_tx_total, 0) AS used_tx, + COALESCE(CAST(json_extract(m.state, '$.traffic.cycle_key') AS INTEGER), 0) AS cycle_key, + COALESCE(CAST(json_extract(m.state, '$.traffic.reset_day') AS INTEGER), s.reset_day) AS traffic_reset_day, + COALESCE(CAST(json_extract(m.state, '$.traffic.timestamp') AS INTEGER), 0) AS traffic_timestamp, + COALESCE(CAST(json_extract(m.state, '$.traffic.raw_rx') AS INTEGER), 0) AS raw_rx, + COALESCE(CAST(json_extract(m.state, '$.traffic.raw_tx') AS INTEGER), 0) AS raw_tx, + COALESCE(CAST(json_extract(m.state, '$.traffic.used_rx') AS INTEGER), 0) AS used_rx, + COALESCE(CAST(json_extract(m.state, '$.traffic.used_tx') AS INTEGER), 0) AS used_tx, s.rx_correction, s.tx_correction, - COALESCE(m.timestamp, 0) AS last_persisted_at + COALESCE(CAST(json_extract(m.state, '$.report.timestamp') AS INTEGER), 0) AS last_persisted_at FROM servers s - LEFT JOIN latest_metrics m ON m.server_id = s.id - LEFT JOIN traffic_cycles c ON c.server_id = s.id + CROSS JOIN latest_state m WHERE s.id = ?1 LIMIT 1"#, ) @@ -485,17 +796,24 @@ pub async fn list_alert_rules(db: &D1Database) -> Result> { .all() .await? .results()?; + let assignments: Vec = db + .prepare( + "SELECT rule_id, server_id FROM alert_rule_servers \ + ORDER BY rule_id ASC, server_id ASC", + ) + .all() + .await? + .results()?; + let mut servers_by_rule = HashMap::>::new(); + for assignment in assignments { + servers_by_rule + .entry(assignment.rule_id) + .or_default() + .push(assignment.server_id); + } let mut rules = Vec::with_capacity(rows.len()); for row in rows { - let servers: Vec = db - .prepare( - "SELECT server_id FROM alert_rule_servers WHERE rule_id = ?1 \ - ORDER BY server_id ASC", - ) - .bind(&[text(&row.id)])? - .all() - .await? - .results()?; + let server_ids = servers_by_rule.remove(&row.id).unwrap_or_default(); rules.push(AlertRuleView { id: row.id, name: row.name, @@ -504,7 +822,7 @@ pub async fn list_alert_rules(db: &D1Database) -> Result> { duration_minutes: row.duration_minutes, aggregation: row.aggregation, enabled: row.enabled != 0, - server_ids: servers.into_iter().map(|row| row.server_id).collect(), + server_ids, }); } Ok(rules) @@ -582,48 +900,11 @@ pub async fn delete_alert_rule(db: &D1Database, id: &str) -> Result { Ok(result.meta()?.and_then(|meta| meta.changes).unwrap_or(0) > 0) } -pub async fn alert_metric_values( - db: &D1Database, - rule: &AlertRuleView, +pub(crate) fn alert_window_covered( + row: &AlertMetricRow, + duration_minutes: i64, current_time: i64, -) -> Result> { - let metric = match rule.metric.as_str() { - "cpu" => "h.cpu", - "memory" => "CASE WHEN h.mem_total > 0 THEN h.mem_used * 100.0 / h.mem_total ELSE 0 END", - "disk" => "CASE WHEN h.disk_total > 0 THEN h.disk_used * 100.0 / h.disk_total ELSE 0 END", - "net_in" => "h.net_in / 1048576.0", - "net_out" => "h.net_out / 1048576.0", - _ => return Ok(Vec::new()), - }; - let aggregate = if rule.aggregation == "continuous" { - "MIN" - } else { - "AVG" - }; - let since = current_time - rule.duration_minutes.clamp(1, 1440) * 60; - let query = format!( - "SELECT s.id AS server_id, s.name, {aggregate}({metric}) AS value, \ - COUNT(*) AS sample_count, MIN(h.timestamp) AS first_timestamp, \ - MAX(h.timestamp) AS last_timestamp, s.report_interval \ - FROM servers s JOIN metric_history h ON h.server_id=s.id \ - WHERE h.timestamp >= ?1 AND s.hidden=0 AND (\ - NOT EXISTS(SELECT 1 FROM alert_rule_servers ars WHERE ars.rule_id=?2) OR \ - EXISTS(SELECT 1 FROM alert_rule_servers ars WHERE ars.rule_id=?2 AND ars.server_id=s.id)\ - ) GROUP BY s.id, s.name, s.report_interval" - ); - let rows: Vec = db - .prepare(query) - .bind(&[number(since), text(&rule.id)])? - .all() - .await? - .results()?; - Ok(rows - .into_iter() - .filter(|row| alert_window_covered(row, rule.duration_minutes, current_time)) - .collect()) -} - -fn alert_window_covered(row: &AlertMetricRow, duration_minutes: i64, current_time: i64) -> bool { +) -> bool { let window_seconds = duration_minutes.clamp(1, 1440) * 60; let report_interval = row.report_interval.clamp(15, 3600).max(60); let expected_samples = ((window_seconds + report_interval - 1) / report_interval).max(1); @@ -694,7 +975,7 @@ pub async fn history(db: &D1Database, id: &str, hours: i64) -> Result 3600, _ => 14_400, }; - let source = if hours <= 24 * 7 { + let source = if hours <= 24 { "SELECT * FROM metric_history".to_string() } else { "SELECT * FROM metric_history UNION ALL SELECT * FROM metric_history_hourly".to_string() @@ -710,19 +991,19 @@ pub async fn history(db: &D1Database, id: &str, hours: i64) -> Result= ?2 GROUP BY timestamp / {bucket} @@ -737,238 +1018,151 @@ pub async fn history(db: &D1Database, id: &str, hours: i64) -> Result Result<()> { + let mut aggregate = HistoryMetricAggregate::default(); + aggregate.extend(reports); + let Some(history) = aggregate.point() else { + return Ok(()); + }; + let mut latency = crate::latency::LatencyMetricAggregates::default(); + let received_at = now(); + for report in reports { + latency.extend(&report.latency_results, received_at); + } + let context: TrafficCounterContext = db + .prepare( + r#"WITH latest_state AS ( + SELECT COALESCE( + (SELECT latest_json FROM metric_history + WHERE server_id = ?1 ORDER BY timestamp DESC LIMIT 1), + (SELECT latest_json FROM metric_history_hourly + WHERE server_id = ?1 ORDER BY timestamp DESC LIMIT 1) + ) AS state + ) SELECT + s.reset_day AS configured_reset_day, + COALESCE(CAST(json_extract(m.state, '$.traffic.cycle_key') AS INTEGER), 0) AS cycle_key, + COALESCE(CAST(json_extract(m.state, '$.traffic.reset_day') AS INTEGER), s.reset_day) AS traffic_reset_day, + COALESCE(CAST(json_extract(m.state, '$.traffic.timestamp') AS INTEGER), 0) AS traffic_timestamp, + COALESCE(CAST(json_extract(m.state, '$.traffic.raw_rx') AS INTEGER), 0) AS raw_rx, + COALESCE(CAST(json_extract(m.state, '$.traffic.raw_tx') AS INTEGER), 0) AS raw_tx, + COALESCE(CAST(json_extract(m.state, '$.traffic.used_rx') AS INTEGER), 0) AS used_rx, + COALESCE(CAST(json_extract(m.state, '$.traffic.used_tx') AS INTEGER), 0) AS used_tx + FROM servers s + CROSS JOIN latest_state m + WHERE s.id = ?1 + LIMIT 1"#, + ) + .bind(&[text(server_id)])? + .first(None) + .await? + .ok_or_else(|| worker::Error::RustError("节点不存在".to_string()))?; + let (mut traffic, configured_reset_day) = context.into_parts(); + traffic.extend(reports, configured_reset_day); + save_reports_with_history(db, server_id, reports, &history, &traffic, &latency).await +} + +pub async fn save_reports_with_history( + db: &D1Database, + server_id: &str, + reports: &[AgentReport], + history_point: &HistoryPoint, + traffic: &TrafficCounterState, + latency: &crate::latency::LatencyMetricAggregates, +) -> Result<()> { let Some(latest_report) = reports.iter().max_by_key(|report| report.timestamp) else { return Ok(()); }; let latest_timestamp = latest_report.timestamp; - let latest = db - .prepare( - r#"INSERT INTO latest_metrics ( - server_id, timestamp, cpu, load1, load5, load15, mem_used, mem_total, - swap_used, swap_total, disk_used, disk_total, net_in, net_out, - net_rx_total, net_tx_total, uptime, processes, tcp_connections, - udp_connections, cpu_cores, cpu_model, os, kernel, arch, - virtualization, gpu_usage, gpu_model, agent_version, - disk_read_bps, disk_write_bps, disk_read_iops, disk_write_iops, - disk_await_ms, disk_utilization, disk_info, gpu_info - ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, - ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, - ?27, ?28, ?29, ?30, ?31, ?32, ?33, ?34, ?35, ?36, ?37 - ) ON CONFLICT(server_id) DO UPDATE SET - timestamp=excluded.timestamp, cpu=excluded.cpu, load1=excluded.load1, - load5=excluded.load5, load15=excluded.load15, mem_used=excluded.mem_used, - mem_total=excluded.mem_total, swap_used=excluded.swap_used, - swap_total=excluded.swap_total, disk_used=excluded.disk_used, - disk_total=excluded.disk_total, net_in=excluded.net_in, net_out=excluded.net_out, - net_rx_total=excluded.net_rx_total, net_tx_total=excluded.net_tx_total, - uptime=excluded.uptime, processes=excluded.processes, - tcp_connections=excluded.tcp_connections, udp_connections=excluded.udp_connections, - cpu_cores=excluded.cpu_cores, cpu_model=excluded.cpu_model, os=excluded.os, - kernel=excluded.kernel, arch=excluded.arch, - virtualization=excluded.virtualization, gpu_usage=excluded.gpu_usage, gpu_model=excluded.gpu_model, - agent_version=excluded.agent_version, - disk_read_bps=excluded.disk_read_bps, disk_write_bps=excluded.disk_write_bps, - disk_read_iops=excluded.disk_read_iops, disk_write_iops=excluded.disk_write_iops, - disk_await_ms=excluded.disk_await_ms, - disk_utilization=excluded.disk_utilization, - disk_info=excluded.disk_info, gpu_info=excluded.gpu_info - WHERE excluded.timestamp >= latest_metrics.timestamp"#, - ) - .bind(&report_values(server_id, latest_report, latest_timestamp))?; - - let history_statement = db.prepare( + let mut latest_report = latest_report.clone(); + latest_report.net_rx_total = traffic.used_rx; + latest_report.net_tx_total = traffic.used_tx; + latest_report.latency_results.clear(); + let latest_json = serde_json::json!({ + "report": latest_report, + "traffic": traffic, + }) + .to_string(); + let latency_json = latency.stored_json()?; + db.prepare( r#"INSERT INTO metric_history ( - server_id, timestamp, cpu, load1, load5, load15, - mem_used, mem_total, swap_used, swap_total, disk_used, disk_total, - net_in, net_out, net_rx_total, net_tx_total, processes, - tcp_connections, udp_connections, gpu_usage, - disk_read_bps, disk_write_bps, disk_read_iops, disk_write_iops, - disk_await_ms, disk_utilization - ) SELECT - ?1, CAST(json_extract(value, '$.timestamp') AS INTEGER), - CAST(json_extract(value, '$.cpu') AS REAL), - CAST(json_extract(value, '$.load1') AS REAL), - CAST(json_extract(value, '$.load5') AS REAL), - CAST(json_extract(value, '$.load15') AS REAL), - CAST(json_extract(value, '$.mem_used') AS INTEGER), - CAST(json_extract(value, '$.mem_total') AS INTEGER), - CAST(json_extract(value, '$.swap_used') AS INTEGER), - CAST(json_extract(value, '$.swap_total') AS INTEGER), - CAST(json_extract(value, '$.disk_used') AS INTEGER), - CAST(json_extract(value, '$.disk_total') AS INTEGER), - CAST(json_extract(value, '$.net_in') AS REAL), - CAST(json_extract(value, '$.net_out') AS REAL), - CAST(json_extract(value, '$.net_rx_total') AS INTEGER), - CAST(json_extract(value, '$.net_tx_total') AS INTEGER), - CAST(json_extract(value, '$.processes') AS INTEGER), - CAST(json_extract(value, '$.tcp_connections') AS INTEGER), - CAST(json_extract(value, '$.udp_connections') AS INTEGER), - CAST(json_extract(value, '$.gpu_usage') AS REAL), - CAST(json_extract(value, '$.disk_read_bps') AS REAL), - CAST(json_extract(value, '$.disk_write_bps') AS REAL), - CAST(json_extract(value, '$.disk_read_iops') AS REAL), - CAST(json_extract(value, '$.disk_write_iops') AS REAL), - CAST(json_extract(value, '$.disk_await_ms') AS REAL), - CAST(json_extract(value, '$.disk_utilization') AS REAL) - FROM json_each(?2) WHERE true - ON CONFLICT(server_id, timestamp) DO UPDATE SET - cpu=excluded.cpu, load1=excluded.load1, load5=excluded.load5, - load15=excluded.load15, mem_used=excluded.mem_used, - mem_total=excluded.mem_total, swap_used=excluded.swap_used, - swap_total=excluded.swap_total, disk_used=excluded.disk_used, - disk_total=excluded.disk_total, net_in=excluded.net_in, - net_out=excluded.net_out, net_rx_total=excluded.net_rx_total, - net_tx_total=excluded.net_tx_total, processes=excluded.processes, - tcp_connections=excluded.tcp_connections, - udp_connections=excluded.udp_connections, gpu_usage=excluded.gpu_usage, - disk_read_bps=excluded.disk_read_bps, - disk_write_bps=excluded.disk_write_bps, - disk_read_iops=excluded.disk_read_iops, - disk_write_iops=excluded.disk_write_iops, - disk_await_ms=excluded.disk_await_ms, - disk_utilization=excluded.disk_utilization"#, - ); - - // D1 keeps one representative sample per minute. The Agent can sample every few - // seconds without multiplying long-term rows or write cost. - let mut minute_samples = BTreeMap::new(); - for report in reports { - minute_samples.insert(report.timestamp / 60 * 60, report); - } - let history_samples = minute_samples - .into_iter() - .map(|(timestamp, report)| HistoryPoint { - timestamp, - cpu: report.cpu, - load1: report.load1, - load5: report.load5, - load15: report.load15, - mem_used: report.mem_used, - mem_total: report.mem_total, - swap_used: report.swap_used, - swap_total: report.swap_total, - disk_used: report.disk_used, - disk_total: report.disk_total, - net_in: report.net_in, - net_out: report.net_out, - net_rx_total: report.net_rx_total, - net_tx_total: report.net_tx_total, - processes: report.processes, - tcp_connections: report.tcp_connections, - udp_connections: report.udp_connections, - gpu_usage: report.gpu_usage, - disk_read_bps: report.disk_read_bps, - disk_write_bps: report.disk_write_bps, - disk_read_iops: report.disk_read_iops, - disk_write_iops: report.disk_write_iops, - disk_await_ms: report.disk_await_ms, - disk_utilization: report.disk_utilization, - }) - .collect::>(); - let history_json = serde_json::to_string(&history_samples)?; - let history = history_statement.bind(&[text(server_id), text(&history_json)])?; - let reset_day = db - .prepare("SELECT reset_day FROM servers WHERE id = ?1") - .bind(&[text(server_id)])? - .first::(Some("reset_day")) - .await? - .unwrap_or(1); - let mut traffic_samples = reports - .iter() - .map(|report| TrafficCounterSample { - timestamp: report.timestamp, - cycle_key: traffic_cycle_key(report.timestamp, reset_day), - reset_day, - raw_rx: report.net_rx_total, - raw_tx: report.net_tx_total, - }) - .collect::>(); - traffic_samples.sort_by_key(|sample| sample.timestamp); - let traffic_json = serde_json::to_string(&traffic_samples)?; - let traffic = db - .prepare( - r#"INSERT INTO traffic_cycles( - server_id, cycle_key, reset_day, timestamp, raw_rx, raw_tx, used_rx, used_tx - ) SELECT - ?1, - CAST(json_extract(value, '$.cycle_key') AS INTEGER), - CAST(json_extract(value, '$.reset_day') AS INTEGER), - CAST(json_extract(value, '$.timestamp') AS INTEGER), - CAST(json_extract(value, '$.raw_rx') AS INTEGER), - CAST(json_extract(value, '$.raw_tx') AS INTEGER), - CAST(json_extract(value, '$.raw_rx') AS INTEGER), - CAST(json_extract(value, '$.raw_tx') AS INTEGER) - FROM json_each(?2) WHERE true ORDER BY CAST(json_extract(value, '$.timestamp') AS INTEGER) - ON CONFLICT(server_id) DO UPDATE SET - cycle_key=excluded.cycle_key, - reset_day=excluded.reset_day, - timestamp=excluded.timestamp, - raw_rx=excluded.raw_rx, - raw_tx=excluded.raw_tx, - used_rx=CASE - WHEN excluded.cycle_key != traffic_cycles.cycle_key THEN 0 - WHEN excluded.reset_day != traffic_cycles.reset_day - THEN 0 - WHEN excluded.raw_rx >= traffic_cycles.raw_rx - THEN traffic_cycles.used_rx + excluded.raw_rx - traffic_cycles.raw_rx - ELSE traffic_cycles.used_rx + excluded.raw_rx - END, - used_tx=CASE - WHEN excluded.cycle_key != traffic_cycles.cycle_key THEN 0 - WHEN excluded.reset_day != traffic_cycles.reset_day - THEN 0 - WHEN excluded.raw_tx >= traffic_cycles.raw_tx - THEN traffic_cycles.used_tx + excluded.raw_tx - traffic_cycles.raw_tx - ELSE traffic_cycles.used_tx + excluded.raw_tx - END - WHERE excluded.timestamp > traffic_cycles.timestamp"#, - ) - .bind(&[text(server_id), text(&traffic_json)])?; - db.batch(vec![latest, history, traffic]).await?; - Ok(()) -} - -fn report_values(server_id: &str, report: &AgentReport, timestamp: i64) -> Vec { - vec![ + server_id, timestamp, cpu, load1, load5, load15, + mem_used, mem_total, swap_used, swap_total, disk_used, disk_total, + net_in, net_out, net_rx_total, net_tx_total, processes, + tcp_connections, udp_connections, gpu_usage, + disk_read_bps, disk_write_bps, disk_read_iops, disk_write_iops, + disk_await_ms, disk_utilization, latest_timestamp, latest_json, latency_json + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, + ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, + ?27, ?28, ?29 + ) ON CONFLICT(server_id, timestamp) DO UPDATE SET + cpu=excluded.cpu, load1=excluded.load1, load5=excluded.load5, + load15=excluded.load15, mem_used=excluded.mem_used, + mem_total=excluded.mem_total, swap_used=excluded.swap_used, + swap_total=excluded.swap_total, disk_used=excluded.disk_used, + disk_total=excluded.disk_total, net_in=excluded.net_in, + net_out=excluded.net_out, net_rx_total=excluded.net_rx_total, + net_tx_total=excluded.net_tx_total, processes=excluded.processes, + tcp_connections=excluded.tcp_connections, + udp_connections=excluded.udp_connections, gpu_usage=excluded.gpu_usage, + disk_read_bps=excluded.disk_read_bps, + disk_write_bps=excluded.disk_write_bps, + disk_read_iops=excluded.disk_read_iops, + disk_write_iops=excluded.disk_write_iops, + disk_await_ms=excluded.disk_await_ms, + disk_utilization=excluded.disk_utilization, + latest_timestamp=MAX(metric_history.latest_timestamp, excluded.latest_timestamp), + latest_json=CASE + WHEN excluded.latest_timestamp >= metric_history.latest_timestamp THEN excluded.latest_json + ELSE metric_history.latest_json END, + latency_json=CASE + WHEN json_array_length(excluded.latency_json) = 0 THEN metric_history.latency_json + ELSE ( + SELECT json_group_array(json(item)) FROM ( + SELECT value AS item FROM json_each(excluded.latency_json) + UNION ALL + SELECT previous.value AS item FROM json_each(metric_history.latency_json) previous + WHERE NOT EXISTS ( + SELECT 1 FROM json_each(excluded.latency_json) current + WHERE json_extract(current.value, '$.task_id') = + json_extract(previous.value, '$.task_id') + ) + ) + ) END"#, + ) + .bind(&[ text(server_id), - number(timestamp), - number(report.cpu), - number(report.load1), - number(report.load5), - number(report.load15), - number(report.mem_used), - number(report.mem_total), - number(report.swap_used), - number(report.swap_total), - number(report.disk_used), - number(report.disk_total), - number(report.net_in), - number(report.net_out), - number(report.net_rx_total), - number(report.net_tx_total), - number(report.uptime), - number(report.processes), - number(report.tcp_connections), - number(report.udp_connections), - number(report.cpu_cores), - text(&report.cpu_model), - text(&report.os), - text(&report.kernel), - text(&report.arch), - text(&report.virtualization), - number(report.gpu_usage), - text(&report.gpu_model), - text(&report.agent_version), - number(report.disk_read_bps), - number(report.disk_write_bps), - number(report.disk_read_iops), - number(report.disk_write_iops), - number(report.disk_await_ms), - number(report.disk_utilization), - text(&serde_json::to_string(&report.disks).unwrap_or_else(|_| "[]".to_string())), - text(&serde_json::to_string(&report.gpus).unwrap_or_else(|_| "[]".to_string())), - ] + number(history_point.timestamp), + number(history_point.cpu), + number(history_point.load1), + number(history_point.load5), + number(history_point.load15), + number(history_point.mem_used), + number(history_point.mem_total), + number(history_point.swap_used), + number(history_point.swap_total), + number(history_point.disk_used), + number(history_point.disk_total), + number(history_point.net_in), + number(history_point.net_out), + number(history_point.net_rx_total), + number(history_point.net_tx_total), + number(history_point.processes), + number(history_point.tcp_connections), + number(history_point.udp_connections), + number(history_point.gpu_usage), + number(history_point.disk_read_bps), + number(history_point.disk_write_bps), + number(history_point.disk_read_iops), + number(history_point.disk_write_iops), + number(history_point.disk_await_ms), + number(history_point.disk_utilization), + number(latest_timestamp), + text(&latest_json), + text(&latency_json), + ])? + .run() + .await?; + Ok(()) } fn bool_setting(values: &HashMap, key: &str, fallback: bool) -> bool { @@ -1008,14 +1202,30 @@ pub async fn settings( default_retention: i64, default_username: &str, ) -> Result { - let rows: Vec = db - .prepare("SELECT key, value FROM settings") - .all() + let cache_key = SettingsCacheKey { + default_name: default_name.to_string(), + default_threshold, + default_retention, + default_username: default_username.to_string(), + }; + let current = now(); + if let Some(settings) = SETTINGS_CACHE.with(|cache| { + cache + .borrow() + .as_ref() + .filter(|entry| entry.key == cache_key && entry.expires_at > current) + .map(|entry| entry.settings.clone()) + }) { + return Ok(settings); + } + let raw = db + .prepare("SELECT value FROM settings WHERE id = 1") + .first::(Some("value")) .await? - .results()?; - let values: HashMap = - rows.into_iter().map(|row| (row.key, row.value)).collect(); - Ok(SettingsView { + .ok_or_else(|| worker::Error::RustError("站点设置尚未初始化".to_string()))?; + let values: HashMap = serde_json::from_str(&raw) + .map_err(|_| worker::Error::RustError("站点设置格式无效".to_string()))?; + let settings = SettingsView { site_name: values .get("site_name") .cloned() @@ -1070,7 +1280,15 @@ pub async fn settings( expiry_alert_days: integer_setting(&values, "expiry_alert_days", 7), cloudflare_account_id: string_setting(&values, "cloudflare_account_id", ""), cloudflare_api_token: string_setting(&values, "cloudflare_api_token", ""), - }) + }; + SETTINGS_CACHE.with(|cache| { + *cache.borrow_mut() = Some(SettingsCacheEntry { + key: cache_key, + expires_at: current + SETTINGS_CACHE_TTL_SECONDS, + settings: settings.clone(), + }); + }); + Ok(settings) } pub async fn update_settings( @@ -1078,18 +1296,10 @@ pub async fn update_settings( input: &SettingsInput, password_hash: Option<&str>, ) -> Result<()> { - let mut statements = Vec::new(); - let timestamp = now(); - let base = db.prepare( - "INSERT INTO settings(key, value, updated_at) VALUES (?1, ?2, ?3) \ - ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at", - ); + let mut updates = Vec::<(&'static str, String)>::new(); macro_rules! push_setting { ($key:expr, $value:expr) => { - statements.push( - base.clone() - .bind(&[text($key), text($value), number(timestamp)])?, - ); + updates.push(($key, $value.to_string())); }; } @@ -1226,8 +1436,19 @@ pub async fn update_settings( { push_setting!("cloudflare_api_token", value.trim()); } - if !statements.is_empty() { - db.batch(statements).await?; + if !updates.is_empty() { + let patch = updates + .into_iter() + .map(|(key, value)| (key.to_string(), serde_json::Value::String(value))) + .collect::>(); + db.prepare("UPDATE settings SET value=json_patch(value, ?1), updated_at=?2 WHERE id=1") + .bind(&[ + text(&serde_json::Value::Object(patch).to_string()), + number(now()), + ])? + .run() + .await?; + invalidate_settings_cache(); } Ok(()) } @@ -1332,23 +1553,83 @@ pub async fn cleanup_history(db: &D1Database, retention_days: i64) -> Result<()> let (cutoff, recent_cutoff) = history_cutoffs(now(), retention_days); let compact = db .prepare( - r#"INSERT INTO metric_history_hourly ( + r#"WITH ranked AS ( + SELECT *, (timestamp / 3600) * 3600 AS bucket, + ROW_NUMBER() OVER ( + PARTITION BY server_id, timestamp / 3600 + ORDER BY latest_timestamp DESC + ) AS latest_position + FROM metric_history WHERE timestamp >= ?1 AND timestamp < ?2 + ), metrics AS ( + SELECT + server_id, bucket, + AVG(cpu) AS cpu, AVG(load1) AS load1, AVG(load5) AS load5, + AVG(load15) AS load15, CAST(AVG(mem_used) AS INTEGER) AS mem_used, + MAX(mem_total) AS mem_total, CAST(AVG(swap_used) AS INTEGER) AS swap_used, + MAX(swap_total) AS swap_total, CAST(AVG(disk_used) AS INTEGER) AS disk_used, + MAX(disk_total) AS disk_total, MAX(net_in) AS net_in, MAX(net_out) AS net_out, + MAX(net_rx_total) AS net_rx_total, MAX(net_tx_total) AS net_tx_total, + CAST(MAX(processes) AS INTEGER) AS processes, + CAST(MAX(tcp_connections) AS INTEGER) AS tcp_connections, + CAST(MAX(udp_connections) AS INTEGER) AS udp_connections, + AVG(gpu_usage) AS gpu_usage, MAX(disk_read_bps) AS disk_read_bps, + MAX(disk_write_bps) AS disk_write_bps, + MAX(disk_read_iops) AS disk_read_iops, + MAX(disk_write_iops) AS disk_write_iops, + MAX(disk_await_ms) AS disk_await_ms, + MAX(disk_utilization) AS disk_utilization, + MAX(CASE WHEN latest_position = 1 THEN latest_timestamp END) AS latest_timestamp, + MAX(CASE WHEN latest_position = 1 THEN latest_json END) AS latest_json + FROM ranked GROUP BY server_id, bucket + ), latency_ranked AS ( + SELECT h.server_id, (h.timestamp / 3600) * 3600 AS bucket, + json_extract(j.value, '$.task_id') AS task_id, + CAST(json_extract(j.value, '$.latency_ms') AS REAL) AS latency_ms, + CAST(json_extract(j.value, '$.packet_loss') AS REAL) AS packet_loss, + CAST(json_extract(j.value, '$.latest_timestamp') AS INTEGER) AS latest_timestamp, + CAST(json_extract(j.value, '$.latest_latency_ms') AS REAL) AS latest_latency_ms, + CAST(json_extract(j.value, '$.latest_packet_loss') AS REAL) AS latest_packet_loss, + ROW_NUMBER() OVER ( + PARTITION BY h.server_id, h.timestamp / 3600, json_extract(j.value, '$.task_id') + ORDER BY CAST(json_extract(j.value, '$.latest_timestamp') AS INTEGER) DESC + ) AS latest_position + FROM metric_history h, json_each(h.latency_json) j + WHERE h.timestamp >= ?1 AND h.timestamp < ?2 + ), latency_tasks AS ( + SELECT server_id, bucket, task_id, + CASE WHEN SUM(CASE WHEN latency_ms >= 0 THEN 1 ELSE 0 END) > 0 + THEN AVG(CASE WHEN latency_ms >= 0 THEN latency_ms END) ELSE -1 END AS latency_ms, + AVG(packet_loss) AS packet_loss, + MAX(CASE WHEN latest_position = 1 THEN latest_timestamp END) AS latest_timestamp, + MAX(CASE WHEN latest_position = 1 THEN latest_latency_ms END) AS latest_latency_ms, + MAX(CASE WHEN latest_position = 1 THEN latest_packet_loss END) AS latest_packet_loss + FROM latency_ranked GROUP BY server_id, bucket, task_id + ), latency_packed AS ( + SELECT server_id, bucket, json_group_array(json_object( + 'task_id', task_id, 'timestamp', bucket, + 'latency_ms', latency_ms, 'packet_loss', packet_loss, + 'latest_timestamp', latest_timestamp, + 'latest_latency_ms', latest_latency_ms, + 'latest_packet_loss', latest_packet_loss + )) AS latency_json + FROM latency_tasks GROUP BY server_id, bucket + ) INSERT INTO metric_history_hourly ( server_id, timestamp, cpu, load1, load5, load15, mem_used, mem_total, swap_used, swap_total, disk_used, disk_total, net_in, net_out, net_rx_total, net_tx_total, processes, tcp_connections, udp_connections, gpu_usage, disk_read_bps, disk_write_bps, disk_read_iops, disk_write_iops, - disk_await_ms, disk_utilization + disk_await_ms, disk_utilization, latest_timestamp, latest_json, latency_json ) SELECT - server_id, (timestamp / 3600) * 3600, - AVG(cpu), AVG(load1), AVG(load5), AVG(load15), CAST(AVG(mem_used) AS INTEGER), - MAX(mem_total), CAST(AVG(swap_used) AS INTEGER), MAX(swap_total), - CAST(AVG(disk_used) AS INTEGER), MAX(disk_total), AVG(net_in), AVG(net_out), - MAX(net_rx_total), MAX(net_tx_total), CAST(AVG(processes) AS INTEGER), - CAST(AVG(tcp_connections) AS INTEGER), CAST(AVG(udp_connections) AS INTEGER), - AVG(gpu_usage), AVG(disk_read_bps), AVG(disk_write_bps), AVG(disk_read_iops), - AVG(disk_write_iops), AVG(disk_await_ms), AVG(disk_utilization) - FROM metric_history WHERE timestamp >= ?1 AND timestamp < ?2 - GROUP BY server_id, timestamp / 3600 + m.server_id, m.bucket, m.cpu, m.load1, m.load5, m.load15, m.mem_used, + m.mem_total, m.swap_used, m.swap_total, m.disk_used, m.disk_total, + m.net_in, m.net_out, m.net_rx_total, m.net_tx_total, m.processes, + m.tcp_connections, m.udp_connections, m.gpu_usage, m.disk_read_bps, + m.disk_write_bps, m.disk_read_iops, m.disk_write_iops, m.disk_await_ms, + m.disk_utilization, m.latest_timestamp, m.latest_json, + COALESCE(l.latency_json, '[]') + FROM metrics m LEFT JOIN latency_packed l + ON l.server_id = m.server_id AND l.bucket = m.bucket + WHERE true ON CONFLICT(server_id, timestamp) DO UPDATE SET cpu=excluded.cpu, load1=excluded.load1, load5=excluded.load5, load15=excluded.load15, mem_used=excluded.mem_used, mem_total=excluded.mem_total, @@ -1360,34 +1641,44 @@ pub async fn cleanup_history(db: &D1Database, retention_days: i64) -> Result<()> udp_connections=excluded.udp_connections, gpu_usage=excluded.gpu_usage, disk_read_bps=excluded.disk_read_bps, disk_write_bps=excluded.disk_write_bps, disk_read_iops=excluded.disk_read_iops, disk_write_iops=excluded.disk_write_iops, - disk_await_ms=excluded.disk_await_ms, disk_utilization=excluded.disk_utilization"#, + disk_await_ms=excluded.disk_await_ms, disk_utilization=excluded.disk_utilization, + latest_timestamp=excluded.latest_timestamp, latest_json=excluded.latest_json, + latency_json=excluded.latency_json"#, ) .bind(&[number(cutoff), number(recent_cutoff)])?; let delete_recent = db - .prepare("DELETE FROM metric_history WHERE timestamp < ?1") + .prepare( + "DELETE FROM metric_history AS h WHERE timestamp < ?1 AND timestamp < ( \ + SELECT MAX(newest.timestamp) FROM metric_history AS newest \ + WHERE newest.server_id = h.server_id \ + )", + ) .bind(&[number(recent_cutoff)])?; let delete_archive = db .prepare("DELETE FROM metric_history_hourly WHERE timestamp < ?1") .bind(&[number(cutoff)])?; db.batch(vec![compact, delete_recent, delete_archive]) .await?; - crate::latency::cleanup_history(db, cutoff).await?; Ok(()) } fn history_cutoffs(current: i64, retention_days: i64) -> (i64, i64) { let cutoff = current - retention_days.clamp(1, 365) * 86_400; - let recent_cutoff = (current - 7 * 86_400).max(cutoff); + let recent_cutoff = (current - 86_400).max(cutoff); (cutoff, recent_cutoff) } pub async fn clear_history(db: &D1Database) -> Result<()> { db.batch(vec![ - db.prepare("DELETE FROM metric_history"), + db.prepare( + "DELETE FROM metric_history AS h WHERE timestamp < ( \ + SELECT MAX(newest.timestamp) FROM metric_history AS newest \ + WHERE newest.server_id = h.server_id \ + )", + ), db.prepare("DELETE FROM metric_history_hourly"), ]) .await?; - crate::latency::clear_history(db).await?; Ok(()) } @@ -1455,7 +1746,13 @@ pub async fn database_stats(db: &D1Database, offline_threshold: i64) -> Result= ?1) AS online_count, + (SELECT COUNT(*) FROM servers s WHERE COALESCE( + (SELECT latest_timestamp FROM metric_history h + WHERE h.server_id = s.id ORDER BY timestamp DESC LIMIT 1), + (SELECT latest_timestamp FROM metric_history_hourly h + WHERE h.server_id = s.id ORDER BY timestamp DESC LIMIT 1), + 0 + ) >= ?1) AS online_count, ((SELECT COUNT(*) FROM metric_history) + (SELECT COUNT(*) FROM metric_history_hourly)) AS history_rows, (SELECT MIN(timestamp) FROM ( @@ -1474,35 +1771,31 @@ pub async fn database_stats(db: &D1Database, offline_threshold: i64) -> Result Result> { - let row: Option = db - .prepare("SELECT key, value FROM settings WHERE key = ?1") - .bind(&[text(key)])? - .first(None) - .await?; - Ok(row.map(|value| value.value)) + db.prepare("SELECT json_extract(value, ?1) AS value FROM settings WHERE id = 1") + .bind(&[text(&format!("$.{key}"))])? + .first::(Some("value")) + .await } pub async fn save_setting(db: &D1Database, key: &str, value: &str) -> Result<()> { - db.prepare( - "INSERT INTO settings(key, value, updated_at) VALUES (?1, ?2, ?3) \ - ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at", - ) - .bind(&[text(key), text(value), number(now())])? - .run() - .await?; + db.prepare("UPDATE settings SET value=json_set(value, ?1, ?2), updated_at=?3 WHERE id=1") + .bind(&[text(&format!("$.{key}")), text(value), number(now())])? + .run() + .await?; + invalidate_settings_cache(); Ok(()) } pub async fn increment_setting(db: &D1Database, key: &str) -> Result<()> { db.prepare( - "INSERT INTO settings(key, value, updated_at) VALUES (?1, '1', ?2) \ - ON CONFLICT(key) DO UPDATE SET \ - value = CAST(CAST(settings.value AS INTEGER) + 1 AS TEXT), \ - updated_at = excluded.updated_at", + "UPDATE settings SET value=json_set( \ + value, ?1, CAST(CAST(COALESCE(json_extract(value, ?1), '0') AS INTEGER) + 1 AS TEXT) \ + ), updated_at=?2 WHERE id=1", ) - .bind(&[text(key), number(now())])? + .bind(&[text(&format!("$.{key}")), number(now())])? .run() .await?; + invalidate_settings_cache(); Ok(()) } @@ -1518,8 +1811,10 @@ pub async fn update_expiry(db: &D1Database, id: &str, expires_at: i64) -> Result mod tests { use super::{ alert_window_covered, history_cutoffs, non_empty_string_setting, secret_for_api, - traffic_cycle_key, AlertMetricRow, SECRET_MASK, + traffic_cycle_key, AlertMetricRow, HistoryMetricAggregate, TrafficCounterState, + SECRET_MASK, }; + use crate::models::AgentReport; use std::collections::HashMap; fn row(samples: i64, last_timestamp: i64, report_interval: i64) -> AlertMetricRow { @@ -1585,11 +1880,11 @@ mod tests { let (thirty_days, thirty_days_recent) = history_cutoffs(current, 30); assert_eq!(thirty_days, current - 30 * 86_400); - assert_eq!(thirty_days_recent, current - 7 * 86_400); + assert_eq!(thirty_days_recent, current - 86_400); } #[test] - fn assigns_monthly_traffic_cycles_with_short_months() { + fn assigns_monthly_traffic_periods_with_short_months() { let timestamp = |year: i64, month: i64, day: i64| { let year = year - i64::from(month <= 2); let era = year.div_euclid(400); @@ -1604,4 +1899,133 @@ mod tests { assert_eq!(traffic_cycle_key(timestamp(2028, 2, 28), 31), 2028 * 12); assert_eq!(traffic_cycle_key(timestamp(2028, 2, 29), 31), 2028 * 12 + 1); } + + #[test] + fn folds_traffic_counters_and_ignores_replayed_samples() { + let mut state = TrafficCounterState::default(); + state.apply( + &AgentReport { + timestamp: 100, + net_rx_total: 1_000, + net_tx_total: 2_000, + ..AgentReport::default() + }, + 1, + ); + state.apply( + &AgentReport { + timestamp: 101, + net_rx_total: 1_300, + net_tx_total: 2_500, + ..AgentReport::default() + }, + 1, + ); + assert_eq!((state.used_rx, state.used_tx), (1_300, 2_500)); + + state.apply( + &AgentReport { + timestamp: 102, + net_rx_total: 20, + net_tx_total: 40, + ..AgentReport::default() + }, + 1, + ); + assert_eq!((state.used_rx, state.used_tx), (1_320, 2_540)); + + state.apply( + &AgentReport { + timestamp: 99, + net_rx_total: 9_999, + net_tx_total: 9_999, + ..AgentReport::default() + }, + 1, + ); + assert_eq!( + (state.timestamp, state.used_rx, state.used_tx), + (102, 1_320, 2_540) + ); + + state.apply( + &AgentReport { + timestamp: 103, + net_rx_total: 30, + net_tx_total: 50, + ..AgentReport::default() + }, + 2, + ); + assert_eq!((state.used_rx, state.used_tx), (0, 0)); + assert_eq!(state.reset_day, 2); + } + + #[test] + fn aggregates_history_with_averages_and_peaks() { + let reports = [ + AgentReport { + timestamp: 121, + cpu: 10.0, + mem_used: 100, + swap_used: 20, + net_in: 50.0, + net_out: 80.0, + processes: 4, + tcp_connections: 7, + udp_connections: 2, + disk_read_bps: 30.0, + disk_write_bps: 90.0, + disk_read_iops: 3.0, + disk_write_iops: 9.0, + disk_await_ms: 2.0, + disk_utilization: 20.0, + ..AgentReport::default() + }, + AgentReport { + timestamp: 124, + cpu: 30.0, + mem_used: 300, + mem_total: 1_000, + swap_used: 60, + swap_total: 500, + net_in: 200.0, + net_out: 40.0, + net_rx_total: 2_000, + net_tx_total: 3_000, + processes: 2, + tcp_connections: 5, + udp_connections: 8, + disk_read_bps: 100.0, + disk_write_bps: 20.0, + disk_read_iops: 10.0, + disk_write_iops: 2.0, + disk_await_ms: 5.0, + disk_utilization: 10.0, + ..AgentReport::default() + }, + ]; + let mut aggregate = HistoryMetricAggregate::default(); + aggregate.extend(&reports[..1]); + aggregate.extend(&reports[1..]); + let point = aggregate.point().unwrap(); + + assert_eq!(point.timestamp, 120); + assert_eq!(point.cpu, 20.0); + assert_eq!(point.mem_used, 200); + assert_eq!(point.swap_used, 40); + assert_eq!(point.net_in, 200.0); + assert_eq!(point.net_out, 80.0); + assert_eq!(point.processes, 4); + assert_eq!(point.tcp_connections, 7); + assert_eq!(point.udp_connections, 8); + assert_eq!(point.disk_read_bps, 100.0); + assert_eq!(point.disk_write_bps, 90.0); + assert_eq!(point.disk_read_iops, 10.0); + assert_eq!(point.disk_write_iops, 9.0); + assert_eq!(point.disk_await_ms, 5.0); + assert_eq!(point.disk_utilization, 20.0); + assert_eq!(point.net_rx_total, 2_000); + assert_eq!(point.net_tx_total, 3_000); + } } diff --git a/src/latency.rs b/src/latency.rs index 3afb4a1..3c65eb3 100644 --- a/src/latency.rs +++ b/src/latency.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap}; use serde::{Deserialize, Serialize}; use worker::{wasm_bindgen::JsValue, D1Database, Date, Result}; @@ -6,7 +6,6 @@ use worker::{wasm_bindgen::JsValue, D1Database, Date, Result}; use crate::models::{AgentLatencyResult, AgentLatencyTask, LatencyTaskInput}; pub const MAX_LATENCY_TASKS: usize = 128; -const MAX_HISTORY_ROWS_PER_REPORT: usize = 4096; const MAX_HISTORY_RESPONSE_ROWS: i64 = 4000; #[derive(Debug, Deserialize)] @@ -26,12 +25,96 @@ struct AssignmentRow { server_id: String, } -#[derive(Clone, Debug, PartialEq, Serialize)] -struct StoredLatencyResult { +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +pub(crate) struct StoredLatencyResult { task_id: String, timestamp: i64, latency_ms: f64, packet_loss: f64, + latest_timestamp: i64, + latest_latency_ms: f64, + latest_packet_loss: f64, +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +// Tuple layout keeps Durable Object WebSocket attachments below Cloudflare's +// 16 KiB serialized-attachment limit even with all 128 latency tasks enabled. +struct LatencyMetricAggregate(u64, u64, f64, f64, i64, f64, f64); + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct LatencyMetricAggregates { + #[serde(rename = "v")] + values: BTreeMap, +} + +impl LatencyMetricAggregates { + pub fn extend(&mut self, results: &[AgentLatencyResult], received_at: i64) { + for result in results { + if !self.values.contains_key(&result.task_id) && self.values.len() >= MAX_LATENCY_TASKS + { + continue; + } + let timestamp = if (result.timestamp - received_at).abs() <= 86_400 { + result.timestamp + } else { + received_at + }; + let aggregate = self.values.entry(result.task_id.clone()).or_default(); + aggregate.0 = aggregate.0.saturating_add(1); + aggregate.3 += result.packet_loss; + if result.latency_ms >= 0.0 { + aggregate.1 = aggregate.1.saturating_add(1); + aggregate.2 += result.latency_ms; + } + if timestamp >= aggregate.4 { + aggregate.4 = timestamp; + aggregate.5 = result.latency_ms; + aggregate.6 = result.packet_loss; + } + } + } + + pub fn merge(&mut self, other: Self) { + for (task_id, other) in other.values { + if !self.values.contains_key(&task_id) && self.values.len() >= MAX_LATENCY_TASKS { + continue; + } + let aggregate = self.values.entry(task_id).or_default(); + aggregate.0 = aggregate.0.saturating_add(other.0); + aggregate.1 = aggregate.1.saturating_add(other.1); + aggregate.2 += other.2; + aggregate.3 += other.3; + if other.4 > aggregate.4 { + aggregate.4 = other.4; + aggregate.5 = other.5; + aggregate.6 = other.6; + } + } + } + + fn rows_matching(&self, mut include: impl FnMut(&String) -> bool) -> Vec { + let mut history = Vec::new(); + for (task_id, aggregate) in self.values.iter().filter(|(task_id, _)| include(task_id)) { + history.push(StoredLatencyResult { + task_id: task_id.clone(), + timestamp: aggregate.4 / 60 * 60, + latency_ms: if aggregate.1 == 0 { + -1.0 + } else { + aggregate.2 / aggregate.1 as f64 + }, + packet_loss: aggregate.3 / aggregate.0.max(1) as f64, + latest_timestamp: aggregate.4, + latest_latency_ms: aggregate.5, + latest_packet_loss: aggregate.6, + }); + } + history + } + + pub fn stored_json(&self) -> Result { + Ok(serde_json::to_string(&self.rows_matching(|_| true))?) + } } #[derive(Debug, Clone, Serialize)] @@ -153,10 +236,10 @@ pub async fn create_task( number(timestamp), ])?, db.prepare( - "INSERT INTO latency_task_servers(task_id, server_id) \ - SELECT ?1, CAST(value AS TEXT) FROM json_each(?2)", + "INSERT INTO latency_task_servers(task_id, server_id, assigned_at) \ + SELECT ?1, CAST(value AS TEXT), ?3 FROM json_each(?2)", ) - .bind(&[text(id), text(&server_ids)])?, + .bind(&[text(id), text(&server_ids), number(timestamp)])?, ]; db.batch(statements).await?; Ok(()) @@ -198,43 +281,22 @@ pub async fn update_task( JsValue::from_bool(input.default_enabled), number(timestamp), ])?, - db.prepare("DELETE FROM latency_task_servers WHERE task_id = ?1") - .bind(&[text(id)])?, db.prepare( - "INSERT INTO latency_task_servers(task_id, server_id) \ - SELECT ?1, CAST(value AS TEXT) FROM json_each(?2)", + "DELETE FROM latency_task_servers WHERE task_id = ?1 AND server_id NOT IN ( \ + SELECT CAST(value AS TEXT) FROM json_each(?2) \ + )", ) .bind(&[text(id), text(&server_ids)])?, + db.prepare( + "INSERT OR IGNORE INTO latency_task_servers(task_id, server_id, assigned_at) \ + SELECT ?1, CAST(value AS TEXT), ?3 FROM json_each(?2)", + ) + .bind(&[text(id), text(&server_ids), number(timestamp)])?, ]; if reset_history { statements.push( - db.prepare("DELETE FROM latency_latest WHERE task_id = ?1") - .bind(&[text(id)])?, - ); - statements.push( - db.prepare("DELETE FROM latency_history WHERE task_id = ?1") - .bind(&[text(id)])?, - ); - } else { - statements.push( - db.prepare( - "DELETE FROM latency_latest WHERE task_id = ?1 AND NOT EXISTS ( \ - SELECT 1 FROM latency_task_servers a \ - WHERE a.task_id = latency_latest.task_id \ - AND a.server_id = latency_latest.server_id \ - )", - ) - .bind(&[text(id)])?, - ); - statements.push( - db.prepare( - "DELETE FROM latency_history WHERE task_id = ?1 AND NOT EXISTS ( \ - SELECT 1 FROM latency_task_servers a \ - WHERE a.task_id = latency_history.task_id \ - AND a.server_id = latency_history.server_id \ - )", - ) - .bind(&[text(id)])?, + db.prepare("UPDATE latency_task_servers SET assigned_at = ?2 WHERE task_id = ?1") + .bind(&[text(id), number(timestamp)])?, ); } db.batch(statements).await?; @@ -252,150 +314,65 @@ pub async fn delete_task(db: &D1Database, id: &str) -> Result { pub async fn assign_defaults(db: &D1Database, server_id: &str) -> Result<()> { db.prepare( - "INSERT OR IGNORE INTO latency_task_servers(task_id, server_id) \ - SELECT id, ?1 FROM latency_tasks WHERE default_enabled = 1", + "INSERT OR IGNORE INTO latency_task_servers(task_id, server_id, assigned_at) \ + SELECT id, ?1, ?2 FROM latency_tasks WHERE default_enabled = 1", ) - .bind(&[text(server_id)])? + .bind(&[text(server_id), number(crate::now())])? .run() .await?; Ok(()) } -pub async fn save_results( - db: &D1Database, - server_id: &str, - results: &[AgentLatencyResult], - received_at: i64, -) -> Result<()> { - if results.is_empty() { - return Ok(()); - } - let assigned: HashSet = tasks_for_server(db, server_id) - .await? - .into_iter() - .map(|task| task.id) - .collect(); - let (latest, history) = compact_results(results, &assigned, received_at); - if latest.is_empty() { - return Ok(()); - } - let latest_json = serde_json::to_string(&latest)?; - let history_json = serde_json::to_string(&history)?; - let latest_statement = db - .prepare( - "INSERT INTO latency_latest( \ - task_id, server_id, timestamp, latency_ms, packet_loss \ - ) SELECT \ - json_extract(value, '$.task_id'), ?1, \ - CAST(json_extract(value, '$.timestamp') AS INTEGER), \ - CAST(json_extract(value, '$.latency_ms') AS REAL), \ - CAST(json_extract(value, '$.packet_loss') AS REAL) \ - FROM json_each(?2) WHERE true \ - ON CONFLICT(task_id, server_id) DO UPDATE SET \ - timestamp=excluded.timestamp, latency_ms=excluded.latency_ms, \ - packet_loss=excluded.packet_loss \ - WHERE excluded.timestamp >= latency_latest.timestamp", - ) - .bind(&[text(server_id), text(&latest_json)])?; - let history_statement = db - .prepare( - "INSERT INTO latency_history( \ - task_id, server_id, timestamp, latency_ms, packet_loss \ - ) SELECT \ - json_extract(value, '$.task_id'), ?1, \ - CAST(json_extract(value, '$.timestamp') AS INTEGER), \ - CAST(json_extract(value, '$.latency_ms') AS REAL), \ - CAST(json_extract(value, '$.packet_loss') AS REAL) \ - FROM json_each(?2) WHERE true \ - ON CONFLICT(task_id, server_id, timestamp) DO UPDATE SET \ - latency_ms=excluded.latency_ms, packet_loss=excluded.packet_loss", - ) - .bind(&[text(server_id), text(&history_json)])?; - db.batch(vec![latest_statement, history_statement]).await?; - Ok(()) -} - +#[cfg(test)] fn compact_results( results: &[AgentLatencyResult], - assigned: &HashSet, + assigned: &std::collections::HashSet, received_at: i64, -) -> (Vec, Vec) { - let mut latest: HashMap = HashMap::new(); - let mut history = std::collections::BTreeMap::new(); - for result in results - .iter() - .filter(|result| assigned.contains(&result.task_id)) - { - let timestamp = if (result.timestamp - received_at).abs() <= 86_400 { - result.timestamp - } else { - received_at - }; - let stored = StoredLatencyResult { - task_id: result.task_id.clone(), - timestamp, - latency_ms: result.latency_ms, - packet_loss: result.packet_loss, - }; - if latest - .get(&result.task_id) - .is_none_or(|current| timestamp >= current.timestamp) - { - latest.insert(result.task_id.clone(), stored.clone()); - } - let minute = timestamp / 60 * 60; - history.insert( - (minute, result.task_id.clone()), - StoredLatencyResult { - timestamp: minute, - ..stored - }, - ); - } - let mut latest = latest.into_values().collect::>(); - latest.sort_by(|left, right| left.task_id.cmp(&right.task_id)); - let skip = history.len().saturating_sub(MAX_HISTORY_ROWS_PER_REPORT); - let history = history - .into_values() - .skip(skip) - .collect::>(); - (latest, history) +) -> Vec { + let mut aggregates = LatencyMetricAggregates::default(); + aggregates.extend(results, received_at); + aggregates.rows_matching(|task_id| assigned.contains(task_id)) } pub async fn latest_all(db: &D1Database) -> Result> { - db.prepare( - "SELECT a.task_id, a.server_id, t.name, t.task_type, t.target, t.port, \ - COALESCE(l.timestamp, 0) AS timestamp, \ - COALESCE(l.latency_ms, -1) AS latency_ms, \ - COALESCE(l.packet_loss, -1) AS packet_loss \ - FROM latency_task_servers a \ - INNER JOIN latency_tasks t ON t.id = a.task_id \ - LEFT JOIN latency_latest l \ - ON l.task_id = a.task_id AND l.server_id = a.server_id \ - ORDER BY t.sort_order ASC, t.created_at ASC", - ) - .all() - .await? - .results() + db.prepare(latest_query(None)).all().await?.results() } pub async fn latest_for_server(db: &D1Database, server_id: &str) -> Result> { - db.prepare( - "SELECT a.task_id, a.server_id, t.name, t.task_type, t.target, t.port, \ - COALESCE(l.timestamp, 0) AS timestamp, \ - COALESCE(l.latency_ms, -1) AS latency_ms, \ - COALESCE(l.packet_loss, -1) AS packet_loss \ - FROM latency_task_servers a \ - INNER JOIN latency_tasks t ON t.id = a.task_id \ - LEFT JOIN latency_latest l \ - ON l.task_id = a.task_id AND l.server_id = a.server_id \ - WHERE a.server_id = ?1 \ - ORDER BY t.sort_order ASC, t.created_at ASC", + db.prepare(latest_query(Some("a.server_id = ?1"))) + .bind(&[text(server_id)])? + .all() + .await? + .results() +} + +fn latest_query(filter: Option<&str>) -> String { + let filter = filter.map_or(String::new(), |value| format!("WHERE {value}")); + format!( + "WITH assignments AS ( \ + SELECT a.task_id, a.server_id, a.assigned_at, t.name, t.task_type, \ + t.target, t.port, t.sort_order, t.created_at \ + FROM latency_task_servers a \ + INNER JOIN latency_tasks t ON t.id = a.task_id {filter} \ + ), latest AS ( \ + SELECT assignments.*, COALESCE( \ + (SELECT j.value FROM metric_history h, json_each(h.latency_json) j \ + WHERE h.server_id = assignments.server_id \ + AND json_extract(j.value, '$.task_id') = assignments.task_id \ + AND CAST(json_extract(j.value, '$.latest_timestamp') AS INTEGER) >= assignments.assigned_at \ + ORDER BY h.timestamp DESC LIMIT 1), \ + (SELECT j.value FROM metric_history_hourly h, json_each(h.latency_json) j \ + WHERE h.server_id = assignments.server_id \ + AND json_extract(j.value, '$.task_id') = assignments.task_id \ + AND CAST(json_extract(j.value, '$.latest_timestamp') AS INTEGER) >= assignments.assigned_at \ + ORDER BY h.timestamp DESC LIMIT 1) \ + ) AS result FROM assignments \ + ) SELECT task_id, server_id, name, task_type, target, port, \ + COALESCE(CAST(json_extract(result, '$.latest_timestamp') AS INTEGER), 0) AS timestamp, \ + COALESCE(CAST(json_extract(result, '$.latest_latency_ms') AS REAL), -1) AS latency_ms, \ + COALESCE(CAST(json_extract(result, '$.latest_packet_loss') AS REAL), -1) AS packet_loss \ + FROM latest ORDER BY sort_order ASC, created_at ASC" ) - .bind(&[text(server_id)])? - .all() - .await? - .results() } pub async fn history(db: &D1Database, server_id: &str, hours: i64) -> Result> { @@ -411,16 +388,37 @@ pub async fn history(db: &D1Database, server_id: &str, hours: i64) -> Result= 0 THEN 1 ELSE 0 END) > 0 \ THEN AVG(CASE WHEN h.latency_ms >= 0 THEN h.latency_ms END) ELSE -1 END AS latency_ms, \ AVG(h.packet_loss) AS packet_loss \ - FROM latency_history h INNER JOIN latency_tasks t ON t.id = h.task_id \ + FROM ({source}) h INNER JOIN latency_tasks t ON t.id = h.task_id \ INNER JOIN latency_task_servers a \ ON a.task_id = h.task_id AND a.server_id = h.server_id \ - WHERE h.server_id = ?1 AND h.timestamp >= ?2 \ + WHERE h.server_id = ?1 AND h.timestamp >= ?2 AND h.latest_timestamp >= a.assigned_at \ GROUP BY h.task_id, h.server_id, t.name, t.task_type, t.target, t.port, h.timestamp / {bucket} \ ORDER BY timestamp ASC, t.sort_order ASC LIMIT {MAX_HISTORY_RESPONSE_ROWS}" ); @@ -447,22 +445,9 @@ fn history_bucket_seconds(hours: i64, task_count: i64) -> i64 { base.max(bounded) } -pub async fn cleanup_history(db: &D1Database, cutoff: i64) -> Result<()> { - db.prepare("DELETE FROM latency_history WHERE timestamp < ?1") - .bind(&[number(cutoff)])? - .run() - .await?; - Ok(()) -} - -pub async fn clear_history(db: &D1Database) -> Result<()> { - db.prepare("DELETE FROM latency_history").run().await?; - Ok(()) -} - #[cfg(test)] mod tests { - use super::{compact_results, history_bucket_seconds, MAX_HISTORY_ROWS_PER_REPORT}; + use super::{compact_results, history_bucket_seconds, MAX_LATENCY_TASKS}; use crate::models::AgentLatencyResult; use std::collections::HashSet; @@ -489,16 +474,18 @@ mod tests { packet_loss: 0.0, }, ]; - let (latest, history) = compact_results(&results, &assigned, 150); - assert_eq!(latest.len(), 1); - assert_eq!(latest[0].timestamp, 149); + let history = compact_results(&results, &assigned, 150); assert_eq!(history.len(), 1); assert_eq!(history[0].timestamp, 120); - assert_eq!(history[0].latency_ms, 20.0); + assert_eq!(history[0].latency_ms, 15.0); + assert_eq!(history[0].packet_loss, 12.5); + assert_eq!(history[0].latest_timestamp, 149); + assert_eq!(history[0].latest_latency_ms, 20.0); + assert_eq!(history[0].latest_packet_loss, 25.0); } #[test] - fn bounds_compacted_latency_history() { + fn bounds_compacted_latency_history_by_task() { let assigned = (0..128) .map(|index| format!("task-{index}")) .collect::>(); @@ -512,8 +499,8 @@ mod tests { }) }) .collect::>(); - let (_, history) = compact_results(&results, &assigned, results.last().unwrap().timestamp); - assert_eq!(history.len(), MAX_HISTORY_ROWS_PER_REPORT); + let history = compact_results(&results, &assigned, results.last().unwrap().timestamp); + assert_eq!(history.len(), MAX_LATENCY_TASKS); assert_eq!( history.last().unwrap().timestamp, results.last().unwrap().timestamp / 60 * 60 diff --git a/src/lib.rs b/src/lib.rs index 038d046..d40ab3a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,11 +28,14 @@ use crate::models::{ pub(crate) const ADMIN_HTML: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/admin.html")); pub(crate) const ADMIN_SCRIPT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/admin.js")); pub(crate) const ADMIN_STYLE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/admin.css")); -const HISTORY_CACHE_SECONDS: i64 = 30; pub(crate) const API_JSON_MAX_BYTES: usize = 1024 * 1024; pub(crate) const AGENT_JSON_MAX_BYTES: usize = 16 * 1024 * 1024; pub(crate) const MAX_AGENT_SAMPLES: usize = 720; pub(crate) const MAX_AGENT_LATENCY_RESULTS: usize = 4096; +pub(crate) const VERSION: &str = match option_env!("NODEFLARE_VERSION") { + Some(version) => version, + None => env!("CARGO_PKG_VERSION"), +}; #[derive(Serialize)] struct PublicServer { @@ -286,6 +289,11 @@ fn remote_theme_failure(message: &str) -> Result { secure_public_response(response) } +fn rewrite_remote_theme_index(html: &str, asset_prefix: &str) -> String { + html.replace("\"/assets/", &format!("\"{asset_prefix}")) + .replace("'/assets/", &format!("'{asset_prefix}")) +} + async fn remote_theme_response(path: &str, base: &str) -> Result> { let (remote_url, relative) = if path == "/" || path == "/index.html" || path.starts_with("/instance/") { @@ -300,7 +308,8 @@ async fn remote_theme_response(path: &str, base: &str) -> Result Result Result Option<&'static str> { if !(15..=3600).contains(&input.report_interval) { return Some("上报间隔应为 15 至 3600 秒"); } - if !(2..=60).contains(&input.collect_interval) || input.collect_interval > input.report_interval + if !(1..=60).contains(&input.collect_interval) || input.collect_interval > input.report_interval { - return Some("采样间隔应为 2 至 60 秒且不能大于上报间隔"); + return Some("采样间隔应为 1 至 60 秒且不能大于上报间隔"); + } + if (input.report_interval + input.collect_interval - 1) / input.collect_interval + > MAX_AGENT_SAMPLES as i64 + { + return Some("每个上报周期最多采集 720 个样本"); } if input.network_interface.chars().count() > 160 { return Some("统计网卡配置字段过长"); @@ -853,7 +872,7 @@ fn rate_limit_binding(method: Method, path: &str) -> Option<&'static str> { if method == Method::Post && matches!(path, "/api/admin/login" | "/api/turnstile/verify") { Some("AUTH_RATE_LIMITER") } else if (method == Method::Post && path == "/api/agent/report") - || (method == Method::Get && path == "/api/agent/live") + || (method == Method::Get && matches!(path, "/api/agent/live" | "/api/agent/config")) { Some("AGENT_RATE_LIMITER") } else if path.starts_with("/api/") { @@ -927,6 +946,15 @@ pub(crate) fn history_cache_key( url.to_string() } +pub(crate) fn history_cache_seconds(hours: i64) -> i64 { + match hours { + 120.. => 10 * 60, + 60.. => 5 * 60, + 30.. => 3 * 60, + _ => 60, + } +} + async fn cached_history_response(key: &str) -> Option { match Cache::default().get(key, false).await { Ok(Some(response)) => { @@ -946,7 +974,7 @@ async fn cached_history_response(key: &str) -> Option { } } -async fn store_history_response(key: &str, response: &mut Response) { +async fn store_history_response(key: &str, response: &mut Response, ttl: i64) { if response.headers_mut().set("X-Cache", "MISS").is_err() { return; } @@ -957,7 +985,7 @@ async fn store_history_response(key: &str, response: &mut Response) { .headers_mut() .set( "Cache-Control", - &format!("public, max-age={HISTORY_CACHE_SECONDS}"), + &format!("public, max-age={}", ttl.clamp(60, 10 * 60)), ) .is_err() { @@ -996,6 +1024,9 @@ async fn handle(req: Request, env: Env, ctx: Context) -> Result { if method == Method::Post && path == "/api/agent/report" { return routes::agent::report(req, env, ctx, &database).await; } + if method == Method::Get && path == "/api/agent/config" { + return routes::agent::config(&req, &database).await; + } if method == Method::Get && path == "/api/agent/live" { return routes::agent::live_websocket(req, &env, &database).await; } @@ -1156,7 +1187,7 @@ async fn scheduled(_event: ScheduledEvent, env: Env, _ctx: ScheduleContext) { console_error!("server renewal failed: {err}"); } if let Ok(settings) = settings { - if let Err(err) = notify::check_alerts(&database, &settings).await { + if let Err(err) = notify::check_alerts(&database, &env, &settings).await { console_error!("alert check failed: {err}"); } } @@ -1165,12 +1196,13 @@ async fn scheduled(_event: ScheduledEvent, env: Env, _ctx: ScheduleContext) { #[cfg(test)] mod tests { use super::{ - allow_on_rate_limit_failure, history_cache_key, rate_limit_binding, same_origin, - submitted_secret, valid_agent_mirror, valid_cloudflare_account_id, - valid_cloudflare_api_token, valid_password_derived, valid_ping_target, valid_server_price, - validate_latency_task, ADMIN_HTML, ADMIN_SCRIPT, ADMIN_STYLE, + allow_on_rate_limit_failure, history_cache_key, history_cache_seconds, rate_limit_binding, + rewrite_remote_theme_index, same_origin, submitted_secret, valid_agent_mirror, + valid_cloudflare_account_id, valid_cloudflare_api_token, valid_password_derived, + valid_ping_target, valid_server_price, validate_latency_task, validate_server, ADMIN_HTML, + ADMIN_SCRIPT, ADMIN_STYLE, }; - use crate::{db::SECRET_MASK, models::LatencyTaskInput}; + use crate::{db::SECRET_MASK, models::LatencyTaskInput, models::ServerInput}; use worker::{Method, Url}; #[test] @@ -1187,6 +1219,10 @@ mod tests { rate_limit_binding(Method::Get, "/api/agent/live"), Some("AGENT_RATE_LIMITER") ); + assert_eq!( + rate_limit_binding(Method::Get, "/api/agent/config"), + Some("AGENT_RATE_LIMITER") + ); assert_eq!( rate_limit_binding(Method::Get, "/api/history/node-a"), Some("API_RATE_LIMITER") @@ -1206,6 +1242,10 @@ mod tests { ); assert!(same_origin("https://status.example", &request_url)); assert!(!same_origin("https://other.example", &request_url)); + assert_eq!(history_cache_seconds(1), 60); + assert_eq!(history_cache_seconds(30), 180); + assert_eq!(history_cache_seconds(60), 300); + assert_eq!(history_cache_seconds(120), 600); } #[test] @@ -1217,6 +1257,39 @@ mod tests { assert!(!valid_server_price(f64::NAN)); } + #[test] + fn bounds_samples_buffered_between_reports() { + let mut server = ServerInput { + name: "node-a".to_string(), + region: String::new(), + group_name: String::new(), + tags: String::new(), + hidden: false, + expires_at: None, + traffic_limit: 0, + traffic_limit_type: "sum".to_string(), + price: 0.0, + billing_cycle: 30, + currency: "CNY".to_string(), + auto_renewal: false, + network_interface: String::new(), + reset_day: 1, + report_interval: 3600, + collect_interval: 5, + rx_correction: 0, + tx_correction: 0, + agent_mirror: String::new(), + offline_notify_disabled: false, + auto_update: true, + }; + assert_eq!(validate_server(&server), None); + server.collect_interval = 4; + assert_eq!( + validate_server(&server), + Some("每个上报周期最多采集 720 个样本") + ); + } + #[test] fn validates_agent_mirrors() { for mirror in [ @@ -1345,4 +1418,13 @@ mod tests { assert!(!ADMIN_SCRIPT.is_empty()); assert!(!ADMIN_STYLE.is_empty()); } + + #[test] + fn namespaces_remote_theme_assets() { + let html = r#""#; + assert_eq!( + rewrite_remote_theme_index(html, "/__theme-active/assets/"), + r#""# + ); + } } diff --git a/src/live.rs b/src/live.rs index 2d94a74..12c0fa3 100644 --- a/src/live.rs +++ b/src/live.rs @@ -1,11 +1,164 @@ +use std::cell::{Cell, RefCell}; +use std::collections::HashSet; + use serde::{Deserialize, Serialize}; use worker::*; -use crate::db::AgentLiveContext; -use crate::models::{AgentLatencyResult, AgentReport}; +use crate::db::{AgentLiveContext, AlertMetricRow, HistoryMetricAggregate}; +use crate::latency::LatencyMetricAggregates; +use crate::models::{AgentLatencyResult, AgentReport, AlertRuleView, HistoryPoint, ServerView}; const MAX_LIVE_SAMPLES: usize = 720; const MAX_LIVE_LATENCY_RESULTS: usize = 4096; +const LIVE_REPORT_DIVISOR: i64 = 15; +const MAX_SERIALIZED_ATTACHMENT_BYTES: usize = 15 * 1024; +const MAX_CACHED_LIVE_SAMPLES: usize = 32; +const MAX_CACHED_LIVE_BYTES: usize = 12 * 1024; +const MAX_CACHED_REPLAY_UPDATES: usize = 64; +const MAX_CACHED_REPLAY_BYTES: usize = 256 * 1024; +const CACHED_LIVE_TTL_SECONDS: i64 = 5 * 60; +const ALERT_WINDOW_SECONDS: i64 = 24 * 60 * 60; +const MAX_ALERT_WINDOW_SAMPLES: usize = 24 * 60 + 1; +const ALERT_STORAGE_PREFIX: &str = "resource-alert:"; +const ALERT_COLLECTION_STATE_KEY: &str = "resource-alert-enabled-until"; +const ALERT_COLLECTION_LEASE_SECONDS: i64 = 10 * 60; + +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +struct AlertWindowSample(i64, u32, f64, f64, f64, f64, f64); + +impl AlertWindowSample { + fn from_history(point: &HistoryPoint) -> Self { + let memory = if point.mem_total > 0 { + point.mem_used as f64 * 100.0 / point.mem_total as f64 + } else { + 0.0 + }; + let disk = if point.disk_total > 0 { + point.disk_used as f64 * 100.0 / point.disk_total as f64 + } else { + 0.0 + }; + Self( + point.timestamp / 60 * 60, + 1, + point.cpu, + memory, + disk, + point.net_in / 1_048_576.0, + point.net_out / 1_048_576.0, + ) + } + + fn merge(&mut self, other: &Self) { + self.1 = self.1.saturating_add(other.1); + self.2 += other.2; + self.3 += other.3; + self.4 += other.4; + self.5 = self.5.max(other.5); + self.6 = self.6.max(other.6); + } + + fn metric(&self, metric: &str) -> Option { + let count = f64::from(self.1.max(1)); + match metric { + "cpu" => Some(self.2 / count), + "memory" => Some(self.3 / count), + "disk" => Some(self.4 / count), + "net_in" => Some(self.5), + "net_out" => Some(self.6), + _ => None, + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +struct AlertRecordRequest { + server_id: String, + point: HistoryPoint, +} + +#[derive(Debug, Deserialize, Serialize)] +struct AlertEvaluationServer { + id: String, + name: String, + report_interval: i64, +} + +#[derive(Debug, Deserialize, Serialize)] +struct AlertEvaluationRequest { + current_time: i64, + rules: Vec, + servers: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct AlertEvaluationValue { + pub rule_id: String, + pub row: AlertMetricRow, +} + +fn alert_storage_key(server_id: &str) -> String { + format!("{ALERT_STORAGE_PREFIX}{server_id}") +} + +fn update_alert_window(samples: &mut Vec, point: &HistoryPoint) { + let sample = AlertWindowSample::from_history(point); + let newest = samples + .last() + .map(|value| value.0) + .unwrap_or(sample.0) + .max(sample.0); + let cutoff = newest - ALERT_WINDOW_SECONDS; + if sample.0 < cutoff { + return; + } + match samples.binary_search_by_key(&sample.0, |value| value.0) { + Ok(index) => samples[index].merge(&sample), + Err(index) => samples.insert(index, sample), + } + samples.retain(|value| value.0 >= cutoff); + if samples.len() > MAX_ALERT_WINDOW_SAMPLES { + let overflow = samples.len() - MAX_ALERT_WINDOW_SAMPLES; + samples.drain(..overflow); + } +} + +fn evaluate_alert_window( + samples: &[AlertWindowSample], + rule: &AlertRuleView, + server: &AlertEvaluationServer, + current_time: i64, +) -> Option { + let since = current_time - rule.duration_minutes.clamp(1, 1440) * 60; + let mut values = samples + .iter() + .filter(|sample| sample.0 >= since && sample.0 <= current_time) + .filter_map(|sample| sample.metric(&rule.metric).map(|value| (sample.0, value))); + let (first_timestamp, first_value) = values.next()?; + let mut count = 1_i64; + let mut last_timestamp = first_timestamp; + let mut total = first_value; + let mut minimum = first_value; + for (timestamp, value) in values { + count += 1; + last_timestamp = timestamp; + total += value; + minimum = minimum.min(value); + } + Some(AlertMetricRow { + server_id: server.id.clone(), + name: server.name.clone(), + value: if rule.aggregation == "continuous" { + minimum + } else { + total / count as f64 + }, + sample_count: count, + first_timestamp, + last_timestamp, + report_interval: server.report_interval, + }) +} #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] @@ -15,39 +168,153 @@ struct AgentLiveBatch { samples: Vec, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] struct TrafficAttachment { + #[serde(rename = "d")] reset_day: i64, + #[serde(rename = "sd")] + state_reset_day: i64, + #[serde(rename = "st")] + timestamp: i64, + #[serde(rename = "c")] cycle_key: i64, + #[serde(rename = "rr")] raw_rx: i64, + #[serde(rename = "rt")] raw_tx: i64, + #[serde(rename = "ur")] used_rx: i64, + #[serde(rename = "ut")] used_tx: i64, + #[serde(rename = "rc")] rx_correction: i64, + #[serde(rename = "tc")] tx_correction: i64, } -#[derive(Debug, Deserialize, Serialize)] -struct SocketAttachment { - role: String, - server_id: Option, - hidden: bool, - report_interval: i64, - collect_interval: i64, - last_d1_write_at: i64, - traffic: Option, +#[derive(Debug, Clone, Deserialize, Serialize)] +struct CachedLiveSample { + ts: i64, + data: serde_json::Value, } -fn live_interval_ms(collect_interval: i64) -> i64 { - collect_interval.clamp(2, 60) * 1000 +#[derive(Clone, Debug, Deserialize, Serialize)] +struct SocketAttachment { + #[serde(rename = "r")] + role: String, + #[serde(rename = "s")] + server_id: Option, + #[serde(rename = "h")] + hidden: bool, + #[serde(rename = "ri")] + report_interval: i64, + #[serde(rename = "ci")] + collect_interval: i64, + #[serde(rename = "dw")] + last_d1_write_at: i64, + #[serde(rename = "df")] + d1_flush: Option, + #[serde(rename = "tr")] + traffic: Option, + #[serde(rename = "mh")] + history_aggregate: HistoryMetricAggregate, + #[serde(rename = "lh")] + latency_aggregates: LatencyMetricAggregates, + #[serde(rename = "ls")] + latest_samples: Vec, + #[serde(rename = "lr")] + latest_received_at: i64, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct D1FlushSnapshot { + #[serde(rename = "h")] + history: HistoryMetricAggregate, + #[serde(rename = "l")] + latency: LatencyMetricAggregates, +} + +fn begin_d1_flush(attachment: &mut SocketAttachment) -> Option { + if let Some(snapshot) = attachment.d1_flush.clone() { + return Some(snapshot); + } + attachment.history_aggregate.point()?; + let snapshot = D1FlushSnapshot { + history: std::mem::take(&mut attachment.history_aggregate), + latency: std::mem::take(&mut attachment.latency_aggregates), + }; + attachment.d1_flush = Some(snapshot.clone()); + Some(snapshot) +} + +fn finish_d1_flush(attachment: &mut SocketAttachment, succeeded: bool, persisted_at: i64) { + let Some(snapshot) = attachment.d1_flush.take() else { + return; + }; + if succeeded { + attachment.last_d1_write_at = persisted_at; + } else { + attachment.history_aggregate.merge(snapshot.history); + attachment.latency_aggregates.merge(snapshot.latency); + } +} + +fn agent_wss_interval_ms(attachment: &SocketAttachment, realtime_active: bool) -> i64 { + let collect = attachment.collect_interval.clamp(1, 60); + let report_interval = attachment.report_interval.clamp(15, 3600); + let realtime_seconds = + report_interval.saturating_add(LIVE_REPORT_DIVISOR - 1) / LIVE_REPORT_DIVISOR; + let realtime = realtime_seconds.clamp(1, 60).max(collect) * 1000; + if realtime_active { + return realtime; + } + report_interval * 1000 +} + +fn next_d1_write_ms(attachment: &SocketAttachment, now: i64) -> i64 { + if attachment.d1_flush.is_some() { + return 0; + } + let interval = attachment.report_interval.clamp(15, 3600); + if attachment.last_d1_write_at <= 0 { + return interval * 1000; + } + attachment + .last_d1_write_at + .saturating_add(interval) + .saturating_sub(now) + .max(0) + .saturating_mul(1000) +} + +fn live_ack_payload( + timestamp: i64, + persisted: bool, + next_d1_write_after_ms: i64, + next_wss_report_after_ms: i64, + realtime_hint: bool, +) -> Result { + Ok(serde_json::to_string(&serde_json::json!({ + "type": "ack", + "ts": timestamp, + "persisted": persisted, + "nextD1WriteAfterMs": next_d1_write_after_ms, + "nextWssReportAfterMs": next_wss_report_after_ms, + "realtimeHint": realtime_hint + }))?) } fn apply_live_traffic(report: &mut AgentReport, traffic: &mut TrafficAttachment) { + if report.timestamp <= traffic.timestamp { + report.net_rx_total = traffic.used_rx.saturating_add(traffic.rx_correction).max(0); + report.net_tx_total = traffic.used_tx.saturating_add(traffic.tx_correction).max(0); + return; + } let cycle_key = crate::db::traffic_cycle_key(report.timestamp, traffic.reset_day); - if cycle_key != traffic.cycle_key { - traffic.cycle_key = cycle_key; - traffic.raw_rx = report.net_rx_total; - traffic.raw_tx = report.net_tx_total; + if traffic.timestamp <= 0 { + traffic.used_rx = report.net_rx_total; + traffic.used_tx = report.net_tx_total; + } else if cycle_key != traffic.cycle_key || traffic.reset_day != traffic.state_reset_day { traffic.used_rx = 0; traffic.used_tx = 0; } else { @@ -63,27 +330,22 @@ fn apply_live_traffic(report: &mut AgentReport, traffic: &mut TrafficAttachment) }; traffic.used_rx = traffic.used_rx.saturating_add(rx_delta); traffic.used_tx = traffic.used_tx.saturating_add(tx_delta); - traffic.raw_rx = report.net_rx_total; - traffic.raw_tx = report.net_tx_total; } + traffic.cycle_key = cycle_key; + traffic.state_reset_day = traffic.reset_day; + traffic.timestamp = report.timestamp; + traffic.raw_rx = report.net_rx_total; + traffic.raw_tx = report.net_tx_total; report.net_rx_total = traffic.used_rx.saturating_add(traffic.rx_correction).max(0); report.net_tx_total = traffic.used_tx.saturating_add(traffic.tx_correction).max(0); } -fn batch_update_payload( - server_id: &str, - reports: &[AgentReport], - traffic: &mut TrafficAttachment, -) -> Result { - batch_update_payload_at(server_id, reports, traffic, crate::now()) -} - -fn batch_update_payload_at( +fn batch_update_parts_at( server_id: &str, reports: &[AgentReport], traffic: &mut TrafficAttachment, envelope_timestamp: i64, -) -> Result { +) -> Result<(String, Vec)> { let mut reports = reports.to_vec(); reports.sort_by_key(|report| report.timestamp); let mut samples = Vec::with_capacity(reports.len()); @@ -94,16 +356,79 @@ fn batch_update_payload_at( if let Some(data) = data.as_object_mut() { data.remove("timestamp"); } - samples.push(serde_json::json!({ "ts": timestamp, "data": data })); + samples.push(CachedLiveSample { + ts: timestamp, + data, + }); } - Ok(serde_json::to_string(&serde_json::json!({ + let payload = serde_json::to_string(&serde_json::json!({ "type": "batchUpdate", "ts": envelope_timestamp, "updates": [{ "serverId": server_id, - "samples": samples + "samples": &samples }] - }))?) + }))?; + Ok((payload, samples)) +} + +fn trim_cached_live_samples(samples: &mut Vec) { + if samples.len() > MAX_CACHED_LIVE_SAMPLES { + let overflow = samples.len() - MAX_CACHED_LIVE_SAMPLES; + samples.drain(..overflow); + } + while samples.len() > 1 + && serde_json::to_vec(samples) + .map(|encoded| encoded.len() > MAX_CACHED_LIVE_BYTES) + .unwrap_or(true) + { + samples.remove(0); + } + if samples.len() == 1 + && serde_json::to_vec(samples) + .map(|encoded| encoded.len() > MAX_CACHED_LIVE_BYTES) + .unwrap_or(true) + { + samples.clear(); + } +} + +fn trim_socket_attachment(attachment: &mut SocketAttachment) -> bool { + while !attachment.latest_samples.is_empty() + && serde_json::to_vec(attachment) + .map(|encoded| encoded.len() > MAX_SERIALIZED_ATTACHMENT_BYTES) + .unwrap_or(true) + { + attachment.latest_samples.remove(0); + } + serde_json::to_vec(attachment) + .map(|encoded| encoded.len() <= MAX_SERIALIZED_ATTACHMENT_BYTES) + .unwrap_or(false) +} + +fn cached_update_value( + attachment: &SocketAttachment, + now: i64, + server_id: Option<&str>, +) -> Option { + let agent_server_id = attachment.server_id.as_deref()?; + if attachment.role != "agent" + || attachment.hidden + || server_id.is_some_and(|requested| requested != agent_server_id) + || attachment.latest_received_at <= 0 + || attachment.latest_samples.is_empty() + { + return None; + } + let age = now.saturating_sub(attachment.latest_received_at); + if !(0..=CACHED_LIVE_TTL_SECONDS).contains(&age) { + return None; + } + Some(serde_json::json!({ + "serverId": agent_server_id, + "reportAgeMs": age.saturating_mul(1000), + "samples": &attachment.latest_samples + })) } fn agent_attachment( @@ -116,10 +441,13 @@ fn agent_attachment( server_id: Some(server_id), hidden, report_interval: context.report_interval.clamp(15, 3600), - collect_interval: context.collect_interval.clamp(2, 60), + collect_interval: context.collect_interval.clamp(1, 60), last_d1_write_at: context.last_persisted_at, + d1_flush: None, traffic: Some(TrafficAttachment { reset_day: context.reset_day.clamp(1, 31), + state_reset_day: context.traffic_reset_day.clamp(1, 31), + timestamp: context.traffic_timestamp, cycle_key: context.cycle_key, raw_rx: context.raw_rx, raw_tx: context.raw_tx, @@ -128,6 +456,10 @@ fn agent_attachment( rx_correction: context.rx_correction, tx_correction: context.tx_correction, }), + history_aggregate: HistoryMetricAggregate::default(), + latency_aggregates: LatencyMetricAggregates::default(), + latest_samples: Vec::new(), + latest_received_at: 0, } } @@ -144,6 +476,8 @@ fn live_context_from_headers(req: &Request) -> Result { collect_interval: required_i64_header(req, "X-Live-Collect-Interval")?, reset_day: required_i64_header(req, "X-Live-Reset-Day")?, cycle_key: required_i64_header(req, "X-Live-Cycle-Key")?, + traffic_reset_day: required_i64_header(req, "X-Live-Traffic-Reset-Day")?, + traffic_timestamp: required_i64_header(req, "X-Live-Traffic-Timestamp")?, raw_rx: required_i64_header(req, "X-Live-Raw-Rx")?, raw_tx: required_i64_header(req, "X-Live-Raw-Tx")?, used_rx: required_i64_header(req, "X-Live-Used-Rx")?, @@ -162,7 +496,12 @@ fn dashboard_attachment() -> SocketAttachment { report_interval: 0, collect_interval: 0, last_d1_write_at: 0, + d1_flush: None, traffic: None, + history_aggregate: HistoryMetricAggregate::default(), + latency_aggregates: LatencyMetricAggregates::default(), + latest_samples: Vec::new(), + latest_received_at: 0, } } @@ -170,6 +509,200 @@ fn dashboard_attachment() -> SocketAttachment { pub struct LiveHub { state: State, env: Env, + alert_collection_until: Cell, + active_d1_flushes: RefCell>, +} + +impl LiveHub { + async fn alert_collection_enabled(&self, current_time: i64) -> Result { + let mut enabled_until = self.alert_collection_until.get(); + if enabled_until < 0 { + enabled_until = self + .state + .storage() + .get::(ALERT_COLLECTION_STATE_KEY) + .await? + .unwrap_or(0); + self.alert_collection_until.set(enabled_until); + } + Ok(enabled_until >= current_time) + } + + async fn configure_alert_collection(&self, enabled: bool, current_time: i64) -> Result<()> { + let enabled_until = if enabled { + current_time.saturating_add(ALERT_COLLECTION_LEASE_SECONDS) + } else { + 0 + }; + self.state + .storage() + .put(ALERT_COLLECTION_STATE_KEY, enabled_until) + .await?; + self.alert_collection_until.set(enabled_until); + Ok(()) + } + + fn dashboard_active_for(&self, server_id: &str) -> bool { + !self.state.get_websockets_with_tag("all").is_empty() + || !self + .state + .get_websockets_with_tag(&format!("server:{server_id}")) + .is_empty() + } + + fn hint_agents(&self, server_id: Option<&str>) { + let sockets = server_id.map_or_else( + || self.state.get_websockets_with_tag("agents"), + |server_id| { + self.state + .get_websockets_with_tag(&format!("agent:{server_id}")) + }, + ); + let now = crate::now(); + for socket in sockets { + let Ok(Some(attachment)) = socket.deserialize_attachment::() else { + continue; + }; + let Some(agent_server_id) = attachment.server_id.as_deref() else { + continue; + }; + let realtime_active = !attachment.hidden && self.dashboard_active_for(agent_server_id); + let Ok(payload) = live_ack_payload( + now, + false, + next_d1_write_ms(&attachment, now), + agent_wss_interval_ms(&attachment, realtime_active), + true, + ) else { + continue; + }; + if socket.send_with_str(&payload).is_err() { + let _ = socket.close(Some(1011), Some("send failed")); + } + } + } + + fn send_cached_updates(dashboard: &WebSocket, now: i64, updates: &[serde_json::Value]) -> bool { + let Ok(payload) = serde_json::to_string(&serde_json::json!({ + "type": "batchUpdate", + "ts": now, + "cached": true, + "updates": updates + })) else { + return false; + }; + if dashboard.send_with_str(&payload).is_ok() { + return true; + } + let _ = dashboard.close(Some(1011), Some("send failed")); + false + } + + fn replay_latest(&self, dashboard: &WebSocket, server_id: Option<&str>) { + let sockets = server_id.map_or_else( + || self.state.get_websockets_with_tag("agents"), + |server_id| { + self.state + .get_websockets_with_tag(&format!("agent:{server_id}")) + }, + ); + let now = crate::now(); + let mut updates = Vec::new(); + let mut encoded_bytes = 0_usize; + for socket in sockets { + let Some(update) = socket + .deserialize_attachment::() + .ok() + .flatten() + .and_then(|attachment| cached_update_value(&attachment, now, server_id)) + else { + continue; + }; + let Ok(update_bytes) = serde_json::to_vec(&update).map(|encoded| encoded.len()) else { + continue; + }; + if !updates.is_empty() + && (updates.len() >= MAX_CACHED_REPLAY_UPDATES + || encoded_bytes.saturating_add(update_bytes) > MAX_CACHED_REPLAY_BYTES) + { + if !Self::send_cached_updates(dashboard, now, &updates) { + return; + } + updates.clear(); + encoded_bytes = 0; + } + encoded_bytes = encoded_bytes.saturating_add(update_bytes); + updates.push(update); + } + if !updates.is_empty() { + Self::send_cached_updates(dashboard, now, &updates); + } + } + + async fn record_alert_point(&self, server_id: &str, point: &HistoryPoint) -> Result<()> { + if server_id.is_empty() || server_id.len() > 80 || server_id.contains('/') { + return Err(Error::RustError( + "invalid alert server identity".to_string(), + )); + } + if !self.alert_collection_enabled(crate::now()).await? { + return Ok(()); + } + let key = alert_storage_key(server_id); + let storage = self.state.storage(); + let mut samples = storage + .get::>(&key) + .await? + .unwrap_or_default(); + update_alert_window(&mut samples, point); + storage.put(&key, &samples).await + } + + async fn evaluate_alerts( + &self, + request: &AlertEvaluationRequest, + ) -> Result> { + let enabled = request.rules.iter().any(|rule| rule.enabled); + self.configure_alert_collection(enabled, request.current_time) + .await?; + if !enabled { + return Ok(Vec::new()); + } + let storage = self.state.storage(); + let mut values = Vec::new(); + for server in &request.servers { + let applicable_rules = request + .rules + .iter() + .filter(|rule| { + rule.enabled + && (rule.server_ids.is_empty() + || rule.server_ids.iter().any(|id| id == &server.id)) + }) + .collect::>(); + if applicable_rules.is_empty() { + continue; + } + let samples = storage + .get::>(&alert_storage_key(&server.id)) + .await? + .unwrap_or_default(); + if samples.is_empty() { + continue; + } + for rule in applicable_rules { + if let Some(row) = + evaluate_alert_window(&samples, rule, server, request.current_time) + { + values.push(AlertEvaluationValue { + rule_id: rule.id.clone(), + row, + }); + } + } + } + Ok(values) + } } impl DurableObject for LiveHub { @@ -177,25 +710,60 @@ impl DurableObject for LiveHub { if let Ok(pair) = WebSocketRequestResponsePair::new("ping", "pong") { state.set_websocket_auto_response(&pair); } - Self { state, env } + Self { + state, + env, + alert_collection_until: Cell::new(-1), + active_d1_flushes: RefCell::new(HashSet::new()), + } } async fn fetch(&self, mut req: Request) -> Result { if req.method() == Method::Post { - if req.headers().get("X-Live-Action")?.as_deref() == Some("disconnect-agents") { - let server_ids = req.json::>().await?; - for server_id in server_ids { - if server_id.is_empty() || server_id.len() > 80 || server_id.contains('/') { - continue; - } - for socket in self - .state - .get_websockets_with_tag(&format!("agent:{server_id}")) - { - let _ = socket.close(Some(1008), Some("server configuration changed")); + match req.headers().get("X-Live-Action")?.as_deref() { + Some("disconnect-agents") => { + let server_ids = req.json::>().await?; + for server_id in server_ids { + if server_id.is_empty() || server_id.len() > 80 || server_id.contains('/') { + continue; + } + for socket in self + .state + .get_websockets_with_tag(&format!("agent:{server_id}")) + { + let _ = socket.close(Some(1008), Some("server configuration changed")); + } } + return Response::empty(); } - return Response::empty(); + Some("record-alert-sample") => { + let record = req.json::().await?; + self.record_alert_point(&record.server_id, &record.point) + .await?; + return Response::empty(); + } + Some("evaluate-alerts") => { + let request = req.json::().await?; + return Response::from_json(&self.evaluate_alerts(&request).await?); + } + Some("clear-alert-windows") => { + self.state.storage().delete_all().await?; + self.alert_collection_until.set(-1); + return Response::empty(); + } + Some("remove-alert-samples") => { + let server_ids = req.json::>().await?; + let keys = server_ids + .into_iter() + .filter(|id| !id.is_empty() && id.len() <= 80 && !id.contains('/')) + .map(|id| alert_storage_key(&id)) + .collect::>(); + if !keys.is_empty() { + self.state.storage().delete_multiple(keys).await?; + } + return Response::empty(); + } + _ => {} } let payload = req.text().await?; let server_id = req.headers().get("X-Server-ID")?.unwrap_or_default(); @@ -241,8 +809,9 @@ impl DurableObject for LiveHub { hidden, context, ))?; + let agent_tag = format!("agent:{server_id}"); self.state - .accept_websocket_with_tags(&pair.server, &[&format!("agent:{server_id}")]); + .accept_websocket_with_tags(&pair.server, &["agents", agent_tag.as_str()]); return Response::from_websocket(pair.client); } let server_id = req @@ -250,9 +819,13 @@ impl DurableObject for LiveHub { .query_pairs() .find_map(|(key, value)| (key == "server_id").then(|| value.to_string())) .filter(|value| !value.is_empty() && value.len() <= 80 && !value.contains('/')); - let tag = server_id.map_or_else(|| "all".to_string(), |id| format!("server:{id}")); + let tag = server_id + .as_ref() + .map_or_else(|| "all".to_string(), |id| format!("server:{id}")); pair.server.serialize_attachment(dashboard_attachment())?; self.state.accept_websocket_with_tags(&pair.server, &[&tag]); + self.replay_latest(&pair.server, server_id.as_deref()); + self.hint_agents(server_id.as_deref()); Response::from_websocket(pair.client) } @@ -303,26 +876,40 @@ impl DurableObject for LiveHub { .traffic .take() .ok_or_else(|| Error::RustError("missing live traffic state".to_string()))?; - let payload = batch_update_payload(&server_id, &batch.samples, &mut traffic)?; + let (payload, mut cached_samples) = + batch_update_parts_at(&server_id, &batch.samples, &mut traffic, received_at)?; + trim_cached_live_samples(&mut cached_samples); + let traffic_state = crate::db::TrafficCounterState { + cycle_key: traffic.cycle_key, + reset_day: traffic.state_reset_day, + timestamp: traffic.timestamp, + raw_rx: traffic.raw_rx, + raw_tx: traffic.raw_tx, + used_rx: traffic.used_rx, + used_tx: traffic.used_tx, + }; attachment.traffic = Some(traffic); + attachment.latest_samples = cached_samples; + attachment.latest_received_at = received_at; + attachment.history_aggregate.extend(&batch.samples); + attachment + .latency_aggregates + .extend(&latency_results, received_at); let report_interval = attachment.report_interval.clamp(15, 3600); let due_for_d1 = received_at.saturating_sub(attachment.last_d1_write_at) >= report_interval; - let mut persisted = false; - if due_for_d1 { + let flush_active = self.active_d1_flushes.borrow().contains(&server_id); + let mut flush = if (due_for_d1 || attachment.d1_flush.is_some()) && !flush_active { let database = self.env.d1("DB")?; - if let Err(error) = crate::db::save_reports(&database, &server_id, &batch.samples).await - { - console_warn!("live D1 metrics write failed for {server_id}: {error:?}"); - } else if let Err(error) = - crate::latency::save_results(&database, &server_id, &latency_results, received_at) - .await - { - console_warn!("live D1 latency write failed for {server_id}: {error:?}"); - } else { - attachment.last_d1_write_at = received_at; - persisted = true; + begin_d1_flush(&mut attachment).map(|snapshot| (database, snapshot)) + } else { + None + }; + if flush.is_some() { + if !trim_socket_attachment(&mut attachment) { + return ws.close(Some(1011), Some("live state exceeds attachment limit")); } + ws.serialize_attachment(&attachment)?; } if !attachment.hidden { @@ -336,38 +923,75 @@ impl DurableObject for LiveHub { let _ = dashboard.close(Some(1011), Some("send failed")); } } - let next_wss_ms = live_interval_ms(attachment.collect_interval); - let next_d1_ms = if persisted { - report_interval * 1000 - } else { - let due_at = attachment.last_d1_write_at.saturating_add(report_interval); - due_at.saturating_sub(received_at).saturating_mul(1000) - }; - ws.serialize_attachment(attachment)?; - ws.send_with_str(&serde_json::to_string(&serde_json::json!({ - "type": "ack", - "ts": received_at, - "persisted": persisted, - "nextD1WriteAfterMs": next_d1_ms, - "nextWssReportAfterMs": next_wss_ms - }))?)?; - } else { - let next_wss_ms = live_interval_ms(attachment.collect_interval); - let next_d1_ms = if persisted { - report_interval * 1000 - } else { - let due_at = attachment.last_d1_write_at.saturating_add(report_interval); - due_at.saturating_sub(received_at).saturating_mul(1000) - }; - ws.serialize_attachment(attachment)?; - ws.send_with_str(&serde_json::to_string(&serde_json::json!({ - "type": "ack", - "ts": received_at, - "persisted": persisted, - "nextD1WriteAfterMs": next_d1_ms, - "nextWssReportAfterMs": next_wss_ms - }))?)?; } + + let mut persisted = false; + let mut attachment_serialized = false; + if let Some((database, snapshot)) = flush.take() { + self.active_d1_flushes + .borrow_mut() + .insert(server_id.clone()); + let history_point = snapshot + .history + .point() + .ok_or_else(|| Error::RustError("missing live history aggregate".to_string()))?; + let metrics_result = crate::db::save_reports_with_history( + &database, + &server_id, + &batch.samples, + &history_point, + &traffic_state, + &snapshot.latency, + ) + .await; + let write_succeeded = match metrics_result { + Err(error) => { + console_warn!("live D1 write failed for {server_id}: {error:?}"); + false + } + Ok(()) => true, + }; + self.active_d1_flushes.borrow_mut().remove(&server_id); + + let Some(mut current_attachment) = ws.deserialize_attachment::()? + else { + return ws.close(Some(1011), Some("missing live state after D1 write")); + }; + if current_attachment.role != "agent" + || current_attachment.server_id.as_deref() != Some(server_id.as_str()) + { + return ws.close(Some(1011), Some("invalid live state after D1 write")); + } + finish_d1_flush(&mut current_attachment, write_succeeded, received_at); + if !trim_socket_attachment(&mut current_attachment) { + return ws.close(Some(1011), Some("live state exceeds attachment limit")); + } + ws.serialize_attachment(¤t_attachment)?; + attachment = current_attachment; + attachment_serialized = true; + persisted = write_succeeded; + + if write_succeeded { + if let Err(error) = self.record_alert_point(&server_id, &history_point).await { + console_warn!("live alert window write failed for {server_id}: {error:?}"); + } + } + } + let realtime_active = !attachment.hidden && self.dashboard_active_for(&server_id); + let next_wss_ms = agent_wss_interval_ms(&attachment, realtime_active); + let next_d1_ms = if persisted { + report_interval * 1000 + } else { + next_d1_write_ms(&attachment, received_at) + }; + let ack = live_ack_payload(received_at, persisted, next_d1_ms, next_wss_ms, false)?; + if !attachment_serialized { + if !trim_socket_attachment(&mut attachment) { + return ws.close(Some(1011), Some("live state exceeds attachment limit")); + } + ws.serialize_attachment(attachment)?; + } + ws.send_with_str(&ack)?; Ok(()) } @@ -386,6 +1010,75 @@ impl DurableObject for LiveHub { } } +async fn post_live_action(env: &Env, path: &str, action: &str, body: &str) -> Result { + let namespace = env.durable_object("LIVE_HUB")?; + let stub = namespace.id_from_name("dashboard")?.get_stub()?; + let mut init = RequestInit::new(); + init.with_method(Method::Post) + .with_body(Some(worker::wasm_bindgen::JsValue::from_str(body))); + let req = Request::new_with_init(&format!("https://live.internal/{path}"), &init)?; + req.headers().set("Content-Type", "application/json")?; + req.headers().set("X-Live-Action", action)?; + stub.fetch_with_request(req).await +} + +pub async fn record_alert_sample(env: &Env, server_id: &str, point: &HistoryPoint) -> Result<()> { + let body = serde_json::to_string(&AlertRecordRequest { + server_id: server_id.to_string(), + point: point.clone(), + })?; + post_live_action(env, "record-alert-sample", "record-alert-sample", &body).await?; + Ok(()) +} + +pub async fn evaluate_resource_alerts( + env: &Env, + rules: &[AlertRuleView], + servers: &[ServerView], + current_time: i64, +) -> Result> { + let request = AlertEvaluationRequest { + current_time, + rules: rules.to_vec(), + servers: servers + .iter() + .filter(|server| server.hidden == 0) + .map(|server| AlertEvaluationServer { + id: server.id.clone(), + name: server.name.clone(), + report_interval: server.report_interval, + }) + .collect(), + }; + let mut response = post_live_action( + env, + "evaluate-alerts", + "evaluate-alerts", + &serde_json::to_string(&request)?, + ) + .await?; + response.json::>().await +} + +pub async fn clear_alert_windows(env: &Env) -> Result<()> { + post_live_action(env, "clear-alert-windows", "clear-alert-windows", "{}").await?; + Ok(()) +} + +pub async fn remove_alert_samples(env: &Env, server_ids: &[String]) -> Result<()> { + if server_ids.is_empty() { + return Ok(()); + } + post_live_action( + env, + "remove-alert-samples", + "remove-alert-samples", + &serde_json::to_string(server_ids)?, + ) + .await?; + Ok(()) +} + pub async fn broadcast(env: &Env, server_id: &str, payload: &str) -> Result<()> { let namespace = env.durable_object("LIVE_HUB")?; let stub = namespace.id_from_name("dashboard")?.get_stub()?; @@ -426,6 +1119,8 @@ pub async fn upgrade(req: Request, env: &Env) -> Result { "X-Live-Collect-Interval", "X-Live-Reset-Day", "X-Live-Cycle-Key", + "X-Live-Traffic-Reset-Day", + "X-Live-Traffic-Timestamp", "X-Live-Raw-Rx", "X-Live-Raw-Tx", "X-Live-Used-Rx", @@ -463,6 +1158,14 @@ pub async fn upgrade_agent( .set("X-Live-Reset-Day", &context.reset_day.to_string())?; req.headers() .set("X-Live-Cycle-Key", &context.cycle_key.to_string())?; + req.headers().set( + "X-Live-Traffic-Reset-Day", + &context.traffic_reset_day.to_string(), + )?; + req.headers().set( + "X-Live-Traffic-Timestamp", + &context.traffic_timestamp.to_string(), + )?; req.headers() .set("X-Live-Raw-Rx", &context.raw_rx.to_string())?; req.headers() @@ -485,8 +1188,31 @@ pub async fn upgrade_agent( #[cfg(test)] mod tests { - use super::{batch_update_payload_at, live_interval_ms, TrafficAttachment}; - use crate::models::AgentReport; + use super::{ + agent_wss_interval_ms, batch_update_parts_at, begin_d1_flush, cached_update_value, + finish_d1_flush, live_ack_payload, next_d1_write_ms, trim_cached_live_samples, + trim_socket_attachment, update_alert_window, AlertEvaluationServer, AlertWindowSample, + CachedLiveSample, SocketAttachment, TrafficAttachment, ALERT_WINDOW_SECONDS, + CACHED_LIVE_TTL_SECONDS, MAX_CACHED_LIVE_BYTES, MAX_CACHED_LIVE_SAMPLES, + MAX_SERIALIZED_ATTACHMENT_BYTES, + }; + use crate::db::HistoryMetricAggregate; + use crate::latency::LatencyMetricAggregates; + use crate::models::{AgentLatencyResult, AgentReport, AlertRuleView, HistoryPoint}; + + fn alert_point(timestamp: i64, cpu: f64, net_in: f64) -> HistoryPoint { + HistoryPoint { + timestamp, + cpu, + mem_used: 50, + mem_total: 100, + disk_used: 25, + disk_total: 100, + net_in, + net_out: net_in / 2.0, + ..HistoryPoint::default() + } + } #[test] fn live_payload_contains_cycle_adjusted_traffic_and_latency() { @@ -531,6 +1257,8 @@ mod tests { }; let mut traffic = TrafficAttachment { reset_day: 1, + state_reset_day: 1, + timestamp: 1_230, cycle_key: crate::db::traffic_cycle_key(1_234, 1), raw_rx: 100, raw_tx: 100, @@ -539,11 +1267,11 @@ mod tests { rx_correction: 10, tx_correction: 20, }; - let payload: serde_json::Value = serde_json::from_str( - &batch_update_payload_at("node-a", std::slice::from_ref(&report), &mut traffic, 1_235) - .unwrap(), - ) - .unwrap(); + let payload = + batch_update_parts_at("node-a", std::slice::from_ref(&report), &mut traffic, 1_235) + .unwrap() + .0; + let payload: serde_json::Value = serde_json::from_str(&payload).unwrap(); assert_eq!(payload["type"], "batchUpdate"); assert_eq!(payload["updates"][0]["serverId"], "node-a"); assert_eq!(payload["updates"][0]["samples"][0]["ts"], 1_234); @@ -555,23 +1283,243 @@ mod tests { assert!(payload["updates"][0]["samples"][0]["data"] .get("timestamp") .is_none()); - assert_eq!(live_interval_ms(5), 5_000); - assert_eq!(live_interval_ms(1), 2_000); - assert_eq!(live_interval_ms(120), 60_000); + let attachment = SocketAttachment { + role: "agent".to_string(), + server_id: Some("node-a".to_string()), + hidden: false, + report_interval: 60, + collect_interval: 1, + last_d1_write_at: 1_200, + d1_flush: None, + traffic: None, + history_aggregate: HistoryMetricAggregate::default(), + latency_aggregates: LatencyMetricAggregates::default(), + latest_samples: Vec::new(), + latest_received_at: 0, + }; + assert_eq!(agent_wss_interval_ms(&attachment, true), 4_000); + assert_eq!(agent_wss_interval_ms(&attachment, false), 60_000); + assert_eq!(next_d1_write_ms(&attachment, 1_235), 25_000); + let hint: serde_json::Value = + serde_json::from_str(&live_ack_payload(1_235, false, 25_000, 4_000, true).unwrap()) + .unwrap(); + assert_eq!(hint["realtimeHint"], true); + assert_eq!(hint["nextWssReportAfterMs"], 4_000); let mut corrected_down = traffic; corrected_down.rx_correction = -800; let mut corrected_report = report.clone(); corrected_report.timestamp = 1_235; corrected_report.net_rx_total = 301; - let payload: serde_json::Value = serde_json::from_str( - &batch_update_payload_at("node-a", &[corrected_report], &mut corrected_down, 1_236) - .unwrap(), - ) - .unwrap(); + let payload = + batch_update_parts_at("node-a", &[corrected_report], &mut corrected_down, 1_236) + .unwrap() + .0; + let payload: serde_json::Value = serde_json::from_str(&payload).unwrap(); assert_eq!( payload["updates"][0]["samples"][0]["data"]["net_rx_total"], 0 ); } + + #[test] + fn bounds_and_expires_cached_live_samples() { + let mut samples = (0..(MAX_CACHED_LIVE_SAMPLES + 8)) + .map(|index| CachedLiveSample { + ts: index as i64, + data: serde_json::json!({ "cpu": index }), + }) + .collect::>(); + trim_cached_live_samples(&mut samples); + assert_eq!(samples.len(), MAX_CACHED_LIVE_SAMPLES); + assert_eq!(samples[0].ts, 8); + + let mut attachment = SocketAttachment { + role: "agent".to_string(), + server_id: Some("node-a".to_string()), + hidden: false, + report_interval: 60, + collect_interval: 1, + last_d1_write_at: 0, + d1_flush: None, + traffic: None, + history_aggregate: HistoryMetricAggregate::default(), + latency_aggregates: LatencyMetricAggregates::default(), + latest_samples: samples, + latest_received_at: 100, + }; + let update = cached_update_value(&attachment, 102, Some("node-a")).unwrap(); + assert_eq!(update["serverId"], "node-a"); + assert_eq!(update["reportAgeMs"], 2_000); + assert_eq!( + update["samples"].as_array().unwrap().len(), + MAX_CACHED_LIVE_SAMPLES + ); + assert!(cached_update_value(&attachment, 102, Some("node-b")).is_none()); + assert!(cached_update_value(&attachment, 101 + CACHED_LIVE_TTL_SECONDS, None).is_none()); + + let mut oversized = vec![CachedLiveSample { + ts: 1, + data: serde_json::json!({ "value": "x".repeat(MAX_CACHED_LIVE_BYTES) }), + }]; + trim_cached_live_samples(&mut oversized); + assert!(oversized.is_empty()); + + let latency_results = (0..128) + .map(|index| AgentLatencyResult { + task_id: format!("00000000-0000-0000-0000-{index:012}"), + timestamp: 100, + latency_ms: index as f64, + packet_loss: 0.0, + }) + .collect::>(); + attachment.latency_aggregates.extend(&latency_results, 100); + assert!(trim_socket_attachment(&mut attachment)); + assert!(serde_json::to_vec(&attachment).unwrap().len() <= MAX_SERIALIZED_ATTACHMENT_BYTES); + } + + #[test] + fn keeps_samples_received_during_d1_flush_for_next_write() { + let first = AgentReport { + timestamp: 120, + cpu: 10.0, + net_in: 100.0, + latency_results: vec![AgentLatencyResult { + task_id: "task-a".to_string(), + timestamp: 120, + latency_ms: 10.0, + packet_loss: 0.0, + }], + ..AgentReport::default() + }; + let second = AgentReport { + timestamp: 121, + cpu: 20.0, + net_in: 500.0, + latency_results: vec![AgentLatencyResult { + task_id: "task-a".to_string(), + timestamp: 121, + latency_ms: 20.0, + packet_loss: 10.0, + }], + ..AgentReport::default() + }; + let mut attachment = SocketAttachment { + role: "agent".to_string(), + server_id: Some("node-a".to_string()), + hidden: false, + report_interval: 60, + collect_interval: 1, + last_d1_write_at: 60, + d1_flush: None, + traffic: None, + history_aggregate: HistoryMetricAggregate::default(), + latency_aggregates: LatencyMetricAggregates::default(), + latest_samples: Vec::new(), + latest_received_at: 0, + }; + attachment + .history_aggregate + .extend(std::slice::from_ref(&first)); + attachment + .latency_aggregates + .extend(&first.latency_results, first.timestamp); + + let mut successful = attachment.clone(); + let successful_snapshot = begin_d1_flush(&mut successful).unwrap(); + assert!(successful.d1_flush.is_some()); + assert!(successful.history_aggregate.point().is_none()); + assert!(begin_d1_flush(&mut successful).is_some()); + let serialized = serde_json::to_vec(&successful).unwrap(); + successful = serde_json::from_slice(&serialized).unwrap(); + assert_eq!( + begin_d1_flush(&mut successful) + .unwrap() + .history + .point() + .unwrap() + .cpu, + 10.0 + ); + successful + .history_aggregate + .extend(std::slice::from_ref(&second)); + successful + .latency_aggregates + .extend(&second.latency_results, second.timestamp); + assert_eq!(successful_snapshot.history.point().unwrap().cpu, 10.0); + finish_d1_flush(&mut successful, true, 180); + + let pending = successful.history_aggregate.point().unwrap(); + assert_eq!(pending.cpu, 20.0); + assert_eq!(pending.net_in, 500.0); + assert_eq!(successful.last_d1_write_at, 180); + assert!(successful.d1_flush.is_none()); + let latency = serde_json::to_value(&successful.latency_aggregates).unwrap(); + assert_eq!(latency["v"]["task-a"][0].as_u64(), Some(1)); + assert_eq!(latency["v"]["task-a"][4].as_i64(), Some(121)); + + let failed_snapshot = begin_d1_flush(&mut attachment).unwrap(); + attachment + .history_aggregate + .extend(std::slice::from_ref(&second)); + attachment + .latency_aggregates + .extend(&second.latency_results, second.timestamp); + assert_eq!(failed_snapshot.history.point().unwrap().cpu, 10.0); + finish_d1_flush(&mut attachment, false, 180); + + let retried = attachment.history_aggregate.point().unwrap(); + assert_eq!(retried.cpu, 15.0); + assert_eq!(retried.net_in, 500.0); + assert_eq!(attachment.last_d1_write_at, 60); + assert!(attachment.d1_flush.is_none()); + let latency = serde_json::to_value(&attachment.latency_aggregates).unwrap(); + assert_eq!(latency["v"]["task-a"][0].as_u64(), Some(2)); + assert_eq!(latency["v"]["task-a"][4].as_i64(), Some(121)); + } + + #[test] + fn merges_and_evaluates_persistent_alert_windows() { + let mut samples = Vec::::new(); + update_alert_window(&mut samples, &alert_point(121, 10.0, 1_048_576.0)); + update_alert_window(&mut samples, &alert_point(149, 30.0, 3_145_728.0)); + update_alert_window(&mut samples, &alert_point(181, 40.0, 2_097_152.0)); + assert_eq!(samples.len(), 2); + assert_eq!(samples[0].metric("cpu"), Some(20.0)); + assert_eq!(samples[0].metric("memory"), Some(50.0)); + assert_eq!(samples[0].metric("disk"), Some(25.0)); + assert_eq!(samples[0].metric("net_in"), Some(3.0)); + + let server = AlertEvaluationServer { + id: "node-a".to_string(), + name: "Node A".to_string(), + report_interval: 60, + }; + let mut rule = AlertRuleView { + id: "rule-a".to_string(), + name: "CPU".to_string(), + metric: "cpu".to_string(), + threshold: 25.0, + duration_minutes: 2, + aggregation: "average".to_string(), + enabled: true, + server_ids: Vec::new(), + }; + let average = super::evaluate_alert_window(&samples, &rule, &server, 240).unwrap(); + assert_eq!(average.value, 30.0); + assert_eq!(average.sample_count, 2); + assert_eq!(average.first_timestamp, 120); + assert_eq!(average.last_timestamp, 180); + + rule.aggregation = "continuous".to_string(); + let continuous = super::evaluate_alert_window(&samples, &rule, &server, 240).unwrap(); + assert_eq!(continuous.value, 20.0); + + for minute in 0..1_500 { + update_alert_window(&mut samples, &alert_point(minute * 60, minute as f64, 0.0)); + } + assert!(samples.len() <= super::MAX_ALERT_WINDOW_SAMPLES); + assert!(samples.last().unwrap().0 - samples.first().unwrap().0 <= ALERT_WINDOW_SECONDS); + } } diff --git a/src/models.rs b/src/models.rs index ad4676b..b4195a1 100644 --- a/src/models.rs +++ b/src/models.rs @@ -181,7 +181,7 @@ pub struct AgentReportBatch { pub samples: Vec, } -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Default, Deserialize, Serialize)] #[serde(deny_unknown_fields)] pub struct AgentReport { pub timestamp: i64, @@ -294,7 +294,7 @@ pub struct ServerView { pub gpu_info: Option, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct HistoryPoint { pub timestamp: i64, pub cpu: f64, @@ -351,7 +351,7 @@ pub struct AlertRuleInput { pub server_ids: Vec, } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize)] pub struct AlertRuleView { pub id: String, pub name: String, diff --git a/src/notify.rs b/src/notify.rs index a37613d..c75c64b 100644 --- a/src/notify.rs +++ b/src/notify.rs @@ -1,9 +1,13 @@ -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use serde::{Deserialize, Serialize}; -use worker::{wasm_bindgen::JsValue, D1Database, Fetch, Method, Request, RequestInit, Result}; +use worker::{ + console_warn, wasm_bindgen::JsValue, D1Database, Env, Fetch, Method, Request, RequestInit, + Result, +}; use crate::db::{self, SettingsView}; +use crate::live; #[derive(Default, Deserialize, Serialize, PartialEq, Eq)] struct AlertState { @@ -84,7 +88,7 @@ pub async fn renew_servers(db_conn: &D1Database) -> Result<()> { Ok(()) } -pub async fn check_alerts(db_conn: &D1Database, settings: &SettingsView) -> Result<()> { +pub async fn check_alerts(db_conn: &D1Database, env: &Env, settings: &SettingsView) -> Result<()> { if !settings.notification_enabled || settings.notification_endpoint.trim().is_empty() { return Ok(()); } @@ -136,7 +140,22 @@ pub async fn check_alerts(db_conn: &D1Database, settings: &SettingsView) -> Resu let mut current_resources = HashSet::new(); let mut evaluated_resources = HashSet::new(); let mut eligible_resources = HashSet::new(); - for rule in db::list_alert_rules(db_conn).await? { + let rules = db::list_alert_rules(db_conn).await?; + let mut values_by_rule = HashMap::new(); + if rules.iter().any(|rule| rule.enabled) { + match live::evaluate_resource_alerts(env, &rules, &servers, current_time).await { + Ok(values) => { + for value in values { + values_by_rule + .entry(value.rule_id) + .or_insert_with(Vec::new) + .push(value.row); + } + } + Err(error) => console_warn!("resource alert evaluation failed: {error:?}"), + } + } + for rule in rules { if !rule.enabled { continue; } @@ -167,7 +186,10 @@ pub async fn check_alerts(db_conn: &D1Database, settings: &SettingsView) -> Resu } else { "%" }; - for value in db::alert_metric_values(db_conn, &rule, current_time).await? { + for value in values_by_rule.remove(&rule.id).unwrap_or_default() { + if !db::alert_window_covered(&value, rule.duration_minutes, current_time) { + continue; + } let key = format!("{}:{}", rule.id, value.server_id); evaluated_resources.insert(key.clone()); if value.value >= rule.threshold { @@ -221,10 +243,6 @@ mod tests { assert_eq!(renewed_expiry(80 * day, 30, 100 * day), Some(110 * day)); assert_eq!(renewed_expiry(101 * day, 30, 100 * day), Some(131 * day)); assert_eq!(renewed_expiry(102 * day, 30, 100 * day), None); - } - - #[test] - fn saturates_pathological_expiry_values() { assert_eq!(renewed_expiry(i64::MIN, 1, i64::MAX), Some(i64::MAX)); } } diff --git a/src/routes/admin.rs b/src/routes/admin.rs index 984d9cd..75a793b 100644 --- a/src/routes/admin.rs +++ b/src/routes/admin.rs @@ -135,6 +135,7 @@ pub(crate) async fn route(mut req: Request, ctx: &RouteContext) -> Result Result Result { name: theme::BUILTIN_THEME_NAME.to_string(), description: "默认主题".to_string(), url: String::new(), - version: env!("CARGO_PKG_VERSION").to_string(), + version: crate::VERSION.to_string(), builtin: true, active: ctx.settings.active_theme_id == theme::BUILTIN_THEME_ID, }]; diff --git a/src/routes/agent.rs b/src/routes/agent.rs index 2681eaa..28ced2d 100644 --- a/src/routes/agent.rs +++ b/src/routes/agent.rs @@ -13,6 +13,33 @@ use crate::{ AGENT_JSON_MAX_BYTES, MAX_AGENT_LATENCY_RESULTS, MAX_AGENT_SAMPLES, }; +async fn config_response( + database: &D1Database, + server_id: &str, + agent_config_hash: &str, + received_at: i64, + changed_status: u16, +) -> Result { + let Some(config) = db::agent_config(database, server_id).await? else { + return error("节点不存在", 404); + }; + let config_json = serde_json::to_string(&config)?; + let config_hash = sha256_hex(&config_json); + let mut response = if agent_config_hash == config_hash { + no_content()? + } else { + crate::json(&config, changed_status)? + }; + response + .headers_mut() + .set("X-Agent-Config-Sha256", &config_hash)?; + response + .headers_mut() + .set("X-NodeFlare-Server-Time", &received_at.to_string())?; + response.headers_mut().set("Cache-Control", "no-store")?; + Ok(response) +} + pub(crate) async fn report( mut req: Request, env: Env, @@ -50,22 +77,25 @@ pub(crate) async fn report( if let Some(ip) = client_ip(&req).and_then(|value| value.parse::().ok()) { db::update_last_ip(database, &server_id, &ip.to_string()).await?; } - let latency_results = batch + let latency_result_count = batch .samples .iter() .flat_map(|report| report.latency_results.iter()) - .cloned() - .collect::>(); - if latency_results.len() > MAX_AGENT_LATENCY_RESULTS { + .count(); + if latency_result_count > MAX_AGENT_LATENCY_RESULTS { return error("每批最多包含 4096 条延迟结果", 400); } + let mut history_aggregate = db::HistoryMetricAggregate::default(); + history_aggregate.extend(&batch.samples); + let alert_point = history_aggregate.point(); db::save_reports(database, &server_id, &batch.samples).await?; - latency::save_results(database, &server_id, &latency_results, received_at).await?; + if let Some(point) = alert_point.as_ref() { + if let Err(error) = live::record_alert_sample(&env, &server_id, point).await { + worker::console_warn!("resource alert sample write failed: {error:?}"); + } + } - let public_dashboard = db::get_setting(database, "public_dashboard") - .await? - .is_none_or(|value| value == "true"); - if identity.hidden == 0 && public_dashboard { + if identity.hidden == 0 { let server = db::get_server(database, &server_id, false).await?; let samples = latency::latest_for_server(database, &server_id).await?; let payload = server @@ -84,30 +114,22 @@ pub(crate) async fn report( }); } } - let Some(config) = db::agent_config(database, &server_id).await? else { - return error("节点不存在", 404); + config_response(database, &server_id, &agent_config_hash, received_at, 202).await +} + +pub(crate) async fn config(req: &Request, database: &D1Database) -> Result { + let agent_config_hash = req + .headers() + .get("X-Agent-Config-Sha256")? + .unwrap_or_default(); + let token = match bearer_token(req) { + Some(value) => value, + None => return error("缺少 Agent Token", 401), }; - let config_json = serde_json::to_string(&config)?; - let config_hash = sha256_hex(&config_json); - if agent_config_hash == config_hash { - let mut response = no_content()?; - response - .headers_mut() - .set("X-Agent-Config-Sha256", &config_hash)?; - response - .headers_mut() - .set("X-NodeFlare-Server-Time", &received_at.to_string())?; - response.headers_mut().set("Cache-Control", "no-store")?; - return Ok(response); - } - let mut response = crate::json(&config, 202)?; - response - .headers_mut() - .set("X-Agent-Config-Sha256", &config_hash)?; - response - .headers_mut() - .set("X-NodeFlare-Server-Time", &received_at.to_string())?; - Ok(response) + let Some(identity) = db::get_agent_identity(database, &token).await? else { + return error("Agent Token 无效", 401); + }; + config_response(database, &identity.id, &agent_config_hash, now(), 200).await } pub(crate) async fn live_websocket( @@ -122,6 +144,9 @@ pub(crate) async fn live_websocket( let Some(identity) = db::get_agent_identity(database, &token).await? else { return error("Agent Token 无效", 401); }; + if let Some(ip) = client_ip(&req).and_then(|value| value.parse::().ok()) { + db::update_last_ip(database, &identity.id, &ip.to_string()).await?; + } let Some(context) = db::agent_live_context(database, &identity.id).await? else { return error("节点不存在", 404); }; diff --git a/src/routes/public.rs b/src/routes/public.rs index c0129c5..a1404c5 100644 --- a/src/routes/public.rs +++ b/src/routes/public.rs @@ -10,15 +10,18 @@ use crate::models::LoginRequest; use crate::routes::{RouteContext, RouteOutcome}; use crate::turnstile; use crate::{ - cached_history_response, db, error, history_cache_key, json, latency, live, no_content, now, - public_server, request_json, requested_hours, server_id, set_admin_session_cookie, - store_history_response, API_JSON_MAX_BYTES, + cached_history_response, db, error, history_cache_key, history_cache_seconds, json, latency, + live, no_content, now, public_server, request_json, requested_hours, server_id, + set_admin_session_cookie, store_history_response, API_JSON_MAX_BYTES, }; pub(crate) async fn route(mut req: Request, ctx: &RouteContext) -> Result { let method = req.method(); let path = req.path(); + if method == Method::Get && path == "/api/bootstrap" { + return Ok(RouteOutcome::Handled(bootstrap(ctx).await?)); + } if method == Method::Get && path == "/api/config" { return Ok(RouteOutcome::Handled(config(ctx)?)); } @@ -67,45 +70,97 @@ fn public_access_denied(ctx: &RouteContext) -> Option> { None } -fn config(ctx: &RouteContext) -> Result { +fn config_value(ctx: &RouteContext) -> serde_json::Value { let settings = &ctx.settings; - let mut response = json( - &serde_json::json!({ - "site_name": settings.site_name, - "site_description": settings.site_description, - "site_announcement": settings.site_announcement, - "favicon_url": settings.favicon_url, - "locale": settings.locale, - "public_dashboard": settings.public_dashboard, - "offline_threshold_seconds": settings.offline_threshold_seconds, - "history_retention_days": settings.history_retention_days, - "default_theme": settings.default_theme, - "active_theme_id": settings.active_theme_id, - "background_url": settings.background_url, - "theme_options": settings.theme_options, - "show_search": settings.show_search, - "show_groups": settings.show_groups, - "show_stats": settings.show_stats, - "show_assets": settings.show_assets, - "show_traffic": settings.show_traffic, - "show_speed": settings.show_speed, - "show_price": settings.show_price, - "show_expiry": settings.show_expiry, - "show_latency": settings.show_latency, - "show_uptime": settings.show_uptime, - "turnstile_enabled": ctx.public_turnstile_enabled, - "turnstile_login_enabled": ctx.login_protection_enabled || ctx.public_turnstile_enabled, - "turnstile_site_key": ctx.turnstile_site_key, - "password_client_salt": settings.password_client_salt - }), - 200, - )?; + serde_json::json!({ + "site_name": settings.site_name, + "site_description": settings.site_description, + "site_announcement": settings.site_announcement, + "favicon_url": settings.favicon_url, + "locale": settings.locale, + "public_dashboard": settings.public_dashboard, + "offline_threshold_seconds": settings.offline_threshold_seconds, + "history_retention_days": settings.history_retention_days, + "default_theme": settings.default_theme, + "active_theme_id": settings.active_theme_id, + "background_url": settings.background_url, + "theme_options": settings.theme_options, + "show_search": settings.show_search, + "show_groups": settings.show_groups, + "show_stats": settings.show_stats, + "show_assets": settings.show_assets, + "show_traffic": settings.show_traffic, + "show_speed": settings.show_speed, + "show_price": settings.show_price, + "show_expiry": settings.show_expiry, + "show_latency": settings.show_latency, + "show_uptime": settings.show_uptime, + "turnstile_enabled": ctx.public_turnstile_enabled, + "turnstile_login_enabled": ctx.login_protection_enabled || ctx.public_turnstile_enabled, + "turnstile_site_key": ctx.turnstile_site_key, + "password_client_salt": settings.password_client_salt + }) +} + +fn config(ctx: &RouteContext) -> Result { + let mut response = json(&config_value(ctx), 200)?; response .headers_mut() .set("X-NodeFlare-Server-Time", &now().to_string())?; Ok(response) } +async fn server_values(ctx: &RouteContext) -> Result> { + let raw_servers = db::list_servers(&ctx.database, false).await?; + let mut latency_by_server: HashMap> = HashMap::new(); + for sample in latency::latest_all(&ctx.database).await? { + latency_by_server + .entry(sample.server_id.clone()) + .or_default() + .push(sample); + } + raw_servers + .into_iter() + .map(|server| { + let samples = latency_by_server.remove(&server.id).unwrap_or_default(); + serde_json::to_value(public_server(server, samples)).map_err(Into::into) + }) + .collect() +} + +async fn bootstrap(ctx: &RouteContext) -> Result { + let access = if !ctx.turnstile_verified { + "turnstile" + } else if !ctx.settings.public_dashboard && !ctx.admin { + "login" + } else { + "ok" + }; + let config = config_value(ctx); + if access != "ok" { + return json( + &serde_json::json!({ + "config": config, + "access": access, + "servers": [], + "exchange_rates": null + }), + 200, + ); + } + let servers = server_values(ctx).await?; + let exchange_rates = crate::exchange::current(&ctx.database, now()).await?; + json( + &serde_json::json!({ + "config": config, + "access": access, + "servers": servers, + "exchange_rates": exchange_rates + }), + 200, + ) +} + async fn admin_login(req: &mut Request, ctx: &RouteContext) -> Result { let settings = &ctx.settings; if settings.admin_username.trim().is_empty() { @@ -217,21 +272,7 @@ async fn servers(ctx: &RouteContext) -> Result { if let Some(denied) = public_access_denied(ctx) { return denied; } - let raw_servers = db::list_servers(&ctx.database, false).await?; - let mut latency_by_server: HashMap> = HashMap::new(); - for sample in latency::latest_all(&ctx.database).await? { - latency_by_server - .entry(sample.server_id.clone()) - .or_default() - .push(sample); - } - let servers: Vec<_> = raw_servers - .into_iter() - .map(|server| { - let samples = latency_by_server.remove(&server.id).unwrap_or_default(); - public_server(server, samples) - }) - .collect(); + let servers = server_values(ctx).await?; json(&serde_json::json!({ "servers": servers }), 200) } @@ -266,7 +307,7 @@ async fn history(req: &Request, ctx: &RouteContext) -> Result { let points = db::history(&ctx.database, &id, hours).await?; let mut response = json(&serde_json::json!({ "points": points }), 200)?; if let Some(cache_key) = cache_key.as_deref() { - store_history_response(cache_key, &mut response).await; + store_history_response(cache_key, &mut response, history_cache_seconds(hours)).await; } Ok(response) } @@ -306,7 +347,7 @@ async fn latency(req: &Request, ctx: &RouteContext) -> Result { 200, )?; if let Some(cache_key) = cache_key.as_deref() { - store_history_response(cache_key, &mut response).await; + store_history_response(cache_key, &mut response, history_cache_seconds(hours)).await; } Ok(response) } diff --git a/src/routes/site.rs b/src/routes/site.rs index 40e1bd1..95a8e90 100644 --- a/src/routes/site.rs +++ b/src/routes/site.rs @@ -51,6 +51,24 @@ pub(crate) async fn route(req: Request, ctx: &RouteContext) -> Result