From 39fcfe50c6282a9db27b3909007cd20e65a0e759 Mon Sep 17 00:00:00 2001 From: imengying Date: Sat, 12 Sep 2026 01:58:01 +0800 Subject: [PATCH] Fixes and improvements --- .github/workflows/release.yml | 2 +- README.md | 227 +- agent/agent.sh | 19 +- agent/install.ps1 | 2 +- agent/src/live.rs | 698 ++++ agent/src/live_batch.rs | 22 +- agent/src/main.rs | 3718 +---------------- agent/src/metrics.rs | 1398 +++++++ agent/src/remote.rs | 587 +++ agent/src/tests.rs | 811 ++++ agent/src/update.rs | 240 ++ backend/src/db/mod.rs | 94 +- backend/src/db/queries.rs | 167 +- backend/src/main.rs | 6 + backend/src/notify.rs | 12 +- backend/src/routes/admin.rs | 30 +- backend/src/routes/auth.rs | 12 +- backend/src/routes/public.rs | 37 +- backend/src/routes/remote.rs | 5 + backend/src/websocket/agent.rs | 31 +- docs/README.en.md | 273 ++ {deploy => docs}/nodeflare.service | 0 frontend/src/App.tsx | 70 +- frontend/src/AppProvider.tsx | 55 +- frontend/src/admin-main.tsx | 14 +- frontend/src/components/AdminPanel.tsx | 739 ++-- frontend/src/components/AlertRuleManager.tsx | 56 +- frontend/src/components/LatencyManager.tsx | 45 +- frontend/src/components/LoginForm.tsx | 70 + frontend/src/components/NodeCard.tsx | 9 +- frontend/src/components/NodeDetails.tsx | 6 +- frontend/src/components/PasswordInput.tsx | 7 +- frontend/src/components/TelegramSettings.tsx | 24 +- frontend/src/components/TurnstileWidget.tsx | 8 +- frontend/src/components/admin/AboutTab.tsx | 17 + frontend/src/components/admin/RemoteTab.tsx | 74 + .../src/components/admin/SettingsTabs.tsx | 125 + frontend/src/components/admin/controls.tsx | 22 + frontend/src/components/admin/shared.ts | 192 + frontend/src/components/useVerification.tsx | 11 +- frontend/src/hooks/useNodeLatency.ts | 13 +- frontend/src/live.test.ts | 25 +- frontend/src/live.ts | 23 +- frontend/src/password.ts | 7 +- frontend/src/styles/admin.css | 4 +- frontend/src/styles/base.css | 7 +- install.ps1 | 2 +- install.sh | 9 + scripts/smoke-test.sh | 10 +- 49 files changed, 5585 insertions(+), 4450 deletions(-) mode change 100644 => 100755 .github/workflows/release.yml mode change 100644 => 100755 README.md mode change 100644 => 100755 agent/agent.sh mode change 100644 => 100755 agent/install.ps1 create mode 100755 agent/src/live.rs mode change 100644 => 100755 agent/src/live_batch.rs mode change 100644 => 100755 agent/src/main.rs create mode 100755 agent/src/metrics.rs create mode 100755 agent/src/remote.rs create mode 100755 agent/src/tests.rs create mode 100755 agent/src/update.rs mode change 100644 => 100755 backend/src/db/mod.rs mode change 100644 => 100755 backend/src/db/queries.rs mode change 100644 => 100755 backend/src/main.rs mode change 100644 => 100755 backend/src/notify.rs mode change 100644 => 100755 backend/src/routes/admin.rs mode change 100644 => 100755 backend/src/routes/auth.rs mode change 100644 => 100755 backend/src/routes/public.rs mode change 100644 => 100755 backend/src/routes/remote.rs mode change 100644 => 100755 backend/src/websocket/agent.rs create mode 100755 docs/README.en.md rename {deploy => docs}/nodeflare.service (100%) mode change 100644 => 100755 frontend/src/App.tsx mode change 100644 => 100755 frontend/src/AppProvider.tsx mode change 100644 => 100755 frontend/src/admin-main.tsx mode change 100644 => 100755 frontend/src/components/AdminPanel.tsx mode change 100644 => 100755 frontend/src/components/AlertRuleManager.tsx mode change 100644 => 100755 frontend/src/components/LatencyManager.tsx create mode 100755 frontend/src/components/LoginForm.tsx mode change 100644 => 100755 frontend/src/components/NodeCard.tsx mode change 100644 => 100755 frontend/src/components/NodeDetails.tsx mode change 100644 => 100755 frontend/src/components/PasswordInput.tsx mode change 100644 => 100755 frontend/src/components/TelegramSettings.tsx mode change 100644 => 100755 frontend/src/components/TurnstileWidget.tsx create mode 100755 frontend/src/components/admin/AboutTab.tsx create mode 100755 frontend/src/components/admin/RemoteTab.tsx create mode 100755 frontend/src/components/admin/SettingsTabs.tsx create mode 100755 frontend/src/components/admin/controls.tsx create mode 100755 frontend/src/components/admin/shared.ts mode change 100644 => 100755 frontend/src/components/useVerification.tsx mode change 100644 => 100755 frontend/src/hooks/useNodeLatency.ts mode change 100644 => 100755 frontend/src/live.test.ts mode change 100644 => 100755 frontend/src/live.ts mode change 100644 => 100755 frontend/src/password.ts mode change 100644 => 100755 frontend/src/styles/admin.css mode change 100644 => 100755 frontend/src/styles/base.css mode change 100644 => 100755 install.ps1 mode change 100644 => 100755 scripts/smoke-test.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml old mode 100644 new mode 100755 index e547495..faf5525 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -764,7 +764,7 @@ jobs: tar -xzf "inputs/frontend/nodeflare-frontend-$VERSION.tar.gz" -C inputs/frontend/unpacked cp -a inputs/frontend/unpacked/dist/. package/share/frontend/ cp -a inputs/frontend/unpacked/admin-dist/. package/share/admin/ - cp deploy/nodeflare.service package/nodeflare.service + cp docs/nodeflare.service package/nodeflare.service cp LICENSE package/LICENSE chmod 0755 "package/$PACKAGE_BINARY" chmod 0644 package/LICENSE package/nodeflare.service diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 6750fad..177c63d --- a/README.md +++ b/README.md @@ -1,14 +1,67 @@ # NodeFlare -NodeFlare 是一款轻量级的自托管服务器监控面板，通过 Web 界面查看服务器状态，并由轻量级 Agent 采集数据上报。支持实时监控、TCP 拨测、Telegram 告警、远程执行、TOTP 两步验证、主题定制、SQLite/PostgreSQL 备份迁移等。 +NodeFlare 是一款轻量级、可自托管的服务器监控面板：通过 Web 界面查看服务器状态，由轻量 Agent 采集数据并主动上报。支持实时监控、TCP/ICMP 拨测、Telegram 告警、远程执行、TOTP 两步验证、主题定制，以及 SQLite / PostgreSQL 的备份恢复与在线迁移。 -CPU、内存和网速每秒采样，默认每 3 秒压缩批量上传。服务器卡片每秒显示一个真实采样值，正常连接下有约 2～3 秒的显示延迟；断线重连后跳过积压的旧数据。磁盘容量、GPU 等较慢指标单独缓存，历史数据按节点设置的保存间隔聚合。增大实时上传间隔会降低卡片连续更新的频率。 +English: [docs/README.en.md](docs/README.en.md) -Linux 内存统计与 Komari 默认口径一致：已用内存为 `MemTotal - MemFree - Cached - SReclaimable - Buffers + Shmem`，Swap 已用量扣除 `SwapCached`。文件缓存不计入已用内存，共享内存计入；面板显示的是整机内存，不是 NodeFlare 进程本身的占用。 +- [特性](#特性) +- [平台支持](#平台支持) +- [安装服务端](#安装服务端) +- [安装 Agent](#安装-agent) +- [监控口径与采样](#监控口径与采样) +- [配置](#配置) +- [默认目录与日志](#默认目录与日志) +- [反向代理](#反向代理) +- [数据库与备份](#数据库与备份) +- [开发](#开发) +- [排障](#排障) + +## 特性 + +**监控与展示** + +- CPU、内存、网速每秒采样，默认每 3 秒压缩批量上传；磁盘、GPU、连接数等慢指标单独缓存 +- 延迟拨测：TCP 与 ICMP 任务按节点分配，支持电信 / 移动 / 联通分线展示 +- 公开看板双语（简体中文 / English），站点名称、公告、Logo、背景图与显示项均可配置 +- 主题商店：内置主题、仓库主题一键安装、本地 ZIP 上传、主题参数自定义 + +**告警** + +- 资源阈值告警：CPU / 内存 / 磁盘 / 上行 / 下行，可选均值或持续超限，并可设置持续时长 +- 离线、到期、流量告警；Telegram 推送，消息模板可自定义 + +**管理** + +- 节点分组、标签、地区、计费周期、价格、到期时间、流量额度与重置日 +- 远程执行命令：需启用 TOTP；单条命令最长执行 10 分钟，离开页面不会中断 +- 每日汇率快照，用于多币种价格折算 + +**安全** + +- TOTP 两步验证；Cloudflare Turnstile 人机验证（可分别保护管理员登录与公开看板） +- 登录限速、会话管理（查看登录设备并踢下线）、会话有效期可配置 +- 默认只监听 `127.0.0.1`；Agent 为出站连接，无需开放入站端口；Agent Token 支持文件或环境变量传递 + +**数据** + +- SQLite 与 PostgreSQL 双支持，两种数据库之间可在线迁移 +- 一键备份 / 恢复（ZIP）、空间回收、历史数据按保留天数自动清理 + +## 平台支持 + +| 角色 | 平台 | 架构 | +| --- | --- | --- | +| 服务端 | Linux | x64 / ARM64 | +| 服务端 | Windows | x64 | +| 服务端 | macOS | ARM64 | +| 服务端 | FreeBSD 13+ | x64 / ARM64 | +| Agent | Linux / Windows / macOS / FreeBSD | 同上（macOS 仅 Apple Silicon） | + +安装脚本会自动检测系统服务管理器（systemd / OpenRC / launchd / FreeBSD rc / Windows 计划任务）并注册为开机自启。 ## 安装服务端 -支持 Linux x64/ARM64、Windows x64、macOS ARM64、FreeBSD 13+ x64/ARM64。安装脚本自动下载最新 Release 并注册系统服务。 +安装脚本自动下载最新 Release 并注册系统服务。 Linux / macOS： @@ -30,21 +83,28 @@ Unblock-File "$env:TEMP\nodeflare-install.ps1" & "$env:TEMP\nodeflare-install.ps1" ``` -首次安装需输入管理员用户名、密码、监听端口（默认 2206）和数据库地址（默认 SQLite）。服务端默认监听 `127.0.0.1:2206`，本机访问 `http://127.0.0.1:2206/admin/login`；对外使用需经 HTTPS 反向代理。 +首次安装会询问管理员用户名、密码、监听端口（默认 2206）和数据库地址（默认 SQLite）。服务端默认监听 `127.0.0.1:2206`，本机访问 `http://127.0.0.1:2206/admin/login`；对外使用需经 HTTPS 反向代理，见[反向代理](#反向代理)。 -更新：重新运行安装脚本即可，配置数据保留。卸载加 `--purge` 删除配置数据。 +管理员密码仅首次初始化数据库时需要，初始化完成后会自动从配置文件中清除。 -Linux systemd： +### 安装脚本参数 -```bash -systemctl status nodeflare -systemctl restart nodeflare -journalctl -u nodeflare -f -``` +| 命令 | 说明 | +| --- | --- | +| `sudo sh install.sh` | 交互菜单 | +| `sudo sh install.sh --install` | 安装或更新 | +| `sudo sh install.sh --status` | 查看服务状态 | +| `sudo sh install.sh --restart` | 重启服务 | +| `sudo sh install.sh --uninstall` | 卸载，保留配置和数据 | +| `sudo sh install.sh --uninstall --purge` | 卸载并删除配置与数据 | + +Windows 对应参数为 `-Install` / `-Status` / `-Restart` / `-Uninstall [-Purge]`。 + +更新：重新运行安装脚本即可，配置数据保留。安装与更新都会校验 Release 摘要，失败时自动回滚到上一版本。 ## 安装 Agent -在管理后台“服务器”页面创建节点，执行弹窗中的安装命令。Agent 出站连接服务端，无需开放入站端口。替换面板地址和 Token。 +在管理后台「服务器」页面创建节点，执行弹窗中的安装命令。Agent 主动出站连接服务端，无需开放入站端口。 Linux： @@ -53,50 +113,114 @@ curl -fsSL https://raw.githubusercontent.com/elysia62/NodeFlare/main/agent/agent | sudo sh -s -- -e 'https://nodeflare.example.com' -t 'Agent Token' ``` -macOS ARM64 / FreeBSD / Windows 脚本见仓库 `agent/` 目录，安装参数相同。 +各平台脚本： -更新： +| 平台 | 脚本 | 服务方式 | +| --- | --- | --- | +| Linux | `agent/agent.sh` | systemd / OpenRC | +| macOS（Apple Silicon） | `agent/install-macos.sh` | launchd | +| FreeBSD | `agent/install-freebsd.sh` | rc.d | +| Windows | `agent/install.ps1` | 计划任务 | -```bash -curl -fsSL https://raw.githubusercontent.com/elysia62/NodeFlare/main/agent/agent.sh | sudo sh -s -- --update -``` +### Agent 参数 -Windows 用 `-Update` 参数。更新自动沿用已配置的地址和 Token，校验 Release 摘要，失败自动回滚。 +| 参数 | 说明 | +| --- | --- | +| `-e` | NodeFlare 服务地址（必填） | +| `-t` | Agent Token（必填） | +| `-i` | 初始历史保存间隔，15–3600 秒（默认 60） | +| `-m` | GitHub 下载加速前缀，如 `https://ghproxy.net`（可选） | +| `--update` | 更新 Agent，沿用已保存的地址与 Token，校验摘要，失败回滚 | +| `--status` | 查看 Agent 状态 | +| `--uninstall` | 卸载 Agent | -卸载： +Windows 对应 `-Endpoint` / `-Token` / `-Interval` / `-Mirror` 与 `-Update` / `-Status` / `-Uninstall`。 -```bash -curl -fsSL https://raw.githubusercontent.com/elysia62/NodeFlare/main/agent/agent.sh | sudo sh -s -- --uninstall -``` +> **关于 Token 传递**：`-t` 会出现在进程列表中。Agent 也支持环境变量 `NODEFLARE_AGENT_TOKEN`，以及 `--token-file <路径>` 从文件读取。Linux 安装脚本会把 Token 写入 `0600` 的 `agent.env` 供服务单元读取。 + +## 监控口径与采样 + +CPU、内存和网速每秒采样，默认每 3 秒压缩批量上传。服务器卡片每秒显示一个真实采样值，正常连接下有约 2～3 秒的显示延迟；断线重连后跳过积压的旧数据。磁盘容量、GPU 等较慢指标单独缓存，历史数据按节点设置的保存间隔聚合。增大实时上传间隔会降低卡片连续更新的频率。 + +Linux 内存统计与 Komari 默认口径一致：已用内存为 `MemTotal - MemFree - Cached - SReclaimable - Buffers + Shmem`，Swap 已用量扣除 `SwapCached`。文件缓存不计入已用内存，共享内存计入；面板显示的是整机内存，不是 NodeFlare 进程本身的占用。 ## 配置 -配置文件位置见[默认目录](#默认目录)，示例见 [`backend/config.example.toml`](backend/config.example.toml)。 +配置文件位置见[默认目录与日志](#默认目录与日志)，完整示例见 [`backend/config.example.toml`](backend/config.example.toml)。 | 配置项 | 说明 | | --- | --- | -| `database_url` | SQLite `sqlite://nodeflare.db` 或 PostgreSQL 连接串 | +| `database_url` | `sqlite://nodeflare.db` 或 PostgreSQL 连接串，如 `postgres://user:password@127.0.0.1:5432/nodeflare?sslmode=disable` | | `bind_addr` | 监听地址，默认 `127.0.0.1:2206` | -| `trusted_proxies` | 可信反向代理 IP / CIDR 列表 | -| `turnstile_site_key` / `turnstile_secret_key` | Turnstile 密钥，留空禁用 | +| `admin_username` | 管理员用户名 | +| `admin_password` | 管理员密码（8–128 字符），仅首次初始化数据库需要，成功后自动清空 | +| `trusted_proxies` | 可信反向代理 IP / CIDR 列表；只有来自这些网段的 `X-Forwarded-For` 才会被信任 | +| `turnstile_site_key` / `turnstile_secret_key` | Turnstile 密钥，留空则禁用 | | `session_ttl_hours` | 会话有效期（1–2160 小时），默认 168 | +| `frontend_dir` / `admin_frontend_dir` | 前端静态资源目录，相对路径按配置文件所在目录解析 | +| `theme_dir` | 主题解压目录，默认配置目录下的 `themes` | -管理员密码仅首次初始化数据库需要，之后自动清除。 +命令行可临时覆盖配置：`nodeflare --config <路径> --bind <地址> --database `。使用 `--database` 启动时，后台的在线迁移功能会被禁用。 -## 默认目录 +## 默认目录与日志 -| 平台 | 程序 | 配置和数据 | -| ------- | ------------------------------ | ----------------------------------------------- | -| Linux | `/opt/nodeflare` | `/etc/nodeflare` | -| Windows | `%ProgramFiles%\NodeFlare` | `%ProgramData%\NodeFlare\Server` | -| macOS | `/usr/local/libexec/nodeflare` | `/Library/Application Support/NodeFlare/Server` | -| FreeBSD | `/usr/local/libexec/nodeflare` | `/var/db/nodeflare/server` | +| 平台 | 程序 | 配置与数据 | +| --- | --- | --- | +| Linux | `/opt/nodeflare` | `/etc/nodeflare` | +| Windows | `%ProgramFiles%\NodeFlare` | `%ProgramData%\NodeFlare\Server` | +| macOS | `/usr/local/libexec/nodeflare` | `/Library/Application Support/NodeFlare/Server` | +| FreeBSD | `/usr/local/libexec/nodeflare` | `/var/db/nodeflare/server` | -SQLite 文件位于配置目录下。 +Agent（Linux）：程序 `/opt/nodeflare/agent`，配置与状态 `/etc/nodeflare/agent`。SQLite 文件位于配置目录下。 + +日志位置： + +- Linux（systemd）：`journalctl -u nodeflare -f`；Agent 为 `journalctl -u nodeflare-agent -f` +- Linux（OpenRC）：`rc-service nodeflare status` +- macOS：`/var/log/nodeflare.log` +- Windows：`Get-ScheduledTaskInfo -TaskName nodeflare` + +## 反向代理 + +对外访问请使用 HTTPS 反向代理，并把代理地址加入 `trusted_proxies`，否则所有访客会被算作同一个 IP，且会话 Cookie 不会带上 `Secure`： + +```toml +trusted_proxies = ["127.0.0.1/32", "::1/128"] +``` + +nginx： + +```nginx +server { + listen 443 ssl; + server_name nodeflare.example.com; + + ssl_certificate /etc/letsencrypt/live/nodeflare.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/nodeflare.example.com/privkey.pem; + + location / { + proxy_pass http://127.0.0.1:2206; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; # WebSocket 实时数据 + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} +``` + +Caddy： + +``` +nodeflare.example.com { + reverse_proxy 127.0.0.1:2206 +} +``` ## 数据库与备份 -后台“数据库”页面支持查看占用、回收空间、导出/恢复 ZIP 备份、SQLite ↔ PostgreSQL 迁移。备份包含设置、节点、历史、通知、主题、任务与安全配置，需 TOTP 或密码验证。迁移覆盖目标库并自动更新连接串，保留 Agent Token。备份上限 512 MiB / 解压 4 GiB。历史数据默认保留 30 天。 +后台「数据库」页面支持查看占用、回收空间、导出 / 恢复 ZIP 备份、SQLite ↔ PostgreSQL 迁移。备份包含设置、节点、历史、通知、主题、任务与安全配置，需要 TOTP 或密码验证。限制为 ZIP 512 MiB、解压后 4 GiB、最多 16384 个条目，单个主题文件 32 MiB。迁移会覆盖目标库并自动更新连接串，保留 Agent Token，重启服务后生效。历史数据默认保留 30 天。 ## 开发 @@ -108,7 +232,20 @@ cp backend/config.example.toml backend/config.toml ./dev.sh ``` -测试构建： +`dev.sh` 会构建前端并以 `cargo run` 启动后端。`start.sh` 则构建 release 版本，优先使用 `/etc/nodeflare/config.toml`，也可用 `NODEFLARE_CONFIG` 指定。 + +仓库结构： + +| 目录 | 内容 | +| --- | --- | +| `backend/` | 服务端（Rust / Axum / SQLx）：`src/routes` 接口、`src/db` 数据层、`src/websocket` 实时通道、`migrations/` 建表脚本 | +| `agent/` | Agent 源码（采集 / 上报 / 远程执行 / 自更新）与各平台安装脚本 | +| `shared/` | Agent 与服务端共用的遥测协议（序列化 + 压缩） | +| `frontend/` | 前端（React + Vite）：`src/components` 组件、`src/styles` 样式 | +| `scripts/` | 构建、版本解析与冒烟测试脚本 | +| `docs/` | 英文 README 与 systemd 服务单元 | + +测试与构建： ```bash bun test --cwd frontend @@ -117,6 +254,20 @@ cargo test --locked --manifest-path agent/Cargo.toml bun run build ``` +冒烟测试（需先启动面板）： + +```bash +MONITOR_ADMIN_USERNAME=admin MONITOR_ADMIN_PASSWORD='你的密码' bun run test:smoke +``` + +## 排障 + +- **服务启动失败**：先看日志（见[默认目录与日志](#默认目录与日志)），多为端口被占用或数据库连接串有误。 +- **不确定监听端口**：查看配置文件中的 `bind_addr`，默认 `127.0.0.1:2206`。 +- **Agent 显示离线**：确认 Agent 能出站访问面板地址（`curl -I https://你的面板地址`），并核对 Token；上报与延迟依赖时钟校准，系统时间偏差过大会有影响。 +- **公开看板看不到节点**：确认节点未启用「隐藏」，且站点设置中已开启「公开仪表盘」。 +- **导出备份报超限**：先缩短历史保留天数或清理历史数据，再重新导出。 + ## License MIT diff --git a/agent/agent.sh b/agent/agent.sh old mode 100644 new mode 100755 index f5b3a4e..f98ff62 --- a/agent/agent.sh +++ b/agent/agent.sh @@ -8,6 +8,9 @@ STATE_ROOT="/etc/nodeflare" STATE_DIR="$STATE_ROOT/agent" SERVICE_FILE="/etc/systemd/system/$SERVICE_NAME.service" OPENRC_FILE="/etc/init.d/$SERVICE_NAME" +# Holds NODEFLARE_AGENT_TOKEN for systemd; a 0600 file inside the 0750 state dir +# is readable only by root/uninstall, unlike Environment= in the unit. +TOKEN_ENV_FILE="$STATE_DIR/agent.env" temporary="" backup_agent="" backup_service="" @@ -347,6 +350,10 @@ install_agent() { mv "$temporary" "$AGENT_FILE" log "正在启动 $init_system 服务" if [ "$init_system" = "systemd" ]; then + # The token goes into a 0600 file rather than Environment=, which any local + # user can read back with `systemctl show`. + printf 'NODEFLARE_AGENT_TOKEN=%s\n' "$token" > "$TOKEN_ENV_FILE" + chmod 600 "$TOKEN_ENV_FILE" printf '%s\n' \ '[Unit]' \ 'Description=NodeFlare Agent' \ @@ -359,7 +366,7 @@ install_agent() { 'Restart=always' \ 'RestartSec=10' \ 'NoNewPrivileges=true' \ - "Environment=NODEFLARE_AGENT_TOKEN=$token" \ + "EnvironmentFile=$TOKEN_ENV_FILE" \ "Environment=NODEFLARE_STATE_DIR=$STATE_DIR" \ '' \ '[Install]' \ @@ -404,7 +411,15 @@ load_installed_agent_config() { service_exec=$(sed -n 's/^ExecStart=//p' "$SERVICE_FILE" | head -n 1) endpoint=$(printf '%s\n' "$service_exec" | sed -n 's/.* -e \([^[:space:]]*\) -i \([0-9][0-9]*\)$/\1/p') interval=$(printf '%s\n' "$service_exec" | sed -n 's/.* -e \([^[:space:]]*\) -i \([0-9][0-9]*\)$/\2/p') - token=$(sed -n 's/^Environment=NODEFLARE_AGENT_TOKEN=//p' "$SERVICE_FILE" | head -n 1) + token="" + if [ -f "$TOKEN_ENV_FILE" ]; then + token=$(sed -n 's/^NODEFLARE_AGENT_TOKEN=//p' "$TOKEN_ENV_FILE" | head -n 1) + fi + # Fall back to the unit file so installs created before the token moved + # into an EnvironmentFile still update cleanly. + if [ -z "$token" ]; then + token=$(sed -n 's/^Environment=NODEFLARE_AGENT_TOKEN=//p' "$SERVICE_FILE" | head -n 1) + fi ;; openrc) [ -f "$OPENRC_FILE" ] || fail "未找到 Agent 服务配置，请先完成安装" diff --git a/agent/install.ps1 b/agent/install.ps1 old mode 100644 new mode 100755 index 712a883..9b61a30 --- a/agent/install.ps1 +++ b/agent/install.ps1 @@ -1,4 +1,4 @@ -<# +<# .EXAMPLE .\install.ps1 -Update Update using the saved endpoint, token and history interval. diff --git a/agent/src/live.rs b/agent/src/live.rs new file mode 100755 index 0000000..5ebf2fe --- /dev/null +++ b/agent/src/live.rs @@ -0,0 +1,698 @@ +use super::*; + +pub(crate) struct LiveSender { + pub(crate) pending: Arc<(Mutex>, Condvar)>, + pub(crate) send_interval: Arc>, + pub(crate) persisted_through: Arc, + pub(crate) remote_config: Arc>>, + pub(crate) stats: Arc, +} + +pub(crate) struct LiveSenderWorker { + pub(crate) pending: Arc<(Mutex>, Condvar)>, + pub(crate) configured_interval: Arc>, + pub(crate) persisted_through: Arc, + pub(crate) remote_config: Arc>>, + pub(crate) clock: SharedClock, + pub(crate) stats: Arc, +} + +impl LiveSender { + pub(crate) fn start( + config: &RuntimeConfig, + clock: SharedClock, + stats: Arc, + ) -> Result { + let pending = Arc::new((Mutex::new(VecDeque::new()), Condvar::new())); + let send_interval = Arc::new(Mutex::new(live_batch_interval(config.collect_interval))); + let persisted_through = Arc::new(AtomicI64::new(0)); + let remote_config = Arc::new(Mutex::new(None)); + let endpoint = live_endpoint(&config.endpoint)?; + let token = config.token.clone(); + let sender_state = Arc::clone(&pending); + let sender_interval = Arc::clone(&send_interval); + let sender_persisted_through = Arc::clone(&persisted_through); + let sender_remote_config = Arc::clone(&remote_config); + let sender_stats = Arc::clone(&stats); + thread::Builder::new() + .name("nodeflare-live".to_string()) + .spawn(move || { + live_sender_loop( + &endpoint, + &token, + LiveSenderWorker { + pending: sender_state, + configured_interval: sender_interval, + persisted_through: sender_persisted_through, + remote_config: sender_remote_config, + clock, + stats: sender_stats, + }, + ) + })?; + Ok(Self { + pending, + send_interval, + persisted_through, + remote_config, + stats, + }) + } + + pub(crate) fn send(&self, report: &Report) { + let (pending, ready) = &*self.pending; + if let Ok(mut pending) = pending.lock() { + if pending.len() >= LIVE_QUEUE_CAPACITY { + pending.pop_front(); + self.stats.live_queue_dropped(1); + } + pending.push_back(report.clone()); + ready.notify_one(); + } + } + + pub(crate) fn set_send_interval(&self, collect_interval: u64) { + if let Ok(mut interval) = self.send_interval.lock() { + *interval = live_batch_interval(collect_interval); + } + self.pending.1.notify_one(); + } + + pub(crate) fn persisted_through(&self) -> i64 { + self.persisted_through.load(Ordering::Acquire) + } + + pub(crate) fn take_remote_config(&self) -> Option { + self.remote_config + .lock() + .ok() + .and_then(|mut config| config.take()) + } +} + +pub(crate) fn live_batch_interval(collect_interval: u64) -> Duration { + Duration::from_secs(collect_interval.clamp(telemetry::MIN_UPLOAD_INTERVAL, 60)) +} + +pub(crate) fn live_endpoint(endpoint: &str) -> Result { + let mut url = url::Url::parse(endpoint)?; + url.set_scheme(if url.scheme() == "https" { "wss" } else { "ws" }) + .map_err(|_| "unsupported live endpoint scheme")?; + let path = format!("{}/api/agent/ws", url.path().trim_end_matches('/')); + url.set_path(&path); + Ok(url.to_string()) +} + +pub(crate) type LiveSocket = tungstenite::WebSocket>; + +pub(crate) fn connect_live(endpoint: &str, token: &str) -> Result<(LiveSocket, Option)> { + let started_ms = unix_timestamp_millis(); + let deadline = Instant::now() + LIVE_CONNECT_TIMEOUT; + let mut url = url::Url::parse(endpoint)?; + let mut redirects = 0; + let (socket, response) = loop { + let mut request = url.as_str().into_client_request()?; + request + .headers_mut() + .insert("Authorization", format!("Bearer {token}").parse()?); + request + .headers_mut() + .insert("User-Agent", format!("nodeflare-agent/{VERSION}").parse()?); + request + .headers_mut() + .insert(AGENT_PROTOCOL_HEADER, AGENT_PROTOCOL_VERSION.parse()?); + request.headers_mut().insert( + AGENT_CAPABILITIES_HEADER, + REQUIRED_AGENT_CAPABILITIES.join(",").parse()?, + ); + let stream = connect_live_stream(&url, deadline)?; + match client_tls(request, stream) { + Ok(connected) => break connected, + Err(tungstenite::HandshakeError::Failure(WebSocketError::Http(response))) + if response.status().is_redirection() && redirects < 3 => + { + let location = response + .headers() + .get("location") + .ok_or("WebSocket redirect has no location")? + .to_str()?; + let redirected = url.join(location)?; + // Never forward the Agent token to a different origin. + if redirected.origin() != url.origin() { + return Err("WebSocket redirect must stay on the configured origin".into()); + } + url = redirected; + redirects += 1; + } + Err(error) => return Err(format!("WebSocket handshake failed: {error}").into()), + } + }; + let ended_ms = unix_timestamp_millis(); + let clock_offset_ms = response + .headers() + .get("date") + .and_then(|value| value.to_str().ok()) + .and_then(|value| clock_offset_from_http_date(value, started_ms, ended_ms)); + if !response + .headers() + .get(AGENT_PROTOCOL_HEADER) + .is_some_and(|value| value == AGENT_PROTOCOL_VERSION) + { + return Err(format!( + "backend protocol mismatch; update the backend to protocol {AGENT_PROTOCOL_VERSION}" + ) + .into()); + } + Ok((socket, clock_offset_ms)) +} + +pub(crate) fn connect_live_stream(url: &url::Url, deadline: Instant) -> Result { + let host = url + .host_str() + .ok_or("WebSocket endpoint has no hostname")? + .trim_start_matches('[') + .trim_end_matches(']'); + let port = url + .port_or_known_default() + .ok_or("WebSocket endpoint has no port")?; + let addresses = (host, port).to_socket_addrs()?; + let mut connected = None; + let mut last_error = io::Error::new( + ErrorKind::NotFound, + "WebSocket hostname resolved to no addresses", + ); + for address in addresses { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err( + io::Error::new(ErrorKind::TimedOut, "WebSocket connection timed out").into(), + ); + } + match TcpStream::connect_timeout(&address, remaining.min(Duration::from_secs(3))) { + Ok(stream) => { + connected = Some(stream); + break; + } + Err(error) => last_error = error, + } + } + let stream = connected.ok_or(last_error)?; + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(io::Error::new(ErrorKind::TimedOut, "WebSocket connection timed out").into()); + } + stream.set_nodelay(true)?; + stream.set_read_timeout(Some(remaining))?; + stream.set_write_timeout(Some(remaining))?; + Ok(stream) +} + +pub(crate) fn set_live_read_timeout(socket: &mut LiveSocket, timeout: Option) -> io::Result<()> { + match socket.get_mut() { + tungstenite::stream::MaybeTlsStream::Plain(stream) => stream.set_read_timeout(timeout), + tungstenite::stream::MaybeTlsStream::Rustls(stream) => { + stream.sock.set_read_timeout(timeout) + } + // An unmatched TLS backend keeps running without a read timeout rather + // than failing outright; the rustls-only feature set always matches an + // arm above, so this is defensive. + _ => Ok(()), + } +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct LiveAck { + #[serde(rename = "type")] + pub(crate) message_type: String, + pub(crate) ts: i64, + pub(crate) persisted: bool, + #[serde(rename = "persistenceError")] + pub(crate) persistence_error: bool, + #[serde(rename = "persistedThroughTs")] + pub(crate) persisted_through_ts: i64, + #[serde(rename = "nextPersistAfterMs")] + pub(crate) next_persist_after_ms: u64, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct LiveConfigMessage { + #[serde(rename = "type")] + pub(crate) message_type: String, + pub(crate) ts: i64, + pub(crate) config: RemoteConfig, +} + +pub(crate) fn ack_persist_interval(ack: &LiveAck) -> Duration { + Duration::from_millis(ack.next_persist_after_ms) + .clamp(Duration::from_secs(1), Duration::from_secs(3600)) +} + +pub(crate) enum LiveRead { + Ack(LiveAck), + Config(RemoteConfig), + RemoteTask(RemoteTaskMessage), + TaskResultAck(String), + Closed, + Pending, +} + +pub(crate) fn read_live_ack(socket: &mut LiveSocket) -> Result { + match socket.read() { + Ok(Message::Text(text)) => { + let Ok(value) = serde_json::from_str::(text.as_ref()) else { + return Ok(LiveRead::Pending); + }; + match value.get("type").and_then(serde_json::Value::as_str) { + Some("ack") => { + let Ok(ack) = serde_json::from_value::(value) else { + return Ok(LiveRead::Pending); + }; + if ack.message_type != "ack" || ack.ts <= 0 { + return Ok(LiveRead::Pending); + } + Ok(LiveRead::Ack(ack)) + } + Some("config") => { + let Ok(message) = serde_json::from_value::(value) else { + return Ok(LiveRead::Pending); + }; + if message.message_type != "config" || message.ts <= 0 { + return Ok(LiveRead::Pending); + } + Ok(LiveRead::Config(message.config)) + } + Some("remote_task") => { + let Ok(task) = serde_json::from_value::(value) else { + return Ok(LiveRead::Pending); + }; + if task.message_type != "remote_task" + || !valid_remote_task_id(&task.task_id) + || task.command.trim().is_empty() + || task.command.len() > 16_384 + { + return Ok(LiveRead::Pending); + } + Ok(LiveRead::RemoteTask(task)) + } + Some("task_result_ack") => { + let Ok(ack) = serde_json::from_value::(value) else { + return Ok(LiveRead::Pending); + }; + if ack.message_type != "task_result_ack" || !valid_remote_task_id(&ack.task_id) + { + return Ok(LiveRead::Pending); + } + Ok(LiveRead::TaskResultAck(ack.task_id)) + } + _ => Ok(LiveRead::Pending), + } + } + Ok(Message::Ping(payload)) => { + socket.send(Message::Pong(payload))?; + Ok(LiveRead::Pending) + } + Ok(Message::Close(_)) => Ok(LiveRead::Closed), + Ok(_) => Ok(LiveRead::Pending), + Err(WebSocketError::Io(error)) + if matches!(error.kind(), ErrorKind::TimedOut | ErrorKind::WouldBlock) => + { + Ok(LiveRead::Pending) + } + Err(error) => Err(error.into()), + } +} + +pub(crate) fn accept_remote_task( + pub(crate) socket: &mut LiveSocket, + pub(crate) executor: &mut RemoteExecutor, + pub(crate) task: RemoteTaskMessage, +) -> Result<()> { + let task_id = task.task_id.clone(); + let rejected = executor.enqueue(task); + socket.send(Message::Text( + serde_json::json!({ + "type": "task_received", + "task_id": task_id, + }) + .to_string() + .into(), + ))?; + if let Some(result) = rejected { + socket.send(Message::Text(serde_json::to_string(&result)?.into()))?; + } + flush_remote_results(socket, executor)?; + Ok(()) +} + +pub(crate) fn wait_for_live_ack( + pub(crate) socket: &mut LiveSocket, + pub(crate) remote_config: &Arc>>, + pub(crate) remote_executor: &mut RemoteExecutor, +) -> Result { + let deadline = Instant::now() + LIVE_ACK_READ_TIMEOUT; + loop { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(LiveRead::Pending); + } + set_live_read_timeout(socket, Some(remaining))?; + match read_live_ack(socket)? { + LiveRead::Ack(ack) => { + return Ok(LiveRead::Ack(ack)); + } + LiveRead::Config(config) => { + if let Ok(mut target) = remote_config.lock() { + *target = Some(config); + } + } + LiveRead::RemoteTask(task) => { + accept_remote_task(socket, remote_executor, task)?; + } + LiveRead::TaskResultAck(task_id) => { + remote_executor.acknowledge(&task_id); + } + LiveRead::Closed => return Ok(LiveRead::Closed), + LiveRead::Pending => {} + } + } +} + +pub(crate) fn live_update_payload( + pub(crate) reports: Vec, + pub(crate) persist: bool, + pub(crate) info: &mut Option, +) -> Result> { + let mut next_info = info.clone(); + let payload = telemetry::encode(&telemetry::Update::from_reports( + reports, + persist, + &mut next_info, + ))?; + *info = next_info; + Ok(payload) +} + +pub(crate) fn observe_persisted_through(target: &AtomicI64, ack: &LiveAck) { + if ack.persisted && !ack.persistence_error { + target.fetch_max(ack.persisted_through_ts.max(0), Ordering::AcqRel); + } +} + +pub(crate) fn prune_live_queue(pending: &Arc<(Mutex>, Condvar)>, persisted_through: i64) { + if persisted_through <= 0 { + return; + } + if let Ok(mut queue) = pending.0.lock() { + queue.retain(|report| report.timestamp > persisted_through); + } +} + +pub(crate) fn live_batch_after(queue: &VecDeque, timestamp: i64) -> Vec { + live_batch::batch_from(queue.iter().filter(|report| report.timestamp > timestamp)) +} + +/// Bounded exponential backoff for the live connection, with jitter derived +/// from the token so a fleet of agents that dropped together does not retry in +/// lockstep and hammer an unreachable server. +pub(crate) fn reconnect_delay(attempt: u32, token: &str) -> Duration { + let base_ms = LIVE_RECONNECT_DELAY.as_millis() as u64; + let backoff_ms = base_ms + .saturating_mul(1_u64 << attempt.min(5)) + .min(LIVE_RECONNECT_MAX_DELAY.as_millis() as u64); + let digest = Sha256::digest(token.as_bytes()); + let seed = u64::from(digest[0]) ^ (u64::from(attempt) << 8); + let jitter_ms = seed % (backoff_ms / 4 + 1); + Duration::from_millis(backoff_ms + jitter_ms) +} + +pub(crate) struct ReconnectBackoff { + pub(crate) attempt: u32, + pub(crate) token: String, +} + +impl ReconnectBackoff { + pub(crate) fn new(token: &str) -> Self { + Self { + attempt: 0, + token: token.to_string(), + } + } + + /// Reset after a confirmed round trip, not merely after a TCP connect, so a + /// server that accepts then immediately drops connections cannot keep us at + /// the shortest delay. + pub(crate) fn reset(&mut self) { + self.attempt = 0; + } + + pub(crate) fn next_delay(&mut self) -> Duration { + self.attempt = self.attempt.saturating_add(1); + reconnect_delay(self.attempt, &self.token) + } +} + +pub(crate) fn live_sender_loop(endpoint: &str, token: &str, worker: LiveSenderWorker) { + let LiveSenderWorker { + pending, + configured_interval, + persisted_through, + remote_config, + clock, + stats, + } = worker; + let mut socket: Option = None; + let mut backoff = ReconnectBackoff::new(token); + let mut wss_interval = configured_interval.lock().map_or( + Duration::from_secs(telemetry::MIN_UPLOAD_INTERVAL), + |interval| *interval, + ); + let mut info = None; + let mut accepted_through = persisted_through.load(Ordering::Acquire); + let mut next_send_at = Instant::now(); + let mut next_probe_at = Instant::now(); + let mut remote_executor = match RemoteExecutor::new() { + Ok(executor) => executor, + Err(error) => { + eprintln!("remote executor disabled: {error}"); + RemoteExecutor::disabled() + } + }; + 'sender: loop { + if socket.is_none() { + match connect_live(endpoint, token) { + Ok((mut connected, offset_ms)) => { + observe_clock(&clock, offset_ms); + if set_live_read_timeout(&mut connected, Some(LIVE_HINT_READ_TIMEOUT)).is_err() + { + thread::sleep(backoff.next_delay()); + continue; + } + socket = Some(connected); + info = None; + remote_executor.reset_delivery(); + accepted_through = persisted_through.load(Ordering::Acquire); + next_send_at = Instant::now(); + next_probe_at = Instant::now(); + } + Err(error) => { + stats.live_connect_failed(); + eprintln!("live connection failed: {error}"); + thread::sleep(backoff.next_delay()); + continue; + } + } + } + + if let Some(connected) = socket.as_mut() + && flush_remote_results(connected, &mut remote_executor).is_err() + { + socket = None; + thread::sleep(backoff.next_delay()); + continue; + } + + let (queue_lock, ready) = &*pending; + let Ok(mut queue) = queue_lock.lock() else { + return; + }; + let (batch, persist, more_pending) = loop { + let now = Instant::now(); + let persisted = persisted_through.load(Ordering::Acquire); + let has_unsent = queue.iter().any(|report| report.timestamp > accepted_through); + let persist = + now >= next_probe_at && queue.iter().any(|report| report.timestamp > persisted); + if persist || (has_unsent && now >= next_send_at) { + // Build (and serialize) the batch only once it is actually due; + // the wait below re-runs several times per second while idle. + let unsent = live_batch_after(&queue, accepted_through); + let last = unsent + .last() + .map_or(accepted_through, |report| report.timestamp); + let more_pending = queue.iter().any(|report| report.timestamp > last); + // A commit can be empty: samples already sent remain buffered by the server. + break (unsent, persist || more_pending, more_pending); + } + + let wake_at = if has_unsent { + next_send_at.min(next_probe_at) + } else if queue.iter().any(|report| report.timestamp > persisted) { + next_probe_at + } else { + now + Duration::from_millis(250) + }; + let wait = wake_at + .saturating_duration_since(now) + .min(Duration::from_millis(250)); + queue = match ready.wait_timeout(queue, wait) { + Ok((queue, _)) => queue, + Err(_) => return, + }; + drop(queue); + + let mut drop_socket = false; + if let Some(connected) = socket.as_mut() { + if flush_remote_results(connected, &mut remote_executor).is_err() + || set_live_read_timeout(connected, Some(LIVE_HINT_READ_TIMEOUT)).is_err() + { + drop_socket = true; + } else { + match read_live_ack(connected) { + Ok(LiveRead::Closed) => drop_socket = true, + Ok(LiveRead::Ack(ack)) => { + observe_persisted_through(&persisted_through, &ack); + if ack.persistence_error { + stats.live_persistence_failed(); + drop_socket = true; + } else { + prune_live_queue(&pending, ack.persisted_through_ts); + accepted_through = accepted_through.max(ack.persisted_through_ts); + next_probe_at = Instant::now() + ack_persist_interval(&ack); + } + } + Ok(LiveRead::Config(config)) => { + if let Ok(mut target) = remote_config.lock() { + *target = Some(config); + } + } + Ok(LiveRead::RemoteTask(task)) => { + if accept_remote_task(connected, &mut remote_executor, task).is_err() { + drop_socket = true; + } + } + Ok(LiveRead::TaskResultAck(task_id)) => { + remote_executor.acknowledge(&task_id); + } + Ok(LiveRead::Pending) => {} + Err(error) => { + eprintln!("live hint read failed: {error}"); + drop_socket = true; + } + } + } + } + if drop_socket { + socket = None; + accepted_through = persisted_through.load(Ordering::Acquire); + next_send_at = Instant::now(); + next_probe_at = Instant::now(); + thread::sleep(backoff.next_delay()); + continue 'sender; + } + queue = match queue_lock.lock() { + Ok(queue) => queue, + Err(_) => return, + }; + }; + drop(queue); + + if let Ok(interval) = configured_interval.lock() { + wss_interval = (*interval).clamp( + Duration::from_secs(telemetry::MIN_UPLOAD_INTERVAL), + Duration::from_secs(60), + ); + } + + let batch_last = batch.last().map(|report| report.timestamp); + let batch_len = batch.len(); + let payload = match live_update_payload(batch, persist, &mut info) { + Ok(payload) => payload, + Err(error) => { + eprintln!("live payload encode failed: {error}"); + next_send_at = Instant::now() + wss_interval; + continue; + } + }; + let sent = socket + .as_mut() + .is_some_and(|socket| socket.send(Message::Binary(payload.into())).is_ok()); + if !sent { + socket = None; + accepted_through = persisted_through.load(Ordering::Acquire); + next_send_at = Instant::now(); + next_probe_at = Instant::now(); + thread::sleep(backoff.next_delay()); + continue; + } + stats.live_batch_sent(batch_len); + backoff.reset(); + if let Some(timestamp) = batch_last { + accepted_through = accepted_through.max(timestamp); + } + next_send_at = Instant::now() + wss_interval; + if !persist { + continue; + } + + let mut drop_socket = false; + if let Some(socket) = socket.as_mut() { + match wait_for_live_ack(socket, &remote_config, &mut remote_executor) { + Ok(LiveRead::Closed) => drop_socket = true, + Ok(LiveRead::Ack(ack)) => { + observe_persisted_through(&persisted_through, &ack); + if ack.persistence_error { + stats.live_persistence_failed(); + drop_socket = true; + } else { + prune_live_queue(&pending, ack.persisted_through_ts); + accepted_through = accepted_through.max(ack.persisted_through_ts); + next_probe_at = Instant::now() + ack_persist_interval(&ack); + if more_pending + && batch_last + .is_some_and(|timestamp| ack.persisted_through_ts >= timestamp) + { + next_probe_at = Instant::now(); + } + } + } + // wait_for_live_ack consumes config and remote-task messages + // internally, so only Ack/Closed/Pending reach this match; these + // arms are kept defensively in case that contract changes. + Ok(LiveRead::Config(config)) => { + if let Ok(mut target) = remote_config.lock() { + *target = Some(config); + } + drop_socket = true; + } + Ok(LiveRead::RemoteTask(_) | LiveRead::TaskResultAck(_)) => {} + Ok(LiveRead::Pending) => { + eprintln!("live ACK timed out"); + drop_socket = true; + } + Err(error) => { + eprintln!("live ACK read failed: {error}"); + drop_socket = true; + } + } + } + if drop_socket { + socket = None; + accepted_through = persisted_through.load(Ordering::Acquire); + next_send_at = Instant::now(); + next_probe_at = Instant::now(); + thread::sleep(backoff.next_delay()); + } + } +} + diff --git a/agent/src/live_batch.rs b/agent/src/live_batch.rs old mode 100644 new mode 100755 index 2b5ab5b..b584d68 --- a/agent/src/live_batch.rs +++ b/agent/src/live_batch.rs @@ -1,13 +1,17 @@ -use std::collections::VecDeque; - use super::Report; pub(crate) const MAX_LIVE_BATCH_BYTES: usize = nodeflare_telemetry::MAX_BATCH_BYTES; -pub(crate) fn batch_len(queue: &VecDeque) -> usize { +/// Collects the longest prefix of `queue` that fits one live batch, cloning only +/// the reports that are actually selected and serializing each candidate once +/// (the size check needs the encoded length). +pub(crate) fn batch_from<'a>(queue: impl Iterator) -> Vec { let mut encoded_bytes = 192_usize; - let mut count = 0_usize; + let mut batch = Vec::new(); for report in queue { + if batch.len() >= super::LIVE_QUEUE_CAPACITY { + break; + } let Ok(report_bytes) = serde_json::to_vec(report) else { break; }; @@ -18,19 +22,19 @@ pub(crate) fn batch_len(queue: &VecDeque) -> usize { break; } encoded_bytes = next_bytes; - count += 1; + batch.push(report.clone()); if encoded_bytes >= MAX_LIVE_BATCH_BYTES { break; } } - count.min(super::LIVE_QUEUE_CAPACITY) + batch } #[cfg(test)] mod tests { use std::collections::VecDeque; - use super::{MAX_LIVE_BATCH_BYTES, batch_len}; + use super::{MAX_LIVE_BATCH_BYTES, batch_from}; use crate::Report; #[test] @@ -40,7 +44,7 @@ mod tests { ..Report::default() }; let queue = std::iter::repeat_n(report, 10).collect::>(); - let count = batch_len(&queue); + let count = batch_from(queue.iter()).len(); assert!((1..10).contains(&count)); let payload = crate::live_update_payload( queue.iter().take(count).cloned().collect::>(), @@ -92,7 +96,7 @@ mod tests { ..Report::default() }) .collect::>(); - assert!(batch_len(&queue) <= crate::LIVE_QUEUE_CAPACITY); + assert!(batch_from(queue.iter()).len() <= crate::LIVE_QUEUE_CAPACITY); let reports = crate::live_batch_after(&queue, 100); assert_eq!(reports.first().unwrap().timestamp, 101); for persist in [true, false] { diff --git a/agent/src/main.rs b/agent/src/main.rs old mode 100644 new mode 100755 index 30decc6..2c3fd93 --- a/agent/src/main.rs +++ b/agent/src/main.rs @@ -1,5 +1,17 @@ +mod live; mod live_batch; +mod metrics; +mod remote; mod runtime_stats; +mod update; + +// Re-exported so the rest of the crate keeps referring to these items by their +// original unqualified names (and so tests can keep using `crate::name`). +pub(crate) use live::*; +pub(crate) use metrics::*; +pub(crate) use remote::*; +pub(crate) use update::*; + use nodeflare_telemetry as telemetry; use telemetry::{ AGENT_CAPABILITIES_HEADER, AGENT_PROTOCOL_HEADER, AGENT_PROTOCOL_VERSION, DiskMetric, @@ -62,12 +74,15 @@ const UPDATE_CHECK_INTERVAL: Duration = Duration::from_secs(6 * 60 * 60); const UPDATE_RETRY_INTERVAL: Duration = Duration::from_secs(30 * 60); const UPDATE_CHECK_JITTER_MAX_SECONDS: u64 = 30 * 60; const LIVE_RECONNECT_DELAY: Duration = Duration::from_secs(3); +const LIVE_RECONNECT_MAX_DELAY: Duration = Duration::from_secs(60); +const SPOOL_REWRITE_MIN_INTERVAL: Duration = Duration::from_secs(30); const LIVE_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); const LIVE_QUEUE_CAPACITY: usize = 720; const MAX_PENDING_SPOOL_BYTES: u64 = 64 * 1024 * 1024; const MAX_PENDING_SPOOL_LINE_BYTES: usize = 1024 * 1024; const LIVE_ACK_READ_TIMEOUT: Duration = Duration::from_secs(5); const LIVE_HINT_READ_TIMEOUT: Duration = Duration::from_millis(10); +const COMMAND_TIMEOUT: Duration = Duration::from_secs(5); const BASIC_INFO_REFRESH_INTERVAL: Duration = Duration::from_secs(5 * 60); const SLOW_METRICS_REFRESH_INTERVAL: Duration = Duration::from_secs(5); const SAMPLE_INTERVAL: Duration = Duration::from_secs(1); @@ -105,8 +120,12 @@ struct CliOptions { #[arg(short = 'e', value_name = "URL")] endpoint: String, /// Agent token - #[arg(short = 't', value_name = "TOKEN")] + #[arg(short = 't', value_name = "TOKEN", conflicts_with = "token_file")] token: Option, + /// Read the Agent token from a file instead of -t, which keeps it out of + /// the process list + #[arg(long = "token-file", value_name = "PATH")] + token_file: Option, /// Initial report interval in seconds (15-3600) #[arg(short = 'i', value_name = "SECONDS", default_value_t = 60)] interval: u64, @@ -234,19 +253,6 @@ struct RemoteConfig { latency_tasks: Vec, } -#[derive(Debug, Deserialize)] -struct GithubRelease { - tag_name: String, - assets: Vec, -} - -#[derive(Debug, Deserialize)] -struct GithubReleaseAsset { - name: String, - browser_download_url: String, - digest: Option, -} - #[derive(Debug, Default)] struct ClockCalibration { offset_ms: i64, @@ -271,7 +277,7 @@ impl ClockCalibration { type SharedClock = Arc>; #[derive(Default)] -struct PublicIpValue { +pub(crate) struct PublicIpValue { address: Option, last_success: Option, } @@ -291,13 +297,13 @@ impl PublicIpValue { } #[derive(Default)] -struct PublicIps { +pub(crate) struct PublicIps { v4: PublicIpValue, v6: PublicIpValue, } #[derive(Clone, Default)] -struct PublicIpProbe { +pub(crate) struct PublicIpProbe { state: Arc>, probing: Arc, last_attempt: Arc>>, @@ -382,739 +388,6 @@ fn parse_public_ip(text: &str, ipv6: bool) -> Option { } } -struct LiveSender { - pending: Arc<(Mutex>, Condvar)>, - send_interval: Arc>, - persisted_through: Arc, - remote_config: Arc>>, - stats: Arc, -} - -struct LiveSenderWorker { - pending: Arc<(Mutex>, Condvar)>, - configured_interval: Arc>, - persisted_through: Arc, - remote_config: Arc>>, - clock: SharedClock, - stats: Arc, -} - -impl LiveSender { - fn start( - config: &RuntimeConfig, - clock: SharedClock, - stats: Arc, - ) -> Result { - let pending = Arc::new((Mutex::new(VecDeque::new()), Condvar::new())); - let send_interval = Arc::new(Mutex::new(live_batch_interval(config.collect_interval))); - let persisted_through = Arc::new(AtomicI64::new(0)); - let remote_config = Arc::new(Mutex::new(None)); - let endpoint = live_endpoint(&config.endpoint)?; - let token = config.token.clone(); - let sender_state = Arc::clone(&pending); - let sender_interval = Arc::clone(&send_interval); - let sender_persisted_through = Arc::clone(&persisted_through); - let sender_remote_config = Arc::clone(&remote_config); - let sender_stats = Arc::clone(&stats); - thread::Builder::new() - .name("nodeflare-live".to_string()) - .spawn(move || { - live_sender_loop( - &endpoint, - &token, - LiveSenderWorker { - pending: sender_state, - configured_interval: sender_interval, - persisted_through: sender_persisted_through, - remote_config: sender_remote_config, - clock, - stats: sender_stats, - }, - ) - })?; - Ok(Self { - pending, - send_interval, - persisted_through, - remote_config, - stats, - }) - } - - fn send(&self, report: &Report) { - let (pending, ready) = &*self.pending; - if let Ok(mut pending) = pending.lock() { - if pending.len() >= LIVE_QUEUE_CAPACITY { - pending.pop_front(); - self.stats.live_queue_dropped(1); - } - pending.push_back(report.clone()); - ready.notify_one(); - } - } - - fn set_send_interval(&self, collect_interval: u64) { - if let Ok(mut interval) = self.send_interval.lock() { - *interval = live_batch_interval(collect_interval); - } - self.pending.1.notify_one(); - } - - fn persisted_through(&self) -> i64 { - self.persisted_through.load(Ordering::Acquire) - } - - fn take_remote_config(&self) -> Option { - self.remote_config - .lock() - .ok() - .and_then(|mut config| config.take()) - } -} - -fn live_batch_interval(collect_interval: u64) -> Duration { - Duration::from_secs(collect_interval.clamp(telemetry::MIN_UPLOAD_INTERVAL, 60)) -} - -#[derive(Debug, Default, Clone)] -struct BasicMetrics { - cpu_cores: i64, - cpu_model: String, - os: String, - kernel: String, - arch: String, - virtualization: String, - gpu_usage: f64, - gpu_model: String, - gpus: Vec, -} - -#[derive(Debug, Default, Clone)] -struct SlowMetrics { - disks: Vec, - processes: i64, - tcp_connections: i64, - udp_connections: i64, -} - -#[cfg(target_os = "linux")] -#[derive(Debug, Default, Clone, Copy)] -struct CpuSample { - total: u64, - idle: u64, -} - -#[cfg(target_os = "linux")] -#[derive(Debug, Default, Clone, Copy)] -struct IoSample { - rx: u64, - tx: u64, - read_ops: u64, - read_sectors: u64, - write_ops: u64, - write_sectors: u64, - read_millis: u64, - write_millis: u64, - io_millis: u64, -} - -#[cfg(target_os = "linux")] -fn text(path: impl AsRef) -> String { - fs::read_to_string(path).unwrap_or_default() -} - -fn command(name: &str, args: &[&str]) -> String { - Command::new(name) - .args(args) - .output() - .ok() - .filter(|output| output.status.success()) - .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_string()) - .unwrap_or_default() -} - -#[cfg(target_os = "linux")] -fn cpu_sample() -> CpuSample { - let line = text("/proc/stat").lines().next().unwrap_or("").to_string(); - let values = line - .split_whitespace() - .skip(1) - .filter_map(|value| value.parse::().ok()) - .collect::>(); - CpuSample { - total: values.iter().sum(), - idle: values.get(3).copied().unwrap_or(0) + values.get(4).copied().unwrap_or(0), - } -} - -fn selected_interface(name: &str, filter: &str) -> bool { - let name = name.trim(); - if name.is_empty() { - return false; - } - let mut has_include = false; - let mut included = false; - for pattern in filter - .split(',') - .map(str::trim) - .filter(|value| !value.is_empty()) - { - if let Some(pattern) = pattern - .strip_prefix('!') - .or_else(|| pattern.strip_prefix('-')) - { - if !pattern.is_empty() && wildcard_match(pattern, name) { - return false; - } - } else { - has_include = true; - included |= wildcard_match(pattern, name); - } - } - // Explicit includes can select a bridge; automatic selection avoids double counting. - if has_include { - return included; - } - let lower = name.to_ascii_lowercase(); - !lower.contains("loopback") - && ![ - "lo", "br", "cni", "docker", "podman", "flannel", "veth", "virbr", "vmbr", "tap", - "fwbr", "fwpr", - ] - .iter() - .any(|prefix| lower.starts_with(prefix)) -} - -fn wildcard_match(pattern: &str, value: &str) -> bool { - let pattern = pattern.as_bytes(); - let value = value.as_bytes(); - let (mut p, mut v, mut star, mut star_value) = (0, 0, None, 0); - while v < value.len() { - if p < pattern.len() && (pattern[p] == b'?' || pattern[p] == value[v]) { - p += 1; - v += 1; - } else if p < pattern.len() && pattern[p] == b'*' { - star = Some(p); - star_value = v; - p += 1; - } else if let Some(star_position) = star { - p = star_position + 1; - star_value += 1; - v = star_value; - } else { - return false; - } - } - while p < pattern.len() && pattern[p] == b'*' { - p += 1; - } - p == pattern.len() -} - -#[cfg(target_os = "linux")] -fn disk_device(name: &str) -> bool { - ((name.starts_with("sd") || name.starts_with("vd") || name.starts_with("xvd")) - && name - .chars() - .last() - .is_some_and(|value| value.is_ascii_alphabetic())) - || (name.starts_with("nvme") && name.contains('n') && !name.contains('p')) - || (name.starts_with("mmcblk") && !name.contains('p')) -} - -#[cfg(target_os = "linux")] -fn io_sample(filter: &str) -> IoSample { - let mut sample = IoSample::default(); - for line in text("/proc/net/dev") - .lines() - .filter(|line| line.contains(':')) - { - let Some((name, values)) = line.split_once(':') else { - continue; - }; - if !selected_interface(name.trim(), filter) { - continue; - } - let fields = values.split_whitespace().collect::>(); - sample.rx = sample - .rx - .saturating_add(fields.first().and_then(|v| v.parse().ok()).unwrap_or(0)); - sample.tx = sample - .tx - .saturating_add(fields.get(8).and_then(|v| v.parse().ok()).unwrap_or(0)); - } - for line in text("/proc/diskstats").lines() { - let fields = line.split_whitespace().collect::>(); - if fields.len() < 13 || !disk_device(fields[2]) { - continue; - } - sample.read_ops = sample - .read_ops - .saturating_add(fields[3].parse().unwrap_or(0)); - sample.read_sectors = sample - .read_sectors - .saturating_add(fields[5].parse().unwrap_or(0)); - sample.write_ops = sample - .write_ops - .saturating_add(fields[7].parse().unwrap_or(0)); - sample.write_sectors = sample - .write_sectors - .saturating_add(fields[9].parse().unwrap_or(0)); - sample.read_millis = sample - .read_millis - .saturating_add(fields[6].parse().unwrap_or(0)); - sample.write_millis = sample - .write_millis - .saturating_add(fields[10].parse().unwrap_or(0)); - sample.io_millis = sample - .io_millis - .saturating_add(fields[12].parse().unwrap_or(0)); - } - sample -} - -#[cfg(target_os = "linux")] -fn mem_value(contents: &str, key: &str) -> i64 { - contents - .lines() - .find(|line| line.starts_with(key)) - .and_then(|line| line.split_whitespace().nth(1)) - .and_then(|value| value.parse::().ok()) - .map_or(0, |value| u64_to_i64(value.saturating_mul(1024))) -} - -#[cfg(target_os = "linux")] -struct MemorySample { - total: i64, - used: i64, - swap_total: i64, - swap_used: i64, -} - -#[cfg(target_os = "linux")] -fn memory_sample(contents: &str) -> MemorySample { - let total = mem_value(contents, "MemTotal:"); - let free = mem_value(contents, "MemFree:"); - let reclaimable = free - .saturating_add(mem_value(contents, "Cached:")) - .saturating_add(mem_value(contents, "SReclaimable:")) - .saturating_add(mem_value(contents, "Buffers:")); - // Komari's default Linux/htop calculation excludes reclaimable caches but - // includes shared memory. MemAvailable uses a different kernel estimate. - let used = total - .saturating_sub(if reclaimable <= total { - reclaimable - } else { - free - }) - .saturating_add(mem_value(contents, "Shmem:")) - .clamp(0, total); - let swap_total = mem_value(contents, "SwapTotal:"); - let swap_free = mem_value(contents, "SwapFree:"); - let swap_deductions = swap_free.saturating_add(mem_value(contents, "SwapCached:")); - let swap_used = swap_total - .saturating_sub(if swap_deductions <= swap_total { - swap_deductions - } else { - swap_free - }) - .clamp(0, swap_total); - MemorySample { - total, - used, - swap_total, - swap_used, - } -} - -#[cfg(target_os = "linux")] -fn file_line_count(path: &str) -> i64 { - text(path).lines().skip(1).count() as i64 -} - -#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] -fn connection_counts_from_netstat(output: &str) -> (i64, i64) { - output.lines().fold((0_i64, 0_i64), |(tcp, udp), line| { - let protocol = line - .split_whitespace() - .next() - .unwrap_or_default() - .to_ascii_lowercase(); - if protocol.starts_with("tcp") { - (tcp.saturating_add(1), udp) - } else if protocol.starts_with("udp") { - (tcp, udp.saturating_add(1)) - } else { - (tcp, udp) - } - }) -} - -#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] -fn connection_counts() -> (i64, i64) { - Command::new("netstat") - .args(["-an"]) - .output() - .ok() - .filter(|output| output.status.success()) - .map(|output| connection_counts_from_netstat(&String::from_utf8_lossy(&output.stdout))) - .unwrap_or_default() -} - -#[cfg(target_os = "linux")] -fn disk_usage() -> Vec { - disk_mounts(&text("/proc/self/mounts")) - .filter_map(|(name, mount_point)| { - // Ignore file bind mounts such as a container's /etc/hosts. - if !Path::new(&mount_point).is_dir() { - return None; - } - let (total, used) = filesystem_space(&mount_point)?; - Some(DiskMetric { - name, - mount_point, - used, - total, - ..DiskMetric::default() - }) - }) - .collect() -} - -#[cfg(target_os = "linux")] -fn disk_mounts(contents: &str) -> impl Iterator + '_ { - fn unescape(value: &str) -> String { - value - .replace("\\040", " ") - .replace("\\011", "\t") - .replace("\\012", "\n") - .replace("\\134", "\\") - } - contents.lines().filter_map(|line| { - let mut fields = line.split_whitespace(); - let source = unescape(fields.next()?); - let mount_point = unescape(fields.next()?); - let filesystem = fields.next()?; - if (!source.starts_with('/') && source.contains(':')) - || source.starts_with("//") - || excluded_filesystem(filesystem, &mount_point) - { - return None; - } - Some((source, mount_point)) - }) -} - -#[cfg(target_os = "linux")] -fn filesystem_space(mount_point: &str) -> Option<(i64, i64)> { - let path = std::ffi::CString::new(mount_point).ok()?; - let mut stat = std::mem::MaybeUninit::::uninit(); - // SAFETY: path is NUL-terminated and stat points to writable storage of the - // platform's statvfs type, including both glibc and musl layouts. - if unsafe { libc::statvfs(path.as_ptr(), stat.as_mut_ptr()) } != 0 { - return None; - } - // SAFETY: statvfs returned success and initialized stat. - let stat = unsafe { stat.assume_init() }; - let block_size = u128::from(if stat.f_frsize == 0 { - stat.f_bsize - } else { - stat.f_frsize - }); - let total = (u128::from(stat.f_blocks) * block_size).min(i64::MAX as u128) as i64; - let used = (u128::from(stat.f_blocks.saturating_sub(stat.f_bfree)) * block_size) - .min(i64::MAX as u128) as i64; - (total > 0).then_some((total, used)) -} - -#[cfg(target_os = "linux")] -fn excluded_filesystem(filesystem: &str, mount_point: &str) -> bool { - // Remote/autofs mounts can block indefinitely while their server is down. - if matches!( - filesystem, - "nfs" - | "nfs4" - | "cifs" - | "smb3" - | "autofs" - | "9p" - | "ceph" - | "afs" - | "fuse.sshfs" - | "fuse.glusterfs" - | "fuse.rclone" - | "fuse.s3fs" - ) { - return true; - } - let mount_point = mount_point.to_ascii_lowercase(); - // Keep the root filesystem visible even when a container reports it as overlay. - if mount_point == "/" { - return false; - } - let filesystem = filesystem.to_ascii_lowercase(); - [ - "tmpfs", - "devtmpfs", - "devpts", - "proc", - "sysfs", - "cgroup", - "cgroup2", - "overlay", - "squashfs", - "efivarfs", - "pstore", - "mqueue", - "hugetlbfs", - "debugfs", - "fusectl", - ] - .iter() - .any(|value| filesystem == *value || filesystem.starts_with(&format!("{value}."))) - || ["/proc", "/sys", "/run", "/dev"] - .iter() - .any(|prefix| mount_point == *prefix || mount_point.starts_with(&format!("{prefix}/"))) -} - -#[cfg(any(target_os = "linux", test))] -fn disk_identity(name: &str) -> String { - let name = name.trim(); - // ZFS reports datasets as pool/dataset. Keep ordinary absolute paths and - // remote sources separate because their slash is part of the path. - if !name.starts_with('/') && !name.contains(':') && name.contains('/') { - return name.split('/').next().unwrap_or(name).to_string(); - } - name.to_string() -} - -#[cfg(any(target_os = "linux", test))] -fn dedupe_disks(disks: impl IntoIterator) -> Vec { - let mut unique = HashMap::::new(); - for disk in disks { - let key = disk_identity(&disk.name); - let replace = unique.get(&key).is_none_or(|current| { - disk.total > current.total - || (disk.total == current.total - && disk.mount_point.len() < current.mount_point.len()) - }); - if replace { - unique.insert(key, disk); - } - } - let mut disks = unique.into_values().collect::>(); - disks.sort_by(|left, right| left.mount_point.cmp(&right.mount_point)); - disks -} - -#[cfg(target_os = "linux")] -fn os_name() -> String { - text("/etc/os-release") - .lines() - .find_map(|line| line.strip_prefix("PRETTY_NAME=")) - .map_or_else( - || command("uname", &["-s"]), - |value| value.trim_matches('"').to_string(), - ) -} - -#[cfg(target_os = "linux")] -fn cpu_model() -> String { - text("/proc/cpuinfo") - .lines() - .find_map(|line| { - let (key, value) = line.split_once(':')?; - matches!(key.trim(), "model name" | "Hardware").then(|| value.trim().to_string()) - }) - .unwrap_or_default() -} - -fn valid_probe_host(host: &str) -> bool { - if host.is_empty() || host.len() > 50 || host.starts_with('.') || host.ends_with('.') { - return false; - } - let labels = host.split('.').collect::>(); - let ipv4_like = labels.len() == 4 - && labels - .iter() - .all(|label| !label.is_empty() && label.chars().all(|c| c.is_ascii_digit())); - if ipv4_like { - return host - .parse() - .is_ok_and(|address| is_public_probe_ip(IpAddr::V4(address))); - } - let lower = host.to_ascii_lowercase(); - if labels.len() < 2 - || ["local", "localhost", "internal", "lan", "localdomain"] - .iter() - .any(|suffix| lower == *suffix || lower.ends_with(&format!(".{suffix}"))) - || lower == "home.arpa" - || lower.ends_with(".home.arpa") - { - return false; - } - labels.iter().all(|label| { - !label.is_empty() - && label - .chars() - .all(|character| character.is_ascii_alphanumeric() || character == '-') - && label - .chars() - .next() - .is_some_and(|character| character.is_ascii_alphanumeric()) - && label - .chars() - .last() - .is_some_and(|character| character.is_ascii_alphanumeric()) - }) -} - -fn is_public_probe_ip(address: IpAddr) -> bool { - match address { - IpAddr::V4(address) => { - let [a, b, c, _] = address.octets(); - !(a == 0 - || a == 10 - || a == 127 - || (a == 100 && (64..=127).contains(&b)) - || (a == 169 && b == 254) - || (a == 172 && (16..=31).contains(&b)) - || (a == 192 && b == 0 && c == 0) - || (a == 192 && b == 0 && c == 2) - || (a == 192 && b == 168) - || (a == 198 && (b == 18 || b == 19)) - || (a == 198 && b == 51 && c == 100) - || (a == 203 && b == 0 && c == 113) - || a >= 224) - } - IpAddr::V6(address) => { - if let Some(mapped) = address.to_ipv4_mapped() { - return is_public_probe_ip(IpAddr::V4(mapped)); - } - address.segments()[0] & 0xe000 == 0x2000 - } - } -} - -fn resolve_public_probe_address(host: &str, port: u16) -> Option { - let addresses = (host, port) - .to_socket_addrs() - .ok()? - .filter(|address| is_public_probe_ip(address.ip())) - .collect::>(); - addresses - .iter() - .find(|address| address.is_ipv4()) - .or_else(|| addresses.first()) - .copied() -} - -fn parse_probe_target(value: &str, port: Option) -> Option<(String, u16)> { - let raw = value.trim(); - if raw.is_empty() - || raw.len() > 60 - || raw.contains("://") - || raw - .chars() - .any(|character| character.is_whitespace() || "/@?#\\[]".contains(character)) - || raw.contains(':') - { - return None; - } - let port = port.unwrap_or(443); - (port > 0 && valid_probe_host(raw)).then(|| (raw.to_ascii_lowercase(), port)) -} - -fn median(values: &mut [f64]) -> f64 { - values.sort_by(f64::total_cmp); - let middle = values.len() / 2; - if values.len().is_multiple_of(2) { - (values[middle - 1] + values[middle]) / 2.0 - } else { - values[middle] - } -} - -fn tcp_latency_probe(target: &str, port: Option) -> (f64, f64) { - if target.trim().is_empty() { - return (-1.0, -1.0); - } - let Some(port) = port.and_then(|value| u16::try_from(value).ok()) else { - return (-1.0, 100.0); - }; - let Some((host, port)) = parse_probe_target(target, Some(port)) else { - return (-1.0, 100.0); - }; - let Some(address) = resolve_public_probe_address(&host, port) else { - return (-1.0, 100.0); - }; - tcp_latency_probe_address(&address) -} - -fn tcp_latency_probe_address(address: &SocketAddr) -> (f64, f64) { - let mut latencies = Vec::with_capacity(PROBE_ATTEMPTS); - for _ in 0..PROBE_ATTEMPTS { - let started = Instant::now(); - if TcpStream::connect_timeout(address, PROBE_TIMEOUT).is_ok() { - latencies.push(started.elapsed().as_secs_f64() * 1000.0); - } - } - let loss = (PROBE_ATTEMPTS - latencies.len()) as f64 * 100.0 / PROBE_ATTEMPTS as f64; - if latencies.is_empty() { - (-1.0, loss) - } else { - (median(&mut latencies), loss) - } -} - -fn ping_latency(output: &str) -> Option { - let marker = output - .find("time=") - .map(|index| (index + 5, false)) - .or_else(|| output.find("time<").map(|index| (index + 5, true)))?; - if marker.1 { - return Some(0.5); - } - let value = output[marker.0..] - .chars() - .take_while(|character| character.is_ascii_digit() || *character == '.') - .collect::(); - value.parse().ok() -} - -fn ping_latencies(output: &str) -> Vec { - output.lines().filter_map(ping_latency).collect() -} - -fn icmp_latency_probe(target: &str) -> (f64, f64) { - let host = target.trim(); - let Some((host, _)) = parse_probe_target(host, None) else { - return (-1.0, 100.0); - }; - let Some(address) = resolve_public_probe_address(&host, 443) else { - return (-1.0, 100.0); - }; - let destination = address.ip().to_string(); - let mut ping = Command::new("ping"); - #[cfg(target_os = "linux")] - ping.args(["-n", "-c", "4", "-W", "1", destination.as_str()]); - #[cfg(any(target_os = "macos", target_os = "freebsd"))] - ping.args(["-n", "-c", "4", "-W", "1000", destination.as_str()]); - #[cfg(target_os = "windows")] - ping.args(["-n", "4", "-w", "1000", destination.as_str()]); - let mut latencies = ping - .output() - .map(|output| ping_latencies(&String::from_utf8_lossy(&output.stdout))) - .unwrap_or_default(); - latencies.truncate(PROBE_ATTEMPTS); - let loss = (PROBE_ATTEMPTS - latencies.len()) as f64 * 100.0 / PROBE_ATTEMPTS as f64; - if latencies.is_empty() { - (-1.0, loss) - } else { - (median(&mut latencies), loss) - } -} - fn unix_timestamp() -> i64 { SystemTime::now() .duration_since(UNIX_EPOCH) @@ -1168,291 +441,6 @@ fn observe_clock(clock: &SharedClock, offset_ms: Option) { } } -fn execute_latency_task(task: LatencyTask) -> LatencyResult { - let (latency_ms, packet_loss) = match task.task_type.as_str() { - "tcp" => tcp_latency_probe(&task.target, task.port), - "icmp" => icmp_latency_probe(&task.target), - _ => (-1.0, 100.0), - }; - LatencyResult { - task_id: task.id, - timestamp: unix_timestamp(), - latency_ms, - packet_loss, - } -} - -fn nvidia_gpu_info() -> Vec { - let gpu = command( - "nvidia-smi", - &[ - "--query-gpu=utilization.gpu,name,memory.used,memory.total", - "--format=csv,noheader,nounits", - ], - ); - gpu.lines() - .filter_map(|line| { - let fields = line.split(',').map(str::trim).collect::>(); - if fields.len() < 4 { - return None; - } - if fields[1].is_empty() { - return None; - } - Some(GpuMetric { - usage: fields[0].parse().ok(), - model: fields[1].to_string(), - memory_used: fields[2] - .parse::() - .unwrap_or(0) - .saturating_mul(1024 * 1024), - memory_total: fields[3] - .parse::() - .unwrap_or(0) - .saturating_mul(1024 * 1024), - }) - }) - .collect() -} - -fn basic_gpu_metrics(names: impl IntoIterator) -> Vec { - let mut seen = HashSet::new(); - names - .into_iter() - .filter_map(|name| { - let name = name.split(" (rev ").next().unwrap_or(&name).trim(); - let lower = name.to_ascii_lowercase(); - if name.is_empty() - || [ - "sensor hub", - "management engine", - "ethernet", - "wireless", - "audio controller", - "usb controller", - "sata controller", - "virtio", - "vmware", - "qxl", - "hyper-v", - "cirrus", - "microsoft basic display", - ] - .iter() - .any(|pattern| lower.contains(pattern)) - || !seen.insert(name.to_string()) - { - return None; - } - Some(GpuMetric { - model: name.to_string(), - usage: None, - memory_used: 0, - memory_total: 0, - }) - }) - .collect() -} - -#[cfg(any(target_os = "linux", test))] -fn parse_lspci_gpu_names(output: &str) -> Vec { - output - .lines() - .filter(|line| { - let lower = line.to_ascii_lowercase(); - lower.contains("vga compatible controller") - || lower.contains("3d controller") - || lower.contains("display controller") - }) - .filter_map(|line| line.split_once(": ").map(|(_, name)| name.to_string())) - .collect() -} - -#[cfg(any(target_os = "linux", test))] -fn gpu_name_from_uevent(output: &str) -> Option { - let driver = output - .lines() - .find_map(|line| line.strip_prefix("DRIVER="))?; - let pci_class = output - .lines() - .find_map(|line| line.strip_prefix("PCI_CLASS=")) - .unwrap_or_default(); - if u32::from_str_radix(pci_class, 16) - .ok() - .map(|class| class >> 16) - != Some(3) - { - return None; - } - match driver { - "i915" | "xe" => Some("Intel Integrated Graphics".to_string()), - "amdgpu" | "radeon" => Some("AMD Radeon Graphics".to_string()), - "nvidia" | "nouveau" => Some("NVIDIA GPU".to_string()), - "msm" | "msm_drm" => Some("Qualcomm Adreno GPU".to_string()), - "panfrost" | "lima" => Some("ARM Mali GPU".to_string()), - "vc4" | "v3d" => Some("Broadcom VideoCore Graphics".to_string()), - "virtio-pci" | "virtio_gpu" | "bochs-drm" | "qxl" | "vmwgfx" | "cirrus" | "vboxvideo" - | "hyperv_fb" | "simpledrm" | "simplefb" => None, - other if !other.is_empty() => Some(format!("GPU ({other})")), - _ => None, - } -} - -#[cfg(target_os = "linux")] -fn sysfs_gpu_names() -> Vec { - let Ok(entries) = fs::read_dir("/sys/class/drm") else { - return Vec::new(); - }; - entries - .filter_map(std::result::Result::ok) - .filter(|entry| { - let name = entry.file_name(); - let name = name.to_string_lossy(); - name.strip_prefix("card").is_some_and(|suffix| { - !suffix.is_empty() && suffix.chars().all(|c| c.is_ascii_digit()) - }) - }) - .filter_map(|entry| gpu_name_from_uevent(&text(entry.path().join("device/uevent")))) - .collect() -} - -#[cfg(any(target_os = "macos", test))] -fn parse_system_profiler_gpu_names(output: &str) -> Vec { - output - .lines() - .filter_map(|line| { - line.trim() - .strip_prefix("Chipset Model:") - .map(|name| name.trim().to_string()) - }) - .collect() -} - -#[cfg(any(target_os = "freebsd", test))] -fn parse_pciconf_gpu_names(output: &str) -> Vec { - fn block_name(lines: &[&str]) -> Option { - let header = lines.first()?.to_ascii_lowercase(); - let mut display = header.contains("class=0x03"); - let mut vendor = None; - let mut device = None; - for line in lines.iter().skip(1) { - let Some((key, value)) = line.trim().split_once('=') else { - continue; - }; - let key = key.trim(); - let value = value.trim().trim_matches(['\'', '"']); - match key { - "class" if value.eq_ignore_ascii_case("display") => display = true, - "vendor" => vendor = Some(value), - "device" => device = Some(value), - _ => {} - } - } - if !display { - return None; - } - match (vendor, device) { - (Some(vendor), Some(device)) - if !device - .to_ascii_lowercase() - .contains(&vendor.to_ascii_lowercase()) => - { - Some(format!("{vendor} {device}")) - } - (_, Some(device)) => Some(device.to_string()), - (Some(vendor), None) => Some(vendor.to_string()), - _ => None, - } - } - - let mut names = Vec::new(); - let mut block = Vec::new(); - for line in output.lines() { - if !line.is_empty() - && !line.chars().next().is_some_and(char::is_whitespace) - && !block.is_empty() - { - if let Some(name) = block_name(&block) { - names.push(name); - } - block.clear(); - } - if !line.trim().is_empty() { - block.push(line); - } - } - if let Some(name) = block_name(&block) { - names.push(name); - } - names -} - -#[cfg(target_os = "linux")] -fn basic_gpu_info() -> Vec { - let names = parse_lspci_gpu_names(&command("lspci", &[])); - basic_gpu_metrics(if names.is_empty() { - sysfs_gpu_names() - } else { - names - }) -} - -#[cfg(target_os = "windows")] -fn basic_gpu_info() -> Vec { - basic_gpu_metrics( - command( - "powershell.exe", - &[ - "-NoProfile", - "-NonInteractive", - "-Command", - "Get-CimInstance Win32_VideoController | ForEach-Object { $_.Name }", - ], - ) - .lines() - .map(str::to_string), - ) -} - -#[cfg(target_os = "macos")] -fn basic_gpu_info() -> Vec { - basic_gpu_metrics(parse_system_profiler_gpu_names(&command( - "system_profiler", - &["SPDisplaysDataType"], - ))) -} - -#[cfg(target_os = "freebsd")] -fn basic_gpu_info() -> Vec { - basic_gpu_metrics(parse_pciconf_gpu_names(&command("pciconf", &["-lv"]))) -} - -fn gpu_info() -> Vec { - let detailed = nvidia_gpu_info(); - if detailed.is_empty() { - basic_gpu_info() - } else { - detailed - } -} - -fn average_gpu_usage(gpus: &[GpuMetric]) -> f64 { - let usages = gpus.iter().filter_map(|gpu| gpu.usage).collect::>(); - if usages.is_empty() { - 0.0 - } else { - usages.iter().sum::() / usages.len() as f64 - } -} - -fn u64_to_i64(value: u64) -> i64 { - value.min(i64::MAX as u64) as i64 -} - -fn per_second(value: u64, elapsed_seconds: f64) -> f64 { - value as f64 / elapsed_seconds.max(0.001) -} - fn valid_sample_schedule(report_interval: u64, collect_interval: u64) -> bool { (15..=3600).contains(&report_interval) && (1..=60).contains(&collect_interval) @@ -1467,431 +455,12 @@ fn advance_deadline(mut deadline: Instant, interval: Duration, now: Instant) -> deadline } -#[cfg(target_os = "linux")] -struct Collector { - previous_cpu: CpuSample, - previous_io: IoSample, - previous_at: Instant, - network_interface: String, - basic: BasicMetrics, - basic_at: Instant, - gpu_at: Instant, - slow: SlowMetrics, - slow_at: Instant, - public_ip: PublicIpProbe, -} - -#[cfg(target_os = "linux")] -impl Collector { - fn new(config: &RuntimeConfig) -> Self { - let mut collector = Self { - previous_cpu: cpu_sample(), - previous_io: io_sample(&config.network_interface), - previous_at: Instant::now(), - network_interface: config.network_interface.clone(), - basic: BasicMetrics::default(), - basic_at: Instant::now(), - gpu_at: Instant::now(), - slow: SlowMetrics::default(), - slow_at: Instant::now(), - public_ip: PublicIpProbe::default(), - }; - collector.refresh_basic(); - collector.refresh_slow(); - collector - } - - fn refresh_basic(&mut self) { - let gpus = gpu_info(); - self.basic = BasicMetrics { - cpu_cores: thread::available_parallelism().map_or(1, |value| value.get() as i64), - cpu_model: cpu_model(), - os: os_name(), - kernel: command("uname", &["-r"]), - arch: env::consts::ARCH.to_string(), - virtualization: command("systemd-detect-virt", &[]), - gpu_usage: average_gpu_usage(&gpus), - gpu_model: gpus - .iter() - .map(|gpu| gpu.model.as_str()) - .collect::>() - .join(" · "), - gpus, - }; - self.basic_at = Instant::now(); - self.gpu_at = Instant::now(); - } - - fn refresh_slow(&mut self) { - let processes = fs::read_dir("/proc").map_or(0, |items| { - items - .filter_map(std::result::Result::ok) - .filter(|item| { - item.file_name() - .to_string_lossy() - .chars() - .all(|ch| ch.is_ascii_digit()) - }) - .count() as i64 - }); - self.slow = SlowMetrics { - disks: dedupe_disks(disk_usage()), - processes, - tcp_connections: file_line_count("/proc/net/tcp") + file_line_count("/proc/net/tcp6"), - udp_connections: file_line_count("/proc/net/udp") + file_line_count("/proc/net/udp6"), - }; - self.slow_at = Instant::now(); - } - - fn collect( - &mut self, - config: &RuntimeConfig, - latency_results: Vec, - timestamp: i64, - ) -> Report { - let sampled_at = Instant::now(); - let elapsed = sampled_at - .saturating_duration_since(self.previous_at) - .as_secs_f64() - .max(0.001); - let cpu_now = cpu_sample(); - let io_now = io_sample(&config.network_interface); - let total = cpu_now.total.saturating_sub(self.previous_cpu.total); - let idle = cpu_now.idle.saturating_sub(self.previous_cpu.idle); - let cpu = if total == 0 { - 0.0 - } else { - (total.saturating_sub(idle)) as f64 * 100.0 / total as f64 - }; - let interface_changed = self.network_interface != config.network_interface; - let io_before = if interface_changed { - io_now - } else { - self.previous_io - }; - self.previous_cpu = cpu_now; - self.previous_io = io_now; - self.previous_at = sampled_at; - self.network_interface.clone_from(&config.network_interface); - - if self.basic_at.elapsed() >= BASIC_INFO_REFRESH_INTERVAL { - self.refresh_basic(); - } - if self.gpu_at.elapsed() >= GPU_METRICS_REFRESH_INTERVAL { - self.refresh_gpu_metrics(); - } - if self.slow_at.elapsed() >= SLOW_METRICS_REFRESH_INTERVAL { - self.refresh_slow(); - } - self.public_ip.refresh(); - - let memory = memory_sample(&text("/proc/meminfo")); - let loads = text("/proc/loadavg") - .split_whitespace() - .take(3) - .filter_map(|value| value.parse::().ok()) - .collect::>(); - let uptime = text("/proc/uptime") - .split_whitespace() - .next() - .and_then(|value| value.parse::().ok()) - .unwrap_or(0.0) as i64; - let processes = self.slow.processes; - let disks = self.slow.disks.clone(); - let disk_used = disks.iter().map(|disk| disk.used).sum(); - let disk_total = disks.iter().map(|disk| disk.total).sum(); - let tcp_connections = self.slow.tcp_connections; - let udp_connections = self.slow.udp_connections; - let read_ops = io_now.read_ops.saturating_sub(io_before.read_ops); - let write_ops = io_now.write_ops.saturating_sub(io_before.write_ops); - let total_ops = read_ops.saturating_add(write_ops); - let io_wait = io_now - .read_millis - .saturating_sub(io_before.read_millis) - .saturating_add(io_now.write_millis.saturating_sub(io_before.write_millis)); - let elapsed_ms = elapsed * 1000.0; - - Report { - timestamp, - cpu, - load1: loads.first().copied().unwrap_or(0.0), - load5: loads.get(1).copied().unwrap_or(0.0), - load15: loads.get(2).copied().unwrap_or(0.0), - mem_used: memory.used, - mem_total: memory.total, - swap_used: memory.swap_used, - swap_total: memory.swap_total, - disk_used, - disk_total, - net_in: per_second(io_now.rx.saturating_sub(io_before.rx), elapsed), - net_out: per_second(io_now.tx.saturating_sub(io_before.tx), elapsed), - net_rx_total: io_now.rx.min(i64::MAX as u64) as i64, - net_tx_total: io_now.tx.min(i64::MAX as u64) as i64, - uptime, - processes, - tcp_connections, - udp_connections, - cpu_cores: self.basic.cpu_cores, - cpu_model: self.basic.cpu_model.clone(), - os: self.basic.os.clone(), - kernel: self.basic.kernel.clone(), - arch: self.basic.arch.clone(), - virtualization: self.basic.virtualization.clone(), - gpu_usage: self.basic.gpu_usage, - gpu_model: self.basic.gpu_model.clone(), - agent_version: VERSION.to_string(), - ip_v4: self.public_ip.v4().unwrap_or_default(), - ip_v6: self.public_ip.v6().unwrap_or_default(), - disk_read_bps: per_second( - io_now - .read_sectors - .saturating_sub(io_before.read_sectors) - .saturating_mul(512), - elapsed, - ), - disk_write_bps: per_second( - io_now - .write_sectors - .saturating_sub(io_before.write_sectors) - .saturating_mul(512), - elapsed, - ), - disk_read_iops: per_second(read_ops, elapsed), - disk_write_iops: per_second(write_ops, elapsed), - disk_await_ms: if total_ops == 0 { - 0.0 - } else { - io_wait as f64 / total_ops as f64 - }, - disk_utilization: if elapsed_ms <= 0.0 { - 0.0 - } else { - (io_now.io_millis.saturating_sub(io_before.io_millis) as f64 * 100.0 / elapsed_ms) - .clamp(0.0, 100.0) - }, - disks, - gpus: self.basic.gpus.clone(), - latency_results, - } - } -} - -#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] -struct Collector { - system: System, - networks: Networks, - disks: Disks, - previous_at: Instant, - basic: BasicMetrics, - basic_at: Instant, - gpu_at: Instant, - slow: SlowMetrics, - slow_at: Instant, - public_ip: PublicIpProbe, -} - -#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] -impl Collector { - fn new(_config: &RuntimeConfig) -> Self { - let mut collector = Self { - system: System::new_all(), - networks: Networks::new_with_refreshed_list(), - disks: Disks::new_with_refreshed_list(), - previous_at: Instant::now(), - basic: BasicMetrics::default(), - basic_at: Instant::now(), - gpu_at: Instant::now(), - slow: SlowMetrics::default(), - slow_at: Instant::now(), - public_ip: PublicIpProbe::default(), - }; - collector.refresh_basic(); - collector.refresh_slow(); - collector - } - - fn refresh_basic(&mut self) { - let gpus = gpu_info(); - self.basic = BasicMetrics { - cpu_cores: self.system.cpus().len().max(1) as i64, - cpu_model: self - .system - .cpus() - .first() - .map(|cpu| cpu.brand().to_string()) - .unwrap_or_default(), - os: System::long_os_version().unwrap_or_else(|| env::consts::OS.to_string()), - kernel: System::kernel_version().unwrap_or_default(), - arch: env::consts::ARCH.to_string(), - virtualization: String::new(), - gpu_usage: average_gpu_usage(&gpus), - gpu_model: gpus - .iter() - .map(|gpu| gpu.model.as_str()) - .collect::>() - .join(" · "), - gpus, - }; - self.basic_at = Instant::now(); - self.gpu_at = Instant::now(); - } - - fn refresh_slow(&mut self) { - self.system.refresh_processes_specifics( - ProcessesToUpdate::All, - true, - ProcessRefreshKind::nothing().without_tasks(), - ); - self.disks.refresh(true); - #[cfg(target_os = "windows")] - let root = format!( - "{}\\", - env::var("SystemDrive").unwrap_or_else(|_| "C:".to_string()) - ); - #[cfg(any(target_os = "macos", target_os = "freebsd"))] - let root = "/".to_string(); - let mut disks = self - .disks - .iter() - .filter(|disk| !disk.is_removable()) - .map(|disk| { - let total = disk.total_space(); - DiskMetric { - name: disk.name().to_string_lossy().to_string(), - mount_point: disk.mount_point().to_string_lossy().to_string(), - used: u64_to_i64(total.saturating_sub(disk.available_space())), - total: u64_to_i64(total), - ..DiskMetric::default() - } - }) - .collect::>(); - disks.sort_by_key(|disk| (disk.mount_point != root, disk.mount_point.clone())); - let (tcp_connections, udp_connections) = connection_counts(); - self.slow = SlowMetrics { - disks, - processes: self.system.processes().len() as i64, - tcp_connections, - udp_connections, - }; - self.slow_at = Instant::now(); - } - - fn collect( - &mut self, - config: &RuntimeConfig, - latency_results: Vec, - timestamp: i64, - ) -> Report { - let sampled_at = Instant::now(); - let elapsed = sampled_at - .saturating_duration_since(self.previous_at) - .as_secs_f64() - .max(0.001); - self.system.refresh_cpu_usage(); - self.system.refresh_memory(); - self.networks.refresh(true); - self.previous_at = sampled_at; - if self.basic_at.elapsed() >= BASIC_INFO_REFRESH_INTERVAL { - self.refresh_basic(); - } - if self.gpu_at.elapsed() >= GPU_METRICS_REFRESH_INTERVAL { - self.refresh_gpu_metrics(); - } - if self.slow_at.elapsed() >= SLOW_METRICS_REFRESH_INTERVAL { - self.refresh_slow(); - } - self.public_ip.refresh(); - - let disk_metrics = self.slow.disks.clone(); - let tcp_connections = self.slow.tcp_connections; - let udp_connections = self.slow.udp_connections; - - let mut net_in = 0_u64; - let mut net_out = 0_u64; - let mut net_rx_total = 0_u64; - let mut net_tx_total = 0_u64; - for (name, network) in &self.networks { - if !selected_interface(name, &config.network_interface) { - continue; - } - net_in = net_in.saturating_add(network.received()); - net_out = net_out.saturating_add(network.transmitted()); - net_rx_total = net_rx_total.saturating_add(network.total_received()); - net_tx_total = net_tx_total.saturating_add(network.total_transmitted()); - } - let load = System::load_average(); - Report { - timestamp, - cpu: self.system.global_cpu_usage() as f64, - load1: load.one, - load5: load.five, - load15: load.fifteen, - mem_used: u64_to_i64( - self.system - .total_memory() - .saturating_sub(self.system.available_memory()), - ), - mem_total: u64_to_i64(self.system.total_memory()), - swap_used: u64_to_i64(self.system.used_swap()), - swap_total: u64_to_i64(self.system.total_swap()), - disk_used: disk_metrics.iter().map(|disk| disk.used).sum(), - disk_total: disk_metrics.iter().map(|disk| disk.total).sum(), - net_in: per_second(net_in, elapsed), - net_out: per_second(net_out, elapsed), - net_rx_total: u64_to_i64(net_rx_total), - net_tx_total: u64_to_i64(net_tx_total), - uptime: u64_to_i64(System::uptime()), - processes: self.slow.processes, - tcp_connections, - udp_connections, - cpu_cores: self.basic.cpu_cores, - cpu_model: self.basic.cpu_model.clone(), - os: self.basic.os.clone(), - kernel: self.basic.kernel.clone(), - arch: self.basic.arch.clone(), - virtualization: self.basic.virtualization.clone(), - gpu_usage: self.basic.gpu_usage, - gpu_model: self.basic.gpu_model.clone(), - agent_version: VERSION.to_string(), - ip_v4: self.public_ip.v4().unwrap_or_default(), - ip_v6: self.public_ip.v6().unwrap_or_default(), - disk_read_bps: 0.0, - disk_write_bps: 0.0, - disk_read_iops: 0.0, - disk_write_iops: 0.0, - disk_await_ms: 0.0, - disk_utilization: 0.0, - disks: disk_metrics, - gpus: self.basic.gpus.clone(), - latency_results, - } - } -} - -impl Collector { - fn refresh_gpu_metrics(&mut self) { - self.gpu_at = Instant::now(); - // Unsupported devices are rediscovered with basic info, not every sample. - if !self - .basic - .gpus - .iter() - .any(|gpu| gpu.usage.is_some() || gpu.memory_total > 0) - { - return; - } - let detailed = nvidia_gpu_info(); - if detailed.is_empty() { - return; - } - self.basic.gpu_usage = average_gpu_usage(&detailed); - self.basic.gpu_model = detailed - .iter() - .map(|gpu| gpu.model.as_str()) - .collect::>() - .join(" · "); - self.basic.gpus = detailed; - } +/// Reads an Agent token from a file, trimming surrounding whitespace (editors +/// and shell redirection usually append a trailing newline). +fn read_token_file(path: &Path) -> Result { + let contents = fs::read_to_string(path) + .map_err(|error| format!("failed to read token file {}: {error}", path.display()))?; + Ok(contents.trim().to_string()) } fn runtime_config(options: &CliOptions) -> Result { @@ -1899,11 +468,13 @@ fn runtime_config(options: &CliOptions) -> Result { if !(15..=3600).contains(&interval) { return Err("interval must be between 15 and 3600 seconds".into()); } - let token = options - .token - .clone() - .or_else(|| env::var("NODEFLARE_AGENT_TOKEN").ok()) - .unwrap_or_default(); + let token = match &options.token { + Some(token) => token.clone(), + None => match options.token_file.as_deref() { + Some(path) => read_token_file(path)?, + None => env::var("NODEFLARE_AGENT_TOKEN").unwrap_or_default(), + }, + }; let endpoint = options.endpoint.clone(); if token.is_empty() || token.len() > 512 || token.chars().any(char::is_whitespace) { return Err("token is invalid".into()); @@ -1925,739 +496,6 @@ fn runtime_config(options: &CliOptions) -> Result { }) } -fn live_endpoint(endpoint: &str) -> Result { - let mut url = url::Url::parse(endpoint)?; - url.set_scheme(if url.scheme() == "https" { "wss" } else { "ws" }) - .map_err(|_| "unsupported live endpoint scheme")?; - let path = format!("{}/api/agent/ws", url.path().trim_end_matches('/')); - url.set_path(&path); - Ok(url.to_string()) -} - -type LiveSocket = tungstenite::WebSocket>; - -fn connect_live(endpoint: &str, token: &str) -> Result<(LiveSocket, Option)> { - let started_ms = unix_timestamp_millis(); - let deadline = Instant::now() + LIVE_CONNECT_TIMEOUT; - let mut url = url::Url::parse(endpoint)?; - let mut redirects = 0; - let (socket, response) = loop { - let mut request = url.as_str().into_client_request()?; - request - .headers_mut() - .insert("Authorization", format!("Bearer {token}").parse()?); - request - .headers_mut() - .insert("User-Agent", format!("nodeflare-agent/{VERSION}").parse()?); - request - .headers_mut() - .insert(AGENT_PROTOCOL_HEADER, AGENT_PROTOCOL_VERSION.parse()?); - request.headers_mut().insert( - AGENT_CAPABILITIES_HEADER, - REQUIRED_AGENT_CAPABILITIES.join(",").parse()?, - ); - let stream = connect_live_stream(&url, deadline)?; - match client_tls(request, stream) { - Ok(connected) => break connected, - Err(tungstenite::HandshakeError::Failure(WebSocketError::Http(response))) - if response.status().is_redirection() && redirects < 3 => - { - let location = response - .headers() - .get("location") - .ok_or("WebSocket redirect has no location")? - .to_str()?; - let redirected = url.join(location)?; - // Never forward the Agent token to a different origin. - if redirected.origin() != url.origin() { - return Err("WebSocket redirect must stay on the configured origin".into()); - } - url = redirected; - redirects += 1; - } - Err(error) => return Err(format!("WebSocket handshake failed: {error}").into()), - } - }; - let ended_ms = unix_timestamp_millis(); - let clock_offset_ms = response - .headers() - .get("date") - .and_then(|value| value.to_str().ok()) - .and_then(|value| clock_offset_from_http_date(value, started_ms, ended_ms)); - if !response - .headers() - .get(AGENT_PROTOCOL_HEADER) - .is_some_and(|value| value == AGENT_PROTOCOL_VERSION) - { - return Err(format!( - "backend protocol mismatch; update the backend to protocol {AGENT_PROTOCOL_VERSION}" - ) - .into()); - } - Ok((socket, clock_offset_ms)) -} - -fn connect_live_stream(url: &url::Url, deadline: Instant) -> Result { - let host = url - .host_str() - .ok_or("WebSocket endpoint has no hostname")? - .trim_start_matches('[') - .trim_end_matches(']'); - let port = url - .port_or_known_default() - .ok_or("WebSocket endpoint has no port")?; - let addresses = (host, port).to_socket_addrs()?; - let mut connected = None; - let mut last_error = io::Error::new( - ErrorKind::NotFound, - "WebSocket hostname resolved to no addresses", - ); - for address in addresses { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return Err( - io::Error::new(ErrorKind::TimedOut, "WebSocket connection timed out").into(), - ); - } - match TcpStream::connect_timeout(&address, remaining.min(Duration::from_secs(3))) { - Ok(stream) => { - connected = Some(stream); - break; - } - Err(error) => last_error = error, - } - } - let stream = connected.ok_or(last_error)?; - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return Err(io::Error::new(ErrorKind::TimedOut, "WebSocket connection timed out").into()); - } - stream.set_nodelay(true)?; - stream.set_read_timeout(Some(remaining))?; - stream.set_write_timeout(Some(remaining))?; - Ok(stream) -} - -fn set_live_read_timeout(socket: &mut LiveSocket, timeout: Option) -> io::Result<()> { - match socket.get_mut() { - tungstenite::stream::MaybeTlsStream::Plain(stream) => stream.set_read_timeout(timeout), - tungstenite::stream::MaybeTlsStream::Rustls(stream) => { - stream.sock.set_read_timeout(timeout) - } - _ => Ok(()), - } -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct LiveAck { - #[serde(rename = "type")] - message_type: String, - ts: i64, - persisted: bool, - #[serde(rename = "persistenceError")] - persistence_error: bool, - #[serde(rename = "persistedThroughTs")] - persisted_through_ts: i64, - #[serde(rename = "nextPersistAfterMs")] - next_persist_after_ms: u64, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct LiveConfigMessage { - #[serde(rename = "type")] - message_type: String, - ts: i64, - config: RemoteConfig, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct RemoteTaskMessage { - #[serde(rename = "type")] - message_type: String, - task_id: String, - command: String, -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] -struct TaskResultMessage { - #[serde(rename = "type")] - message_type: String, - task_id: String, - status: String, - result: String, - exit_code: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(deny_unknown_fields)] -struct TaskResultAckMessage { - #[serde(rename = "type")] - message_type: String, - task_id: String, -} - -#[derive(Debug, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] -struct RemoteTaskJournal { - version: u32, - entries: Vec, -} - -#[derive(Debug, Deserialize, Serialize)] -#[serde(tag = "state", rename_all = "snake_case")] -enum RemoteTaskJournalEntry { - Active { task_id: String }, - Completed { result: TaskResultMessage }, -} - -#[derive(Default)] -struct CapturedOutput { - bytes: Vec, - truncated: bool, -} - -fn capture_remote_output(mut reader: R) -> mpsc::Receiver -where - R: Read + Send + 'static, -{ - let (sender, receiver) = mpsc::sync_channel(1); - thread::spawn(move || { - let mut bytes = Vec::new(); - let mut truncated = false; - let mut buffer = [0_u8; 8192]; - // Keep draining after the limit so verbose commands do not receive SIGPIPE. - loop { - match reader.read(&mut buffer) { - Ok(0) => break, - Ok(read) => { - let retained = read.min(REMOTE_STREAM_OUTPUT_BYTES as usize - bytes.len()); - bytes.extend_from_slice(&buffer[..retained]); - truncated |= retained < read; - } - Err(error) if error.kind() == ErrorKind::Interrupted => continue, - Err(_) => break, - } - } - let _ = sender.send(CapturedOutput { bytes, truncated }); - }); - receiver -} - -fn terminate_remote_process_tree(child: &mut Child) { - #[cfg(unix)] - { - let process_group = format!("-{}", child.id()); - let _ = Command::new("kill") - .args(["-KILL", "--", process_group.as_str()]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } - #[cfg(target_os = "windows")] - { - let _ = Command::new("taskkill") - .args(["/PID", &child.id().to_string(), "/T", "/F"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } - let _ = child.kill(); - let _ = child.wait(); -} - -fn receive_remote_output( - receiver: &mpsc::Receiver, - child: &mut Child, -) -> CapturedOutput { - match receiver.recv_timeout(REMOTE_OUTPUT_DRAIN_TIMEOUT) { - Ok(output) => output, - Err(_) => { - // Descendants may keep the pipe open after the shell exits. - terminate_remote_process_tree(child); - receiver - .recv_timeout(REMOTE_OUTPUT_DRAIN_TIMEOUT) - .unwrap_or_default() - } - } -} - -fn remote_result_text(stdout: &CapturedOutput, stderr: &CapturedOutput) -> String { - let truncated = stdout.truncated || stderr.truncated; - let stdout = String::from_utf8_lossy(&stdout.bytes); - let stderr = String::from_utf8_lossy(&stderr.bytes); - let mut combined = if stderr.is_empty() { - stdout.trim().to_string() - } else if stdout.is_empty() { - stderr.trim().to_string() - } else { - format!("{}\n{}", stdout.trim(), stderr.trim()) - }; - let result_truncated = combined.len() > REMOTE_RESULT_OUTPUT_BYTES; - if result_truncated { - let mut end = REMOTE_RESULT_OUTPUT_BYTES; - while !combined.is_char_boundary(end) { - end -= 1; - } - combined.truncate(end); - } - if truncated || result_truncated { - combined.push_str("\n[输出已截断]"); - } - combined -} - -fn wait_for_remote_child( - child: &mut Child, - timeout: Duration, -) -> io::Result> { - let deadline = Instant::now() + timeout; - loop { - if let Some(status) = child.try_wait()? { - return Ok(Some(status)); - } - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return Ok(None); - } - thread::sleep(remaining.min(Duration::from_millis(50))); - } -} - -fn execute_remote_task_with_timeout( - task: &RemoteTaskMessage, - timeout: Duration, -) -> TaskResultMessage { - let task_id = task.task_id.clone(); - - if task.command.trim().is_empty() || task.command.len() > 16_384 { - return TaskResultMessage { - message_type: "task_result".to_string(), - task_id, - status: "failed".to_string(), - result: "命令为空或长度超出限制".to_string(), - exit_code: Some(-1), - }; - } - - #[cfg(unix)] - let mut command = { - let mut command = Command::new("sh"); - command.args(["-c", task.command.as_str()]); - command.process_group(0); - command - }; - #[cfg(target_os = "windows")] - let mut command = { - let mut command = Command::new("cmd.exe"); - command.args(["/C", task.command.as_str()]); - command - }; - command.stdout(Stdio::piped()).stderr(Stdio::piped()); - - let mut child = match command.spawn() { - Ok(child) => child, - Err(error) => { - return TaskResultMessage { - message_type: "task_result".to_string(), - task_id, - status: "failed".to_string(), - result: format!("无法启动命令：{error}"), - exit_code: Some(-1), - }; - } - }; - let stdout = capture_remote_output(child.stdout.take().expect("stdout is piped")); - let stderr = capture_remote_output(child.stderr.take().expect("stderr is piped")); - - let status = match wait_for_remote_child(&mut child, timeout) { - Ok(Some(status)) => Some(status), - Ok(None) => { - terminate_remote_process_tree(&mut child); - None - } - Err(error) => { - terminate_remote_process_tree(&mut child); - let stdout = receive_remote_output(&stdout, &mut child); - let stderr = receive_remote_output(&stderr, &mut child); - let output = remote_result_text(&stdout, &stderr); - return TaskResultMessage { - message_type: "task_result".to_string(), - task_id, - status: "failed".to_string(), - result: if output.is_empty() { - format!("等待命令结束失败：{error}") - } else { - format!("{output}\n等待命令结束失败：{error}") - }, - exit_code: Some(-1), - }; - } - }; - let stdout = receive_remote_output(&stdout, &mut child); - let stderr = receive_remote_output(&stderr, &mut child); - let output = remote_result_text(&stdout, &stderr); - - let Some(status) = status else { - let timeout_seconds = timeout.as_secs().max(1); - return TaskResultMessage { - message_type: "task_result".to_string(), - task_id, - status: "failed".to_string(), - result: if output.is_empty() { - format!("命令执行超过 {timeout_seconds} 秒，已终止") - } else { - format!("{output}\n命令执行超过 {timeout_seconds} 秒，已终止") - }, - exit_code: Some(-1), - }; - }; - - TaskResultMessage { - message_type: "task_result".to_string(), - task_id, - status: if status.success() { - "success" - } else { - "failed" - } - .to_string(), - result: output, - exit_code: status.code().map(i64::from), - } -} - -fn execute_remote_task(task: &RemoteTaskMessage) -> TaskResultMessage { - execute_remote_task_with_timeout(task, REMOTE_TASK_TIMEOUT) -} - -struct RemoteExecutor { - task_tx: mpsc::SyncSender, - result_rx: mpsc::Receiver, - active: HashSet, - completed: HashMap, - task_order: VecDeque, - last_sent: HashMap, - journal_path: PathBuf, - disabled: bool, -} - -impl RemoteExecutor { - fn new() -> Result { - Self::new_at(remote_task_journal_path()?) - } - - fn new_at(journal_path: PathBuf) -> Result { - let journal = load_remote_task_journal(&journal_path)?; - let mut active = HashSet::new(); - let mut completed = HashMap::new(); - let mut task_order = VecDeque::new(); - for entry in journal.entries { - match entry { - RemoteTaskJournalEntry::Active { task_id } => { - active.insert(task_id.clone()); - task_order.push_back(task_id); - } - RemoteTaskJournalEntry::Completed { result } => { - task_order.push_back(result.task_id.clone()); - completed.insert(result.task_id.clone(), result); - } - } - } - - let (task_tx, task_rx) = mpsc::sync_channel::(16); - let (result_tx, result_rx) = mpsc::channel::(); - thread::Builder::new() - .name("nodeflare-remote".to_string()) - .spawn(move || { - while let Ok(task) = task_rx.recv() { - if result_tx.send(execute_remote_task(&task)).is_err() { - return; - } - } - })?; - let interrupted = active.drain().collect::>(); - for task_id in interrupted { - completed.insert( - task_id.clone(), - TaskResultMessage { - message_type: "task_result".to_string(), - task_id, - status: "failed".to_string(), - result: "Agent 在命令执行期间重启，无法确认原命令状态；为避免重复操作，任务不会再次执行" - .to_string(), - exit_code: Some(-1), - }, - ); - } - let executor = Self { - task_tx, - result_rx, - active, - completed, - task_order, - last_sent: HashMap::new(), - journal_path, - disabled: false, - }; - if !executor.completed.is_empty() { - executor.persist_snapshot(None)?; - } - Ok(executor) - } - - fn disabled() -> Self { - let (task_tx, task_rx) = mpsc::sync_channel(1); - let (result_tx, result_rx) = mpsc::channel(); - drop(task_rx); - drop(result_tx); - Self { - task_tx, - result_rx, - active: HashSet::new(), - completed: HashMap::new(), - task_order: VecDeque::new(), - last_sent: HashMap::new(), - journal_path: PathBuf::new(), - disabled: true, - } - } - - fn enqueue(&mut self, task: RemoteTaskMessage) -> Option { - if self.disabled { - return Some(remote_task_rejected( - task.task_id, - "Agent 任务日志异常，远程执行已禁用", - )); - } - if self.completed.contains_key(&task.task_id) { - self.last_sent.remove(&task.task_id); - return None; - } - if self.active.contains(&task.task_id) { - return None; - } - let task_id = task.task_id.clone(); - if self.task_order.len() >= MAX_REMOTE_TASK_JOURNAL_ENTRIES { - return Some(remote_task_rejected( - task_id, - "待确认的远程任务过多，已拒绝执行", - )); - } - self.active.insert(task_id.clone()); - self.task_order.push_back(task_id.clone()); - if let Err(error) = self.persist_snapshot(None) { - self.active.remove(&task_id); - self.task_order.retain(|id| id != &task_id); - eprintln!("remote task journal write failed: {error}"); - return Some(remote_task_rejected( - task_id, - "Agent 无法持久化任务状态，已拒绝执行", - )); - } - match self.task_tx.try_send(task) { - Ok(()) => None, - Err(error) => { - let message = match error { - mpsc::TrySendError::Full(_) => "远程命令队列已满，请稍后重试", - mpsc::TrySendError::Disconnected(_) => "远程命令执行线程不可用", - }; - let result = remote_task_rejected(task_id, message); - self.remember_completed(result.clone()); - None - } - } - } - - fn drain_completed(&mut self) { - let results = self.result_rx.try_iter().collect::>(); - for result in results { - self.remember_completed(result); - } - } - - fn acknowledge(&mut self, task_id: &str) { - if !self.completed.contains_key(task_id) { - return; - } - match self.persist_snapshot(Some(task_id)) { - Ok(()) => { - self.completed.remove(task_id); - self.task_order.retain(|id| id != task_id); - self.last_sent.remove(task_id); - } - Err(error) => { - eprintln!("remote task ACK persistence failed: {error}"); - } - } - } - - fn remember_completed(&mut self, result: TaskResultMessage) { - let task_id = result.task_id.clone(); - let was_active = self.active.remove(&task_id); - if !was_active && !self.completed.contains_key(&task_id) { - self.task_order.push_back(task_id.clone()); - } - self.completed.insert(task_id.clone(), result); - self.last_sent.remove(&task_id); - if let Err(error) = self.persist_snapshot(None) { - eprintln!("remote task result persistence failed: {error}"); - } - } - - fn due_results(&mut self) -> Vec { - self.drain_completed(); - let now = Instant::now(); - self.task_order - .iter() - .filter_map(|task_id| { - let result = self.completed.get(task_id)?; - let due = self.last_sent.get(task_id).is_none_or(|sent_at| { - now.duration_since(*sent_at) >= REMOTE_RESULT_RETRY_INTERVAL - }); - due.then(|| result.clone()) - }) - .collect() - } - - fn mark_sent(&mut self, task_id: &str) { - self.last_sent.insert(task_id.to_string(), Instant::now()); - } - - fn reset_delivery(&mut self) { - self.last_sent.clear(); - } - - fn persist_snapshot(&self, excluded_task_id: Option<&str>) -> Result<()> { - let entries = self - .task_order - .iter() - .filter(|task_id| excluded_task_id != Some(task_id.as_str())) - .filter_map(|task_id| { - if self.active.contains(task_id) { - Some(RemoteTaskJournalEntry::Active { - task_id: task_id.clone(), - }) - } else { - self.completed - .get(task_id) - .cloned() - .map(|result| RemoteTaskJournalEntry::Completed { result }) - } - }) - .collect::>(); - write_remote_task_journal(&self.journal_path, entries) - } -} - -fn remote_task_rejected(task_id: String, result: &str) -> TaskResultMessage { - TaskResultMessage { - message_type: "task_result".to_string(), - task_id, - status: "failed".to_string(), - result: result.to_string(), - exit_code: Some(-1), - } -} - -fn valid_remote_task_id(value: &str) -> bool { - value.len() == 36 - && value.bytes().enumerate().all(|(index, byte)| { - if matches!(index, 8 | 13 | 18 | 23) { - byte == b'-' - } else { - byte.is_ascii_hexdigit() - } - }) -} - -fn load_remote_task_journal(path: &Path) -> Result { - let bytes = match fs::read(path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == ErrorKind::NotFound => { - return Ok(RemoteTaskJournal { - version: REMOTE_TASK_JOURNAL_VERSION, - entries: Vec::new(), - }); - } - Err(error) => return Err(error.into()), - }; - if bytes.len() as u64 > MAX_REMOTE_TASK_JOURNAL_BYTES { - return Err("remote task journal is too large".into()); - } - let journal = serde_json::from_slice::(&bytes)?; - if journal.version != REMOTE_TASK_JOURNAL_VERSION { - return Err("unsupported remote task journal version".into()); - } - if journal.entries.len() > MAX_REMOTE_TASK_JOURNAL_ENTRIES { - return Err("remote task journal has too many entries".into()); - } - let mut task_ids = HashSet::new(); - for entry in &journal.entries { - let (task_id, valid_result) = match entry { - RemoteTaskJournalEntry::Active { task_id } => (task_id, true), - RemoteTaskJournalEntry::Completed { result } => ( - &result.task_id, - result.message_type == "task_result" - && matches!(result.status.as_str(), "success" | "failed") - && result.result.len() <= REMOTE_RESULT_OUTPUT_BYTES + 64, - ), - }; - if !valid_remote_task_id(task_id) || !valid_result || !task_ids.insert(task_id) { - return Err("remote task journal contains an invalid entry".into()); - } - } - Ok(journal) -} - -fn write_remote_task_journal(path: &Path, entries: Vec) -> Result<()> { - if entries.is_empty() { - return match fs::remove_file(path) { - Ok(()) => Ok(()), - Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), - Err(error) => Err(error.into()), - }; - } - let journal = RemoteTaskJournal { - version: REMOTE_TASK_JOURNAL_VERSION, - entries, - }; - let mut encoded = serde_json::to_vec(&journal)?; - encoded.push(b'\n'); - if encoded.len() as u64 > MAX_REMOTE_TASK_JOURNAL_BYTES { - return Err("remote task journal is too large".into()); - } - let temporary = path.with_file_name(format!( - ".{}.{}.tmp", - path.file_name() - .and_then(|value| value.to_str()) - .unwrap_or("remote-tasks"), - std::process::id() - )); - let mut file = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&temporary)?; - #[cfg(unix)] - fs::set_permissions(&temporary, fs::Permissions::from_mode(0o600))?; - file.write_all(&encoded)?; - file.sync_all()?; - drop(file); - #[cfg(target_os = "windows")] - if path.exists() { - fs::remove_file(path)?; - } - if let Err(error) = fs::rename(&temporary, path) { - let _ = fs::remove_file(&temporary); - return Err(error.into()); - } - Ok(()) -} fn flush_remote_results(socket: &mut LiveSocket, executor: &mut RemoteExecutor) -> Result<()> { for result in executor.due_results() { @@ -2667,416 +505,6 @@ fn flush_remote_results(socket: &mut LiveSocket, executor: &mut RemoteExecutor) Ok(()) } -fn ack_persist_interval(ack: &LiveAck) -> Duration { - Duration::from_millis(ack.next_persist_after_ms) - .clamp(Duration::from_secs(1), Duration::from_secs(3600)) -} - -enum LiveRead { - Ack(LiveAck), - Config(RemoteConfig), - RemoteTask(RemoteTaskMessage), - TaskResultAck(String), - Closed, - Pending, -} - -fn read_live_ack(socket: &mut LiveSocket) -> Result { - match socket.read() { - Ok(Message::Text(text)) => { - let Ok(value) = serde_json::from_str::(text.as_ref()) else { - return Ok(LiveRead::Pending); - }; - match value.get("type").and_then(serde_json::Value::as_str) { - Some("ack") => { - let Ok(ack) = serde_json::from_value::(value) else { - return Ok(LiveRead::Pending); - }; - if ack.message_type != "ack" || ack.ts <= 0 { - return Ok(LiveRead::Pending); - } - Ok(LiveRead::Ack(ack)) - } - Some("config") => { - let Ok(message) = serde_json::from_value::(value) else { - return Ok(LiveRead::Pending); - }; - if message.message_type != "config" || message.ts <= 0 { - return Ok(LiveRead::Pending); - } - Ok(LiveRead::Config(message.config)) - } - Some("remote_task") => { - let Ok(task) = serde_json::from_value::(value) else { - return Ok(LiveRead::Pending); - }; - if task.message_type != "remote_task" - || !valid_remote_task_id(&task.task_id) - || task.command.trim().is_empty() - || task.command.len() > 16_384 - { - return Ok(LiveRead::Pending); - } - Ok(LiveRead::RemoteTask(task)) - } - Some("task_result_ack") => { - let Ok(ack) = serde_json::from_value::(value) else { - return Ok(LiveRead::Pending); - }; - if ack.message_type != "task_result_ack" || !valid_remote_task_id(&ack.task_id) - { - return Ok(LiveRead::Pending); - } - Ok(LiveRead::TaskResultAck(ack.task_id)) - } - _ => Ok(LiveRead::Pending), - } - } - Ok(Message::Ping(payload)) => { - socket.send(Message::Pong(payload))?; - Ok(LiveRead::Pending) - } - Ok(Message::Close(_)) => Ok(LiveRead::Closed), - Ok(_) => Ok(LiveRead::Pending), - Err(WebSocketError::Io(error)) - if matches!(error.kind(), ErrorKind::TimedOut | ErrorKind::WouldBlock) => - { - Ok(LiveRead::Pending) - } - Err(error) => Err(error.into()), - } -} - -fn accept_remote_task( - socket: &mut LiveSocket, - executor: &mut RemoteExecutor, - task: RemoteTaskMessage, -) -> Result<()> { - let task_id = task.task_id.clone(); - let rejected = executor.enqueue(task); - socket.send(Message::Text( - serde_json::json!({ - "type": "task_received", - "task_id": task_id, - }) - .to_string() - .into(), - ))?; - if let Some(result) = rejected { - socket.send(Message::Text(serde_json::to_string(&result)?.into()))?; - } - flush_remote_results(socket, executor)?; - Ok(()) -} - -fn wait_for_live_ack( - socket: &mut LiveSocket, - remote_config: &Arc>>, - remote_executor: &mut RemoteExecutor, -) -> Result { - let deadline = Instant::now() + LIVE_ACK_READ_TIMEOUT; - loop { - let remaining = deadline.saturating_duration_since(Instant::now()); - if remaining.is_zero() { - return Ok(LiveRead::Pending); - } - set_live_read_timeout(socket, Some(remaining))?; - match read_live_ack(socket)? { - LiveRead::Ack(ack) => { - return Ok(LiveRead::Ack(ack)); - } - LiveRead::Config(config) => { - if let Ok(mut target) = remote_config.lock() { - *target = Some(config); - } - } - LiveRead::RemoteTask(task) => { - accept_remote_task(socket, remote_executor, task)?; - } - LiveRead::TaskResultAck(task_id) => { - remote_executor.acknowledge(&task_id); - } - LiveRead::Closed => return Ok(LiveRead::Closed), - LiveRead::Pending => {} - } - } -} - -fn live_update_payload( - reports: Vec, - persist: bool, - info: &mut Option, -) -> Result> { - let mut next_info = info.clone(); - let payload = telemetry::encode(&telemetry::Update::from_reports( - reports, - persist, - &mut next_info, - ))?; - *info = next_info; - Ok(payload) -} - -fn observe_persisted_through(target: &AtomicI64, ack: &LiveAck) { - if ack.persisted && !ack.persistence_error { - target.fetch_max(ack.persisted_through_ts.max(0), Ordering::AcqRel); - } -} - -fn prune_live_queue(pending: &Arc<(Mutex>, Condvar)>, persisted_through: i64) { - if persisted_through <= 0 { - return; - } - if let Ok(mut queue) = pending.0.lock() { - queue.retain(|report| report.timestamp > persisted_through); - } -} - -fn live_batch_after(queue: &VecDeque, timestamp: i64) -> Vec { - let candidates = queue - .iter() - .filter(|report| report.timestamp > timestamp) - .cloned() - .collect::>(); - let count = live_batch::batch_len(&candidates); - candidates.into_iter().take(count).collect() -} - -fn live_sender_loop(endpoint: &str, token: &str, worker: LiveSenderWorker) { - let LiveSenderWorker { - pending, - configured_interval, - persisted_through, - remote_config, - clock, - stats, - } = worker; - let mut socket: Option = None; - let mut wss_interval = configured_interval.lock().map_or( - Duration::from_secs(telemetry::MIN_UPLOAD_INTERVAL), - |interval| *interval, - ); - let mut info = None; - let mut accepted_through = persisted_through.load(Ordering::Acquire); - let mut next_send_at = Instant::now(); - let mut next_probe_at = Instant::now(); - let mut remote_executor = match RemoteExecutor::new() { - Ok(executor) => executor, - Err(error) => { - eprintln!("remote executor disabled: {error}"); - RemoteExecutor::disabled() - } - }; - 'sender: loop { - if socket.is_none() { - match connect_live(endpoint, token) { - Ok((mut connected, offset_ms)) => { - observe_clock(&clock, offset_ms); - if set_live_read_timeout(&mut connected, Some(LIVE_HINT_READ_TIMEOUT)).is_err() - { - thread::sleep(LIVE_RECONNECT_DELAY); - continue; - } - socket = Some(connected); - info = None; - remote_executor.reset_delivery(); - accepted_through = persisted_through.load(Ordering::Acquire); - next_send_at = Instant::now(); - next_probe_at = Instant::now(); - } - Err(error) => { - stats.live_connect_failed(); - eprintln!("live connection failed: {error}"); - thread::sleep(LIVE_RECONNECT_DELAY); - continue; - } - } - } - - if let Some(connected) = socket.as_mut() - && flush_remote_results(connected, &mut remote_executor).is_err() - { - socket = None; - thread::sleep(LIVE_RECONNECT_DELAY); - continue; - } - - let (queue_lock, ready) = &*pending; - let Ok(mut queue) = queue_lock.lock() else { - return; - }; - let (batch, persist, more_pending) = loop { - let now = Instant::now(); - let persisted = persisted_through.load(Ordering::Acquire); - let unsent = live_batch_after(&queue, accepted_through); - let persist = - now >= next_probe_at && queue.iter().any(|report| report.timestamp > persisted); - if persist || (!unsent.is_empty() && now >= next_send_at) { - let last = unsent - .last() - .map_or(accepted_through, |report| report.timestamp); - let more_pending = queue.iter().any(|report| report.timestamp > last); - // A commit can be empty: samples already sent remain buffered by the server. - break (unsent, persist || more_pending, more_pending); - } - - let wake_at = if !unsent.is_empty() { - next_send_at.min(next_probe_at) - } else if queue.iter().any(|report| report.timestamp > persisted) { - next_probe_at - } else { - now + Duration::from_millis(250) - }; - let wait = wake_at - .saturating_duration_since(now) - .min(Duration::from_millis(250)); - queue = match ready.wait_timeout(queue, wait) { - Ok((queue, _)) => queue, - Err(_) => return, - }; - drop(queue); - - let mut drop_socket = false; - if let Some(connected) = socket.as_mut() { - if flush_remote_results(connected, &mut remote_executor).is_err() - || set_live_read_timeout(connected, Some(LIVE_HINT_READ_TIMEOUT)).is_err() - { - drop_socket = true; - } else { - match read_live_ack(connected) { - Ok(LiveRead::Closed) => drop_socket = true, - Ok(LiveRead::Ack(ack)) => { - observe_persisted_through(&persisted_through, &ack); - if ack.persistence_error { - stats.live_persistence_failed(); - drop_socket = true; - } else { - prune_live_queue(&pending, ack.persisted_through_ts); - accepted_through = accepted_through.max(ack.persisted_through_ts); - next_probe_at = Instant::now() + ack_persist_interval(&ack); - } - } - Ok(LiveRead::Config(config)) => { - if let Ok(mut target) = remote_config.lock() { - *target = Some(config); - } - } - Ok(LiveRead::RemoteTask(task)) => { - if accept_remote_task(connected, &mut remote_executor, task).is_err() { - drop_socket = true; - } - } - Ok(LiveRead::TaskResultAck(task_id)) => { - remote_executor.acknowledge(&task_id); - } - Ok(LiveRead::Pending) => {} - Err(error) => { - eprintln!("live hint read failed: {error}"); - drop_socket = true; - } - } - } - } - if drop_socket { - socket = None; - accepted_through = persisted_through.load(Ordering::Acquire); - next_send_at = Instant::now(); - next_probe_at = Instant::now(); - thread::sleep(LIVE_RECONNECT_DELAY); - continue 'sender; - } - queue = match queue_lock.lock() { - Ok(queue) => queue, - Err(_) => return, - }; - }; - drop(queue); - - if let Ok(interval) = configured_interval.lock() { - wss_interval = (*interval).clamp( - Duration::from_secs(telemetry::MIN_UPLOAD_INTERVAL), - Duration::from_secs(60), - ); - } - - let batch_last = batch.last().map(|report| report.timestamp); - let batch_len = batch.len(); - let payload = match live_update_payload(batch, persist, &mut info) { - Ok(payload) => payload, - Err(error) => { - eprintln!("live payload encode failed: {error}"); - next_send_at = Instant::now() + wss_interval; - continue; - } - }; - let sent = socket - .as_mut() - .is_some_and(|socket| socket.send(Message::Binary(payload.into())).is_ok()); - if !sent { - socket = None; - accepted_through = persisted_through.load(Ordering::Acquire); - next_send_at = Instant::now(); - next_probe_at = Instant::now(); - thread::sleep(LIVE_RECONNECT_DELAY); - continue; - } - stats.live_batch_sent(batch_len); - if let Some(timestamp) = batch_last { - accepted_through = accepted_through.max(timestamp); - } - next_send_at = Instant::now() + wss_interval; - if !persist { - continue; - } - - let mut drop_socket = false; - if let Some(socket) = socket.as_mut() { - match wait_for_live_ack(socket, &remote_config, &mut remote_executor) { - Ok(LiveRead::Closed) => drop_socket = true, - Ok(LiveRead::Ack(ack)) => { - observe_persisted_through(&persisted_through, &ack); - if ack.persistence_error { - stats.live_persistence_failed(); - drop_socket = true; - } else { - prune_live_queue(&pending, ack.persisted_through_ts); - accepted_through = accepted_through.max(ack.persisted_through_ts); - next_probe_at = Instant::now() + ack_persist_interval(&ack); - if more_pending - && batch_last - .is_some_and(|timestamp| ack.persisted_through_ts >= timestamp) - { - next_probe_at = Instant::now(); - } - } - } - Ok(LiveRead::Config(config)) => { - if let Ok(mut target) = remote_config.lock() { - *target = Some(config); - } - drop_socket = true; - } - Ok(LiveRead::RemoteTask(_) | LiveRead::TaskResultAck(_)) => {} - Ok(LiveRead::Pending) => { - eprintln!("live ACK timed out"); - drop_socket = true; - } - Err(error) => { - eprintln!("live ACK read failed: {error}"); - drop_socket = true; - } - } - } - if drop_socket { - socket = None; - accepted_through = persisted_through.load(Ordering::Acquire); - next_send_at = Instant::now(); - next_probe_at = Instant::now(); - thread::sleep(LIVE_RECONNECT_DELAY); - } - } -} - fn apply_remote(config: &mut RuntimeConfig, remote: &RemoteConfig) -> bool { if valid_sample_schedule(remote.report_interval, remote.collect_interval) { config.report_interval = remote.report_interval; @@ -3271,239 +699,28 @@ fn rewrite_pending_spool(path: &Path, samples: &[Report]) -> Result<()> { file.write_all(b"\n")?; } file.sync_data()?; - #[cfg(target_os = "windows")] - if path.exists() { - fs::remove_file(path)?; - } + // fs::rename replaces the destination on Windows too, so no pre-delete is + // needed here (same reasoning as write_remote_task_journal). fs::rename(&temporary, path)?; Ok(()) } -fn normalized_version(value: &str) -> &str { - let value = value.trim(); - value.strip_prefix('v').unwrap_or(value) -} - -fn version_triplet(value: &str) -> Option<(u64, u64, u64)> { - let mut parts = normalized_version(value).split('.'); - let version = ( - parts.next()?.parse().ok()?, - parts.next()?.parse().ok()?, - parts.next()?.parse().ok()?, - ); - parts.next().is_none().then_some(version) -} - -fn agent_artifact_name() -> Option<&'static str> { - match (env::consts::OS, env::consts::ARCH) { - ("linux", "x86_64") => Some(if cfg!(target_env = "musl") { - "agent-linux-x64-musl" - } else { - "agent-linux-x64-glibc" - }), - ("linux", "aarch64") => Some(if cfg!(target_env = "musl") { - "agent-linux-aarch64-musl" - } else { - "agent-linux-aarch64-glibc" - }), - ("windows", "x86_64") => Some("agent-windows-x64.exe"), - ("macos", "aarch64") => Some("agent-macos-aarch64"), - ("freebsd", "x86_64") => Some("agent-freebsd-x64"), - ("freebsd", "aarch64") => Some("agent-freebsd-aarch64"), - _ => None, +/// Compacts the spool at most once per `SPOOL_REWRITE_MIN_INTERVAL`. +/// +/// While the live connection is down and the queue sits at capacity the sample +/// set changes every second; rewriting and fsyncing the full spool that often +/// wears the disk for no benefit. The in-memory queue plus the per-sample line +/// appends stay authoritative until the next due rewrite. +fn rewrite_spool_if_due(path: &Path, samples: &[Report], last: &mut Option) { + if last.is_some_and(|previous| previous.elapsed() < SPOOL_REWRITE_MIN_INTERVAL) { + return; + } + match rewrite_pending_spool(path, samples) { + Ok(()) => *last = Some(Instant::now()), + Err(error) => eprintln!("pending report spool compaction failed: {error}"), } } -fn executable_format_valid(path: &Path) -> bool { - let mut magic = [0_u8; 4]; - if fs::File::open(path) - .and_then(|mut file| file.read_exact(&mut magic)) - .is_err() - { - return false; - } - match env::consts::OS { - "linux" | "freebsd" => magic == *b"\x7fELF", - "windows" => magic.starts_with(b"MZ"), - "macos" => matches!( - magic, - [0xcf, 0xfa, 0xed, 0xfe] - | [0xfe, 0xed, 0xfa, 0xcf] - | [0xca, 0xfe, 0xba, 0xbe] - | [0xbe, 0xba, 0xfe, 0xca] - ), - _ => false, - } -} - -fn sha256_file(path: &Path) -> Result { - let mut file = fs::File::open(path)?; - let mut digest = Sha256::new(); - let mut buffer = [0_u8; 64 * 1024]; - loop { - let read = file.read(&mut buffer)?; - if read == 0 { - break; - } - digest.update(&buffer[..read]); - } - Ok(digest - .finalize() - .iter() - .map(|byte| format!("{byte:02x}")) - .collect()) -} - -fn release_asset_sha256(asset: &GithubReleaseAsset) -> Option { - let digest = asset.digest.as_deref()?.strip_prefix("sha256:")?; - (digest.len() == 64 - && digest - .chars() - .all(|character| character.is_ascii_hexdigit())) - .then(|| digest.to_ascii_lowercase()) -} - -fn download_agent( - agent: &ureq::Agent, - url: &str, - destination: &Path, - expected_version: &str, - expected_sha256: &str, -) -> Result { - let Ok(response) = agent.get(url).call() else { - return Ok(false); - }; - let response = response.into_body().into_reader(); - let mut file = fs::File::create(destination)?; - let copied = io::copy(&mut response.take(MAX_AGENT_BINARY_BYTES + 1), &mut file)?; - file.flush()?; - drop(file); - if copied > MAX_AGENT_BINARY_BYTES - || !executable_format_valid(destination) - || sha256_file(destination)? != expected_sha256 - { - let _ = fs::remove_file(destination); - return Ok(false); - } - #[cfg(unix)] - fs::set_permissions(destination, fs::Permissions::from_mode(0o755))?; - - let downloaded_version = Command::new(destination) - .arg("--version") - .output() - .ok() - .filter(|output| output.status.success()) - .and_then(|output| { - String::from_utf8_lossy(&output.stdout) - .split_whitespace() - .next_back() - .map(str::to_string) - }); - let valid = downloaded_version.as_deref().map(normalized_version) - == Some(normalized_version(expected_version)); - if !valid { - let _ = fs::remove_file(destination); - } - Ok(valid) -} - -fn mirrored_download_url(mirror: &str, url: &str) -> String { - let mirror = mirror.trim().trim_end_matches('/'); - if mirror.is_empty() { - url.to_string() - } else { - format!("{mirror}/{url}") - } -} - -fn update(agent: &ureq::Agent, mirror: &str) -> Result { - let mut response = agent - .get(LATEST_RELEASE_API) - .header("Accept", "application/vnd.github+json") - .header("User-Agent", format!("nodeflare-agent/{VERSION}")) - .call()?; - let release = response.body_mut().read_json::()?; - let Some(remote_version) = version_triplet(&release.tag_name) else { - return Ok(false); - }; - let Some(current_version) = version_triplet(VERSION) else { - return Ok(false); - }; - if remote_version <= current_version { - return Ok(false); - } - let Some(artifact) = agent_artifact_name() else { - return Ok(false); - }; - let current = env::current_exe()?; - let temporary = current.with_file_name(format!( - ".nodeflare-agent.{}.download{}", - std::process::id(), - if cfg!(target_os = "windows") { - ".exe" - } else { - "" - } - )); - let Some(release_asset) = release.assets.iter().find(|asset| asset.name == artifact) else { - return Err(format!("latest release does not contain {artifact}").into()); - }; - let Some(expected_sha256) = release_asset_sha256(release_asset) else { - return Err( - format!("latest release does not contain a SHA-256 digest for {artifact}").into(), - ); - }; - if !download_agent( - agent, - &mirrored_download_url(mirror, &release_asset.browser_download_url), - &temporary, - &release.tag_name, - &expected_sha256, - )? { - return Err("downloaded agent version does not match the configured version".into()); - } - - #[cfg(unix)] - { - fs::rename(&temporary, ¤t)?; - let error = Command::new(¤t).args(env::args_os().skip(1)).exec(); - Err(error.into()) - } - - #[cfg(target_os = "windows")] - { - let script = concat!( - "$ErrorActionPreference='Stop'; ", - "$targetPid=[int]$env:NODEFLARE_UPDATE_PID; ", - "Wait-Process -Id $targetPid; ", - "Move-Item -LiteralPath $env:NODEFLARE_UPDATE_NEW ", - "-Destination $env:NODEFLARE_UPDATE_CURRENT -Force; ", - "try { Start-ScheduledTask -TaskName 'nodeflare-agent' -ErrorAction Stop } ", - "catch { $restartArgs=@($env:NODEFLARE_UPDATE_ARGS | ConvertFrom-Json); ", - "Start-Process -FilePath $env:NODEFLARE_UPDATE_CURRENT -ArgumentList $restartArgs }" - ); - Command::new("powershell.exe") - .args([ - "-NoLogo", - "-NoProfile", - "-NonInteractive", - "-Command", - script, - ]) - .env("NODEFLARE_UPDATE_PID", std::process::id().to_string()) - .env("NODEFLARE_UPDATE_NEW", &temporary) - .env("NODEFLARE_UPDATE_CURRENT", ¤t) - .env( - "NODEFLARE_UPDATE_ARGS", - serde_json::to_string(&env::args().skip(1).collect::>())?, - ) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; - Ok(true) - } -} fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { let mut config = runtime_config(options)?; @@ -3522,6 +739,7 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { let mut latency_executor = LatencyExecutor::new(Arc::clone(&stats))?; let mut pending_results = Vec::new(); let spool_path = pending_spool_path()?; + let mut last_spool_rewrite: Option = None; let mut pending_samples = load_pending_spool(&spool_path).unwrap_or_else(|error| { eprintln!("pending report spool ignored: {error}"); Vec::new() @@ -3567,10 +785,8 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { let before = pending_samples.len(); pending_samples.retain(|report| report.timestamp > persisted_through); stats.persisted_samples_pruned(before.saturating_sub(pending_samples.len())); - if before != pending_samples.len() - && let Err(error) = rewrite_pending_spool(&spool_path, &pending_samples) - { - eprintln!("pending report spool compaction failed: {error}"); + if before != pending_samples.len() { + rewrite_spool_if_due(&spool_path, &pending_samples, &mut last_spool_rewrite); } if once_target_timestamp.is_some_and(|target| persisted_through >= target) { return Ok(()); @@ -3645,8 +861,12 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { println!("{}", serde_json::to_string_pretty(&report)?); return Ok(()); } - append_pending_spool(&spool_path, &report) - .map_err(|error| format!("pending report spool append failed: {error}"))?; + if let Err(error) = append_pending_spool(&spool_path, &report) { + // A spool write failure (disk full, transient IO error) must not + // kill the agent: the sample is still queued in memory and will + // be retried on the next compaction. + eprintln!("pending report spool append failed: {error}"); + } if let Some(live) = &live { live.send(&report); } @@ -3668,15 +888,14 @@ fn run(options: &CliOptions, once: bool, print_only: bool) -> Result<()> { pending_samples.drain(..overflow); stats.live_queue_dropped(overflow); } - if (before_prune != pending_samples.len() + if before_prune != pending_samples.len() || latency_before_prune != pending_samples .iter() .map(|report| report.latency_results.len()) - .sum::()) - && let Err(error) = rewrite_pending_spool(&spool_path, &pending_samples) + .sum::() { - eprintln!("pending report spool compaction failed: {error}"); + rewrite_spool_if_due(&spool_path, &pending_samples, &mut last_spool_rewrite); } next_collect = advance_deadline(next_collect, SAMPLE_INTERVAL, Instant::now()); } @@ -3721,814 +940,5 @@ fn main() { } #[cfg(test)] -mod tests { - use clap::Parser; - use std::fs; - use std::path::PathBuf; - use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +mod tests; - use super::{ - CLOCK_CALIBRATION_MAX_AGE, CapturedOutput, CliOptions, ClockCalibration, DiskMetric, - GithubReleaseAsset, LatencyResult, LatencyTask, LiveAck, MAX_PENDING_LATENCY_RESULTS, - PUBLIC_IP_STALE_AFTER, PublicIpValue, REMOTE_RESULT_OUTPUT_BYTES, - REMOTE_STREAM_OUTPUT_BYTES, RemoteExecutor, RemoteTaskJournalEntry, RemoteTaskMessage, - Report, TaskResultMessage, UPDATE_CHECK_JITTER_MAX_SECONDS, ack_persist_interval, - advance_deadline, clock_offset_from_http_date, corrected_timestamp, dedupe_disks, - execute_remote_task_with_timeout, gpu_name_from_uevent, is_public_probe_ip, live_endpoint, - live_update_payload, monotonic_report_timestamp, normalized_version, parse_lspci_gpu_names, - parse_pciconf_gpu_names, parse_probe_target, parse_public_ip, - parse_system_profiler_gpu_names, ping_latencies, ping_latency, prune_report_samples, - release_asset_sha256, remote_result_text, sanitize_latency_tasks, selected_interface, - update_check_jitter, valid_endpoint, version_triplet, wildcard_match, - write_remote_task_journal, - }; - - #[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] - use super::connection_counts_from_netstat; - #[cfg(target_os = "linux")] - use super::disk_device; - - #[test] - fn parses_family_matched_public_ips() { - let v4: std::net::IpAddr = "203.0.113.7".parse().unwrap(); - let v6: std::net::IpAddr = "2001:db8::1".parse().unwrap(); - assert_eq!(parse_public_ip("203.0.113.7\n", false), Some(v4)); - assert_eq!(parse_public_ip("2001:db8::1", true), Some(v6)); - assert_eq!(parse_public_ip("2001:db8::1", false), None); - assert_eq!(parse_public_ip("203.0.113.7", true), None); - assert_eq!(parse_public_ip("not an ip", false), None); - assert_eq!(parse_public_ip("", false), None); - } - - #[test] - fn expires_public_ip_after_sustained_probe_failures() { - let started_at = Instant::now(); - let address = "203.0.113.7".parse().unwrap(); - let mut value = PublicIpValue::default(); - - value.observe(Some(address), started_at); - value.observe( - None, - started_at + PUBLIC_IP_STALE_AFTER - Duration::from_secs(1), - ); - assert_eq!(value.address, Some(address)); - - value.observe(None, started_at + PUBLIC_IP_STALE_AFTER); - assert_eq!(value.address, None); - } - - #[test] - fn parses_platform_gpu_names() { - let lspci = "00:02.0 VGA compatible controller: Intel Corporation CometLake-S GT2 [UHD Graphics 630] (rev 05)\n\ - 01:00.0 Audio device: NVIDIA Corporation HDMI Audio\n\ - 02:00.0 3D controller: NVIDIA Corporation GA102 [GeForce RTX 3090] (rev a1)"; - assert_eq!( - parse_lspci_gpu_names(lspci), - [ - "Intel Corporation CometLake-S GT2 [UHD Graphics 630] (rev 05)", - "NVIDIA Corporation GA102 [GeForce RTX 3090] (rev a1)", - ] - ); - - let profiler = "Graphics/Displays:\n\n Apple M3 Max:\n\n Chipset Model: Apple M3 Max\n Metal Support: Metal 3"; - assert_eq!(parse_system_profiler_gpu_names(profiler), ["Apple M3 Max"]); - - let pciconf = "vgapci0@pci0:0:2:0:\tclass=0x030000 rev=0x02 hdr=0x00 vendor=0x8086\n\ - \tvendor = 'Intel Corporation'\n\ - \tdevice = 'UHD Graphics 630'\n\ - \tclass = display\n\ - em0@pci0:0:25:0:\tclass=0x020000 rev=0x05 hdr=0x00 vendor=0x8086\n\ - \tvendor = 'Intel Corporation'\n\ - \tdevice = 'Ethernet Connection'\n\ - \tclass = network"; - assert_eq!( - parse_pciconf_gpu_names(pciconf), - ["Intel Corporation UHD Graphics 630"] - ); - - assert_eq!( - gpu_name_from_uevent("DRIVER=i915\nPCI_CLASS=30000\nPCI_ID=8086:591B"), - Some("Intel Integrated Graphics".to_string()) - ); - assert_eq!( - gpu_name_from_uevent("DRIVER=virtio_gpu\nPCI_CLASS=30000"), - None - ); - } - - #[cfg(target_os = "linux")] - #[test] - fn selects_whole_disk_devices() { - assert!(disk_device("sda")); - assert!(disk_device("nvme0n1")); - assert!(!disk_device("sda1")); - assert!(!disk_device("nvme0n1p1")); - } - - #[test] - fn filters_virtual_interfaces_and_supports_include_exclude_patterns() { - assert!(!selected_interface("lo", "")); - assert!(!selected_interface("docker0", "")); - assert!(!selected_interface("veth123", "")); - assert!(!selected_interface("br0", "!eth*")); - assert!(selected_interface("br0", "br0")); - assert!(!selected_interface("br0", "br*,!br0")); - assert!(!selected_interface("br0", "!br0,br*")); - assert!(!selected_interface("Software Loopback Interface 1", "")); - assert!(selected_interface("eth0", "")); - assert!(selected_interface("ens3", "eth*,ens*")); - assert!(!selected_interface("wlan0", "eth*,ens*")); - assert!(!selected_interface("eth0", "eth*,!eth0")); - assert!(selected_interface("ens3", "!eth*")); - assert!(wildcard_match("en?3", "ens3")); - assert!(!wildcard_match("en?3", "enp4s0")); - } - - #[test] - fn deduplicates_mounts_for_the_same_device() { - let disks = dedupe_disks([ - DiskMetric { - name: "/dev/vg/root".into(), - mount_point: "/var".into(), - total: 10, - used: 4, - ..DiskMetric::default() - }, - DiskMetric { - name: "/dev/vg/root".into(), - mount_point: "/".into(), - total: 20, - used: 8, - ..DiskMetric::default() - }, - DiskMetric { - name: "tank/data".into(), - mount_point: "/data".into(), - total: 30, - used: 9, - ..DiskMetric::default() - }, - DiskMetric { - name: "tank/archive".into(), - mount_point: "/archive".into(), - total: 25, - used: 7, - ..DiskMetric::default() - }, - DiskMetric { - name: "/var/lib/data".into(), - mount_point: "/var/lib/data".into(), - total: 40, - used: 12, - ..DiskMetric::default() - }, - ]); - assert_eq!(disks.len(), 3); - assert_eq!(disks[0].mount_point, "/"); - assert_eq!(disks[0].total, 20); - assert_eq!(disks[1].name, "tank/data"); - assert_eq!(disks[2].name, "/var/lib/data"); - } - - #[cfg(target_os = "linux")] - #[test] - fn keeps_root_filesystem_but_filters_virtual_mounts() { - assert!(!super::excluded_filesystem("overlay", "/")); - assert!(super::excluded_filesystem("tmpfs", "/tmp")); - assert!(super::excluded_filesystem("proc", "/proc")); - assert!(!super::excluded_filesystem("btrfs", "/var")); - } - - #[cfg(target_os = "linux")] - #[test] - fn memory_matches_komari_default_including_shared_memory_and_swap_cache() { - let memory = super::memory_sample( - "MemTotal: 1000000 kB\nMemFree: 100000 kB\nMemAvailable: 750000 kB\n\ - Cached: 300000 kB\nSReclaimable: 50000 kB\nBuffers: 25000 kB\n\ - Shmem: 25000 kB\nSwapTotal: 100000 kB\nSwapFree: 60000 kB\nSwapCached: 20000 kB\n", - ); - assert_eq!(memory.total, 1_000_000 * 1024); - assert_eq!(memory.used, 550_000 * 1024); - assert_eq!(memory.swap_total, 100_000 * 1024); - assert_eq!(memory.swap_used, 20_000 * 1024); - } - - #[cfg(target_os = "linux")] - #[test] - fn filling_file_cache_does_not_inflate_reported_memory() { - let before = super::memory_sample( - "MemTotal: 1000 kB\nMemFree: 400 kB\nCached: 300 kB\n\ - Buffers: 50 kB\nSReclaimable: 50 kB\nShmem: 100 kB\n", - ); - let after = super::memory_sample( - "MemTotal: 1000 kB\nMemFree: 200 kB\nCached: 500 kB\n\ - Buffers: 50 kB\nSReclaimable: 50 kB\nShmem: 100 kB\n", - ); - assert_eq!(before.used, 300 * 1024); - assert_eq!(after.used, before.used); - } - - #[cfg(target_os = "linux")] - #[test] - fn memory_handles_missing_and_inconsistent_kernel_counters() { - let memory = super::memory_sample( - "MemTotal: 1000 kB\nMemFree: 200 kB\nCached: 1000 kB\n\ - Shmem: 100 kB\nSwapTotal: 100 kB\nSwapFree: 20 kB\nSwapCached: 100 kB\n", - ); - assert_eq!(memory.used, 900 * 1024); - assert_eq!(memory.swap_used, 80 * 1024); - for contents in ["", "MemTotal: invalid kB\n", "MemTotal: -1 kB\n"] { - let memory = super::memory_sample(contents); - assert_eq!( - ( - memory.total, - memory.used, - memory.swap_total, - memory.swap_used - ), - (0, 0, 0, 0) - ); - } - let memory = super::memory_sample( - "MemTotal: 100 kB\nMemFree: 1000 kB\nSwapTotal: 100 kB\nSwapFree: 1000 kB\n", - ); - assert_eq!((memory.used, memory.swap_used), (0, 0)); - } - - #[cfg(target_os = "linux")] - #[test] - fn reads_local_mounts_and_decodes_escaped_paths_once() { - let mounts = super::disk_mounts( - r"overlay / overlay rw 0 0 -proc /proc proc rw 0 0 -tmpfs /run tmpfs rw 0 0 -/dev/vda1 /data\040volume ext4 rw 0 0 -/dev/vda2 /literal\134040path ext4 rw 0 0 -/dev/vda3 /tab\011dir ext4 rw 0 0 -server:/export /remote nfs4 rw 0 0 -//server/share /remote-share cifs rw 0 0 -none /auto autofs rw 0 0 -invalid", - ) - .collect::>(); - assert_eq!( - mounts, - vec![ - ("overlay".into(), "/".into()), - ("/dev/vda1".into(), "/data volume".into()), - ("/dev/vda2".into(), "/literal\\040path".into()), - ("/dev/vda3".into(), "/tab\tdir".into()), - ] - ); - } - - #[cfg(target_os = "linux")] - #[test] - fn reads_root_disk_capacity_without_external_commands() { - let (total, used) = super::filesystem_space("/").unwrap(); - assert!(total > 0); - assert!((0..=total).contains(&used)); - assert!( - super::disk_usage() - .iter() - .any(|disk| disk.mount_point == "/") - ); - assert!(super::filesystem_space("/proc/self/not-a-mount").is_none()); - assert!(super::filesystem_space("/nul\0path").is_none()); - } - - #[test] - fn overdue_latency_tasks_wait_and_resume_without_duplicate_probes() { - let (task_tx, task_rx) = std::sync::mpsc::channel(); - let (result_tx, result_rx) = std::sync::mpsc::channel(); - let mut executor = super::LatencyExecutor { - task_tx, - result_rx, - in_flight: Default::default(), - stats: Default::default(), - }; - let task = LatencyTask { - id: "queued-probe".into(), - name: "probe".into(), - task_type: "tcp".into(), - target: "example.com".into(), - port: Some(443), - interval_seconds: 30, - }; - let mut deadlines = std::collections::HashMap::new(); - let started = Instant::now(); - executor.schedule(std::slice::from_ref(&task), &mut deadlines, started); - assert_eq!(task_rx.try_recv().unwrap().id, task.id); - assert_eq!(deadlines[&task.id], started + Duration::from_secs(30)); - - let overdue = started + Duration::from_secs(30); - executor.schedule(std::slice::from_ref(&task), &mut deadlines, overdue); - assert!(task_rx.try_recv().is_err()); - assert_eq!(deadlines[&task.id], overdue + Duration::from_secs(1)); - - result_tx - .send(LatencyResult { - task_id: task.id.clone(), - ..LatencyResult::default() - }) - .unwrap(); - assert_eq!(executor.drain().len(), 1); - let retry_at = deadlines[&task.id]; - executor.schedule(std::slice::from_ref(&task), &mut deadlines, retry_at); - executor.schedule(std::slice::from_ref(&task), &mut deadlines, retry_at); - assert_eq!(task_rx.try_recv().unwrap().id, task.id); - assert!(task_rx.try_recv().is_err()); - assert_eq!(deadlines[&task.id], retry_at + Duration::from_secs(30)); - - drop(task_rx); - executor.in_flight.clear(); - let retry_at = deadlines[&task.id]; - executor.schedule(std::slice::from_ref(&task), &mut deadlines, retry_at); - assert_eq!(deadlines[&task.id], retry_at + Duration::from_secs(1)); - } - - #[test] - fn parses_release_versions() { - assert_eq!(normalized_version("v1.2.3"), "1.2.3"); - assert_eq!(version_triplet("v1.2.3"), Some((1, 2, 3))); - assert_eq!(version_triplet("1.2"), None); - assert_eq!(version_triplet("1.2.3.4"), None); - assert_eq!(version_triplet("rust-3"), None); - } - - #[test] - fn validates_release_asset_digests() { - let hash = "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF"; - let mut asset = GithubReleaseAsset { - name: "agent-linux-x64-glibc".to_string(), - browser_download_url: "https://example.com/agent".to_string(), - digest: Some(format!("sha256:{hash}")), - }; - let expected = hash.to_ascii_lowercase(); - assert_eq!( - release_asset_sha256(&asset).as_deref(), - Some(expected.as_str()) - ); - asset.digest = Some("sha512:0123".to_string()); - assert_eq!(release_asset_sha256(&asset), None); - asset.digest = None; - assert_eq!(release_asset_sha256(&asset), None); - } - - #[test] - fn validates_endpoints() { - assert!(valid_endpoint("https://monitor.example.com/")); - assert!(valid_endpoint("http://127.0.0.1:8787")); - assert!(valid_endpoint("http://localhost:8787")); - assert!(valid_endpoint("http://[::1]:8787")); - assert!(!valid_endpoint("http://monitor.example.com")); - assert!(!valid_endpoint("http://127.0.0.2:8787")); - assert!(!valid_endpoint("monitor.example.com")); - assert!(!valid_endpoint("https://")); - assert!(!valid_endpoint("https://user@example.com")); - assert!(!valid_endpoint("https://monitor.example.com/?token=abc")); - assert!(!valid_endpoint("https://bad host.example")); - } - - #[test] - fn builds_live_websocket_endpoints() { - assert_eq!( - live_endpoint("https://monitor.example.com").unwrap(), - "wss://monitor.example.com/api/agent/ws" - ); - assert_eq!( - live_endpoint("https://monitor.example.com/base/").unwrap(), - "wss://monitor.example.com/base/api/agent/ws" - ); - assert_eq!( - live_endpoint("http://127.0.0.1:8787").unwrap(), - "ws://127.0.0.1:8787/api/agent/ws" - ); - } - - #[test] - fn accepts_durable_ack_interval_and_limits_upload_frequency() { - let ack: LiveAck = serde_json::from_str( - r#"{"type":"ack","ts":100,"persisted":true,"persistenceError":false,"persistedThroughTs":100,"nextPersistAfterMs":60000}"#, - ) - .unwrap(); - assert_eq!(ack_persist_interval(&ack), Duration::from_secs(60)); - let idle: LiveAck = serde_json::from_str( - r#"{"type":"ack","ts":100,"persisted":true,"persistenceError":false,"persistedThroughTs":100,"nextPersistAfterMs":120000}"#, - ) - .unwrap(); - assert_eq!(ack_persist_interval(&idle), Duration::from_secs(120)); - assert_eq!(super::live_batch_interval(1), Duration::from_secs(3)); - assert_eq!(super::live_batch_interval(3), Duration::from_secs(3)); - assert_eq!(super::live_batch_interval(15), Duration::from_secs(15)); - } - - #[cfg(unix)] - #[test] - fn remote_commands_capture_output_and_exit_status() { - let result = execute_remote_task_with_timeout( - &RemoteTaskMessage { - message_type: "remote_task".to_string(), - task_id: "task-output".to_string(), - command: "printf stdout; printf stderr >&2; exit 7".to_string(), - }, - Duration::from_secs(2), - ); - assert_eq!(result.status, "failed"); - assert_eq!(result.exit_code, Some(7)); - assert_eq!(result.result, "stdout\nstderr"); - } - - #[cfg(unix)] - #[test] - fn remote_commands_are_terminated_after_the_deadline() { - let result = execute_remote_task_with_timeout( - &RemoteTaskMessage { - message_type: "remote_task".to_string(), - task_id: "task-timeout".to_string(), - command: "sleep 5".to_string(), - }, - Duration::from_millis(50), - ); - assert_eq!(result.status, "failed"); - assert_eq!(result.exit_code, Some(-1)); - assert!(result.result.contains("已终止")); - } - - #[test] - fn combined_remote_output_is_bounded_at_utf8_boundaries() { - let result = remote_result_text( - &CapturedOutput { - bytes: vec![b'x'; REMOTE_STREAM_OUTPUT_BYTES as usize], - truncated: true, - }, - &CapturedOutput { - bytes: "输出" - .repeat(REMOTE_STREAM_OUTPUT_BYTES as usize / 6) - .into_bytes(), - truncated: false, - }, - ); - assert!(result.len() < REMOTE_RESULT_OUTPUT_BYTES + 64); - assert!(result.ends_with("[输出已截断]")); - assert!(!result.contains('\u{fffd}')); - } - - #[cfg(unix)] - #[test] - fn verbose_remote_commands_finish_after_output_is_truncated() { - for command in [ - "set -e; head -c 1048576 /dev/zero; printf completed >&2", - "set -e; head -c 1048576 /dev/zero >&2; printf completed", - ] { - let result = execute_remote_task_with_timeout( - &RemoteTaskMessage { - message_type: "remote_task".to_string(), - task_id: "verbose-output".to_string(), - command: command.to_string(), - }, - Duration::from_secs(5), - ); - assert_eq!(result.status, "success"); - assert_eq!(result.exit_code, Some(0)); - assert!(result.result.contains("completed")); - assert!(result.result.ends_with("[输出已截断]")); - assert!(result.result.len() < REMOTE_RESULT_OUTPUT_BYTES + 64); - } - } - - fn temporary_remote_task_journal(label: &str) -> PathBuf { - let nonce = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_nanos(); - let directory = std::env::temp_dir().join(format!( - "nodeflare-agent-{label}-{}-{nonce}", - std::process::id() - )); - fs::create_dir_all(&directory).unwrap(); - directory.join("remote-tasks.json") - } - - #[test] - fn interrupted_remote_tasks_are_failed_without_reexecution() { - let path = temporary_remote_task_journal("interrupted-task"); - let task_id = "8dd70536-f721-4d47-af63-e2f17c83de25"; - write_remote_task_journal( - &path, - vec![RemoteTaskJournalEntry::Active { - task_id: task_id.to_string(), - }], - ) - .unwrap(); - - let mut executor = RemoteExecutor::new_at(path.clone()).unwrap(); - assert!(executor.active.is_empty()); - let due = executor.due_results(); - assert_eq!(due.len(), 1); - assert_eq!(due[0].task_id, task_id); - assert_eq!(due[0].status, "failed"); - assert!(due[0].result.contains("不会再次执行")); - - executor.mark_sent(task_id); - assert!(executor.due_results().is_empty()); - executor.reset_delivery(); - assert_eq!(executor.due_results().len(), 1); - executor.acknowledge(task_id); - assert!(!path.exists()); - - let directory = path.parent().unwrap().to_path_buf(); - drop(executor); - fs::remove_dir_all(directory).unwrap(); - } - - #[test] - fn completed_remote_tasks_return_the_cached_result() { - let path = temporary_remote_task_journal("completed-task"); - let task_id = "6cbf4d36-f33a-40ef-b952-6af1b8491820"; - let result = TaskResultMessage { - message_type: "task_result".to_string(), - task_id: task_id.to_string(), - status: "success".to_string(), - result: "already finished".to_string(), - exit_code: Some(0), - }; - write_remote_task_journal( - &path, - vec![RemoteTaskJournalEntry::Completed { - result: result.clone(), - }], - ) - .unwrap(); - - let mut executor = RemoteExecutor::new_at(path.clone()).unwrap(); - assert!( - executor - .enqueue(RemoteTaskMessage { - message_type: "remote_task".to_string(), - task_id: task_id.to_string(), - command: "this command must not run".to_string(), - }) - .is_none() - ); - assert!(executor.active.is_empty()); - assert_eq!(executor.due_results(), [result]); - executor.acknowledge(task_id); - - let directory = path.parent().unwrap().to_path_buf(); - drop(executor); - fs::remove_dir_all(directory).unwrap(); - } - - #[test] - fn encodes_realtime_samples_as_a_batch() { - let payload = live_update_payload( - vec![ - Report { - timestamp: 10, - cpu: 20.0, - ..Report::default() - }, - Report { - timestamp: 15, - cpu: 30.0, - ..Report::default() - }, - ], - false, - &mut None, - ) - .unwrap(); - let value: serde_json::Value = nodeflare_telemetry::decode(&payload).unwrap(); - assert_eq!(value["persist"], false); - assert_eq!(value["samples"].as_array().unwrap().len(), 2); - assert_eq!(value["samples"][1]["metrics"]["cpu"], 30.0); - } - - #[test] - fn calibrates_timestamps_from_http_date() { - let server_ms = 784_111_777_000_i64; - assert_eq!( - clock_offset_from_http_date( - "Sun, 06 Nov 1994 08:49:37 GMT", - server_ms - 1_000, - server_ms + 1_000, - ), - Some(0), - ); - assert_eq!(corrected_timestamp(100, 2_500), 102); - assert_eq!(corrected_timestamp(100, -2_500), 97); - assert_eq!(monotonic_report_timestamp(97, 100, 90), 101); - assert_eq!(monotonic_report_timestamp(110, 100, 120), 121); - assert_eq!(monotonic_report_timestamp(130, 120, 110), 130); - } - - #[test] - fn ignores_small_clock_jitter_until_the_calibration_expires() { - let mut calibration = ClockCalibration::default(); - assert!(calibration.observe(1_000)); - assert!(!calibration.observe(2_000)); - assert_eq!(calibration.offset_ms, 1_000); - assert!(calibration.observe(25_000)); - calibration.calibrated_at = Some(Instant::now() - CLOCK_CALIBRATION_MAX_AGE); - assert!(calibration.observe(25_001)); - } - - #[test] - fn parses_cli_options() { - let parsed = CliOptions::try_parse_from([ - "agent", - "-e", - "https://monitor.example.com", - "-t", - "agent-token", - "-i", - "60", - "--once", - ]) - .unwrap(); - assert_eq!(parsed.endpoint, "https://monitor.example.com"); - assert_eq!(parsed.token.as_deref(), Some("agent-token")); - assert_eq!(parsed.interval, 60); - assert!(parsed.once); - assert!(CliOptions::try_parse_from(["agent", "-t"]).is_err()); - assert!(CliOptions::try_parse_from(["agent", "-t", "first", "-t", "second"]).is_err()); - assert!( - CliOptions::try_parse_from([ - "agent", - "-e", - "https://monitor.example.com", - "-t", - "agent-token", - "--token-file", - "/run/nodeflare/token", - ]) - .is_err() - ); - assert!(CliOptions::try_parse_from(["agent", "--once", "--collect"]).is_err()); - } - - #[test] - fn filters_and_deduplicates_remote_latency_tasks_without_a_count_limit() { - let valid = LatencyTask { - id: "task-1".to_string(), - name: "Cloudflare".to_string(), - task_type: "tcp".to_string(), - target: "1.1.1.1".to_string(), - port: Some(443), - interval_seconds: 60, - }; - let mut tasks = vec![valid.clone(), valid]; - tasks.push(LatencyTask { - id: "bad".to_string(), - name: "Bad".to_string(), - task_type: "http".to_string(), - target: "https://example.com".to_string(), - port: Some(443), - interval_seconds: 1, - }); - for index in 2..=300 { - tasks.push(LatencyTask { - id: format!("task-{index}"), - name: format!("Task {index}"), - task_type: "icmp".to_string(), - target: "1.1.1.1".to_string(), - port: None, - interval_seconds: 60, - }); - } - let sanitized = sanitize_latency_tasks(&tasks); - assert_eq!(sanitized.len(), 300); - assert_eq!(sanitized[0].id, "task-1"); - assert!(!sanitized.iter().any(|task| task.id == "bad")); - } - - #[test] - fn advances_collection_deadlines_without_drift() { - let start = Instant::now(); - let interval = Duration::from_secs(1); - assert_eq!( - advance_deadline(start, interval, start + Duration::from_millis(2500)), - start + Duration::from_secs(3) - ); - } - - #[test] - fn drops_samples_the_server_would_reject_as_expired() { - let mut samples = vec![ - Report { - timestamp: 1_000, - ..Report::default() - }, - Report { - timestamp: 8_500, - ..Report::default() - }, - ]; - prune_report_samples(&mut samples, 9_000); - assert_eq!(samples.len(), 1); - assert_eq!(samples[0].timestamp, 8_500); - } - - #[test] - fn bounds_pending_latency_results() { - let mut samples = (0..MAX_PENDING_LATENCY_RESULTS + 10) - .map(|index| Report { - timestamp: index as i64 + 1, - latency_results: vec![LatencyResult { - task_id: format!("task-{index}"), - timestamp: index as i64 + 1, - latency_ms: 10.0, - packet_loss: 0.0, - }], - ..Report::default() - }) - .collect::>(); - prune_report_samples(&mut samples, MAX_PENDING_LATENCY_RESULTS as i64 + 10); - assert_eq!( - samples - .iter() - .map(|sample| sample.latency_results.len()) - .sum::(), - MAX_PENDING_LATENCY_RESULTS - ); - assert!(samples.first().unwrap().latency_results.is_empty()); - assert_eq!(samples.last().unwrap().latency_results.len(), 1); - } - - #[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] - #[test] - fn parses_netstat_connection_counts() { - let output = "tcp4 0 0 host.443 peer.1 ESTABLISHED\nudp4 0 0 *.5353 *.*\nTCP host peer ESTABLISHED\n"; - assert_eq!(connection_counts_from_netstat(output), (2, 1)); - } - - #[test] - fn parses_probe_targets() { - assert_eq!( - parse_probe_target("Example.COM", None), - Some(("example.com".to_string(), 443)) - ); - assert_eq!( - parse_probe_target("1.1.1.1", Some(8080)), - Some(("1.1.1.1".to_string(), 8080)) - ); - for target in [ - "", - "https://example.com", - "example.com:0", - "example.com:65536", - "999.1.1.1", - "127.0.0.1:8080", - "169.254.169.254", - "192.168.1.1", - "router.local", - "localhost", - "[::1]:443", - "bad host", - ] { - assert_eq!(parse_probe_target(target, None), None, "target: {target}"); - } - } - - #[test] - fn accepts_only_public_probe_addresses() { - for address in ["1.1.1.1", "8.8.8.8", "2606:4700:4700::1111"] { - assert!(is_public_probe_ip(address.parse().expect("IP address"))); - } - for address in [ - "0.0.0.0", - "10.0.0.1", - "100.64.0.1", - "127.0.0.1", - "169.254.169.254", - "172.16.0.1", - "192.168.1.1", - "198.18.0.1", - "224.0.0.1", - "::1", - "fe80::1", - "fd00:ec2::254", - ] { - assert!(!is_public_probe_ip(address.parse().expect("IP address"))); - } - } - - #[test] - fn parses_ping_latency() { - assert_eq!(ping_latency("64 bytes time=12.34 ms"), Some(12.34)); - assert_eq!(ping_latency("64 bytes time<1 ms"), Some(0.5)); - assert_eq!(ping_latency("unreachable"), None); - assert_eq!( - ping_latencies("reply time=12.34 ms\nrequest timeout\nreply time=8.5 ms"), - vec![12.34, 8.5] - ); - } - - #[test] - fn update_check_jitter_is_stable_and_bounded() { - let first = update_check_jitter("agent-token"); - assert_eq!(first, update_check_jitter("agent-token")); - assert!(first.as_secs() <= UPDATE_CHECK_JITTER_MAX_SECONDS); - } -} diff --git a/agent/src/metrics.rs b/agent/src/metrics.rs new file mode 100755 index 0000000..64bc686 --- /dev/null +++ b/agent/src/metrics.rs @@ -0,0 +1,1398 @@ +use super::*; + +#[derive(Debug, Default, Clone)] +pub(crate) struct BasicMetrics { + pub(crate) cpu_cores: i64, + pub(crate) cpu_model: String, + pub(crate) os: String, + pub(crate) kernel: String, + pub(crate) arch: String, + pub(crate) virtualization: String, + pub(crate) gpu_usage: f64, + pub(crate) gpu_model: String, + pub(crate) gpus: Vec, +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct SlowMetrics { + pub(crate) disks: Vec, + pub(crate) processes: i64, + pub(crate) tcp_connections: i64, + pub(crate) udp_connections: i64, +} + +#[cfg(target_os = "linux")] +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct CpuSample { + pub(crate) total: u64, + pub(crate) idle: u64, +} + +#[cfg(target_os = "linux")] +#[derive(Debug, Default, Clone, Copy)] +pub(crate) struct IoSample { + pub(crate) rx: u64, + pub(crate) tx: u64, + pub(crate) read_ops: u64, + pub(crate) read_sectors: u64, + pub(crate) write_ops: u64, + pub(crate) write_sectors: u64, + pub(crate) read_millis: u64, + pub(crate) write_millis: u64, + pub(crate) io_millis: u64, +} + +#[cfg(target_os = "linux")] +pub(crate) fn text(path: impl AsRef) -> String { + fs::read_to_string(path).unwrap_or_default() +} + +/// Runs a command and captures stdout, killing it after `timeout`. +/// +/// A dedicated thread drains the pipe while we poll, so a command whose output +/// exceeds the pipe buffer is not mistaken for a hang. Without the timeout a +/// wedged child (a stuck GPU driver, an unresponsive netstat) would block the +/// sampler thread forever and silently stop telemetry. +pub(crate) fn output_with_timeout(mut process: Command, timeout: Duration) -> Option { + let mut child = process + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .ok()?; + let reader = child.stdout.take().map(|mut pipe| { + thread::spawn(move || { + let mut buffer = Vec::new(); + let _ = pipe.read_to_end(&mut buffer); + buffer + }) + }); + let started = Instant::now(); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Some(status), + Ok(None) if started.elapsed() >= timeout => { + let _ = child.kill(); + let _ = child.wait(); + break None; + } + Ok(None) => thread::sleep(Duration::from_millis(20)), + Err(_) => { + let _ = child.kill(); + let _ = child.wait(); + break None; + } + } + }; + let stdout = reader + .and_then(|handle| handle.join().ok()) + .unwrap_or_default(); + Some(std::process::Output { + status: status?, + stdout, + stderr: Vec::new(), + }) +} + +pub(crate) fn command(name: &str, args: &[&str]) -> String { + let mut process = Command::new(name); + process.args(args); + output_with_timeout(process, COMMAND_TIMEOUT) + .filter(|output| output.status.success()) + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_string()) + .unwrap_or_default() +} + +#[cfg(target_os = "linux")] +pub(crate) fn cpu_sample() -> CpuSample { + let line = text("/proc/stat").lines().next().unwrap_or("").to_string(); + let values = line + .split_whitespace() + .skip(1) + .filter_map(|value| value.parse::().ok()) + .collect::>(); + CpuSample { + total: values.iter().sum(), + idle: values.get(3).copied().unwrap_or(0) + values.get(4).copied().unwrap_or(0), + } +} + +pub(crate) fn selected_interface(name: &str, filter: &str) -> bool { + let name = name.trim(); + if name.is_empty() { + return false; + } + let mut has_include = false; + let mut included = false; + for pattern in filter + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + { + if let Some(pattern) = pattern + .strip_prefix('!') + .or_else(|| pattern.strip_prefix('-')) + { + if !pattern.is_empty() && wildcard_match(pattern, name) { + return false; + } + } else { + has_include = true; + included |= wildcard_match(pattern, name); + } + } + // Explicit includes can select a bridge; automatic selection avoids double counting. + if has_include { + return included; + } + let lower = name.to_ascii_lowercase(); + !lower.contains("loopback") + && ![ + "lo", "br", "cni", "docker", "podman", "flannel", "veth", "virbr", "vmbr", "tap", + "fwbr", "fwpr", + ] + .iter() + .any(|prefix| lower.starts_with(prefix)) +} + +pub(crate) fn wildcard_match(pattern: &str, value: &str) -> bool { + let pattern = pattern.as_bytes(); + let value = value.as_bytes(); + let (mut p, mut v, mut star, mut star_value) = (0, 0, None, 0); + while v < value.len() { + if p < pattern.len() && (pattern[p] == b'?' || pattern[p] == value[v]) { + p += 1; + v += 1; + } else if p < pattern.len() && pattern[p] == b'*' { + star = Some(p); + star_value = v; + p += 1; + } else if let Some(star_position) = star { + p = star_position + 1; + star_value += 1; + v = star_value; + } else { + return false; + } + } + while p < pattern.len() && pattern[p] == b'*' { + p += 1; + } + p == pattern.len() +} + +#[cfg(target_os = "linux")] +pub(crate) fn disk_device(name: &str) -> bool { + ((name.starts_with("sd") || name.starts_with("vd") || name.starts_with("xvd")) + && name + .chars() + .last() + .is_some_and(|value| value.is_ascii_alphabetic())) + || (name.starts_with("nvme") && name.contains('n') && !name.contains('p')) + || (name.starts_with("mmcblk") && !name.contains('p')) +} + +#[cfg(target_os = "linux")] +pub(crate) fn io_sample(filter: &str) -> IoSample { + let mut sample = IoSample::default(); + for line in text("/proc/net/dev") + .lines() + .filter(|line| line.contains(':')) + { + let Some((name, values)) = line.split_once(':') else { + continue; + }; + if !selected_interface(name.trim(), filter) { + continue; + } + let fields = values.split_whitespace().collect::>(); + sample.rx = sample + .rx + .saturating_add(fields.first().and_then(|v| v.parse().ok()).unwrap_or(0)); + sample.tx = sample + .tx + .saturating_add(fields.get(8).and_then(|v| v.parse().ok()).unwrap_or(0)); + } + for line in text("/proc/diskstats").lines() { + let fields = line.split_whitespace().collect::>(); + if fields.len() < 13 || !disk_device(fields[2]) { + continue; + } + sample.read_ops = sample + .read_ops + .saturating_add(fields[3].parse().unwrap_or(0)); + sample.read_sectors = sample + .read_sectors + .saturating_add(fields[5].parse().unwrap_or(0)); + sample.write_ops = sample + .write_ops + .saturating_add(fields[7].parse().unwrap_or(0)); + sample.write_sectors = sample + .write_sectors + .saturating_add(fields[9].parse().unwrap_or(0)); + sample.read_millis = sample + .read_millis + .saturating_add(fields[6].parse().unwrap_or(0)); + sample.write_millis = sample + .write_millis + .saturating_add(fields[10].parse().unwrap_or(0)); + sample.io_millis = sample + .io_millis + .saturating_add(fields[12].parse().unwrap_or(0)); + } + sample +} + +#[cfg(target_os = "linux")] +pub(crate) fn mem_value(contents: &str, key: &str) -> i64 { + contents + .lines() + .find(|line| line.starts_with(key)) + .and_then(|line| line.split_whitespace().nth(1)) + .and_then(|value| value.parse::().ok()) + .map_or(0, |value| u64_to_i64(value.saturating_mul(1024))) +} + +#[cfg(target_os = "linux")] +pub(crate) struct MemorySample { + pub(crate) total: i64, + pub(crate) used: i64, + pub(crate) swap_total: i64, + pub(crate) swap_used: i64, +} + +#[cfg(target_os = "linux")] +pub(crate) fn memory_sample(contents: &str) -> MemorySample { + let total = mem_value(contents, "MemTotal:"); + let free = mem_value(contents, "MemFree:"); + let reclaimable = free + .saturating_add(mem_value(contents, "Cached:")) + .saturating_add(mem_value(contents, "SReclaimable:")) + .saturating_add(mem_value(contents, "Buffers:")); + // Komari's default Linux/htop calculation excludes reclaimable caches but + // includes shared memory. MemAvailable uses a different kernel estimate. + let used = total + .saturating_sub(if reclaimable <= total { + reclaimable + } else { + free + }) + .saturating_add(mem_value(contents, "Shmem:")) + .clamp(0, total); + let swap_total = mem_value(contents, "SwapTotal:"); + let swap_free = mem_value(contents, "SwapFree:"); + let swap_deductions = swap_free.saturating_add(mem_value(contents, "SwapCached:")); + let swap_used = swap_total + .saturating_sub(if swap_deductions <= swap_total { + swap_deductions + } else { + swap_free + }) + .clamp(0, swap_total); + MemorySample { + total, + used, + swap_total, + swap_used, + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn file_line_count(path: &str) -> i64 { + text(path).lines().skip(1).count() as i64 +} + +#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] +pub(crate) fn connection_counts_from_netstat(output: &str) -> (i64, i64) { + output.lines().fold((0_i64, 0_i64), |(tcp, udp), line| { + let protocol = line + .split_whitespace() + .next() + .unwrap_or_default() + .to_ascii_lowercase(); + if protocol.starts_with("tcp") { + (tcp.saturating_add(1), udp) + } else if protocol.starts_with("udp") { + (tcp, udp.saturating_add(1)) + } else { + (tcp, udp) + } + }) +} + +#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] +pub(crate) fn connection_counts() -> (i64, i64) { + let mut process = Command::new("netstat"); + process.args(["-an"]); + output_with_timeout(process, COMMAND_TIMEOUT) + .filter(|output| output.status.success()) + .map(|output| connection_counts_from_netstat(&String::from_utf8_lossy(&output.stdout))) + .unwrap_or_default() +} + +#[cfg(target_os = "linux")] +pub(crate) fn disk_usage() -> Vec { + disk_mounts(&text("/proc/self/mounts")) + .filter_map(|(name, mount_point)| { + // Ignore file bind mounts such as a container's /etc/hosts. + if !Path::new(&mount_point).is_dir() { + return None; + } + let (total, used) = filesystem_space(&mount_point)?; + Some(DiskMetric { + name, + mount_point, + used, + total, + ..DiskMetric::default() + }) + }) + .collect() +} + +#[cfg(target_os = "linux")] +pub(crate) fn disk_mounts(contents: &str) -> impl Iterator + '_ { + pub(crate) fn unescape(value: &str) -> String { + value + .replace("\\040", " ") + .replace("\\011", "\t") + .replace("\\012", "\n") + .replace("\\134", "\\") + } + contents.lines().filter_map(|line| { + let mut fields = line.split_whitespace(); + let source = unescape(fields.next()?); + let mount_point = unescape(fields.next()?); + let filesystem = fields.next()?; + if (!source.starts_with('/') && source.contains(':')) + || source.starts_with("//") + || excluded_filesystem(filesystem, &mount_point) + { + return None; + } + Some((source, mount_point)) + }) +} + +#[cfg(target_os = "linux")] +pub(crate) fn filesystem_space(mount_point: &str) -> Option<(i64, i64)> { + let path = std::ffi::CString::new(mount_point).ok()?; + let mut stat = std::mem::MaybeUninit::::uninit(); + // SAFETY: path is NUL-terminated and stat points to writable storage of the + // platform's statvfs type, including both glibc and musl layouts. + if unsafe { libc::statvfs(path.as_ptr(), stat.as_mut_ptr()) } != 0 { + return None; + } + // SAFETY: statvfs returned success and initialized stat. + let stat = unsafe { stat.assume_init() }; + let block_size = u128::from(if stat.f_frsize == 0 { + stat.f_bsize + } else { + stat.f_frsize + }); + let total = (u128::from(stat.f_blocks) * block_size).min(i64::MAX as u128) as i64; + let used = (u128::from(stat.f_blocks.saturating_sub(stat.f_bfree)) * block_size) + .min(i64::MAX as u128) as i64; + (total > 0).then_some((total, used)) +} + +#[cfg(target_os = "linux")] +pub(crate) fn excluded_filesystem(filesystem: &str, mount_point: &str) -> bool { + // Remote/autofs mounts can block indefinitely while their server is down. + if matches!( + filesystem, + "nfs" + | "nfs4" + | "cifs" + | "smb3" + | "autofs" + | "9p" + | "ceph" + | "afs" + | "fuse.sshfs" + | "fuse.glusterfs" + | "fuse.rclone" + | "fuse.s3fs" + ) { + return true; + } + let mount_point = mount_point.to_ascii_lowercase(); + // Keep the root filesystem visible even when a container reports it as overlay. + if mount_point == "/" { + return false; + } + let filesystem = filesystem.to_ascii_lowercase(); + [ + "tmpfs", + "devtmpfs", + "devpts", + "proc", + "sysfs", + "cgroup", + "cgroup2", + "overlay", + "squashfs", + "efivarfs", + "pstore", + "mqueue", + "hugetlbfs", + "debugfs", + "fusectl", + ] + .iter() + .any(|value| filesystem == *value || filesystem.starts_with(&format!("{value}."))) + || ["/proc", "/sys", "/run", "/dev"] + .iter() + .any(|prefix| mount_point == *prefix || mount_point.starts_with(&format!("{prefix}/"))) +} + +#[cfg(any(target_os = "linux", test))] +pub(crate) fn disk_identity(name: &str) -> String { + let name = name.trim(); + // ZFS reports datasets as pool/dataset. Keep ordinary absolute paths and + // remote sources separate because their slash is part of the path. + if !name.starts_with('/') && !name.contains(':') && name.contains('/') { + return name.split('/').next().unwrap_or(name).to_string(); + } + name.to_string() +} + +#[cfg(any(target_os = "linux", test))] +pub(crate) fn dedupe_disks(disks: impl IntoIterator) -> Vec { + let mut unique = HashMap::::new(); + for disk in disks { + let key = disk_identity(&disk.name); + let replace = unique.get(&key).is_none_or(|current| { + disk.total > current.total + || (disk.total == current.total + && disk.mount_point.len() < current.mount_point.len()) + }); + if replace { + unique.insert(key, disk); + } + } + let mut disks = unique.into_values().collect::>(); + disks.sort_by(|left, right| left.mount_point.cmp(&right.mount_point)); + disks +} + +#[cfg(target_os = "linux")] +pub(crate) fn os_name() -> String { + text("/etc/os-release") + .lines() + .find_map(|line| line.strip_prefix("PRETTY_NAME=")) + .map_or_else( + || command("uname", &["-s"]), + |value| value.trim_matches('"').to_string(), + ) +} + +#[cfg(target_os = "linux")] +pub(crate) fn cpu_model() -> String { + text("/proc/cpuinfo") + .lines() + .find_map(|line| { + let (key, value) = line.split_once(':')?; + matches!(key.trim(), "model name" | "Hardware").then(|| value.trim().to_string()) + }) + .unwrap_or_default() +} + +pub(crate) fn valid_probe_host(host: &str) -> bool { + if host.is_empty() || host.len() > 50 || host.starts_with('.') || host.ends_with('.') { + return false; + } + let labels = host.split('.').collect::>(); + let ipv4_like = labels.len() == 4 + && labels + .iter() + .all(|label| !label.is_empty() && label.chars().all(|c| c.is_ascii_digit())); + if ipv4_like { + return host + .parse() + .is_ok_and(|address| is_public_probe_ip(IpAddr::V4(address))); + } + let lower = host.to_ascii_lowercase(); + if labels.len() < 2 + || ["local", "localhost", "internal", "lan", "localdomain"] + .iter() + .any(|suffix| lower == *suffix || lower.ends_with(&format!(".{suffix}"))) + || lower == "home.arpa" + || lower.ends_with(".home.arpa") + { + return false; + } + labels.iter().all(|label| { + !label.is_empty() + && label + .chars() + .all(|character| character.is_ascii_alphanumeric() || character == '-') + && label + .chars() + .next() + .is_some_and(|character| character.is_ascii_alphanumeric()) + && label + .chars() + .last() + .is_some_and(|character| character.is_ascii_alphanumeric()) + }) +} + +pub(crate) fn is_public_probe_ip(address: IpAddr) -> bool { + match address { + IpAddr::V4(address) => { + let [a, b, c, _] = address.octets(); + !(a == 0 + || a == 10 + || a == 127 + || (a == 100 && (64..=127).contains(&b)) + || (a == 169 && b == 254) + || (a == 172 && (16..=31).contains(&b)) + || (a == 192 && b == 0 && c == 0) + || (a == 192 && b == 0 && c == 2) + || (a == 192 && b == 168) + || (a == 198 && (b == 18 || b == 19)) + || (a == 198 && b == 51 && c == 100) + || (a == 203 && b == 0 && c == 113) + || a >= 224) + } + IpAddr::V6(address) => { + if let Some(mapped) = address.to_ipv4_mapped() { + return is_public_probe_ip(IpAddr::V4(mapped)); + } + address.segments()[0] & 0xe000 == 0x2000 + } + } +} + +pub(crate) fn resolve_public_probe_address(host: &str, port: u16) -> Option { + let addresses = (host, port) + .to_socket_addrs() + .ok()? + .filter(|address| is_public_probe_ip(address.ip())) + .collect::>(); + addresses + .iter() + .find(|address| address.is_ipv4()) + .or_else(|| addresses.first()) + .copied() +} + +pub(crate) fn parse_probe_target(value: &str, port: Option) -> Option<(String, u16)> { + let raw = value.trim(); + if raw.is_empty() + || raw.len() > 60 + || raw.contains("://") + || raw + .chars() + .any(|character| character.is_whitespace() || "/@?#\\[]".contains(character)) + || raw.contains(':') + { + return None; + } + let port = port.unwrap_or(443); + (port > 0 && valid_probe_host(raw)).then(|| (raw.to_ascii_lowercase(), port)) +} + +pub(crate) fn median(values: &mut [f64]) -> f64 { + values.sort_by(f64::total_cmp); + let middle = values.len() / 2; + if values.len().is_multiple_of(2) { + (values[middle - 1] + values[middle]) / 2.0 + } else { + values[middle] + } +} + +pub(crate) fn tcp_latency_probe(target: &str, port: Option) -> (f64, f64) { + if target.trim().is_empty() { + return (-1.0, -1.0); + } + let Some(port) = port.and_then(|value| u16::try_from(value).ok()) else { + return (-1.0, 100.0); + }; + let Some((host, port)) = parse_probe_target(target, Some(port)) else { + return (-1.0, 100.0); + }; + let Some(address) = resolve_public_probe_address(&host, port) else { + return (-1.0, 100.0); + }; + tcp_latency_probe_address(&address) +} + +pub(crate) fn tcp_latency_probe_address(address: &SocketAddr) -> (f64, f64) { + let mut latencies = Vec::with_capacity(PROBE_ATTEMPTS); + for _ in 0..PROBE_ATTEMPTS { + let started = Instant::now(); + if TcpStream::connect_timeout(address, PROBE_TIMEOUT).is_ok() { + latencies.push(started.elapsed().as_secs_f64() * 1000.0); + } + } + let loss = (PROBE_ATTEMPTS - latencies.len()) as f64 * 100.0 / PROBE_ATTEMPTS as f64; + if latencies.is_empty() { + (-1.0, loss) + } else { + (median(&mut latencies), loss) + } +} + +pub(crate) fn ping_latency(output: &str) -> Option { + let marker = output + .find("time=") + .map(|index| (index + 5, false)) + .or_else(|| output.find("time<").map(|index| (index + 5, true)))?; + if marker.1 { + return Some(0.5); + } + let value = output[marker.0..] + .chars() + .take_while(|character| character.is_ascii_digit() || *character == '.') + .collect::(); + value.parse().ok() +} + +pub(crate) fn ping_latencies(output: &str) -> Vec { + output.lines().filter_map(ping_latency).collect() +} + +pub(crate) fn icmp_latency_probe(target: &str) -> (f64, f64) { + let host = target.trim(); + let Some((host, _)) = parse_probe_target(host, None) else { + return (-1.0, 100.0); + }; + let Some(address) = resolve_public_probe_address(&host, 443) else { + return (-1.0, 100.0); + }; + let destination = address.ip().to_string(); + let mut ping = Command::new("ping"); + #[cfg(target_os = "linux")] + ping.args(["-n", "-c", "4", "-W", "1", destination.as_str()]); + #[cfg(any(target_os = "macos", target_os = "freebsd"))] + ping.args(["-n", "-c", "4", "-W", "1000", destination.as_str()]); + #[cfg(target_os = "windows")] + ping.args(["-n", "4", "-w", "1000", destination.as_str()]); + let mut latencies = ping + .output() + .map(|output| ping_latencies(&String::from_utf8_lossy(&output.stdout))) + .unwrap_or_default(); + latencies.truncate(PROBE_ATTEMPTS); + let loss = (PROBE_ATTEMPTS - latencies.len()) as f64 * 100.0 / PROBE_ATTEMPTS as f64; + if latencies.is_empty() { + (-1.0, loss) + } else { + (median(&mut latencies), loss) + } +} + +pub(crate) fn execute_latency_task(task: LatencyTask) -> LatencyResult { + let (latency_ms, packet_loss) = match task.task_type.as_str() { + "tcp" => tcp_latency_probe(&task.target, task.port), + "icmp" => icmp_latency_probe(&task.target), + _ => (-1.0, 100.0), + }; + LatencyResult { + task_id: task.id, + timestamp: unix_timestamp(), + latency_ms, + packet_loss, + } +} + +pub(crate) fn nvidia_gpu_info() -> Vec { + let gpu = command( + "nvidia-smi", + &[ + "--query-gpu=utilization.gpu,name,memory.used,memory.total", + "--format=csv,noheader,nounits", + ], + ); + gpu.lines() + .filter_map(|line| { + let fields = line.split(',').map(str::trim).collect::>(); + if fields.len() < 4 { + return None; + } + if fields[1].is_empty() { + return None; + } + Some(GpuMetric { + usage: fields[0].parse().ok(), + model: fields[1].to_string(), + memory_used: fields[2] + .parse::() + .unwrap_or(0) + .saturating_mul(1024 * 1024), + memory_total: fields[3] + .parse::() + .unwrap_or(0) + .saturating_mul(1024 * 1024), + }) + }) + .collect() +} + +pub(crate) fn basic_gpu_metrics(names: impl IntoIterator) -> Vec { + let mut seen = HashSet::new(); + names + .into_iter() + .filter_map(|name| { + let name = name.split(" (rev ").next().unwrap_or(&name).trim(); + let lower = name.to_ascii_lowercase(); + if name.is_empty() + || [ + "sensor hub", + "management engine", + "ethernet", + "wireless", + "audio controller", + "usb controller", + "sata controller", + "virtio", + "vmware", + "qxl", + "hyper-v", + "cirrus", + "microsoft basic display", + ] + .iter() + .any(|pattern| lower.contains(pattern)) + || !seen.insert(name.to_string()) + { + return None; + } + Some(GpuMetric { + model: name.to_string(), + usage: None, + memory_used: 0, + memory_total: 0, + }) + }) + .collect() +} + +#[cfg(any(target_os = "linux", test))] +pub(crate) fn parse_lspci_gpu_names(output: &str) -> Vec { + output + .lines() + .filter(|line| { + let lower = line.to_ascii_lowercase(); + lower.contains("vga compatible controller") + || lower.contains("3d controller") + || lower.contains("display controller") + }) + .filter_map(|line| line.split_once(": ").map(|(_, name)| name.to_string())) + .collect() +} + +#[cfg(any(target_os = "linux", test))] +pub(crate) fn gpu_name_from_uevent(output: &str) -> Option { + let driver = output + .lines() + .find_map(|line| line.strip_prefix("DRIVER="))?; + let pci_class = output + .lines() + .find_map(|line| line.strip_prefix("PCI_CLASS=")) + .unwrap_or_default(); + if u32::from_str_radix(pci_class, 16) + .ok() + .map(|class| class >> 16) + != Some(3) + { + return None; + } + match driver { + "i915" | "xe" => Some("Intel Integrated Graphics".to_string()), + "amdgpu" | "radeon" => Some("AMD Radeon Graphics".to_string()), + "nvidia" | "nouveau" => Some("NVIDIA GPU".to_string()), + "msm" | "msm_drm" => Some("Qualcomm Adreno GPU".to_string()), + "panfrost" | "lima" => Some("ARM Mali GPU".to_string()), + "vc4" | "v3d" => Some("Broadcom VideoCore Graphics".to_string()), + "virtio-pci" | "virtio_gpu" | "bochs-drm" | "qxl" | "vmwgfx" | "cirrus" | "vboxvideo" + | "hyperv_fb" | "simpledrm" | "simplefb" => None, + other if !other.is_empty() => Some(format!("GPU ({other})")), + _ => None, + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn sysfs_gpu_names() -> Vec { + let Ok(entries) = fs::read_dir("/sys/class/drm") else { + return Vec::new(); + }; + entries + .filter_map(std::result::Result::ok) + .filter(|entry| { + let name = entry.file_name(); + let name = name.to_string_lossy(); + name.strip_prefix("card").is_some_and(|suffix| { + !suffix.is_empty() && suffix.chars().all(|c| c.is_ascii_digit()) + }) + }) + .filter_map(|entry| gpu_name_from_uevent(&text(entry.path().join("device/uevent")))) + .collect() +} + +#[cfg(any(target_os = "macos", test))] +pub(crate) fn parse_system_profiler_gpu_names(output: &str) -> Vec { + output + .lines() + .filter_map(|line| { + line.trim() + .strip_prefix("Chipset Model:") + .map(|name| name.trim().to_string()) + }) + .collect() +} + +#[cfg(any(target_os = "freebsd", test))] +pub(crate) fn parse_pciconf_gpu_names(output: &str) -> Vec { + pub(crate) fn block_name(lines: &[&str]) -> Option { + let header = lines.first()?.to_ascii_lowercase(); + let mut display = header.contains("class=0x03"); + let mut vendor = None; + let mut device = None; + for line in lines.iter().skip(1) { + let Some((key, value)) = line.trim().split_once('=') else { + continue; + }; + let key = key.trim(); + let value = value.trim().trim_matches(['\'', '"']); + match key { + "class" if value.eq_ignore_ascii_case("display") => display = true, + "vendor" => vendor = Some(value), + "device" => device = Some(value), + _ => {} + } + } + if !display { + return None; + } + match (vendor, device) { + (Some(vendor), Some(device)) + if !device + .to_ascii_lowercase() + .contains(&vendor.to_ascii_lowercase()) => + { + Some(format!("{vendor} {device}")) + } + (_, Some(device)) => Some(device.to_string()), + (Some(vendor), None) => Some(vendor.to_string()), + _ => None, + } + } + + let mut names = Vec::new(); + let mut block = Vec::new(); + for line in output.lines() { + if !line.is_empty() + && !line.chars().next().is_some_and(char::is_whitespace) + && !block.is_empty() + { + if let Some(name) = block_name(&block) { + names.push(name); + } + block.clear(); + } + if !line.trim().is_empty() { + block.push(line); + } + } + if let Some(name) = block_name(&block) { + names.push(name); + } + names +} + +#[cfg(target_os = "linux")] +pub(crate) fn basic_gpu_info() -> Vec { + let names = parse_lspci_gpu_names(&command("lspci", &[])); + basic_gpu_metrics(if names.is_empty() { + sysfs_gpu_names() + } else { + names + }) +} + +#[cfg(target_os = "windows")] +pub(crate) fn basic_gpu_info() -> Vec { + basic_gpu_metrics( + command( + "powershell.exe", + &[ + "-NoProfile", + "-NonInteractive", + "-Command", + "Get-CimInstance Win32_VideoController | ForEach-Object { $_.Name }", + ], + ) + .lines() + .map(str::to_string), + ) +} + +#[cfg(target_os = "macos")] +pub(crate) fn basic_gpu_info() -> Vec { + basic_gpu_metrics(parse_system_profiler_gpu_names(&command( + "system_profiler", + &["SPDisplaysDataType"], + ))) +} + +#[cfg(target_os = "freebsd")] +pub(crate) fn basic_gpu_info() -> Vec { + basic_gpu_metrics(parse_pciconf_gpu_names(&command("pciconf", &["-lv"]))) +} + +pub(crate) fn gpu_info() -> Vec { + let detailed = nvidia_gpu_info(); + if detailed.is_empty() { + basic_gpu_info() + } else { + detailed + } +} + +pub(crate) fn average_gpu_usage(gpus: &[GpuMetric]) -> f64 { + let usages = gpus.iter().filter_map(|gpu| gpu.usage).collect::>(); + if usages.is_empty() { + 0.0 + } else { + usages.iter().sum::() / usages.len() as f64 + } +} + +pub(crate) fn u64_to_i64(value: u64) -> i64 { + value.min(i64::MAX as u64) as i64 +} + +pub(crate) fn per_second(value: u64, elapsed_seconds: f64) -> f64 { + value as f64 / elapsed_seconds.max(0.001) +} + +#[cfg(target_os = "linux")] +pub(crate) struct Collector { + pub(crate) previous_cpu: CpuSample, + pub(crate) previous_io: IoSample, + pub(crate) previous_at: Instant, + pub(crate) network_interface: String, + pub(crate) basic: BasicMetrics, + pub(crate) basic_at: Instant, + pub(crate) gpu_at: Instant, + pub(crate) slow: SlowMetrics, + pub(crate) slow_at: Instant, + pub(crate) public_ip: PublicIpProbe, +} + +#[cfg(target_os = "linux")] +impl Collector { + pub(crate) fn new(config: &RuntimeConfig) -> Self { + let mut collector = Self { + previous_cpu: cpu_sample(), + previous_io: io_sample(&config.network_interface), + previous_at: Instant::now(), + network_interface: config.network_interface.clone(), + basic: BasicMetrics::default(), + basic_at: Instant::now(), + gpu_at: Instant::now(), + slow: SlowMetrics::default(), + slow_at: Instant::now(), + public_ip: PublicIpProbe::default(), + }; + collector.refresh_basic(); + collector.refresh_slow(); + collector + } + + pub(crate) fn refresh_basic(&mut self) { + let gpus = gpu_info(); + self.basic = BasicMetrics { + cpu_cores: thread::available_parallelism().map_or(1, |value| value.get() as i64), + cpu_model: cpu_model(), + os: os_name(), + kernel: command("uname", &["-r"]), + arch: env::consts::ARCH.to_string(), + virtualization: command("systemd-detect-virt", &[]), + gpu_usage: average_gpu_usage(&gpus), + gpu_model: gpus + .iter() + .map(|gpu| gpu.model.as_str()) + .collect::>() + .join(" · "), + gpus, + }; + self.basic_at = Instant::now(); + self.gpu_at = Instant::now(); + } + + pub(crate) fn refresh_slow(&mut self) { + let processes = fs::read_dir("/proc").map_or(0, |items| { + items + .filter_map(std::result::Result::ok) + .filter(|item| { + item.file_name() + .to_string_lossy() + .chars() + .all(|ch| ch.is_ascii_digit()) + }) + .count() as i64 + }); + self.slow = SlowMetrics { + disks: dedupe_disks(disk_usage()), + processes, + tcp_connections: file_line_count("/proc/net/tcp") + file_line_count("/proc/net/tcp6"), + udp_connections: file_line_count("/proc/net/udp") + file_line_count("/proc/net/udp6"), + }; + self.slow_at = Instant::now(); + } + + pub(crate) fn collect( + &mut self, + config: &RuntimeConfig, + latency_results: Vec, + timestamp: i64, + ) -> Report { + let sampled_at = Instant::now(); + let elapsed = sampled_at + .saturating_duration_since(self.previous_at) + .as_secs_f64() + .max(0.001); + let cpu_now = cpu_sample(); + let io_now = io_sample(&config.network_interface); + let total = cpu_now.total.saturating_sub(self.previous_cpu.total); + let idle = cpu_now.idle.saturating_sub(self.previous_cpu.idle); + let cpu = if total == 0 { + 0.0 + } else { + (total.saturating_sub(idle)) as f64 * 100.0 / total as f64 + }; + let interface_changed = self.network_interface != config.network_interface; + let io_before = if interface_changed { + io_now + } else { + self.previous_io + }; + self.previous_cpu = cpu_now; + self.previous_io = io_now; + self.previous_at = sampled_at; + self.network_interface.clone_from(&config.network_interface); + + if self.basic_at.elapsed() >= BASIC_INFO_REFRESH_INTERVAL { + self.refresh_basic(); + } + if self.gpu_at.elapsed() >= GPU_METRICS_REFRESH_INTERVAL { + self.refresh_gpu_metrics(); + } + if self.slow_at.elapsed() >= SLOW_METRICS_REFRESH_INTERVAL { + self.refresh_slow(); + } + self.public_ip.refresh(); + + let memory = memory_sample(&text("/proc/meminfo")); + let loads = text("/proc/loadavg") + .split_whitespace() + .take(3) + .filter_map(|value| value.parse::().ok()) + .collect::>(); + let uptime = text("/proc/uptime") + .split_whitespace() + .next() + .and_then(|value| value.parse::().ok()) + .unwrap_or(0.0) as i64; + let processes = self.slow.processes; + let disks = self.slow.disks.clone(); + let disk_used = disks.iter().map(|disk| disk.used).sum(); + let disk_total = disks.iter().map(|disk| disk.total).sum(); + let tcp_connections = self.slow.tcp_connections; + let udp_connections = self.slow.udp_connections; + let read_ops = io_now.read_ops.saturating_sub(io_before.read_ops); + let write_ops = io_now.write_ops.saturating_sub(io_before.write_ops); + let total_ops = read_ops.saturating_add(write_ops); + let io_wait = io_now + .read_millis + .saturating_sub(io_before.read_millis) + .saturating_add(io_now.write_millis.saturating_sub(io_before.write_millis)); + let elapsed_ms = elapsed * 1000.0; + + Report { + timestamp, + cpu, + load1: loads.first().copied().unwrap_or(0.0), + load5: loads.get(1).copied().unwrap_or(0.0), + load15: loads.get(2).copied().unwrap_or(0.0), + mem_used: memory.used, + mem_total: memory.total, + swap_used: memory.swap_used, + swap_total: memory.swap_total, + disk_used, + disk_total, + net_in: per_second(io_now.rx.saturating_sub(io_before.rx), elapsed), + net_out: per_second(io_now.tx.saturating_sub(io_before.tx), elapsed), + net_rx_total: io_now.rx.min(i64::MAX as u64) as i64, + net_tx_total: io_now.tx.min(i64::MAX as u64) as i64, + uptime, + processes, + tcp_connections, + udp_connections, + cpu_cores: self.basic.cpu_cores, + cpu_model: self.basic.cpu_model.clone(), + os: self.basic.os.clone(), + kernel: self.basic.kernel.clone(), + arch: self.basic.arch.clone(), + virtualization: self.basic.virtualization.clone(), + gpu_usage: self.basic.gpu_usage, + gpu_model: self.basic.gpu_model.clone(), + agent_version: VERSION.to_string(), + ip_v4: self.public_ip.v4().unwrap_or_default(), + ip_v6: self.public_ip.v6().unwrap_or_default(), + disk_read_bps: per_second( + io_now + .read_sectors + .saturating_sub(io_before.read_sectors) + .saturating_mul(512), + elapsed, + ), + disk_write_bps: per_second( + io_now + .write_sectors + .saturating_sub(io_before.write_sectors) + .saturating_mul(512), + elapsed, + ), + disk_read_iops: per_second(read_ops, elapsed), + disk_write_iops: per_second(write_ops, elapsed), + disk_await_ms: if total_ops == 0 { + 0.0 + } else { + io_wait as f64 / total_ops as f64 + }, + disk_utilization: if elapsed_ms <= 0.0 { + 0.0 + } else { + (io_now.io_millis.saturating_sub(io_before.io_millis) as f64 * 100.0 / elapsed_ms) + .clamp(0.0, 100.0) + }, + disks, + gpus: self.basic.gpus.clone(), + latency_results, + } + } +} + +#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] +pub(crate) struct Collector { + pub(crate) system: System, + pub(crate) networks: Networks, + pub(crate) disks: Disks, + pub(crate) previous_at: Instant, + pub(crate) basic: BasicMetrics, + pub(crate) basic_at: Instant, + pub(crate) gpu_at: Instant, + pub(crate) slow: SlowMetrics, + pub(crate) slow_at: Instant, + pub(crate) public_ip: PublicIpProbe, +} + +#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] +impl Collector { + pub(crate) fn new(_config: &RuntimeConfig) -> Self { + let mut collector = Self { + system: System::new_all(), + networks: Networks::new_with_refreshed_list(), + disks: Disks::new_with_refreshed_list(), + previous_at: Instant::now(), + basic: BasicMetrics::default(), + basic_at: Instant::now(), + gpu_at: Instant::now(), + slow: SlowMetrics::default(), + slow_at: Instant::now(), + public_ip: PublicIpProbe::default(), + }; + collector.refresh_basic(); + collector.refresh_slow(); + collector + } + + pub(crate) fn refresh_basic(&mut self) { + let gpus = gpu_info(); + self.basic = BasicMetrics { + cpu_cores: self.system.cpus().len().max(1) as i64, + cpu_model: self + .system + .cpus() + .first() + .map(|cpu| cpu.brand().to_string()) + .unwrap_or_default(), + os: System::long_os_version().unwrap_or_else(|| env::consts::OS.to_string()), + kernel: System::kernel_version().unwrap_or_default(), + arch: env::consts::ARCH.to_string(), + virtualization: String::new(), + gpu_usage: average_gpu_usage(&gpus), + gpu_model: gpus + .iter() + .map(|gpu| gpu.model.as_str()) + .collect::>() + .join(" · "), + gpus, + }; + self.basic_at = Instant::now(); + self.gpu_at = Instant::now(); + } + + pub(crate) fn refresh_slow(&mut self) { + self.system.refresh_processes_specifics( + ProcessesToUpdate::All, + true, + ProcessRefreshKind::nothing().without_tasks(), + ); + self.disks.refresh(true); + #[cfg(target_os = "windows")] + let root = format!( + "{}\\", + env::var("SystemDrive").unwrap_or_else(|_| "C:".to_string()) + ); + #[cfg(any(target_os = "macos", target_os = "freebsd"))] + let root = "/".to_string(); + let mut disks = self + .disks + .iter() + .filter(|disk| !disk.is_removable()) + .map(|disk| { + let total = disk.total_space(); + DiskMetric { + name: disk.name().to_string_lossy().to_string(), + mount_point: disk.mount_point().to_string_lossy().to_string(), + used: u64_to_i64(total.saturating_sub(disk.available_space())), + total: u64_to_i64(total), + ..DiskMetric::default() + } + }) + .collect::>(); + disks.sort_by_key(|disk| (disk.mount_point != root, disk.mount_point.clone())); + let (tcp_connections, udp_connections) = connection_counts(); + self.slow = SlowMetrics { + disks, + processes: self.system.processes().len() as i64, + tcp_connections, + udp_connections, + }; + self.slow_at = Instant::now(); + } + + pub(crate) fn collect( + &mut self, + config: &RuntimeConfig, + latency_results: Vec, + timestamp: i64, + ) -> Report { + let sampled_at = Instant::now(); + let elapsed = sampled_at + .saturating_duration_since(self.previous_at) + .as_secs_f64() + .max(0.001); + self.system.refresh_cpu_usage(); + self.system.refresh_memory(); + self.networks.refresh(true); + self.previous_at = sampled_at; + if self.basic_at.elapsed() >= BASIC_INFO_REFRESH_INTERVAL { + self.refresh_basic(); + } + if self.gpu_at.elapsed() >= GPU_METRICS_REFRESH_INTERVAL { + self.refresh_gpu_metrics(); + } + if self.slow_at.elapsed() >= SLOW_METRICS_REFRESH_INTERVAL { + self.refresh_slow(); + } + self.public_ip.refresh(); + + let disk_metrics = self.slow.disks.clone(); + let tcp_connections = self.slow.tcp_connections; + let udp_connections = self.slow.udp_connections; + + let mut net_in = 0_u64; + let mut net_out = 0_u64; + let mut net_rx_total = 0_u64; + let mut net_tx_total = 0_u64; + for (name, network) in &self.networks { + if !selected_interface(name, &config.network_interface) { + continue; + } + net_in = net_in.saturating_add(network.received()); + net_out = net_out.saturating_add(network.transmitted()); + net_rx_total = net_rx_total.saturating_add(network.total_received()); + net_tx_total = net_tx_total.saturating_add(network.total_transmitted()); + } + let load = System::load_average(); + Report { + timestamp, + cpu: self.system.global_cpu_usage() as f64, + load1: load.one, + load5: load.five, + load15: load.fifteen, + mem_used: u64_to_i64( + self.system + .total_memory() + .saturating_sub(self.system.available_memory()), + ), + mem_total: u64_to_i64(self.system.total_memory()), + swap_used: u64_to_i64(self.system.used_swap()), + swap_total: u64_to_i64(self.system.total_swap()), + disk_used: disk_metrics.iter().map(|disk| disk.used).sum(), + disk_total: disk_metrics.iter().map(|disk| disk.total).sum(), + net_in: per_second(net_in, elapsed), + net_out: per_second(net_out, elapsed), + net_rx_total: u64_to_i64(net_rx_total), + net_tx_total: u64_to_i64(net_tx_total), + uptime: u64_to_i64(System::uptime()), + processes: self.slow.processes, + tcp_connections, + udp_connections, + cpu_cores: self.basic.cpu_cores, + cpu_model: self.basic.cpu_model.clone(), + os: self.basic.os.clone(), + kernel: self.basic.kernel.clone(), + arch: self.basic.arch.clone(), + virtualization: self.basic.virtualization.clone(), + gpu_usage: self.basic.gpu_usage, + gpu_model: self.basic.gpu_model.clone(), + agent_version: VERSION.to_string(), + ip_v4: self.public_ip.v4().unwrap_or_default(), + ip_v6: self.public_ip.v6().unwrap_or_default(), + disk_read_bps: 0.0, + disk_write_bps: 0.0, + disk_read_iops: 0.0, + disk_write_iops: 0.0, + disk_await_ms: 0.0, + disk_utilization: 0.0, + disks: disk_metrics, + gpus: self.basic.gpus.clone(), + latency_results, + } + } +} + +impl Collector { + pub(crate) fn refresh_gpu_metrics(&mut self) { + self.gpu_at = Instant::now(); + // Unsupported devices are rediscovered with basic info, not every sample. + if !self + .basic + .gpus + .iter() + .any(|gpu| gpu.usage.is_some() || gpu.memory_total > 0) + { + return; + } + let detailed = nvidia_gpu_info(); + if detailed.is_empty() { + return; + } + self.basic.gpu_usage = average_gpu_usage(&detailed); + self.basic.gpu_model = detailed + .iter() + .map(|gpu| gpu.model.as_str()) + .collect::>() + .join(" · "); + self.basic.gpus = detailed; + } +} + diff --git a/agent/src/remote.rs b/agent/src/remote.rs new file mode 100755 index 0000000..7b3038b --- /dev/null +++ b/agent/src/remote.rs @@ -0,0 +1,587 @@ +use super::*; + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RemoteTaskMessage { + #[serde(rename = "type")] + pub(crate) message_type: String, + pub(crate) task_id: String, + pub(crate) command: String, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +pub(crate) struct TaskResultMessage { + #[serde(rename = "type")] + pub(crate) message_type: String, + pub(crate) task_id: String, + pub(crate) status: String, + pub(crate) result: String, + pub(crate) exit_code: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TaskResultAckMessage { + #[serde(rename = "type")] + pub(crate) message_type: String, + pub(crate) task_id: String, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct RemoteTaskJournal { + pub(crate) version: u32, + pub(crate) entries: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(tag = "state", rename_all = "snake_case")] +pub(crate) enum RemoteTaskJournalEntry { + Active { task_id: String }, + Completed { result: TaskResultMessage }, +} + +#[derive(Default)] +pub(crate) struct CapturedOutput { + pub(crate) bytes: Vec, + pub(crate) truncated: bool, +} + +pub(crate) fn capture_remote_output(mut reader: R) -> mpsc::Receiver +where + R: Read + Send + 'static, +{ + let (sender, receiver) = mpsc::sync_channel(1); + thread::spawn(move || { + let mut bytes = Vec::new(); + let mut truncated = false; + let mut buffer = [0_u8; 8192]; + // Keep draining after the limit so verbose commands do not receive SIGPIPE. + loop { + match reader.read(&mut buffer) { + Ok(0) => break, + Ok(read) => { + let retained = read.min(REMOTE_STREAM_OUTPUT_BYTES as usize - bytes.len()); + bytes.extend_from_slice(&buffer[..retained]); + truncated |= retained < read; + } + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(_) => break, + } + } + let _ = sender.send(CapturedOutput { bytes, truncated }); + }); + receiver +} + +pub(crate) fn terminate_remote_process_tree(child: &mut Child) { + #[cfg(unix)] + { + let process_group = format!("-{}", child.id()); + let _ = Command::new("kill") + .args(["-KILL", "--", process_group.as_str()]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + #[cfg(target_os = "windows")] + { + let _ = Command::new("taskkill") + .args(["/PID", &child.id().to_string(), "/T", "/F"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + let _ = child.kill(); + let _ = child.wait(); +} + +pub(crate) fn receive_remote_output( + pub(crate) receiver: &mpsc::Receiver, + pub(crate) child: &mut Child, +) -> CapturedOutput { + match receiver.recv_timeout(REMOTE_OUTPUT_DRAIN_TIMEOUT) { + Ok(output) => output, + Err(_) => { + // Descendants may keep the pipe open after the shell exits. + terminate_remote_process_tree(child); + receiver + .recv_timeout(REMOTE_OUTPUT_DRAIN_TIMEOUT) + .unwrap_or_default() + } + } +} + +pub(crate) fn remote_result_text(stdout: &CapturedOutput, stderr: &CapturedOutput) -> String { + let truncated = stdout.truncated || stderr.truncated; + let stdout = String::from_utf8_lossy(&stdout.bytes); + let stderr = String::from_utf8_lossy(&stderr.bytes); + let mut combined = if stderr.is_empty() { + stdout.trim().to_string() + } else if stdout.is_empty() { + stderr.trim().to_string() + } else { + format!("{}\n{}", stdout.trim(), stderr.trim()) + }; + let result_truncated = combined.len() > REMOTE_RESULT_OUTPUT_BYTES; + if result_truncated { + let mut end = REMOTE_RESULT_OUTPUT_BYTES; + while !combined.is_char_boundary(end) { + end -= 1; + } + combined.truncate(end); + } + if truncated || result_truncated { + combined.push_str("\n[输出已截断]"); + } + combined +} + +pub(crate) fn wait_for_remote_child( + pub(crate) child: &mut Child, + pub(crate) timeout: Duration, +) -> io::Result> { + let deadline = Instant::now() + timeout; + loop { + if let Some(status) = child.try_wait()? { + return Ok(Some(status)); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Ok(None); + } + thread::sleep(remaining.min(Duration::from_millis(50))); + } +} + +pub(crate) fn execute_remote_task_with_timeout( + pub(crate) task: &RemoteTaskMessage, + pub(crate) timeout: Duration, +) -> TaskResultMessage { + let task_id = task.task_id.clone(); + + if task.command.trim().is_empty() || task.command.len() > 16_384 { + return TaskResultMessage { + message_type: "task_result".to_string(), + task_id, + status: "failed".to_string(), + result: "命令为空或长度超出限制".to_string(), + exit_code: Some(-1), + }; + } + + #[cfg(unix)] + let mut command = { + let mut command = Command::new("sh"); + command.args(["-c", task.command.as_str()]); + command.process_group(0); + command + }; + #[cfg(target_os = "windows")] + let mut command = { + let mut command = Command::new("cmd.exe"); + command.args(["/C", task.command.as_str()]); + command + }; + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + + let mut child = match command.spawn() { + Ok(child) => child, + Err(error) => { + return TaskResultMessage { + message_type: "task_result".to_string(), + task_id, + status: "failed".to_string(), + result: format!("无法启动命令：{error}"), + exit_code: Some(-1), + }; + } + }; + let stdout = capture_remote_output(child.stdout.take().expect("stdout is piped")); + let stderr = capture_remote_output(child.stderr.take().expect("stderr is piped")); + + let status = match wait_for_remote_child(&mut child, timeout) { + Ok(Some(status)) => Some(status), + Ok(None) => { + terminate_remote_process_tree(&mut child); + None + } + Err(error) => { + terminate_remote_process_tree(&mut child); + let stdout = receive_remote_output(&stdout, &mut child); + let stderr = receive_remote_output(&stderr, &mut child); + let output = remote_result_text(&stdout, &stderr); + return TaskResultMessage { + message_type: "task_result".to_string(), + task_id, + status: "failed".to_string(), + result: if output.is_empty() { + format!("等待命令结束失败：{error}") + } else { + format!("{output}\n等待命令结束失败：{error}") + }, + exit_code: Some(-1), + }; + } + }; + let stdout = receive_remote_output(&stdout, &mut child); + let stderr = receive_remote_output(&stderr, &mut child); + let output = remote_result_text(&stdout, &stderr); + + let Some(status) = status else { + let timeout_seconds = timeout.as_secs().max(1); + return TaskResultMessage { + message_type: "task_result".to_string(), + task_id, + status: "failed".to_string(), + result: if output.is_empty() { + format!("命令执行超过 {timeout_seconds} 秒，已终止") + } else { + format!("{output}\n命令执行超过 {timeout_seconds} 秒，已终止") + }, + exit_code: Some(-1), + }; + }; + + TaskResultMessage { + message_type: "task_result".to_string(), + task_id, + status: if status.success() { + "success" + } else { + "failed" + } + .to_string(), + result: output, + exit_code: status.code().map(i64::from), + } +} + +pub(crate) fn execute_remote_task(task: &RemoteTaskMessage) -> TaskResultMessage { + execute_remote_task_with_timeout(task, REMOTE_TASK_TIMEOUT) +} + +pub(crate) struct RemoteExecutor { + pub(crate) task_tx: mpsc::SyncSender, + pub(crate) result_rx: mpsc::Receiver, + pub(crate) active: HashSet, + pub(crate) completed: HashMap, + pub(crate) task_order: VecDeque, + pub(crate) last_sent: HashMap, + pub(crate) journal_path: PathBuf, + pub(crate) disabled: bool, +} + +impl RemoteExecutor { + pub(crate) fn new() -> Result { + Self::new_at(remote_task_journal_path()?) + } + + pub(crate) fn new_at(journal_path: PathBuf) -> Result { + let journal = load_remote_task_journal(&journal_path)?; + let mut active = HashSet::new(); + let mut completed = HashMap::new(); + let mut task_order = VecDeque::new(); + for entry in journal.entries { + match entry { + RemoteTaskJournalEntry::Active { task_id } => { + active.insert(task_id.clone()); + task_order.push_back(task_id); + } + RemoteTaskJournalEntry::Completed { result } => { + task_order.push_back(result.task_id.clone()); + completed.insert(result.task_id.clone(), result); + } + } + } + + let (task_tx, task_rx) = mpsc::sync_channel::(16); + let (result_tx, result_rx) = mpsc::channel::(); + thread::Builder::new() + .name("nodeflare-remote".to_string()) + .spawn(move || { + while let Ok(task) = task_rx.recv() { + if result_tx.send(execute_remote_task(&task)).is_err() { + return; + } + } + })?; + let interrupted = active.drain().collect::>(); + for task_id in interrupted { + completed.insert( + task_id.clone(), + TaskResultMessage { + message_type: "task_result".to_string(), + task_id, + status: "failed".to_string(), + result: "Agent 在命令执行期间重启，无法确认原命令状态；为避免重复操作，任务不会再次执行" + .to_string(), + exit_code: Some(-1), + }, + ); + } + let executor = Self { + task_tx, + result_rx, + active, + completed, + task_order, + last_sent: HashMap::new(), + journal_path, + disabled: false, + }; + if !executor.completed.is_empty() { + executor.persist_snapshot(None)?; + } + Ok(executor) + } + + pub(crate) fn disabled() -> Self { + let (task_tx, task_rx) = mpsc::sync_channel(1); + let (result_tx, result_rx) = mpsc::channel(); + drop(task_rx); + drop(result_tx); + Self { + task_tx, + result_rx, + active: HashSet::new(), + completed: HashMap::new(), + task_order: VecDeque::new(), + last_sent: HashMap::new(), + journal_path: PathBuf::new(), + disabled: true, + } + } + + pub(crate) fn enqueue(&mut self, task: RemoteTaskMessage) -> Option { + if self.disabled { + return Some(remote_task_rejected( + task.task_id, + "Agent 任务日志异常，远程执行已禁用", + )); + } + if self.completed.contains_key(&task.task_id) { + self.last_sent.remove(&task.task_id); + return None; + } + if self.active.contains(&task.task_id) { + return None; + } + let task_id = task.task_id.clone(); + if self.task_order.len() >= MAX_REMOTE_TASK_JOURNAL_ENTRIES { + return Some(remote_task_rejected( + task_id, + "待确认的远程任务过多，已拒绝执行", + )); + } + self.active.insert(task_id.clone()); + self.task_order.push_back(task_id.clone()); + if let Err(error) = self.persist_snapshot(None) { + self.active.remove(&task_id); + self.task_order.retain(|id| id != &task_id); + eprintln!("remote task journal write failed: {error}"); + return Some(remote_task_rejected( + task_id, + "Agent 无法持久化任务状态，已拒绝执行", + )); + } + match self.task_tx.try_send(task) { + Ok(()) => None, + Err(error) => { + let message = match error { + mpsc::TrySendError::Full(_) => "远程命令队列已满，请稍后重试", + mpsc::TrySendError::Disconnected(_) => "远程命令执行线程不可用", + }; + let result = remote_task_rejected(task_id, message); + self.remember_completed(result.clone()); + None + } + } + } + + pub(crate) fn drain_completed(&mut self) { + let results = self.result_rx.try_iter().collect::>(); + for result in results { + self.remember_completed(result); + } + } + + pub(crate) fn acknowledge(&mut self, task_id: &str) { + if !self.completed.contains_key(task_id) { + return; + } + match self.persist_snapshot(Some(task_id)) { + Ok(()) => { + self.completed.remove(task_id); + self.task_order.retain(|id| id != task_id); + self.last_sent.remove(task_id); + } + Err(error) => { + eprintln!("remote task ACK persistence failed: {error}"); + } + } + } + + pub(crate) fn remember_completed(&mut self, result: TaskResultMessage) { + let task_id = result.task_id.clone(); + let was_active = self.active.remove(&task_id); + if !was_active && !self.completed.contains_key(&task_id) { + self.task_order.push_back(task_id.clone()); + } + self.completed.insert(task_id.clone(), result); + self.last_sent.remove(&task_id); + if let Err(error) = self.persist_snapshot(None) { + eprintln!("remote task result persistence failed: {error}"); + } + } + + pub(crate) fn due_results(&mut self) -> Vec { + self.drain_completed(); + let now = Instant::now(); + self.task_order + .iter() + .filter_map(|task_id| { + let result = self.completed.get(task_id)?; + let due = self.last_sent.get(task_id).is_none_or(|sent_at| { + now.duration_since(*sent_at) >= REMOTE_RESULT_RETRY_INTERVAL + }); + due.then(|| result.clone()) + }) + .collect() + } + + pub(crate) fn mark_sent(&mut self, task_id: &str) { + self.last_sent.insert(task_id.to_string(), Instant::now()); + } + + pub(crate) fn reset_delivery(&mut self) { + self.last_sent.clear(); + } + + pub(crate) fn persist_snapshot(&self, excluded_task_id: Option<&str>) -> Result<()> { + let entries = self + .task_order + .iter() + .filter(|task_id| excluded_task_id != Some(task_id.as_str())) + .filter_map(|task_id| { + if self.active.contains(task_id) { + Some(RemoteTaskJournalEntry::Active { + task_id: task_id.clone(), + }) + } else { + self.completed + .get(task_id) + .cloned() + .map(|result| RemoteTaskJournalEntry::Completed { result }) + } + }) + .collect::>(); + write_remote_task_journal(&self.journal_path, entries) + } +} + +pub(crate) fn remote_task_rejected(task_id: String, result: &str) -> TaskResultMessage { + TaskResultMessage { + message_type: "task_result".to_string(), + task_id, + status: "failed".to_string(), + result: result.to_string(), + exit_code: Some(-1), + } +} + +pub(crate) fn valid_remote_task_id(value: &str) -> bool { + value.len() == 36 + && value.bytes().enumerate().all(|(index, byte)| { + if matches!(index, 8 | 13 | 18 | 23) { + byte == b'-' + } else { + byte.is_ascii_hexdigit() + } + }) +} + +pub(crate) fn load_remote_task_journal(path: &Path) -> Result { + let bytes = match fs::read(path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == ErrorKind::NotFound => { + return Ok(RemoteTaskJournal { + version: REMOTE_TASK_JOURNAL_VERSION, + entries: Vec::new(), + }); + } + Err(error) => return Err(error.into()), + }; + if bytes.len() as u64 > MAX_REMOTE_TASK_JOURNAL_BYTES { + return Err("remote task journal is too large".into()); + } + let journal = serde_json::from_slice::(&bytes)?; + if journal.version != REMOTE_TASK_JOURNAL_VERSION { + return Err("unsupported remote task journal version".into()); + } + if journal.entries.len() > MAX_REMOTE_TASK_JOURNAL_ENTRIES { + return Err("remote task journal has too many entries".into()); + } + let mut task_ids = HashSet::new(); + for entry in &journal.entries { + let (task_id, valid_result) = match entry { + RemoteTaskJournalEntry::Active { task_id } => (task_id, true), + RemoteTaskJournalEntry::Completed { result } => ( + &result.task_id, + result.message_type == "task_result" + && matches!(result.status.as_str(), "success" | "failed") + && result.result.len() <= REMOTE_RESULT_OUTPUT_BYTES + 64, + ), + }; + if !valid_remote_task_id(task_id) || !valid_result || !task_ids.insert(task_id) { + return Err("remote task journal contains an invalid entry".into()); + } + } + Ok(journal) +} + +pub(crate) fn write_remote_task_journal(path: &Path, entries: Vec) -> Result<()> { + if entries.is_empty() { + return match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + }; + } + let journal = RemoteTaskJournal { + version: REMOTE_TASK_JOURNAL_VERSION, + entries, + }; + let mut encoded = serde_json::to_vec(&journal)?; + encoded.push(b'\n'); + if encoded.len() as u64 > MAX_REMOTE_TASK_JOURNAL_BYTES { + return Err("remote task journal is too large".into()); + } + let temporary = path.with_file_name(format!( + ".{}.{}.tmp", + path.file_name() + .and_then(|value| value.to_str()) + .unwrap_or("remote-tasks"), + std::process::id() + )); + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&temporary)?; + #[cfg(unix)] + fs::set_permissions(&temporary, fs::Permissions::from_mode(0o600))?; + file.write_all(&encoded)?; + file.sync_all()?; + drop(file); + // fs::rename maps to MoveFileEx(MOVEFILE_REPLACE_EXISTING) on Windows, so it + // replaces the destination atomically; deleting it first would open a window + // where a crash loses the whole journal. + if let Err(error) = fs::rename(&temporary, path) { + let _ = fs::remove_file(&temporary); + return Err(error.into()); + } + Ok(()) +} diff --git a/agent/src/tests.rs b/agent/src/tests.rs new file mode 100755 index 0000000..1aee07b --- /dev/null +++ b/agent/src/tests.rs @@ -0,0 +1,811 @@ +use super::*; + +use clap::Parser; +use std::fs; +use std::path::PathBuf; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use super::{ + CLOCK_CALIBRATION_MAX_AGE, CapturedOutput, CliOptions, ClockCalibration, DiskMetric, + GithubReleaseAsset, LatencyResult, LatencyTask, LiveAck, MAX_PENDING_LATENCY_RESULTS, + PUBLIC_IP_STALE_AFTER, PublicIpValue, REMOTE_RESULT_OUTPUT_BYTES, + REMOTE_STREAM_OUTPUT_BYTES, RemoteExecutor, RemoteTaskJournalEntry, RemoteTaskMessage, + Report, TaskResultMessage, UPDATE_CHECK_JITTER_MAX_SECONDS, ack_persist_interval, + advance_deadline, clock_offset_from_http_date, corrected_timestamp, dedupe_disks, + execute_remote_task_with_timeout, gpu_name_from_uevent, is_public_probe_ip, live_endpoint, + live_update_payload, monotonic_report_timestamp, normalized_version, parse_lspci_gpu_names, + parse_pciconf_gpu_names, parse_probe_target, parse_public_ip, + parse_system_profiler_gpu_names, ping_latencies, ping_latency, prune_report_samples, + release_asset_sha256, remote_result_text, sanitize_latency_tasks, selected_interface, + update_check_jitter, valid_endpoint, version_triplet, wildcard_match, + write_remote_task_journal, +}; + +#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] +use super::connection_counts_from_netstat; +#[cfg(target_os = "linux")] +use super::disk_device; + +#[test] +fn parses_family_matched_public_ips() { + let v4: std::net::IpAddr = "203.0.113.7".parse().unwrap(); + let v6: std::net::IpAddr = "2001:db8::1".parse().unwrap(); + assert_eq!(parse_public_ip("203.0.113.7\n", false), Some(v4)); + assert_eq!(parse_public_ip("2001:db8::1", true), Some(v6)); + assert_eq!(parse_public_ip("2001:db8::1", false), None); + assert_eq!(parse_public_ip("203.0.113.7", true), None); + assert_eq!(parse_public_ip("not an ip", false), None); + assert_eq!(parse_public_ip("", false), None); +} + +#[test] +fn expires_public_ip_after_sustained_probe_failures() { + let started_at = Instant::now(); + let address = "203.0.113.7".parse().unwrap(); + let mut value = PublicIpValue::default(); + + value.observe(Some(address), started_at); + value.observe( + None, + started_at + PUBLIC_IP_STALE_AFTER - Duration::from_secs(1), + ); + assert_eq!(value.address, Some(address)); + + value.observe(None, started_at + PUBLIC_IP_STALE_AFTER); + assert_eq!(value.address, None); +} + +#[test] +fn parses_platform_gpu_names() { + let lspci = "00:02.0 VGA compatible controller: Intel Corporation CometLake-S GT2 [UHD Graphics 630] (rev 05)\n\ + 01:00.0 Audio device: NVIDIA Corporation HDMI Audio\n\ + 02:00.0 3D controller: NVIDIA Corporation GA102 [GeForce RTX 3090] (rev a1)"; + assert_eq!( + parse_lspci_gpu_names(lspci), + [ + "Intel Corporation CometLake-S GT2 [UHD Graphics 630] (rev 05)", + "NVIDIA Corporation GA102 [GeForce RTX 3090] (rev a1)", + ] + ); + + let profiler = "Graphics/Displays:\n\n Apple M3 Max:\n\n Chipset Model: Apple M3 Max\n Metal Support: Metal 3"; + assert_eq!(parse_system_profiler_gpu_names(profiler), ["Apple M3 Max"]); + + let pciconf = "vgapci0@pci0:0:2:0:\tclass=0x030000 rev=0x02 hdr=0x00 vendor=0x8086\n\ + \tvendor = 'Intel Corporation'\n\ + \tdevice = 'UHD Graphics 630'\n\ + \tclass = display\n\ + em0@pci0:0:25:0:\tclass=0x020000 rev=0x05 hdr=0x00 vendor=0x8086\n\ + \tvendor = 'Intel Corporation'\n\ + \tdevice = 'Ethernet Connection'\n\ + \tclass = network"; + assert_eq!( + parse_pciconf_gpu_names(pciconf), + ["Intel Corporation UHD Graphics 630"] + ); + + assert_eq!( + gpu_name_from_uevent("DRIVER=i915\nPCI_CLASS=30000\nPCI_ID=8086:591B"), + Some("Intel Integrated Graphics".to_string()) + ); + assert_eq!( + gpu_name_from_uevent("DRIVER=virtio_gpu\nPCI_CLASS=30000"), + None + ); +} + +#[cfg(target_os = "linux")] +#[test] +fn selects_whole_disk_devices() { + assert!(disk_device("sda")); + assert!(disk_device("nvme0n1")); + assert!(!disk_device("sda1")); + assert!(!disk_device("nvme0n1p1")); +} + +#[test] +fn filters_virtual_interfaces_and_supports_include_exclude_patterns() { + assert!(!selected_interface("lo", "")); + assert!(!selected_interface("docker0", "")); + assert!(!selected_interface("veth123", "")); + assert!(!selected_interface("br0", "!eth*")); + assert!(selected_interface("br0", "br0")); + assert!(!selected_interface("br0", "br*,!br0")); + assert!(!selected_interface("br0", "!br0,br*")); + assert!(!selected_interface("Software Loopback Interface 1", "")); + assert!(selected_interface("eth0", "")); + assert!(selected_interface("ens3", "eth*,ens*")); + assert!(!selected_interface("wlan0", "eth*,ens*")); + assert!(!selected_interface("eth0", "eth*,!eth0")); + assert!(selected_interface("ens3", "!eth*")); + assert!(wildcard_match("en?3", "ens3")); + assert!(!wildcard_match("en?3", "enp4s0")); +} + +#[test] +fn deduplicates_mounts_for_the_same_device() { + let disks = dedupe_disks([ + DiskMetric { + name: "/dev/vg/root".into(), + mount_point: "/var".into(), + total: 10, + used: 4, + ..DiskMetric::default() + }, + DiskMetric { + name: "/dev/vg/root".into(), + mount_point: "/".into(), + total: 20, + used: 8, + ..DiskMetric::default() + }, + DiskMetric { + name: "tank/data".into(), + mount_point: "/data".into(), + total: 30, + used: 9, + ..DiskMetric::default() + }, + DiskMetric { + name: "tank/archive".into(), + mount_point: "/archive".into(), + total: 25, + used: 7, + ..DiskMetric::default() + }, + DiskMetric { + name: "/var/lib/data".into(), + mount_point: "/var/lib/data".into(), + total: 40, + used: 12, + ..DiskMetric::default() + }, + ]); + assert_eq!(disks.len(), 3); + assert_eq!(disks[0].mount_point, "/"); + assert_eq!(disks[0].total, 20); + assert_eq!(disks[1].name, "tank/data"); + assert_eq!(disks[2].name, "/var/lib/data"); +} + +#[cfg(target_os = "linux")] +#[test] +fn keeps_root_filesystem_but_filters_virtual_mounts() { + assert!(!super::excluded_filesystem("overlay", "/")); + assert!(super::excluded_filesystem("tmpfs", "/tmp")); + assert!(super::excluded_filesystem("proc", "/proc")); + assert!(!super::excluded_filesystem("btrfs", "/var")); +} + +#[cfg(target_os = "linux")] +#[test] +fn memory_matches_komari_default_including_shared_memory_and_swap_cache() { + let memory = super::memory_sample( + "MemTotal: 1000000 kB\nMemFree: 100000 kB\nMemAvailable: 750000 kB\n\ + Cached: 300000 kB\nSReclaimable: 50000 kB\nBuffers: 25000 kB\n\ + Shmem: 25000 kB\nSwapTotal: 100000 kB\nSwapFree: 60000 kB\nSwapCached: 20000 kB\n", + ); + assert_eq!(memory.total, 1_000_000 * 1024); + assert_eq!(memory.used, 550_000 * 1024); + assert_eq!(memory.swap_total, 100_000 * 1024); + assert_eq!(memory.swap_used, 20_000 * 1024); +} + +#[cfg(target_os = "linux")] +#[test] +fn filling_file_cache_does_not_inflate_reported_memory() { + let before = super::memory_sample( + "MemTotal: 1000 kB\nMemFree: 400 kB\nCached: 300 kB\n\ + Buffers: 50 kB\nSReclaimable: 50 kB\nShmem: 100 kB\n", + ); + let after = super::memory_sample( + "MemTotal: 1000 kB\nMemFree: 200 kB\nCached: 500 kB\n\ + Buffers: 50 kB\nSReclaimable: 50 kB\nShmem: 100 kB\n", + ); + assert_eq!(before.used, 300 * 1024); + assert_eq!(after.used, before.used); +} + +#[cfg(target_os = "linux")] +#[test] +fn memory_handles_missing_and_inconsistent_kernel_counters() { + let memory = super::memory_sample( + "MemTotal: 1000 kB\nMemFree: 200 kB\nCached: 1000 kB\n\ + Shmem: 100 kB\nSwapTotal: 100 kB\nSwapFree: 20 kB\nSwapCached: 100 kB\n", + ); + assert_eq!(memory.used, 900 * 1024); + assert_eq!(memory.swap_used, 80 * 1024); + for contents in ["", "MemTotal: invalid kB\n", "MemTotal: -1 kB\n"] { + let memory = super::memory_sample(contents); + assert_eq!( + ( + memory.total, + memory.used, + memory.swap_total, + memory.swap_used + ), + (0, 0, 0, 0) + ); + } + let memory = super::memory_sample( + "MemTotal: 100 kB\nMemFree: 1000 kB\nSwapTotal: 100 kB\nSwapFree: 1000 kB\n", + ); + assert_eq!((memory.used, memory.swap_used), (0, 0)); +} + +#[cfg(target_os = "linux")] +#[test] +fn reads_local_mounts_and_decodes_escaped_paths_once() { + let mounts = super::disk_mounts( + r"overlay / overlay rw 0 0 +proc /proc proc rw 0 0 +tmpfs /run tmpfs rw 0 0 +/dev/vda1 /data\040volume ext4 rw 0 0 +/dev/vda2 /literal\134040path ext4 rw 0 0 +/dev/vda3 /tab\011dir ext4 rw 0 0 +server:/export /remote nfs4 rw 0 0 +//server/share /remote-share cifs rw 0 0 +none /auto autofs rw 0 0 +invalid", + ) + .collect::>(); + assert_eq!( + mounts, + vec![ + ("overlay".into(), "/".into()), + ("/dev/vda1".into(), "/data volume".into()), + ("/dev/vda2".into(), "/literal\\040path".into()), + ("/dev/vda3".into(), "/tab\tdir".into()), + ] + ); +} + +#[cfg(target_os = "linux")] +#[test] +fn reads_root_disk_capacity_without_external_commands() { + let (total, used) = super::filesystem_space("/").unwrap(); + assert!(total > 0); + assert!((0..=total).contains(&used)); + assert!( + super::disk_usage() + .iter() + .any(|disk| disk.mount_point == "/") + ); + assert!(super::filesystem_space("/proc/self/not-a-mount").is_none()); + assert!(super::filesystem_space("/nul\0path").is_none()); +} + +#[test] +fn overdue_latency_tasks_wait_and_resume_without_duplicate_probes() { + let (task_tx, task_rx) = std::sync::mpsc::channel(); + let (result_tx, result_rx) = std::sync::mpsc::channel(); + let mut executor = super::LatencyExecutor { + task_tx, + result_rx, + in_flight: Default::default(), + stats: Default::default(), + }; + let task = LatencyTask { + id: "queued-probe".into(), + name: "probe".into(), + task_type: "tcp".into(), + target: "example.com".into(), + port: Some(443), + interval_seconds: 30, + }; + let mut deadlines = std::collections::HashMap::new(); + let started = Instant::now(); + executor.schedule(std::slice::from_ref(&task), &mut deadlines, started); + assert_eq!(task_rx.try_recv().unwrap().id, task.id); + assert_eq!(deadlines[&task.id], started + Duration::from_secs(30)); + + let overdue = started + Duration::from_secs(30); + executor.schedule(std::slice::from_ref(&task), &mut deadlines, overdue); + assert!(task_rx.try_recv().is_err()); + assert_eq!(deadlines[&task.id], overdue + Duration::from_secs(1)); + + result_tx + .send(LatencyResult { + task_id: task.id.clone(), + ..LatencyResult::default() + }) + .unwrap(); + assert_eq!(executor.drain().len(), 1); + let retry_at = deadlines[&task.id]; + executor.schedule(std::slice::from_ref(&task), &mut deadlines, retry_at); + executor.schedule(std::slice::from_ref(&task), &mut deadlines, retry_at); + assert_eq!(task_rx.try_recv().unwrap().id, task.id); + assert!(task_rx.try_recv().is_err()); + assert_eq!(deadlines[&task.id], retry_at + Duration::from_secs(30)); + + drop(task_rx); + executor.in_flight.clear(); + let retry_at = deadlines[&task.id]; + executor.schedule(std::slice::from_ref(&task), &mut deadlines, retry_at); + assert_eq!(deadlines[&task.id], retry_at + Duration::from_secs(1)); +} + +#[test] +fn parses_release_versions() { + assert_eq!(normalized_version("v1.2.3"), "1.2.3"); + assert_eq!(version_triplet("v1.2.3"), Some((1, 2, 3))); + assert_eq!(version_triplet("1.2"), None); + assert_eq!(version_triplet("1.2.3.4"), None); + assert_eq!(version_triplet("rust-3"), None); +} + +#[test] +fn validates_release_asset_digests() { + let hash = "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF"; + let mut asset = GithubReleaseAsset { + name: "agent-linux-x64-glibc".to_string(), + browser_download_url: "https://example.com/agent".to_string(), + digest: Some(format!("sha256:{hash}")), + }; + let expected = hash.to_ascii_lowercase(); + assert_eq!( + release_asset_sha256(&asset).as_deref(), + Some(expected.as_str()) + ); + asset.digest = Some("sha512:0123".to_string()); + assert_eq!(release_asset_sha256(&asset), None); + asset.digest = None; + assert_eq!(release_asset_sha256(&asset), None); +} + +#[test] +fn validates_endpoints() { + assert!(valid_endpoint("https://monitor.example.com/")); + assert!(valid_endpoint("http://127.0.0.1:8787")); + assert!(valid_endpoint("http://localhost:8787")); + assert!(valid_endpoint("http://[::1]:8787")); + assert!(!valid_endpoint("http://monitor.example.com")); + assert!(!valid_endpoint("http://127.0.0.2:8787")); + assert!(!valid_endpoint("monitor.example.com")); + assert!(!valid_endpoint("https://")); + assert!(!valid_endpoint("https://user@example.com")); + assert!(!valid_endpoint("https://monitor.example.com/?token=abc")); + assert!(!valid_endpoint("https://bad host.example")); +} + +#[test] +fn builds_live_websocket_endpoints() { + assert_eq!( + live_endpoint("https://monitor.example.com").unwrap(), + "wss://monitor.example.com/api/agent/ws" + ); + assert_eq!( + live_endpoint("https://monitor.example.com/base/").unwrap(), + "wss://monitor.example.com/base/api/agent/ws" + ); + assert_eq!( + live_endpoint("http://127.0.0.1:8787").unwrap(), + "ws://127.0.0.1:8787/api/agent/ws" + ); +} + +#[test] +fn accepts_durable_ack_interval_and_limits_upload_frequency() { + let ack: LiveAck = serde_json::from_str( + r#"{"type":"ack","ts":100,"persisted":true,"persistenceError":false,"persistedThroughTs":100,"nextPersistAfterMs":60000}"#, + ) + .unwrap(); + assert_eq!(ack_persist_interval(&ack), Duration::from_secs(60)); + let idle: LiveAck = serde_json::from_str( + r#"{"type":"ack","ts":100,"persisted":true,"persistenceError":false,"persistedThroughTs":100,"nextPersistAfterMs":120000}"#, + ) + .unwrap(); + assert_eq!(ack_persist_interval(&idle), Duration::from_secs(120)); + assert_eq!(super::live_batch_interval(1), Duration::from_secs(3)); + assert_eq!(super::live_batch_interval(3), Duration::from_secs(3)); + assert_eq!(super::live_batch_interval(15), Duration::from_secs(15)); +} + +#[cfg(unix)] +#[test] +fn remote_commands_capture_output_and_exit_status() { + let result = execute_remote_task_with_timeout( + &RemoteTaskMessage { + message_type: "remote_task".to_string(), + task_id: "task-output".to_string(), + command: "printf stdout; printf stderr >&2; exit 7".to_string(), + }, + Duration::from_secs(2), + ); + assert_eq!(result.status, "failed"); + assert_eq!(result.exit_code, Some(7)); + assert_eq!(result.result, "stdout\nstderr"); +} + +#[cfg(unix)] +#[test] +fn remote_commands_are_terminated_after_the_deadline() { + let result = execute_remote_task_with_timeout( + &RemoteTaskMessage { + message_type: "remote_task".to_string(), + task_id: "task-timeout".to_string(), + command: "sleep 5".to_string(), + }, + Duration::from_millis(50), + ); + assert_eq!(result.status, "failed"); + assert_eq!(result.exit_code, Some(-1)); + assert!(result.result.contains("已终止")); +} + +#[test] +fn combined_remote_output_is_bounded_at_utf8_boundaries() { + let result = remote_result_text( + &CapturedOutput { + bytes: vec![b'x'; REMOTE_STREAM_OUTPUT_BYTES as usize], + truncated: true, + }, + &CapturedOutput { + bytes: "输出" + .repeat(REMOTE_STREAM_OUTPUT_BYTES as usize / 6) + .into_bytes(), + truncated: false, + }, + ); + assert!(result.len() < REMOTE_RESULT_OUTPUT_BYTES + 64); + assert!(result.ends_with("[输出已截断]")); + assert!(!result.contains('\u{fffd}')); +} + +#[cfg(unix)] +#[test] +fn verbose_remote_commands_finish_after_output_is_truncated() { + for command in [ + "set -e; head -c 1048576 /dev/zero; printf completed >&2", + "set -e; head -c 1048576 /dev/zero >&2; printf completed", + ] { + let result = execute_remote_task_with_timeout( + &RemoteTaskMessage { + message_type: "remote_task".to_string(), + task_id: "verbose-output".to_string(), + command: command.to_string(), + }, + Duration::from_secs(5), + ); + assert_eq!(result.status, "success"); + assert_eq!(result.exit_code, Some(0)); + assert!(result.result.contains("completed")); + assert!(result.result.ends_with("[输出已截断]")); + assert!(result.result.len() < REMOTE_RESULT_OUTPUT_BYTES + 64); + } +} + +fn temporary_remote_task_journal(label: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "nodeflare-agent-{label}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&directory).unwrap(); + directory.join("remote-tasks.json") +} + +#[test] +fn interrupted_remote_tasks_are_failed_without_reexecution() { + let path = temporary_remote_task_journal("interrupted-task"); + let task_id = "8dd70536-f721-4d47-af63-e2f17c83de25"; + write_remote_task_journal( + &path, + vec![RemoteTaskJournalEntry::Active { + task_id: task_id.to_string(), + }], + ) + .unwrap(); + + let mut executor = RemoteExecutor::new_at(path.clone()).unwrap(); + assert!(executor.active.is_empty()); + let due = executor.due_results(); + assert_eq!(due.len(), 1); + assert_eq!(due[0].task_id, task_id); + assert_eq!(due[0].status, "failed"); + assert!(due[0].result.contains("不会再次执行")); + + executor.mark_sent(task_id); + assert!(executor.due_results().is_empty()); + executor.reset_delivery(); + assert_eq!(executor.due_results().len(), 1); + executor.acknowledge(task_id); + assert!(!path.exists()); + + let directory = path.parent().unwrap().to_path_buf(); + drop(executor); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn completed_remote_tasks_return_the_cached_result() { + let path = temporary_remote_task_journal("completed-task"); + let task_id = "6cbf4d36-f33a-40ef-b952-6af1b8491820"; + let result = TaskResultMessage { + message_type: "task_result".to_string(), + task_id: task_id.to_string(), + status: "success".to_string(), + result: "already finished".to_string(), + exit_code: Some(0), + }; + write_remote_task_journal( + &path, + vec![RemoteTaskJournalEntry::Completed { + result: result.clone(), + }], + ) + .unwrap(); + + let mut executor = RemoteExecutor::new_at(path.clone()).unwrap(); + assert!( + executor + .enqueue(RemoteTaskMessage { + message_type: "remote_task".to_string(), + task_id: task_id.to_string(), + command: "this command must not run".to_string(), + }) + .is_none() + ); + assert!(executor.active.is_empty()); + assert_eq!(executor.due_results(), [result]); + executor.acknowledge(task_id); + + let directory = path.parent().unwrap().to_path_buf(); + drop(executor); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn encodes_realtime_samples_as_a_batch() { + let payload = live_update_payload( + vec![ + Report { + timestamp: 10, + cpu: 20.0, + ..Report::default() + }, + Report { + timestamp: 15, + cpu: 30.0, + ..Report::default() + }, + ], + false, + &mut None, + ) + .unwrap(); + let value: serde_json::Value = nodeflare_telemetry::decode(&payload).unwrap(); + assert_eq!(value["persist"], false); + assert_eq!(value["samples"].as_array().unwrap().len(), 2); + assert_eq!(value["samples"][1]["metrics"]["cpu"], 30.0); +} + +#[test] +fn calibrates_timestamps_from_http_date() { + let server_ms = 784_111_777_000_i64; + assert_eq!( + clock_offset_from_http_date( + "Sun, 06 Nov 1994 08:49:37 GMT", + server_ms - 1_000, + server_ms + 1_000, + ), + Some(0), + ); + assert_eq!(corrected_timestamp(100, 2_500), 102); + assert_eq!(corrected_timestamp(100, -2_500), 97); + assert_eq!(monotonic_report_timestamp(97, 100, 90), 101); + assert_eq!(monotonic_report_timestamp(110, 100, 120), 121); + assert_eq!(monotonic_report_timestamp(130, 120, 110), 130); +} + +#[test] +fn ignores_small_clock_jitter_until_the_calibration_expires() { + let mut calibration = ClockCalibration::default(); + assert!(calibration.observe(1_000)); + assert!(!calibration.observe(2_000)); + assert_eq!(calibration.offset_ms, 1_000); + assert!(calibration.observe(25_000)); + calibration.calibrated_at = Some(Instant::now() - CLOCK_CALIBRATION_MAX_AGE); + assert!(calibration.observe(25_001)); +} + +#[test] +fn parses_cli_options() { + let parsed = CliOptions::try_parse_from([ + "agent", + "-e", + "https://monitor.example.com", + "-t", + "agent-token", + "-i", + "60", + "--once", + ]) + .unwrap(); + assert_eq!(parsed.endpoint, "https://monitor.example.com"); + assert_eq!(parsed.token.as_deref(), Some("agent-token")); + assert_eq!(parsed.interval, 60); + assert!(parsed.once); + assert!(CliOptions::try_parse_from(["agent", "-t"]).is_err()); + assert!(CliOptions::try_parse_from(["agent", "-t", "first", "-t", "second"]).is_err()); + assert!( + CliOptions::try_parse_from([ + "agent", + "-e", + "https://monitor.example.com", + "-t", + "agent-token", + "--token-file", + "/run/nodeflare/token", + ]) + .is_err() + ); + assert!(CliOptions::try_parse_from(["agent", "--once", "--collect"]).is_err()); +} + +#[test] +fn filters_and_deduplicates_remote_latency_tasks_without_a_count_limit() { + let valid = LatencyTask { + id: "task-1".to_string(), + name: "Cloudflare".to_string(), + task_type: "tcp".to_string(), + target: "1.1.1.1".to_string(), + port: Some(443), + interval_seconds: 60, + }; + let mut tasks = vec![valid.clone(), valid]; + tasks.push(LatencyTask { + id: "bad".to_string(), + name: "Bad".to_string(), + task_type: "http".to_string(), + target: "https://example.com".to_string(), + port: Some(443), + interval_seconds: 1, + }); + for index in 2..=300 { + tasks.push(LatencyTask { + id: format!("task-{index}"), + name: format!("Task {index}"), + task_type: "icmp".to_string(), + target: "1.1.1.1".to_string(), + port: None, + interval_seconds: 60, + }); + } + let sanitized = sanitize_latency_tasks(&tasks); + assert_eq!(sanitized.len(), 300); + assert_eq!(sanitized[0].id, "task-1"); + assert!(!sanitized.iter().any(|task| task.id == "bad")); +} + +#[test] +fn advances_collection_deadlines_without_drift() { + let start = Instant::now(); + let interval = Duration::from_secs(1); + assert_eq!( + advance_deadline(start, interval, start + Duration::from_millis(2500)), + start + Duration::from_secs(3) + ); +} + +#[test] +fn drops_samples_the_server_would_reject_as_expired() { + let mut samples = vec![ + Report { + timestamp: 1_000, + ..Report::default() + }, + Report { + timestamp: 8_500, + ..Report::default() + }, + ]; + prune_report_samples(&mut samples, 9_000); + assert_eq!(samples.len(), 1); + assert_eq!(samples[0].timestamp, 8_500); +} + +#[test] +fn bounds_pending_latency_results() { + let mut samples = (0..MAX_PENDING_LATENCY_RESULTS + 10) + .map(|index| Report { + timestamp: index as i64 + 1, + latency_results: vec![LatencyResult { + task_id: format!("task-{index}"), + timestamp: index as i64 + 1, + latency_ms: 10.0, + packet_loss: 0.0, + }], + ..Report::default() + }) + .collect::>(); + prune_report_samples(&mut samples, MAX_PENDING_LATENCY_RESULTS as i64 + 10); + assert_eq!( + samples + .iter() + .map(|sample| sample.latency_results.len()) + .sum::(), + MAX_PENDING_LATENCY_RESULTS + ); + assert!(samples.first().unwrap().latency_results.is_empty()); + assert_eq!(samples.last().unwrap().latency_results.len(), 1); +} + +#[cfg(any(target_os = "windows", target_os = "macos", target_os = "freebsd"))] +#[test] +fn parses_netstat_connection_counts() { + let output = "tcp4 0 0 host.443 peer.1 ESTABLISHED\nudp4 0 0 *.5353 *.*\nTCP host peer ESTABLISHED\n"; + assert_eq!(connection_counts_from_netstat(output), (2, 1)); +} + +#[test] +fn parses_probe_targets() { + assert_eq!( + parse_probe_target("Example.COM", None), + Some(("example.com".to_string(), 443)) + ); + assert_eq!( + parse_probe_target("1.1.1.1", Some(8080)), + Some(("1.1.1.1".to_string(), 8080)) + ); + for target in [ + "", + "https://example.com", + "example.com:0", + "example.com:65536", + "999.1.1.1", + "127.0.0.1:8080", + "169.254.169.254", + "192.168.1.1", + "router.local", + "localhost", + "[::1]:443", + "bad host", + ] { + assert_eq!(parse_probe_target(target, None), None, "target: {target}"); + } +} + +#[test] +fn accepts_only_public_probe_addresses() { + for address in ["1.1.1.1", "8.8.8.8", "2606:4700:4700::1111"] { + assert!(is_public_probe_ip(address.parse().expect("IP address"))); + } + for address in [ + "0.0.0.0", + "10.0.0.1", + "100.64.0.1", + "127.0.0.1", + "169.254.169.254", + "172.16.0.1", + "192.168.1.1", + "198.18.0.1", + "224.0.0.1", + "::1", + "fe80::1", + "fd00:ec2::254", + ] { + assert!(!is_public_probe_ip(address.parse().expect("IP address"))); + } +} + +#[test] +fn parses_ping_latency() { + assert_eq!(ping_latency("64 bytes time=12.34 ms"), Some(12.34)); + assert_eq!(ping_latency("64 bytes time<1 ms"), Some(0.5)); + assert_eq!(ping_latency("unreachable"), None); + assert_eq!( + ping_latencies("reply time=12.34 ms\nrequest timeout\nreply time=8.5 ms"), + vec![12.34, 8.5] + ); +} + +#[test] +fn update_check_jitter_is_stable_and_bounded() { + let first = update_check_jitter("agent-token"); + assert_eq!(first, update_check_jitter("agent-token")); + assert!(first.as_secs() <= UPDATE_CHECK_JITTER_MAX_SECONDS); +} diff --git a/agent/src/update.rs b/agent/src/update.rs new file mode 100755 index 0000000..89d41b5 --- /dev/null +++ b/agent/src/update.rs @@ -0,0 +1,240 @@ +use super::*; + +#[derive(Debug, Deserialize)] +pub(crate) struct GithubRelease { + pub(crate) tag_name: String, + pub(crate) assets: Vec, +} + +#[derive(Debug, Deserialize)] +pub(crate) struct GithubReleaseAsset { + pub(crate) name: String, + pub(crate) browser_download_url: String, + pub(crate) digest: Option, +} + +pub(crate) fn normalized_version(value: &str) -> &str { + let value = value.trim(); + value.strip_prefix('v').unwrap_or(value) +} + +pub(crate) fn version_triplet(value: &str) -> Option<(u64, u64, u64)> { + let mut parts = normalized_version(value).split('.'); + let version = ( + parts.next()?.parse().ok()?, + parts.next()?.parse().ok()?, + parts.next()?.parse().ok()?, + ); + parts.next().is_none().then_some(version) +} + +pub(crate) fn agent_artifact_name() -> Option<&'static str> { + match (env::consts::OS, env::consts::ARCH) { + ("linux", "x86_64") => Some(if cfg!(target_env = "musl") { + "agent-linux-x64-musl" + } else { + "agent-linux-x64-glibc" + }), + ("linux", "aarch64") => Some(if cfg!(target_env = "musl") { + "agent-linux-aarch64-musl" + } else { + "agent-linux-aarch64-glibc" + }), + ("windows", "x86_64") => Some("agent-windows-x64.exe"), + ("macos", "aarch64") => Some("agent-macos-aarch64"), + ("freebsd", "x86_64") => Some("agent-freebsd-x64"), + ("freebsd", "aarch64") => Some("agent-freebsd-aarch64"), + _ => None, + } +} + +pub(crate) fn executable_format_valid(path: &Path) -> bool { + let mut magic = [0_u8; 4]; + if fs::File::open(path) + .and_then(|mut file| file.read_exact(&mut magic)) + .is_err() + { + return false; + } + match env::consts::OS { + "linux" | "freebsd" => magic == *b"\x7fELF", + "windows" => magic.starts_with(b"MZ"), + "macos" => matches!( + magic, + [0xcf, 0xfa, 0xed, 0xfe] + | [0xfe, 0xed, 0xfa, 0xcf] + | [0xca, 0xfe, 0xba, 0xbe] + | [0xbe, 0xba, 0xfe, 0xca] + ), + _ => false, + } +} + +pub(crate) fn sha256_file(path: &Path) -> Result { + let mut file = fs::File::open(path)?; + let mut digest = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + digest.update(&buffer[..read]); + } + Ok(digest + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect()) +} + +pub(crate) fn release_asset_sha256(asset: &GithubReleaseAsset) -> Option { + let digest = asset.digest.as_deref()?.strip_prefix("sha256:")?; + (digest.len() == 64 + && digest + .chars() + .all(|character| character.is_ascii_hexdigit())) + .then(|| digest.to_ascii_lowercase()) +} + +pub(crate) fn download_agent( + pub(crate) agent: &ureq::Agent, + pub(crate) url: &str, + pub(crate) destination: &Path, + pub(crate) expected_version: &str, + pub(crate) expected_sha256: &str, +) -> Result { + let Ok(response) = agent.get(url).call() else { + return Ok(false); + }; + let response = response.into_body().into_reader(); + let mut file = fs::File::create(destination)?; + let copied = io::copy(&mut response.take(MAX_AGENT_BINARY_BYTES + 1), &mut file)?; + file.flush()?; + drop(file); + if copied > MAX_AGENT_BINARY_BYTES + || !executable_format_valid(destination) + || sha256_file(destination)? != expected_sha256 + { + let _ = fs::remove_file(destination); + return Ok(false); + } + #[cfg(unix)] + fs::set_permissions(destination, fs::Permissions::from_mode(0o755))?; + + let downloaded_version = Command::new(destination) + .arg("--version") + .output() + .ok() + .filter(|output| output.status.success()) + .and_then(|output| { + String::from_utf8_lossy(&output.stdout) + .split_whitespace() + .next_back() + .map(str::to_string) + }); + let valid = downloaded_version.as_deref().map(normalized_version) + == Some(normalized_version(expected_version)); + if !valid { + let _ = fs::remove_file(destination); + } + Ok(valid) +} + +pub(crate) fn mirrored_download_url(mirror: &str, url: &str) -> String { + let mirror = mirror.trim().trim_end_matches('/'); + if mirror.is_empty() { + url.to_string() + } else { + format!("{mirror}/{url}") + } +} + +pub(crate) fn update(agent: &ureq::Agent, mirror: &str) -> Result { + let mut response = agent + .get(LATEST_RELEASE_API) + .header("Accept", "application/vnd.github+json") + .header("User-Agent", format!("nodeflare-agent/{VERSION}")) + .call()?; + let release = response.body_mut().read_json::()?; + let Some(remote_version) = version_triplet(&release.tag_name) else { + return Ok(false); + }; + let Some(current_version) = version_triplet(VERSION) else { + return Ok(false); + }; + if remote_version <= current_version { + return Ok(false); + } + let Some(artifact) = agent_artifact_name() else { + return Ok(false); + }; + let current = env::current_exe()?; + let temporary = current.with_file_name(format!( + ".nodeflare-agent.{}.download{}", + std::process::id(), + if cfg!(target_os = "windows") { + ".exe" + } else { + "" + } + )); + let Some(release_asset) = release.assets.iter().find(|asset| asset.name == artifact) else { + return Err(format!("latest release does not contain {artifact}").into()); + }; + let Some(expected_sha256) = release_asset_sha256(release_asset) else { + return Err( + format!("latest release does not contain a SHA-256 digest for {artifact}").into(), + ); + }; + if !download_agent( + agent, + &mirrored_download_url(mirror, &release_asset.browser_download_url), + &temporary, + &release.tag_name, + &expected_sha256, + )? { + return Err("downloaded agent version does not match the configured version".into()); + } + + #[cfg(unix)] + { + fs::rename(&temporary, ¤t)?; + let error = Command::new(¤t).args(env::args_os().skip(1)).exec(); + Err(error.into()) + } + + #[cfg(target_os = "windows")] + { + let script = concat!( + "$ErrorActionPreference='Stop'; ", + "$targetPid=[int]$env:NODEFLARE_UPDATE_PID; ", + "Wait-Process -Id $targetPid; ", + "Move-Item -LiteralPath $env:NODEFLARE_UPDATE_NEW ", + "-Destination $env:NODEFLARE_UPDATE_CURRENT -Force; ", + "try { Start-ScheduledTask -TaskName 'nodeflare-agent' -ErrorAction Stop } ", + "catch { $restartArgs=@($env:NODEFLARE_UPDATE_ARGS | ConvertFrom-Json); ", + "Start-Process -FilePath $env:NODEFLARE_UPDATE_CURRENT -ArgumentList $restartArgs }" + ); + Command::new("powershell.exe") + .args([ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + script, + ]) + .env("NODEFLARE_UPDATE_PID", std::process::id().to_string()) + .env("NODEFLARE_UPDATE_NEW", &temporary) + .env("NODEFLARE_UPDATE_CURRENT", ¤t) + .env( + "NODEFLARE_UPDATE_ARGS", + serde_json::to_string(&env::args().skip(1).collect::>())?, + ) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + Ok(true) + } +} diff --git a/backend/src/db/mod.rs b/backend/src/db/mod.rs old mode 100644 new mode 100755 index 393d14f..eabd3c4 --- a/backend/src/db/mod.rs +++ b/backend/src/db/mod.rs @@ -56,6 +56,50 @@ pub struct SessionIdentity { pub username: String, } +/// Rewrites `?` placeholders to `$1`, `$2`, … while skipping quoted spans. +fn translate_placeholders(statement: &str) -> String { + let mut output = String::with_capacity(statement.len() + 16); + let mut index = 0_u32; + let mut characters = statement.chars().peekable(); + let mut single_quoted = false; + let mut double_quoted = false; + while let Some(character) = characters.next() { + if single_quoted || double_quoted { + output.push(character); + let delimiter = if single_quoted { '\'' } else { '"' }; + if character == delimiter { + // A doubled delimiter is an escape, not the end of the span. + if characters.peek() == Some(&delimiter) { + if let Some(escaped) = characters.next() { + output.push(escaped); + } + } else { + single_quoted = false; + double_quoted = false; + } + } + continue; + } + match character { + '\'' => { + single_quoted = true; + output.push(character); + } + '"' => { + double_quoted = true; + output.push(character); + } + '?' => { + index += 1; + output.push('$'); + output.push_str(&index.to_string()); + } + _ => output.push(character), + } + } + output +} + impl Database { pub fn pool(&self) -> &AnyPool { &self.pool @@ -98,6 +142,8 @@ impl Database { kind: self.kind.as_str().to_string(), size_bytes, reclaimable_bytes, + // The database layer cannot see pending restarts; routes fill this + // in from the activity state (see routes/admin.rs database_stats). restart_required: false, }) } @@ -164,6 +210,11 @@ impl Database { Ok(()) } + /// Translates SQLite-style `?` placeholders to PostgreSQL `$N`. + /// + /// Placeholders inside single-quoted literals or double-quoted identifiers + /// are left untouched, so a statement such as `WHERE label='?'` cannot + /// silently shift the numbering of the real parameters. pub fn sql(&self, statement: &'static str) -> &'static str { if self.kind == DatabaseKind::Sqlite { return statement; @@ -174,18 +225,7 @@ impl Database { if let Some(value) = translated.get(statement) { return value; } - let mut index = 0; - let mut output = String::with_capacity(statement.len() + 16); - for character in statement.chars() { - if character == '?' { - index += 1; - output.push('$'); - output.push_str(&index.to_string()); - } else { - output.push(character); - } - } - let output = Box::leak(output.into_boxed_str()); + let output = Box::leak(translate_placeholders(statement).into_boxed_str()); translated.insert(statement, output); output } @@ -553,6 +593,7 @@ pub async fn update_settings( pool: &Database, input: &SettingsInput, password_hash: Option<&str>, + totp_rename: Option<(&str, &str)>, ) -> Result<()> { let mut updates = Vec::<(&str, String)>::new(); macro_rules! text { @@ -648,6 +689,18 @@ pub async fn update_settings( .execute(&mut *transaction) .await?; } + // The TOTP record must follow the admin rename atomically; otherwise a + // failure between the two writes would leave login unable to find the + // secret and silently skip two-factor verification. + if let Some((previous, next)) = totp_rename + && previous != next + { + sqlx::query(pool.sql("UPDATE admin_2fa SET username=? WHERE username=?")) + .bind(next) + .bind(previous) + .execute(&mut *transaction) + .await?; + } transaction.commit().await?; Ok(()) } @@ -885,7 +938,7 @@ mod tests { "turnstile_login_enabled": false })) .unwrap(); - update_settings(&db, &input, None).await.unwrap(); + update_settings(&db, &input, None, None).await.unwrap(); assert_eq!( get_setting(&db, "turnstile_site_key") .await @@ -905,7 +958,7 @@ mod tests { "turnstile_site_key": "new-site", "turnstile_secret_key": "new-secret" })) .unwrap(); - update_settings(&db, &input, None).await.unwrap(); + update_settings(&db, &input, None, None).await.unwrap(); assert_eq!( get_setting(&db, "turnstile_secret_key") .await @@ -915,6 +968,19 @@ mod tests { ); } + #[test] + fn placeholder_translation_skips_quoted_spans() { + assert_eq!( + translate_placeholders("SELECT '?' , ? FROM t WHERE a=? AND b='it''s ?'"), + "SELECT '?' , $1 FROM t WHERE a=$2 AND b='it''s ?'" + ); + assert_eq!( + translate_placeholders("SELECT \"weird?col\" FROM t WHERE a=?"), + "SELECT \"weird?col\" FROM t WHERE a=$1" + ); + assert_eq!(translate_placeholders("SELECT 1"), "SELECT 1"); + } + #[tokio::test] async fn sqlite_file_pool_configures_every_connection() { let directory = tempfile::tempdir().unwrap(); diff --git a/backend/src/db/queries.rs b/backend/src/db/queries.rs old mode 100644 new mode 100755 index a81e500..b0d94a6 --- a/backend/src/db/queries.rs +++ b/backend/src/db/queries.rs @@ -18,6 +18,9 @@ const AGENT_REPORT_MAX_GPUS: usize = 32; const AGENT_REPORT_MAX_LATENCY_RESULTS: usize = 2048; const HISTORY_INSERT_BATCH_ROWS: usize = 500; const LATENCY_INSERT_BATCH_ROWS: usize = 500; +// Kept small enough that a 3-column batch stays under the legacy SQLite +// 999-variable limit as well as PostgreSQL's 65535-parameter limit. +const ASSIGNMENT_INSERT_BATCH_ROWS: usize = 200; const CLEANUP_DELETE_BATCH_ROWS: i64 = 1000; const CLEANUP_MAX_PASSES: usize = 10; const CLEANUP_TIME_BUDGET: std::time::Duration = std::time::Duration::from_secs(2); @@ -532,11 +535,19 @@ pub async fn reorder_servers(db: &Database, ids: &[String]) -> Result<()> { pub async fn agent_install_token(db: &Database, id: &str) -> Result> { let token = auth::random_token(32); + // The existence check and the insert share one transaction so a server + // deleted in between cannot turn a missing node into a foreign-key 500. + let mut transaction = if db.is_postgres() { + db.pool().begin().await? + } else { + db.pool().begin_with("BEGIN IMMEDIATE").await? + }; let exists = sqlx::query_scalar::<_, i64>(db.sql("SELECT COUNT(*) FROM servers WHERE id=?")) .bind(id) - .fetch_one(db.pool()) + .fetch_one(&mut *transaction) .await?; if exists == 0 { + transaction.rollback().await?; return Ok(None); } sqlx::query(db.sql( @@ -545,8 +556,9 @@ pub async fn agent_install_token(db: &Database, id: &str) -> Result String { + let mut placeholders = Vec::with_capacity(columns); + for _ in 0..columns { + if db.is_postgres() { + placeholders.push(format!("${}", *parameter_index)); + *parameter_index += 1; + } else { + placeholders.push("?".to_string()); + } + } + format!("({})", placeholders.join(",")) +} + async fn save_history_rows( db: &Database, transaction: &mut Transaction<'_, Any>, server_id: &str, rows: &[HistoryRow], ) -> Result<()> { + let merge_sql = history_merge_sql(); for batch in rows.chunks(HISTORY_INSERT_BATCH_ROWS) { let mut arguments = AnyArguments::default(); let mut parameter_index = 1_usize; @@ -993,7 +1023,7 @@ async fn save_history_rows( ON CONFLICT(server_id, timestamp) DO UPDATE SET {}", HISTORY_AGGREGATE_COLUMNS.join(","), value_groups.join(","), - history_merge_sql(), + merge_sql, ); sqlx::query_with(AssertSqlSafe(sql), arguments) .execute(&mut **transaction) @@ -1542,15 +1572,30 @@ async fn replace_task_servers( .bind(task_id) .execute(&mut **transaction) .await?; - for server_id in server_ids { - sqlx::query(db.sql( - "INSERT INTO latency_task_servers(task_id, server_id, assigned_at) VALUES (?, ?, ?)", - )) - .bind(task_id) - .bind(server_id) - .bind(existing.get(server_id).copied().unwrap_or(timestamp)) - .execute(&mut **transaction) - .await?; + for batch in server_ids.chunks(ASSIGNMENT_INSERT_BATCH_ROWS) { + let mut arguments = AnyArguments::default(); + let mut parameter_index = 1_usize; + let mut groups = Vec::with_capacity(batch.len()); + for server_id in batch { + groups.push(placeholder_group(db, &mut parameter_index, 3)); + let assigned_at = existing.get(server_id).copied().unwrap_or(timestamp); + arguments + .add(task_id.to_string()) + .map_err(|error| anyhow::anyhow!("无法编码拨测任务节点：{error}"))?; + arguments + .add(server_id.clone()) + .map_err(|error| anyhow::anyhow!("无法编码拨测任务节点：{error}"))?; + arguments + .add(assigned_at) + .map_err(|error| anyhow::anyhow!("无法编码拨测任务节点：{error}"))?; + } + let sql = format!( + "INSERT INTO latency_task_servers(task_id, server_id, assigned_at) VALUES {}", + groups.join(",") + ); + sqlx::query_with(AssertSqlSafe(sql), arguments) + .execute(&mut **transaction) + .await?; } Ok(()) } @@ -1740,10 +1785,24 @@ async fn replace_alert_servers( .bind(rule_id) .execute(&mut **transaction) .await?; - for server_id in server_ids { - sqlx::query(db.sql("INSERT INTO alert_rule_servers(rule_id, server_id) VALUES (?, ?)")) - .bind(rule_id) - .bind(server_id) + for batch in server_ids.chunks(ASSIGNMENT_INSERT_BATCH_ROWS) { + let mut arguments = AnyArguments::default(); + let mut parameter_index = 1_usize; + let mut groups = Vec::with_capacity(batch.len()); + for server_id in batch { + groups.push(placeholder_group(db, &mut parameter_index, 2)); + arguments + .add(rule_id.to_string()) + .map_err(|error| anyhow::anyhow!("无法编码告警规则节点：{error}"))?; + arguments + .add(server_id.clone()) + .map_err(|error| anyhow::anyhow!("无法编码告警规则节点：{error}"))?; + } + let sql = format!( + "INSERT INTO alert_rule_servers(rule_id, server_id) VALUES {}", + groups.join(",") + ); + sqlx::query_with(AssertSqlSafe(sql), arguments) .execute(&mut **transaction) .await?; } @@ -1769,17 +1828,37 @@ pub async fn evaluate_resource_rules( db: &Database, server_id: &str, ) -> Result> { - let rules = list_alert_rules(db).await?; let report_interval = sqlx::query_scalar::<_, i64>(db.sql("SELECT report_interval FROM servers WHERE id=?")) .bind(server_id) .fetch_optional(db.pool()) .await? .unwrap_or(60); + // Load only the rules that apply to this server instead of the full rule + // table plus every assignment row on every evaluation pass; this runs per + // persisted batch, so the previous full scan scaled badly with fleet size. + let rows = sqlx::query(db.sql( + "SELECT id, name, metric, threshold, duration_minutes, aggregation, all_servers \ + FROM alert_rules WHERE enabled=1 \ + AND (all_servers=1 OR id IN (SELECT rule_id FROM alert_rule_servers WHERE server_id=?)) \ + ORDER BY created_at", + )) + .bind(server_id) + .fetch_all(db.pool()) + .await?; let mut evaluations = Vec::new(); - for rule in rules.into_iter().filter(|rule| { - rule.enabled && (rule.all_servers || rule.server_ids.iter().any(|id| id == server_id)) - }) { + for row in rows { + let rule = AlertRuleView { + server_ids: Vec::new(), + id: row.try_get("id")?, + name: row.try_get("name")?, + metric: row.try_get("metric")?, + threshold: row.try_get("threshold")?, + duration_minutes: row.try_get("duration_minutes")?, + aggregation: row.try_get("aggregation")?, + all_servers: row.try_get::("all_servers")? != 0, + enabled: true, + }; let (average_expression, minimum_expression) = match rule.metric.as_str() { "cpu" => ("cpu", "cpu_min"), "memory" => ("memory_avg", "memory_min"), @@ -2006,18 +2085,6 @@ pub async fn set_totp_enabled(db: &Database, username: &str, enabled: bool) -> R Ok(result.rows_affected() > 0) } -pub async fn rename_totp_user(db: &Database, old: &str, new: &str) -> Result<()> { - if old == new { - return Ok(()); - } - sqlx::query(db.sql("UPDATE admin_2fa SET username=? WHERE username=?")) - .bind(new) - .bind(old) - .execute(db.pool()) - .await?; - Ok(()) -} - pub async fn create_remote_task( db: &Database, server_id: &str, @@ -2290,16 +2357,40 @@ pub async fn set_active_theme(db: &Database, id: &str) -> Result { } pub async fn delete_theme(db: &Database, id: &str) -> Result { + // Deleting the theme and resetting active_theme_id must be atomic; a failure + // between them would leave the active theme pointing at a deleted row. + let mut transaction = if db.is_postgres() { + db.pool().begin().await? + } else { + db.pool().begin_with("BEGIN IMMEDIATE").await? + }; let result = sqlx::query(db.sql("DELETE FROM themes WHERE id=?")) .bind(id) - .execute(db.pool()) + .execute(&mut *transaction) .await?; - if result.rows_affected() > 0 - && super::get_setting(db, "active_theme_id").await?.as_deref() == Some(id) - { - super::set_setting(db, "active_theme_id", "builtin-nodeflare-glass").await?; + if result.rows_affected() == 0 { + transaction.rollback().await?; + return Ok(false); } - Ok(result.rows_affected() > 0) + let active = sqlx::query_scalar::<_, String>( + db.sql("SELECT value FROM settings WHERE key=? AND value=?"), + ) + .bind("active_theme_id") + .bind(id) + .fetch_optional(&mut *transaction) + .await?; + if active.is_some() { + sqlx::query(db.sql( + "INSERT INTO settings(key, value) VALUES (?, ?) \ + ON CONFLICT(key) DO UPDATE SET value=excluded.value", + )) + .bind("active_theme_id") + .bind("builtin-nodeflare-glass") + .execute(&mut *transaction) + .await?; + } + transaction.commit().await?; + Ok(true) } pub async fn create_theme_preview(db: &Database, theme_id: &str) -> Result { diff --git a/backend/src/main.rs b/backend/src/main.rs old mode 100644 new mode 100755 index 8cfb807..f2db0e2 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -45,6 +45,7 @@ pub struct AppState { pub restart_tx: watch::Sender, pub login_attempts: security::AttemptLimiter, pub sensitive_attempts: security::AttemptLimiter, + pub turnstile_attempts: security::AttemptLimiter, pub password_verifications: Arc, } @@ -186,6 +187,11 @@ async fn main() -> Result<()> { std::time::Duration::from_secs(5 * 60), std::time::Duration::from_secs(10 * 60), ), + turnstile_attempts: security::AttemptLimiter::new( + 10, + std::time::Duration::from_secs(5 * 60), + std::time::Duration::from_secs(5 * 60), + ), password_verifications: Arc::new(Semaphore::new(4)), }); diff --git a/backend/src/notify.rs b/backend/src/notify.rs old mode 100644 new mode 100755 index 897b414..b80fa7b --- a/backend/src/notify.rs +++ b/backend/src/notify.rs @@ -299,7 +299,17 @@ async fn send( } let value: Value = serde_json::from_slice(&body)?; if value.get("ok").and_then(Value::as_bool) != Some(true) { - anyhow::bail!("Telegram rejected the message"); + // Surface Telegram's error_code/description (e.g. "chat not found", + // "bot was blocked by the user") so admins can act on the failure; + // neither field contains the bot token. + let description = value + .get("description") + .and_then(Value::as_str) + .unwrap_or("no description"); + match value.get("error_code").and_then(Value::as_i64) { + Some(code) => anyhow::bail!("Telegram rejected the message (error {code}): {description}"), + None => anyhow::bail!("Telegram rejected the message: {description}"), + } } Ok(()) } diff --git a/backend/src/routes/admin.rs b/backend/src/routes/admin.rs old mode 100644 new mode 100755 index 6b773a6..d2a2f0f --- a/backend/src/routes/admin.rs +++ b/backend/src/routes/admin.rs @@ -192,12 +192,17 @@ pub async fn settings_patch( if !username_changed { input.admin_username = None; } - crate::db::update_settings(&state.db, &input, password_hash.as_deref()) - .await - .map_err(ApiResponse::internal)?; - crate::db::queries::rename_totp_user(&state.db, &user.username, &next_username) - .await - .map_err(ApiResponse::internal)?; + // The TOTP rename is folded into the same transaction as the settings + // update so a mid-way failure cannot leave 2FA looking up a stale username. + let totp_rename = username_changed.then(|| (user.username.as_str(), next_username.as_str())); + crate::db::update_settings( + &state.db, + &input, + password_hash.as_deref(), + totp_rename, + ) + .await + .map_err(ApiResponse::internal)?; let credentials_changed = password_hash.is_some() || username_changed; let token = if credentials_changed { Some( @@ -707,8 +712,17 @@ pub async fn database_migrate( .await .map_err(|error| ApiResponse::unprocessable(format!("迁移失败：{error}")))?; let database = target.stats().await.map_err(ApiResponse::internal)?; - crate::config::update_database_url(&state.config_path, &target_url) - .map_err(ApiResponse::internal)?; + crate::config::update_database_url(&state.config_path, &target_url).map_err(|error| { + // The copy already succeeded, so a bare 500 would make the admin think + // nothing happened and possibly retry against a half-switched setup. + tracing::error!(%error, "database migration copied data but the configuration update failed"); + ApiResponse::error( + StatusCode::INTERNAL_SERVER_ERROR, + format!( + "目标数据库已写入数据，但配置更新失败（{error}）；请手动修改 database_url 后重启面板" + ), + ) + })?; state.database_activity.require_restart(); Ok(Json(serde_json::json!({ diff --git a/backend/src/routes/auth.rs b/backend/src/routes/auth.rs old mode 100644 new mode 100755 index bc7238d..6c6f267 --- a/backend/src/routes/auth.rs +++ b/backend/src/routes/auth.rs @@ -174,6 +174,15 @@ pub async fn verify_turnstile( headers: HeaderMap, Json(input): Json, ) -> Result { + // This endpoint is public and each call triggers an outbound request to + // Cloudflare, so rate-limit per IP to stop it being used as an amplifier. + let client_ip = client_ip(&headers, peer, &state.config.trusted_proxies); + if let Some(seconds) = state.turnstile_attempts.retry_after(&client_ip) { + return Err(ApiResponse::error( + StatusCode::TOO_MANY_REQUESTS, + format!("人机验证尝试过多，请在 {seconds} 秒后重试"), + )); + } let settings = crate::db::load_settings(&state.db) .await .map_err(ApiResponse::internal)?; @@ -181,7 +190,6 @@ pub async fn verify_turnstile( return Err(ApiResponse::bad_request("公开仪表盘未启用人机验证")); } let host = hostname(&headers).ok_or_else(|| ApiResponse::bad_request("请求主机名无效"))?; - let client_ip = client_ip(&headers, peer, &state.config.trusted_proxies); if !crate::turnstile::verify( &state.http, &input.token, @@ -192,8 +200,10 @@ pub async fn verify_turnstile( ) .await { + state.turnstile_attempts.record_failure(&client_ip); return Err(ApiResponse::forbidden("Cloudflare 人机验证失败，请重试")); } + state.turnstile_attempts.clear(&client_ip); let proof = crate::db::create_dashboard_proof(&state.db) .await .map_err(ApiResponse::internal)?; diff --git a/backend/src/routes/public.rs b/backend/src/routes/public.rs old mode 100644 new mode 100755 index cc0a426..cdb6c3d --- a/backend/src/routes/public.rs +++ b/backend/src/routes/public.rs @@ -199,24 +199,31 @@ async fn require_public_server(state: &AppState, server_id: &str) -> Result<(), Ok(()) } +/// Fields that must never reach the unauthenticated dashboard. +/// +/// This is a deny list, so a new `ServerView` field is public until added here; +/// prefer inverting it into an explicit allow list when the public projection is +/// next reworked (it needs the frontend's field usage verified first). +const PRIVATE_SERVER_FIELDS: [&str; 13] = [ + "hidden", + "last_ip", + "ip_v4", + "ip_v6", + "network_interface", + "reset_day", + "report_interval", + "collect_interval", + "rx_correction", + "tx_correction", + "agent_mirror", + "offline_notify_disabled", + "auto_update", +]; + fn public_server(server: crate::models::ServerView) -> serde_json::Value { let mut value = serde_json::to_value(server).unwrap_or_else(|_| serde_json::json!({})); if let Some(object) = value.as_object_mut() { - for key in [ - "hidden", - "last_ip", - "ip_v4", - "ip_v6", - "network_interface", - "reset_day", - "report_interval", - "collect_interval", - "rx_correction", - "tx_correction", - "agent_mirror", - "offline_notify_disabled", - "auto_update", - ] { + for key in PRIVATE_SERVER_FIELDS { object.remove(key); } } diff --git a/backend/src/routes/remote.rs b/backend/src/routes/remote.rs old mode 100644 new mode 100755 index 3909b87..cb84115 --- a/backend/src/routes/remote.rs +++ b/backend/src/routes/remote.rs @@ -139,6 +139,11 @@ pub async fn get_task( State(state): State>, Path(id): Path, ) -> Result { + // Remote task IDs are UUIDs; validate the path parameter for consistency + // with the other ID endpoints instead of sending arbitrary input to the DB. + if id.len() > 80 || uuid::Uuid::parse_str(&id).is_err() { + return Err(ApiResponse::bad_request("任务 ID 无效")); + } let task = crate::db::queries::remote_task(&state.db, &id) .await .map_err(ApiResponse::internal)? diff --git a/backend/src/websocket/agent.rs b/backend/src/websocket/agent.rs old mode 100644 new mode 100755 index 4b3dc1d..ba3bb8f --- a/backend/src/websocket/agent.rs +++ b/backend/src/websocket/agent.rs @@ -20,6 +20,24 @@ use tokio::sync::mpsc; const MAX_AGENT_MESSAGE_BYTES: usize = 2 * 1024 * 1024; const MAX_TASK_RESULT_BYTES: usize = 1024 * 1024; +/// Static identity fields the public dashboard already receives through +/// `/api/bootstrap`, so the live stream omits them. +/// +/// This is a deny list, so a new `Report` field is broadcast until added here; +/// prefer inverting it into an explicit allow list when the sample shape is next +/// reworked (verify the frontend's usage first). +const LIVE_SAMPLE_PRIVATE_FIELDS: [&str; 10] = [ + "timestamp", + "cpu_model", + "os", + "kernel", + "arch", + "virtualization", + "gpu_model", + "agent_version", + "ip_v4", + "ip_v6", +]; #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -406,18 +424,7 @@ fn broadcast_reports(state: &AppState, identity: &AgentIdentity, reports: &[Agen .map(|report| { let mut data = serde_json::to_value(report).unwrap_or_else(|_| serde_json::json!({})); if let Some(object) = data.as_object_mut() { - for key in [ - "timestamp", - "cpu_model", - "os", - "kernel", - "arch", - "virtualization", - "gpu_model", - "agent_version", - "ip_v4", - "ip_v6", - ] { + for key in LIVE_SAMPLE_PRIVATE_FIELDS { object.remove(key); } } diff --git a/docs/README.en.md b/docs/README.en.md new file mode 100755 index 0000000..acdafd5 --- /dev/null +++ b/docs/README.en.md @@ -0,0 +1,273 @@ +# NodeFlare + +NodeFlare is a lightweight, self-hosted server monitoring panel: view server status in a web UI while a small agent collects and pushes telemetry. It supports live metrics, TCP/ICMP probing, Telegram alerts, remote command execution, TOTP two-factor authentication, theme customization, and SQLite / PostgreSQL backup, restore, and online migration. + +Chinese: [README.md](../README.md) + +- [Features](#features) +- [Supported platforms](#supported-platforms) +- [Install the server](#install-the-server) +- [Install the agent](#install-the-agent) +- [Metrics and sampling](#metrics-and-sampling) +- [Configuration](#configuration) +- [Default paths and logs](#default-paths-and-logs) +- [Reverse proxy](#reverse-proxy) +- [Database and backups](#database-and-backups) +- [Development](#development) +- [Troubleshooting](#troubleshooting) + +## Features + +**Monitoring** + +- CPU, memory, and network speed are sampled every second and uploaded in compressed batches every 3 seconds by default; slower metrics such as disks, GPU, and connection counts are cached separately +- Latency probing: TCP and ICMP tasks assigned per node, with per-carrier (China Telecom / Mobile / Unicom) display +- Public dashboard in Simplified Chinese and English; site name, announcement, logo, background image, and visible sections are configurable +- Theme store: built-in themes, one-click install from the repository, local ZIP upload, and per-theme settings + +**Alerts** + +- Resource threshold rules for CPU / memory / disk / inbound / outbound, using averages or continuous breach with a configurable duration +- Offline, expiry, and traffic alerts delivered through Telegram with a customizable message template + +**Management** + +- Node grouping, tags, region, billing cycle, price, expiry date, traffic quota, and reset day +- Remote command execution: requires TOTP; a command runs for up to 10 minutes and is not cancelled when you leave the page +- Daily exchange-rate snapshot for multi-currency pricing + +**Security** + +- TOTP two-factor authentication; Cloudflare Turnstile (can protect admin login and the public dashboard independently) +- Login rate limiting, session management (list login devices and revoke them), configurable session lifetime +- Listens on `127.0.0.1` by default; the agent connects outbound, so no inbound port is required; the agent token can be passed as a file or an environment variable + +**Data** + +- SQLite and PostgreSQL, with online migration between them +- One-click ZIP backup / restore, space reclamation, and automatic cleanup of history by retention days + +## Supported platforms + +| Role | Platform | Architecture | +| --- | --- | --- | +| Server | Linux | x64 / ARM64 | +| Server | Windows | x64 | +| Server | macOS | ARM64 | +| Server | FreeBSD 13+ | x64 / ARM64 | +| Agent | Linux / Windows / macOS / FreeBSD | Same as above (macOS is Apple Silicon only) | + +The installer detects the service manager (systemd / OpenRC / launchd / FreeBSD rc / Windows scheduled task) and registers the panel to start on boot. + +## Install the server + +The installer downloads the latest release and registers the system service. + +Linux / macOS: + +```bash +curl -fsSL https://raw.githubusercontent.com/elysia62/NodeFlare/main/install.sh | sudo sh +``` + +FreeBSD: + +```sh +fetch -qo - https://raw.githubusercontent.com/elysia62/NodeFlare/main/install.sh | sudo sh +``` + +Windows PowerShell (administrator): + +```powershell +Invoke-WebRequest -UseBasicParsing https://raw.githubusercontent.com/elysia62/NodeFlare/main/install.ps1 -OutFile "$env:TEMP\nodeflare-install.ps1" +Unblock-File "$env:TEMP\nodeflare-install.ps1" +& "$env:TEMP\nodeflare-install.ps1" +``` + +The first install asks for the admin username, password, listening port (default 2206), and database URL (SQLite by default). The panel listens on `127.0.0.1:2206`, so open `http://127.0.0.1:2206/admin/login` locally; public access requires an HTTPS reverse proxy, see [Reverse proxy](#reverse-proxy). + +The admin password is only needed to initialize the database for the first time; it is cleared from the configuration afterwards. + +### Installer options + +| Command | Description | +| --- | --- | +| `sudo sh install.sh` | Interactive menu | +| `sudo sh install.sh --install` | Install or update | +| `sudo sh install.sh --status` | Show service status | +| `sudo sh install.sh --restart` | Restart the service | +| `sudo sh install.sh --uninstall` | Uninstall, keeping configuration and data | +| `sudo sh install.sh --uninstall --purge` | Uninstall and delete configuration and data | + +On Windows the equivalent options are `-Install` / `-Status` / `-Restart` / `-Uninstall [-Purge]`. + +To update, run the installer again; configuration and data are preserved. Installs and updates verify the release digest and roll back to the previous version on failure. + +## Install the agent + +Create a node on the "Servers" page in the admin panel and run the command shown in the dialog. The agent connects outbound to the panel, so no inbound port is required. + +Linux: + +```bash +curl -fsSL https://raw.githubusercontent.com/elysia62/NodeFlare/main/agent/agent.sh \ + | sudo sh -s -- -e 'https://nodeflare.example.com' -t 'Agent Token' +``` + +Per-platform scripts: + +| Platform | Script | Service | +| --- | --- | --- | +| Linux | `agent/agent.sh` | systemd / OpenRC | +| macOS (Apple Silicon) | `agent/install-macos.sh` | launchd | +| FreeBSD | `agent/install-freebsd.sh` | rc.d | +| Windows | `agent/install.ps1` | Scheduled task | + +### Agent options + +| Option | Description | +| --- | --- | +| `-e` | NodeFlare endpoint (required) | +| `-t` | Agent token (required) | +| `-i` | Initial history interval in seconds, 15–3600 (default 60) | +| `-m` | Optional GitHub download mirror prefix, e.g. `https://ghproxy.net` | +| `--update` | Update the agent, reusing the saved endpoint and token, verifying the digest and rolling back on failure | +| `--status` | Show agent status | +| `--uninstall` | Uninstall the agent | + +On Windows the equivalents are `-Endpoint` / `-Token` / `-Interval` / `-Mirror` and `-Update` / `-Status` / `-Uninstall`. + +> **About the token**: `-t` is visible in the process list. The agent also accepts the `NODEFLARE_AGENT_TOKEN` environment variable and `--token-file `. The Linux installer writes the token to a `0600` `agent.env` file that the service unit reads. + +## Metrics and sampling + +CPU, memory, and network speed are sampled every second and uploaded in compressed batches every 3 seconds by default. Server cards show one real sample per second, giving roughly 2–3 seconds of display latency on a healthy connection; after a reconnect, backlogged samples are skipped. Slower metrics such as disk capacity and GPU are cached separately, and history is aggregated by each node's configured interval. A longer real-time upload interval reduces how often cards update. + +Linux memory accounting follows the Komari convention: used memory is `MemTotal - MemFree - Cached - SReclaimable - Buffers + Shmem`, and used swap subtracts `SwapCached`. File cache is not counted as used memory and shared memory is; the panel reports whole-machine memory, not NodeFlare's own usage. + +## Configuration + +The configuration file location is listed in [Default paths and logs](#default-paths-and-logs); a full example is in [`backend/config.example.toml`](../backend/config.example.toml). + +| Key | Description | +| --- | --- | +| `database_url` | `sqlite://nodeflare.db` or a PostgreSQL URL such as `postgres://user:password@127.0.0.1:5432/nodeflare?sslmode=disable` | +| `bind_addr` | Listen address, `127.0.0.1:2206` by default | +| `admin_username` | Admin username | +| `admin_password` | Admin password (8–128 characters), only needed for the first database initialization and cleared afterwards | +| `trusted_proxies` | Trusted reverse proxy IP / CIDR list; only `X-Forwarded-For` from these networks is trusted | +| `turnstile_site_key` / `turnstile_secret_key` | Turnstile keys; leave empty to disable | +| `session_ttl_hours` | Session lifetime in hours (1–2160), 168 by default | +| `frontend_dir` / `admin_frontend_dir` | Frontend asset directories; relative paths resolve against the configuration file | +| `theme_dir` | Theme extraction directory, `themes` under the configuration directory by default | + +A few settings can be overridden on the command line: `nodeflare --config --bind
--database `. When started with `--database`, the panel's online migration feature is disabled. + +## Default paths and logs + +| Platform | Program | Configuration and data | +| --- | --- | --- | +| Linux | `/opt/nodeflare` | `/etc/nodeflare` | +| Windows | `%ProgramFiles%\NodeFlare` | `%ProgramData%\NodeFlare\Server` | +| macOS | `/usr/local/libexec/nodeflare` | `/Library/Application Support/NodeFlare/Server` | +| FreeBSD | `/usr/local/libexec/nodeflare` | `/var/db/nodeflare/server` | + +Agent on Linux: program at `/opt/nodeflare/agent`, configuration and state in `/etc/nodeflare/agent`. The SQLite file lives in the configuration directory. + +Logs: + +- Linux (systemd): `journalctl -u nodeflare -f`; the agent is `journalctl -u nodeflare-agent -f` +- Linux (OpenRC): `rc-service nodeflare status` +- macOS: `/var/log/nodeflare.log` +- Windows: `Get-ScheduledTaskInfo -TaskName nodeflare` + +## Reverse proxy + +Expose the panel through an HTTPS reverse proxy and add the proxy address to `trusted_proxies`, otherwise every visitor is treated as the same IP and the session cookie is not marked `Secure`: + +```toml +trusted_proxies = ["127.0.0.1/32", "::1/128"] +``` + +nginx: + +```nginx +server { + listen 443 ssl; + server_name nodeflare.example.com; + + ssl_certificate /etc/letsencrypt/live/nodeflare.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/nodeflare.example.com/privkey.pem; + + location / { + proxy_pass http://127.0.0.1:2206; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; # WebSocket live data + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} +``` + +Caddy: + +``` +nodeflare.example.com { + reverse_proxy 127.0.0.1:2206 +} +``` + +## Database and backups + +The "Database" page shows usage, reclaims space, exports and restores ZIP backups, and migrates between SQLite and PostgreSQL. A backup contains settings, nodes, history, notifications, themes, tasks, and security configuration, and requires TOTP or the password. Limits are a 512 MiB ZIP, 4 GiB extracted, 16,384 entries, and 32 MiB per theme file. Migration overwrites the target database, updates the connection string automatically, keeps agent tokens, and takes effect after a restart. History is kept for 30 days by default. + +## Development + +```bash +git clone https://github.com/elysia62/NodeFlare.git +cd NodeFlare +bun install --frozen-lockfile +cp backend/config.example.toml backend/config.toml +./dev.sh +``` + +`dev.sh` builds the frontend and starts the backend with `cargo run`. `start.sh` builds release binaries and prefers `/etc/nodeflare/config.toml`, which can be overridden with `NODEFLARE_CONFIG`. + +Repository layout: + +| Directory | Contents | +| --- | --- | +| `backend/` | Server (Rust / Axum / SQLx): `src/routes` endpoints, `src/db` data layer, `src/websocket` live channels, `migrations/` schema | +| `agent/` | Agent source (collection / upload / remote execution / self-update) and per-platform install scripts | +| `shared/` | Telemetry protocol shared by the agent and the server (serialization + compression) | +| `frontend/` | Frontend (React + Vite): `src/components`, `src/styles` | +| `scripts/` | Build, version resolution, and smoke-test scripts | +| `docs/` | English README and systemd service unit | + +Tests and builds: + +```bash +bun test --cwd frontend +cargo test --locked --manifest-path backend/Cargo.toml +cargo test --locked --manifest-path agent/Cargo.toml +bun run build +``` + +Smoke test (start the panel first): + +```bash +MONITOR_ADMIN_USERNAME=admin MONITOR_ADMIN_PASSWORD='your-password' bun run test:smoke +``` + +## Troubleshooting + +- **The service fails to start**: check the logs (see [Default paths and logs](#default-paths-and-logs)); usually the port is in use or the database URL is wrong. +- **Unknown listening port**: check `bind_addr` in the configuration file, `127.0.0.1:2206` by default. +- **The agent shows offline**: verify the agent can reach the panel (`curl -I https://your-panel`), check the token, and check the system clock, since reporting and latency rely on clock calibration. +- **Nodes are missing from the public dashboard**: make sure the node is not hidden and that "Public dashboard" is enabled in site settings. +- **Backup export reports the size limit**: shorten the history retention or clear history, then export again. + +## License + +MIT diff --git a/deploy/nodeflare.service b/docs/nodeflare.service similarity index 100% rename from deploy/nodeflare.service rename to docs/nodeflare.service diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx old mode 100644 new mode 100755 index 8670667..abf8aec --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,47 +1,18 @@ -import { KeyRound, Megaphone, Moon, Search, Sun, UserCircle } from "lucide-react"; -import { lazy, Suspense, useMemo, useState, type FormEvent } from "react"; +import { Megaphone, Moon, Search, Sun, UserCircle } from "lucide-react"; +import { lazy, Suspense, useEffect, useMemo, useState } from "react"; import { demoMode, useApp } from "./AppProvider"; +import { LoginForm } from "./components/LoginForm"; import { NodeCard } from "./components/NodeCard"; import { SiteLogo } from "./components/SiteLogo"; import { StatsBar } from "./components/StatsBar"; import { TurnstileWidget } from "./components/TurnstileWidget"; -import { ApiError } from "./api"; import { ui } from "./locale"; const NodeDetails = lazy(() => import("./components/NodeDetails").then((module) => ({ default: module.NodeDetails }))); function LoginGate() { const { config, dark, error, setError, login } = useApp(); - const [username, setUsername] = useState(""); - const [password, setPassword] = useState(""); - const [turnstileToken, setTurnstileToken] = useState(""); - const [turnstileReset, setTurnstileReset] = useState(0); - const [totpCode, setTotpCode] = useState(""); - const [totpChallenge, setTotpChallenge] = useState(false); - const [busy, setBusy] = useState(false); const locale = config.locale; - const totpRequired = config.totp_login_enabled || totpChallenge; - - async function submit(event: FormEvent) { - event.preventDefault(); - if (busy) return; - setBusy(true); - setError(""); - try { - await login(username, password, turnstileToken, totpCode); - setPassword(""); - setTurnstileToken(""); - setTotpCode(""); - setTotpChallenge(false); - } catch (reason) { - if (reason instanceof ApiError && reason.status === 428) setTotpChallenge(true); - setError(reason instanceof Error ? reason.message : ui(locale, "登录失败", "Unable to sign in")); - setTurnstileToken(""); - setTurnstileReset((value) => value + 1); - } finally { - setBusy(false); - } - } return (
@@ -50,14 +21,17 @@ function LoginGate() {

{ui(locale, "登录仪表盘", "Sign in to dashboard")}

{ui(locale, "此仪表盘仅限登录后访问", "This dashboard requires an administrator sign-in")}

-
void submit(event)}> - - - {totpRequired ? : null} - {config.turnstile_login_enabled ?
: null} - {error ?

{error}

: null} - -
+
); } @@ -86,7 +60,7 @@ function VerificationGate() {

{ui(locale, "访问验证", "Access verification")}

{config.site_name}

{config.turnstile_site_key - ? void run(token)} onError={setError} /> + ? void run(token)} onError={setError} locale={locale} /> :

{ui(locale, "Turnstile 尚未正确配置", "Turnstile is not configured")}

} {busy ? {ui(locale, "正在验证", "Verifying")} : null} {error ?

{error}

: null} @@ -100,17 +74,21 @@ function HomeView() { const [query, setQuery] = useState(""); const [group, setGroup] = useState("__all__"); const locale = config.locale; + const fallbackGroup = ui(locale, "默认", "Default"); const groups = useMemo( - () => ["__all__", ...Array.from(new Set(servers.map((server) => server.group_name || "默认")))], - [servers], + () => ["__all__", ...Array.from(new Set(servers.map((server) => server.group_name || fallbackGroup)))], + [fallbackGroup, servers], ); const visible = useMemo(() => servers.filter((server) => { const text = `${server.name} ${server.region} ${server.tags} ${server.group_name}`.toLowerCase(); - const groupMatches = !config.show_groups || group === "__all__" || (server.group_name || "默认") === group; + const groupMatches = !config.show_groups || group === "__all__" || (server.group_name || fallbackGroup) === group; const queryMatches = !config.show_search || text.includes(query.trim().toLowerCase()); return groupMatches && queryMatches; - }), [config.show_groups, config.show_search, group, query, servers]); + }), [config.show_groups, config.show_search, fallbackGroup, group, query, servers]); + + // A locale change renames the fallback group; drop the stale selection. + useEffect(() => setGroup("__all__"), [fallbackGroup]); return (
@@ -125,7 +103,7 @@ function HomeView() {
) : visible.length ? (
- {visible.map((server) => openServer(server)} />)} + {visible.map((server) => )}
) : !error ? (
{servers.length ? ui(locale, "没有匹配的节点", "No matching servers") : ui(locale, "尚未添加节点", "No servers added")}
diff --git a/frontend/src/AppProvider.tsx b/frontend/src/AppProvider.tsx old mode 100644 new mode 100755 index 84e2e87..13ff1bc --- a/frontend/src/AppProvider.tsx +++ b/frontend/src/AppProvider.tsx @@ -6,8 +6,10 @@ import { createLivePlayback, mergeServerLive, pruneLiveMetrics, + type LiveMetrics, type LiveMetricsMap, } from "./live"; +import { isOnline } from "./format"; import { ui } from "./locale"; import { useFavicon, useStoredAppearance, useSystemDark } from "./hooks/useBrowserAppearance"; import { resolveBackground, themeToggle } from "./theme"; @@ -85,6 +87,13 @@ export function AppProvider({ children }: { children: ReactNode }) { const systemDark = useSystemDark(); const liveConnectedRef = useRef(false); const serversRef = useRef([]); + const cardCacheRef = useRef(new Map()); + const liveServersRef = useRef([]); const localeRef = useRef(config.locale); const reloadQueueRef = useRef | null>(null); @@ -142,7 +151,7 @@ export function AppProvider({ children }: { children: ReactNode }) { }, []); const login = useCallback(async (username: string, password: string, turnstileToken: string, totpCode: string) => { - const derived = await derivePassword(password, config.password_client_salt); + const derived = await derivePassword(password, config.password_client_salt, config.locale); await api.login(username.trim(), derived, turnstileToken, totpCode); setAccess("ok"); await reload(); @@ -286,14 +295,42 @@ export function AppProvider({ children }: { children: ReactNode }) { return () => window.removeEventListener("popstate", onPop); }, []); - const metricServers = useMemo( - () => servers.map((server) => mergeServerLive(server, liveMetrics[server.id], 0, config.offline_threshold_seconds)), - [config.offline_threshold_seconds, liveMetrics, servers], - ); - const liveServers = useMemo( - () => metricServers.map((server) => mergeServerLive(server, undefined, clockNow, config.offline_threshold_seconds)), - [clockNow, config.offline_threshold_seconds, metricServers], - ); + // Merge live metrics into servers with identity preservation: a server's object + // only changes when its own data or online status changes, and the array itself + // stays referentially stable on quiet ticks. Memoized cards then skip re-renders + // for nodes that reported nothing new, while a silent node still flips to + // offline within a second of crossing the threshold. + const liveServers = useMemo(() => { + const nowSec = clockNow / 1000; + const cache = cardCacheRef.current; + const next = servers.map((server) => { + const live = liveMetrics[server.id]; + const merged = mergeServerLive(server, live); + const online = isOnline(merged, config.offline_threshold_seconds, nowSec); + const cached = cache.get(server.id); + if (cached && cached.server === server && cached.live === live) { + if (cached.online === online) return cached.result; + // Online status flipped without new data: mint a fresh object so the + // card re-renders into the offline/online state. + const result = { ...merged }; + cache.set(server.id, { server, live, online, result }); + return result; + } + cache.set(server.id, { server, live, online, result: merged }); + return merged; + }); + if (cache.size > next.length) { + const ids = new Set(next.map((server) => server.id)); + for (const id of cache.keys()) { + if (!ids.has(id)) cache.delete(id); + } + } + const previous = liveServersRef.current; + const stable = previous.length === next.length + && next.every((server, index) => server === previous[index]); + liveServersRef.current = stable ? previous : next; + return liveServersRef.current; + }, [clockNow, config.offline_threshold_seconds, liveMetrics, servers]); useEffect(() => { if (!configReady) return; diff --git a/frontend/src/admin-main.tsx b/frontend/src/admin-main.tsx old mode 100644 new mode 100755 index b761c61..f9f7edf --- a/frontend/src/admin-main.tsx +++ b/frontend/src/admin-main.tsx @@ -3,6 +3,7 @@ import { createRoot } from "react-dom/client"; import { api } from "./api"; import { AdminPanel } from "./components/AdminPanel"; import { useFavicon, useStoredAppearance, useSystemDark } from "./hooks/useBrowserAppearance"; +import { ui } from "./locale"; import type { Config } from "./types"; import "./styles/admin.css"; @@ -13,6 +14,8 @@ function AdminApp() { const systemDark = useSystemDark(); const dark = appearance ? appearance === "dark" : config?.default_theme === "system" || !config ? systemDark : config.default_theme === "dark"; useFavicon(config?.logo_url); + // Before the config arrives there is no configured locale; fall back to the browser. + const locale = config?.locale ?? (navigator.language.toLowerCase().startsWith("zh") ? "zh-CN" : "en"); async function loadConfig() { try { @@ -20,7 +23,7 @@ function AdminApp() { setConfig(next.config); setError(""); } catch (reason) { - setError(reason instanceof Error ? reason.message : "无法加载管理面板"); + setError(reason instanceof Error ? reason.message : ui(locale, "无法加载管理面板", "Unable to load the admin panel")); } } @@ -30,11 +33,12 @@ function AdminApp() { document.documentElement.style.colorScheme = dark ? "dark" : "light"; }, [dark]); useEffect(() => { - document.title = config ? `管理面板 · ${config.site_name}` : "管理面板"; - }, [config]); + document.documentElement.lang = locale; + document.title = config ? ui(locale, `管理面板 · ${config.site_name}`, `Admin · ${config.site_name}`) : ui(locale, "管理面板", "Admin"); + }, [config, locale]); - if (error) return
{error}
; - if (!config) return
正在加载管理面板
; + if (error) return
{error}
; + if (!config) return
{ui(locale, "正在加载管理面板", "Loading the admin panel")}
; return { const next = dark ? "light" : "dark"; setAppearance(next); diff --git a/frontend/src/components/AdminPanel.tsx b/frontend/src/components/AdminPanel.tsx old mode 100644 new mode 100755 index 2b15378..bd83607 --- a/frontend/src/components/AdminPanel.tsx +++ b/frontend/src/components/AdminPanel.tsx @@ -1,35 +1,19 @@ import { - AlertTriangle, - Info, ArrowLeft, - ArrowRightLeft, ChevronDown, ChevronUp, CircleAlert, CircleCheck, - Coins, Copy, - Database, Download, Eye, GripVertical, - KeyRound, LogOut, - MonitorSmartphone, Moon, - Palette, Pencil, Plus, - Power, - RadioTower, - RotateCw, Save, - Search, - ServerCog, - ShieldCheck, - SlidersHorizontal, Sun, - Terminal, Trash2, Upload, Check, @@ -37,128 +21,43 @@ import { } from "lucide-react"; import { ChangeEvent, DragEvent, FormEvent, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ADMIN_UNAUTHORIZED_EVENT, api, ApiError } from "../api"; +import { ui } from "../locale"; import { adminTabFromPath, adminTabPaths, canonicalAdminPath, type AdminTab } from "../adminRoutes"; import { formatBytes, formatByteSize, parseByteSize } from "../format"; import { derivePassword } from "../password"; import { hasActiveRemoteTasks, isRemoteTaskActive, REMOTE_TASK_POLL_INTERVAL_MS, REMOTE_TASK_POLL_TIMEOUT_MS } from "../refresh"; -import { ASSET_CURRENCIES, type AdminServer, type Config, type DatabaseMigrationResult, type DatabaseStats, type ExchangeRates, type LoginSession, type RemoteTask, type ServerInput, type Settings, type Theme, type ThemeSettingField, type ThemeSettingsSchema, type ThemeSettingValue, type TotpSetup, type TotpStatus } from "../types"; +import { ASSET_CURRENCIES, type AdminServer, type Config, type DatabaseMigrationResult, type DatabaseStats, type ExchangeRates, type LoginSession, type RemoteTask, type ServerInput, type Settings, type Theme, type ThemeSettingsSchema, type ThemeSettingValue, type TotpSetup, type TotpStatus } from "../types"; import { Checkbox } from "./Checkbox"; -import { TurnstileWidget } from "./TurnstileWidget"; +import { LoginForm } from "./LoginForm"; import { useDialog } from "./useDialog"; import { SiteLogo } from "./SiteLogo"; import { LatencyManager } from "./LatencyManager"; -import { AlertRuleManager } from "./AlertRuleManager"; -import { TelegramSettings } from "./TelegramSettings"; -import { PasswordInput } from "./PasswordInput"; import { useVerification } from "./useVerification"; import { Flag } from "./Flag"; -import pkg from "../../package.json"; +import { AboutTab } from "./admin/AboutTab"; +import { RemoteTab } from "./admin/RemoteTab"; +import { SettingsTabs } from "./admin/SettingsTabs"; +import { Toggle } from "./admin/controls"; +import { + AGENT_SCRIPT_BASE, + adminNavigation, + adminPages, + billingCycles, + copyText, + describeLoginDevice, + emptyServer, + formatDate, + settingPatch, + shellLiteral, + powershellLiteral, + toInput, + waitForDatabaseSwitch, + type AgentInstallInfo, + type AgentPlatform, + type ThemeSourceMode, +} from "./admin/shared"; -const VERSION = import.meta.env.VITE_NODEFLARE_VERSION || pkg.version; -const AGENT_SCRIPT_BASE = "https://raw.githubusercontent.com/elysia62/NodeFlare/main/agent"; - -type AgentPlatform = "linux" | "windows" | "macos" | "freebsd"; -type ThemeSourceMode = "repository" | "upload"; - -interface AgentInstallInfo { - agent_token: string; - agent_mirror: string; -} - -const adminPages: Record = { - servers: { title: "服务器", description: "管理监控节点和运行参数" }, - latency: { title: "延迟检测", description: "配置分配给各服务器的 TCP 与 ICMP 延迟任务" }, - appearance: { title: "站点设置", description: "调整站点信息、公开内容和前台显示项目" }, - themes: { title: "主题商店", description: "选择内置主题或安装本地主题包" }, - themeSettings: { title: "主题设置", description: "调整当前前端主题提供的显示选项" }, - alerts: { title: "通知", description: "配置 Telegram 通知和资源告警阈值" }, - security: { title: "登录与安全", description: "管理管理员账号、登录设备和安全验证" }, - data: { title: "数据库", description: "查看空间、备份恢复和迁移数据库" }, - remote: { title: "远程执行", description: "输入命令并执行" }, - about: { title: "关于", description: "版本信息与项目地址" }, -}; - -const adminNavigation = [ - { tab: "servers" as const, label: "服务器", icon: ServerCog }, - { tab: "latency" as const, label: "延迟检测", icon: RadioTower }, - { tab: "remote" as const, label: "远程执行", icon: Terminal }, - { tab: "alerts" as const, label: "通知", icon: AlertTriangle }, - { tab: "themes" as const, label: "主题商店", icon: Palette }, - { tab: "themeSettings" as const, label: "主题设置", icon: SlidersHorizontal }, - { tab: "appearance" as const, label: "站点设置", icon: Eye }, - { tab: "security" as const, label: "登录与安全", icon: ShieldCheck }, - { tab: "data" as const, label: "数据库", icon: Database }, - { tab: "about" as const, label: "关于", icon: Info }, -]; - -const remoteTaskStatusLabels: Record = { - pending: "等待接收", - sent: "执行中", - success: "执行成功", - failed: "执行失败", -}; - - -const emptyServer: ServerInput = { - name: "", - region: "", - group_name: "默认", - tags: "", - hidden: false, - expires_at: null, - traffic_limit: 0, - traffic_limit_type: "sum", - price: 0, - billing_cycle: 30, - currency: "CNY", - auto_renewal: false, - network_interface: "", - reset_day: 1, - report_interval: 60, - collect_interval: 3, - rx_correction: 0, - tx_correction: 0, - agent_mirror: "", - offline_notify_disabled: false, - auto_update: true, -}; - -function toInput(server: AdminServer): ServerInput { - return { - ...emptyServer, - name: server.name, - region: server.region, - group_name: server.group_name, - tags: server.tags, - hidden: server.hidden, - expires_at: server.expires_at, - traffic_limit: server.traffic_limit, - traffic_limit_type: server.traffic_limit_type, - price: server.price, - billing_cycle: server.billing_cycle, - currency: server.currency, - auto_renewal: server.auto_renewal, - network_interface: server.network_interface, - reset_day: server.reset_day, - report_interval: server.report_interval, - collect_interval: server.collect_interval, - rx_correction: server.rx_correction, - tx_correction: server.tx_correction, - agent_mirror: server.agent_mirror, - offline_notify_disabled: server.offline_notify_disabled, - auto_update: server.auto_update, - }; -} - -const BILLING_CYCLES: Array<{ days: number; label: string }> = [ - { days: 30, label: "月" }, - { days: 90, label: "季" }, - { days: 180, label: "半年" }, - { days: 365, label: "年" }, - { days: 0, label: "一次性" }, -]; - -function ServerIpMeta({ server, agentVersion, onCopy }: { server: AdminServer; agentVersion: string | null; onCopy: (ip: string) => void }) { +function ServerIpMeta({ server, agentVersion, onCopy, locale }: { server: AdminServer; agentVersion: string | null; onCopy: (ip: string) => void; locale: string }) { const entries = [ { family: "v4" as const, ip: server.ip_v4 || "" }, { family: "v6" as const, ip: server.ip_v6 || "" }, @@ -168,89 +67,16 @@ function ServerIpMeta({ server, agentVersion, onCopy }: { server: AdminServer; a {entries.map((entry) => ( {entry.family === "v6" ? "IPv6" : "IPv4"} - + ))} - {!entries.length ? 尚未探测到公网 IP : null} + {!entries.length ? {ui(locale, "尚未探测到公网 IP", "No public IP detected yet")} : null} · - {agentVersion ? `Agent v${agentVersion}` : "Agent 未上报版本"} + {agentVersion ? `Agent v${agentVersion}` : ui(locale, "Agent 未上报版本", "Agent version not reported")}
); } -function formatDate(value: number | null) { - return value ? new Date(value * 1000).toISOString().slice(0, 10) : ""; -} - -function formatSessionTime(value: number) { - return new Date(value * 1000).toLocaleString(); -} - -function describeLoginDevice(userAgent: string) { - const browser = userAgent.includes("Edg/") ? "Edge" - : userAgent.includes("Firefox/") ? "Firefox" - : userAgent.includes("Chrome/") ? "Chrome" - : userAgent.includes("Safari/") ? "Safari" - : "其他客户端"; - const system = userAgent.includes("Android") ? "Android" - : /iPhone|iPad/.test(userAgent) ? "iOS" - : userAgent.includes("Windows") ? "Windows" - : userAgent.includes("Mac OS") ? "macOS" - : userAgent.includes("Linux") ? "Linux" - : ""; - return system ? `${browser} · ${system}` : browser; -} - -function settingPatch(settings: Settings, key: keyof Settings, value: unknown) { - return { ...settings, [key]: value } as Settings; -} - -function shellLiteral(value: string) { - return `'${value.replaceAll("'", `'\\''`)}'`; -} - -function powershellLiteral(value: string) { - return `'${value.replaceAll("'", "''")}'`; -} - -async function copyText(value: string) { - if (navigator.clipboard?.writeText) { - try { - await navigator.clipboard.writeText(value); - return; - } catch {} - } - const textarea = document.createElement("textarea"); - textarea.value = value; - textarea.readOnly = true; - textarea.style.position = "fixed"; - textarea.style.opacity = "0"; - document.body.append(textarea); - textarea.select(); - const copied = document.execCommand("copy"); - textarea.remove(); - if (!copied) throw new Error("copy failed"); -} - -async function waitForDatabaseSwitch(targetKind: DatabaseStats["kind"]) { - const deadline = Date.now() + 120_000; - while (Date.now() < deadline) { - await new Promise((resolve) => window.setTimeout(resolve, 750)); - try { - const response = await fetch(`/api/admin/database?restart=${Date.now()}`, { - cache: "no-store", - credentials: "same-origin", - signal: AbortSignal.timeout(2_500), - }); - if (response.status === 401) return; - if (response.ok) { - const database = await response.json() as DatabaseStats; - if (database.kind === targetKind && !database.restart_required) return; - } - } catch {} - } - throw new Error("NodeFlare 未自动恢复，请在主机上检查服务状态"); -} export function AdminPanel({ config, @@ -265,13 +91,10 @@ export function AdminPanel({ }) { const [authenticated, setAuthenticated] = useState(false); const [authChecked, setAuthChecked] = useState(false); - const [username, setUsername] = useState(""); - const [password, setPassword] = useState(""); - const [turnstileToken, setTurnstileToken] = useState(""); - const [turnstileReset, setTurnstileReset] = useState(0); - const [loginTotpCode, setLoginTotpCode] = useState(""); - const [loginTotpChallenge, setLoginTotpChallenge] = useState(false); const [busy, setBusy] = useState(false); + const locale = config.locale; + const pages = adminPages(locale); + const navigation = adminNavigation(locale); const [error, setError] = useState(""); const [notice, setNotice] = useState(""); const [tab, setTab] = useState(() => adminTabFromPath(window.location.pathname)); @@ -322,8 +145,7 @@ export function AdminPanel({ const remoteTasksRequestRef = useRef(0); const remoteTasksActive = useMemo(() => hasActiveRemoteTasks(remoteTasks), [remoteTasks]); const remotePayloadReady = Boolean(remoteCommand.trim()); - const loginTotpRequired = config.totp_login_enabled || loginTotpChallenge; - const verificationDialog = useVerification(authenticated); + const verificationDialog = useVerification(authenticated, locale); remoteTasksRef.current = remoteTasks; @@ -350,7 +172,7 @@ export function AdminPanel({ setError(""); return; } - setError(reason instanceof Error ? reason.message : "加载失败"); + setError(reason instanceof Error ? reason.message : ui(locale, "加载失败", "Failed to load")); } finally { setAuthChecked(true); setBusy(false); @@ -376,11 +198,30 @@ export function AdminPanel({ return () => window.removeEventListener("popstate", handlePopState); }, [authChecked, authenticated]); + // These loaders are ordinary functions recreated on every render. Route the + // tab effect through a ref so it always calls the latest version while its + // dependencies stay [authenticated, tab]; listing them directly would either + // be a stale-closure bug or refetch the active tab on every render. + const tabLoaders = useRef({ + themeSettings: () => {}, + database: () => {}, + loginSessions: () => {}, + }); + useEffect(() => { + tabLoaders.current = { + themeSettings: loadThemeSettings, + database: loadDatabase, + loginSessions: loadLoginSessions, + }; + }); + useEffect(() => { if (!authenticated) return; - if (tab === "themeSettings") void loadThemeSettings(); - if (tab === "data" && !database) void loadDatabase(); - if (tab === "security") void loadLoginSessions(); + if (tab === "themeSettings") void tabLoaders.current.themeSettings(); + // `database` is read purely as a one-shot guard; re-running on every + // database update would needlessly refetch the other tabs' data. + if (tab === "data" && !database) void tabLoaders.current.database(); + if (tab === "security") void tabLoaders.current.loginSessions(); }, [authenticated, tab]); useEffect(() => { @@ -412,25 +253,11 @@ export function AdminPanel({ return () => window.clearTimeout(timer); }, [twoFactorSecretCopied]); - async function login(event: FormEvent) { - event.preventDefault(); - setBusy(true); - setError(""); - try { - const passwordDerived = await derivePassword(password, config.password_client_salt); - await api.login(username.trim(), passwordDerived, turnstileToken, loginTotpCode); - setPassword(""); - setTurnstileToken(""); - setLoginTotpCode(""); - setLoginTotpChallenge(false); - setAuthenticated(true); - await load(); - } catch (reason) { - if (reason instanceof ApiError && reason.status === 428) setLoginTotpChallenge(true); - setError(reason instanceof Error ? reason.message : "登录失败"); - setTurnstileToken(""); - setTurnstileReset((value) => value + 1); - } finally { setBusy(false); } + async function handleLoginSubmit(loginUsername: string, loginPassword: string, turnstileToken: string, totpCode: string) { + const passwordDerived = await derivePassword(loginPassword, config.password_client_salt, locale); + await api.login(loginUsername.trim(), passwordDerived, turnstileToken, totpCode); + setAuthenticated(true); + await load(); } function openEditor(server?: AdminServer) { @@ -490,26 +317,26 @@ export function AdminPanel({ await load(); onChanged(); } catch (reason) { - setError(reason instanceof Error ? reason.message : "保存节点失败"); + setError(reason instanceof Error ? reason.message : ui(locale, "保存节点失败", "Failed to save the server")); } finally { setBusy(false); } } async function removeSelected() { - if (!selectedIds.length || !window.confirm(`确认删除选中的 ${selectedIds.length} 个节点及其全部历史数据？`)) return; + if (!selectedIds.length || !window.confirm(ui(locale, `确认删除选中的 ${selectedIds.length} 个节点及其全部历史数据？`, `Delete the selected ${selectedIds.length} server(s) and all their history?`))) return; setBusy(true); try { await api.deleteServers(selectedIds); setSelectedIds([]); await load(); onChanged(); - } catch (reason) { setError(reason instanceof Error ? reason.message : "批量删除失败"); } + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "批量删除失败", "Batch delete failed")); } finally { setBusy(false); } } async function remove(server: AdminServer) { - if (!window.confirm(`确认删除“${server.name}”及其全部历史数据？`)) return; + if (!window.confirm(ui(locale, `确认删除“${server.name}”及其全部历史数据？`, `Delete "${server.name}" and all its history?`))) return; try { await api.deleteServer(server.id); await load(); onChanged(); } - catch (reason) { setError(reason instanceof Error ? reason.message : "删除失败"); } + catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "删除失败", "Delete failed")); } } async function showInstallCommand(server: AdminServer) { @@ -519,7 +346,7 @@ export function AdminPanel({ const { agent_token } = await api.createAgentInstallToken(server.id); setInstall({ agent_token, agent_mirror: server.agent_mirror }); } catch (reason) { - setError(reason instanceof Error ? reason.message : "生成 Agent 安装命令失败"); + setError(reason instanceof Error ? reason.message : ui(locale, "生成 Agent 安装命令失败", "Failed to generate the Agent install command")); } finally { setBusy(false); } @@ -528,18 +355,18 @@ export function AdminPanel({ async function copyInstallCommand() { try { if (installCommand) await copyText(installCommand); - setNotice("安装命令已复制"); + setNotice(ui(locale, "安装命令已复制", "Install command copied")); } catch { - setError("复制失败，请手动选择命令复制"); + setError(ui(locale, "复制失败，请手动选择命令复制", "Copy failed; select and copy the command manually")); } } async function copyServerIp(ip: string) { try { await copyText(ip); - setNotice("IP 已复制"); + setNotice(ui(locale, "IP 已复制", "IP copied")); } catch { - setError("复制失败，请手动选择复制"); + setError(ui(locale, "复制失败，请手动选择复制", "Copy failed; please copy it manually")); } } @@ -550,7 +377,7 @@ export function AdminPanel({ [next[index], next[target]] = [next[target], next[index]]; setServers(next); try { await api.reorderServers(next.map((server) => server.id)); onChanged(); } - catch (reason) { setServers(servers); setError(reason instanceof Error ? reason.message : "排序失败"); } + catch (reason) { setServers(servers); setError(reason instanceof Error ? reason.message : ui(locale, "排序失败", "Failed to reorder")); } } function startDrag(event: DragEvent, id: string) { @@ -573,7 +400,7 @@ export function AdminPanel({ next.splice(targetIndex, 0, moved); setServers(next); try { await api.reorderServers(next.map((server) => server.id)); onChanged(); } - catch (reason) { setServers(previous); setError(reason instanceof Error ? reason.message : "排序失败"); } + catch (reason) { setServers(previous); setError(reason instanceof Error ? reason.message : ui(locale, "排序失败", "Failed to reorder")); } } async function saveSite(event: FormEvent) { @@ -589,15 +416,15 @@ export function AdminPanel({ delete payload.current_totp_code; try { if (payload.new_password) { - if (payload.new_password !== newPasswordConfirmation) throw new Error("两次输入的新密码不一致"); - payload.new_password_derived = await derivePassword(payload.new_password, config.password_client_salt); + if (payload.new_password !== newPasswordConfirmation) throw new Error(ui(locale, "两次输入的新密码不一致", "The two new passwords do not match")); + payload.new_password_derived = await derivePassword(payload.new_password, config.password_client_salt, locale); delete payload.new_password; } else { delete payload.new_password; delete payload.new_password_derived; } if (tab === "security") { - const proof = await sensitiveProof("保存设置"); + const proof = await sensitiveProof(ui(locale, "保存设置", "Save settings")); if (!proof) return; if (proof.totpCode) { payload.current_totp_code = proof.totpCode; @@ -609,9 +436,9 @@ export function AdminPanel({ setSettings(result); setNewPasswordConfirmation(""); if (tab === "security") await loadLoginSessions(); - setNotice("设置已保存"); + setNotice(ui(locale, "设置已保存", "Settings saved")); onChanged(); - } catch (reason) { setError(reason instanceof Error ? reason.message : "保存设置失败"); } + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "保存设置失败", "Failed to save settings")); } finally { setBusy(false); } } @@ -623,26 +450,26 @@ export function AdminPanel({ if (!stats.restart_required) setDatabaseMigrationResult(null); setExchangeRates(rates); } - catch (reason) { setError(reason instanceof Error ? reason.message : "读取数据库统计失败"); } + catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "读取数据库统计失败", "Failed to load database stats")); } finally { setBusy(false); } } async function sensitiveProof(action: string) { - if (!twoFactorStatus) throw new Error("正在读取两步验证状态，请稍候"); + if (!twoFactorStatus) throw new Error(ui(locale, "正在读取两步验证状态，请稍候", "Reading two-factor status, please wait")); const entered = await verificationDialog.ask(action, twoFactorStatus.enabled); if (entered === null) return null; if (twoFactorStatus.enabled) return { totpCode: entered }; - return { passwordDerived: await derivePassword(entered, config.password_client_salt) }; + return { passwordDerived: await derivePassword(entered, config.password_client_salt, locale) }; } async function reclaimDatabase() { - if (!window.confirm("回收空间期间数据库会短暂不可用，确认继续？")) return; + if (!window.confirm(ui(locale, "回收空间期间数据库会短暂不可用，确认继续？", "The database will be briefly unavailable while reclaiming space. Continue?"))) return; setBusy(true); setError(""); setNotice(""); try { const result = await api.reclaimDatabase(); setDatabase(result.database); - setNotice(result.reclaimed_bytes > 0 ? `已回收 ${formatBytes(result.reclaimed_bytes)}` : "数据库已整理，当前没有可释放空间"); - } catch (reason) { setError(reason instanceof Error ? reason.message : "回收数据库空间失败"); } + setNotice(result.reclaimed_bytes > 0 ? ui(locale, `已回收 ${formatBytes(result.reclaimed_bytes)}`, `Reclaimed ${formatBytes(result.reclaimed_bytes)}`) : ui(locale, "数据库已整理，当前没有可释放空间", "Database optimized; nothing to reclaim right now")); + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "回收数据库空间失败", "Failed to reclaim database space")); } finally { setBusy(false); } } @@ -651,35 +478,35 @@ export function AdminPanel({ if (!databaseUrl) return; const source = database?.kind === "postgresql" ? "PostgreSQL" : "SQLite"; const target = database?.kind === "postgresql" ? "SQLite" : "PostgreSQL"; - if (!window.confirm(`将 ${source} 迁移到 ${target}？目标库中已有的 NodeFlare 数据会被覆盖，完成后业务写入将暂停，直至重启服务。`)) return; + if (!window.confirm(ui(locale, `将 ${source} 迁移到 ${target}？目标库中已有的 NodeFlare 数据会被覆盖，完成后业务写入将暂停，直至重启服务。`, `Migrate from ${source} to ${target}? Existing NodeFlare data in the target database will be overwritten, and writes will pause until the service restarts.`))) return; setBusy(true); setError(""); setNotice(""); try { - const proof = await sensitiveProof("迁移数据库"); + const proof = await sensitiveProof(ui(locale, "迁移数据库", "Migrate database")); if (!proof) return; const result = await api.migrateDatabase(databaseUrl, proof); setDatabaseMigrationUrl(""); setDatabaseMigrationResult(result); setDatabase((current) => current ? { ...current, restart_required: result.restart_required } : current); const targetName = result.target_kind === "postgresql" ? "PostgreSQL" : "SQLite"; - setNotice(`数据库已迁移到 ${targetName}，共 ${result.migrated_rows.toLocaleString()} 行（${formatBytes(result.size_bytes)}），业务写入已暂停，请重启服务`); - } catch (reason) { setError(reason instanceof Error ? reason.message : "数据库迁移失败"); } + setNotice(ui(locale, `数据库已迁移到 ${targetName}，共 ${result.migrated_rows.toLocaleString()} 行（${formatBytes(result.size_bytes)}），业务写入已暂停，请重启服务`, `Migrated to ${targetName}: ${result.migrated_rows.toLocaleString()} rows (${formatBytes(result.size_bytes)}). Writes are paused; restart the service`)); + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "数据库迁移失败", "Database migration failed")); } finally { setBusy(false); } } async function restartAfterDatabaseMigration() { - if (!database?.restart_required || !window.confirm("立即重启 NodeFlare 并切换到新数据库？管理界面和 Agent 会短暂断开。")) return; + if (!database?.restart_required || !window.confirm(ui(locale, "立即重启 NodeFlare 并切换到新数据库？管理界面和 Agent 会短暂断开。", "Restart NodeFlare now and switch to the new database? The admin UI and agents will disconnect briefly."))) return; const targetKind = databaseMigrationResult?.target_kind === "postgresql" ? "postgresql" : databaseMigrationResult?.target_kind === "sqlite" ? "sqlite" : database.kind === "postgresql" ? "sqlite" : "postgresql"; - setBusy(true); setRestarting(true); setError(""); setNotice("NodeFlare 正在重启"); + setBusy(true); setRestarting(true); setError(""); setNotice(ui(locale, "NodeFlare 正在重启", "NodeFlare is restarting")); try { await api.restartAfterDatabaseMigration(); - await waitForDatabaseSwitch(targetKind); + await waitForDatabaseSwitch(targetKind, locale); window.location.replace("/admin/login"); } catch (reason) { - setError(reason instanceof Error ? reason.message : "重启 NodeFlare 失败"); + setError(reason instanceof Error ? reason.message : ui(locale, "重启 NodeFlare 失败", "Failed to restart NodeFlare")); } finally { setBusy(false); setRestarting(false); @@ -691,15 +518,15 @@ export function AdminPanel({ try { const rates = await api.refreshExchangeRates(); setExchangeRates(rates); - setNotice(`汇率已更新 · ${rates.source} · ${rates.date}`); - } catch (reason) { setError(reason instanceof Error ? reason.message : "汇率更新失败"); } + setNotice(ui(locale, `汇率已更新 · ${rates.source} · ${rates.date}`, `Exchange rates updated · ${rates.source} · ${rates.date}`)); + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "汇率更新失败", "Failed to refresh exchange rates")); } finally { setBusy(false); } } async function exportDatabaseBackup() { setBusy(true); setError(""); setNotice(""); try { - const proof = await sensitiveProof("导出备份"); + const proof = await sensitiveProof(ui(locale, "导出备份", "Export backup")); if (!proof) return; const { blob, filename } = await api.databaseBackup(proof); const url = URL.createObjectURL(blob); @@ -710,43 +537,43 @@ export function AdminPanel({ link.click(); link.remove(); window.setTimeout(() => URL.revokeObjectURL(url), 1000); - setNotice("数据库备份已导出"); - } catch (reason) { setError(reason instanceof Error ? reason.message : "导出数据库备份失败"); } + setNotice(ui(locale, "数据库备份已导出", "Database backup exported")); + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "导出数据库备份失败", "Failed to export the database backup")); } finally { setBusy(false); } } async function restoreDatabaseBackup(file: File) { if (file.size > 512 * 1024 * 1024) { - setError("数据库备份 ZIP 不能超过 512 MiB"); + setError(ui(locale, "数据库备份 ZIP 不能超过 512 MiB", "Database backup ZIP must not exceed 512 MiB")); return; } - if (!window.confirm("恢复将覆盖当前数据库，并使现有登录会话失效。确认继续？")) return; + if (!window.confirm(ui(locale, "恢复将覆盖当前数据库，并使现有登录会话失效。确认继续？", "Restoring overwrites the current database and invalidates existing sessions. Continue?"))) return; setBusy(true); setError(""); setNotice(""); try { - const proof = await sensitiveProof("恢复备份"); + const proof = await sensitiveProof(ui(locale, "恢复备份", "Restore backup")); if (!proof) return; const result = await api.restoreDatabaseBackup(file, proof); - window.alert(`数据库已恢复 ${result.restored_rows.toLocaleString()} 行，请使用备份中的管理员账号重新登录。`); + window.alert(ui(locale, `数据库已恢复 ${result.restored_rows.toLocaleString()} 行，请使用备份中的管理员账号重新登录。`, `Restored ${result.restored_rows.toLocaleString()} rows. Sign in again with the admin account from the backup.`)); window.location.reload(); - } catch (reason) { setError(reason instanceof Error ? reason.message : "恢复数据库备份失败"); } + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "恢复数据库备份失败", "Failed to restore the database backup")); } finally { setBusy(false); } } async function loadThemeSettings() { setBusy(true); setError(""); try { setThemeSettingsSchema(await api.themeSettings()); } - catch (reason) { setError(reason instanceof Error ? reason.message : "读取主题设置失败"); } + catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "读取主题设置失败", "Failed to load theme settings")); } finally { setBusy(false); } } async function addTheme(event: FormEvent) { event.preventDefault(); if (themeSourceMode === "upload" && !themeFile) { - setError("请选择 ZIP 主题文件"); + setError(ui(locale, "请选择 ZIP 主题文件", "Choose a ZIP theme file")); return; } if (themeSourceMode === "upload" && themeFile && themeFile.size > 32 * 1024 * 1024) { - setError("主题 ZIP 不能超过 32 MiB"); + setError(ui(locale, "主题 ZIP 不能超过 32 MiB", "Theme ZIP must not exceed 32 MiB")); return; } setBusy(true); setError(""); setNotice(""); @@ -760,8 +587,8 @@ export function AdminPanel({ setThemeFile(null); if (themeFileInputRef.current) themeFileInputRef.current.value = ""; await load(); - setNotice(themeSourceMode === "upload" ? "主题 ZIP 已安装" : "最新 Release 主题已安装"); - } catch (reason) { setError(reason instanceof Error ? reason.message : "添加主题失败"); } + setNotice(themeSourceMode === "upload" ? ui(locale, "主题 ZIP 已安装", "Theme ZIP installed") : ui(locale, "最新 Release 主题已安装", "Latest release theme installed")); + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "添加主题失败", "Failed to add the theme")); } finally { setBusy(false); } } @@ -771,9 +598,9 @@ export function AdminPanel({ try { await api.activateTheme(theme.id); await load(); - setNotice(`已启用主题：${theme.name}`); + setNotice(ui(locale, `已启用主题：${theme.name}`, `Theme activated: ${theme.name}`)); onChanged(); - } catch (reason) { setError(reason instanceof Error ? reason.message : "启用主题失败"); } + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "启用主题失败", "Failed to activate the theme")); } finally { setBusy(false); } } @@ -791,18 +618,18 @@ export function AdminPanel({ } } catch (reason) { previewWindow?.close(); - setError(reason instanceof Error ? reason.message : "创建主题预览失败"); + setError(reason instanceof Error ? reason.message : ui(locale, "创建主题预览失败", "Failed to create the theme preview")); } finally { setBusy(false); } } async function removeTheme(theme: Theme) { - if (theme.builtin || !window.confirm(`确认删除主题“${theme.name}”？`)) return; + if (theme.builtin || !window.confirm(ui(locale, `确认删除主题“${theme.name}”？`, `Delete the theme "${theme.name}"?`))) return; setBusy(true); setError(""); setNotice(""); try { await api.deleteTheme(theme.id); await load(); - setNotice("主题已删除"); - } catch (reason) { setError(reason instanceof Error ? reason.message : "删除主题失败"); } + setNotice(ui(locale, "主题已删除", "Theme deleted")); + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "删除主题失败", "Failed to delete the theme")); } finally { setBusy(false); } } @@ -813,20 +640,20 @@ export function AdminPanel({ setLoginSessionsLoaded(true); } catch (reason) { if (!(reason instanceof ApiError && reason.status === 401)) { - setError(reason instanceof Error ? reason.message : "读取登录设备失败"); + setError(reason instanceof Error ? reason.message : ui(locale, "读取登录设备失败", "Failed to load signed-in devices")); } } } async function revokeLoginSession(session: LoginSession) { - if (session.current || !window.confirm(`确认让“${describeLoginDevice(session.user_agent)}”下线？`)) return; + if (session.current || !window.confirm(ui(locale, `确认让“${describeLoginDevice(session.user_agent, locale)}”下线？`, `Sign out "${describeLoginDevice(session.user_agent, locale)}"?`))) return; setRevokingSessionId(session.id); setError(""); setNotice(""); try { await api.revokeLoginSession(session.id); setLoginSessions((current) => current.filter((item) => item.id !== session.id)); - setNotice("设备已踢下线"); + setNotice(ui(locale, "设备已踢下线", "Device signed out")); } catch (reason) { - setError(reason instanceof Error ? reason.message : "设备下线失败"); + setError(reason instanceof Error ? reason.message : ui(locale, "设备下线失败", "Failed to sign out the device")); } finally { setRevokingSessionId(""); } } @@ -847,42 +674,42 @@ export function AdminPanel({ async function setupTwoFactor() { setBusy(true); setError(""); setNotice(""); try { - const proof = await sensitiveProof("生成两步验证密钥"); + const proof = await sensitiveProof(ui(locale, "生成两步验证密钥", "Generate two-factor secret")); if (!proof) return; const setup = await api.setupTwoFactor(proof); setTwoFactorSetup(setup); setTwoFactorStatus({ enabled: false, has_secret: true }); setTwoFactorSecretCopied(false); - setNotice("两步验证密钥已生成，请先加入验证器再启用"); - } catch (reason) { setError(reason instanceof Error ? reason.message : "生成两步验证密钥失败"); } + setNotice(ui(locale, "两步验证密钥已生成，请先加入验证器再启用", "Two-factor secret generated; add it to your authenticator before enabling")); + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "生成两步验证密钥失败", "Failed to generate the two-factor secret")); } finally { setBusy(false); } } async function enableTwoFactor() { setBusy(true); setError(""); setNotice(""); try { - const code = await verificationDialog.ask("启用两步验证", true); + const code = await verificationDialog.ask(ui(locale, "启用两步验证", "Enable two-factor authentication"), true); if (code === null) return; await api.enableTwoFactor(code); setTwoFactorStatus({ enabled: true, has_secret: true }); setTwoFactorSetup(null); setTwoFactorSecretCopied(false); - setNotice("两步验证已启用"); + setNotice(ui(locale, "两步验证已启用", "Two-factor authentication enabled")); onChanged(); - } catch (reason) { setError(reason instanceof Error ? reason.message : "启用两步验证失败"); } + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "启用两步验证失败", "Failed to enable two-factor authentication")); } finally { setBusy(false); } } async function disableTwoFactor() { setBusy(true); setError(""); setNotice(""); try { - const code = await verificationDialog.ask("禁用两步验证", true); + const code = await verificationDialog.ask(ui(locale, "禁用两步验证", "Disable two-factor authentication"), true); if (code === null) return; await api.disableTwoFactor(code); setTwoFactorStatus({ enabled: false, has_secret: true }); - setNotice("两步验证已禁用"); + setNotice(ui(locale, "两步验证已禁用", "Two-factor authentication disabled")); onChanged(); - } catch (reason) { setError(reason instanceof Error ? reason.message : "禁用两步验证失败"); } + } catch (reason) { setError(reason instanceof Error ? reason.message : ui(locale, "禁用两步验证失败", "Failed to disable two-factor authentication")); } finally { setBusy(false); } } @@ -892,7 +719,7 @@ export function AdminPanel({ await copyText(twoFactorSetup.secret); setTwoFactorSecretCopied(true); } catch { - setError("复制失败，请手动选择复制"); + setError(ui(locale, "复制失败，请手动选择复制", "Copy failed; please copy it manually")); } } @@ -962,32 +789,32 @@ export function AdminPanel({ else failed = true; }); setRemoteTasks((current) => current.map((task) => updates.get(task.id) ?? task)); - if (failed && !quiet) setError("部分执行结果刷新失败"); + if (failed && !quiet) setError(ui(locale, "部分执行结果刷新失败", "Some results failed to refresh")); }, []); const createRemoteTask = async (event: FormEvent) => { event.preventDefault(); if (!remoteSelectedIds.length) { - setError("请选择至少一台服务器"); + setError(ui(locale, "请选择至少一台服务器", "Select at least one server")); return; } if (!remoteCommand.trim()) { - setError("请输入命令"); + setError(ui(locale, "请输入命令", "Enter a command")); return; } if (twoFactorStatus === null) { - setError("正在读取两步验证状态，请稍候"); + setError(ui(locale, "正在读取两步验证状态，请稍候", "Reading two-factor status, please wait")); return; } if (!twoFactorStatus.enabled) { - setError("请先在登录与安全中启用 TOTP 两步验证"); + setError(ui(locale, "请先在登录与安全中启用 TOTP 两步验证", "Enable TOTP two-factor authentication under Sign-in & security first")); return; } setBusy(true); setError(""); try { - const code = await verificationDialog.ask("确认执行远程命令", true); + const code = await verificationDialog.ask(ui(locale, "确认执行远程命令", "Confirm remote command execution"), true); if (code === null) return; const command = remoteCommand; const data = await api.createRemoteTask({ @@ -1013,12 +840,12 @@ export function AdminPanel({ setRemoteTasks(tasks); setRemotePollingUntil(Date.now() + REMOTE_TASK_POLL_TIMEOUT_MS); setRemoteCommand(""); - setNotice("已提交命令，请查看各节点的执行结果"); + setNotice(ui(locale, "已提交命令，请查看各节点的执行结果", "Command submitted; check each node for results")); await refreshRemoteTasks(true, true); } catch (err) { setError(err instanceof Error && err.name === "TimeoutError" - ? "下发请求超时，命令可能已提交，请先检查节点，避免重复执行" - : err instanceof Error ? err.message : "下发命令失败"); + ? ui(locale, "下发请求超时，命令可能已提交，请先检查节点，避免重复执行", "Dispatch timed out; the command may have been submitted. Check the nodes before retrying") + : err instanceof Error ? err.message : ui(locale, "下发命令失败", "Failed to dispatch the command")); } finally { setBusy(false); } @@ -1096,34 +923,38 @@ export function AdminPanel({ {authenticated ? (error ?
: notice ?
: null) : null} {!authenticated ?
- -
+ + -

管理员登录

{config.site_name}

- - - {loginTotpRequired ? : null} - {config.turnstile_login_enabled ?
: null} - {error ? : null} - - -
:
+

{ui(locale, "管理员登录", "Admin sign in")}

{config.site_name}

+ + :
{config.site_name}
- - - + + +
-

{adminPages[tab].title}

{adminPages[tab].description}

+

{pages[tab].title}

{pages[tab].description}

{tab === "servers" ? (
-

监控节点

{servers.length} 个节点 · 可拖动上下排序
-
{selectedIds.length ? : 批量操作}
+

{ui(locale, "监控节点", "Monitored servers")}

{ui(locale, `${servers.length} 个节点 · 可拖动上下排序`, `${servers.length} server(s) · drag to reorder`)}
+
{selectedIds.length ? : {ui(locale, "批量操作", "Batch actions")}}
{servers.map((server, index) => (
{ event.preventDefault(); event.dataTransfer.dropEffect = "move"; }} onDrop={(event) => void dropServer(event, server.id)}> - - toggleSelected(server.id)} ariaLabel={`选择 ${server.name}`} /> -
{server.name}
void copyServerIp(value)} />
-
+ + toggleSelected(server.id)} ariaLabel={ui(locale, `选择 ${server.name}`, `Select ${server.name}`)} /> +
{server.name}
void copyServerIp(value)} locale={locale} />
+
))} - {!servers.length && !busy ?
暂无节点
: null} + {!servers.length && !busy ?
{ui(locale, "暂无节点", "No servers yet")}
: null}
) : tab === "latency" ? ( - + ) : tab === "themes" && settings ? (
-

主题列表

+

{ui(locale, "主题列表", "Themes")}

{themes.map((theme) => { const uploaded = theme.url.startsWith("upload:"); return
- {!theme.builtin ? : null} - - {!theme.builtin ? : null} + {!theme.builtin ? : null} + + {!theme.builtin ? : null}
; })}
-

安装主题

主题包含可执行前端代码，只安装可信来源；安装后不依赖运行时远程资源。
-
- - +

{ui(locale, "安装主题", "Install theme")}

{ui(locale, "主题包含可执行前端代码，只安装可信来源；安装后不依赖运行时远程资源。", "Themes contain executable frontend code; install only from trusted sources. No runtime remote resources are needed after installation.")}
+
+ +
-

{themeSourceMode === "repository" ? "填写仓库主页地址，NodeFlare 会下载 latest Release 中的第一个 ZIP 文件。" : "ZIP 根目录需包含 index.html，也支持外层只有一个目录的打包方式；最大 32 MiB。"}

-
{themeSourceMode === "repository" ? : }
-