From db7afde7afed59a8f1b2c2d32192d252e9d135a0 Mon Sep 17 00:00:00 2001 From: imengying Date: Tue, 8 Sep 2026 12:33:22 +0800 Subject: [PATCH] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E5=A4=87=E4=BB=BD?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 10 +- backend/src/backup.rs | 227 ++++++++++++++++++++++- backend/tests/fixtures/README.md | 16 ++ backend/tests/fixtures/v1.0.3-backup.zip | Bin 0 -> 6002 bytes 4 files changed, 250 insertions(+), 3 deletions(-) create mode 100644 backend/tests/fixtures/README.md create mode 100644 backend/tests/fixtures/v1.0.3-backup.zip diff --git a/README.md b/README.md index eec1ca0..9d6a706 100644 --- a/README.md +++ b/README.md @@ -176,10 +176,18 @@ database_url = "postgres://nodeflare:password@127.0.0.1:5432/nodeflare?sslmode=d - ZIP 备份上限为 512 MiB，解压后不超过 4 GiB；导出也会检查这些上限，超限时请减少历史数据后重试。 - 新节点首次有效上报前不发送离线告警；后续按最近实时上报时间判断，后端重启后会等待一个离线告警周期供 Agent 重连。 - 历史指标默认保留 30 天，过期数据自动清理；完整空间回收在后台手动触发。 -- 升级建议先更新服务端再更新 Agent；恢复备份建议使用相同版本。 +- 升级建议先更新服务端再更新 Agent；支持当前版本和 v1.0.3 导出的 ZIP 备份。 SQLite / PostgreSQL 均使用一份 `01_initial.sql`，首次启动时自动创建完整数据库结构。ZIP 恢复校验表结构并导入业务数据，保留目标库的初始化记录。服务端与 Agent 须配套使用，当前 Agent 协议为 `2`。 +从 v1.0.3 导入数据： + +1. 在 v1.0.3 后台导出 ZIP 备份，并保留原配置和数据库。 +2. 为新版配置全新的 SQLite 文件或空 PostgreSQL 数据库，设置临时管理员账号和密码后启动。 +3. 登录新版后台恢复 ZIP；恢复后使用备份中的管理员账号、密码和两步验证重新登录，再更新各节点 Agent。 + +导入会保留设置、节点、历史数据、主题文件和安全配置；旧版最新状态中的重复指标列会忽略，完整指标从 `latest_json` 保留。原库的迁移记录不会导入。额外安装 Token 的恢复规则见上文。 + ## 从源码开发 ```bash diff --git a/backend/src/backup.rs b/backend/src/backup.rs index 2d50ff5..08e735c 100644 --- a/backend/src/backup.rs +++ b/backend/src/backup.rs @@ -786,7 +786,27 @@ async fn restore_table( if read_bounded_line(&mut reader, &mut line)? == 0 { bail!("{table} 备份表头无效"); } - let header: BackupTableHeader = serde_json::from_str(line.trim_end())?; + let mut header: BackupTableHeader = serde_json::from_str(line.trim_end())?; + // v1.0.3 stores these metrics both as columns and in latest_json. Accept only + // that exact layout; retained columns still undergo the full schema check. + let legacy_latest_state = table == "server_latest_state" + && header.columns.len() == 12 + && header.columns[2..10] + .iter() + .map(|column| (column.name.as_str(), column.kind)) + .eq([ + ("cpu", BackupKind::Double), + ("mem_used", BackupKind::BigInt), + ("mem_total", BackupKind::BigInt), + ("disk_used", BackupKind::BigInt), + ("disk_total", BackupKind::BigInt), + ("net_in", BackupKind::Double), + ("net_out", BackupKind::Double), + ("uptime", BackupKind::BigInt), + ]); + if legacy_latest_state { + header.columns.drain(2..10); + } if header.table != table || header.columns != columns { bail!("{table} 备份结构与当前数据库不兼容"); } @@ -802,7 +822,14 @@ async fn restore_table( if trimmed.is_empty() { continue; } - batch.push(serde_json::from_str::>(trimmed)?); + let mut row = serde_json::from_str::>(trimmed)?; + if legacy_latest_state { + if row.len() != 12 { + bail!("{table} 备份行字段数量不匹配"); + } + row.drain(2..10); + } + batch.push(row); batch_bytes = batch_bytes.saturating_add(read); if batch.len() >= INSERT_BATCH_ROWS || batch_bytes >= MAX_NDJSON_LINE_BYTES { insert_rows(db, transaction, table, columns, &batch).await?; @@ -923,6 +950,202 @@ pub async fn copy_database(source: &Database, target: &Database) -> Result)) -> Vec { + let mut archive = ZipArchive::new(Cursor::new(V1_0_3_BACKUP)).unwrap(); + let entry_name = table_entry_name("server_latest_state"); + let mut lines = BufReader::new(archive.by_name(&entry_name).unwrap()) + .lines() + .map(|line| serde_json::from_str::(&line.unwrap()).unwrap()) + .collect::>(); + edit(&mut lines); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + for index in 0..archive.len() { + let entry = archive.by_index(index).unwrap(); + if entry.name() == entry_name { + writer + .start_file(&entry_name, SimpleFileOptions::default()) + .unwrap(); + for line in &lines { + serde_json::to_writer(&mut writer, line).unwrap(); + writer.write_all(b"\n").unwrap(); + } + } else { + writer.raw_copy_file(entry).unwrap(); + } + } + writer.finish().unwrap().into_inner() + } + + #[tokio::test] + async fn rejects_invalid_v1_0_3_layouts_and_rows_without_changing_the_target() { + type FixtureEdit = fn(&mut Vec); + let cases: [(&str, FixtureEdit); 7] = [ + ("unknown metric column", |lines| { + lines[0]["columns"][2]["name"] = Value::from("unknown_metric"); + }), + ("wrong metric type", |lines| { + lines[0]["columns"][2]["kind"] = Value::from("text"); + }), + ("wrong retained column", |lines| { + lines[0]["columns"][10]["name"] = Value::from("unknown_json"); + }), + ("extra column", |lines| { + lines[0]["columns"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({"name": "extra", "kind": "text"})); + }), + ("missing value", |lines| { + lines[1].as_array_mut().unwrap().pop(); + }), + ("extra value", |lines| { + lines[1].as_array_mut().unwrap().push(Value::Null); + }), + ("invalid timestamp", |lines| { + lines[1][1] = Value::from("invalid"); + }), + ]; + for (name, edit) in cases { + let db = crate::db::connect("sqlite::memory:").await.unwrap(); + db.migrate().await.unwrap(); + crate::db::set_setting(&db, "site_name", "Preserved") + .await + .unwrap(); + sqlx::query("INSERT INTO servers(id,name,token_hash,created_at,updated_at) VALUES ('preserved','Preserved','hash',1,1)") + .execute(db.pool()).await.unwrap(); + sqlx::query("INSERT INTO sessions(id,token_hash,username,ip_address,user_agent,created_at,last_seen_at,expires_at) VALUES ('session','hash','admin','127.0.0.1','test',1,1,9999999999)") + .execute(db.pool()).await.unwrap(); + let directory = tempfile::tempdir().unwrap(); + let themes = directory.path().join("themes"); + fs::create_dir(&themes).unwrap(); + fs::write(themes.join("current.txt"), "Preserved").unwrap(); + + assert!( + restore_archive(&db, &themes, &edit_v1_0_3_latest_state(edit)) + .await + .is_err(), + "{name} was accepted" + ); + assert_eq!( + crate::db::get_setting(&db, "site_name") + .await + .unwrap() + .as_deref(), + Some("Preserved"), + "{name} changed settings" + ); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT id FROM servers") + .fetch_all(db.pool()) + .await + .unwrap(), + ["preserved"], + "{name} changed servers" + ); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT id FROM sessions") + .fetch_all(db.pool()) + .await + .unwrap(), + ["session"], + "{name} changed sessions" + ); + assert_eq!( + fs::read_to_string(themes.join("current.txt")).unwrap(), + "Preserved" + ); + assert_eq!(fs::read_dir(&themes).unwrap().count(), 1); + assert_eq!(fs::read_dir(directory.path()).unwrap().count(), 1); + } + } + + #[tokio::test] + async fn restores_v1_0_3_backup_without_legacy_migrations() { + let db = crate::db::connect("sqlite::memory:").await.unwrap(); + db.migrate().await.unwrap(); + let directory = tempfile::tempdir().unwrap(); + let themes = directory.path().join("themes"); + restore_archive(&db, &themes, V1_0_3_BACKUP).await.unwrap(); + + let servers = crate::db::queries::list_servers(&db, true).await.unwrap(); + assert_eq!(servers.len(), 1); + assert_eq!(servers[0].cpu, Some(42.5)); + assert_eq!(servers[0].mem_used, Some(512)); + let latest = sqlx::query("SELECT * FROM server_latest_state") + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(latest.columns().len(), 4); + assert_eq!(latest.get::("last_batch_id"), "fixture-batch"); + let report: crate::models::AgentReport = + serde_json::from_str(&latest.get::("latest_json")).unwrap(); + assert_eq!(report.gpu_usage, 32.5); + assert_eq!(latest.get::("latest_timestamp"), report.timestamp); + let history = sqlx::query("SELECT * FROM metric_history ORDER BY timestamp") + .fetch_all(db.pool()) + .await + .unwrap(); + assert_eq!(history.len(), 2); + assert_eq!(history[0].get::("cpu_min"), 75.0); + assert_eq!(history[0].get::("memory_avg"), 50.0); + assert_eq!(history[1].get::("sample_count"), 4); + assert_eq!(history[1].get::("cpu_min"), 10.0); + assert_eq!(history[1].get::("cpu_max"), 90.0); + assert_eq!(history[1].get::("memory_min"), 10.0); + assert_eq!( + crate::db::get_setting(&db, "site_name") + .await + .unwrap() + .as_deref(), + Some("v1.0.3 fixture") + ); + assert!( + crate::db::queries::agent_identity(&db, "fixture-primary-token") + .await + .unwrap() + .is_some() + ); + assert!( + crate::db::queries::agent_identity(&db, "fixture-install-token") + .await + .unwrap() + .is_none() + ); + assert_eq!( + sqlx::query_scalar::<_, String>("SELECT totp_secret FROM admin_2fa WHERE enabled=1") + .fetch_one(db.pool()) + .await + .unwrap(), + "JBSWY3DPEHPK3PXP" + ); + assert_eq!( + crate::db::queries::remote_task(&db, "fixture-task") + .await + .unwrap() + .unwrap() + .result, + "fixture" + ); + assert_eq!( + fs::read_to_string(themes.join("theme-12345678/index.html")).unwrap(), + "
v1.0.3 fixture
" + ); + assert_eq!( + fs::read_to_string(themes.join("theme-12345678/assets/app.css")).unwrap(), + "main { color: red; }" + ); + db.migrate().await.unwrap(); + assert_eq!( + sqlx::query_scalar::<_, i64>("SELECT version FROM _sqlx_migrations") + .fetch_all(db.pool()) + .await + .unwrap(), + [1] + ); + } + #[test] fn exported_archives_must_fit_both_restore_limits() { let compressed = DATABASE_BACKUP_MAX_BYTES as u64; diff --git a/backend/tests/fixtures/README.md b/backend/tests/fixtures/README.md new file mode 100644 index 0000000..ce5e71d --- /dev/null +++ b/backend/tests/fixtures/README.md @@ -0,0 +1,16 @@ +# v1.0.3 backup fixture + +`v1.0.3-backup.zip` was exported with the unchanged `backup::export_archive` +implementation and migrations from tag `v1.0.3` (commit `58dea92`). A temporary +Rust test populated an in-memory SQLite database and called that exporter. +The archive uses backup format 2 and contains only synthetic data. + +It includes settings, a hidden free node, latest metrics, both a snapshot and an +aggregated history row, a completed remote task, enabled TOTP, and a theme with +an asset. The redundant SQL CPU value is deliberately 99 while `latest_json` +contains 42.5, so tests verify that the complete JSON report is preserved. + +The fake administrator is `fixture-admin` with password `FixturePassword123`; +the TOTP secret is `JBSWY3DPEHPK3PXP`. The primary Agent token is +`fixture-primary-token`. An extra `fixture-install-token` was created in the +source database but is excluded by v1.0.3's export policy. diff --git a/backend/tests/fixtures/v1.0.3-backup.zip b/backend/tests/fixtures/v1.0.3-backup.zip new file mode 100644 index 0000000000000000000000000000000000000000..d85b168e50543386f7fbb718e39a1f5fb595a20c GIT binary patch literal 6002 zcma)=1yqy!|HnteK%_)UVoXFNMk*mPq(f4XcEad#8!Zi!kaQqiLb-sT)FmV&1SCWn zQ96_^5tUTr_i(Q=+0Xm?|KWVkbI#6rUg!OZ=M!IDO%hTj5C}vG93D_nT&t!K5eNh& z27!(Nzc?dY>}`-}jIaaR)kQnL&!q`Qqdd9A>MsmqVJ~MEWE@V)V%xRWTQz8-E8{e7za|pX)sn_!ABb|&nI0U zJn&pQVaM4bI`Hs;D9dFs6>aOVT@qZEqKtsyL%}fe6W7}=-oM<#*N8YdP->RTdDprC z>wtL?zjnDvTT?g4?Ll;(nod~5$w56CF^!T2C0J&~F&9Oqo2bF|^Ml)7!vV(E+w5`_ z<6CrFOMA?^v;ta2vmpi*z2vIQD8=cQs+pX$TTe@lZP~e?N+xQZEGIp_o_Jik7>PwJ zVl)XVyHhm#8r*!(Lz{2?f?D@#a|?W%W80D0Sj~Zn z*F^r7{)KOHd&Ig&{OTyaxqAxmYdUb40l#7pH=U4Z5i}Bmv3IdW3%gh!IM_T^N2f`F zu4U~uSbVWdsMMK=XOZY*4^ddMT^wD%;CN6`IrZX|O8$2>>|llssg$kMt+~~pW{C1a zFdUL&M9$KjTK=*;=!4U76KhrX=m!lq3d=)<={``>u0TxUnYS%7xE#dkM9)UipH4<( zLt%L;_kI0)ou0ZP{6noDVaiXm;!@32B4AW7H>LU!J00~lZ0)o)&l+hLb5^g_Xq;?E zY%Ybg_g+GjjY)XJb5?Hs#k;xr3DXU0I(#*m(GS>NUw{3cXGHUHU8cIMQ8R-k#A9td zhW&Xdb%^(yutxpWyqA52q2Ovc=3KC(vPw27jbQQlrTVWgj7Be2MZ{hWYJ42-E8|%i z_0-I#X!e9N69N+>FKv;23Sse;{HR0U%4ftC1J2|m1A$Kb>QIyi67|!ci3XZ6>^yYi zJ!c_rp?~f*I=k{BQ~V^@c;uDYQ?S?IzGG}%7g2!)=pZxxYUR+2k#Y&CDFysq#S>YE zS9PG8vq39r#B#)ZiV-pXE0-ZPU(YaDTPoO6o~AfSGaOHotCg{MNf2knth``G^V#dp zEe<7f-VWieEMHq1Gim-os*@Ia`ptq6wg!P?S3)kUDb5K9WtG3Re`mh-J>={s3ByQ^ z-oMl9)FhnDE+gEQv3W|FfMC_nQ1U47Mo3uB*49OI73m+3rL zMzSONKC_D0e{cPW4#+rL~ku^v5w`NH`{w%v~ZZ~_zE0B9rY;}H5GPv``#7Jb+D9!4- zoevq;Jzw!Qwb5zsyt&`8@uDM9_tX5~*5svtv}vpuo66eRENJw_%oWBgO_lcf;z>2{ zTI*UjY1uPr4>Z-PoY`8V^*TK2%v}etpRbh)4W@~}O()cln3~yq8zSmamupc;`O#*T zjunPlPaq?5V|!#rBYe7Vktr95tQ8RDg97#Alfl>_osmBS?ANOe4uR2O-__;vPs%u# z?9YJeLc^8BoiAHKcIY2_ZQTs>(8EYZ%_GSlsnX@}xrI`^nQ7EDKIJ&wSjMR`kad;{ zn=6J|$)jc&8tlAab2Xtj%^|qdMn3XNPZqDMyJqS20uz+~y2`8?X+jv~^GAkTiYb9S zTnSD-4Km}a$J<`-5M%R~6fxzyAeII?cIU8}N_8obH-Yd`4ZHgB??P&g{c_$UM-8j@ znbS-LjH;smfjIDnIU_MBdn-#jdo;!s<^7`~B;r=Dyx?ZS?)$QSE22(++ch338cNj7 zxC7=Gn=AD_KhWa71iEb3|7k|rl@><=uYVg^5U$6f@Rw2eLQ@7t=|cM5XzL9OcT#Gn z(us+yH@MYRCmJ+LA$cBZG2`chZx|}Md}B3!feosA7^*YxzNjDLCMV+9hO{KXKFL&L zL0*s4U`E?eo@% z;*8;40p_zz8C14~43ynT!_d*;XdOZ2`V{MLpEuBhT4!EKn3D*J-T&x1`gwgWqPBt7K)3R(y7Dvs z5;NiX7j8N=EwHRp>77CSMY%W+@6J1~JLE*hqdPI3MLqsdr~cjj*m=hYE<4)oto~_< zb>)KAysgP&-ey|oHY_&{B1%4NKTgYryL)vl`o%gAL zpOqpYY1|w)E_&>byX)=Bv{&O3mt~bh9(78cYo!RdVCX&hXmNb1r1$-mst&{N;?c~; ze+=EcHxzgKz3$%1$zEyGtgVGR8F?R+oTD56z)5;aNfZw~qyDpkcK-S(xs^062Mfh= z?q~Y+l(y{r=X@Cbm|&+jmaLi|<1b<0q$0;3&-)YInV0z#h;61y52z}Txh25$&LP-- zm=?*qY5oU~aq3W+L1+(#;oMh~z0Qs@L+FrVm62$S>WC+O5FsbBb%zbG~IP8 zZ(Jrac!9x9_w4EHuB*Y?0*hqeoPG0=%%v%hkMa{W^ac4zahKU`>qey5h}{QO=6z>c zD2vBA+A^9@V7bEcFB>Ibd?nxy+c`c@NY(?C{4>P4bcgShes3EHlDMb0n{SJp#WC5f zG>G=v?64B&j?en^Y<_aX&1+3Sl?3|52+LH8gT3H0K>QMr*lc(sP)KK24AK&VKs)}d z!qI(j2NF>lvR=O{df!dLDmtn)C|j-0wKOLgXwn)eez3q+yr%1iX)a&VY?6QREt!>s z?Y_fG=Q12lDY*rSMEGL2#iE$t7oZU=H{@7_%dKCnHg{cwWG20z)jR(A?Z$rXx~i>Y zjLSE(tlGWDqIEUU%IOg>vxBsa1rfVrkI{^WXhuS{F7M=GX^m6GP;R@=^!yIBLWf>` zF^4I`=%qbFcrPdM@;8g;{cQ=PUo`sBk19`t>Q#b|sUe4e)qNlpoKBSTw0cFg?O_*actL(O7E1UQ`i7AY| zAg5Z4#OzrbkROcM$GpSz{ZkdR<+e%kDd~XJyT|gYf-Wnr?AaZ%Cf|fDfQo7tjmAN9ae{!JrT}Hb6r;Qg=H4 z?w91fezEI?7vV+vuV8#utXlmEcMBmC<_mvb(tZDZ{Tj=b-jr^RTz1?EPVx=IXzkv{ z$@e%^m~8ROwEApv42vYih2&0bp)cd}ZGMSYpLpsl)IY}x=(q5@nWpePZe5(H-&RM; zL*MclB)SdMc23Thrh6P!-m;M_5DzF$0F<-il^;|}7c1{0$>rB03Vs2j342%9)|}ojg$CH2@7VtNh1DquXb>kuwcSwfV=XcnO6kHf=7lv`{;otN zEbXH_nu}Gnr(!;gmS@86Orb*T01v&?AYjB-BJ zKAHXU=rl_C#P)M}96XsA_x8{MbL4AI)nNM}%@^LqrOqmcJv`JGBKIuhILec{gl8H%TOA zXQ_H+z*n`kJ*s7-Z8_Ei1{Hgd&-qt}hY!J~6i`ijG6oQ83#0}sUNFK5iNaW-+<~p> z=NA|YKM+g0=7;U}hu7;z>N&j&A3?<<`9r_eL2(I+c4B zS(Wx7 z0*aafol;+2Ti!<6jUq@tMlAYn3Z}UzoB`z-LER!~woeg2^wbec=W?I!Yksym)uHxMJ*q3WmVgySi9nkWNTj6vFvu?>vwSqdO=I$?ia^hKH() zJ-U2P-QEJuUsqG5cg}rJ{(B|;1tuQVw+}s|X`JVHls`2kl1CO6ypgG1R%zW(mh4lw zX97`$ml=_)zVM|z#_7!PmH*|m!uyvWCsQnSb3D7>iwZX_D(G^6&RN0Q~R2z@UT2ky{_aR|&5&{&^7u3Jkgqgzxtk z8-xtP?H$i}3|u@NGJf6V2@!d|9FxV zu;U#u=XcuxA&zi-#N$+`4snFTBq4-we8WSi!H1CF40D7W!l4My*~4>wJ0cO12*(vX zsgU|m(f=4;2zi7<1fFM0bIAMI=zg772|U=TL#1c literal 0 HcmV?d00001