diff --git a/src-tauri/src/config/cloud_sync.rs b/src-tauri/src/config/cloud_sync.rs index 6079591d..0dcaebc5 100644 --- a/src-tauri/src/config/cloud_sync.rs +++ b/src-tauri/src/config/cloud_sync.rs @@ -264,6 +264,10 @@ pub struct CloudSyncStatus { pub state: String, #[serde(default)] pub message: String, + /// Stable machine-readable identifier for failures the UI reacts to, so the + /// frontend never has to match prose error text. + #[serde(default)] + pub error_code: Option, #[serde(default)] pub current_operation: Option, #[serde(default)] @@ -281,6 +285,7 @@ impl Default for CloudSyncStatus { provider: default_provider(), state: default_status_state(), message: String::new(), + error_code: None, current_operation: None, last_checked_at_ms: None, last_synced_at_ms: None, diff --git a/src-tauri/src/core/cloud_sync/gc.rs b/src-tauri/src/core/cloud_sync/gc.rs index a5684aeb..676f5fe1 100644 --- a/src-tauri/src/core/cloud_sync/gc.rs +++ b/src-tauri/src/core/cloud_sync/gc.rs @@ -12,8 +12,9 @@ use super::remote::{ pub(super) const SYNC_SNAPSHOT_KEEP_RECENT: usize = 5; pub(super) const SYNC_SNAPSHOT_GC_GRACE_PERIOD: Duration = Duration::from_secs(24 * 60 * 60); -/// Gitee gist hard limit is 10 files; keep headroom for the next upload. -pub(super) const GIST_REMOTE_MAX_FILES: usize = 10; +/// Fixed sync documents that always occupy a gist slot: `sync/latest.redb` and +/// `sync/current.redb.enc`. +const GIST_FIXED_FILE_COUNT: usize = 2; #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct SnapshotGcEntry { @@ -58,70 +59,117 @@ pub(super) async fn cleanup_sync_snapshots( /// Free gist capacity before uploading a new snapshot file. /// -/// Keeps at most `keep_recent - 1` snapshots so the upcoming upload stays within -/// `keep_recent`, and deletes additional oldest snapshots when total gist files -/// would exceed `GIST_REMOTE_MAX_FILES - 1`. +/// Only runs for backends with a hard file-count limit, and only when the +/// upcoming upload would exceed it. Reading every snapshot to rank generations is +/// expensive, so the common case stops after a single cheap remote file count. pub(super) async fn prune_gist_snapshots_before_upload( remote: &CloudRemote, remote_root: &str, latest: Option<&RemoteSyncPointer>, ) -> AppResult<()> { - let total_files = remote.list_files("").await?.len(); + let Some(capacity_limit) = remote.file_capacity_limit() else { + // No hard limit: retention is handled by `cleanup_sync_snapshots`. + return Ok(()); + }; + let Some(total_files) = remote.gist_file_count().await? else { + return Ok(()); + }; + + let projected_total = total_files.saturating_add(1); + if projected_total <= capacity_limit { + return Ok(()); + } + + tracing::info!( + total_files, + capacity_limit, + "Gist is at capacity; pruning snapshots before upload" + ); let snapshots = collect_snapshots(remote, remote_root).await?; - let keep_before_upload = SYNC_SNAPSHOT_KEEP_RECENT.saturating_sub(1); - let mut paths = plan_snapshot_gc( - snapshots.clone(), + let paths = plan_gist_capacity_prune( + &snapshots, + total_files, + capacity_limit, latest.map(|pointer| pointer.revision_id.as_str()), - current_time_ms(), - keep_before_upload, - Duration::from_secs(0), ); - let projected_total = total_files.saturating_sub(paths.len()).saturating_add(1); - if projected_total > GIST_REMOTE_MAX_FILES { - let extra_needed = projected_total - GIST_REMOTE_MAX_FILES; - let mut protected: HashSet = HashSet::new(); - if let Some(latest) = latest { - protected.insert(latest.revision_id.clone()); + let needed = projected_total - capacity_limit; + let unmanaged_files = total_files.saturating_sub(snapshots.len()); + if unmanaged_files > GIST_FIXED_FILE_COUNT { + if paths.len() < needed { + tracing::warn!( + total_files, + unmanaged_files, + freed = paths.len(), + needed, + "Gist holds files outside the current sync root; the snapshot prune cannot free enough capacity" + ); + } else { + tracing::info!( + total_files, + unmanaged_files, + freed = paths.len(), + "Gist holds files outside the current sync root; that capacity cannot be reclaimed" + ); } - let mut candidates = snapshots; - candidates.sort_by_key(|snapshot| snapshot.created_at_ms); - let mut extras = Vec::new(); - for snapshot in candidates { - if extras.len() >= extra_needed { - break; - } - if !snapshot.deletable { - // Capacity prune: drop unreadable orphans too when over the hard limit. - if snapshot.revision_id.is_empty() || !protected.contains(&snapshot.revision_id) { - if !paths.contains(&snapshot.path) { - extras.push(snapshot.path); - } - } - continue; - } - if protected.contains(&snapshot.revision_id) { - continue; - } - if paths.contains(&snapshot.path) { - continue; - } - extras.push(snapshot.path); - } - paths.extend(extras); } - if !paths.is_empty() { - tracing::info!( - delete_count = paths.len(), - total_files, - "Pruning gist snapshots before upload" - ); - } delete_snapshot_paths(remote, paths).await; Ok(()) } +/// Best-effort capacity pruning for gist remotes. +/// +/// Inspecting the remote can fail on its own (network, quota on reads). That must +/// never block the upload that follows, so failures are only logged. +pub(super) async fn prune_gist_snapshots_best_effort( + remote: &CloudRemote, + remote_root: &str, + latest: Option<&RemoteSyncPointer>, +) { + if !remote.is_gist_backend() { + return; + } + + if let Err(error) = prune_gist_snapshots_before_upload(remote, remote_root, latest).await { + tracing::warn!( + error = %error, + "Gist snapshot pruning did not complete; continuing with upload" + ); + } +} + +/// Pure planning half of [`prune_gist_snapshots_before_upload`]. +/// +/// Returns the snapshot files to delete so that `total_files + 1` fits into +/// `capacity_limit`. Oldest generations go first (unreadable entries carry +/// `created_at_ms == 0`, so they are reclaimed before readable history), and the +/// latest pointer target is never touched. +pub(super) fn plan_gist_capacity_prune( + snapshots: &[SnapshotGcEntry], + total_files: usize, + capacity_limit: usize, + latest_revision: Option<&str>, +) -> Vec { + let projected_total = total_files.saturating_add(1); + if projected_total <= capacity_limit { + return Vec::new(); + } + let needed = projected_total - capacity_limit; + + let mut candidates: Vec<&SnapshotGcEntry> = snapshots + .iter() + .filter(|snapshot| Some(snapshot.revision_id.as_str()) != latest_revision) + .collect(); + candidates.sort_by_key(|snapshot| snapshot.created_at_ms); + + candidates + .into_iter() + .take(needed) + .map(|snapshot| snapshot.path.clone()) + .collect() +} + async fn delete_snapshot_paths(remote: &CloudRemote, paths: Vec) { for path in paths { if let Err(error) = remote.delete(&path).await { @@ -213,6 +261,26 @@ async fn read_snapshot_for_gc( Ok((snapshot.created_at_ms, snapshot.payload_hash)) } +/// Revisions that must survive any cleanup: the latest pointer target plus the +/// most recent `keep_recent` generations. +pub(super) fn protected_revision_ids( + snapshots: &[SnapshotGcEntry], + latest_revision: Option<&str>, + keep_recent: usize, +) -> HashSet { + let mut protected: HashSet = HashSet::new(); + if let Some(latest_revision) = latest_revision { + protected.insert(latest_revision.to_string()); + } + + let mut by_age: Vec<&SnapshotGcEntry> = snapshots.iter().collect(); + by_age.sort_by_key(|snapshot| snapshot.created_at_ms); + for snapshot in by_age.into_iter().rev().take(keep_recent) { + protected.insert(snapshot.revision_id.clone()); + } + protected +} + pub(super) fn plan_snapshot_gc( mut snapshots: Vec, latest_revision: Option<&str>, @@ -220,17 +288,10 @@ pub(super) fn plan_snapshot_gc( keep_recent: usize, grace_period: Duration, ) -> Vec { - let mut protected: HashSet = HashSet::new(); - if let Some(latest_revision) = latest_revision { - protected.insert(latest_revision.to_string()); - } - - snapshots.sort_by_key(|snapshot| snapshot.created_at_ms); - for snapshot in snapshots.iter().rev().take(keep_recent) { - protected.insert(snapshot.revision_id.clone()); - } + let protected = protected_revision_ids(&snapshots, latest_revision, keep_recent); let grace_ms = u64::try_from(grace_period.as_millis()).unwrap_or(u64::MAX); + snapshots.sort_by_key(|snapshot| snapshot.created_at_ms); snapshots .into_iter() .filter(|snapshot| snapshot.deletable) @@ -250,6 +311,7 @@ fn snapshot_revision_from_path(path: &str) -> Option { #[cfg(test)] mod tests { + use super::super::operator::MemoryRemote; use super::*; fn entry(revision: &str, created_at_ms: u64) -> SnapshotGcEntry { @@ -314,6 +376,175 @@ mod tests { assert_eq!(delete, vec!["nyaterm/sync/snapshots/r1.redb.enc"]); } + fn full_gist_snapshots() -> Vec { + (1..=8) + .map(|index| entry(&format!("r{index}"), index)) + .collect() + } + + #[test] + fn gist_capacity_prune_frees_exactly_the_needed_slot() { + // latest + current + 8 snapshots = 10 files, so the next upload needs one + // slot freed. + let delete = plan_gist_capacity_prune(&full_gist_snapshots(), 10, 10, Some("r8")); + + assert_eq!( + delete, + vec!["nyaterm/sync/snapshots/r1.redb.enc".to_string()] + ); + } + + #[test] + fn gist_capacity_prune_is_a_no_op_when_there_is_room() { + let delete = plan_gist_capacity_prune(&full_gist_snapshots(), 7, 10, Some("r8")); + + assert!(delete.is_empty()); + } + + #[test] + fn gist_capacity_prune_frees_as_many_slots_as_needed() { + // 12 files against a limit of 10: the upload needs three slots, oldest first. + let delete = plan_gist_capacity_prune(&full_gist_snapshots(), 12, 10, Some("r8")); + + assert_eq!( + delete, + vec![ + "nyaterm/sync/snapshots/r1.redb.enc".to_string(), + "nyaterm/sync/snapshots/r2.redb.enc".to_string(), + "nyaterm/sync/snapshots/r3.redb.enc".to_string(), + ] + ); + } + + #[test] + fn gist_capacity_prune_never_drops_the_latest_pointer_snapshot() { + // r1 is both the oldest and the pointer target: r2 goes instead. + let delete = plan_gist_capacity_prune(&full_gist_snapshots(), 10, 10, Some("r1")); + + assert_eq!( + delete, + vec!["nyaterm/sync/snapshots/r2.redb.enc".to_string()] + ); + } + + #[test] + fn gist_capacity_prune_drops_unreadable_orphans_first() { + let mut snapshots = vec![ + SnapshotGcEntry { + path: "nyaterm/sync/snapshots/broken.redb.enc".to_string(), + revision_id: "broken".to_string(), + created_at_ms: 0, + deletable: false, + }, + SnapshotGcEntry { + path: "nyaterm/sync/snapshots/.redb.enc".to_string(), + revision_id: String::new(), + created_at_ms: 0, + deletable: false, + }, + ]; + snapshots.extend(full_gist_snapshots()); + + // Two slots needed: the orphans go first, no readable generation is lost. + let delete = plan_gist_capacity_prune(&snapshots, 11, 10, Some("r8")); + + assert_eq!( + delete, + vec![ + "nyaterm/sync/snapshots/broken.redb.enc".to_string(), + "nyaterm/sync/snapshots/.redb.enc".to_string(), + ] + ); + } + + const SNAPSHOT_LIST_PREFIX: &str = "nyaterm/sync/snapshots/"; + + /// A gist holding a single managed file; tests set the capacity limit they + /// need explicitly. + fn gist_remote_with_one_file(mark_gist_backend: bool) -> (MemoryRemote, CloudRemote) { + let memory = MemoryRemote::with_files(std::collections::HashMap::from([( + "nyaterm/sync/latest.redb".to_string(), + vec![1u8], + )])); + if mark_gist_backend { + memory.mark_gist_backend(); + } + let remote = CloudRemote::Memory(memory.clone()); + (memory, remote) + } + + #[tokio::test(flavor = "current_thread")] + async fn gist_prune_skips_the_remote_without_a_capacity_limit() { + let (memory, remote) = gist_remote_with_one_file(true); + memory.fail_next_list_containing(SNAPSHOT_LIST_PREFIX); + + // No limit yet: the prune must not read a single snapshot. + prune_gist_snapshots_before_upload(&remote, "nyaterm", None) + .await + .expect("nothing to prune without a capacity limit"); + + memory.set_file_capacity_limit(1); + assert!( + prune_gist_snapshots_before_upload(&remote, "nyaterm", None) + .await + .is_err(), + "the pending injected failure proves the first call never listed snapshots" + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn gist_prune_stops_after_the_cheap_count_when_there_is_room() { + let (memory, remote) = gist_remote_with_one_file(true); + memory.set_file_capacity_limit(10); + memory.fail_next_list_containing(SNAPSHOT_LIST_PREFIX); + + prune_gist_snapshots_before_upload(&remote, "nyaterm", None) + .await + .expect("one file is far below the limit"); + + memory.set_file_capacity_limit(1); + assert!( + prune_gist_snapshots_before_upload(&remote, "nyaterm", None) + .await + .is_err(), + "the pending injected failure proves the first call never listed snapshots" + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn gist_best_effort_prune_swallows_remote_failures() { + let (memory, remote) = gist_remote_with_one_file(true); + memory.set_file_capacity_limit(1); + memory.fail_next_list_containing(SNAPSHOT_LIST_PREFIX); + + prune_gist_snapshots_best_effort(&remote, "nyaterm", None).await; + + memory.fail_next_list_containing(SNAPSHOT_LIST_PREFIX); + assert!( + prune_gist_snapshots_before_upload(&remote, "nyaterm", None) + .await + .is_err(), + "the injected list failure must be real" + ); + } + + #[tokio::test(flavor = "current_thread")] + async fn gist_best_effort_prune_skips_non_gist_backends() { + let (memory, remote) = gist_remote_with_one_file(false); + memory.fail_next_list_containing(SNAPSHOT_LIST_PREFIX); + + // A non-gist backend must not even talk to the remote here. + prune_gist_snapshots_best_effort(&remote, "nyaterm", None).await; + + memory.set_file_capacity_limit(1); + assert!( + prune_gist_snapshots_before_upload(&remote, "nyaterm", None) + .await + .is_err(), + "the pending injected failure proves the best-effort call stayed local" + ); + } + #[test] fn is_gist_provider_detects_snippet_backends() { assert!(is_gist_provider("gitee_snippet")); diff --git a/src-tauri/src/core/cloud_sync/manager.rs b/src-tauri/src/core/cloud_sync/manager.rs index 5951f01b..c52069a6 100644 --- a/src-tauri/src/core/cloud_sync/manager.rs +++ b/src-tauri/src/core/cloud_sync/manager.rs @@ -16,7 +16,7 @@ use crate::error::{AppError, AppResult, CloudSyncError}; use super::crypto::require_master_password; use super::gc::{ SYNC_SNAPSHOT_GC_GRACE_PERIOD, cleanup_sync_snapshots, is_gist_provider, - prune_gist_snapshots_before_upload, + prune_gist_snapshots_best_effort, }; use super::history_log::{log_history_entry, read_cloud_sync_history_from_logs}; use super::migration::{ @@ -964,13 +964,12 @@ impl CloudSyncManager { let pointer = pointer_from_snapshot(&envelope); let gist_backend = is_gist_provider(&settings.provider); if gist_backend { - trace_cloud_sync_step(trigger, "prune_gist_snapshots_before_upload", async { - prune_gist_snapshots_before_upload(&remote, &settings.remote_root, latest.as_ref()) - .await - }) - .await?; + prune_gist_snapshots_step(trigger, &remote, &settings.remote_root, latest.as_ref()) + .await; } + // Gist snippets silently drop a new file once they are at capacity, so a + // failed upload/verification is retried once after freeing capacity. let mut upload_attempts = 0; loop { upload_attempts += 1; @@ -993,12 +992,13 @@ impl CloudSyncManager { attempt = upload_attempts, "Gist upload rejected; pruning and retrying once" ); - prune_gist_snapshots_before_upload( + prune_gist_snapshots_step( + trigger, &remote, &settings.remote_root, latest.as_ref(), ) - .await?; + .await; continue; } Err(error) => return Err(error), @@ -1015,8 +1015,10 @@ impl CloudSyncManager { && upload_attempts < 2 && matches!( &error, - AppError::CloudSync(CloudSyncError::SnapshotNotAccepted { .. }) - | AppError::CloudSync(CloudSyncError::RemoteFileRejected { .. }) + AppError::CloudSync( + CloudSyncError::SnapshotNotAccepted { .. } + | CloudSyncError::RemoteFileRejected { .. } + ) ) => { tracing::warn!( @@ -1024,12 +1026,13 @@ impl CloudSyncManager { attempt = upload_attempts, "Gist upload verify failed; pruning and retrying once" ); - prune_gist_snapshots_before_upload( + prune_gist_snapshots_step( + trigger, &remote, &settings.remote_root, latest.as_ref(), ) - .await?; + .await; } Err(error) => return Err(error), } @@ -1503,6 +1506,7 @@ impl CloudSyncManager { let provider = self.settings.lock().await.provider.clone(); let message = error.to_string(); + let error_code = cloud_sync_failure_code(&provider, error); self.append_history(CloudSyncHistoryEntry { id: uuid::Uuid::new_v4().to_string(), @@ -1516,7 +1520,14 @@ impl CloudSyncManager { message: message.clone(), }) .await; - self.set_status("failed", message, None, None).await; + self.set_status_with_code( + "failed", + message, + None, + None, + error_code.map(str::to_string), + ) + .await; self.record_automatic_retry_failure(trigger, error).await; } @@ -1641,6 +1652,18 @@ impl CloudSyncManager { message: String, current_operation: Option, conflict: Option, + ) { + self.set_status_with_code(state_value, message, current_operation, conflict, None) + .await; + } + + async fn set_status_with_code( + &self, + state_value: &str, + message: String, + current_operation: Option, + conflict: Option, + error_code: Option, ) { let app = self.app().ok(); let settings = self.settings.lock().await.clone(); @@ -1650,6 +1673,7 @@ impl CloudSyncManager { provider: settings.provider.clone(), state: state_value.to_string(), message, + error_code, current_operation, last_checked_at_ms: state.last_checked_at_ms, last_synced_at_ms: state.last_synced_at_ms, @@ -1671,6 +1695,7 @@ impl CloudSyncManager { status.provider = provider; status.state = if enabled { "idle" } else { "disabled" }.to_string(); status.message.clear(); + status.error_code = None; status.current_operation = None; status.conflict = None; status.clone() @@ -1750,6 +1775,21 @@ where result } +/// Free gist capacity before an upload. Remote inspection can fail on its own, so +/// this is best-effort: it is traced, but it never fails the push. +async fn prune_gist_snapshots_step( + trigger: &str, + remote: &super::operator::CloudRemote, + remote_root: &str, + latest: Option<&RemoteSyncPointer>, +) { + let _ = trace_cloud_sync_step(trigger, "prune_gist_snapshots_before_upload", async { + prune_gist_snapshots_best_effort(remote, remote_root, latest).await; + Ok::<(), AppError>(()) + }) + .await; +} + fn decide_remote_check( state: &CloudSyncState, local_hash: &str, @@ -1901,6 +1941,26 @@ fn is_non_retryable_automatic_error(error: &AppError) -> bool { } } +/// Machine-readable code for failures the UI offers a recovery action for. +/// +/// The frontend must not parse prose error text: keeping the code here means +/// rewording a message can never silently disable the recovery button. +fn cloud_sync_failure_code(provider: &str, error: &AppError) -> Option<&'static str> { + if !is_gist_provider(provider) { + return None; + } + + match error { + AppError::CloudSync( + CloudSyncError::SnapshotNotAccepted { .. } | CloudSyncError::RemoteFileRejected { .. }, + ) => Some(GIST_CAPACITY_ERROR_CODE), + _ => None, + } +} + +/// `CloudSyncStatus.error_code` value for "the gist is full". +pub(super) const GIST_CAPACITY_ERROR_CODE: &str = "gist_capacity"; + fn should_record_startup_check_failure(error: &AppError) -> bool { !matches!(error, AppError::Io(_)) } @@ -2213,6 +2273,70 @@ mod tests { assert!(!is_automatic_trigger("manual_test_connection")); } + #[test] + fn gist_capacity_failure_code_is_limited_to_gist_providers() { + let capacity_error = AppError::CloudSync(CloudSyncError::SnapshotNotAccepted { + revision: "r1".to_string(), + }); + + assert_eq!( + cloud_sync_failure_code("gitee_snippet", &capacity_error), + Some(GIST_CAPACITY_ERROR_CODE) + ); + assert_eq!( + cloud_sync_failure_code("github_gist", &capacity_error), + Some(GIST_CAPACITY_ERROR_CODE) + ); + assert_eq!(cloud_sync_failure_code("webdav", &capacity_error), None); + assert_eq!( + cloud_sync_failure_code( + "gitee_snippet", + &AppError::CloudSync(CloudSyncError::ConcurrentUpdate { + expected_revision: None, + actual_revision: None, + }) + ), + None + ); + } + + #[tokio::test] + async fn record_failure_publishes_the_gist_capacity_code() { + let manager = CloudSyncManager::new(); + manager.settings.lock().await.provider = "gitee_snippet".to_string(); + + manager + .record_failure( + "sync", + "manual_push", + &AppError::CloudSync(CloudSyncError::RemoteFileRejected { + filename: "nyaterm-x.blob".to_string(), + }), + ) + .await; + + let status = manager.status.lock().await.clone(); + assert_eq!(status.state, "failed"); + assert_eq!(status.error_code.as_deref(), Some(GIST_CAPACITY_ERROR_CODE)); + } + + #[tokio::test] + async fn record_failure_leaves_the_code_empty_for_unrelated_errors() { + let manager = CloudSyncManager::new(); + + manager + .record_failure( + "sync", + "manual_push", + &AppError::Io(std::io::Error::new(std::io::ErrorKind::TimedOut, "timeout")), + ) + .await; + + let status = manager.status.lock().await.clone(); + assert_eq!(status.state, "failed"); + assert!(status.error_code.is_none()); + } + #[tokio::test] async fn record_failure_clears_current_operation() { let manager = CloudSyncManager::new(); diff --git a/src-tauri/src/core/cloud_sync/migration.rs b/src-tauri/src/core/cloud_sync/migration.rs index 85c4f591..a4874e59 100644 --- a/src-tauri/src/core/cloud_sync/migration.rs +++ b/src-tauri/src/core/cloud_sync/migration.rs @@ -1,6 +1,7 @@ use crate::core::portable_snapshot::PortableSnapshot; use crate::error::{AppError, AppResult, CloudSyncError}; +use super::gc::prune_gist_snapshots_best_effort; use super::operator::CloudRemote; use super::protocol::{ commit_sync_pointer, pointer_from_snapshot, read_current_sync_snapshot_compat, @@ -95,6 +96,7 @@ pub(super) async fn migrate_legacy_snapshot( pointer: &RemoteSyncPointer, snapshot: &PortableSnapshot, ) -> AppResult<()> { + prune_gist_snapshots_best_effort(remote, remote_root, Some(pointer)).await; upload_sync_snapshot(remote, remote_root, snapshot).await?; verify_uploaded_sync_snapshot(remote, remote_root, pointer).await?; Ok(()) @@ -110,6 +112,8 @@ pub(super) async fn recover_current_remote_snapshot( )); }; let pointer = pointer_from_snapshot(&snapshot); + // Recovery rewrites a snapshot file, which needs a free slot on gists. + prune_gist_snapshots_best_effort(remote, remote_root, Some(&pointer)).await; upload_sync_snapshot(remote, remote_root, &snapshot).await?; verify_uploaded_sync_snapshot(remote, remote_root, &pointer).await?; commit_sync_pointer(remote, remote_root, &pointer).await?; diff --git a/src-tauri/src/core/cloud_sync/operator.rs b/src-tauri/src/core/cloud_sync/operator.rs index 7d3392e8..9e02f9e7 100644 --- a/src-tauri/src/core/cloud_sync/operator.rs +++ b/src-tauri/src/core/cloud_sync/operator.rs @@ -29,6 +29,10 @@ use super::remote::remote_path; const GITEE_REMOTE_FILE_PREFIX: &str = "nyaterm-"; const GITEE_REMOTE_FILE_SUFFIX: &str = ".blob"; const GITEE_REMOTE_TIMEOUT: Duration = Duration::from_secs(30); +/// Gitee stops storing new snippet files once the snippet holds this many files. +/// Observed empirically (the API answers 200 and silently drops the new file); +/// Gitee does not document the limit, so keep the value in one place. +const GITEE_GIST_FILE_LIMIT: usize = 10; const GITHUB_GIST_API_ENDPOINT: &str = "https://api.github.com"; const GITHUB_GIST_REMOTE_TIMEOUT: Duration = Duration::from_secs(30); const GITHUB_GIST_CONFLICT_RETRY_DELAY: Duration = Duration::from_millis(750); @@ -121,6 +125,47 @@ impl CloudRemote { Self::Memory(remote) => remote.list_files(path), } } + + /// Gist-style remotes keep every sync object in one flat file list, which is + /// subject to provider file-count limits. + pub(super) fn is_gist_backend(&self) -> bool { + match self { + Self::GiteeSnippet(_) | Self::GithubGist(_) => true, + Self::OpenDal(_) => false, + #[cfg(test)] + Self::Memory(remote) => remote.is_gist_backend(), + } + } + + /// Hard file-count limit of the underlying gist, if it has one. + /// + /// `None` means capacity pruning is skipped and only the regular retention + /// cleanup applies: GitHub gists have no per-gist file cap, so the value must + /// not be forced on them. + pub(super) fn file_capacity_limit(&self) -> Option { + match self { + Self::GiteeSnippet(_) => Some(GITEE_GIST_FILE_LIMIT), + Self::GithubGist(_) | Self::OpenDal(_) => None, + #[cfg(test)] + Self::Memory(remote) => remote.file_capacity_limit(), + } + } + + /// Number of files the gist holds, including files the sync layer does not + /// manage. + /// + /// [`Self::list_files`] only reports decodable `nyaterm-*.blob` entries, which + /// would under-count the snippet and make capacity math too optimistic. + /// `None` for backends without a flat gist file list. + pub(super) async fn gist_file_count(&self) -> AppResult> { + match self { + Self::GiteeSnippet(remote) => remote.file_count().await.map(Some), + Self::GithubGist(remote) => remote.file_count().await.map(Some), + Self::OpenDal(_) => Ok(None), + #[cfg(test)] + Self::Memory(remote) => Ok(Some(remote.file_count())), + } + } } fn map_optional_read(result: Result) -> AppResult>> { @@ -137,6 +182,10 @@ pub(super) struct MemoryRemote { files: Arc>>>, fail_writes: Arc>>, drop_writes: Arc>>, + fail_lists: Arc>>, + drop_new_writes_at_file_count: Arc>>, + gist_backend: Arc>, + capacity_limit: Arc>>, } #[cfg(test)] @@ -146,9 +195,31 @@ impl MemoryRemote { files: Arc::new(StdMutex::new(files)), fail_writes: Arc::new(StdMutex::new(Vec::new())), drop_writes: Arc::new(StdMutex::new(Vec::new())), + fail_lists: Arc::new(StdMutex::new(Vec::new())), + drop_new_writes_at_file_count: Arc::new(StdMutex::new(None)), + gist_backend: Arc::new(StdMutex::new(false)), + capacity_limit: Arc::new(StdMutex::new(None)), } } + /// Mark this fake remote as a gist-style backend so capacity handling applies. + pub(super) fn mark_gist_backend(&self) { + *self.gist_backend.lock().expect("lock gist backend") = true; + } + + pub(super) fn is_gist_backend(&self) -> bool { + *self.gist_backend.lock().expect("lock gist backend") + } + + /// Give the fake gist a hard file-count limit, like Gitee snippets have. + pub(super) fn set_file_capacity_limit(&self, limit: usize) { + *self.capacity_limit.lock().expect("lock capacity limit") = Some(limit); + } + + pub(super) fn file_capacity_limit(&self) -> Option { + *self.capacity_limit.lock().expect("lock capacity limit") + } + pub(super) fn fail_next_write_containing(&self, needle: &str) { self.fail_writes .lock() @@ -163,6 +234,27 @@ impl MemoryRemote { .push(needle.to_string()); } + pub(super) fn fail_next_list_containing(&self, needle: &str) { + self.fail_lists + .lock() + .expect("lock fail lists") + .push(needle.to_string()); + } + + /// Mimic gist providers that answer a PATCH with HTTP 200 but never store a + /// *new* file once the file-count limit is reached. Updates to existing files + /// keep working, exactly like the real snippet API. + pub(super) fn drop_new_writes_when_file_count_reaches(&self, limit: usize) { + *self + .drop_new_writes_at_file_count + .lock() + .expect("lock drop-new-writes limit") = Some(limit); + } + + pub(super) fn file_count(&self) -> usize { + self.files.lock().expect("lock files").len() + } + pub(super) fn file(&self, path: &str) -> Option> { self.files.lock().expect("lock files").get(path).cloned() } @@ -203,6 +295,17 @@ impl MemoryRemote { } drop(drop_writes); + let drop_new_writes_at_file_count = *self + .drop_new_writes_at_file_count + .lock() + .expect("lock drop-new-writes limit"); + if let Some(limit) = drop_new_writes_at_file_count { + let files = self.files.lock().expect("lock files"); + if !files.contains_key(path) && files.len() >= limit { + return Ok(()); + } + } + self.files .lock() .expect("lock files") @@ -216,6 +319,19 @@ impl MemoryRemote { } fn list_files(&self, path: &str) -> AppResult> { + let mut fail_lists = self.fail_lists.lock().expect("lock fail lists"); + if let Some(index) = fail_lists + .iter() + .position(|needle| path.contains(needle.as_str())) + { + fail_lists.remove(index); + return Err(AppError::Io(io::Error::new( + io::ErrorKind::Other, + format!("injected memory list failure for {path}"), + ))); + } + drop(fail_lists); + Ok(self .files .lock() @@ -540,8 +656,10 @@ pub(super) struct GiteeSnippetRemote { #[derive(Debug, serde::Deserialize)] struct GiteeSnippet { + /// `None` when the payload carries no `files` map at all, which must stay + /// distinguishable from an empty map (there the file really is absent). #[serde(default)] - files: HashMap, + files: Option>, } #[derive(Debug, serde::Deserialize)] @@ -591,7 +709,10 @@ impl GiteeSnippetRemote { async fn exists(&self, path: &str) -> AppResult { let snippet = self.fetch_snippet().await?; - Ok(snippet.files.contains_key(&gitee_remote_filename(path))) + Ok(snippet + .files + .as_ref() + .is_some_and(|files| files.contains_key(&gitee_remote_filename(path)))) } async fn read_if_exists(&self, path: &str) -> AppResult>> { @@ -601,7 +722,11 @@ impl GiteeSnippetRemote { } let snippet = self.fetch_snippet().await?; - let Some(file) = snippet.files.get(&filename) else { + let Some(file) = snippet + .files + .as_ref() + .and_then(|files| files.get(&filename)) + else { return Ok(None); }; let content = if file.truncated { @@ -633,10 +758,21 @@ impl GiteeSnippetRemote { let prefix = path.trim_start_matches('/'); Ok(snippet .files - .keys() - .filter_map(|filename| gitee_remote_path(filename)) - .filter(|remote_path| remote_path.starts_with(prefix)) - .collect()) + .as_ref() + .map(|files| { + files + .keys() + .filter_map(|filename| gitee_remote_path(filename)) + .filter(|remote_path| remote_path.starts_with(prefix)) + .collect() + }) + .unwrap_or_default()) + } + + /// Every file in the snippet, including entries the sync layer does not manage. + async fn file_count(&self) -> AppResult { + let snippet = self.fetch_snippet().await?; + Ok(snippet.files.as_ref().map_or(0, HashMap::len)) } async fn fetch_snippet(&self) -> AppResult { @@ -711,8 +847,11 @@ impl GiteeSnippetRemote { .await .map_err(map_gitee_client_error)?; let snippet: GiteeSnippet = decode_gitee_response(response).await?; - let response_files: HashSet = snippet.files.keys().cloned().collect(); - ensure_gist_patch_accepted(&files, &response_files)?; + let response_files = snippet + .files + .as_ref() + .map(|files| files.keys().cloned().collect::>()); + ensure_gist_patch_accepted(&files, response_files.as_ref())?; Ok(()) } } @@ -744,10 +883,20 @@ fn gitee_patch_body( }) } +/// Verify that the gist update really applied what we asked for. +/// +/// `response_filenames` is `None` when the provider answered without a `files` +/// map at all; that must not be mistaken for "every requested file is missing", +/// otherwise every write would be reported as rejected. fn ensure_gist_patch_accepted( requested: &serde_json::Map, - response_filenames: &HashSet, + response_filenames: Option<&HashSet>, ) -> AppResult<()> { + let Some(response_filenames) = response_filenames else { + tracing::warn!("Gist update response carried no files map; skipping write verification"); + return Ok(()); + }; + for (filename, value) in requested { let should_exist = !value.is_null(); let exists = response_filenames.contains(filename); @@ -811,8 +960,9 @@ pub(super) struct GithubGistRemote { #[derive(Debug, serde::Deserialize)] struct GithubGist { + /// `None` when the payload carries no `files` map at all; see [`GiteeSnippet`]. #[serde(default)] - files: HashMap, + files: Option>, } #[derive(Debug, serde::Deserialize)] @@ -852,13 +1002,16 @@ impl GithubGistRemote { async fn exists(&self, path: &str) -> AppResult { let gist = self.fetch_gist().await?; - Ok(gist.files.contains_key(&github_gist_remote_filename(path))) + Ok(gist + .files + .as_ref() + .is_some_and(|files| files.contains_key(&github_gist_remote_filename(path)))) } async fn read_if_exists(&self, path: &str) -> AppResult>> { let filename = github_gist_remote_filename(path); let gist = self.fetch_gist().await?; - let Some(file) = gist.files.get(&filename) else { + let Some(file) = gist.files.as_ref().and_then(|files| files.get(&filename)) else { return Ok(None); }; let content = if file.truncated { @@ -891,10 +1044,21 @@ impl GithubGistRemote { let prefix = path.trim_start_matches('/'); Ok(gist .files - .keys() - .filter_map(|filename| github_gist_remote_path(filename)) - .filter(|remote_path| remote_path.starts_with(prefix)) - .collect()) + .as_ref() + .map(|files| { + files + .keys() + .filter_map(|filename| github_gist_remote_path(filename)) + .filter(|remote_path| remote_path.starts_with(prefix)) + .collect() + }) + .unwrap_or_default()) + } + + /// Every file in the gist, including entries the sync layer does not manage. + async fn file_count(&self) -> AppResult { + let gist = self.fetch_gist().await?; + Ok(gist.files.as_ref().map_or(0, HashMap::len)) } async fn fetch_gist(&self) -> AppResult { @@ -971,8 +1135,11 @@ impl GithubGistRemote { .await .map_err(map_github_gist_client_error)?; let gist: GithubGist = decode_github_gist_response(response).await?; - let response_files: HashSet = gist.files.keys().cloned().collect(); - ensure_gist_patch_accepted(files, &response_files)?; + let response_files = gist + .files + .as_ref() + .map(|files| files.keys().cloned().collect::>()); + ensure_gist_patch_accepted(files, response_files.as_ref())?; Ok(()) } } @@ -1326,7 +1493,8 @@ mod tests { serde_json::json!({ "content": "abc" }), ); let response = HashSet::from(["nyaterm-old.blob".to_string()]); - let err = ensure_gist_patch_accepted(&requested, &response).expect_err("missing file"); + let err = + ensure_gist_patch_accepted(&requested, Some(&response)).expect_err("missing file"); assert!(matches!( err, AppError::CloudSync(crate::error::CloudSyncError::RemoteFileRejected { filename }) @@ -1342,7 +1510,29 @@ mod tests { serde_json::json!({ "content": "abc" }), ); let response = HashSet::from(["nyaterm-new.blob".to_string()]); - ensure_gist_patch_accepted(&requested, &response).expect("accepted"); + ensure_gist_patch_accepted(&requested, Some(&response)).expect("accepted"); + } + + #[test] + fn gist_patch_without_files_map_is_not_a_rejection() { + let mut requested = serde_json::Map::new(); + requested.insert( + "nyaterm-new.blob".to_string(), + serde_json::json!({ "content": "abc" }), + ); + + ensure_gist_patch_accepted(&requested, None) + .expect("a response without a files map must not fail the write"); + } + + #[test] + fn gist_patch_detects_ignored_deletion() { + let mut requested = serde_json::Map::new(); + requested.insert("nyaterm-old.blob".to_string(), serde_json::Value::Null); + let response = HashSet::from(["nyaterm-old.blob".to_string()]); + let err = ensure_gist_patch_accepted(&requested, Some(&response)) + .expect_err("deleted file still present"); + assert!(matches!(err, AppError::Config(message) if message.contains("still contains"))); } #[test] @@ -1482,6 +1672,48 @@ mod tests { assert_eq!(github_gist_remote_path(&filename).as_deref(), Some(path)); } + #[tokio::test(flavor = "current_thread")] + async fn gist_file_count_includes_files_nyaterm_does_not_manage() { + let memory = MemoryRemote::with_files(HashMap::from([ + ("nyaterm/sync/latest.redb".to_string(), vec![1u8]), + ("notes.txt".to_string(), vec![2u8]), + ])); + let remote = CloudRemote::Memory(memory); + + assert_eq!(remote.gist_file_count().await.expect("count"), Some(2)); + assert!( + remote + .list_files("nyaterm/sync/snapshots/") + .await + .expect("list") + .is_empty(), + "the prefix listing still only reports managed sync documents" + ); + } + + #[test] + fn gist_capacity_limit_is_gitee_only() { + let mut settings = CloudSyncSettings { + provider: "gitee_snippet".to_string(), + ..CloudSyncSettings::default() + }; + settings.gitee_snippet.api_endpoint = "https://gitee.com/api/v5".to_string(); + settings.gitee_snippet.gist_id = "abc".to_string(); + settings.gitee_snippet.access_token = Some("token".to_string()); + let gitee = build_remote(&settings).expect("build gitee remote"); + assert_eq!(gitee.file_capacity_limit(), Some(GITEE_GIST_FILE_LIMIT)); + + settings.provider = "github_gist".to_string(); + settings.github_gist.gist_id = "abc".to_string(); + settings.github_gist.access_token = Some("token".to_string()); + let github = build_remote(&settings).expect("build github remote"); + assert_eq!( + github.file_capacity_limit(), + None, + "GitHub gists have no documented per-gist file cap" + ); + } + #[test] fn github_gist_file_deserializes_truncated_flag() { let file: GithubGistFile = serde_json::from_str( diff --git a/src-tauri/src/core/cloud_sync/protocol.rs b/src-tauri/src/core/cloud_sync/protocol.rs index 45d37ac9..217e3f48 100644 --- a/src-tauri/src/core/cloud_sync/protocol.rs +++ b/src-tauri/src/core/cloud_sync/protocol.rs @@ -264,7 +264,9 @@ mod tests { use crate::error::{AppError, CloudSyncError}; use crate::utils::crypto::set_master_password; - use super::super::migration::{RemoteSnapshotResolution, resolve_remote_snapshot}; + use super::super::migration::{ + RemoteSnapshotResolution, recover_current_remote_snapshot, resolve_remote_snapshot, + }; use super::super::operator::MemoryRemote; use super::super::remote::{load_sync_pointer, remote_path}; use super::*; @@ -521,6 +523,82 @@ mod tests { set_master_password(None); } + fn encrypted_snapshot_file(snapshot: &PortableSnapshot) -> Vec { + encrypt_snapshot_bytes(&encode_portable_snapshot(snapshot).expect("encode snapshot")) + .expect("encrypt snapshot") + } + + /// A gist at its file limit: latest pointer + current snapshot + 8 generations. + /// The fake remote silently drops new files from then on, like Gitee does. + fn full_gist_remote(with_capacity_handling: bool) -> (MemoryRemote, CloudRemote) { + let mut files = HashMap::new(); + for index in 1..=8u64 { + let revision = format!("r{index}"); + files.insert( + sync_snapshot_path("nyaterm", &revision), + encrypted_snapshot_file(&sample_snapshot(&revision, index)), + ); + } + files.insert( + remote_path("nyaterm", SYNC_CURRENT_FILE), + encrypted_snapshot_file(&sample_snapshot("r-new", 100)), + ); + + let memory = MemoryRemote::with_files(files); + if with_capacity_handling { + memory.mark_gist_backend(); + memory.set_file_capacity_limit(10); + } + memory.drop_new_writes_when_file_count_reaches(10); + let remote = CloudRemote::Memory(memory.clone()); + (memory, remote) + } + + #[tokio::test(flavor = "current_thread")] + async fn recovery_fails_on_a_full_gist_without_capacity_handling() { + let _guard = MASTER_PASSWORD_TEST_LOCK.lock().expect("lock password"); + set_master_password(Some("secret".to_string())); + let (memory, remote) = full_gist_remote(false); + let latest = pointer_from_snapshot(&sample_snapshot("r8", 8)); + commit_sync_pointer(&remote, "nyaterm", &latest) + .await + .expect("seed pointer"); + assert_eq!(memory.file_count(), 10); + + let result = recover_current_remote_snapshot(&remote, "nyaterm").await; + + assert!(matches!( + result, + Err(AppError::CloudSync(CloudSyncError::SnapshotNotAccepted { revision })) + if revision == "r-new" + )); + set_master_password(None); + } + + #[tokio::test(flavor = "current_thread")] + async fn recovery_prunes_a_full_gist_before_uploading() { + let _guard = MASTER_PASSWORD_TEST_LOCK.lock().expect("lock password"); + set_master_password(Some("secret".to_string())); + let (memory, remote) = full_gist_remote(true); + let latest = pointer_from_snapshot(&sample_snapshot("r8", 8)); + commit_sync_pointer(&remote, "nyaterm", &latest) + .await + .expect("seed pointer"); + assert_eq!(memory.file_count(), 10); + + let recovered = recover_current_remote_snapshot(&remote, "nyaterm") + .await + .expect("recover from a full gist"); + + assert_eq!(recovered.revision_id, "r-new"); + assert!(memory.file_count() <= 10); + let pointer = load_sync_pointer(&remote, "nyaterm") + .await + .expect("load pointer") + .expect("pointer"); + assert_eq!(pointer.revision_id, "r-new"); + set_master_password(None); + } #[tokio::test(flavor = "current_thread")] async fn pointer_write_failure_keeps_old_revision_readable() { let _guard = MASTER_PASSWORD_TEST_LOCK.lock().expect("lock password"); diff --git a/src/components/settings/SyncBackupTab.tsx b/src/components/settings/SyncBackupTab.tsx index 86b4658a..71135e69 100644 --- a/src/components/settings/SyncBackupTab.tsx +++ b/src/components/settings/SyncBackupTab.tsx @@ -15,8 +15,8 @@ import { formatCloudProvider, formatTimestamp, getCloudSyncValidationErrors, - isGistCapacitySyncFailure, isRemoteInconsistentConflict, + needsGistCapacityRecovery, secretInputValue, secretPlaceholder, shortValue, @@ -124,7 +124,8 @@ export function SyncBackupTab({ onNavigateSecurity }: SyncBackupTabProps) { const isBusy = loading || isSaving || runningAction !== null; const isRemoteInconsistent = isRemoteInconsistentConflict(status.conflict); const showGistCapacityRecovery = - status.state === "failed" && isGistCapacitySyncFailure(status.message); + status.state === "failed" && + needsGistCapacityRecovery(status.provider, status.message, status.error_code); const updateCloudSync = useCallback( (patch: Partial) => { @@ -1075,10 +1076,8 @@ export function SyncBackupTab({ onNavigateSecurity }: SyncBackupTabProps) { {showGistCapacityRecovery ? (