diff --git a/Termind/Application/Connections/RemoteConnectionCoordinator.swift b/Termind/Application/Connections/RemoteConnectionCoordinator.swift index 3f7beb0d..45ff9acb 100644 --- a/Termind/Application/Connections/RemoteConnectionCoordinator.swift +++ b/Termind/Application/Connections/RemoteConnectionCoordinator.swift @@ -18,11 +18,15 @@ nonisolated final class RemoteConnectionCoordinator: Sendable { func connectSFTP( using configuration: SSHConnectionConfiguration, - progress: SSHRouteProgressHandler? = nil + progress: SSHRouteProgressHandler? = nil, + connectionFailure: (@Sendable (String) -> Void)? = nil ) async throws -> any SFTPClientProtocol { let client = makeSFTPClient() do { - try await client.connect(using: configuration, progress: progress) + try await client.connect( + using: configuration, progress: progress, + connectionFailure: connectionFailure + ) return client } catch { await client.disconnect() diff --git a/Termind/Application/Connections/SessionReconnectController.swift b/Termind/Application/Connections/SessionReconnectController.swift new file mode 100644 index 00000000..0880a238 --- /dev/null +++ b/Termind/Application/Connections/SessionReconnectController.swift @@ -0,0 +1,139 @@ +import Foundation + +nonisolated struct ReconnectPolicy: Sendable, Equatable { + static let sessionDefault = ReconnectPolicy() + + var delays: [Duration] + var stableResetInterval: TimeInterval + + init( + delays: [Duration] = [.seconds(1), .seconds(2), .seconds(5), .seconds(10), .seconds(30)], + stableResetInterval: TimeInterval = 60 + ) { + precondition(!delays.isEmpty) + self.delays = delays + self.stableResetInterval = stableResetInterval + } + + func delay(forAttempt attempt: Int) -> Duration { + delays[min(max(0, attempt - 1), delays.count - 1)] + } +} + +nonisolated struct ReconnectState: Equatable, Sendable { + var attempt: Int + var nextRetry: Date + var lastError: String +} + +/// Owns only retry timing and stale-callback rejection. The session models keep +/// transport construction because they also own authentication and host-key UI. +@Observable +final class SessionReconnectController { + typealias RetryAction = @MainActor @Sendable () -> Void + typealias Sleep = @Sendable (Duration) async throws -> Void + typealias Now = @Sendable () -> Date + + private(set) var state: ReconnectState? + + private let policy: ReconnectPolicy + private let sleep: Sleep + private let now: Now + private var retryTask: Task? + private var retryAction: RetryAction? + private var attempts = 0 + private var connectedAt: Date? + private var generation: UInt64 = 0 + + init( + policy: ReconnectPolicy = .sessionDefault, + sleep: @escaping Sleep = { try await Task.sleep(for: $0) }, + now: @escaping Now = Date.init + ) { + self.policy = policy + self.sleep = sleep + self.now = now + } + + func markConnected(initial: Bool) { + cancelScheduledRetry() + state = nil + retryAction = nil + connectedAt = now() + if initial { attempts = 0 } + } + + func schedule(after error: String, retry: @escaping RetryAction) { + cancelScheduledRetry() + if let connectedAt, + now().timeIntervalSince(connectedAt) >= policy.stableResetInterval + { + attempts = 0 + } + connectedAt = nil + attempts += 1 + retryAction = retry + let delay = policy.delay(forAttempt: attempts) + state = ReconnectState( + attempt: attempts, + nextRetry: now().addingTimeInterval(delay.timeInterval), + lastError: error + ) + schedule(delay: delay) + } + + func retryNow() { + guard let retryAction else { return } + cancelScheduledRetry() + generation &+= 1 + let expectedGeneration = generation + retryTask = Task { [weak self] in + guard let self, self.generation == expectedGeneration else { return } + retryAction() + } + } + + func stop() { + cancelScheduledRetry() + generation &+= 1 + state = nil + retryAction = nil + attempts = 0 + connectedAt = nil + } + + /// Pauses retry timing without discarding the attempt counter. Used while + /// the connection journey waits for credentials or a host-key decision. + func pauseForUserInteraction() { + cancelScheduledRetry() + generation &+= 1 + state = nil + retryAction = nil + } + + private func schedule(delay: Duration) { + generation &+= 1 + let expectedGeneration = generation + let sleep = sleep + retryTask = Task { [weak self] in + do { try await sleep(delay) } catch { return } + guard let self, self.generation == expectedGeneration, + let retryAction = self.retryAction + else { return } + retryAction() + } + } + + private func cancelScheduledRetry() { + retryTask?.cancel() + retryTask = nil + } +} + +private nonisolated extension Duration { + var timeInterval: TimeInterval { + let components = self.components + return TimeInterval(components.seconds) + + TimeInterval(components.attoseconds) / 1_000_000_000_000_000_000 + } +} diff --git a/Termind/Core/SSH/SFTPClient.swift b/Termind/Core/SSH/SFTPClient.swift index b85772d3..ad3bf499 100644 --- a/Termind/Core/SSH/SFTPClient.swift +++ b/Termind/Core/SSH/SFTPClient.swift @@ -62,6 +62,11 @@ nonisolated protocol SFTPClientProtocol: Sendable { using configuration: SSHConnectionConfiguration, progress: SSHRouteProgressHandler? ) async throws + func connect( + using configuration: SSHConnectionConfiguration, + progress: SSHRouteProgressHandler?, + connectionFailure: (@Sendable (String) -> Void)? + ) async throws func disconnect() async func homeDirectory() async throws -> String func listDirectory(at path: String) async throws -> [RemoteFile] @@ -92,6 +97,14 @@ nonisolated protocol SFTPClientProtocol: Sendable { } nonisolated extension SFTPClientProtocol { + func connect( + using configuration: SSHConnectionConfiguration, + progress: SSHRouteProgressHandler?, + connectionFailure: (@Sendable (String) -> Void)? + ) async throws { + try await connect(using: configuration, progress: progress) + } + func copyFile( from sourcePath: String, to destinationPath: String, @@ -107,6 +120,7 @@ nonisolated extension SFTPClientProtocol { nonisolated enum SFTPError: LocalizedError, Sendable { case notConnected + case connectionLost(String) case cancelled case localFile(String) case operation(String) @@ -114,6 +128,7 @@ nonisolated enum SFTPError: LocalizedError, Sendable { var errorDescription: String? { switch self { case .notConnected: String(localized: "The SFTP connection is not available.") + case .connectionLost(let message): message case .cancelled: String(localized: "Transfer cancelled.") case .localFile(let message), .operation(let message): message } @@ -130,6 +145,17 @@ actor SFTPClient: SFTPClientProtocol { try await worker.connect(using: configuration, progress: progress) } + func connect( + using configuration: SSHConnectionConfiguration, + progress: SSHRouteProgressHandler?, + connectionFailure: (@Sendable (String) -> Void)? + ) async throws { + try await worker.connect( + using: configuration, progress: progress, + connectionFailure: connectionFailure + ) + } + func disconnect() async { await worker.disconnect() } func homeDirectory() async throws -> String { @@ -271,12 +297,15 @@ private nonisolated final class SFTPWorker: @unchecked Sendable { private var session: OpaquePointer? private var sftp: OpaquePointer? private var keepaliveTimer: (any DispatchSourceTimer)? + private var connectionFailure: (@Sendable (String) -> Void)? + private var didReportConnectionFailure = false deinit { cleanup() } func connect( using configuration: SSHConnectionConfiguration, - progress: SSHRouteProgressHandler? + progress: SSHRouteProgressHandler?, + connectionFailure: (@Sendable (String) -> Void)? = nil ) async throws { try await perform { self.cleanup() @@ -313,6 +342,8 @@ private nonisolated final class SFTPWorker: @unchecked Sendable { throw SSHSessionWorker.sessionError(session, prefix: "Unable to start SFTP") } self.sftp = sftp + self.connectionFailure = connectionFailure + self.didReportConnectionFailure = false self.startKeepalive(for: session) } } @@ -718,7 +749,21 @@ private nonisolated final class SFTPWorker: @unchecked Sendable { private func error(_ prefix: String) -> SFTPError { guard let sftp else { return .operation(prefix) } - return .operation("\(prefix): SFTP error \(libssh2_sftp_last_error(sftp))") + let sftpCode = libssh2_sftp_last_error(sftp) + let sessionCode = session.map(libssh2_session_last_errno) ?? 0 + let message = "\(prefix): SFTP error \(sftpCode)" + let lostSFTPCodes = [ + UInt(LIBSSH2_FX_NO_CONNECTION), UInt(LIBSSH2_FX_CONNECTION_LOST), + ] + let lostSessionCodes = [ + LIBSSH2_ERROR_SOCKET_SEND, LIBSSH2_ERROR_SOCKET_RECV, + LIBSSH2_ERROR_TIMEOUT, LIBSSH2_ERROR_CHANNEL_CLOSED, + ] + if lostSFTPCodes.contains(sftpCode) || lostSessionCodes.contains(sessionCode) { + reportConnectionFailure(message) + return .connectionLost(message) + } + return .operation(message) } private func cleanup() { @@ -732,6 +777,8 @@ private nonisolated final class SFTPWorker: @unchecked Sendable { } routeTransport?.close() routeTransport = nil + connectionFailure = nil + didReportConnectionFailure = false } private func startKeepalive(for session: OpaquePointer) { @@ -741,12 +788,24 @@ private nonisolated final class SFTPWorker: @unchecked Sendable { timer.setEventHandler { [weak self] in guard let self, self.session == session else { return } var secondsToNext: Int32 = 0 - _ = libssh2_keepalive_send(session, &secondsToNext) + let result = libssh2_keepalive_send(session, &secondsToNext) + if result < 0, result != LIBSSH2_ERROR_EAGAIN { + let error = SSHSessionWorker.sessionError( + session, prefix: "SFTP keepalive failed" + ) + self.reportConnectionFailure(error.localizedDescription) + } } keepaliveTimer = timer timer.resume() } + private func reportConnectionFailure(_ message: String) { + guard !didReportConnectionFailure else { return } + didReportConnectionFailure = true + connectionFailure?(message) + } + private func perform(_ operation: @escaping @Sendable () throws -> T) async throws -> T { try await withCheckedThrowingContinuation { continuation in queue.async { diff --git a/Termind/Core/SSH/SSHConnectionJourney.swift b/Termind/Core/SSH/SSHConnectionJourney.swift index 50c9150d..24e6070a 100644 --- a/Termind/Core/SSH/SSHConnectionJourney.swift +++ b/Termind/Core/SSH/SSHConnectionJourney.swift @@ -46,6 +46,11 @@ nonisolated struct SSHAuthenticationSubmission: Sendable { /// a completed path backwards. @Observable final class SSHConnectionJourney { + enum Mode: Equatable { + case connection + case reconnection + } + enum Phase: Equatable { case hidden case connecting @@ -72,6 +77,7 @@ final class SSHConnectionJourney { } private(set) var purpose: SSHConnectionPurpose = .sshSession + private(set) var mode: Mode = .connection private(set) var endpoints: [SSHConnectionEndpoint] = [] private(set) var phase: Phase = .hidden private(set) var activeEndpointIndex = 0 @@ -87,7 +93,10 @@ final class SSHConnectionJourney { var isPresented: Bool { phase != .hidden } var destination: SSHConnectionEndpoint? { endpoints.last } - func begin(host: Host, route: [Host], purpose: SSHConnectionPurpose, username: String?) { + func begin( + host: Host, route: [Host], purpose: SSHConnectionPurpose, + username: String?, mode: Mode = .connection + ) { completionTask?.cancel() let nextEndpoints = [ @@ -113,6 +122,7 @@ final class SSHConnectionJourney { let retainedLogVisibility = continuesJourney && showsLogs self.purpose = purpose + self.mode = mode endpoints = nextEndpoints phase = .connecting activeEndpointIndex = retainedCompletedEndpoint @@ -122,9 +132,10 @@ final class SSHConnectionJourney { waitsForLocalIdentity = false logs = retainedLogs showsLogs = retainedLogVisibility + let action = mode == .reconnection ? "Reconnecting " : "Starting " log( endpoint: endpoints.first?.name ?? String(localized: "This Mac"), - message: "Starting " + message: action + (purpose == .sshSession ? "SSH session" : "SFTP session") + " connection to " + host.name ) diff --git a/Termind/Core/SSH/SSHSessionActor.swift b/Termind/Core/SSH/SSHSessionActor.swift index cbd00f62..e33dcab2 100644 --- a/Termind/Core/SSH/SSHSessionActor.swift +++ b/Termind/Core/SSH/SSHSessionActor.swift @@ -1087,7 +1087,12 @@ nonisolated final class SSHSessionWorker: @unchecked Sendable { } var next = Int32(0) - _ = libssh2_keepalive_send(session, &next) + let keepalive = libssh2_keepalive_send(session, &next) + if keepalive < 0, keepalive != LIBSSH2_ERROR_EAGAIN { + reportExit(.failure(Self.sessionError(session, prefix: "SSH keepalive failed"))) + cleanup() + return + } if libssh2_channel_eof(channel) == 1 { let status = libssh2_channel_get_exit_status(channel) reportExit(.success(status)) diff --git a/Termind/Features/SFTP/SFTPBrowserView.swift b/Termind/Features/SFTP/SFTPBrowserView.swift index eb610370..840ab989 100644 --- a/Termind/Features/SFTP/SFTPBrowserView.swift +++ b/Termind/Features/SFTP/SFTPBrowserView.swift @@ -68,7 +68,10 @@ struct SFTPBrowserView: View { cancelConnection() }, retry: { sftpSession.connect(using: app.hosts) }, - cancel: cancelConnection + cancel: cancelConnection, + reconnectState: sftpSession.reconnectState, + retryReconnectNow: sftpSession.retryReconnectNow, + stopReconnecting: sftpSession.stopReconnecting ) .zIndex(10) } @@ -413,10 +416,12 @@ struct SFTPBrowserView: View { renameTarget = file renameName = file.name } + .disabled(!sftpSession.allowsRemoteMutations) } Button(targets.count == 1 ? "Delete" : "Delete \(targets.count) Items", role: .destructive) { deleteTargets = targets } + .disabled(!sftpSession.allowsRemoteMutations) Divider() Button("New Folder") { newFolderPresented = true } .disabled(sftpSession.status != .connected) diff --git a/Termind/Features/SFTP/SFTPSessionModel.swift b/Termind/Features/SFTP/SFTPSessionModel.swift index 91edfb15..7cfc4317 100644 --- a/Termind/Features/SFTP/SFTPSessionModel.swift +++ b/Termind/Features/SFTP/SFTPSessionModel.swift @@ -74,6 +74,9 @@ final class SFTPSessionModel: Identifiable { private var suppliedAuthentications: [Host.ID: SSHAuthenticationSubmission] = [:] private var temporaryHostKeys: [Host.ID: SSHHostKeyIdentity] = [:] private var connectionAttemptID: UUID? + private let reconnectController = SessionReconnectController() + private var hasConnectedOnce = false + private var isRecoveringConnection = false private var operationCancellation: TransferCancellationToken? private weak var statusStore: HostStore? private weak var transferCoordinator: SFTPTransferCoordinator? @@ -99,8 +102,12 @@ final class SFTPSessionModel: Identifiable { self.didResolveInitialPath = !configuredPath.isEmpty self.automaticRefreshInterval = automaticRefreshInterval if client != nil { status = .connected } + if client != nil { hasConnectedOnce = true } } + var reconnectState: ReconnectState? { reconnectController.state } + var allowsRemoteMutations: Bool { status == .connected } + var canGoBack: Bool { !backHistory.isEmpty } var canGoForward: Bool { !forwardHistory.isEmpty } var canGoUp: Bool { path != "/" } @@ -132,9 +139,16 @@ final class SFTPSessionModel: Identifiable { } func connect(using store: HostStore) { + startConnection(using: store) + } + + private func startConnection(using store: HostStore) { + let reconnecting = hasConnectedOnce && isRecoveringConnection bindStatus(to: store) connectionTask?.cancel() guard let host = store.host(hostID) else { + reconnectController.stop() + isRecoveringConnection = false let message = String(localized: "The host is no longer available.") status = .failed(message) connectionJourney.fail(message) @@ -146,10 +160,16 @@ final class SFTPSessionModel: Identifiable { let suppliedAuthentication = suppliedAuthentications[host.id] connectionJourney.begin( host: host, route: route, purpose: .sftpSession, - username: suppliedAuthentication?.username ?? store.identity(for: host)?.username + username: suppliedAuthentication?.username ?? store.identity(for: host)?.username, + mode: reconnecting ? .reconnection : .connection ) - status = .connecting - hint = String(localized: "Connecting") + if reconnecting, usesEphemeralCredentials, suppliedAuthentication == nil { + requestAuthentication(host: host, store: store) + return + } + status = reconnecting ? .reconnecting : .connecting + hint = reconnecting + ? String(localized: "Reconnecting") : String(localized: "Connecting") connectionTask = Task { [weak self] in guard let self else { return } do { @@ -190,7 +210,12 @@ final class SFTPSessionModel: Identifiable { } let coordinator = self.connectionCoordinator ?? RemoteConnectionCoordinator() let client = try await coordinator.connectSFTP( - using: configuration, progress: progressHandler(attemptID: attemptID) + using: configuration, progress: progressHandler(attemptID: attemptID), + connectionFailure: { [weak self] message in + Task { @MainActor in + self?.handleConnectionFailure(message, attemptID: attemptID) + } + } ) try Task.checkCancellation() try await self.resolveInitialPathIfNeeded(using: client) @@ -200,12 +225,22 @@ final class SFTPSessionModel: Identifiable { self.suppliedAuthentications.removeAll() self.temporaryHostKeys.removeAll() self.pendingAuthentication = nil + let initialConnection = !self.hasConnectedOnce + let presentsConnectionJourney = self.connectionJourney.isPresented + self.hasConnectedOnce = true + self.isRecoveringConnection = false + self.reconnectController.markConnected(initial: initialConnection) self.status = .connected - self.connectionJourney.succeed() + if presentsConnectionJourney { + self.connectionJourney.succeed() + } else { + self.connectionJourney.hide() + } self.hint = "" await self.reload() self.startAutomaticRefresh() } catch is CancellationError { + guard self.connectionAttemptID == attemptID else { return } self.status = .disconnected } catch TermindSSHError.untrustedHost { await self.discoverHostKey(host: host, store: store) @@ -238,11 +273,15 @@ final class SFTPSessionModel: Identifiable { ? .serverChallenge : .localIdentity ) } catch { - self.status = .failed(error.localizedDescription) - self.hint = error.localizedDescription - self.connectionJourney.fail( - error.localizedDescription, at: self.routeHostID(from: error) - ) + if self.hasConnectedOnce || reconnecting { + self.scheduleReconnect(after: error.localizedDescription, using: store) + } else { + self.status = .failed(error.localizedDescription) + self.hint = error.localizedDescription + self.connectionJourney.fail( + error.localizedDescription, at: self.routeHostID(from: error) + ) + } } } } @@ -265,6 +304,9 @@ final class SFTPSessionModel: Identifiable { connectionTask?.cancel() connectionTask = nil connectionAttemptID = nil + reconnectController.stop() + hasConnectedOnce = false + isRecoveringConnection = false automaticRefreshTask?.cancel() automaticRefreshTask = nil uploadRefreshTask?.cancel() @@ -283,6 +325,28 @@ final class SFTPSessionModel: Identifiable { } } + func retryReconnectNow() { + reconnectController.retryNow() + } + + func stopReconnecting() { + guard let reconnectState else { return } + reconnectController.stop() + isRecoveringConnection = false + connectionAttemptID = nil + connectionTask?.cancel() + connectionTask = nil + automaticRefreshTask?.cancel() + automaticRefreshTask = nil + let client = client + self.client = nil + configuration = nil + status = .failed(reconnectState.lastError) + hint = reconnectState.lastError + connectionJourney.fail(reconnectState.lastError) + Task { await client?.disconnect() } + } + func resolveInitialPathIfNeeded(using client: any SFTPClientProtocol) async throws { guard !didResolveInitialPath else { return } path = RemotePath.normalized(try await client.homeDirectory()) @@ -384,6 +448,8 @@ final class SFTPSessionModel: Identifiable { } func cancelAuthentication() { + reconnectController.stop() + isRecoveringConnection = false pendingAuthentication = nil status = .disconnected connectionJourney.hide() @@ -441,6 +507,20 @@ final class SFTPSessionModel: Identifiable { hint = snapshot.count == 1 ? String(localized: "1 item") : String(localized: "\(snapshot.count) items") + } catch let error as SFTPError { + if case .connectionLost(let message) = error { + handleConnectionFailure(message, attemptID: connectionAttemptID) + return + } + if case .notConnected = error { + handleConnectionFailure( + error.localizedDescription, attemptID: connectionAttemptID + ) + return + } + guard !silently else { return } + operationError = error.localizedDescription + hint = error.localizedDescription } catch { guard !silently else { return } operationError = error.localizedDescription @@ -1050,16 +1130,24 @@ final class SFTPSessionModel: Identifiable { } pendingHostKey = SSHHostKeyPrompt(observation: observation, kind: kind) pendingHostKeyHostID = host.id + reconnectController.pauseForUserInteraction() connectionJourney.waitForHostKey( at: host.id, fingerprint: observation.fingerprint ) - } catch { status = .failed(error.localizedDescription) } + } catch { + if hasConnectedOnce { + scheduleReconnect(after: error.localizedDescription, using: store) + } else { + status = .failed(error.localizedDescription) + } + } } private func requestAuthentication( host: Host, store: HostStore, failure: String? = nil, mode: SSHAuthenticationPrompt.Mode = .localIdentity ) { + reconnectController.pauseForUserInteraction() suppliedAuthentications[host.id] = nil let identity = store.identity(for: host) pendingAuthentication = SSHAuthenticationPrompt( @@ -1090,6 +1178,43 @@ final class SFTPSessionModel: Identifiable { (error as? TermindSSHError)?.routeHostID } + private func handleConnectionFailure(_ message: String, attemptID: UUID?) { + guard hasConnectedOnce, connectionAttemptID == attemptID, + let store = statusStore + else { return } + scheduleReconnect(after: message, using: store) + } + + private func scheduleReconnect(after message: String, using store: HostStore) { + isRecoveringConnection = true + connectionAttemptID = nil + connectionTask?.cancel() + connectionTask = nil + automaticRefreshTask?.cancel() + automaticRefreshTask = nil + uploadRefreshTask?.cancel() + uploadRefreshTask = nil + let client = client + self.client = nil + configuration = nil + status = .reconnecting + hint = String(localized: "Reconnecting") + presentReconnectJourney(using: store) + reconnectController.schedule(after: message) { [weak self, weak store] in + guard let self, let store else { return } + self.startConnection(using: store) + } + Task { await client?.disconnect() } + } + + private func presentReconnectJourney(using store: HostStore) { + guard let host = store.host(hostID) else { return } + connectionJourney.begin( + host: host, route: store.route(for: host), purpose: .sftpSession, + username: store.identity(for: host)?.username, mode: .reconnection + ) + } + private func deleteRecursively( _ file: RemoteFile, client: any SFTPClientProtocol, cancellation: TransferCancellationToken? = nil diff --git a/Termind/Features/StatusBar/StatusBarView.swift b/Termind/Features/StatusBar/StatusBarView.swift index 0a24535f..c8907c0d 100644 --- a/Termind/Features/StatusBar/StatusBarView.swift +++ b/Termind/Features/StatusBar/StatusBarView.swift @@ -65,6 +65,7 @@ struct StatusBarView: View { private var connection: some View { let status = sshSession?.status + ?? sftpSession?.status ?? pane?.hostID.map { app.hosts.status(of: $0) } ?? .disconnected diff --git a/Termind/Features/Terminal/SSHSessionModel.swift b/Termind/Features/Terminal/SSHSessionModel.swift index 2ad476b0..3ed17a15 100644 --- a/Termind/Features/Terminal/SSHSessionModel.swift +++ b/Termind/Features/Terminal/SSHSessionModel.swift @@ -184,6 +184,9 @@ final class SSHSessionModel: Identifiable { var connectionCoordinator: RemoteConnectionCoordinator? private var connectionTask: Task? private var connectionAttemptID: UUID? + private let reconnectController = SessionReconnectController() + private var hasConnectedOnce = false + private var isRecoveringConnection = false private var workingDirectoryObservation: AnyCancellable? /// The exact authenticated route used by the live terminal, retained only /// in memory so a drop can open an independent SFTP connection without @@ -211,6 +214,8 @@ final class SSHSessionModel: Identifiable { private var pendingInitialCommand: PendingInitialCommand? private var commandExcludedFromHistory: String? + var reconnectState: ReconnectState? { reconnectController.state } + convenience init(hostID: Host.ID, title: String, initialCommand: String? = nil) { self.init( hostID: hostID, title: title, initialCommand: initialCommand, @@ -267,7 +272,7 @@ final class SSHSessionModel: Identifiable { } init( - host: Host, usernameHint: String?, initialCommand: String?, + host: Host, usernameHint: String? = nil, initialCommand: String?, recordsInitialCommandInHistory: Bool ) { self.transientHost = host @@ -375,6 +380,11 @@ final class SSHSessionModel: Identifiable { } func connect(using store: HostStore) { + startConnection(using: store) + } + + private func startConnection(using store: HostStore) { + let reconnecting = hasConnectedOnce && isRecoveringConnection workingDirectory = nil shellIntegrationAvailable = false transferConfiguration = nil @@ -385,6 +395,8 @@ final class SSHSessionModel: Identifiable { if let ssh { Task { await ssh.disconnect() } } ssh = nil guard let host = store.host(hostID) ?? transientHost else { + reconnectController.stop() + isRecoveringConnection = false pendingInitialCommand = nil connectionAttemptID = nil status = .failed(String(localized: "The host is no longer available.")) @@ -399,7 +411,8 @@ final class SSHSessionModel: Identifiable { ?? identity?.username ?? usernameHint connectionJourney.begin( - host: host, route: route, purpose: .sshSession, username: username + host: host, route: route, purpose: .sshSession, username: username, + mode: reconnecting ? .reconnection : .connection ) guard let username, !username.trimmingCharacters(in: .whitespaces).isEmpty else { requestAuthentication( @@ -487,7 +500,7 @@ final class SSHSessionModel: Identifiable { let id = sshError.routeHostID, let jump = store.host(id) { - status = .connecting + status = reconnecting ? .reconnecting : .connecting connectionTask = Task { [weak self] in await self?.discoverHostKey( for: jump, using: store, attemptID: attemptID @@ -514,14 +527,15 @@ final class SSHSessionModel: Identifiable { } pendingInitialCommand = nil connectionAttemptID = nil - status = .failed(error.localizedDescription) - connectionJourney.fail(error.localizedDescription, at: routeHostID(from: error)) + handleConnectionAttemptFailure( + error, store: store, reconnecting: reconnecting + ) return } pendingHostKey = nil connectionTask?.cancel() - status = .connecting + status = reconnecting ? .reconnecting : .connecting connectionTask = Task { [weak self] in guard let self else { return } do { @@ -605,9 +619,9 @@ final class SSHSessionModel: Identifiable { ? .serverChallenge : .localIdentity ) } else { - pendingInitialCommand = nil - status = .failed(error.localizedDescription) - connectionJourney.fail(error.localizedDescription, at: routeHostID(from: error)) + handleConnectionAttemptFailure( + error, store: store, reconnecting: reconnecting + ) } } } @@ -738,6 +752,9 @@ final class SSHSessionModel: Identifiable { suppliedAuthentications.removeAll() temporaryHostKeys.removeAll() connectionAttemptID = nil + reconnectController.stop() + hasConnectedOnce = false + isRecoveringConnection = false connectionTask?.cancel() connectionTask = nil if let ssh { Task { await ssh.disconnect() } } @@ -746,6 +763,23 @@ final class SSHSessionModel: Identifiable { connectionJourney.hide() } + func retryReconnectNow() { + reconnectController.retryNow() + } + + func stopReconnecting() { + guard let reconnectState else { return } + reconnectController.stop() + isRecoveringConnection = false + connectionAttemptID = nil + connectionTask?.cancel() + connectionTask = nil + if let ssh { Task { await ssh.disconnect() } } + ssh = nil + status = .failed(reconnectState.lastError) + connectionJourney.fail(reconnectState.lastError) + } + func applyAppearance(_ preferences: TerminalAppearancePreferences) { let light = TermindTerminalThemeCatalog.terminalConfiguration( themeName: preferences.lightThemeName, @@ -777,6 +811,8 @@ final class SSHSessionModel: Identifiable { proxy: SSHTransportProxy?, jumps: [SSHRouteHopConfiguration], attemptID: UUID ) async throws { + let wasReconnecting = isRecoveringConnection + let presentsConnectionJourney = connectionJourney.isPresented let configuration = SSHConnectionConfiguration( hostID: host.id, host: host.host, port: host.port, username: username, credentials: credentials, expectedHostKeys: expectedHostKeys, @@ -822,8 +858,21 @@ final class SSHSessionModel: Identifiable { transferConfiguration = configuration suppliedAuthentications.removeAll() temporaryHostKeys.removeAll() + let initialConnection = !hasConnectedOnce + hasConnectedOnce = true + isRecoveringConnection = false + reconnectController.markConnected(initial: initialConnection) status = .connected - connectionJourney.succeed() + if presentsConnectionJourney { + connectionJourney.succeed() + } else { + connectionJourney.hide() + } + if wasReconnecting { + appendTerminalNotice( + String(localized: "Reconnected; a new remote shell has started.") + ) + } submitPendingInitialCommand() } catch { await actor.disconnect() @@ -845,19 +894,78 @@ final class SSHSessionModel: Identifiable { ssh = nil switch result { case .success: + reconnectController.stop() + hasConnectedOnce = false + isRecoveringConnection = false status = .disconnected inputRouter?.closeSSHSessionPane(afterExit: self) case .failure(let error): - status = .failed(error.localizedDescription) + beginAutomaticReconnect(after: error.localizedDescription) } } + private func handleConnectionAttemptFailure( + _ error: any Error, store: HostStore, reconnecting: Bool + ) { + connectionAttemptID = nil + if hasConnectedOnce || reconnecting { + scheduleReconnect(after: error.localizedDescription, using: store) + } else { + pendingInitialCommand = nil + status = .failed(error.localizedDescription) + connectionJourney.fail( + error.localizedDescription, at: routeHostID(from: error) + ) + } + } + + private func beginAutomaticReconnect(after message: String) { + guard hasConnectedOnce, let store = statusStore else { + status = .failed(message) + return + } + appendTerminalNotice(String(localized: "Connection lost; reconnecting…")) + isRecoveringConnection = true + scheduleReconnect(after: message, using: store) + } + + private func scheduleReconnect(after message: String, using store: HostStore) { + isRecoveringConnection = true + connectionAttemptID = nil + connectionTask?.cancel() + connectionTask = nil + if let ssh { Task { await ssh.disconnect() } } + ssh = nil + transferConfiguration = nil + shellIntegrationAvailable = false + status = .reconnecting + presentReconnectJourney(using: store) + reconnectController.schedule(after: message) { [weak self, weak store] in + guard let self, let store else { return } + self.startConnection(using: store) + } + } + + private func presentReconnectJourney(using store: HostStore) { + guard let host = store.host(hostID) ?? transientHost else { return } + connectionJourney.begin( + host: host, route: store.route(for: host), purpose: .sshSession, + username: store.identity(for: host)?.username ?? usernameHint, + mode: .reconnection + ) + } + + private func appendTerminalNotice(_ message: String) { + terminalBackend.receive(Data("\r\n\u{001B}[2m[Termind] \(message)\u{001B}[0m\r\n".utf8)) + } + private func requestAuthentication( for host: Host, username: String, selectedKeyID: SSHKeyRef.ID?, failureMessage: String? = nil, requiresPassphrase: Bool = false, mode: SSHAuthenticationPrompt.Mode = .localIdentity ) { + reconnectController.pauseForUserInteraction() suppliedAuthentications[host.id] = nil pendingAuthentication = SSHAuthenticationPrompt( host: host, username: username, @@ -917,14 +1025,19 @@ final class SSHSessionModel: Identifiable { } catch { guard connectionAttemptID == attemptID else { return } connectionAttemptID = nil - status = .failed(error.localizedDescription) - connectionJourney.fail(error.localizedDescription, at: host.id) + if hasConnectedOnce { + scheduleReconnect(after: error.localizedDescription, using: store) + } else { + status = .failed(error.localizedDescription) + connectionJourney.fail(error.localizedDescription, at: host.id) + } } } private func presentHostKeyPrompt( _ observation: SSHHostKeyObservation, kind: SSHHostKeyPrompt.Kind, hostID: Host.ID ) { + reconnectController.pauseForUserInteraction() pendingHostKey = SSHHostKeyPrompt(observation: observation, kind: kind) pendingHostKeyHostID = hostID connectionAttemptID = nil diff --git a/Termind/Features/Terminal/TerminalPlaceholderView.swift b/Termind/Features/Terminal/TerminalPlaceholderView.swift index 49a94646..9f0598dc 100644 --- a/Termind/Features/Terminal/TerminalPlaceholderView.swift +++ b/Termind/Features/Terminal/TerminalPlaceholderView.swift @@ -117,7 +117,10 @@ struct TerminalPlaceholderView: View { log: sshSession.connectionJourney.plainTextLog ) } - } + }, + reconnectState: sshSession.reconnectState, + retryReconnectNow: sshSession.retryReconnectNow, + stopReconnecting: sshSession.stopReconnecting ) .zIndex(10) } diff --git a/Termind/UI/Components/SSHConnectionJourneyView.swift b/Termind/UI/Components/SSHConnectionJourneyView.swift index ddd7e830..01eae7d0 100644 --- a/Termind/UI/Components/SSHConnectionJourneyView.swift +++ b/Termind/UI/Components/SSHConnectionJourneyView.swift @@ -15,6 +15,9 @@ struct SSHConnectionJourneyView: View { var retry: () -> Void var cancel: () -> Void var askAI: (() -> Void)? + var reconnectState: ReconnectState? = nil + var retryReconnectNow: (() -> Void)? = nil + var stopReconnecting: (() -> Void)? = nil @Environment(\.accessibilityReduceMotion) private var reduceMotion @@ -78,7 +81,7 @@ struct SSHConnectionJourneyView: View { .font(.title3.weight(.semibold)) .lineLimit(1) Text( - "\(journey.purpose == .sshSession ? "SSH" : "SFTP") \(journey.destination?.address ?? "")" + headerDetail ) .font(.callout.monospaced()) .foregroundStyle(.secondary) @@ -165,6 +168,12 @@ struct SSHConnectionJourneyView: View { message: message, retry: retry, cancel: cancel, copyLogs: copyLogs, askAI: askAI ) + } else if journey.mode == .reconnection, let reconnectState { + JourneyReconnectStatus( + state: reconnectState, + retryNow: retryReconnectNow ?? retry, + stop: stopReconnecting ?? cancel + ) } else { HStack(spacing: 9) { Image(systemName: statusSymbol) @@ -221,6 +230,15 @@ struct SSHConnectionJourneyView: View { } } + private var headerDetail: String { + let protocolName = journey.purpose == .sshSession ? "SSH" : "SFTP" + let address = journey.destination?.address ?? "" + if journey.mode == .reconnection { + return String(localized: "Reconnecting \(protocolName) · \(address)") + } + return "\(protocolName) \(address)" + } + private var statusSymbol: String { switch journey.phase { case .succeeded: "checkmark.circle.fill" @@ -277,6 +295,51 @@ struct SSHConnectionJourneyView: View { } } +private struct JourneyReconnectStatus: View { + let state: ReconnectState + let retryNow: () -> Void + let stop: () -> Void + + var body: some View { + TimelineView(.periodic(from: .now, by: 1)) { context in + VStack(spacing: 12) { + HStack(spacing: 9) { + Image(systemName: "arrow.trianglehead.2.clockwise.rotate.90") + .foregroundStyle(.blue) + Text(statusText(at: context.date)) + .font(.headline) + } + + Text(state.lastError) + .font(.callout) + .foregroundStyle(.secondary) + .lineLimit(2) + .multilineTextAlignment(.center) + .textSelection(.enabled) + + HStack(spacing: 10) { + Button("Retry Now", action: retryNow) + .buttonStyle(.borderedProminent) + Button("Stop Reconnecting", role: .cancel, action: stop) + .buttonStyle(.bordered) + } + .controlSize(.large) + } + .padding(.top, 4) + } + } + + private func statusText(at date: Date) -> String { + let seconds = max(0, Int(ceil(state.nextRetry.timeIntervalSince(date)))) + if seconds == 0 { + return String(localized: "Reconnect attempt \(state.attempt)…") + } + return String( + localized: "Reconnect attempt \(state.attempt) · retrying in \(seconds)s" + ) + } +} + private enum JourneyVisualState { case pending case active diff --git a/TermindTests/SFTPFeatureTests.swift b/TermindTests/SFTPFeatureTests.swift index bcb0af8c..496dc998 100644 --- a/TermindTests/SFTPFeatureTests.swift +++ b/TermindTests/SFTPFeatureTests.swift @@ -191,6 +191,51 @@ struct SFTPFeatureTests { #expect(await client.listRequestCount == countAfterDisconnect) } + @Test + func connectionLossStartsReconnectWithoutClearingDirectorySnapshot() async { + let host = Host(name: "Files", host: "files.example") + let existing = remoteFile(path: "/home/existing.txt", kind: .file) + let client = SnapshotSFTPClient( + responses: [.success([existing]), .connectionLost] + ) + let store = HostStore(hosts: [host]) + let sftpSession = SFTPSessionModel( + hostID: host.id, initialPath: "/home", client: client, + configuration: testConfiguration + ) + sftpSession.bindStatus(to: store) + + await sftpSession.reload() + await sftpSession.reload(silently: true) + + #expect(sftpSession.status == .reconnecting) + #expect(sftpSession.reconnectState?.attempt == 1) + #expect(sftpSession.connectionJourney.mode == .reconnection) + #expect(sftpSession.connectionJourney.isPresented) + #expect(sftpSession.files == [existing]) + #expect(sftpSession.path == "/home") + sftpSession.disconnect() + } + + @Test + func ordinarySFTPFailureDoesNotStartReconnect() async { + let host = Host(name: "Files", host: "files.example") + let client = SnapshotSFTPClient(responses: [.failure]) + let store = HostStore(hosts: [host]) + let sftpSession = SFTPSessionModel( + hostID: host.id, initialPath: "/home", client: client, + configuration: testConfiguration + ) + sftpSession.bindStatus(to: store) + + await sftpSession.reload() + + #expect(sftpSession.status == .connected) + #expect(sftpSession.reconnectState == nil) + #expect(sftpSession.operationError == "Refresh failed") + sftpSession.disconnect() + } + @Test func successfulUploadRefreshesTheVisibleDirectory() async throws { let root = FileManager.default.temporaryDirectory @@ -349,6 +394,9 @@ struct SFTPFeatureTests { store.setConnectionStatus(.disconnected, connectionID: sshSessionID, hostID: hostID) #expect(store.status(of: hostID) == .connecting) + store.setConnectionStatus(.reconnecting, connectionID: sftpSessionID, hostID: hostID) + #expect(store.status(of: hostID) == .reconnecting) + store.setConnectionStatus(.connected, connectionID: sftpSessionID, hostID: hostID) #expect(store.status(of: hostID) == .connected) @@ -761,6 +809,7 @@ private actor SnapshotSFTPClient: SFTPClientProtocol { enum Response: Sendable { case success([RemoteFile]) case failure + case connectionLost } private var responses: [Response] @@ -780,6 +829,7 @@ private actor SnapshotSFTPClient: SFTPClientProtocol { switch response { case .success(let files): return files case .failure: throw SFTPError.operation("Refresh failed") + case .connectionLost: throw SFTPError.connectionLost("Network lost") } } func attributes(at path: String) async throws -> RemoteFile? { nil } diff --git a/TermindTests/SSHConnectionJourneyTests.swift b/TermindTests/SSHConnectionJourneyTests.swift index 7583e37a..29e4d376 100644 --- a/TermindTests/SSHConnectionJourneyTests.swift +++ b/TermindTests/SSHConnectionJourneyTests.swift @@ -27,6 +27,22 @@ struct SSHConnectionJourneyTests { #expect(journey.endpoints[2].hostID == destination.id) } + @Test + func reconnectUsesTheSameJourneyInReconnectMode() { + let destination = Host(name: "Production", host: "prod.example") + let journey = SSHConnectionJourney() + + journey.begin( + host: destination, route: [destination], + purpose: .sshSession, username: "deploy", mode: .reconnection + ) + + #expect(journey.mode == .reconnection) + #expect(journey.phase == .connecting) + #expect(journey.isPresented) + #expect(journey.plainTextLog.contains("Reconnecting SSH session")) + } + @Test func authenticationCheckpointAppearsOnlyWhenRequested() { let destination = Host(name: "Server", host: "server.example") diff --git a/TermindTests/SessionReconnectControllerTests.swift b/TermindTests/SessionReconnectControllerTests.swift new file mode 100644 index 00000000..3192e889 --- /dev/null +++ b/TermindTests/SessionReconnectControllerTests.swift @@ -0,0 +1,78 @@ +import Foundation +import Testing + +@testable import Termind + +@Suite +@MainActor +struct SessionReconnectControllerTests { + @Test + func retriesUseCappedBackoffAndRetainLatestFailure() { + let clock = ReconnectTestClock() + let controller = SessionReconnectController( + sleep: { _ in try await Task.sleep(for: .seconds(3_600)) }, + now: { clock.value } + ) + + let expectedDelays: [TimeInterval] = [1, 2, 5, 10, 30, 30] + for (index, expectedDelay) in expectedDelays.enumerated() { + controller.schedule(after: "failure-\(index + 1)") {} + #expect(controller.state?.attempt == index + 1) + #expect(controller.state?.lastError == "failure-\(index + 1)") + #expect( + controller.state?.nextRetry.timeIntervalSince(clock.value) + == expectedDelay + ) + } + controller.stop() + } + + @Test + func stableConnectionResetsAttemptCounter() { + let clock = ReconnectTestClock() + let controller = SessionReconnectController( + sleep: { _ in try await Task.sleep(for: .seconds(3_600)) }, + now: { clock.value } + ) + + controller.schedule(after: "first") {} + controller.schedule(after: "second") {} + #expect(controller.state?.attempt == 2) + + controller.markConnected(initial: false) + clock.advance(by: 61) + controller.schedule(after: "after stable connection") {} + + #expect(controller.state?.attempt == 1) + controller.stop() + } + + @Test + func retryNowRunsImmediatelyAndStopCancelsFutureRetry() async { + let controller = SessionReconnectController( + sleep: { _ in try await Task.sleep(for: .seconds(3_600)) } + ) + var retryCount = 0 + controller.schedule(after: "offline") { retryCount += 1 } + + controller.retryNow() + await Task.yield() + #expect(retryCount == 1) + + controller.stop() + #expect(controller.state == nil) + try? await Task.sleep(for: .milliseconds(10)) + #expect(retryCount == 1) + } +} + +private nonisolated final class ReconnectTestClock: @unchecked Sendable { + private let lock = NSLock() + private var date = Date(timeIntervalSince1970: 1_000) + + var value: Date { lock.withLock { date } } + + func advance(by interval: TimeInterval) { + lock.withLock { date.addTimeInterval(interval) } + } +}