okooooooo/Termind
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add automatic SSH and SFTP reconnection

Browse Source
This commit is contained in:
Kaede Akino committed 2026-08-24 23:55:21 +08:00
1 parent 639dd9b5c6
commit 9f0f96eaea
14 files changed
+707 -35
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Termind/Application/Connections/RemoteConnectionCoordinator.swift
+6 -2
View File
@@ -18,11 +18,15 @@ nonisolated final class RemoteConnectionCoordinator: Sendable {
func connectSFTP(
using configuration: SSHConnectionConfiguration,
progress: SSHRouteProgressHandler? = nil
progress: SSHRouteProgressHandler? = nil,
connectionFailure: (@Sendable (String) -> Void)? = nil
) async throws -> any SFTPClientProtocol {
let client = makeSFTPClient()
do {
try await client.connect(using: configuration, progress: progress)
try await client.connect(
using: configuration, progress: progress,
connectionFailure: connectionFailure
)
return client
} catch {
await client.disconnect()
Termind/Application/Connections/SessionReconnectController.swift
+139
View File
@@ -0,0 +1,139 @@
import Foundation
nonisolated struct ReconnectPolicy: Sendable, Equatable {
static let sessionDefault = ReconnectPolicy()
var delays: [Duration]
var stableResetInterval: TimeInterval
init(
delays: [Duration] = [.seconds(1), .seconds(2), .seconds(5), .seconds(10), .seconds(30)],
stableResetInterval: TimeInterval = 60
) {
precondition(!delays.isEmpty)
self.delays = delays
self.stableResetInterval = stableResetInterval
}
func delay(forAttempt attempt: Int) -> Duration {
delays[min(max(0, attempt - 1), delays.count - 1)]
}
}
nonisolated struct ReconnectState: Equatable, Sendable {
var attempt: Int
var nextRetry: Date
var lastError: String
}
/// Owns only retry timing and stale-callback rejection. The session models keep
/// transport construction because they also own authentication and host-key UI.
@Observable
final class SessionReconnectController {
typealias RetryAction = @MainActor @Sendable () -> Void
typealias Sleep = @Sendable (Duration) async throws -> Void
typealias Now = @Sendable () -> Date
private(set) var state: ReconnectState?
private let policy: ReconnectPolicy
private let sleep: Sleep
private let now: Now
private var retryTask: Task<Void, Never>?
private var retryAction: RetryAction?
private var attempts = 0
private var connectedAt: Date?
private var generation: UInt64 = 0
init(
policy: ReconnectPolicy = .sessionDefault,
sleep: @escaping Sleep = { try await Task.sleep(for: $0) },
now: @escaping Now = Date.init
) {
self.policy = policy
self.sleep = sleep
self.now = now
}
func markConnected(initial: Bool) {
cancelScheduledRetry()
state = nil
retryAction = nil
connectedAt = now()
if initial { attempts = 0 }
}
func schedule(after error: String, retry: @escaping RetryAction) {
cancelScheduledRetry()
if let connectedAt,
now().timeIntervalSince(connectedAt) >= policy.stableResetInterval
{
attempts = 0
}
connectedAt = nil
attempts += 1
retryAction = retry
let delay = policy.delay(forAttempt: attempts)
state = ReconnectState(
attempt: attempts,
nextRetry: now().addingTimeInterval(delay.timeInterval),
lastError: error
)
schedule(delay: delay)
}
func retryNow() {
guard let retryAction else { return }
cancelScheduledRetry()
generation &+= 1
let expectedGeneration = generation
retryTask = Task { [weak self] in
guard let self, self.generation == expectedGeneration else { return }
retryAction()
}
}
func stop() {
cancelScheduledRetry()
generation &+= 1
state = nil
retryAction = nil
attempts = 0
connectedAt = nil
}
/// Pauses retry timing without discarding the attempt counter. Used while
/// the connection journey waits for credentials or a host-key decision.
func pauseForUserInteraction() {
cancelScheduledRetry()
generation &+= 1
state = nil
retryAction = nil
}
private func schedule(delay: Duration) {
generation &+= 1
let expectedGeneration = generation
let sleep = sleep
retryTask = Task { [weak self] in
do { try await sleep(delay) } catch { return }
guard let self, self.generation == expectedGeneration,
let retryAction = self.retryAction
else { return }
retryAction()
}
}
private func cancelScheduledRetry() {
retryTask?.cancel()
retryTask = nil
}
}
private nonisolated extension Duration {
var timeInterval: TimeInterval {
let components = self.components
return TimeInterval(components.seconds)
+ TimeInterval(components.attoseconds) / 1_000_000_000_000_000_000
}
}
Termind/Core/SSH/SFTPClient.swift
+62 -3
View File
@@ -62,6 +62,11 @@ nonisolated protocol SFTPClientProtocol: Sendable {
using configuration: SSHConnectionConfiguration,
progress: SSHRouteProgressHandler?
) async throws
func connect(
using configuration: SSHConnectionConfiguration,
progress: SSHRouteProgressHandler?,
connectionFailure: (@Sendable (String) -> Void)?
) async throws
func disconnect() async
func homeDirectory() async throws -> String
func listDirectory(at path: String) async throws -> [RemoteFile]
@@ -92,6 +97,14 @@ nonisolated protocol SFTPClientProtocol: Sendable {
}
nonisolated extension SFTPClientProtocol {
func connect(
using configuration: SSHConnectionConfiguration,
progress: SSHRouteProgressHandler?,
connectionFailure: (@Sendable (String) -> Void)?
) async throws {
try await connect(using: configuration, progress: progress)
}
func copyFile(
from sourcePath: String,
to destinationPath: String,
@@ -107,6 +120,7 @@ nonisolated extension SFTPClientProtocol {
nonisolated enum SFTPError: LocalizedError, Sendable {
case notConnected
case connectionLost(String)
case cancelled
case localFile(String)
case operation(String)
@@ -114,6 +128,7 @@ nonisolated enum SFTPError: LocalizedError, Sendable {
var errorDescription: String? {
switch self {
case .notConnected: String(localized: "The SFTP connection is not available.")
case .connectionLost(let message): message
case .cancelled: String(localized: "Transfer cancelled.")
case .localFile(let message), .operation(let message): message
}
@@ -130,6 +145,17 @@ actor SFTPClient: SFTPClientProtocol {
try await worker.connect(using: configuration, progress: progress)
}
func connect(
using configuration: SSHConnectionConfiguration,
progress: SSHRouteProgressHandler?,
connectionFailure: (@Sendable (String) -> Void)?
) async throws {
try await worker.connect(
using: configuration, progress: progress,
connectionFailure: connectionFailure
)
}
func disconnect() async { await worker.disconnect() }
func homeDirectory() async throws -> String {
@@ -271,12 +297,15 @@ private nonisolated final class SFTPWorker: @unchecked Sendable {
private var session: OpaquePointer?
private var sftp: OpaquePointer?
private var keepaliveTimer: (any DispatchSourceTimer)?
private var connectionFailure: (@Sendable (String) -> Void)?
private var didReportConnectionFailure = false
deinit { cleanup() }
func connect(
using configuration: SSHConnectionConfiguration,
progress: SSHRouteProgressHandler?
progress: SSHRouteProgressHandler?,
connectionFailure: (@Sendable (String) -> Void)? = nil
) async throws {
try await perform {
self.cleanup()
@@ -313,6 +342,8 @@ private nonisolated final class SFTPWorker: @unchecked Sendable {
throw SSHSessionWorker.sessionError(session, prefix: "Unable to start SFTP")
}
self.sftp = sftp
self.connectionFailure = connectionFailure
self.didReportConnectionFailure = false
self.startKeepalive(for: session)
}
}
@@ -718,7 +749,21 @@ private nonisolated final class SFTPWorker: @unchecked Sendable {
private func error(_ prefix: String) -> SFTPError {
guard let sftp else { return .operation(prefix) }
return .operation("\(prefix): SFTP error \(libssh2_sftp_last_error(sftp))")
let sftpCode = libssh2_sftp_last_error(sftp)
let sessionCode = session.map(libssh2_session_last_errno) ?? 0
let message = "\(prefix): SFTP error \(sftpCode)"
let lostSFTPCodes = [
UInt(LIBSSH2_FX_NO_CONNECTION), UInt(LIBSSH2_FX_CONNECTION_LOST),
]
let lostSessionCodes = [
LIBSSH2_ERROR_SOCKET_SEND, LIBSSH2_ERROR_SOCKET_RECV,
LIBSSH2_ERROR_TIMEOUT, LIBSSH2_ERROR_CHANNEL_CLOSED,
]
if lostSFTPCodes.contains(sftpCode) || lostSessionCodes.contains(sessionCode) {
reportConnectionFailure(message)
return .connectionLost(message)
}
return .operation(message)
}
private func cleanup() {
@@ -732,6 +777,8 @@ private nonisolated final class SFTPWorker: @unchecked Sendable {
}
routeTransport?.close()
routeTransport = nil
connectionFailure = nil
didReportConnectionFailure = false
}
private func startKeepalive(for session: OpaquePointer) {
@@ -741,12 +788,24 @@ private nonisolated final class SFTPWorker: @unchecked Sendable {
timer.setEventHandler { [weak self] in
guard let self, self.session == session else { return }
var secondsToNext: Int32 = 0
_ = libssh2_keepalive_send(session, &secondsToNext)
let result = libssh2_keepalive_send(session, &secondsToNext)
if result < 0, result != LIBSSH2_ERROR_EAGAIN {
let error = SSHSessionWorker.sessionError(
session, prefix: "SFTP keepalive failed"
)
self.reportConnectionFailure(error.localizedDescription)
}
}
keepaliveTimer = timer
timer.resume()
}
private func reportConnectionFailure(_ message: String) {
guard !didReportConnectionFailure else { return }
didReportConnectionFailure = true
connectionFailure?(message)
}
private func perform<T: Sendable>(_ operation: @escaping @Sendable () throws -> T) async throws -> T {
try await withCheckedThrowingContinuation { continuation in
queue.async {
Termind/Core/SSH/SSHConnectionJourney.swift
+13 -2
View File
@@ -46,6 +46,11 @@ nonisolated struct SSHAuthenticationSubmission: Sendable {
/// a completed path backwards.
@Observable
final class SSHConnectionJourney {
enum Mode: Equatable {
case connection
case reconnection
}
enum Phase: Equatable {
case hidden
case connecting
@@ -72,6 +77,7 @@ final class SSHConnectionJourney {
}
private(set) var purpose: SSHConnectionPurpose = .sshSession
private(set) var mode: Mode = .connection
private(set) var endpoints: [SSHConnectionEndpoint] = []
private(set) var phase: Phase = .hidden
private(set) var activeEndpointIndex = 0
@@ -87,7 +93,10 @@ final class SSHConnectionJourney {
var isPresented: Bool { phase != .hidden }
var destination: SSHConnectionEndpoint? { endpoints.last }
func begin(host: Host, route: [Host], purpose: SSHConnectionPurpose, username: String?) {
func begin(
host: Host, route: [Host], purpose: SSHConnectionPurpose,
username: String?, mode: Mode = .connection
) {
completionTask?.cancel()
let nextEndpoints =
[
@@ -113,6 +122,7 @@ final class SSHConnectionJourney {
let retainedLogVisibility = continuesJourney && showsLogs
self.purpose = purpose
self.mode = mode
endpoints = nextEndpoints
phase = .connecting
activeEndpointIndex = retainedCompletedEndpoint
@@ -122,9 +132,10 @@ final class SSHConnectionJourney {
waitsForLocalIdentity = false
logs = retainedLogs
showsLogs = retainedLogVisibility
let action = mode == .reconnection ? "Reconnecting " : "Starting "
log(
endpoint: endpoints.first?.name ?? String(localized: "This Mac"),
message: "Starting "
message: action
+ (purpose == .sshSession ? "SSH session" : "SFTP session")
+ " connection to " + host.name
)
Termind/Core/SSH/SSHSessionActor.swift
+6 -1
View File
@@ -1087,7 +1087,12 @@ nonisolated final class SSHSessionWorker: @unchecked Sendable {
}
var next = Int32(0)
_ = libssh2_keepalive_send(session, &next)
let keepalive = libssh2_keepalive_send(session, &next)
if keepalive < 0, keepalive != LIBSSH2_ERROR_EAGAIN {
reportExit(.failure(Self.sessionError(session, prefix: "SSH keepalive failed")))
cleanup()
return
}
if libssh2_channel_eof(channel) == 1 {
let status = libssh2_channel_get_exit_status(channel)
reportExit(.success(status))
Termind/Features/SFTP/SFTPBrowserView.swift
+6 -1
View File
@@ -68,7 +68,10 @@ struct SFTPBrowserView: View {
cancelConnection()
},
retry: { sftpSession.connect(using: app.hosts) },
cancel: cancelConnection
cancel: cancelConnection,
reconnectState: sftpSession.reconnectState,
retryReconnectNow: sftpSession.retryReconnectNow,
stopReconnecting: sftpSession.stopReconnecting
)
.zIndex(10)
}
@@ -413,10 +416,12 @@ struct SFTPBrowserView: View {
renameTarget = file
renameName = file.name
}
.disabled(!sftpSession.allowsRemoteMutations)
}
Button(targets.count == 1 ? "Delete" : "Delete \(targets.count) Items", role: .destructive) {
deleteTargets = targets
}
.disabled(!sftpSession.allowsRemoteMutations)
Divider()
Button("New Folder") { newFolderPresented = true }
.disabled(sftpSession.status != .connected)
Termind/Features/SFTP/SFTPSessionModel.swift
+136 -11
View File
@@ -74,6 +74,9 @@ final class SFTPSessionModel: Identifiable {
private var suppliedAuthentications: [Host.ID: SSHAuthenticationSubmission] = [:]
private var temporaryHostKeys: [Host.ID: SSHHostKeyIdentity] = [:]
private var connectionAttemptID: UUID?
private let reconnectController = SessionReconnectController()
private var hasConnectedOnce = false
private var isRecoveringConnection = false
private var operationCancellation: TransferCancellationToken?
private weak var statusStore: HostStore?
private weak var transferCoordinator: SFTPTransferCoordinator?
@@ -99,8 +102,12 @@ final class SFTPSessionModel: Identifiable {
self.didResolveInitialPath = !configuredPath.isEmpty
self.automaticRefreshInterval = automaticRefreshInterval
if client != nil { status = .connected }
if client != nil { hasConnectedOnce = true }
}
var reconnectState: ReconnectState? { reconnectController.state }
var allowsRemoteMutations: Bool { status == .connected }
var canGoBack: Bool { !backHistory.isEmpty }
var canGoForward: Bool { !forwardHistory.isEmpty }
var canGoUp: Bool { path != "/" }
@@ -132,9 +139,16 @@ final class SFTPSessionModel: Identifiable {
}
func connect(using store: HostStore) {
startConnection(using: store)
}
private func startConnection(using store: HostStore) {
let reconnecting = hasConnectedOnce && isRecoveringConnection
bindStatus(to: store)
connectionTask?.cancel()
guard let host = store.host(hostID) else {
reconnectController.stop()
isRecoveringConnection = false
let message = String(localized: "The host is no longer available.")
status = .failed(message)
connectionJourney.fail(message)
@@ -146,10 +160,16 @@ final class SFTPSessionModel: Identifiable {
let suppliedAuthentication = suppliedAuthentications[host.id]
connectionJourney.begin(
host: host, route: route, purpose: .sftpSession,
username: suppliedAuthentication?.username ?? store.identity(for: host)?.username
username: suppliedAuthentication?.username ?? store.identity(for: host)?.username,
mode: reconnecting ? .reconnection : .connection
)
status = .connecting
hint = String(localized: "Connecting")
if reconnecting, usesEphemeralCredentials, suppliedAuthentication == nil {
requestAuthentication(host: host, store: store)
return
}
status = reconnecting ? .reconnecting : .connecting
hint = reconnecting
? String(localized: "Reconnecting") : String(localized: "Connecting")
connectionTask = Task { [weak self] in
guard let self else { return }
do {
@@ -190,7 +210,12 @@ final class SFTPSessionModel: Identifiable {
}
let coordinator = self.connectionCoordinator ?? RemoteConnectionCoordinator()
let client = try await coordinator.connectSFTP(
using: configuration, progress: progressHandler(attemptID: attemptID)
using: configuration, progress: progressHandler(attemptID: attemptID),
connectionFailure: { [weak self] message in
Task { @MainActor in
self?.handleConnectionFailure(message, attemptID: attemptID)
}
}
)
try Task.checkCancellation()
try await self.resolveInitialPathIfNeeded(using: client)
@@ -200,12 +225,22 @@ final class SFTPSessionModel: Identifiable {
self.suppliedAuthentications.removeAll()
self.temporaryHostKeys.removeAll()
self.pendingAuthentication = nil
let initialConnection = !self.hasConnectedOnce
let presentsConnectionJourney = self.connectionJourney.isPresented
self.hasConnectedOnce = true
self.isRecoveringConnection = false
self.reconnectController.markConnected(initial: initialConnection)
self.status = .connected
self.connectionJourney.succeed()
if presentsConnectionJourney {
self.connectionJourney.succeed()
} else {
self.connectionJourney.hide()
}
self.hint = ""
await self.reload()
self.startAutomaticRefresh()
} catch is CancellationError {
guard self.connectionAttemptID == attemptID else { return }
self.status = .disconnected
} catch TermindSSHError.untrustedHost {
await self.discoverHostKey(host: host, store: store)
@@ -238,11 +273,15 @@ final class SFTPSessionModel: Identifiable {
? .serverChallenge : .localIdentity
)
} catch {
self.status = .failed(error.localizedDescription)
self.hint = error.localizedDescription
self.connectionJourney.fail(
error.localizedDescription, at: self.routeHostID(from: error)
)
if self.hasConnectedOnce || reconnecting {
self.scheduleReconnect(after: error.localizedDescription, using: store)
} else {
self.status = .failed(error.localizedDescription)
self.hint = error.localizedDescription
self.connectionJourney.fail(
error.localizedDescription, at: self.routeHostID(from: error)
)
}
}
}
}
@@ -265,6 +304,9 @@ final class SFTPSessionModel: Identifiable {
connectionTask?.cancel()
connectionTask = nil
connectionAttemptID = nil
reconnectController.stop()
hasConnectedOnce = false
isRecoveringConnection = false
automaticRefreshTask?.cancel()
automaticRefreshTask = nil
uploadRefreshTask?.cancel()
@@ -283,6 +325,28 @@ final class SFTPSessionModel: Identifiable {
}
}
func retryReconnectNow() {
reconnectController.retryNow()
}
func stopReconnecting() {
guard let reconnectState else { return }
reconnectController.stop()
isRecoveringConnection = false
connectionAttemptID = nil
connectionTask?.cancel()
connectionTask = nil
automaticRefreshTask?.cancel()
automaticRefreshTask = nil
let client = client
self.client = nil
configuration = nil
status = .failed(reconnectState.lastError)
hint = reconnectState.lastError
connectionJourney.fail(reconnectState.lastError)
Task { await client?.disconnect() }
}
func resolveInitialPathIfNeeded(using client: any SFTPClientProtocol) async throws {
guard !didResolveInitialPath else { return }
path = RemotePath.normalized(try await client.homeDirectory())
@@ -384,6 +448,8 @@ final class SFTPSessionModel: Identifiable {
}
func cancelAuthentication() {
reconnectController.stop()
isRecoveringConnection = false
pendingAuthentication = nil
status = .disconnected
connectionJourney.hide()
@@ -441,6 +507,20 @@ final class SFTPSessionModel: Identifiable {
hint =
snapshot.count == 1
? String(localized: "1 item") : String(localized: "\(snapshot.count) items")
} catch let error as SFTPError {
if case .connectionLost(let message) = error {
handleConnectionFailure(message, attemptID: connectionAttemptID)
return
}
if case .notConnected = error {
handleConnectionFailure(
error.localizedDescription, attemptID: connectionAttemptID
)
return
}
guard !silently else { return }
operationError = error.localizedDescription
hint = error.localizedDescription
} catch {
guard !silently else { return }
operationError = error.localizedDescription
@@ -1050,16 +1130,24 @@ final class SFTPSessionModel: Identifiable {
}
pendingHostKey = SSHHostKeyPrompt(observation: observation, kind: kind)
pendingHostKeyHostID = host.id
reconnectController.pauseForUserInteraction()
connectionJourney.waitForHostKey(
at: host.id, fingerprint: observation.fingerprint
)
} catch { status = .failed(error.localizedDescription) }
} catch {
if hasConnectedOnce {
scheduleReconnect(after: error.localizedDescription, using: store)
} else {
status = .failed(error.localizedDescription)
}
}
}
private func requestAuthentication(
host: Host, store: HostStore, failure: String? = nil,
mode: SSHAuthenticationPrompt.Mode = .localIdentity
) {
reconnectController.pauseForUserInteraction()
suppliedAuthentications[host.id] = nil
let identity = store.identity(for: host)
pendingAuthentication = SSHAuthenticationPrompt(
@@ -1090,6 +1178,43 @@ final class SFTPSessionModel: Identifiable {
(error as? TermindSSHError)?.routeHostID
}
private func handleConnectionFailure(_ message: String, attemptID: UUID?) {
guard hasConnectedOnce, connectionAttemptID == attemptID,
let store = statusStore
else { return }
scheduleReconnect(after: message, using: store)
}
private func scheduleReconnect(after message: String, using store: HostStore) {
isRecoveringConnection = true
connectionAttemptID = nil
connectionTask?.cancel()
connectionTask = nil
automaticRefreshTask?.cancel()
automaticRefreshTask = nil
uploadRefreshTask?.cancel()
uploadRefreshTask = nil
let client = client
self.client = nil
configuration = nil
status = .reconnecting
hint = String(localized: "Reconnecting")
presentReconnectJourney(using: store)
reconnectController.schedule(after: message) { [weak self, weak store] in
guard let self, let store else { return }
self.startConnection(using: store)
}
Task { await client?.disconnect() }
}
private func presentReconnectJourney(using store: HostStore) {
guard let host = store.host(hostID) else { return }
connectionJourney.begin(
host: host, route: store.route(for: host), purpose: .sftpSession,
username: store.identity(for: host)?.username, mode: .reconnection
)
}
private func deleteRecursively(
_ file: RemoteFile, client: any SFTPClientProtocol,
cancellation: TransferCancellationToken? = nil
Termind/Features/StatusBar/StatusBarView.swift
+1
View File
@@ -65,6 +65,7 @@ struct StatusBarView: View {
private var connection: some View {
let status =
sshSession?.status
?? sftpSession?.status
?? pane?.hostID.map { app.hosts.status(of: $0) }
?? .disconnected
Termind/Features/Terminal/SSHSessionModel.swift
+126 -13
View File
@@ -184,6 +184,9 @@ final class SSHSessionModel: Identifiable {
var connectionCoordinator: RemoteConnectionCoordinator?
private var connectionTask: Task<Void, Never>?
private var connectionAttemptID: UUID?
private let reconnectController = SessionReconnectController()
private var hasConnectedOnce = false
private var isRecoveringConnection = false
private var workingDirectoryObservation: AnyCancellable?
/// The exact authenticated route used by the live terminal, retained only
/// in memory so a drop can open an independent SFTP connection without
@@ -211,6 +214,8 @@ final class SSHSessionModel: Identifiable {
private var pendingInitialCommand: PendingInitialCommand?
private var commandExcludedFromHistory: String?
var reconnectState: ReconnectState? { reconnectController.state }
convenience init(hostID: Host.ID, title: String, initialCommand: String? = nil) {
self.init(
hostID: hostID, title: title, initialCommand: initialCommand,
@@ -267,7 +272,7 @@ final class SSHSessionModel: Identifiable {
}
init(
host: Host, usernameHint: String?, initialCommand: String?,
host: Host, usernameHint: String? = nil, initialCommand: String?,
recordsInitialCommandInHistory: Bool
) {
self.transientHost = host
@@ -375,6 +380,11 @@ final class SSHSessionModel: Identifiable {
}
func connect(using store: HostStore) {
startConnection(using: store)
}
private func startConnection(using store: HostStore) {
let reconnecting = hasConnectedOnce && isRecoveringConnection
workingDirectory = nil
shellIntegrationAvailable = false
transferConfiguration = nil
@@ -385,6 +395,8 @@ final class SSHSessionModel: Identifiable {
if let ssh { Task { await ssh.disconnect() } }
ssh = nil
guard let host = store.host(hostID) ?? transientHost else {
reconnectController.stop()
isRecoveringConnection = false
pendingInitialCommand = nil
connectionAttemptID = nil
status = .failed(String(localized: "The host is no longer available."))
@@ -399,7 +411,8 @@ final class SSHSessionModel: Identifiable {
?? identity?.username
?? usernameHint
connectionJourney.begin(
host: host, route: route, purpose: .sshSession, username: username
host: host, route: route, purpose: .sshSession, username: username,
mode: reconnecting ? .reconnection : .connection
)
guard let username, !username.trimmingCharacters(in: .whitespaces).isEmpty else {
requestAuthentication(
@@ -487,7 +500,7 @@ final class SSHSessionModel: Identifiable {
let id = sshError.routeHostID,
let jump = store.host(id)
{
status = .connecting
status = reconnecting ? .reconnecting : .connecting
connectionTask = Task { [weak self] in
await self?.discoverHostKey(
for: jump, using: store, attemptID: attemptID
@@ -514,14 +527,15 @@ final class SSHSessionModel: Identifiable {
}
pendingInitialCommand = nil
connectionAttemptID = nil
status = .failed(error.localizedDescription)
connectionJourney.fail(error.localizedDescription, at: routeHostID(from: error))
handleConnectionAttemptFailure(
error, store: store, reconnecting: reconnecting
)
return
}
pendingHostKey = nil
connectionTask?.cancel()
status = .connecting
status = reconnecting ? .reconnecting : .connecting
connectionTask = Task { [weak self] in
guard let self else { return }
do {
@@ -605,9 +619,9 @@ final class SSHSessionModel: Identifiable {
? .serverChallenge : .localIdentity
)
} else {
pendingInitialCommand = nil
status = .failed(error.localizedDescription)
connectionJourney.fail(error.localizedDescription, at: routeHostID(from: error))
handleConnectionAttemptFailure(
error, store: store, reconnecting: reconnecting
)
}
}
}
@@ -738,6 +752,9 @@ final class SSHSessionModel: Identifiable {
suppliedAuthentications.removeAll()
temporaryHostKeys.removeAll()
connectionAttemptID = nil
reconnectController.stop()
hasConnectedOnce = false
isRecoveringConnection = false
connectionTask?.cancel()
connectionTask = nil
if let ssh { Task { await ssh.disconnect() } }
@@ -746,6 +763,23 @@ final class SSHSessionModel: Identifiable {
connectionJourney.hide()
}
func retryReconnectNow() {
reconnectController.retryNow()
}
func stopReconnecting() {
guard let reconnectState else { return }
reconnectController.stop()
isRecoveringConnection = false
connectionAttemptID = nil
connectionTask?.cancel()
connectionTask = nil
if let ssh { Task { await ssh.disconnect() } }
ssh = nil
status = .failed(reconnectState.lastError)
connectionJourney.fail(reconnectState.lastError)
}
func applyAppearance(_ preferences: TerminalAppearancePreferences) {
let light = TermindTerminalThemeCatalog.terminalConfiguration(
themeName: preferences.lightThemeName,
@@ -777,6 +811,8 @@ final class SSHSessionModel: Identifiable {
proxy: SSHTransportProxy?, jumps: [SSHRouteHopConfiguration],
attemptID: UUID
) async throws {
let wasReconnecting = isRecoveringConnection
let presentsConnectionJourney = connectionJourney.isPresented
let configuration = SSHConnectionConfiguration(
hostID: host.id, host: host.host, port: host.port, username: username,
credentials: credentials, expectedHostKeys: expectedHostKeys,
@@ -822,8 +858,21 @@ final class SSHSessionModel: Identifiable {
transferConfiguration = configuration
suppliedAuthentications.removeAll()
temporaryHostKeys.removeAll()
let initialConnection = !hasConnectedOnce
hasConnectedOnce = true
isRecoveringConnection = false
reconnectController.markConnected(initial: initialConnection)
status = .connected
connectionJourney.succeed()
if presentsConnectionJourney {
connectionJourney.succeed()
} else {
connectionJourney.hide()
}
if wasReconnecting {
appendTerminalNotice(
String(localized: "Reconnected; a new remote shell has started.")
)
}
submitPendingInitialCommand()
} catch {
await actor.disconnect()
@@ -845,19 +894,78 @@ final class SSHSessionModel: Identifiable {
ssh = nil
switch result {
case .success:
reconnectController.stop()
hasConnectedOnce = false
isRecoveringConnection = false
status = .disconnected
inputRouter?.closeSSHSessionPane(afterExit: self)
case .failure(let error):
status = .failed(error.localizedDescription)
beginAutomaticReconnect(after: error.localizedDescription)
}
}
private func handleConnectionAttemptFailure(
_ error: any Error, store: HostStore, reconnecting: Bool
) {
connectionAttemptID = nil
if hasConnectedOnce || reconnecting {
scheduleReconnect(after: error.localizedDescription, using: store)
} else {
pendingInitialCommand = nil
status = .failed(error.localizedDescription)
connectionJourney.fail(
error.localizedDescription, at: routeHostID(from: error)
)
}
}
private func beginAutomaticReconnect(after message: String) {
guard hasConnectedOnce, let store = statusStore else {
status = .failed(message)
return
}
appendTerminalNotice(String(localized: "Connection lost; reconnecting…"))
isRecoveringConnection = true
scheduleReconnect(after: message, using: store)
}
private func scheduleReconnect(after message: String, using store: HostStore) {
isRecoveringConnection = true
connectionAttemptID = nil
connectionTask?.cancel()
connectionTask = nil
if let ssh { Task { await ssh.disconnect() } }
ssh = nil
transferConfiguration = nil
shellIntegrationAvailable = false
status = .reconnecting
presentReconnectJourney(using: store)
reconnectController.schedule(after: message) { [weak self, weak store] in
guard let self, let store else { return }
self.startConnection(using: store)
}
}
private func presentReconnectJourney(using store: HostStore) {
guard let host = store.host(hostID) ?? transientHost else { return }
connectionJourney.begin(
host: host, route: store.route(for: host), purpose: .sshSession,
username: store.identity(for: host)?.username ?? usernameHint,
mode: .reconnection
)
}
private func appendTerminalNotice(_ message: String) {
terminalBackend.receive(Data("\r\n\u{001B}[2m[Termind] \(message)\u{001B}[0m\r\n".utf8))
}
private func requestAuthentication(
for host: Host, username: String,
selectedKeyID: SSHKeyRef.ID?, failureMessage: String? = nil,
requiresPassphrase: Bool = false,
mode: SSHAuthenticationPrompt.Mode = .localIdentity
) {
reconnectController.pauseForUserInteraction()
suppliedAuthentications[host.id] = nil
pendingAuthentication = SSHAuthenticationPrompt(
host: host, username: username,
@@ -917,14 +1025,19 @@ final class SSHSessionModel: Identifiable {
} catch {
guard connectionAttemptID == attemptID else { return }
connectionAttemptID = nil
status = .failed(error.localizedDescription)
connectionJourney.fail(error.localizedDescription, at: host.id)
if hasConnectedOnce {
scheduleReconnect(after: error.localizedDescription, using: store)
} else {
status = .failed(error.localizedDescription)
connectionJourney.fail(error.localizedDescription, at: host.id)
}
}
}
private func presentHostKeyPrompt(
_ observation: SSHHostKeyObservation, kind: SSHHostKeyPrompt.Kind, hostID: Host.ID
) {
reconnectController.pauseForUserInteraction()
pendingHostKey = SSHHostKeyPrompt(observation: observation, kind: kind)
pendingHostKeyHostID = hostID
connectionAttemptID = nil
Termind/Features/Terminal/TerminalPlaceholderView.swift
+4 -1
View File
@@ -117,7 +117,10 @@ struct TerminalPlaceholderView: View {
log: sshSession.connectionJourney.plainTextLog
)
}
}
},
reconnectState: sshSession.reconnectState,
retryReconnectNow: sshSession.retryReconnectNow,
stopReconnecting: sshSession.stopReconnecting
)
.zIndex(10)
}
Termind/UI/Components/SSHConnectionJourneyView.swift
+64 -1
View File
@@ -15,6 +15,9 @@ struct SSHConnectionJourneyView: View {
var retry: () -> Void
var cancel: () -> Void
var askAI: (() -> Void)?
var reconnectState: ReconnectState? = nil
var retryReconnectNow: (() -> Void)? = nil
var stopReconnecting: (() -> Void)? = nil
@Environment(\.accessibilityReduceMotion) private var reduceMotion
@@ -78,7 +81,7 @@ struct SSHConnectionJourneyView: View {
.font(.title3.weight(.semibold))
.lineLimit(1)
Text(
"\(journey.purpose == .sshSession ? "SSH" : "SFTP") \(journey.destination?.address ?? "")"
headerDetail
)
.font(.callout.monospaced())
.foregroundStyle(.secondary)
@@ -165,6 +168,12 @@ struct SSHConnectionJourneyView: View {
message: message, retry: retry, cancel: cancel,
copyLogs: copyLogs, askAI: askAI
)
} else if journey.mode == .reconnection, let reconnectState {
JourneyReconnectStatus(
state: reconnectState,
retryNow: retryReconnectNow ?? retry,
stop: stopReconnecting ?? cancel
)
} else {
HStack(spacing: 9) {
Image(systemName: statusSymbol)
@@ -221,6 +230,15 @@ struct SSHConnectionJourneyView: View {
}
}
private var headerDetail: String {
let protocolName = journey.purpose == .sshSession ? "SSH" : "SFTP"
let address = journey.destination?.address ?? ""
if journey.mode == .reconnection {
return String(localized: "Reconnecting \(protocolName) · \(address)")
}
return "\(protocolName) \(address)"
}
private var statusSymbol: String {
switch journey.phase {
case .succeeded: "checkmark.circle.fill"
@@ -277,6 +295,51 @@ struct SSHConnectionJourneyView: View {
}
}
private struct JourneyReconnectStatus: View {
let state: ReconnectState
let retryNow: () -> Void
let stop: () -> Void
var body: some View {
TimelineView(.periodic(from: .now, by: 1)) { context in
VStack(spacing: 12) {
HStack(spacing: 9) {
Image(systemName: "arrow.trianglehead.2.clockwise.rotate.90")
.foregroundStyle(.blue)
Text(statusText(at: context.date))
.font(.headline)
}
Text(state.lastError)
.font(.callout)
.foregroundStyle(.secondary)
.lineLimit(2)
.multilineTextAlignment(.center)
.textSelection(.enabled)
HStack(spacing: 10) {
Button("Retry Now", action: retryNow)
.buttonStyle(.borderedProminent)
Button("Stop Reconnecting", role: .cancel, action: stop)
.buttonStyle(.bordered)
}
.controlSize(.large)
}
.padding(.top, 4)
}
}
private func statusText(at date: Date) -> String {
let seconds = max(0, Int(ceil(state.nextRetry.timeIntervalSince(date))))
if seconds == 0 {
return String(localized: "Reconnect attempt \(state.attempt)…")
}
return String(
localized: "Reconnect attempt \(state.attempt) · retrying in \(seconds)s"
)
}
}
private enum JourneyVisualState {
case pending
case active
TermindTests/SFTPFeatureTests.swift
+50
View File
@@ -191,6 +191,51 @@ struct SFTPFeatureTests {
#expect(await client.listRequestCount == countAfterDisconnect)
}
@Test
func connectionLossStartsReconnectWithoutClearingDirectorySnapshot() async {
let host = Host(name: "Files", host: "files.example")
let existing = remoteFile(path: "/home/existing.txt", kind: .file)
let client = SnapshotSFTPClient(
responses: [.success([existing]), .connectionLost]
)
let store = HostStore(hosts: [host])
let sftpSession = SFTPSessionModel(
hostID: host.id, initialPath: "/home", client: client,
configuration: testConfiguration
)
sftpSession.bindStatus(to: store)
await sftpSession.reload()
await sftpSession.reload(silently: true)
#expect(sftpSession.status == .reconnecting)
#expect(sftpSession.reconnectState?.attempt == 1)
#expect(sftpSession.connectionJourney.mode == .reconnection)
#expect(sftpSession.connectionJourney.isPresented)
#expect(sftpSession.files == [existing])
#expect(sftpSession.path == "/home")
sftpSession.disconnect()
}
@Test
func ordinarySFTPFailureDoesNotStartReconnect() async {
let host = Host(name: "Files", host: "files.example")
let client = SnapshotSFTPClient(responses: [.failure])
let store = HostStore(hosts: [host])
let sftpSession = SFTPSessionModel(
hostID: host.id, initialPath: "/home", client: client,
configuration: testConfiguration
)
sftpSession.bindStatus(to: store)
await sftpSession.reload()
#expect(sftpSession.status == .connected)
#expect(sftpSession.reconnectState == nil)
#expect(sftpSession.operationError == "Refresh failed")
sftpSession.disconnect()
}
@Test
func successfulUploadRefreshesTheVisibleDirectory() async throws {
let root = FileManager.default.temporaryDirectory
@@ -349,6 +394,9 @@ struct SFTPFeatureTests {
store.setConnectionStatus(.disconnected, connectionID: sshSessionID, hostID: hostID)
#expect(store.status(of: hostID) == .connecting)
store.setConnectionStatus(.reconnecting, connectionID: sftpSessionID, hostID: hostID)
#expect(store.status(of: hostID) == .reconnecting)
store.setConnectionStatus(.connected, connectionID: sftpSessionID, hostID: hostID)
#expect(store.status(of: hostID) == .connected)
@@ -761,6 +809,7 @@ private actor SnapshotSFTPClient: SFTPClientProtocol {
enum Response: Sendable {
case success([RemoteFile])
case failure
case connectionLost
}
private var responses: [Response]
@@ -780,6 +829,7 @@ private actor SnapshotSFTPClient: SFTPClientProtocol {
switch response {
case .success(let files): return files
case .failure: throw SFTPError.operation("Refresh failed")
case .connectionLost: throw SFTPError.connectionLost("Network lost")
}
}
func attributes(at path: String) async throws -> RemoteFile? { nil }
TermindTests/SSHConnectionJourneyTests.swift
+16
View File
@@ -27,6 +27,22 @@ struct SSHConnectionJourneyTests {
#expect(journey.endpoints[2].hostID == destination.id)
}
@Test
func reconnectUsesTheSameJourneyInReconnectMode() {
let destination = Host(name: "Production", host: "prod.example")
let journey = SSHConnectionJourney()
journey.begin(
host: destination, route: [destination],
purpose: .sshSession, username: "deploy", mode: .reconnection
)
#expect(journey.mode == .reconnection)
#expect(journey.phase == .connecting)
#expect(journey.isPresented)
#expect(journey.plainTextLog.contains("Reconnecting SSH session"))
}
@Test
func authenticationCheckpointAppearsOnlyWhenRequested() {
let destination = Host(name: "Server", host: "server.example")
TermindTests/SessionReconnectControllerTests.swift
+78
View File
@@ -0,0 +1,78 @@
import Foundation
import Testing
@testable import Termind
@Suite
@MainActor
struct SessionReconnectControllerTests {
@Test
func retriesUseCappedBackoffAndRetainLatestFailure() {
let clock = ReconnectTestClock()
let controller = SessionReconnectController(
sleep: { _ in try await Task.sleep(for: .seconds(3_600)) },
now: { clock.value }
)
let expectedDelays: [TimeInterval] = [1, 2, 5, 10, 30, 30]
for (index, expectedDelay) in expectedDelays.enumerated() {
controller.schedule(after: "failure-\(index + 1)") {}
#expect(controller.state?.attempt == index + 1)
#expect(controller.state?.lastError == "failure-\(index + 1)")
#expect(
controller.state?.nextRetry.timeIntervalSince(clock.value)
== expectedDelay
)
}
controller.stop()
}
@Test
func stableConnectionResetsAttemptCounter() {
let clock = ReconnectTestClock()
let controller = SessionReconnectController(
sleep: { _ in try await Task.sleep(for: .seconds(3_600)) },
now: { clock.value }
)
controller.schedule(after: "first") {}
controller.schedule(after: "second") {}
#expect(controller.state?.attempt == 2)
controller.markConnected(initial: false)
clock.advance(by: 61)
controller.schedule(after: "after stable connection") {}
#expect(controller.state?.attempt == 1)
controller.stop()
}
@Test
func retryNowRunsImmediatelyAndStopCancelsFutureRetry() async {
let controller = SessionReconnectController(
sleep: { _ in try await Task.sleep(for: .seconds(3_600)) }
)
var retryCount = 0
controller.schedule(after: "offline") { retryCount += 1 }
controller.retryNow()
await Task.yield()
#expect(retryCount == 1)
controller.stop()
#expect(controller.state == nil)
try? await Task.sleep(for: .milliseconds(10))
#expect(retryCount == 1)
}
}
private nonisolated final class ReconnectTestClock: @unchecked Sendable {
private let lock = NSLock()
private var date = Date(timeIntervalSince1970: 1_000)
var value: Date { lock.withLock { date } }
func advance(by interval: TimeInterval) {
lock.withLock { date.addTimeInterval(interval) }
}
}