import CSSH2 import Darwin import Foundation import Logging import TermindApplication import TermindDomain /// One authenticated SSH connection with any number of non-PTY exec Channels. /// /// libssh2 itself is never entered concurrently. Every call is made by `queue`, /// while command state machines take bounded turns from `ready`. This gives /// commands Session-level concurrency without violating libssh2's single-owner /// requirement or letting a quiet/blocked command pin its peers. nonisolated final class SSHConnectionWorker: @unchecked Sendable { private final class StableCString: @unchecked Sendable { let pointer: UnsafeMutablePointer let count: Int init(_ data: Data) { let bytes = Array(data) count = bytes.count pointer = .allocate(capacity: count + 1) for (index, byte) in bytes.enumerated() { pointer[index] = CChar(bitPattern: byte) } pointer[count] = 0 } deinit { pointer.deallocate() } } private final class SSHExecChannel: @unchecked Sendable { enum Phase { case opening, starting, running, closing, freeing } let id = UUID() let command: StableCString let cancellation: TransferCancellationToken let deadline: ContinuousClock.Instant let outputLimit: Int let onOutput: SSHExecOutputHandler? let continuation: CheckedContinuation let channelLease: SSHChannelLease let input = SSHByteQueue(chunkSize: 32_700, zeroizesConsumedBytes: true) var phase = Phase.opening var execChannel: OpaquePointer? var sentEOF = false var stdout = Data() var stderr = Data() var stdoutRemoteBytes = 0 var stderrRemoteBytes = 0 let stdoutDecoder: RemoteCharsetStreamConverter let stderrDecoder: RemoteCharsetStreamConverter let charset: RemoteCharset var stdoutTruncated = false var stderrTruncated = false var terminalError: (any Error)? var exitCode: Int32 = 0 var signal: String? var started = false var completed = false init( command: String, workingDirectory: String?, stdin: SSHCommandInput?, charset: RemoteCharset, timeout: TimeInterval, outputLimit: Int, cancellation: TransferCancellationToken, onOutput: SSHExecOutputHandler?, channelLease: SSHChannelLease, continuation: CheckedContinuation ) throws { let effective: String if let cwd = workingDirectory?.trimmingCharacters(in: .whitespacesAndNewlines), !cwd.isEmpty { effective = "cd -- \(Self.shellQuote(cwd)) && /bin/sh -lc \(Self.shellQuote(command))" } else { effective = "/bin/sh -lc \(Self.shellQuote(command))" } self.command = StableCString( try RemoteCharsetStreamConverter.encode(effective, charset: charset)) self.charset = charset stdoutDecoder = try RemoteCharsetStreamConverter(charset: charset, direction: .decoding) stderrDecoder = try RemoteCharsetStreamConverter(charset: charset, direction: .decoding) self.cancellation = cancellation deadline = .now.advanced(by: .seconds(max(1, min(timeout, 15 * 60)))) self.outputLimit = outputLimit self.onOutput = onOutput self.channelLease = channelLease self.continuation = continuation try stdin?.append(to: input, charset: charset) sentEOF = input.isEmpty } private static func shellQuote(_ value: String) -> String { "'" + value.replacingOccurrences(of: "'", with: "'\\''") + "'" } } private let queue = DispatchQueue(label: "com.akinokaede.termind.ssh.connection") private let logger: Logger private let maxSessionsPolicy: SSHMaxSessionsPolicy private var connection: SSHAuthenticatedConnection? private var buffer: SSHConnectionBuffer? private var channelBudget: SSHChannelBudget? private var socketEvents: SSHSocketEvents? private var timer: (any DispatchSourceTimer)? private var hostID: UUID? private var charset: RemoteCharset = .utf8 private var commands: [UUID: SSHExecChannel] = [:] private var pendingOpens = SSHReadyQueue() private var opening: SSHExecChannel? private var ready = SSHReadyQueue() private var scheduled = false private var generation = 0 private var controlDirections: Int32 = 0 private var keepaliveDue = true private var nextKeepalive = DispatchTime.now() private var scratch = [CChar](repeating: 0, count: 16 * 1024) private var sftpChannelOwner: SSHMultiplexedSFTPChannelOwner! private var connectionFailure: (@Sendable (String) -> Void)? private var didReportConnectionFailure = false init(logger: Logger, maxSessionsPolicy: SSHMaxSessionsPolicy) { self.logger = logger self.maxSessionsPolicy = maxSessionsPolicy sftpChannelOwner = SSHMultiplexedSFTPChannelOwner( queue: queue, context: { [weak self] in guard let self, let connection = self.connection, let buffer = self.buffer, let channelBudget = self.channelBudget else { return nil } return .init( connection: connection, buffer: buffer, channelBudget: channelBudget, charset: self.charset) }, updateEvents: { [weak self] session, buffer in self?.updateSharedConnectionEvents(session: session, buffer: buffer) }) } deinit { queue.sync { cleanup(error: CancellationError()) } } func connect( using request: SSHConnectionRequest, progress: SSHRouteProgressHandler? = nil, connectionFailure: (@Sendable (String) -> Void)? = nil ) async throws { let operationID = UUID().uuidString.lowercased() let hostID = request.configuration.hostID logger.info( "ssh.multiplexed.connect.started", metadata: [ "host_id": RemoteLogging.hostID(hostID), "operation_id": "\(operationID)", "protocol": "ssh", "stage": "connecting", ]) await disconnect() do { let opened = try await SSHConnectionOpener.openAuthenticatedConnection( using: request, timeout: 20, progress: progress, logger: logger) let channelBudget = maxSessionsPolicy.makeBudget() try await queue.perform { try Task.checkCancellation() self.connection = opened self.hostID = hostID self.charset = request.configuration.charset self.buffer = opened.installBuffer() self.channelBudget = channelBudget self.connectionFailure = connectionFailure self.didReportConnectionFailure = false libssh2_session_set_blocking(opened.libssh2Session, 0) libssh2_keepalive_config(opened.libssh2Session, 0, 30) self.startEvents(socket: opened.byteTransport.socket) } try Task.checkCancellation() logger.info( "ssh.multiplexed.connect.completed", metadata: [ "host_id": RemoteLogging.hostID(hostID), "operation_id": "\(operationID)", "outcome": "success", "protocol": "ssh", "stage": "connected", ]) } catch { await disconnect() logger.error( "ssh.multiplexed.connect.failed", metadata: [ "error_code": RemoteLogging.errorCode(error), "host_id": RemoteLogging.hostID(hostID), "operation_id": "\(operationID)", "outcome": "failure", "protocol": "ssh", "stage": "connecting", ]) throw error } } func execute( _ command: String, workingDirectory: String?, stdin: SSHCommandInput?, timeout: TimeInterval, outputLimit: Int, cancellation: TransferCancellationToken, onOutput: SSHExecOutputHandler? ) async throws -> SSHExecResult { guard !command.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw TermindSSHError.library("The command is empty.") } let operationID = UUID().uuidString.lowercased() var metadata: Logger.Metadata = [ "operation_id": "\(operationID)", "protocol": "ssh", "stage": "executing", ] if let hostID = try? await queue.perform({ self.hostID }) { metadata["host_id"] = RemoteLogging.hostID(hostID) } logger.debug("ssh.exec.started", metadata: metadata) do { guard let channelBudget = try await queue.perform({ self.channelBudget }) else { throw SSHCommandExecutionError.connectionUnavailable(TermindSSHError.disconnected) } let channelLease = try await channelBudget.acquire(cancellation: cancellation) let result = try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in queue.async { [self] in guard connection != nil, self.channelBudget === channelBudget else { channelLease.release() continuation.resume( throwing: SSHCommandExecutionError.connectionUnavailable(TermindSSHError.disconnected)) return } let context: SSHExecChannel do { context = try SSHExecChannel( command: command, workingDirectory: workingDirectory, stdin: stdin, charset: charset, timeout: timeout, outputLimit: outputLimit, cancellation: cancellation, onOutput: onOutput, channelLease: channelLease, continuation: continuation) } catch { channelLease.release() continuation.resume(throwing: error) return } commands[context.id] = context pendingOpens.insert(context.id) schedulePump() } } metadata["outcome"] = "success" metadata["stage"] = "completed" logger.debug("ssh.exec.completed", metadata: metadata) return result } catch { metadata["error_code"] = RemoteLogging.errorCode(error) metadata["outcome"] = "failure" logger.error("ssh.exec.failed", metadata: metadata) throw error } } func openSFTPChannel() async throws -> any SFTPChannelProtocol { try await sftpChannelOwner.openChannel() } private func updateSharedConnectionEvents(session: OpaquePointer, buffer: SSHConnectionBuffer) { let directions = libssh2_session_block_directions(session) socketEvents?.update( read: !buffer.isBackpressured, write: buffer.wantsWrite || directions & LIBSSH2_SESSION_BLOCK_OUTBOUND != 0) } func disconnect() async { await withCheckedContinuation { continuation in queue.async { [self] in cleanup(error: CancellationError()) continuation.resume() } } } private func startEvents(socket: Int32) { generation += 1 let token = generation socketEvents = SSHSocketEvents(socket: socket, queue: queue) { [weak self] in guard let self, self.generation == token else { return } self.wakeAll() } socketEvents?.update(read: false, write: false) let source = DispatchSource.makeTimerSource(queue: queue) source.schedule(deadline: .now() + .milliseconds(100), repeating: .milliseconds(100)) source.setEventHandler { [weak self] in guard let self, self.generation == token else { return } if DispatchTime.now() >= self.nextKeepalive { self.keepaliveDue = true } self.wakeAll() } timer = source source.resume() keepaliveDue = true } private func wakeAll() { for id in commands.keys { ready.insert(id) } schedulePump() } private func schedulePump() { guard connection != nil, !scheduled else { return } scheduled = true let token = generation queue.async { [weak self] in guard let self, self.generation == token else { return } self.scheduled = false self.pump() } } private func pump() { guard let connection, let buffer else { return } let session = connection.libssh2Session let deadline = DispatchTime.now().uptimeNanoseconds + SSHForwardingTuning.current.sliceNanoseconds buffer.beginPass(deadline: deadline) do { try buffer.flush(socket: connection.byteTransport.socket) } catch { failConnection(session) return } if buffer.isBackpressured { socketEvents?.update(read: false, write: true) return } controlDirections = 0 var progressed = advanceOpen(session) while !buffer.isBackpressured, DispatchTime.now().uptimeNanoseconds < deadline, let id = ready.pop() { guard let command = commands[id] else { continue } if advance(command, session: session, deadline: deadline) { progressed = true } } if advanceOpen(session) { progressed = true } if keepaliveDue { var next: Int32 = 0 let result = libssh2_keepalive_send(session, &next) if result == LIBSSH2_ERROR_EAGAIN { blocked(session) } else if result < 0 { failConnection(session) return } else { keepaliveDue = false nextKeepalive = .now() + .seconds(max(1, Int(next))) progressed = true } } do { try buffer.flush(socket: connection.byteTransport.socket) } catch { failConnection(session) return } var byte: UInt8 = 0 let available = Darwin.recv(connection.byteTransport.socket, &byte, 1, MSG_PEEK | MSG_DONTWAIT) if available == 0 || (available < 0 && errno != EAGAIN && errno != EWOULDBLOCK && errno != EINTR) { failConnection(session) return } let directions = commands.values.reduce(controlDirections) { partial, command in guard command.execChannel != nil else { return partial } return partial | libssh2_session_block_directions(session) } socketEvents?.update( read: !commands.isEmpty && !buffer.isBackpressured, write: buffer.wantsWrite || directions & LIBSSH2_SESSION_BLOCK_OUTBOUND != 0) if progressed || buffer.exhaustedReadBudget { wakeAll() } } private func advanceOpen(_ session: OpaquePointer) -> Bool { if opening == nil { while let id = pendingOpens.pop() { if let command = commands[id], command.phase == .opening { opening = command break } } } guard let command = opening else { return false } if cancellationError(for: command) != nil { opening = nil finish(command, throwing: cancellationError(for: command)!) return true } guard let execChannel = libssh2_channel_open_ex( session, "session", 7, 2 * 1_024 * 1_024, 32 * 1_024, nil, 0) else { let code = libssh2_session_last_errno(session) if code == LIBSSH2_ERROR_EAGAIN { blocked(session) return false } opening = nil let error = SSHConnectionOpener.connectionError(session, prefix: "Unable to open SSH exec Channel") if isConnectionError(code) { failConnection(session, error: error) } else { finish(command, throwing: SSHCommandExecutionError.channelUnavailable(error)) } return true } opening = nil command.execChannel = execChannel command.phase = .starting ready.insert(command.id) return true } private func advance(_ command: SSHExecChannel, session: OpaquePointer, deadline: UInt64) -> Bool { if let cancellation = cancellationError(for: command), command.terminalError == nil { command.terminalError = cancellation if command.execChannel == nil { finish(command, throwing: cancellation) return true } command.phase = .closing } guard let execChannel = command.execChannel else { return false } switch command.phase { case .opening: return false case .starting: let result = libssh2_channel_process_startup( execChannel, "exec", 4, command.command.pointer, UInt32(command.command.count)) if result == LIBSSH2_ERROR_EAGAIN { blocked(session) return false } guard result == 0 else { let code = libssh2_session_last_errno(session) let error = SSHConnectionOpener.connectionError(session, prefix: "Unable to start remote command") command.terminalError = SSHCommandExecutionError.channelUnavailable(error) command.phase = .closing if isConnectionError(code) { failConnection(session, error: error) } ready.insert(command.id) return true } command.started = true command.phase = .running ready.insert(command.id) return true case .running: return advanceRunning(command, execChannel: execChannel, session: session, deadline: deadline) case .closing: let result = libssh2_channel_close(execChannel) if result == LIBSSH2_ERROR_EAGAIN { blocked(session) return false } if command.terminalError == nil { command.exitCode = libssh2_channel_get_exit_status(execChannel) command.signal = exitSignal(execChannel: execChannel, session: session) } command.phase = .freeing ready.insert(command.id) return true case .freeing: let result = libssh2_channel_free(execChannel) if result == LIBSSH2_ERROR_EAGAIN { blocked(session) return false } command.execChannel = nil if let error = command.terminalError { finish(command, throwing: error) } else { guard let stdout = String(data: command.stdout, encoding: .utf8), let stderr = String(data: command.stderr, encoding: .utf8) else { finish( command, throwing: RemoteCharsetConversionError( charset: command.charset, direction: .decoding)) return true } finish( command, returning: SSHExecResult( stdout: stdout, stderr: stderr, exitCode: command.exitCode, signal: command.signal, stdoutTruncated: command.stdoutTruncated, stderrTruncated: command.stderrTruncated)) } return true } } private func advanceRunning( _ command: SSHExecChannel, execChannel: OpaquePointer, session: OpaquePointer, deadline: UInt64 ) -> Bool { var progressed = false var remaining = SSHForwardingTuning.current.quantum if !command.input.isEmpty { let written = command.input.withFirstBytes { libssh2_channel_write_ex( execChannel, 0, $0.baseAddress!.assumingMemoryBound(to: CChar.self), $0.count) } if written > 0 { command.input.consume(written) remaining -= min(remaining, written) progressed = true } else if written == LIBSSH2_ERROR_EAGAIN { blocked(session) } else { return fail(command, session: session, prefix: "Unable to write command input") } } if command.input.isEmpty, !command.sentEOF { let result = libssh2_channel_send_eof(execChannel) if result == 0 { command.sentEOF = true progressed = true } else if result == LIBSSH2_ERROR_EAGAIN { blocked(session) } else { return fail(command, session: session, prefix: "Unable to finish command input") } } for stream in [Int32(0), Int32(1)] { while remaining > 0 && DispatchTime.now().uptimeNanoseconds < deadline { let capacity = min(scratch.count, remaining) let count = libssh2_channel_read_ex(execChannel, stream, &scratch, capacity) if count > 0 { progressed = true remaining -= count let chunk = Data(bytes: scratch, count: count) do { try appendDecoded(chunk, stream: stream, to: command) } catch { command.terminalError = error command.phase = .closing ready.insert(command.id) return true } } else if count == LIBSSH2_ERROR_EAGAIN || count == 0 { if count == LIBSSH2_ERROR_EAGAIN { blocked(session) } break } else { return fail(command, session: session, prefix: "Unable to read command output") } } } if libssh2_channel_eof(execChannel) == 1 { do { try finishDecodedOutput(command) } catch { command.terminalError = error } command.phase = .closing ready.insert(command.id) return true } if !command.input.isEmpty || !command.sentEOF || libssh2_poll_channel_read(execChannel, 0) == 1 { ready.insert(command.id) } return progressed } private func appendDecoded(_ chunk: Data, stream: Int32, to command: SSHExecChannel) throws { let isError = stream != 0 let consumed = isError ? command.stderrRemoteBytes : command.stdoutRemoteBytes let available = max(0, command.outputLimit - consumed) let accepted = Data(chunk.prefix(available)) if isError { command.stderrRemoteBytes += accepted.count if accepted.count < chunk.count { command.stderrTruncated = true } } else { command.stdoutRemoteBytes += accepted.count if accepted.count < chunk.count { command.stdoutTruncated = true } } guard !accepted.isEmpty else { return } let decoded = try (isError ? command.stderrDecoder : command.stdoutDecoder).convert(accepted).data guard !decoded.isEmpty else { return } command.onOutput?(decoded, isError) if isError { command.stderr.append(decoded) } else { command.stdout.append(decoded) } } private func finishDecodedOutput(_ command: SSHExecChannel) throws { if !command.stdoutTruncated { let tail = try command.stdoutDecoder.finish().data if !tail.isEmpty { command.onOutput?(tail, false) command.stdout.append(tail) } } if !command.stderrTruncated { let tail = try command.stderrDecoder.finish().data if !tail.isEmpty { command.onOutput?(tail, true) command.stderr.append(tail) } } } private func fail(_ command: SSHExecChannel, session: OpaquePointer, prefix: String) -> Bool { let code = libssh2_session_last_errno(session) let error = SSHConnectionOpener.connectionError(session, prefix: prefix) if isConnectionError(code) { failConnection(session, error: error) } else { command.terminalError = error command.phase = .closing ready.insert(command.id) } return true } private func cancellationError(for command: SSHExecChannel) -> (any Error)? { if command.cancellation.isCancelled { return CancellationError() } if ContinuousClock.now >= command.deadline { return TermindSSHError.library("The remote command timed out.") } return nil } private func exitSignal(execChannel: OpaquePointer, session: OpaquePointer) -> String? { var pointer: UnsafeMutablePointer? var length = 0 let result = libssh2_channel_get_exit_signal(execChannel, &pointer, &length, nil, nil, nil, nil) guard result == 0, let pointer else { return nil } defer { libssh2_free(session, pointer) } return String( decoding: UnsafeBufferPointer(start: pointer, count: length).map(UInt8.init(bitPattern:)), as: UTF8.self) } private func finish(_ command: SSHExecChannel, returning result: SSHExecResult) { guard !command.completed else { return } command.completed = true commands[command.id] = nil command.continuation.resume(returning: result) } private func finish(_ command: SSHExecChannel, throwing error: any Error) { guard !command.completed else { return } command.completed = true commands[command.id] = nil command.continuation.resume(throwing: error) } private func blocked(_ session: OpaquePointer) { controlDirections |= libssh2_session_block_directions(session) } private func isConnectionError(_ code: Int32) -> Bool { [ LIBSSH2_ERROR_SOCKET_SEND, LIBSSH2_ERROR_SOCKET_RECV, LIBSSH2_ERROR_SOCKET_DISCONNECT, LIBSSH2_ERROR_SOCKET_TIMEOUT, LIBSSH2_ERROR_TIMEOUT, LIBSSH2_ERROR_BAD_SOCKET, ].contains(code) } private func failConnection(_ session: OpaquePointer, error: (any Error)? = nil) { let failure = error ?? SSHConnectionOpener.connectionError(session, prefix: "The SSH connection closed") if !didReportConnectionFailure { didReportConnectionFailure = true connectionFailure?(failure.localizedDescription) } cleanup(error: failure) } private func cleanup(error: any Error) { generation += 1 scheduled = false timer?.cancel() timer = nil for command in Array(commands.values) { finish(command, throwing: error) } commands.removeAll() opening = nil pendingOpens = SSHReadyQueue() ready = SSHReadyQueue() sftpChannelOwner.closeAllOnQueue() channelBudget?.close() channelBudget = nil let connection = connection self.connection = nil buffer = nil connectionFailure = nil didReportConnectionFailure = false hostID = nil if let events = socketEvents { events.cancel { connection?.close() } socketEvents = nil } else { connection?.close() } } }