feat: 添加 WebSocket 重连管理器,优化 SSH 和进程监控的自动重连逻辑

This commit is contained in:
cmliu committed 2026-08-04 16:35:18 +08:00
1 parent a99b01a682
commit 6822e3808a
4 files changed
+420 -14

No files matched your search

+22 -5
View File
@@ -102,10 +102,9 @@ const DISCONNECTED_CHANNEL: LocalizedText = {
en: 'File management connection lost',
};
// Auto-reconnect backoff for the file manager after an unexpected drop.
const RECONNECT_BASE_MS = 1_000;
const RECONNECT_MAX_MS = 15_000;
const RECONNECT_MAX_ATTEMPTS = 10;
// Auto-reconnect for the file manager after an unexpected drop.
const RECONNECT_MAX_ATTEMPTS = 3;
const RECONNECT_DELAYS: readonly number[] = [1000, 2000, 4000];
function requiredElement<T extends Element>(root: ParentNode, id: string): T {
const node = root.querySelector<T>(`#${id}`);
@@ -288,6 +287,9 @@ export class FileManager {
});
socket.addEventListener('close', (event: CloseEvent) => {
if (!this.isCurrent(socket, generation)) return;
console.log(
`[WS-Reconnect] ${new Date().toISOString()} | SFTP | disconnect | code=${event.code} | reason="${event.reason}"`,
);
this.socket = null;
this.ready = false;
this.abortTransfers(false);
@@ -1219,11 +1221,20 @@ export class FileManager {
if (!this.wantConnection || !this.url || this.reconnectTimer !== null) return;
this.reconnectAttempts++;
if (this.reconnectAttempts > RECONNECT_MAX_ATTEMPTS) {
console.log(
`[WS-Reconnect] ${new Date().toISOString()} | SFTP | give_up | attempt=${this.reconnectAttempts - 1}/${RECONNECT_MAX_ATTEMPTS}`,
);
this.reconnectAttempts = 0;
this.renderDisconnected(true, { zh: '文件管理重连失败。', en: 'File manager reconnect failed.' });
return;
}
const delay = Math.min(RECONNECT_MAX_MS, RECONNECT_BASE_MS * (2 ** (this.reconnectAttempts - 1)));
const delayIndex = this.reconnectAttempts - 1;
const delay = delayIndex < RECONNECT_DELAYS.length
? RECONNECT_DELAYS[delayIndex]
: RECONNECT_DELAYS[RECONNECT_DELAYS.length - 1];
console.log(
`[WS-Reconnect] ${new Date().toISOString()} | SFTP | reconnect_attempt | attempt=${this.reconnectAttempts}/${RECONNECT_MAX_ATTEMPTS}`,
);
this.setStatus({
zh: '文件管理连接已断开,正在重连…',
en: 'File manager disconnected; reconnecting…',
@@ -1235,6 +1246,9 @@ export class FileManager {
try {
this.attach(this.url);
} catch {
console.log(
`[WS-Reconnect] ${new Date().toISOString()} | SFTP | reconnect_failed | attempt=${attempts}/${RECONNECT_MAX_ATTEMPTS}`,
);
// attach() → reset() zeroed the counter; restore so the next
// close handler's scheduleReconnect() continues accumulating.
if (this.wantConnection) {
@@ -1243,6 +1257,9 @@ export class FileManager {
this.scheduleReconnect();
return;
}
console.log(
`[WS-Reconnect] ${new Date().toISOString()} | SFTP | reconnect_success | attempt=${attempts}/${RECONNECT_MAX_ATTEMPTS}`,
);
// Successful attach(); reset() already zeroed the counter.
// Restore so that if a close fires before the 'open' handler
// can confirm the connection, the back-off chain continues.
+135 -4
View File
@@ -10,6 +10,8 @@ import { FileManager, collectFileManagerElements } from './file-manager';
import { FileTree } from './file-tree';
import { ProcessManager, collectProcessManagerElements, type NetworkSample } from './process-manager';
import { resetTerminalForConnection } from './terminal-session';
import { WebSocketReconnectManager } from './ws-reconnect';
import type { ReconnectLogEntry } from './ws-reconnect';
import './style.css';
type AuthMethod = 'password' | 'publickey';
@@ -411,6 +413,12 @@ let panelOpen = false;
let fileManager: FileManager;
let fileTree: FileTree;
let processManager: ProcessManager;
let sshReconnectManager: WebSocketReconnectManager | null = null;
let reconnectParams: {
host: string; port: number; username: string; authMethod: string;
password?: string; privateKey?: string; pinnedKey?: string;
term: string; encoding: string;
} | null = null;
// Network rate state. The backend sends cumulative byte counters per interface
// per tick; we keep a per-interface baseline (counters + local clock timestamp)
@@ -1725,6 +1733,91 @@ async function issueTicket(signal: AbortSignal): Promise<{ ticket: string; sessi
return { ticket: payload.ticket, sessionId: payload.sessionId };
}
/** Factory that builds a fresh SSH WebSocket for reconnection attempts. */
function createSshReconnectFactory(): (attempt: number) => Promise<WebSocket> {
return async (attempt: number): Promise<WebSocket> => {
const params = reconnectParams!;
const abortController = new AbortController();
const { ticket, sessionId } = await issueTicket(abortController.signal);
currentSessionId = sessionId;
const protocol = location.protocol === 'https:' ? 'wss:' : 'ws:';
const url = new URL('/api/ssh', location.href);
url.protocol = protocol;
url.searchParams.set('ticket', ticket);
url.searchParams.set('session', sessionId);
const generation = ++connectGeneration;
const ws = new WebSocket(url);
ws.binaryType = 'arraybuffer';
ws.addEventListener('open', () => {
if (generation !== connectGeneration) {
ws.close(1000, 'Connection superseded');
return;
}
// Reset terminal for the new session.
resetTerminalForConnection(terminal);
fitTerminal(true);
const config: ConnectionConfig = {
type: 'connect',
host: params.host,
port: params.port,
username: params.username,
authMethod: params.authMethod as AuthMethod,
cols: terminal.cols,
rows: terminal.rows,
term: params.term,
};
if (params.authMethod === 'password') config.password = params.password;
else config.privateKey = params.privateKey;
if (params.pinnedKey) config.expectedFingerprint = params.pinnedKey;
ws.send(JSON.stringify(config));
updateConnectionStatus(localized('正在打开 TCP 连接...', 'Opening TCP connection...'));
event(bilingual('WebSocket 已建立,正在打开 SSH 传输(自动重连)。', 'WebSocket established; opening SSH transport (auto-reconnect).'), 'transport');
}, { once: true });
ws.addEventListener('message', (socketEvent) => {
void handleSocketData(socketEvent.data as string | ArrayBuffer | Blob, ws, generation);
});
ws.addEventListener('error', () => {
if (socket !== ws) return;
const message = localized('WebSocket 传输错误。', 'WebSocket transport error.');
event(localize(message), 'transport', true);
});
socket = ws;
currentExpectedFingerprint = params.pinnedKey ?? '';
decoder = createDecoder(params.encoding);
return ws;
};
}
/** Handles reconnect lifecycle events and updates the SSH UI accordingly. */
function handleSshReconnectLog(entry: ReconnectLogEntry): void {
if (entry.event === 'give_up') {
// Reconnect exhausted — perform full cleanup that was deferred.
const reason = bilingual('SSH 重连失败,已达最大重试次数。', 'SSH reconnect failed; maximum retries reached.');
event(reason, 'disconnect', true);
updateConnectionStatus(messageTranslation(reason));
setState('error');
toast(reason, 'error');
fileManager.reset();
fileTree?.setReady(false);
processManager.reset();
} else if (entry.event === 'reconnect_attempt') {
updateConnectionStatus(localized(
`正在重连 SSH(${entry.attempt}/${entry.maxAttempts})…`,
`Reconnecting SSH (${entry.attempt}/${entry.maxAttempts})…`,
));
} else if (entry.event === 'reconnect_success') {
event(bilingual('SSH 重连成功。', 'SSH reconnected successfully.'), 'connect');
}
}
async function connect(): Promise<void> {
if (connectionState === 'connecting' || connectionState === 'connected' || connectionState === 'disconnecting') return;
if (socket || authorizationAbort) return;
@@ -1797,6 +1890,22 @@ async function connect(): Promise<void> {
const activeSocket = new WebSocket(url);
socket = activeSocket;
activeSocket.binaryType = 'arraybuffer';
// Persist connection parameters so the reconnect factory can reuse them.
reconnectParams = { host, port, username, authMethod: method, term, encoding: ui.encoding.value };
if (method === 'password') reconnectParams.password = password;
else reconnectParams.privateKey = privateKey;
if (pinnedKey) reconnectParams.pinnedKey = pinnedKey;
// Set up (or replace) the SSH reconnect manager.
sshReconnectManager?.reset();
sshReconnectManager = new WebSocketReconnectManager({
id: 'SSH',
onConnect: createSshReconnectFactory(),
onLog: handleSshReconnectLog,
});
sshReconnectManager.attach(activeSocket);
activeSocket.addEventListener('open', () => {
if (socket !== activeSocket || generation !== connectGeneration) {
activeSocket.close(1000, 'Connection attempt superseded');
@@ -1838,6 +1947,26 @@ async function connect(): Promise<void> {
pendingHistory = null;
currentExpectedFingerprint = '';
currentRememberedFingerprint = '';
const wasActive = connectionState === 'connected';
const isUnexpected = closeEvent.code !== 1000 && closeEvent.code !== 1005;
if (isUnexpected && sshReconnectManager) {
// The reconnect manager will attempt reconnection autonomously.
// Skip tearing down child connections — they each have their own
// reconnect logic that fires independently.
stopTimers();
resetNetworkMetric();
clearHostKeyPrompt();
invalidateHistoryPasswordLoad();
const reason = bilingual('SSH 连接断开,正在重连…', 'SSH connection lost; reconnecting…');
event(reason, 'disconnect', true);
updateConnectionStatus(messageTranslation(reason));
setState('connecting');
if (wasActive) toast(reason, 'error');
return;
}
// Normal close or reconnect not available — full cleanup.
stopTimers();
fileManager.reset();
fileTree?.setReady(false);
@@ -1845,16 +1974,15 @@ async function connect(): Promise<void> {
resetNetworkMetric();
clearHostKeyPrompt();
invalidateHistoryPasswordLoad();
const wasActive = connectionState === 'connected';
const reason = closeEvent.reason
? bilingualServerMessage(closeEvent.reason)
: closeEvent.code === 1000
? bilingual('会话已关闭。', 'Session closed.')
: bilingual(`会话已关闭(${closeEvent.code})。`, `Session closed (${closeEvent.code}).`);
event(reason, 'disconnect', closeEvent.code !== 1000 && closeEvent.code !== 1005);
event(reason, 'disconnect', isUnexpected);
updateConnectionStatus(messageTranslation(reason));
setState(closeEvent.code === 1000 || closeEvent.code === 1005 ? 'idle' : 'error');
if (wasActive) toast(reason, closeEvent.code === 1000 || closeEvent.code === 1005 ? 'info' : 'error');
setState(isUnexpected ? 'error' : 'idle');
if (wasActive) toast(reason, isUnexpected ? 'error' : 'info');
});
} catch (error) {
if (authorizationAbort === abortController) authorizationAbort = null;
@@ -1884,6 +2012,9 @@ function disconnect(reason = bilingual('已由用户断开连接', 'Disconnected
authorizationAbort = null;
const activeSocket = socket;
socket = null;
sshReconnectManager?.reset();
sshReconnectManager = null;
reconnectParams = null;
stopTimers();
fileManager.reset();
fileTree?.setReady(false);
+20 -5
View File
@@ -101,10 +101,9 @@ const MAX_PROCESSES = 512;
// per tick. Mirrors the shell/parser cap (src/backend) so a hostile or
// misbehaving host cannot flood the frontend with rows.
const MAX_NETWORK_INTERFACES = 32;
// Auto-reconnect backoff for the process monitor after an unexpected drop.
const RECONNECT_BASE_MS = 1_000;
const RECONNECT_MAX_MS = 15_000;
const RECONNECT_MAX_ATTEMPTS = 10;
// Auto-reconnect for the process monitor after an unexpected drop.
const RECONNECT_MAX_ATTEMPTS = 3;
const RECONNECT_DELAYS: readonly number[] = [1000, 2000, 4000];
// Hard cap on the stdout/stderr text we render from a kill result. Backend already truncates
// the wire payload, but the frontend keeps its own ceiling so a malformed payload cannot fill
// the dialog / toast with megabytes of text.
@@ -307,6 +306,9 @@ export class ProcessManager {
});
socket.addEventListener('close', (event) => {
if (!this.isCurrent(socket, generation)) return;
console.log(
`[WS-Reconnect] ${new Date().toISOString()} | Process | disconnect | code=${event.code} | reason="${event.reason}"`,
);
this.socket = null;
if (event.code !== 1000 && event.code !== 1005) {
// Unexpected drop. Auto-reconnect unless the user intentionally tore the session down.
@@ -836,21 +838,34 @@ export class ProcessManager {
if (!this.wantConnection || !this.url || this.reconnectTimer !== null) return;
this.reconnectAttempts++;
if (this.reconnectAttempts > RECONNECT_MAX_ATTEMPTS) {
console.log(
`[WS-Reconnect] ${new Date().toISOString()} | Process | give_up | attempt=${this.reconnectAttempts - 1}/${RECONNECT_MAX_ATTEMPTS}`,
);
this.reconnectAttempts = 0;
this.showError('进程监控已意外停止。', 'Process monitor stopped unexpectedly.');
return;
}
const delay = Math.min(RECONNECT_MAX_MS, RECONNECT_BASE_MS * (2 ** (this.reconnectAttempts - 1)));
const delayIndex = this.reconnectAttempts - 1;
const delay = delayIndex < RECONNECT_DELAYS.length
? RECONNECT_DELAYS[delayIndex]
: RECONNECT_DELAYS[RECONNECT_DELAYS.length - 1];
console.log(
`[WS-Reconnect] ${new Date().toISOString()} | Process | reconnect_attempt | attempt=${this.reconnectAttempts}/${RECONNECT_MAX_ATTEMPTS}`,
);
if (this.reconnectAttempts === 1 && this.onReconnect) {
this.onReconnect('进程监控已意外停止,正在尝试自动重连…', 'Process monitor stopped unexpectedly; attempting to reconnect automatically…');
}
this.setStatus('进程监控连接已断开,正在重连…', 'Process monitor disconnected; reconnecting…');
const attempts = this.reconnectAttempts;
this.reconnectTimer = window.setTimeout(() => {
this.reconnectTimer = null;
if (!this.wantConnection || !this.url) return;
try {
this.attach(this.url);
} catch {
console.log(
`[WS-Reconnect] ${new Date().toISOString()} | Process | reconnect_failed | attempt=${attempts}/${RECONNECT_MAX_ATTEMPTS}`,
);
this.scheduleReconnect();
}
}, delay);
+243
View File
@@ -0,0 +1,243 @@
/**
* WebSocket Reconnect Manager
*
* A framework-agnostic utility that monitors a WebSocket connection and
* automatically attempts to re-establish it after an unexpected drop.
* Each instance manages one connection independently; multiple instances
* can coexist without interfering with each other.
*/
export interface ReconnectConfig {
/** Human-readable identifier used in log output, e.g. "SSH" / "SFTP" / "Process". */
id: string;
/** Maximum number of consecutive reconnection attempts. Default: 3. */
maxAttempts?: number;
/** Per-attempt delay in milliseconds. Index 0 = first retry delay, etc.
* If the attempt count exceeds the array length the last value is reused.
* Default: [1000, 2000, 4000]. */
delays?: number[];
/**
* Factory that creates a fresh WebSocket for each reconnection attempt.
* Called with the current attempt number (1-based).
* The manager calls `attach()` on the returned socket to monitor its
* close events for future reconnects.
*/
onConnect: (attempt: number) => WebSocket | Promise<WebSocket>;
/** Optional callback invoked for every lifecycle event. */
onLog?: (entry: ReconnectLogEntry) => void;
}
export interface ReconnectLogEntry {
/** ISO-8601 timestamp of the event. */
timestamp: string;
/** Connection identifier from the config. */
id: string;
/** Lifecycle event type. */
event: 'disconnect' | 'reconnect_attempt' | 'reconnect_success' | 'reconnect_failed' | 'give_up';
/** WebSocket close code (disconnect event only). */
code?: number;
/** WebSocket close reason (disconnect event only). */
reason?: string;
/** Current attempt number (1-based). */
attempt?: number;
/** Configured maximum attempts. */
maxAttempts?: number;
}
const DEFAULT_MAX_ATTEMPTS = 3;
const DEFAULT_DELAYS: readonly number[] = [1000, 2000, 4000];
export class WebSocketReconnectManager {
private id: string;
private maxAttempts: number;
private delays: number[];
private onConnect: (attempt: number) => WebSocket | Promise<WebSocket>;
private onLog?: (entry: ReconnectLogEntry) => void;
private socket: WebSocket | null = null;
private destroyed = false;
private reconnecting = false;
private attempt = 0;
private timer: ReturnType<typeof setTimeout> | null = null;
constructor(config: ReconnectConfig) {
this.id = config.id;
this.maxAttempts = config.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
this.delays = config.delays ?? [...DEFAULT_DELAYS];
this.onConnect = config.onConnect;
this.onLog = config.onLog;
}
/**
* Bind an existing WebSocket to this manager.
* The manager will listen for the `close` event and automatically
* begin the reconnection sequence when the socket drops unexpectedly.
*
* Safe to call multiple times; each call replaces the tracked socket
* and re-registers the close listener.
*/
attach(socket: WebSocket): void {
if (this.destroyed) return;
this.socket = socket;
this.socket.addEventListener('close', (event: CloseEvent) => this.handleClose(event));
}
/**
* Stop all reconnection activity and forget the current socket.
* Call this when the user intentionally disconnects (code 1000).
* After reset the manager is permanently deactivated — create a new
* instance when the user initiates a fresh connection.
*/
reset(): void {
this.clearTimer();
this.destroyed = true;
this.socket = null;
this.reconnecting = false;
this.attempt = 0;
}
/**
* Alias for `reset()` to match common teardown naming conventions.
*/
destroy(): void {
this.reset();
}
// ------------------------------------------------------------------ private
private handleClose(event: CloseEvent): void {
if (this.destroyed) return;
this.log({
timestamp: new Date().toISOString(),
id: this.id,
event: 'disconnect',
code: event.code,
reason: event.reason,
});
// Normal closures (user-initiated or "no status") should never reconnect.
if (event.code === 1000 || event.code === 1005) {
return;
}
this.startReconnect();
}
private startReconnect(): void {
if (this.destroyed || this.reconnecting) return;
this.reconnecting = true;
this.attempt = 0;
this.tryReconnect();
}
private tryReconnect(): void {
if (this.destroyed) {
this.reconnecting = false;
return;
}
this.attempt++;
if (this.attempt > this.maxAttempts) {
this.reconnecting = false;
this.log({
timestamp: new Date().toISOString(),
id: this.id,
event: 'give_up',
attempt: this.attempt - 1,
maxAttempts: this.maxAttempts,
});
return;
}
const delayIndex = this.attempt - 1;
const delay = delayIndex < this.delays.length
? this.delays[delayIndex]
: this.delays[this.delays.length - 1];
this.log({
timestamp: new Date().toISOString(),
id: this.id,
event: 'reconnect_attempt',
attempt: this.attempt,
maxAttempts: this.maxAttempts,
});
this.timer = setTimeout(() => {
this.timer = null;
if (this.destroyed) {
this.reconnecting = false;
return;
}
void this.executeReconnect();
}, delay);
}
private async executeReconnect(): Promise<void> {
const attemptSnapshot = this.attempt;
try {
const newSocket = await this.onConnect(attemptSnapshot);
if (this.destroyed) {
this.reconnecting = false;
if (newSocket.readyState < WebSocket.CLOSING) {
newSocket.close(1000, 'Reconnect superseded');
}
return;
}
// Re-attach the close listener so future drops are also handled.
this.socket = newSocket;
this.socket.addEventListener('close', (event: CloseEvent) => this.handleClose(event));
this.reconnecting = false;
this.log({
timestamp: new Date().toISOString(),
id: this.id,
event: 'reconnect_success',
attempt: attemptSnapshot,
maxAttempts: this.maxAttempts,
});
} catch {
if (this.destroyed) {
this.reconnecting = false;
return;
}
this.log({
timestamp: new Date().toISOString(),
id: this.id,
event: 'reconnect_failed',
attempt: attemptSnapshot,
maxAttempts: this.maxAttempts,
});
this.tryReconnect();
}
}
private log(entry: ReconnectLogEntry): void {
const details = this.formatDetails(entry);
console.log(`[WS-Reconnect] ${entry.timestamp} | ${entry.id} | ${entry.event} | ${details}`);
this.onLog?.(entry);
}
private formatDetails(entry: ReconnectLogEntry): string {
const parts: string[] = [];
if (entry.code !== undefined) parts.push(`code=${entry.code}`);
if (entry.reason !== undefined && entry.reason !== '') parts.push(`reason="${entry.reason}"`);
if (entry.attempt !== undefined && entry.maxAttempts !== undefined) {
parts.push(`attempt=${entry.attempt}/${entry.maxAttempts}`);
}
return parts.length > 0 ? parts.join(' | ') : '-';
}
private clearTimer(): void {
if (this.timer !== null) {
clearTimeout(this.timer);
this.timer = null;
}
}
}