refactor: 重构 WebSocket 可靠投递会话模型

This commit is contained in:
dushixiang committed 2026-08-31 21:52:03 +08:00
1 parent 787fa6918d
commit 0fa920e972
7 files changed
+892 -485

No files matched your search

+2 -1
View File
@@ -246,7 +246,7 @@ func (h *AgentHandler) Delete(c *echo.Context) error {
}
// 断开连接
client.Conn.Close()
client.Close()
}
// 删除探针及其所有相关数据
@@ -257,6 +257,7 @@ func (h *AgentHandler) Delete(c *echo.Context) error {
zap.Error(err))
return err
}
h.wsManager.ForgetAgent(agentID)
h.logger.Info("探针已删除",
zap.String("agentID", agentID),
+61 -44
View File
@@ -3,6 +3,7 @@ package handler
import (
"context"
"encoding/json"
"errors"
"net/http"
"time"
@@ -38,13 +39,24 @@ func (h *AgentHandler) HandleWebSocket(c *echo.Context) error {
return err
}
// 同步探针会话:BootID 变化(探针重启)时重置确认位点,
// 再把当前位点随注册响应返回
h.wsManager.BeginAgentSession(agent.ID, registerReq.BootID)
// 先原子激活连接及其 BootID 会话,再发送注册响应。这样响应中的
// AckSeq 与当前连接属于同一次会话切换,旧连接无法在其间污染位点。
client := ws.NewClient(agent.ID, conn, h.wsManager)
ackSeq := h.wsManager.Register(client, registerReq.BootID)
// RegisterAgent 的在线写入发生在连接切换之前,可能被恰好退出的旧连接
// 覆盖;会话激活后再确认一次,确保数据库状态属于 current client。
if err := h.agentService.UpdateAgentStatus(context.Background(), agent.ID, 1); err != nil {
h.logger.Warn("failed to confirm agent online status", zap.String("agentID", agent.ID), zap.Error(err))
}
defer func() {
if h.wsManager.Unregister(client) {
h.markAgentOffline(client.ID)
}
}()
// 发送注册成功响应:声明可靠投递支持,并回传该探针的累计确认
// 位点,探针据此跳过已处理消息、只重放未确认部分
if err := h.sendRegisterSuccess(conn, agent.ID); err != nil {
if err := h.sendRegisterSuccess(conn, agent.ID, ackSeq); err != nil {
h.logger.Error("failed to send register ack", zap.Error(err))
conn.Close()
return err
@@ -68,17 +80,6 @@ func (h *AgentHandler) HandleWebSocket(c *echo.Context) error {
}
}
// 创建客户端并注册到管理器
client := h.newClient(agent.ID, conn)
h.wsManager.Register(client)
// 连接结束后标记离线;若探针已由新连接替换则跳过,避免旧连接的
// 退出把新连接打成离线
defer func() {
h.markAgentOffline(client)
}()
// 启动读写协程
go client.WritePump()
client.ReadPump(context.Background())
@@ -159,22 +160,8 @@ func (h *AgentHandler) readRegisterRequest(conn *websocket.Conn) (*protocol.Regi
return &registerReq, nil
}
func (h *AgentHandler) markAgentOffline(client *ws.Client) {
// 探针已由新连接替换时跳过:离线状态由新连接的活跃心跳管理
if cur, ok := h.wsManager.GetClient(client.ID); ok && cur != client {
return
}
_ = h.agentService.UpdateAgentStatus(context.Background(), client.ID, 0)
}
func (h *AgentHandler) newClient(agentID string, conn *websocket.Conn) *ws.Client {
return &ws.Client{
ID: agentID,
Conn: conn,
Send: make(chan []byte, 512),
Manager: h.wsManager,
LastActive: time.Now(),
}
func (h *AgentHandler) markAgentOffline(agentID string) {
_ = h.agentService.UpdateAgentStatus(context.Background(), agentID, 0)
}
// pongStatusRefreshInterval 在线状态心跳刷新间隔:本连接首次 pong 立即
@@ -189,7 +176,13 @@ func (h *AgentHandler) handleWebSocketPong(client *ws.Client) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := h.agentService.UpdateAgentStatus(ctx, client.ID, 1); err != nil {
current, err := h.wsManager.DoIfCurrent(client, func() error {
return h.agentService.UpdateAgentStatus(ctx, client.ID, 1)
})
if !current {
return
}
if err != nil {
h.logger.Warn("failed to update agent status on pong", zap.String("agentID", client.ID), zap.Error(err))
return
}
@@ -199,36 +192,60 @@ func (h *AgentHandler) handleWebSocketPong(client *ws.Client) {
func (h *AgentHandler) handleMetricsMessage(ctx context.Context, agentID string, data json.RawMessage) error {
var batch protocol.MetricsBatch
if err := json.Unmarshal(data, &batch); err != nil {
return err
return ws.Permanent(err)
}
var transientErr error
var permanentErr error
for _, sample := range batch.Samples {
metricsData, err := json.Marshal(sample.Data)
if err != nil {
h.logger.Warn("failed to marshal metric sample", zap.Error(err))
permanentErr = errors.Join(permanentErr, err)
continue
}
if err := h.metricService.HandleMetricData(ctx, agentID, string(sample.Type), metricsData, sample.Timestamp); err != nil {
h.logger.Warn("failed to handle metric sample", zap.Error(err), zap.String("type", string(sample.Type)))
if isPayloadError(err) {
permanentErr = errors.Join(permanentErr, err)
} else {
transientErr = errors.Join(transientErr, err)
}
continue
}
}
return nil
if transientErr != nil {
return transientErr
}
return ws.Permanent(permanentErr)
}
func (h *AgentHandler) handleCommandResponseMessage(ctx context.Context, agentID string, data json.RawMessage) error {
var cmdResp protocol.CommandResponse
if err := json.Unmarshal(data, &cmdResp); err != nil {
return err
return ws.Permanent(err)
}
return h.agentService.HandleCommandResponse(ctx, agentID, &cmdResp)
return classifyMessageError(h.agentService.HandleCommandResponse(ctx, agentID, &cmdResp))
}
func isPayloadError(err error) bool {
var syntaxErr *json.SyntaxError
var typeErr *json.UnmarshalTypeError
return errors.As(err, &syntaxErr) || errors.As(err, &typeErr)
}
func classifyMessageError(err error) error {
if isPayloadError(err) {
return ws.Permanent(err)
}
return err
}
func (h *AgentHandler) handleTamperEventMessage(ctx context.Context, agentID string, data json.RawMessage) error {
var eventData protocol.TamperEventData
if err := json.Unmarshal(data, &eventData); err != nil {
h.logger.Error("failed to unmarshal tamper event", zap.Error(err))
return err
return ws.Permanent(err)
}
return h.tamperService.CreateEvent(ctx, agentID, &eventData)
}
@@ -237,7 +254,7 @@ func (h *AgentHandler) handleDDNSIPReportMessage(ctx context.Context, agentID st
var ipReport protocol.DDNSIPReportData
if err := json.Unmarshal(data, &ipReport); err != nil {
h.logger.Error("failed to unmarshal ddns ip report", zap.Error(err))
return err
return ws.Permanent(err)
}
return h.ddnsService.HandleIPReport(ctx, agentID, &ipReport)
}
@@ -246,7 +263,7 @@ func (h *AgentHandler) handlePublicIPReportMessage(ctx context.Context, agentID
var ipReport protocol.PublicIPReportData
if err := json.Unmarshal(data, &ipReport); err != nil {
h.logger.Error("failed to unmarshal public ip report", zap.Error(err))
return err
return ws.Permanent(err)
}
return h.agentService.UpdatePublicIP(ctx, agentID, ipReport.IPv4, ipReport.IPv6)
}
@@ -255,7 +272,7 @@ func (h *AgentHandler) handleSSHLoginEventMessage(ctx context.Context, agentID s
var eventData protocol.SSHLoginEvent
if err := json.Unmarshal(data, &eventData); err != nil {
h.logger.Error("failed to unmarshal ssh login event", zap.Error(err))
return err
return ws.Permanent(err)
}
return h.sshLoginService.HandleEvent(ctx, agentID, eventData)
}
@@ -264,7 +281,7 @@ func (h *AgentHandler) handleSSHLoginConfigResultMessage(ctx context.Context, ag
var resultData protocol.SSHLoginConfigResult
if err := json.Unmarshal(data, &resultData); err != nil {
h.logger.Error("failed to unmarshal ssh login config result", zap.Error(err))
return err
return ws.Permanent(err)
}
return h.sshLoginService.HandleConfigResult(ctx, agentID, resultData)
}
@@ -273,18 +290,18 @@ func (h *AgentHandler) handleTamperProtectMessage(ctx context.Context, agentID s
var protectResp protocol.TamperProtectResponse
if err := json.Unmarshal(data, &protectResp); err != nil {
h.logger.Error("failed to unmarshal tamper protect response", zap.Error(err))
return err
return ws.Permanent(err)
}
return h.tamperService.HandleConfigResult(ctx, agentID, protectResp)
}
// sendRegisterSuccess 发送注册成功响应
func (h *AgentHandler) sendRegisterSuccess(conn *websocket.Conn, agentID string) error {
func (h *AgentHandler) sendRegisterSuccess(conn *websocket.Conn, agentID string, ackSeq uint64) error {
resp := protocol.RegisterResponse{
AgentID: agentID,
Status: "success",
Reliable: true,
AckSeq: h.wsManager.AckSeq(agentID),
AckSeq: ackSeq,
}
return conn.WriteJSON(protocol.OutboundMessage{
Type: protocol.MessageTypeRegisterAck,
+478 -294
View File
@@ -3,6 +3,7 @@ package websocket
import (
"context"
"encoding/json"
"errors"
"sync"
"sync/atomic"
"time"
@@ -17,103 +18,202 @@ const (
serverPongWait = 30 * time.Second
serverWriteWait = 10 * time.Second
// agentQueueSize 每个探针的待处理消息队列长度。队列满时断开连接,
// 由探针重连后重放未确认消息,读循环不会被慢处理(DB/VM 写入)拖死。
agentQueueSize = 512
// agentQueueWait 消息入队的最长等待,超时判定为处理能力不足
agentQueueWait = 2 * time.Second
// serverSendWait 下行消息写入 Send 通道的最长等待
// 可靠事件必须保持会话内顺序;队列满表示服务端已无法及时消费,
// 断开当前传输后由 agent 保留 outbox 并重连重放。
reliableQueueSize = 512
reliableQueueWait = 2 * time.Second
// 普通遥测是 latest-wins,不应阻塞可靠事件。队列满时丢弃本批,
// agent 下一轮会继续上报更新后的快照。
bestEffortQueueSize = 128
serverSendWait = 3 * time.Second
messageRetryMin = 500 * time.Millisecond
messageRetryMax = 10 * time.Second
)
// agentMessage 从探针收到的一条待处理消息
// MessageHandler 处理一条 agent 上行消息。
type MessageHandler func(ctx context.Context, agentID string, messageType string, data json.RawMessage) error
// PongHandler 处理连接 pong。
type PongHandler func(client *Client)
// PermanentMessageError 表示重放也无法成功的坏消息,例如 JSON 结构错误。
// 可靠处理器会记录并确认这类消息,避免单条毒消息永久阻塞后续事件。
type PermanentMessageError struct {
err error
}
func (e *PermanentMessageError) Error() string { return e.err.Error() }
func (e *PermanentMessageError) Unwrap() error { return e.err }
// Permanent 把不可重试的消息错误标记为永久错误。
func Permanent(err error) error {
if err == nil {
return nil
}
return &PermanentMessageError{err: err}
}
func isPermanent(err error) bool {
var target *PermanentMessageError
return errors.As(err, &target)
}
type agentMessage struct {
seq uint64
typ string
data json.RawMessage
}
// processor 单个探针连接的有序处理单元:ReadPump 只负责解析并入队,
// processLoop 串行消费,慢处理不再阻塞 TCP 读取造成反压。
type processor struct {
queue chan agentMessage
quit chan struct{}
quitOnce sync.Once
// agentSession 是可靠投递状态的唯一所有者。它绑定 agentID + BootID,
// 与具体 WebSocket 连接解耦:同一进程重连复用 session,新进程注册时
// 原子替换 session,旧 worker 即使仍在返回路径上也无法提交 ACK。
type agentSession struct {
agentID string
bootID string
ctx context.Context
cancel context.CancelFunc
reliableQueue chan agentMessage
bestEffortQueue chan agentMessage
ackSeq atomic.Uint64
}
func newProcessor() *processor {
return &processor{
queue: make(chan agentMessage, agentQueueSize),
quit: make(chan struct{}),
func newAgentSession(agentID, bootID string) *agentSession {
ctx, cancel := context.WithCancel(context.Background())
return &agentSession{
agentID: agentID,
bootID: bootID,
ctx: ctx,
cancel: cancel,
reliableQueue: make(chan agentMessage, reliableQueueSize),
bestEffortQueue: make(chan agentMessage, bestEffortQueueSize),
}
}
func (p *processor) stop() {
p.quitOnce.Do(func() { close(p.quit) })
}
// Client WebSocket客户端
// Client 只代表一次 WebSocket 传输,不拥有可靠会话状态。
type Client struct {
ID string // 探针ID
Conn *websocket.Conn // WebSocket连接
Send chan []byte // 发送消息通道
Manager *Manager // 管理器引用
LastActive time.Time // 最后活跃时间
proc *processor // 上行消息的有序处理器
// lastStatusWrite 上次把在线状态写入数据库的时间(毫秒),用于
// 对 pong 触发的状态更新去抖
ID string
conn *websocket.Conn
manager *Manager
session *agentSession
send chan []byte
done chan struct{}
closeOnce sync.Once
lastActive atomic.Int64
lastStatusWrite atomic.Int64
closed bool // 标记channel是否已关闭
closeMu sync.Mutex
}
// Manager WebSocket连接管理器
// NewClient 创建尚未注册到 Manager 的连接。
func NewClient(agentID string, conn *websocket.Conn, manager *Manager) *Client {
c := &Client{
ID: agentID,
conn: conn,
manager: manager,
send: make(chan []byte, 512),
done: make(chan struct{}),
}
c.touch()
return c
}
func (c *Client) touch() {
c.lastActive.Store(time.Now().UnixMilli())
}
func (c *Client) inactiveFor(now time.Time) time.Duration {
last := c.lastActive.Load()
if last == 0 {
return 0
}
return now.Sub(time.UnixMilli(last))
}
func (c *Client) close() {
c.closeOnce.Do(func() {
close(c.done)
if c.conn != nil {
_ = c.conn.Close()
}
})
}
// Close 主动关闭本次传输;Handler 的退出路径负责同步注销 current client。
func (c *Client) Close() {
c.close()
}
func (c *Client) sendWithTimeout(message []byte, timeout time.Duration) error {
select {
case <-c.done:
return ErrClientNotFound
default:
}
timer := time.NewTimer(timeout)
defer timer.Stop()
select {
case <-c.done:
return ErrClientNotFound
case c.send <- message:
return nil
case <-timer.C:
return ErrSendTimeout
}
}
func (c *Client) trySend(message []byte) bool {
select {
case <-c.done:
return false
default:
}
select {
case <-c.done:
return false
case c.send <- message:
return true
default:
return false
}
}
// Manager 同步维护当前连接和每个 agent 的会话状态。
type Manager struct {
clients map[string]*Client // 客户端映射 probeID -> Client
register chan *Client // 注册通道
unregister chan *Client // 注销通道
broadcast chan []byte // 广播通道
mu sync.RWMutex // 读写锁
logger *zap.Logger // 日志
onMessage MessageHandler // 消息处理器
onPong PongHandler // Pong 处理器
// ackSeqs 每个探针事件流的累计确认位点(探针重启或服务端重启后
// 丢失,退化为重放+去重,可能产生少量重复处理)
ackSeqs map[string]uint64
// sessions 每个探针最近一次注册携带的 BootID,用于识别探针进程重启
sessions map[string]string
mu sync.RWMutex
clients map[string]*Client
sessions map[string]*agentSession
logger *zap.Logger
onMessage MessageHandler
onPong PongHandler
}
// MessageHandler 消息处理器接口
type MessageHandler func(ctx context.Context, probeID string, messageType string, data json.RawMessage) error
// PongHandler Pong 处理器接口
type PongHandler func(client *Client)
// NewManager 创建新的WebSocket管理器
func NewManager(logger *zap.Logger) *Manager {
return &Manager{
clients: make(map[string]*Client),
register: make(chan *Client, 10),
unregister: make(chan *Client, 10),
broadcast: make(chan []byte, 256),
ackSeqs: make(map[string]uint64),
sessions: make(map[string]string),
logger: logger,
clients: make(map[string]*Client),
sessions: make(map[string]*agentSession),
logger: logger,
}
}
// SetMessageHandler 设置消息处理器
func (m *Manager) SetMessageHandler(handler MessageHandler) {
m.onMessage = handler
}
// SetPongHandler 设置 Pong 处理器
func (m *Manager) SetPongHandler(handler PongHandler) {
m.onPong = handler
}
// Run 启动管理器
// Run 只负责周期性连接保活检查和进程退出清理;注册/注销本身是同步的,
// 注册响应拿到的 ACK 与刚完成的会话切换因此属于同一个原子操作。
func (m *Manager) Run(ctx context.Context) {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
@@ -121,135 +221,189 @@ func (m *Manager) Run(ctx context.Context) {
for {
select {
case <-ctx.Done():
m.shutdown()
m.logger.Info("websocket manager stopped")
return
case client := <-m.register:
m.registerClient(client)
case client := <-m.unregister:
m.unregisterClient(client)
case message := <-m.broadcast:
m.broadcastMessage(message)
case <-ticker.C:
m.checkInactiveClients()
}
}
}
// Register 注册客户端(公开方法)。同步初始化处理器,确保随后的
// ReadPump 可以立即入队;注册表维护仍走 Run 循环的注册通道。
func (m *Manager) Register(client *Client) {
client.proc = newProcessor()
go m.processLoop(client)
m.register <- client
}
// Register 原子激活连接并返回该 BootID 会话已确认的累计位点。
// 同 BootID 重连复用处理队列;BootID 变化创建全新 session 并取消旧 session。
func (m *Manager) Register(client *Client, bootID string) uint64 {
var oldClient *Client
var oldSession *agentSession
var startSession *agentSession
// registerClient 注册客户端
func (m *Manager) registerClient(client *Client) {
m.mu.Lock()
defer m.mu.Unlock()
// 如果已存在该探针的连接,先关闭旧连接
if oldClient, exists := m.clients[client.ID]; exists && oldClient != client {
m.logger.Info("agent reconnected, closing old connection", zap.String("agentID", client.ID))
oldClient.closeChannel()
oldClient.Conn.Close()
oldClient.stopProcessor()
oldClient = m.clients[client.ID]
session := m.sessions[client.ID]
if session == nil || (bootID != "" && session.bootID != bootID) {
oldSession = session
session = newAgentSession(client.ID, bootID)
m.sessions[client.ID] = session
startSession = session
}
client.session = session
m.clients[client.ID] = client
m.logger.Info("agent connected", zap.String("agentID", client.ID), zap.Int("totalClients", len(m.clients)))
}
ackSeq := session.ackSeq.Load()
clientCount := len(m.clients)
m.mu.Unlock()
// unregisterClient 注销客户端
func (m *Manager) unregisterClient(client *Client) {
m.mu.Lock()
defer m.mu.Unlock()
// 只注销当前注册的连接:探针快速重连时旧连接的退出路径不能把
// 新连接从注册表中移除,否则新连接的下行通道会被关闭
if cur, exists := m.clients[client.ID]; exists && cur == client {
delete(m.clients, client.ID)
client.closeChannel()
m.logger.Info("agent disconnected", zap.String("agentID", client.ID), zap.Int("totalClients", len(m.clients)))
// map/session 指针已切换后再关闭旧对象。旧 worker 的提交路径会先
// 校验 session 指针,所以无法污染新会话。
if oldSession != nil {
oldSession.cancel()
}
client.stopProcessor()
if oldClient != nil && oldClient != client {
oldClient.close()
}
if startSession != nil {
go m.reliableProcessLoop(startSession)
go m.bestEffortProcessLoop(startSession)
}
m.logger.Info("agent connected",
zap.String("agentID", client.ID),
zap.String("bootID", bootID),
zap.Uint64("ackSeq", ackSeq),
zap.Int("totalClients", clientCount))
return ackSeq
}
// broadcastMessage 广播消息
func (m *Manager) broadcastMessage(message []byte) {
// Unregister 只移除仍为 current 的连接,返回值用于决定是否标记离线。
func (m *Manager) Unregister(client *Client) bool {
m.mu.Lock()
current := m.clients[client.ID] == client
if current {
delete(m.clients, client.ID)
}
clientCount := len(m.clients)
m.mu.Unlock()
client.close()
if current {
m.logger.Info("agent disconnected",
zap.String("agentID", client.ID),
zap.Int("totalClients", clientCount))
}
return current
}
// ForgetAgent 删除已移除 agent 的连接与会话状态,终止常驻 worker。
func (m *Manager) ForgetAgent(agentID string) {
m.mu.Lock()
client := m.clients[agentID]
session := m.sessions[agentID]
delete(m.clients, agentID)
delete(m.sessions, agentID)
m.mu.Unlock()
if client != nil {
client.close()
}
if session != nil {
session.cancel()
}
}
func (m *Manager) shutdown() {
m.mu.Lock()
clients := make([]*Client, 0, len(m.clients))
for _, client := range m.clients {
clients = append(clients, client)
}
sessions := make([]*agentSession, 0, len(m.sessions))
for _, session := range m.sessions {
sessions = append(sessions, session)
}
m.clients = make(map[string]*Client)
m.sessions = make(map[string]*agentSession)
m.mu.Unlock()
for _, client := range clients {
client.close()
}
for _, session := range sessions {
session.cancel()
}
}
func (m *Manager) isCurrentSession(session *agentSession) bool {
m.mu.RLock()
defer m.mu.RUnlock()
for _, client := range m.clients {
select {
case client.Send <- message:
default:
// 发送失败,客户端可能已断开
m.logger.Warn("failed to send message, client may be disconnected", zap.String("agentID", client.ID))
}
}
return m.sessions[session.agentID] == session
}
func (m *Manager) currentClientForSession(session *agentSession) *Client {
m.mu.RLock()
defer m.mu.RUnlock()
if m.sessions[session.agentID] != session {
return nil
}
client := m.clients[session.agentID]
if client == nil || client.session != session {
return nil
}
return client
}
// checkInactiveClients 检查不活跃的客户端
func (m *Manager) checkInactiveClients() {
now := time.Now()
m.mu.RLock()
inactiveClients := make([]*Client, 0)
timeout := 2 * time.Minute
inactive := make([]*Client, 0)
for _, client := range m.clients {
if time.Since(client.LastActive) > timeout {
inactiveClients = append(inactiveClients, client)
if client.inactiveFor(now) > 2*time.Minute {
inactive = append(inactive, client)
}
}
m.mu.RUnlock()
// 断开不活跃的客户端
for _, client := range inactiveClients {
// 再次检查客户端是否仍然存在(避免竞态条件)
for _, client := range inactive {
m.mu.RLock()
cur, exists := m.clients[client.ID]
current := m.clients[client.ID] == client
m.mu.RUnlock()
if exists && cur == client {
if current {
m.logger.Warn("agent inactive timeout, disconnecting", zap.String("agentID", client.ID))
client.Conn.Close()
m.unregister <- client
client.close()
}
}
}
// SendToClient 发送消息给指定客户端。通道满时最多等待 serverSendWait,
// 避免高负载时指令/配置被静默丢弃
func (m *Manager) SendToClient(probeID string, message []byte) error {
func (m *Manager) SendToClient(agentID string, message []byte) error {
m.mu.RLock()
client, exists := m.clients[probeID]
client := m.clients[agentID]
m.mu.RUnlock()
if !exists {
if client == nil {
return ErrClientNotFound
}
select {
case client.Send <- message:
return nil
case <-time.After(serverSendWait):
return ErrSendTimeout
}
return client.sendWithTimeout(message, serverSendWait)
}
// GetClient 获取客户端
func (m *Manager) GetClient(probeID string) (*Client, bool) {
func (m *Manager) GetClient(agentID string) (*Client, bool) {
m.mu.RLock()
defer m.mu.RUnlock()
client, exists := m.clients[probeID]
return client, exists
client, ok := m.clients[agentID]
return client, ok
}
// DoIfCurrent 在连接仍为 current 的前提下执行状态变更,并用注册锁保证
// 回调完成前不会切换连接。用于 pong 在线写库,避免旧连接的迟到写入覆盖
// 新连接失败后的离线状态。
func (m *Manager) DoIfCurrent(client *Client, fn func() error) (bool, error) {
m.mu.RLock()
defer m.mu.RUnlock()
if m.clients[client.ID] != client {
return false, nil
}
return true, fn()
}
// GetAllClients 获取所有客户端ID
func (m *Manager) GetAllClients() []string {
m.mu.RLock()
defer m.mu.RUnlock()
ids := make([]string, 0, len(m.clients))
for id := range m.clients {
ids = append(ids, id)
@@ -257,158 +411,209 @@ func (m *Manager) GetAllClients() []string {
return ids
}
// ClientCount 获取客户端数量
func (m *Manager) ClientCount() int {
m.mu.RLock()
defer m.mu.RUnlock()
return len(m.clients)
}
// AckSeq 返回探针事件流的累计确认位点。注册响应携带该值,探针据此
// 跳过服务端已处理的消息,避免无谓重放。
func (m *Manager) AckSeq(probeID string) uint64 {
func (m *Manager) AckSeq(agentID string) uint64 {
m.mu.RLock()
defer m.mu.RUnlock()
return m.ackSeqs[probeID]
session := m.sessions[agentID]
m.mu.RUnlock()
if session == nil {
return 0
}
return session.ackSeq.Load()
}
// BeginAgentSession 在注册时同步探针会话:BootID 变化说明探针进程
// 重启过(纯内存事件队列清空、seq 从 1 重新分配),必须重置确认位点,
// 否则重启后新分配的小序号会被旧位点误判为重复消息而丢弃。
// 旧版探针不携带 BootID,保持原位点不变。
func (m *Manager) BeginAgentSession(probeID, bootID string) {
if bootID == "" {
// enqueueBestEffort/ enqueueReliable 在持有 current-client 读锁时完成入队。
// Register 必须取得写锁,因此同 BootID 重连时,旧连接已经读出的消息必定
// 先入队,然后才会切换连接;切换完成后旧连接不能再把较大 seq 插到新连接
// 重放的较小 seq 前面,累计 ACK 顺序不会被跨连接竞态破坏。
func (m *Manager) enqueueBestEffort(client *Client, msg agentMessage) bool {
m.mu.RLock()
defer m.mu.RUnlock()
if m.clients[client.ID] != client || m.sessions[client.ID] != client.session {
return false
}
select {
case client.session.bestEffortQueue <- msg:
return true
default:
return false
}
}
func (m *Manager) enqueueReliable(ctx context.Context, client *Client, msg agentMessage) error {
m.mu.RLock()
defer m.mu.RUnlock()
if m.clients[client.ID] != client || m.sessions[client.ID] != client.session {
return ErrClientNotFound
}
timer := time.NewTimer(reliableQueueWait)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-client.done:
return ErrClientNotFound
case <-client.session.ctx.Done():
return ErrClientNotFound
case client.session.reliableQueue <- msg:
return nil
case <-timer.C:
return ErrQueueFull
}
}
// ReadPump 只解析并按可靠性分流。可靠队列提供反压;best-effort 遥测
// 队列满时直接丢弃,避免 VM 慢写阻塞安全事件。
func (c *Client) ReadPump(ctx context.Context) {
defer c.close()
if c.conn == nil {
return
}
m.mu.Lock()
defer m.mu.Unlock()
if m.sessions[probeID] != bootID {
m.sessions[probeID] = bootID
m.ackSeqs[probeID] = 0
}
}
// advanceAckSeq 推进探针的累计确认位点(只前进不回退)
func (m *Manager) advanceAckSeq(probeID string, seq uint64) {
m.mu.Lock()
defer m.mu.Unlock()
if m.ackSeqs[probeID] < seq {
m.ackSeqs[probeID] = seq
}
}
// ReadPump 读取客户端消息。只做解析和入队,处理在 processLoop 中进行,
// 保证慢处理不会阻塞读取造成 TCP 反压。
func (c *Client) ReadPump(ctx context.Context) {
defer func() {
c.Manager.unregister <- c
c.Conn.Close()
c.stopProcessor()
}()
c.Conn.SetReadDeadline(time.Now().Add(serverPongWait))
c.Conn.SetPongHandler(func(string) error {
c.Conn.SetReadDeadline(time.Now().Add(serverPongWait))
c.LastActive = time.Now()
if c.Manager.onPong != nil {
go c.Manager.onPong(c)
_ = c.conn.SetReadDeadline(time.Now().Add(serverPongWait))
c.conn.SetPongHandler(func(string) error {
if err := c.conn.SetReadDeadline(time.Now().Add(serverPongWait)); err != nil {
return err
}
c.touch()
if c.manager.onPong != nil {
// 同步执行,确保连接退出后的离线写库一定发生在本次 pong
// 状态刷新之后,避免遗留 goroutine 把已断开的连接重新标在线。
c.manager.onPong(c)
}
return nil
})
for {
_, message, err := c.Conn.ReadMessage()
_, raw, err := c.conn.ReadMessage()
if err != nil {
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure) {
c.Manager.logger.Error("websocket read error", zap.Error(err), zap.String("agentID", c.ID))
c.manager.logger.Error("websocket read error", zap.Error(err), zap.String("agentID", c.ID))
}
return
}
c.touch()
c.LastActive = time.Now()
// 解析消息
var msg protocol.InputMessage
if err := json.Unmarshal(message, &msg); err != nil {
c.Manager.logger.Error("failed to parse message", zap.Error(err), zap.String("agentID", c.ID))
if err := json.Unmarshal(raw, &msg); err != nil {
c.manager.logger.Error("failed to parse message", zap.Error(err), zap.String("agentID", c.ID))
continue
}
if err := c.enqueue(agentMessage{seq: msg.Seq, typ: string(msg.Type), data: msg.Data}); err != nil {
c.Manager.logger.Warn("agent processing queue stalled, disconnecting for replay",
zap.String("agentID", c.ID), zap.Error(err))
item := agentMessage{seq: msg.Seq, typ: string(msg.Type), data: msg.Data}
if msg.Seq == 0 {
if !c.manager.enqueueBestEffort(c, item) {
c.manager.logger.Warn("best-effort agent queue full, dropping telemetry",
zap.String("agentID", c.ID), zap.String("type", string(msg.Type)))
}
continue
}
if err := c.manager.enqueueReliable(ctx, c, item); err != nil {
c.manager.logger.Warn("reliable agent queue stalled, disconnecting for replay",
zap.String("agentID", c.ID), zap.Uint64("seq", msg.Seq), zap.Error(err))
return
}
}
}
// enqueue 将消息放入该连接的有序处理队列。队列满时最多等待
// agentQueueWait,仍无法入队则返回错误,由调用方断开连接触发探针重放。
func (c *Client) enqueue(msg agentMessage) error {
// 先检查处理器是否已停止:select 各分支同时就绪时是随机选择,
// 不前置检查会导致停止后的消息仍被入队却永远无人消费
select {
case <-c.proc.quit:
return ErrProcessorStopped
default:
}
select {
case c.proc.queue <- msg:
return nil
case <-time.After(agentQueueWait):
return ErrQueueFull
case <-c.proc.quit:
return ErrProcessorStopped
}
}
// stopProcessor 停止该连接的处理器
func (c *Client) stopProcessor() {
if c.proc != nil {
c.proc.stop()
}
}
// processLoop 串行处理单个探针的消息,保证 per-agent 顺序。
// 带 seq 的消息处理成功后推进确认位点并回累计 ack;重放消息按
// 确认位点去重。处理失败按丢弃处理并记录(与旧行为一致),避免
// 毒消息卡死整个事件流。
func (m *Manager) processLoop(c *Client) {
func (m *Manager) bestEffortProcessLoop(session *agentSession) {
for {
select {
case msg := <-c.proc.queue:
if msg.seq > 0 && msg.seq <= m.AckSeq(c.ID) {
// 重放去重:该消息已处理过。仍回 ack(当前确认位点
// 不低于该 seq),让探针能修剪队列
m.sendAck(c, m.AckSeq(c.ID))
case <-session.ctx.Done():
return
case msg := <-session.bestEffortQueue:
if session.ctx.Err() != nil || !m.isCurrentSession(session) {
return
}
if m.onMessage == nil {
continue
}
if err := m.onMessage(session.ctx, session.agentID, msg.typ, msg.data); err != nil {
m.logger.Error("failed to handle best-effort message, dropped",
zap.Error(err), zap.String("agentID", session.agentID), zap.String("type", msg.typ))
}
}
}
}
func (m *Manager) reliableProcessLoop(session *agentSession) {
for {
select {
case <-session.ctx.Done():
return
case msg := <-session.reliableQueue:
if session.ctx.Err() != nil || !m.isCurrentSession(session) {
return
}
if msg.seq <= session.ackSeq.Load() {
m.sendAck(session, session.ackSeq.Load())
continue
}
if m.onMessage != nil {
if err := m.onMessage(context.Background(), c.ID, msg.typ, msg.data); err != nil {
m.logger.Error("failed to handle message, dropped",
zap.Error(err), zap.String("agentID", c.ID),
zap.String("type", msg.typ), zap.Uint64("seq", msg.seq))
}
if !m.processReliableMessage(session, msg) {
return
}
if msg.seq > 0 {
m.advanceAckSeq(c.ID, msg.seq)
m.sendAck(c, msg.seq)
if !m.isCurrentSession(session) {
return
}
case <-c.proc.quit:
return
session.ackSeq.Store(msg.seq)
m.sendAck(session, msg.seq)
}
}
}
// sendAck 向探针发送累计确认。走下行 Send 通道,满时丢弃:丢失的
// ack 只会让探针多保留一会儿消息,重连重放时按 seq 去重,无副作用。
func (m *Manager) sendAck(c *Client, seq uint64) {
// processReliableMessage 对瞬时错误原位退避重试,保证后续序号不能越过
// 失败消息;永久格式错误明确丢弃并允许推进位点。
func (m *Manager) processReliableMessage(session *agentSession, msg agentMessage) bool {
retryDelay := messageRetryMin
for {
if session.ctx.Err() != nil || !m.isCurrentSession(session) {
return false
}
if m.onMessage == nil {
return true
}
err := m.onMessage(session.ctx, session.agentID, msg.typ, msg.data)
if err == nil {
return true
}
if isPermanent(err) {
m.logger.Error("permanent reliable message error, dropped",
zap.Error(err), zap.String("agentID", session.agentID),
zap.String("type", msg.typ), zap.Uint64("seq", msg.seq))
return true
}
m.logger.Warn("reliable message handling failed, retrying",
zap.Error(err), zap.String("agentID", session.agentID),
zap.String("type", msg.typ), zap.Uint64("seq", msg.seq),
zap.Duration("retryAfter", retryDelay))
timer := time.NewTimer(retryDelay)
select {
case <-session.ctx.Done():
timer.Stop()
return false
case <-timer.C:
}
if retryDelay < messageRetryMax {
retryDelay *= 2
if retryDelay > messageRetryMax {
retryDelay = messageRetryMax
}
}
}
}
func (m *Manager) sendAck(session *agentSession, seq uint64) {
payload, err := json.Marshal(protocol.OutboundMessage{
Type: protocol.MessageTypeAck,
Data: protocol.AckData{Seq: seq},
@@ -416,72 +621,51 @@ func (m *Manager) sendAck(c *Client, seq uint64) {
if err != nil {
return
}
select {
case c.Send <- payload:
default:
if client := m.currentClientForSession(session); client != nil {
client.trySend(payload)
}
}
// WritePump 向客户端写入消息
func (c *Client) WritePump() {
if c.conn == nil {
return
}
ticker := time.NewTicker(serverPingInterval)
defer func() {
ticker.Stop()
c.Conn.Close()
}()
defer ticker.Stop()
for {
select {
case message, ok := <-c.Send:
c.Conn.SetWriteDeadline(time.Now().Add(serverWriteWait))
if !ok {
// 通道已关闭
c.Conn.WriteMessage(websocket.CloseMessage, []byte{})
case <-c.done:
return
case message := <-c.send:
_ = c.conn.SetWriteDeadline(time.Now().Add(serverWriteWait))
if err := c.conn.WriteMessage(websocket.TextMessage, message); err != nil {
c.manager.logger.Error("failed to write message", zap.Error(err), zap.String("agentID", c.ID))
c.close()
return
}
if err := c.Conn.WriteMessage(websocket.TextMessage, message); err != nil {
c.Manager.logger.Error("failed to write message", zap.Error(err), zap.String("agentID", c.ID))
return
}
case <-ticker.C:
c.Conn.SetWriteDeadline(time.Now().Add(serverWriteWait))
if err := c.Conn.WriteMessage(websocket.PingMessage, nil); err != nil {
_ = c.conn.SetWriteDeadline(time.Now().Add(serverWriteWait))
if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil {
c.close()
return
}
}
}
}
// ShouldWriteStatus 报告是否需要把在线状态写入数据库。去抖:本连接
// 首次调用立即通过,之后每 refreshInterval 通过一次。
func (c *Client) ShouldWriteStatus(refreshInterval time.Duration) bool {
now := time.Now().UnixMilli()
last := c.lastStatusWrite.Load()
return last == 0 || now-last >= refreshInterval.Milliseconds()
}
// MarkStatusWritten 记录本次在线状态已成功写库
func (c *Client) MarkStatusWritten() {
c.lastStatusWrite.Store(time.Now().UnixMilli())
}
// closeChannel 安全地关闭发送通道
func (c *Client) closeChannel() {
c.closeMu.Lock()
defer c.closeMu.Unlock()
if !c.closed {
close(c.Send)
c.closed = true
}
}
// 错误定义
var (
ErrClientNotFound = &websocket.CloseError{Code: 1000, Text: "client not found"}
ErrSendTimeout = &websocket.CloseError{Code: 1001, Text: "send timeout"}
ErrQueueFull = &websocket.CloseError{Code: 1013, Text: "processing queue full"}
ErrProcessorStopped = &websocket.CloseError{Code: 1000, Text: "processor stopped"}
ErrClientNotFound = &websocket.CloseError{Code: 1000, Text: "client not found"}
ErrSendTimeout = &websocket.CloseError{Code: 1001, Text: "send timeout"}
ErrQueueFull = &websocket.CloseError{Code: 1013, Text: "reliable queue full"}
)
+208 -90
View File
@@ -3,6 +3,9 @@ package websocket
import (
"context"
"encoding/json"
"errors"
"sync"
"sync/atomic"
"testing"
"time"
@@ -10,20 +13,16 @@ import (
"go.uber.org/zap"
)
func newTestClient(m *Manager, id string) *Client {
c := &Client{
ID: id,
Send: make(chan []byte, 16),
Manager: m,
}
c.proc = newProcessor()
func newTestClient(m *Manager, id, bootID string) *Client {
c := NewClient(id, nil, m)
m.Register(c, bootID)
return c
}
func readAck(t *testing.T, c *Client) uint64 {
t.Helper()
select {
case raw := <-c.Send:
case raw := <-c.send:
var msg protocol.InputMessage
if err := json.Unmarshal(raw, &msg); err != nil {
t.Fatalf("unmarshal ack: %v", err)
@@ -42,29 +41,28 @@ func readAck(t *testing.T, c *Client) uint64 {
}
}
func TestProcessLoopDedupAckAndOrder(t *testing.T) {
func TestReliableSessionDedupAckAndOrder(t *testing.T) {
m := NewManager(zap.NewNop())
defer m.shutdown()
var mu sync.Mutex
var processed []string
done := make(chan struct{})
m.SetMessageHandler(func(ctx context.Context, probeID, typ string, data json.RawMessage) error {
m.SetMessageHandler(func(_ context.Context, _, typ string, _ json.RawMessage) error {
mu.Lock()
processed = append(processed, typ)
if len(processed) == 2 {
close(done)
}
mu.Unlock()
return nil
})
client := newTestClient(m, "agent-1")
go m.processLoop(client)
defer client.stopProcessor()
// 模拟断线前的确认位点:seq 1 已处理
m.advanceAckSeq("agent-1", 1)
// seq 1 是重放(应去重跳过),seq 2、3 是新消息(按序处理)
client.proc.queue <- agentMessage{seq: 1, typ: "replayed"}
client.proc.queue <- agentMessage{seq: 2, typ: "first"}
client.proc.queue <- agentMessage{seq: 3, typ: "second"}
client := newTestClient(m, "agent-1", "boot-1")
client.session.ackSeq.Store(1)
client.session.reliableQueue <- agentMessage{seq: 1, typ: "replayed"}
client.session.reliableQueue <- agentMessage{seq: 2, typ: "first"}
client.session.reliableQueue <- agentMessage{seq: 3, typ: "second"}
select {
case <-done:
@@ -72,110 +70,230 @@ func TestProcessLoopDedupAckAndOrder(t *testing.T) {
t.Fatal("timed out waiting for processing")
}
if len(processed) != 2 || processed[0] != "first" || processed[1] != "second" {
t.Fatalf("processed = %v, want [first second]", processed)
mu.Lock()
got := append([]string(nil), processed...)
mu.Unlock()
if len(got) != 2 || got[0] != "first" || got[1] != "second" {
t.Fatalf("processed = %v, want [first second]", got)
}
if got := m.AckSeq("agent-1"); got != 3 {
t.Fatalf("AckSeq = %d, want 3", got)
if ack := m.AckSeq("agent-1"); ack != 3 {
t.Fatalf("AckSeq = %d, want 3", ack)
}
// 去重跳过的消息也回 ack(位点 1),随后是新消息的 ack(2、3)
if seq := readAck(t, client); seq != 1 {
t.Fatalf("ack for replayed message = %d, want 1", seq)
}
if seq := readAck(t, client); seq != 2 {
t.Fatalf("ack for first message = %d, want 2", seq)
}
if seq := readAck(t, client); seq != 3 {
t.Fatalf("ack for second message = %d, want 3", seq)
for i, want := range []uint64{1, 2, 3} {
if ack := readAck(t, client); ack != want {
t.Fatalf("ack #%d = %d, want %d", i, ack, want)
}
}
}
func TestProcessLoopSeqZeroBypassesAck(t *testing.T) {
func TestReliableFailureDoesNotAdvanceOrSkip(t *testing.T) {
m := NewManager(zap.NewNop())
handled := make(chan string, 1)
m.SetMessageHandler(func(ctx context.Context, probeID, typ string, data json.RawMessage) error {
handled <- typ
defer m.shutdown()
var allowFirst atomic.Bool
processed := make(chan string, 4)
m.SetMessageHandler(func(_ context.Context, _, typ string, _ json.RawMessage) error {
if typ == "first" && !allowFirst.Load() {
return errors.New("temporary database failure")
}
processed <- typ
return nil
})
client := newTestClient(m, "agent-2")
go m.processLoop(client)
defer client.stopProcessor()
client := newTestClient(m, "agent-2", "boot-1")
client.session.reliableQueue <- agentMessage{seq: 1, typ: "first"}
client.session.reliableQueue <- agentMessage{seq: 2, typ: "second"}
// seq=0 表示不参与可靠投递(如常规指标消息):正常处理但不 ack
client.proc.queue <- agentMessage{seq: 0, typ: "metrics"}
select {
case typ := <-handled:
if typ != "metrics" {
t.Fatalf("handled = %s, want metrics", typ)
}
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for processing")
time.Sleep(100 * time.Millisecond)
if ack := m.AckSeq("agent-2"); ack != 0 {
t.Fatalf("AckSeq during failure = %d, want 0", ack)
}
select {
case raw := <-client.Send:
t.Fatalf("seq=0 message should not be acked, got %s", raw)
case typ := <-processed:
t.Fatalf("processed %s before seq 1 recovered", typ)
default:
}
}
func TestAdvanceAckSeqMonotonic(t *testing.T) {
m := NewManager(zap.NewNop())
m.advanceAckSeq("a", 5)
m.advanceAckSeq("a", 3) // 回退无效
if got := m.AckSeq("a"); got != 5 {
t.Fatalf("AckSeq = %d, want 5", got)
allowFirst.Store(true)
for _, want := range []string{"first", "second"} {
select {
case got := <-processed:
if got != want {
t.Fatalf("processed %q, want %q", got, want)
}
case <-time.After(2 * time.Second):
t.Fatalf("timed out waiting for %s", want)
}
}
if got := m.AckSeq("missing"); got != 0 {
t.Fatalf("AckSeq(missing) = %d, want 0", got)
if ack := m.AckSeq("agent-2"); ack != 2 {
t.Fatalf("AckSeq after recovery = %d, want 2", ack)
}
}
func TestBeginAgentSessionResetsAckSeqOnReboot(t *testing.T) {
func TestBootIDReplacementRejectsOldSessionCommit(t *testing.T) {
m := NewManager(zap.NewNop())
defer m.shutdown()
// 探针运行期间位点推进
m.BeginAgentSession("a", "boot-1")
m.advanceAckSeq("a", 5000)
oldStarted := make(chan struct{})
releaseOld := make(chan struct{})
newProcessed := make(chan struct{})
m.SetMessageHandler(func(_ context.Context, _, typ string, _ json.RawMessage) error {
switch typ {
case "old":
close(oldStarted)
<-releaseOld // 模拟不遵守 context 的旧业务处理器
case "new":
close(newProcessed)
}
return nil
})
// 同一进程重连:位点保留,重放去重继续有效
m.BeginAgentSession("a", "boot-1")
if got := m.AckSeq("a"); got != 5000 {
t.Fatalf("AckSeq after reconnect = %d, want 5000", got)
oldClient := newTestClient(m, "agent-3", "boot-old")
oldSession := oldClient.session
oldSession.reliableQueue <- agentMessage{seq: 100, typ: "old"}
<-oldStarted
newClient := newTestClient(m, "agent-3", "boot-new")
if oldClient.session == newClient.session {
t.Fatal("BootID change should replace the session")
}
if ack := m.AckSeq("agent-3"); ack != 0 {
t.Fatalf("new session AckSeq = %d, want 0", ack)
}
// 探针重启(新 BootID,内存队列清空、seq 从 1 重新分配):
// 位点必须重置,否则重启后的小序号会被误判为重复消息丢弃
m.BeginAgentSession("a", "boot-2")
if got := m.AckSeq("a"); got != 0 {
t.Fatalf("AckSeq after reboot = %d, want 0", got)
close(releaseOld)
newClient.session.reliableQueue <- agentMessage{seq: 1, typ: "new"}
select {
case <-newProcessed:
case <-time.After(2 * time.Second):
t.Fatal("new session message was not processed")
}
// 旧版探针不携带 BootID:不影响现有位点
m.advanceAckSeq("a", 7)
m.BeginAgentSession("a", "")
if got := m.AckSeq("a"); got != 7 {
t.Fatalf("AckSeq after legacy register = %d, want 7", got)
if ack := readAck(t, newClient); ack != 1 {
t.Fatalf("new session ack = %d, want 1", ack)
}
time.Sleep(50 * time.Millisecond)
if ack := m.AckSeq("agent-3"); ack != 1 {
t.Fatalf("old session polluted AckSeq: got %d, want 1", ack)
}
}
func TestEnqueueReturnsAfterProcessorStop(t *testing.T) {
func TestSameBootReconnectReusesSession(t *testing.T) {
m := NewManager(zap.NewNop())
client := newTestClient(m, "agent-3")
client.stopProcessor()
defer m.shutdown()
if err := client.enqueue(agentMessage{seq: 1, typ: "x"}); err != ErrProcessorStopped {
t.Fatalf("enqueue after stop = %v, want ErrProcessorStopped", err)
first := newTestClient(m, "agent-4", "boot-1")
first.session.ackSeq.Store(9)
second := newTestClient(m, "agent-4", "boot-1")
if first.session != second.session {
t.Fatal("same BootID reconnect should reuse session")
}
if ack := m.AckSeq("agent-4"); ack != 9 {
t.Fatalf("AckSeq after reconnect = %d, want 9", ack)
}
select {
case <-first.done:
default:
t.Fatal("old transport should be closed after replacement")
}
if m.Unregister(first) {
t.Fatal("old transport must not unregister the replacement")
}
if current, ok := m.GetClient("agent-4"); !ok || current != second {
t.Fatal("replacement transport was removed by old unregister")
}
}
func TestReplacedTransportCannotEnqueueOutOfOrderSequence(t *testing.T) {
m := NewManager(zap.NewNop())
defer m.shutdown()
processed := make(chan string, 2)
m.SetMessageHandler(func(_ context.Context, _, typ string, _ json.RawMessage) error {
processed <- typ
return nil
})
oldClient := newTestClient(m, "agent-order", "boot-1")
newClient := newTestClient(m, "agent-order", "boot-1")
if err := m.enqueueReliable(context.Background(), oldClient, agentMessage{seq: 10, typ: "stale"}); !errors.Is(err, ErrClientNotFound) {
t.Fatalf("old transport enqueue = %v, want ErrClientNotFound", err)
}
if err := m.enqueueReliable(context.Background(), newClient, agentMessage{seq: 1, typ: "current"}); err != nil {
t.Fatalf("new transport enqueue: %v", err)
}
select {
case typ := <-processed:
if typ != "current" {
t.Fatalf("processed %q, want current", typ)
}
case <-time.After(time.Second):
t.Fatal("current transport message was not processed")
}
select {
case typ := <-processed:
t.Fatalf("processed stale transport message %q", typ)
case <-time.After(50 * time.Millisecond):
}
}
func TestBestEffortDoesNotBlockReliableEvents(t *testing.T) {
m := NewManager(zap.NewNop())
defer m.shutdown()
blockTelemetry := make(chan struct{})
reliableDone := make(chan struct{})
m.SetMessageHandler(func(_ context.Context, _, typ string, _ json.RawMessage) error {
if typ == "metrics" {
<-blockTelemetry
} else if typ == "event" {
close(reliableDone)
}
return nil
})
client := newTestClient(m, "agent-5", "boot-1")
client.session.bestEffortQueue <- agentMessage{typ: "metrics"}
client.session.reliableQueue <- agentMessage{seq: 1, typ: "event"}
select {
case <-reliableDone:
case <-time.After(500 * time.Millisecond):
t.Fatal("reliable event was blocked by best-effort telemetry")
}
close(blockTelemetry)
}
func TestPermanentMessageAdvancesAck(t *testing.T) {
m := NewManager(zap.NewNop())
defer m.shutdown()
m.SetMessageHandler(func(context.Context, string, string, json.RawMessage) error {
return Permanent(errors.New("invalid payload"))
})
client := newTestClient(m, "agent-6", "boot-1")
client.session.reliableQueue <- agentMessage{seq: 1, typ: "bad"}
if ack := readAck(t, client); ack != 1 {
t.Fatalf("permanent message ack = %d, want 1", ack)
}
}
func TestClosedClientRejectsSendWithoutPanic(t *testing.T) {
m := NewManager(zap.NewNop())
client := newTestClient(m, "agent-7", "boot-1")
client.close()
if client.trySend([]byte("ack")) {
t.Fatal("closed client accepted non-blocking send")
}
if err := client.sendWithTimeout([]byte("command"), time.Second); !errors.Is(err, ErrClientNotFound) {
t.Fatalf("send to closed client = %v, want ErrClientNotFound", err)
}
m.shutdown()
}
func TestClientShouldWriteStatusDebounce(t *testing.T) {
m := NewManager(zap.NewNop())
client := newTestClient(m, "agent-4")
client := NewClient("agent-8", nil, m)
if !client.ShouldWriteStatus(time.Minute) {
t.Fatal("first check should pass")
}
+40 -56
View File
@@ -11,7 +11,6 @@ import (
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pika-monitor/pika/internal/protocol"
@@ -95,7 +94,6 @@ type Agent struct {
collectorManager *collector.Manager
metricsStore *metricsStore
outbox *outbox
reliable atomic.Bool
tamperProtector *tamper.Protector
sshMonitor *sshmonitor.Monitor
}
@@ -123,6 +121,10 @@ func (a *Agent) Start(ctx context.Context) error {
// 采集与发送解耦:采集永远运行(断线也写快照),发送独立读快照上报
go a.collectLoop(ctx)
go a.sendLoop(ctx)
// 安全事件消费属于 Agent 生命周期,而不是单次 WebSocket 连接。
// 断线和退避期间仍持续写入 outbox,避免上游小缓冲区溢出丢事件。
go a.tamperEventLoop(ctx)
go a.sshLoginEventLoop(ctx)
// 启动探针主循环
b := &backoff.Backoff{
@@ -230,9 +232,8 @@ func (a *Agent) runOnce(ctx context.Context, onRegistered func()) error {
// 重置发送游标,使下一个 sendLoop tick 立即发送当前全量快照,
// 保证重连瞬间服务端就能拿到最新数据
a.metricsStore.reset()
// 服务端声明支持可靠投递时,事件类消息改走 outbox;注册响应携带的
// 累计确认位点先修剪队列,避免重放服务端已处理的消息
a.reliable.Store(registerResp.Reliable)
// 可靠服务端返回累计确认位点;旧服务端没有 ACK,发送器会在每次
// WebSocket 写成功后本地确认,从而完整支持升级后的降级/回滚。
if registerResp.Reliable && registerResp.AckSeq > 0 {
a.outbox.ack(registerResp.AckSeq)
}
@@ -267,21 +268,10 @@ func (a *Agent) runOnce(ctx context.Context, onRegistered func()) error {
}
})
// 事件类消息可靠发送(仅服务端支持时启用)
if registerResp.Reliable {
wg.Go(func() {
a.eventSendLoop(done, conn)
})
}
// 启动防篡改事件监控
// 每个连接只有一个 outbox 发送器。可靠服务端等待累计 ACK;旧服务端
// 按写成功确认,队列不会因版本回退而被永久搁置。
wg.Go(func() {
a.tamperEventLoop(ctx, done)
})
// 启动 SSH 登录事件监控
wg.Go(func() {
a.sshLoginEventLoop(ctx, done)
a.eventSendLoop(done, conn, registerResp.Reliable)
})
// 等待第一个错误或上下文取消
@@ -461,23 +451,36 @@ func (a *Agent) handleAck(data json.RawMessage) {
a.outbox.ack(ack.Seq)
}
// eventSendLoop 事件类消息的可靠发送循环(每个连接一个)。
// 连接建立后先从队头重放全部未确认消息(服务端按 seq 去重),之后增量
// 发送新入队的消息;写失败直接返回并断开连接,未确认消息留给下一个
// 连接重放。
func (a *Agent) eventSendLoop(done chan struct{}, conn *safeConn) {
// eventSendLoop 是单连接 outbox 发送器。可靠模式下由服务端 ACK 修剪;
// 兼容旧服务端时以 WebSocket 写成功作为本地确认,至少保证断线期间已经
// 入队的消息会在降级连接上继续发送。
func (a *Agent) eventSendLoop(done chan struct{}, conn *safeConn, reliable bool) {
if err := sendOutboxLoop(a.outbox, done, conn, reliable); err != nil {
slog.Warn("事件消息发送失败,等待重连后重放", "error", err)
a.killConn(conn)
}
}
type jsonWriter interface {
WriteJSON(v interface{}) error
}
// sendOutboxLoop 是与连接生命周期无关的发送算法,便于独立验证可靠和
// 旧版降级语义。写失败不修改待确认消息,由调用方关闭连接并重连。
func sendOutboxLoop(outbox *outbox, done <-chan struct{}, writer jsonWriter, reliable bool) error {
var lastSent uint64
for {
for _, msg := range a.outbox.after(lastSent) {
if err := conn.WriteJSON(msg); err != nil {
slog.Warn("事件消息发送失败,等待重连后重放", "error", err, "seq", msg.Seq)
a.killConn(conn)
return
for _, msg := range outbox.after(lastSent) {
if err := writer.WriteJSON(msg); err != nil {
return fmt.Errorf("send outbox seq %d: %w", msg.Seq, err)
}
lastSent = msg.Seq
if !reliable {
outbox.ack(msg.Seq)
}
}
if !a.outbox.wait(done) {
return
if !outbox.wait(done) {
return nil
}
}
}
@@ -512,7 +515,7 @@ func (a *Agent) handleMonitorConfig(data json.RawMessage) {
slog.Info("收到服务监控配置,立即执行检测", "count", len(payload.Items))
// 立即执行一次监控检测,结果以单元素 batch 上报。
// 走 sendOutboundMessage(可靠模式下进 outbox),断线期间不丢
// 走常驻 outbox,断线和版本降级期间也不会绕过队列
sample := manager.CollectMonitor(payload.Items)
if err := a.sendOutboundMessage(protocol.OutboundMessage{
Type: protocol.MessageTypeMetrics,
@@ -536,25 +539,10 @@ func (a *Agent) getActiveConn() *safeConn {
return a.activeConn
}
// sendOutboundMessage 发送事件类出站消息。服务端支持可靠投递时入队
// outbox,由专用协程发送并等待服务端确认,断线期间不丢;否则降级为
// 直接发送(与旧服务端兼容)。
// sendOutboundMessage 把事件类消息无条件写入常驻 outbox。网络版本协商
// 只影响连接级发送器如何确认,不影响事件是否在断线期间被接收。
func (a *Agent) sendOutboundMessage(msg protocol.OutboundMessage) error {
if a.reliable.Load() {
a.outbox.enqueue(msg)
return nil
}
conn := a.getActiveConn()
if conn == nil {
return fmt.Errorf("connection is unavailable")
}
if err := conn.WriteJSON(msg); err != nil {
a.killConn(conn)
return err
}
a.outbox.enqueue(msg)
return nil
}
@@ -798,14 +786,12 @@ func (a *Agent) sendTamperProtectResponse(success bool, message string, paths []
}
// tamperEventLoop 防篡改事件监控循环(包含事件和告警)
func (a *Agent) tamperEventLoop(ctx context.Context, done chan struct{}) {
func (a *Agent) tamperEventLoop(ctx context.Context) {
eventCh := a.tamperProtector.GetEvents()
alertCh := a.tamperProtector.GetAlerts()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case event := <-eventCh:
@@ -1090,13 +1076,11 @@ func (a *Agent) sendSSHLoginConfigResult(success bool, enabled bool, message str
}
// sshLoginEventLoop SSH登录事件监控循环
func (a *Agent) sshLoginEventLoop(ctx context.Context, done chan struct{}) {
func (a *Agent) sshLoginEventLoop(ctx context.Context) {
eventCh := a.sshMonitor.GetEvents()
for {
select {
case <-done:
return
case <-ctx.Done():
return
case event := <-eventCh:
+5
View File
@@ -92,6 +92,11 @@ func (o *outbox) ack(seq uint64) int {
o.mu.Lock()
defer o.mu.Unlock()
// 不信任超过本进程已分配范围的 ACK。异常或过期服务端位点不能
// 把未来序号也标成已确认,否则后续消息会长期无法正常修剪。
if seq > o.nextSeq {
seq = o.nextSeq
}
if seq <= o.ackedSeq {
return 0
}
+98
View File
@@ -1,11 +1,36 @@
package service
import (
"errors"
"sync"
"testing"
"time"
"github.com/pika-monitor/pika/internal/protocol"
)
type recordingJSONWriter struct {
mu sync.Mutex
messages []protocol.OutboundMessage
err error
written chan struct{}
}
func (w *recordingJSONWriter) WriteJSON(value interface{}) error {
if w.err != nil {
return w.err
}
msg := value.(protocol.OutboundMessage)
w.mu.Lock()
w.messages = append(w.messages, msg)
w.mu.Unlock()
select {
case w.written <- struct{}{}:
default:
}
return nil
}
func TestOutboxSeqAssignment(t *testing.T) {
o := newOutbox()
for i := 1; i <= 3; i++ {
@@ -104,3 +129,76 @@ func TestOutboxEnqueueSetsMsgSeq(t *testing.T) {
t.Fatalf("stored = %+v, want one message with seq 1", stored)
}
}
func TestOutboxClampsAckToAllocatedSequence(t *testing.T) {
o := newOutbox()
o.enqueue(protocol.OutboundMessage{Type: protocol.MessageTypeTamperEvent})
o.enqueue(protocol.OutboundMessage{Type: protocol.MessageTypeSSHLoginEvent})
if trimmed := o.ack(999); trimmed != 2 {
t.Fatalf("ack beyond allocated range trimmed %d, want 2", trimmed)
}
if _, acked, _ := o.stats(); acked != 2 {
t.Fatalf("acked seq = %d, want clamp to 2", acked)
}
seq := o.enqueue(protocol.OutboundMessage{Type: protocol.MessageTypeCommandResp})
if seq != 3 {
t.Fatalf("next seq = %d, want 3", seq)
}
if pending, _, _ := o.stats(); pending != 1 {
t.Fatalf("future message was incorrectly pre-acked; pending = %d, want 1", pending)
}
}
func TestSendOutboundMessageQueuesWhileDisconnected(t *testing.T) {
a := &Agent{outbox: newOutbox()}
if err := a.sendOutboundMessage(protocol.OutboundMessage{Type: protocol.MessageTypeSSHLoginEvent}); err != nil {
t.Fatalf("queue disconnected event: %v", err)
}
if pending, _, _ := a.outbox.stats(); pending != 1 {
t.Fatalf("pending = %d, want 1", pending)
}
}
func TestLegacyOutboxSenderDrainsPendingMessages(t *testing.T) {
o := newOutbox()
o.enqueue(protocol.OutboundMessage{Type: protocol.MessageTypeTamperEvent})
o.enqueue(protocol.OutboundMessage{Type: protocol.MessageTypeSSHLoginEvent})
done := make(chan struct{})
writer := &recordingJSONWriter{written: make(chan struct{}, 2)}
result := make(chan error, 1)
go func() {
result <- sendOutboxLoop(o, done, writer, false)
}()
for i := 0; i < 2; i++ {
select {
case <-writer.written:
case <-time.After(time.Second):
t.Fatal("legacy sender did not flush pending messages")
}
}
close(done)
if err := <-result; err != nil {
t.Fatalf("legacy sender: %v", err)
}
if pending, acked, _ := o.stats(); pending != 0 || acked != 2 {
t.Fatalf("after legacy flush: pending=%d acked=%d, want 0/2", pending, acked)
}
}
func TestOutboxWriteFailureKeepsMessagePending(t *testing.T) {
o := newOutbox()
o.enqueue(protocol.OutboundMessage{Type: protocol.MessageTypeCommandResp})
writer := &recordingJSONWriter{err: errors.New("network down"), written: make(chan struct{}, 1)}
err := sendOutboxLoop(o, make(chan struct{}), writer, false)
if err == nil {
t.Fatal("expected write failure")
}
if pending, acked, _ := o.stats(); pending != 1 || acked != 0 {
t.Fatalf("after write failure: pending=%d acked=%d, want 1/0", pending, acked)
}
}