fix: 避免在线探针离线告警抖动

This commit is contained in:
dushixiang committed 2026-08-31 22:44:38 +08:00
1 parent 0fa920e972
commit 78006a7120
2 files changed
+70 -19

No files matched your search

+27 -19
View File
@@ -1077,16 +1077,11 @@ func (s *AlertService) checkAgentOfflineAlerts(ctx context.Context, now int64) e
stateKey := fmt.Sprintf("%s:%s:agent_offline:%s", agent.ID, config.ConfigID, agent.ID)
// 防止时钟回拨导致负数
rawOfflineSeconds := int64(0)
if now > agent.LastSeenAt {
rawOfflineSeconds = (now - agent.LastSeenAt) / 1000
}
offlineStartTime := config.ConditionStartAfterMaintenance(time.UnixMilli(now), agent.LastSeenAt)
offlineSeconds := int64(0)
if now > offlineStartTime {
offlineSeconds = (now - offlineStartTime) / 1000
}
// status 由 WebSocket 连接的注册/注销同步维护,是当前是否离线的
// 权威状态;last_seen_at 仅用于在已经离线时计算持续时间。在线连接
// 的 last_seen_at 会按间隔去抖写库,不能单独据此判定离线,否则
// 写库间隔与告警阈值相同时会在边界上反复误报和恢复。
isOffline, offlineStartTime, offlineSeconds := agentOfflineCondition(config, &agent, now)
// 从数据库加载状态
state, err := s.AlertStateRepo.GetAlertState(ctx, stateKey)
@@ -1111,15 +1106,12 @@ func (s *AlertService) checkAgentOfflineAlerts(ctx context.Context, now int64) e
var shouldFire, shouldResolve bool
if state.IsFiring {
// 已有告警只在探针实际恢复后解除,不能因为维护结束后重新累计而伪造恢复。
if rawOfflineSeconds < int64(config.Rules.AgentOfflineDuration) {
shouldResolve = true
}
} else if offlineSeconds >= int64(config.Rules.AgentOfflineDuration) {
if !state.IsFiring {
shouldFire = true
state.IsFiring = true
}
// 已有告警只在 WebSocket 连接实际恢复后解除,不能因为维护结束后
// 重新累计或 last_seen_at 刷新而伪造恢复。
shouldResolve = !isOffline
} else if isOffline && offlineSeconds >= int64(config.Rules.AgentOfflineDuration) {
shouldFire = true
state.IsFiring = true
}
// 保存状态到数据库
@@ -1140,6 +1132,22 @@ func (s *AlertService) checkAgentOfflineAlerts(ctx context.Context, now int64) e
return nil
}
// agentOfflineCondition 返回探针当前是否离线,以及扣除维护窗口后的离线
// 起点和持续秒数。在线状态下时间统一归零,避免去抖写入的 last_seen_at
// 被误当作连接活性判断。
func agentOfflineCondition(config *EffectiveAlertConfig, agent *models.Agent, now int64) (bool, int64, int64) {
if agent.Status == 1 {
return false, 0, 0
}
offlineStartTime := config.ConditionStartAfterMaintenance(time.UnixMilli(now), agent.LastSeenAt)
offlineSeconds := int64(0)
if now > offlineStartTime {
offlineSeconds = (now - offlineStartTime) / 1000
}
return true, offlineStartTime, offlineSeconds
}
// fireAgentOfflineAlert 触发探针离线告警
func (s *AlertService) fireAgentOfflineAlert(ctx context.Context, config *EffectiveAlertConfig, agent *models.Agent, state *models.AlertState, offlineSeconds int64, now int64) {
s.logger.Info("触发探针离线告警",
@@ -0,0 +1,43 @@
package service
import (
"testing"
"time"
"github.com/pika-monitor/pika/internal/models"
)
func TestAgentOfflineConditionUsesConnectionStatus(t *testing.T) {
now := time.Date(2026, time.August, 31, 22, 31, 1, 0, time.Local).UnixMilli()
staleLastSeen := now - int64(306*time.Second/time.Millisecond)
config := &EffectiveAlertConfig{}
t.Run("在线探针不因去抖后的旧心跳时间误判离线", func(t *testing.T) {
agent := &models.Agent{Status: 1, LastSeenAt: staleLastSeen}
isOffline, startTime, seconds := agentOfflineCondition(config, agent, now)
if isOffline || startTime != 0 || seconds != 0 {
t.Fatalf(
"agentOfflineCondition() = (%v, %d, %d), want (false, 0, 0)",
isOffline,
startTime,
seconds,
)
}
})
t.Run("离线探针从最后状态更新时间累计", func(t *testing.T) {
agent := &models.Agent{Status: 0, LastSeenAt: staleLastSeen}
isOffline, startTime, seconds := agentOfflineCondition(config, agent, now)
if !isOffline || startTime != staleLastSeen || seconds != 306 {
t.Fatalf(
"agentOfflineCondition() = (%v, %d, %d), want (true, %d, 306)",
isOffline,
startTime,
seconds,
staleLastSeen,
)
}
})
}