fix: 避免在线探针离线告警抖动
This commit is contained in:
1 parent
0fa920e972
commit
78006a7120
2 files changed
+70
-19
No files matched your search
@@ -1077,16 +1077,11 @@ func (s *AlertService) checkAgentOfflineAlerts(ctx context.Context, now int64) e
|
||||
|
||||
stateKey := fmt.Sprintf("%s:%s:agent_offline:%s", agent.ID, config.ConfigID, agent.ID)
|
||||
|
||||
// 防止时钟回拨导致负数
|
||||
rawOfflineSeconds := int64(0)
|
||||
if now > agent.LastSeenAt {
|
||||
rawOfflineSeconds = (now - agent.LastSeenAt) / 1000
|
||||
}
|
||||
offlineStartTime := config.ConditionStartAfterMaintenance(time.UnixMilli(now), agent.LastSeenAt)
|
||||
offlineSeconds := int64(0)
|
||||
if now > offlineStartTime {
|
||||
offlineSeconds = (now - offlineStartTime) / 1000
|
||||
}
|
||||
// status 由 WebSocket 连接的注册/注销同步维护,是当前是否离线的
|
||||
// 权威状态;last_seen_at 仅用于在已经离线时计算持续时间。在线连接
|
||||
// 的 last_seen_at 会按间隔去抖写库,不能单独据此判定离线,否则
|
||||
// 写库间隔与告警阈值相同时会在边界上反复误报和恢复。
|
||||
isOffline, offlineStartTime, offlineSeconds := agentOfflineCondition(config, &agent, now)
|
||||
|
||||
// 从数据库加载状态
|
||||
state, err := s.AlertStateRepo.GetAlertState(ctx, stateKey)
|
||||
@@ -1111,15 +1106,12 @@ func (s *AlertService) checkAgentOfflineAlerts(ctx context.Context, now int64) e
|
||||
var shouldFire, shouldResolve bool
|
||||
|
||||
if state.IsFiring {
|
||||
// 已有告警只在探针实际恢复后解除,不能因为维护结束后重新累计而伪造恢复。
|
||||
if rawOfflineSeconds < int64(config.Rules.AgentOfflineDuration) {
|
||||
shouldResolve = true
|
||||
}
|
||||
} else if offlineSeconds >= int64(config.Rules.AgentOfflineDuration) {
|
||||
if !state.IsFiring {
|
||||
shouldFire = true
|
||||
state.IsFiring = true
|
||||
}
|
||||
// 已有告警只在 WebSocket 连接实际恢复后解除,不能因为维护结束后
|
||||
// 重新累计或 last_seen_at 刷新而伪造恢复。
|
||||
shouldResolve = !isOffline
|
||||
} else if isOffline && offlineSeconds >= int64(config.Rules.AgentOfflineDuration) {
|
||||
shouldFire = true
|
||||
state.IsFiring = true
|
||||
}
|
||||
|
||||
// 保存状态到数据库
|
||||
@@ -1140,6 +1132,22 @@ func (s *AlertService) checkAgentOfflineAlerts(ctx context.Context, now int64) e
|
||||
return nil
|
||||
}
|
||||
|
||||
// agentOfflineCondition 返回探针当前是否离线,以及扣除维护窗口后的离线
|
||||
// 起点和持续秒数。在线状态下时间统一归零,避免去抖写入的 last_seen_at
|
||||
// 被误当作连接活性判断。
|
||||
func agentOfflineCondition(config *EffectiveAlertConfig, agent *models.Agent, now int64) (bool, int64, int64) {
|
||||
if agent.Status == 1 {
|
||||
return false, 0, 0
|
||||
}
|
||||
|
||||
offlineStartTime := config.ConditionStartAfterMaintenance(time.UnixMilli(now), agent.LastSeenAt)
|
||||
offlineSeconds := int64(0)
|
||||
if now > offlineStartTime {
|
||||
offlineSeconds = (now - offlineStartTime) / 1000
|
||||
}
|
||||
return true, offlineStartTime, offlineSeconds
|
||||
}
|
||||
|
||||
// fireAgentOfflineAlert 触发探针离线告警
|
||||
func (s *AlertService) fireAgentOfflineAlert(ctx context.Context, config *EffectiveAlertConfig, agent *models.Agent, state *models.AlertState, offlineSeconds int64, now int64) {
|
||||
s.logger.Info("触发探针离线告警",
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pika-monitor/pika/internal/models"
|
||||
)
|
||||
|
||||
func TestAgentOfflineConditionUsesConnectionStatus(t *testing.T) {
|
||||
now := time.Date(2026, time.August, 31, 22, 31, 1, 0, time.Local).UnixMilli()
|
||||
staleLastSeen := now - int64(306*time.Second/time.Millisecond)
|
||||
config := &EffectiveAlertConfig{}
|
||||
|
||||
t.Run("在线探针不因去抖后的旧心跳时间误判离线", func(t *testing.T) {
|
||||
agent := &models.Agent{Status: 1, LastSeenAt: staleLastSeen}
|
||||
|
||||
isOffline, startTime, seconds := agentOfflineCondition(config, agent, now)
|
||||
if isOffline || startTime != 0 || seconds != 0 {
|
||||
t.Fatalf(
|
||||
"agentOfflineCondition() = (%v, %d, %d), want (false, 0, 0)",
|
||||
isOffline,
|
||||
startTime,
|
||||
seconds,
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("离线探针从最后状态更新时间累计", func(t *testing.T) {
|
||||
agent := &models.Agent{Status: 0, LastSeenAt: staleLastSeen}
|
||||
|
||||
isOffline, startTime, seconds := agentOfflineCondition(config, agent, now)
|
||||
if !isOffline || startTime != staleLastSeen || seconds != 306 {
|
||||
t.Fatalf(
|
||||
"agentOfflineCondition() = (%v, %d, %d), want (true, %d, 306)",
|
||||
isOffline,
|
||||
startTime,
|
||||
seconds,
|
||||
staleLastSeen,
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user