feat: add agent expire alerts

This commit is contained in:
dushixiang committed 2026-05-23 15:32:46 +08:00
1 parent 0e670a82a0
commit c8ef936a71
4 files changed
+232 -3

No files matched your search

+27
View File
@@ -100,6 +100,8 @@ func setup(app *orz.App) error {
// 启动流量重置检查任务(每小时检查一次)
go startTrafficResetCheck(ctx, components, app.Logger())
// 启动机器到期提醒检查任务(每小时检查一次)
go startAgentExpireCheck(ctx, components, app.Logger())
// 启动 DDNS 定时任务
go components.DDNSService.Run(ctx)
@@ -449,6 +451,31 @@ func startTrafficResetCheck(ctx context.Context, components *AppComponents, logg
}
}
// startAgentExpireCheck 启动机器到期提醒定时任务
func startAgentExpireCheck(ctx context.Context, components *AppComponents, logger *zap.Logger) {
logger.Info("启动机器到期提醒检查任务")
ticker := time.NewTicker(1 * time.Hour) // 每小时检查一次
defer ticker.Stop()
check := func() {
if err := components.AlertService.CheckAgentExpireAlerts(ctx); err != nil {
logger.Error("机器到期提醒检查失败", zap.Error(err))
}
}
check()
for {
select {
case <-ctx.Done():
logger.Info("机器到期提醒检查任务已停止")
return
case <-ticker.C:
check()
}
}
}
// JWTAuthMiddleware JWT 认证中间件(必须登录)
func JWTAuthMiddleware(accountHandler *handler.AccountHandler) echo.MiddlewareFunc {
return func(next echo.HandlerFunc) echo.HandlerFunc {
+194
View File
@@ -13,6 +13,11 @@ import (
"gorm.io/gorm"
)
const (
agentExpireAlertType = "agent_expire"
agentExpireReminderThreshold = 7
)
// AlertService 告警服务
type AlertService struct {
Service *orz.Service
@@ -341,6 +346,195 @@ func (s *AlertService) sendAlertNotificationSync(ctx context.Context, record *mo
return s.notifier.SendNotificationByConfigs(ctx, enabledChannels, record, agent, alertConfig.MaskIP)
}
// CheckAgentExpireAlerts 检查机器到期提醒
func (s *AlertService) CheckAgentExpireAlerts(ctx context.Context) error {
alertConfig, err := s.propertyService.GetAlertConfig(ctx)
if err != nil {
s.logger.Error("获取全局告警配置失败", zap.Error(err))
return err
}
if !alertConfig.Enabled {
return nil
}
now := time.Now().UnixMilli()
agents, err := s.agentRepo.FindAll(ctx)
if err != nil {
return err
}
for _, agent := range agents {
if err := s.checkAgentExpireAlert(ctx, &agent, now); err != nil {
s.logger.Error("检查机器过期提醒失败",
zap.String("agentId", agent.ID),
zap.Error(err),
)
}
}
return nil
}
func (s *AlertService) checkAgentExpireAlert(ctx context.Context, agent *models.Agent, now int64) error {
stateKey := fmt.Sprintf("%s:global:%s:%s", agent.ID, agentExpireAlertType, agent.ID)
state, err := s.AlertStateRepo.GetAlertState(ctx, stateKey)
if err != nil {
state = &models.AlertState{
ID: stateKey,
AgentID: agent.ID,
AlertType: agentExpireAlertType,
}
}
daysLeft := agentExpireDaysLeft(agent.ExpireTime, now)
state.AgentID = agent.ID
state.AlertType = agentExpireAlertType
state.Threshold = float64(agentExpireReminderThreshold)
state.Duration = 0
state.Value = daysLeft
state.LastCheckTime = now
dayMs := int64(24 * time.Hour / time.Millisecond)
var shouldFire, shouldResolve bool
if agent.ExpireTime > 0 && agent.ExpireTime-now <= int64(agentExpireReminderThreshold)*dayMs {
if !state.IsFiring {
shouldFire = true
state.IsFiring = true
}
} else {
if state.IsFiring {
shouldResolve = true
}
}
if err := s.AlertStateRepo.SaveAlertState(ctx, state); err != nil {
return fmt.Errorf("保存告警状态失败: %w", err)
}
if shouldFire {
s.fireAgentExpireAlert(ctx, agent, state, now)
}
if shouldResolve {
s.resolveAgentExpireAlert(ctx, agent, state)
}
return nil
}
func (s *AlertService) fireAgentExpireAlert(ctx context.Context, agent *models.Agent, state *models.AlertState, now int64) {
s.logger.Info("触发机器过期提醒",
zap.String("agentId", agent.ID),
zap.String("agentName", agent.Name),
zap.Int64("expireTime", agent.ExpireTime),
zap.Float64("daysLeft", state.Value),
)
record := &models.AlertRecord{
AgentID: agent.ID,
AgentName: agent.Name,
AlertType: agentExpireAlertType,
Message: s.buildAgentExpireMessage(agent, now),
Threshold: float64(agentExpireReminderThreshold),
ActualValue: state.Value,
Level: s.calculateAgentExpireLevel(state.Value),
Status: "firing",
FiredAt: now,
CreatedAt: now,
}
if err := s.AlertRecordRepo.CreateAlertRecord(ctx, record); err != nil {
s.logger.Error("创建机器过期提醒记录失败", zap.Error(err))
return
}
state.LastRecordID = record.ID
if err := s.AlertStateRepo.SaveAlertState(ctx, state); err != nil {
s.logger.Error("保存告警状态失败", zap.Error(err))
return
}
go s.sendAlertNotification(record, agent)
}
func (s *AlertService) resolveAgentExpireAlert(ctx context.Context, agent *models.Agent, state *models.AlertState) {
s.logger.Info("机器过期提醒恢复",
zap.String("agentId", agent.ID),
zap.String("agentName", agent.Name),
zap.Int64("expireTime", agent.ExpireTime),
zap.Float64("daysLeft", state.Value),
)
if state.LastRecordID > 0 {
existingRecord, err := s.AlertRecordRepo.GetAlertRecordByID(ctx, state.LastRecordID)
if err != nil {
s.logger.Error("获取机器过期提醒记录失败", zap.Error(err))
} else if existingRecord != nil && existingRecord.Status == "firing" {
now := time.Now().UnixMilli()
existingRecord.Status = "resolved"
existingRecord.ResolvedValue = state.Value
existingRecord.ResolvedAt = now
existingRecord.UpdatedAt = now
if err := s.AlertRecordRepo.UpdateAlertRecord(ctx, existingRecord); err != nil {
s.logger.Error("更新机器过期提醒记录失败", zap.Error(err))
} else {
go s.sendAlertNotification(existingRecord, agent)
}
}
}
state.IsFiring = false
state.LastRecordID = 0
if err := s.AlertStateRepo.SaveAlertState(ctx, state); err != nil {
s.logger.Error("保存告警状态失败", zap.Error(err))
}
}
func (s *AlertService) buildAgentExpireMessage(agent *models.Agent, now int64) string {
agentName := agent.Name
if agentName == "" {
agentName = agent.Hostname
}
if agentName == "" {
agentName = agent.ID
}
expireAt := time.UnixMilli(agent.ExpireTime).Format("2006-01-02 15:04:05")
if agent.ExpireTime <= now {
return fmt.Sprintf("机器 %s 已于 %s 过期", agentName, expireAt)
}
remaining := agent.ExpireTime - now
dayMs := int64(24 * time.Hour / time.Millisecond)
if remaining < dayMs {
return fmt.Sprintf("机器 %s 将于 %s 过期,剩余不足1天", agentName, expireAt)
}
days := remaining / dayMs
if remaining%dayMs != 0 {
days++
}
return fmt.Sprintf("机器 %s 将于 %s 过期,剩余%d天", agentName, expireAt, days)
}
func (s *AlertService) calculateAgentExpireLevel(daysLeft float64) string {
if daysLeft <= 0 {
return "critical"
}
if daysLeft <= 3 {
return "warning"
}
return "info"
}
func agentExpireDaysLeft(expireTime int64, now int64) float64 {
if expireTime <= 0 {
return 0
}
return float64(expireTime-now) / float64(24*time.Hour/time.Millisecond)
}
// CheckMonitorAlerts 检查监控相关告警(证书和服务下线)
func (s *AlertService) CheckMonitorAlerts(ctx context.Context) error {
// 获取全局告警配置
+7
View File
@@ -106,6 +106,13 @@ var alertTypeMetadataMap = map[string]AlertTypeMetadata{
ShowThreshold: true,
ShowActual: true,
},
"agent_expire": {
Name: "机器过期提醒",
ThresholdUnit: "天",
ValueUnit: "天",
ShowThreshold: true,
ShowActual: true,
},
"ssh_login": {
Name: "SSH登录成功",
ThresholdUnit: "",
+4 -3
View File
@@ -39,6 +39,7 @@ const AlertRecordList = () => {
cert: 'HTTPS证书',
service: '服务下线',
agent_offline: '探针离线',
agent_expire: '机器到期',
};
// 告警级别映射
@@ -175,7 +176,7 @@ const AlertRecordList = () => {
if (record.alertType === 'network') {
return `${record.threshold.toFixed(2)} MB/s`;
}
if (record.alertType === 'cert') {
if (record.alertType === 'cert' || record.alertType === 'agent_expire') {
return `${record.threshold.toFixed(0)} 天`;
}
if (record.alertType === 'service' || record.alertType === 'agent_offline') {
@@ -192,7 +193,7 @@ const AlertRecordList = () => {
if (record.alertType === 'network') {
return `${record.actualValue.toFixed(2)} MB/s`;
}
if (record.alertType === 'cert') {
if (record.alertType === 'cert' || record.alertType === 'agent_expire') {
return `${record.actualValue.toFixed(0)} 天`;
}
if (record.alertType === 'service' || record.alertType === 'agent_offline') {
@@ -213,7 +214,7 @@ const AlertRecordList = () => {
if (record.alertType === 'network') {
return `${record.resolvedValue.toFixed(2)} MB/s`;
}
if (record.alertType === 'cert') {
if (record.alertType === 'cert' || record.alertType === 'agent_expire') {
return `${record.resolvedValue.toFixed(0)} 天`;
}
if (record.alertType === 'service' || record.alertType === 'agent_offline') {