fix: enhance notification system reliability

- Fix: Trigger notification when Agent and Monitor recover from offline status
- Fix: Prevent duplicate 'Down' notifications in Monitor task
- Fix: Use 'Asia/Shanghai' timezone for notification time
- Feat: Add status emojis (🟢/🔴) to Agent and Monitor notifications
This commit is contained in:
zhangsherry committed 2026-01-11 01:54:09 +08:00
1 parent 4803321951
commit a1e986bca9
4 files changed
+207 -101

No files matched your search

+82 -3
View File
@@ -109,11 +109,11 @@ async function handleAgentOfflineNotification(
name: agentName,
status: "offline",
previous_status: "online", // 添加previous_status变量
time: new Date().toLocaleString("zh-CN"),
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
hostname: agent.hostname || "未知",
ip_addresses: getFormattedIPAddresses(agent.ip_addresses),
os: agent.os || "未知",
error: "客户端连接超时",
error: "客户端连接超时 🔴",
details: `主机名: ${
agent.hostname || "未知"
}\nIP地址: ${getFormattedIPAddresses(
@@ -145,6 +145,85 @@ async function handleAgentOfflineNotification(
}
}
/**
* 处理客户端上线通知
* @param env 环境变量
* @param agentId 客户端ID
* @param agentName 客户端名称
* @param userId 用户ID
*/
export async function handleAgentOnlineNotification(
env: any,
agentId: number,
agentName: string,
userId: number
) {
try {
// 检查是否需要发送通知
// 注意:这里状态是从 offline 变为 online
const notificationCheck = await shouldSendNotification(
userId,
"agent",
agentId,
"offline", // 上一个状态
"online" // 当前状态
);
if (
!notificationCheck.shouldSend ||
notificationCheck.channels.length === 0
) {
return;
}
console.log(`客户端 ${agentName} (ID: ${agentId}) 已恢复上线,正在发送通知...`);
// 获取客户端完整信息
const agent = await getAgentById(agentId);
if (!agent) {
console.error(`找不到客户端数据 (ID: ${agentId})`);
return;
}
// 准备通知变量
const variables = {
name: agentName,
status: "online",
previous_status: "offline",
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
hostname: agent.hostname || "未知",
ip_addresses: getFormattedIPAddresses(agent.ip_addresses),
os: agent.os || "未知",
error: "客户端连接已恢复 🟢",
details: `主机名: ${
agent.hostname || "未知"
}\nIP地址: ${getFormattedIPAddresses(
agent.ip_addresses
)}\n操作系统: ${agent.os || "未知"}\n恢复时间: ${new Date().toLocaleString("zh-CN")}`,
};
// 发送通知
const notificationResult = await sendNotification(
"agent",
agentId,
variables,
notificationCheck.channels,
userId
);
if (notificationResult.success) {
console.log(`客户端 ${agentName} (ID: ${agentId}) 上线通知发送成功`);
} else {
console.error(`客户端 ${agentName} (ID: ${agentId}) 上线通知发送失败`);
}
} catch (error) {
console.error(
`处理客户端上线通知时出错 (${agentName}, ID: ${agentId}):`,
error
);
}
}
/**
* 处理客户端阈值超出通知
* 此函数可以单独调用,也可以在客户端上报数据时触发
@@ -261,7 +340,7 @@ export async function handleAgentThresholdNotification(
name: agent.name,
status: `${metricName}告警`,
previous_status: "normal", // 添加previous_status变量
time: new Date().toLocaleString("zh-CN"),
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
hostname: agent.hostname || "未知",
ip_addresses: getFormattedIPAddresses(agent.ip_addresses),
os: agent.os || "未知",
+26 -13
View File
@@ -60,18 +60,22 @@ async function handleMonitorNotification(
try {
console.log(`======= 通知检查开始 =======`);
console.log(`监控: ${monitor.name} (ID: ${monitor.id})`);
console.log(
`上一状态: ${checkResult.previous_status}, 当前状态: ${checkResult.status}`
);
// console.log(
// `上一状态: ${checkResult.previous_status}, 当前状态: ${checkResult.status}`
// );
// 如果监控状态没有变化,不需要继续处理
if (checkResult.status === checkResult.previous_status) {
console.log(`状态未变化,不发送通知`);
// 如果监控状态没有变化,不需要继续处理,使用 monitor.status (数据库里的最新状态) 与刚才检查到的状态 (checkResult.status)
if (monitor.status === checkResult.status) {
console.log(`[Monitor] ${monitor.name} 状态未变化 (${monitor.status}),忽略通知`);
return;
}
// 定义当前状态和前一个状态
const currentStatus = checkResult.status;
const previousStatus = monitor.status || "unknown"; // 使用 monitor.status 作为前一个状态
console.log(
`状态已变化: ${checkResult.previous_status} -> ${checkResult.status}`
`状态已变化: ${previousStatus} -> ${currentStatus}`
);
// 检查是否需要发送通知
@@ -80,8 +84,8 @@ async function handleMonitorNotification(
monitor.created_by, // 修复: 传入 userId
"monitor",
monitor.id,
checkResult.previous_status,
checkResult.status
previousStatus,
currentStatus
);
console.log(
@@ -105,19 +109,28 @@ async function handleMonitorNotification(
);
console.log(`通知渠道: ${JSON.stringify(notificationCheck.channels)}`);
// 信息添加红绿灯
let errorMsg = checkResult.error || "无";
if (currentStatus === "up") {
errorMsg = "服务已恢复访问 🟢";
}
else if (currentStatus === "down") {
errorMsg = `${checkResult.error || "服务无法访问"} 🔴`;
}
// 准备通知变量
const variables = {
name: monitor.name,
status: checkResult.status,
previous_status: checkResult.previous_status || "未知",
time: new Date().toLocaleString("zh-CN"),
status: currentStatus,
previous_status: previousStatus,
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
url: monitor.url,
response_time: `${checkResult.responseTime}ms`,
status_code: checkResult.statusCode
? checkResult.statusCode.toString()
: "无",
expected_status: monitor.expected_status.toString(),
error: checkResult.error || "无",
error: errorMsg,
details: `URL: ${monitor.url}\n响应时间: ${
checkResult.responseTime
}ms\n状态码: ${checkResult.statusCode || "无"}\n错误信息: ${
+14 -1
View File
@@ -2,7 +2,7 @@ import { Bindings } from "../models/db";
import type { Agent } from "../models/agent";
import * as AgentRepository from "../repositories";
import { generateToken, verifyToken } from "../utils/jwt";
import { handleAgentThresholdNotification } from "../jobs/agent-task";
import { handleAgentThresholdNotification, handleAgentOnlineNotification } from "../jobs/agent-task"; // 引入上线通知处理函数
/**
* 获取所有客户端
@@ -305,6 +305,19 @@ export async function updateAgentStatusService(status: any) {
// 通过token查找客户端
const agent = await AgentRepository.getAgentByToken(norlmalInfo.token);
if (!agent) {
throw new Error("找不到对应的Agent");
}
// 检测上线状态
if (agent.status !== "active") {
console.log(`[AgentService] 客户端 ${agent.name} (ID: ${agent.id}) 恢复上线,触发通知...`);
// 异步触发上线通知,不阻塞主流程
handleAgentOnlineNotification({}, agent.id, agent.name, agent.created_by)
.catch(err => console.error(`[AgentService] 触发上线通知失败:`, err));
}
if (
agent.status != "active" ||
agent.hostname != norlmalInfo.hostname ||
+85 -84
View File
@@ -7,69 +7,58 @@ export async function getMonitorsToCheck() {
}
export async function checkMonitor(monitor: models.Monitor) {
console.log(`开始检查监控项: ${monitor.name} (${monitor.url})`);
// 记录监控之前的状态
const previousStatus = monitor.status;
const startTime = Date.now();
// 初始化结果变量
let status = "down"; // 默认为 down,除非请求成功且符合预期
let responseTime = 0;
let statusCode: number | null = null;
let error: string | null = null;
let response: Response | null = null;
try {
console.log(`开始检查监控项: ${monitor.name} (${monitor.url})`);
// 设置超时
const controller = new AbortController();
const timeoutId = setTimeout(
() => controller.abort(),
(monitor.timeout || 30) * 1000
);
// 记录监控之前的状态
const previousStatus = monitor.status;
const startTime = Date.now();
let response;
// 准备 Headers
let headers: Headers = new Headers();
let error = null;
try {
// 设置超时
const controller = new AbortController();
const timeoutId = setTimeout(
() => controller.abort(),
(monitor.timeout || 30) * 1000
);
// 如果headers是字符串,则转换为对象
if (typeof monitor.headers === "string") {
if (typeof monitor.headers === "string") {
try {
const parseHeaders = JSON.parse(monitor.headers);
if (
parseHeaders &&
typeof parseHeaders === "object" &&
!Array.isArray(parseHeaders)
) {
const headerObj = Object.assign({}, parseHeaders);
if (Object.keys(headerObj).length === 0) {
headers = new Headers();
} else {
headers = new Headers(headerObj);
}
headers = new Headers(parseHeaders);
}
} catch (e) {
// header 解析失败忽略
}
// 发送请求
response = await fetch(monitor.url, {
method: monitor.method || "GET",
headers: headers,
body: monitor.method !== "GET" && monitor.method !== "HEAD" ? monitor.body || "" : undefined,
signal: controller.signal,
});
// 清除超时
clearTimeout(timeoutId);
} catch (e) {
// 处理请求错误
error = e instanceof Error ? e.message : String(e);
console.error(`监控 ${monitor.name} (${monitor.url}) 请求失败: ${error}`);
return {
success: false,
status: "down",
previous_status: previousStatus,
error,
responseTime: Date.now() - startTime,
statusCode: null,
};
}
// 发送请求
response = await fetch(monitor.url, {
method: monitor.method || "GET",
headers: headers,
body: monitor.method !== "GET" && monitor.method !== "HEAD" ? monitor.body || "" : undefined,
signal: controller.signal,
});
// 清除超时
clearTimeout(timeoutId);
// 计算响应时间
const responseTime = Date.now() - startTime;
responseTime = Date.now() - startTime;
statusCode = response.status;
// 检查状态码是否符合预期
let isExpectedStatus = false;
@@ -77,53 +66,57 @@ export async function checkMonitor(monitor: models.Monitor) {
// 处理范围状态码:如果预期状态码为个位数(1-5),则视为范围检查
if (expectedStatus >= 1 && expectedStatus <= 5) {
// 例如,当预期状态码为2时,匹配所有2xx状态码
const statusCodeFirstDigit = Math.floor(response.status / 100);
const statusCodeFirstDigit = Math.floor(statusCode / 100);
isExpectedStatus = statusCodeFirstDigit === expectedStatus;
} else {
// 精确匹配状态码
isExpectedStatus = response.status === expectedStatus;
isExpectedStatus = statusCode === expectedStatus;
}
const status = isExpectedStatus ? "up" : "down";
// 确定最终状态
status = isExpectedStatus ? "up" : "down";
// 如果状态码不符合预期,记录错误信息
if (!isExpectedStatus) {
error = `状态码不符合预期: ${statusCode}, 预期: ${getExpectedStatusDisplay(expectedStatus)}`;
}
// 记录状态历史
} catch (e) {
// 处理请求错误 (连接超时, DNS错误等)
status = "down";
error = e instanceof Error ? e.message : String(e);
responseTime = Date.now() - startTime;
console.error(`监控 ${monitor.name} (${monitor.url}) 请求失败: ${error}`);
}
// 确保数据库一定会被更新
try {
// 1. 记录状态历史
await repositories.insertMonitorStatusHistory(
monitor.id,
status,
responseTime,
response.status,
statusCode ?? 0,
error
);
console.log(`监控 ${monitor.name} (${monitor.url}) 检查完成.`);
// 更新监控状态
// 2. 更新监控状态,防止重复通知
await repositories.updateMonitorStatus(monitor.id, status, responseTime);
console.log(`监控 ${monitor.name} (${monitor.url}) 检查完成. 结果: ${status}`);
return {
success: true,
status,
previous_status: previousStatus,
responseTime,
statusCode: response.status,
error: isExpectedStatus
? null
: `状态码不符合预期: ${
response.status
}, 预期: ${getExpectedStatusDisplay(expectedStatus)}`,
};
} catch (error) {
console.error(`检查监控出错 (${monitor.name}):`, error);
return {
success: false,
status: "error",
previous_status: monitor.status,
error: error instanceof Error ? error.message : String(error),
responseTime: 0,
statusCode: null,
};
} catch (dbError) {
console.error(`更新数据库失败 (${monitor.name}):`, dbError);
// 即使数据库更新失败也返回检查结果,以免阻塞流程
}
return {
success: true,
status,
previous_status: previousStatus,
responseTime,
statusCode,
error,
};
}
export async function getAllMonitors(userId: number) {
@@ -363,7 +356,7 @@ export async function manualCheckMonitor(id: number, userId: number, userRole: s
userId, // 修复: 传入 userId
"monitor",
monitor.id,
result.previous_status,
result.previous_status || "unknown",
result.status
);
@@ -380,20 +373,28 @@ export async function manualCheckMonitor(id: number, userId: number, userRole: s
console.log(
`监控 ${monitor.name} (ID: ${monitor.id}) 状态变更,正在发送通知...`
);
// 信息添加红绿灯
let errorMsg = result.error || "无";
if (result.status === "up") {
errorMsg = "服务已恢复访问 🟢";
} else if (result.status === "down") {
errorMsg = `${result.error || "服务无法访问"} 🔴`;
}
// 准备通知变量
const variables = {
name: monitor.name,
status: result.status,
previous_status: result.previous_status || "未知",
time: new Date().toLocaleString("zh-CN"),
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
response_time: `${result.responseTime}ms`,
url: monitor.url,
status_code: result.statusCode
? result.statusCode.toString()
: "无",
expected_status: monitor.expected_status.toString(),
error: result.error || "无",
error: errorMsg,
details: `URL: ${monitor.url}\n响应时间: ${
result.responseTime
}ms\n状态码: ${result.statusCode || "无"}\n错误信息: ${