fix: enhance notification system reliability
- Fix: Trigger notification when Agent and Monitor recover from offline status - Fix: Prevent duplicate 'Down' notifications in Monitor task - Fix: Use 'Asia/Shanghai' timezone for notification time - Feat: Add status emojis (🟢/🔴) to Agent and Monitor notifications
This commit is contained in:
1 parent
4803321951
commit
a1e986bca9
4 files changed
+207
-101
No files matched your search
@@ -109,11 +109,11 @@ async function handleAgentOfflineNotification(
|
||||
name: agentName,
|
||||
status: "offline",
|
||||
previous_status: "online", // 添加previous_status变量
|
||||
time: new Date().toLocaleString("zh-CN"),
|
||||
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
|
||||
hostname: agent.hostname || "未知",
|
||||
ip_addresses: getFormattedIPAddresses(agent.ip_addresses),
|
||||
os: agent.os || "未知",
|
||||
error: "客户端连接超时",
|
||||
error: "客户端连接超时 🔴",
|
||||
details: `主机名: ${
|
||||
agent.hostname || "未知"
|
||||
}\nIP地址: ${getFormattedIPAddresses(
|
||||
@@ -145,6 +145,85 @@ async function handleAgentOfflineNotification(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理客户端上线通知
|
||||
* @param env 环境变量
|
||||
* @param agentId 客户端ID
|
||||
* @param agentName 客户端名称
|
||||
* @param userId 用户ID
|
||||
*/
|
||||
export async function handleAgentOnlineNotification(
|
||||
env: any,
|
||||
agentId: number,
|
||||
agentName: string,
|
||||
userId: number
|
||||
) {
|
||||
try {
|
||||
// 检查是否需要发送通知
|
||||
// 注意:这里状态是从 offline 变为 online
|
||||
const notificationCheck = await shouldSendNotification(
|
||||
userId,
|
||||
"agent",
|
||||
agentId,
|
||||
"offline", // 上一个状态
|
||||
"online" // 当前状态
|
||||
);
|
||||
|
||||
if (
|
||||
!notificationCheck.shouldSend ||
|
||||
notificationCheck.channels.length === 0
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`客户端 ${agentName} (ID: ${agentId}) 已恢复上线,正在发送通知...`);
|
||||
|
||||
// 获取客户端完整信息
|
||||
const agent = await getAgentById(agentId);
|
||||
if (!agent) {
|
||||
console.error(`找不到客户端数据 (ID: ${agentId})`);
|
||||
return;
|
||||
}
|
||||
|
||||
// 准备通知变量
|
||||
const variables = {
|
||||
name: agentName,
|
||||
status: "online",
|
||||
previous_status: "offline",
|
||||
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
|
||||
hostname: agent.hostname || "未知",
|
||||
ip_addresses: getFormattedIPAddresses(agent.ip_addresses),
|
||||
os: agent.os || "未知",
|
||||
error: "客户端连接已恢复 🟢",
|
||||
details: `主机名: ${
|
||||
agent.hostname || "未知"
|
||||
}\nIP地址: ${getFormattedIPAddresses(
|
||||
agent.ip_addresses
|
||||
)}\n操作系统: ${agent.os || "未知"}\n恢复时间: ${new Date().toLocaleString("zh-CN")}`,
|
||||
};
|
||||
|
||||
// 发送通知
|
||||
const notificationResult = await sendNotification(
|
||||
"agent",
|
||||
agentId,
|
||||
variables,
|
||||
notificationCheck.channels,
|
||||
userId
|
||||
);
|
||||
|
||||
if (notificationResult.success) {
|
||||
console.log(`客户端 ${agentName} (ID: ${agentId}) 上线通知发送成功`);
|
||||
} else {
|
||||
console.error(`客户端 ${agentName} (ID: ${agentId}) 上线通知发送失败`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`处理客户端上线通知时出错 (${agentName}, ID: ${agentId}):`,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理客户端阈值超出通知
|
||||
* 此函数可以单独调用,也可以在客户端上报数据时触发
|
||||
@@ -261,7 +340,7 @@ export async function handleAgentThresholdNotification(
|
||||
name: agent.name,
|
||||
status: `${metricName}告警`,
|
||||
previous_status: "normal", // 添加previous_status变量
|
||||
time: new Date().toLocaleString("zh-CN"),
|
||||
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
|
||||
hostname: agent.hostname || "未知",
|
||||
ip_addresses: getFormattedIPAddresses(agent.ip_addresses),
|
||||
os: agent.os || "未知",
|
||||
|
||||
@@ -60,18 +60,22 @@ async function handleMonitorNotification(
|
||||
try {
|
||||
console.log(`======= 通知检查开始 =======`);
|
||||
console.log(`监控: ${monitor.name} (ID: ${monitor.id})`);
|
||||
console.log(
|
||||
`上一状态: ${checkResult.previous_status}, 当前状态: ${checkResult.status}`
|
||||
);
|
||||
// console.log(
|
||||
// `上一状态: ${checkResult.previous_status}, 当前状态: ${checkResult.status}`
|
||||
// );
|
||||
|
||||
// 如果监控状态没有变化,不需要继续处理
|
||||
if (checkResult.status === checkResult.previous_status) {
|
||||
console.log(`状态未变化,不发送通知`);
|
||||
// 如果监控状态没有变化,不需要继续处理,使用 monitor.status (数据库里的最新状态) 与刚才检查到的状态 (checkResult.status)
|
||||
if (monitor.status === checkResult.status) {
|
||||
console.log(`[Monitor] ${monitor.name} 状态未变化 (${monitor.status}),忽略通知`);
|
||||
return;
|
||||
}
|
||||
|
||||
// 定义当前状态和前一个状态
|
||||
const currentStatus = checkResult.status;
|
||||
const previousStatus = monitor.status || "unknown"; // 使用 monitor.status 作为前一个状态
|
||||
|
||||
console.log(
|
||||
`状态已变化: ${checkResult.previous_status} -> ${checkResult.status}`
|
||||
`状态已变化: ${previousStatus} -> ${currentStatus}`
|
||||
);
|
||||
|
||||
// 检查是否需要发送通知
|
||||
@@ -80,8 +84,8 @@ async function handleMonitorNotification(
|
||||
monitor.created_by, // 修复: 传入 userId
|
||||
"monitor",
|
||||
monitor.id,
|
||||
checkResult.previous_status,
|
||||
checkResult.status
|
||||
previousStatus,
|
||||
currentStatus
|
||||
);
|
||||
|
||||
console.log(
|
||||
@@ -105,19 +109,28 @@ async function handleMonitorNotification(
|
||||
);
|
||||
console.log(`通知渠道: ${JSON.stringify(notificationCheck.channels)}`);
|
||||
|
||||
// 信息添加红绿灯
|
||||
let errorMsg = checkResult.error || "无";
|
||||
if (currentStatus === "up") {
|
||||
errorMsg = "服务已恢复访问 🟢";
|
||||
}
|
||||
else if (currentStatus === "down") {
|
||||
errorMsg = `${checkResult.error || "服务无法访问"} 🔴`;
|
||||
}
|
||||
|
||||
// 准备通知变量
|
||||
const variables = {
|
||||
name: monitor.name,
|
||||
status: checkResult.status,
|
||||
previous_status: checkResult.previous_status || "未知",
|
||||
time: new Date().toLocaleString("zh-CN"),
|
||||
status: currentStatus,
|
||||
previous_status: previousStatus,
|
||||
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
|
||||
url: monitor.url,
|
||||
response_time: `${checkResult.responseTime}ms`,
|
||||
status_code: checkResult.statusCode
|
||||
? checkResult.statusCode.toString()
|
||||
: "无",
|
||||
expected_status: monitor.expected_status.toString(),
|
||||
error: checkResult.error || "无",
|
||||
error: errorMsg,
|
||||
details: `URL: ${monitor.url}\n响应时间: ${
|
||||
checkResult.responseTime
|
||||
}ms\n状态码: ${checkResult.statusCode || "无"}\n错误信息: ${
|
||||
|
||||
@@ -2,7 +2,7 @@ import { Bindings } from "../models/db";
|
||||
import type { Agent } from "../models/agent";
|
||||
import * as AgentRepository from "../repositories";
|
||||
import { generateToken, verifyToken } from "../utils/jwt";
|
||||
import { handleAgentThresholdNotification } from "../jobs/agent-task";
|
||||
import { handleAgentThresholdNotification, handleAgentOnlineNotification } from "../jobs/agent-task"; // 引入上线通知处理函数
|
||||
|
||||
/**
|
||||
* 获取所有客户端
|
||||
@@ -305,6 +305,19 @@ export async function updateAgentStatusService(status: any) {
|
||||
// 通过token查找客户端
|
||||
const agent = await AgentRepository.getAgentByToken(norlmalInfo.token);
|
||||
|
||||
if (!agent) {
|
||||
throw new Error("找不到对应的Agent");
|
||||
}
|
||||
|
||||
// 检测上线状态
|
||||
if (agent.status !== "active") {
|
||||
console.log(`[AgentService] 客户端 ${agent.name} (ID: ${agent.id}) 恢复上线,触发通知...`);
|
||||
|
||||
// 异步触发上线通知,不阻塞主流程
|
||||
handleAgentOnlineNotification({}, agent.id, agent.name, agent.created_by)
|
||||
.catch(err => console.error(`[AgentService] 触发上线通知失败:`, err));
|
||||
}
|
||||
|
||||
if (
|
||||
agent.status != "active" ||
|
||||
agent.hostname != norlmalInfo.hostname ||
|
||||
|
||||
@@ -7,69 +7,58 @@ export async function getMonitorsToCheck() {
|
||||
}
|
||||
|
||||
export async function checkMonitor(monitor: models.Monitor) {
|
||||
console.log(`开始检查监控项: ${monitor.name} (${monitor.url})`);
|
||||
|
||||
// 记录监控之前的状态
|
||||
const previousStatus = monitor.status;
|
||||
const startTime = Date.now();
|
||||
|
||||
// 初始化结果变量
|
||||
let status = "down"; // 默认为 down,除非请求成功且符合预期
|
||||
let responseTime = 0;
|
||||
let statusCode: number | null = null;
|
||||
let error: string | null = null;
|
||||
let response: Response | null = null;
|
||||
|
||||
try {
|
||||
console.log(`开始检查监控项: ${monitor.name} (${monitor.url})`);
|
||||
// 设置超时
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(
|
||||
() => controller.abort(),
|
||||
(monitor.timeout || 30) * 1000
|
||||
);
|
||||
|
||||
// 记录监控之前的状态
|
||||
const previousStatus = monitor.status;
|
||||
|
||||
const startTime = Date.now();
|
||||
let response;
|
||||
// 准备 Headers
|
||||
let headers: Headers = new Headers();
|
||||
let error = null;
|
||||
|
||||
try {
|
||||
// 设置超时
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(
|
||||
() => controller.abort(),
|
||||
(monitor.timeout || 30) * 1000
|
||||
);
|
||||
|
||||
// 如果headers是字符串,则转换为对象
|
||||
if (typeof monitor.headers === "string") {
|
||||
if (typeof monitor.headers === "string") {
|
||||
try {
|
||||
const parseHeaders = JSON.parse(monitor.headers);
|
||||
if (
|
||||
parseHeaders &&
|
||||
typeof parseHeaders === "object" &&
|
||||
!Array.isArray(parseHeaders)
|
||||
) {
|
||||
const headerObj = Object.assign({}, parseHeaders);
|
||||
if (Object.keys(headerObj).length === 0) {
|
||||
headers = new Headers();
|
||||
} else {
|
||||
headers = new Headers(headerObj);
|
||||
}
|
||||
headers = new Headers(parseHeaders);
|
||||
}
|
||||
} catch (e) {
|
||||
// header 解析失败忽略
|
||||
}
|
||||
|
||||
// 发送请求
|
||||
response = await fetch(monitor.url, {
|
||||
method: monitor.method || "GET",
|
||||
headers: headers,
|
||||
body: monitor.method !== "GET" && monitor.method !== "HEAD" ? monitor.body || "" : undefined,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
// 清除超时
|
||||
clearTimeout(timeoutId);
|
||||
} catch (e) {
|
||||
// 处理请求错误
|
||||
error = e instanceof Error ? e.message : String(e);
|
||||
console.error(`监控 ${monitor.name} (${monitor.url}) 请求失败: ${error}`);
|
||||
|
||||
return {
|
||||
success: false,
|
||||
status: "down",
|
||||
previous_status: previousStatus,
|
||||
error,
|
||||
responseTime: Date.now() - startTime,
|
||||
statusCode: null,
|
||||
};
|
||||
}
|
||||
|
||||
// 发送请求
|
||||
response = await fetch(monitor.url, {
|
||||
method: monitor.method || "GET",
|
||||
headers: headers,
|
||||
body: monitor.method !== "GET" && monitor.method !== "HEAD" ? monitor.body || "" : undefined,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
// 清除超时
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
// 计算响应时间
|
||||
const responseTime = Date.now() - startTime;
|
||||
responseTime = Date.now() - startTime;
|
||||
statusCode = response.status;
|
||||
|
||||
// 检查状态码是否符合预期
|
||||
let isExpectedStatus = false;
|
||||
@@ -77,53 +66,57 @@ export async function checkMonitor(monitor: models.Monitor) {
|
||||
|
||||
// 处理范围状态码:如果预期状态码为个位数(1-5),则视为范围检查
|
||||
if (expectedStatus >= 1 && expectedStatus <= 5) {
|
||||
// 例如,当预期状态码为2时,匹配所有2xx状态码
|
||||
const statusCodeFirstDigit = Math.floor(response.status / 100);
|
||||
const statusCodeFirstDigit = Math.floor(statusCode / 100);
|
||||
isExpectedStatus = statusCodeFirstDigit === expectedStatus;
|
||||
} else {
|
||||
// 精确匹配状态码
|
||||
isExpectedStatus = response.status === expectedStatus;
|
||||
isExpectedStatus = statusCode === expectedStatus;
|
||||
}
|
||||
|
||||
const status = isExpectedStatus ? "up" : "down";
|
||||
// 确定最终状态
|
||||
status = isExpectedStatus ? "up" : "down";
|
||||
|
||||
// 如果状态码不符合预期,记录错误信息
|
||||
if (!isExpectedStatus) {
|
||||
error = `状态码不符合预期: ${statusCode}, 预期: ${getExpectedStatusDisplay(expectedStatus)}`;
|
||||
}
|
||||
|
||||
// 记录状态历史
|
||||
} catch (e) {
|
||||
// 处理请求错误 (连接超时, DNS错误等)
|
||||
status = "down";
|
||||
error = e instanceof Error ? e.message : String(e);
|
||||
responseTime = Date.now() - startTime;
|
||||
console.error(`监控 ${monitor.name} (${monitor.url}) 请求失败: ${error}`);
|
||||
}
|
||||
|
||||
// 确保数据库一定会被更新
|
||||
try {
|
||||
// 1. 记录状态历史
|
||||
await repositories.insertMonitorStatusHistory(
|
||||
monitor.id,
|
||||
status,
|
||||
responseTime,
|
||||
response.status,
|
||||
statusCode ?? 0,
|
||||
error
|
||||
);
|
||||
|
||||
console.log(`监控 ${monitor.name} (${monitor.url}) 检查完成.`);
|
||||
|
||||
// 更新监控状态
|
||||
// 2. 更新监控状态,防止重复通知
|
||||
await repositories.updateMonitorStatus(monitor.id, status, responseTime);
|
||||
|
||||
console.log(`监控 ${monitor.name} (${monitor.url}) 检查完成. 结果: ${status}`);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
status,
|
||||
previous_status: previousStatus,
|
||||
responseTime,
|
||||
statusCode: response.status,
|
||||
error: isExpectedStatus
|
||||
? null
|
||||
: `状态码不符合预期: ${
|
||||
response.status
|
||||
}, 预期: ${getExpectedStatusDisplay(expectedStatus)}`,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`检查监控出错 (${monitor.name}):`, error);
|
||||
return {
|
||||
success: false,
|
||||
status: "error",
|
||||
previous_status: monitor.status,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
responseTime: 0,
|
||||
statusCode: null,
|
||||
};
|
||||
} catch (dbError) {
|
||||
console.error(`更新数据库失败 (${monitor.name}):`, dbError);
|
||||
// 即使数据库更新失败也返回检查结果,以免阻塞流程
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
status,
|
||||
previous_status: previousStatus,
|
||||
responseTime,
|
||||
statusCode,
|
||||
error,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getAllMonitors(userId: number) {
|
||||
@@ -363,7 +356,7 @@ export async function manualCheckMonitor(id: number, userId: number, userRole: s
|
||||
userId, // 修复: 传入 userId
|
||||
"monitor",
|
||||
monitor.id,
|
||||
result.previous_status,
|
||||
result.previous_status || "unknown",
|
||||
result.status
|
||||
);
|
||||
|
||||
@@ -380,20 +373,28 @@ export async function manualCheckMonitor(id: number, userId: number, userRole: s
|
||||
console.log(
|
||||
`监控 ${monitor.name} (ID: ${monitor.id}) 状态变更,正在发送通知...`
|
||||
);
|
||||
|
||||
// 信息添加红绿灯
|
||||
let errorMsg = result.error || "无";
|
||||
if (result.status === "up") {
|
||||
errorMsg = "服务已恢复访问 🟢";
|
||||
} else if (result.status === "down") {
|
||||
errorMsg = `${result.error || "服务无法访问"} 🔴`;
|
||||
}
|
||||
|
||||
// 准备通知变量
|
||||
const variables = {
|
||||
name: monitor.name,
|
||||
status: result.status,
|
||||
previous_status: result.previous_status || "未知",
|
||||
time: new Date().toLocaleString("zh-CN"),
|
||||
time: new Date().toLocaleString("zh-CN", { timeZone: "Asia/Shanghai" }),
|
||||
response_time: `${result.responseTime}ms`,
|
||||
url: monitor.url,
|
||||
status_code: result.statusCode
|
||||
? result.statusCode.toString()
|
||||
: "无",
|
||||
expected_status: monitor.expected_status.toString(),
|
||||
error: result.error || "无",
|
||||
error: errorMsg,
|
||||
details: `URL: ${monitor.url}\n响应时间: ${
|
||||
result.responseTime
|
||||
}ms\n状态码: ${result.statusCode || "无"}\n错误信息: ${
|
||||
|
||||
Reference in new issue
Block a user