diff --git a/.gitignore b/.gitignore index 11ccca6..a0dba26 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ go.work.sum config.yaml agent.yaml +playground diff --git a/docker-compose.yml b/docker-compose.yml index 53800f4..a3e1fef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,7 +20,20 @@ services: restart: unless-stopped networks: - pika-network - + victoriametrics: + image: victoriametrics/victoria-metrics:v1.131.0 + container_name: victoriametrics + restart: always + command: + - "-retentionPeriod=7d" # 保留 7 天数据 + - "-storageDataPath=/victoria-metrics-data" + - "-httpListenAddr=:8428" + ports: + - "8428:8428" # API, PromQL, insert, select + volumes: + - ./vmdata:/victoria-metrics-data + networks: + - pika-network pika: image: dushixiang/pika:latest container_name: pika-server @@ -33,6 +46,8 @@ services: depends_on: postgresql: condition: service_healthy + victoriametrics: + condition: service_started restart: unless-stopped networks: - pika-network diff --git a/internal/app.go b/internal/app.go index b1723a1..977ba8e 100644 --- a/internal/app.go +++ b/internal/app.go @@ -81,12 +81,6 @@ func setup(app *orz.App) error { // 启动WebSocket管理器 go components.WSManager.Run(ctx) - // 启动数据清理任务 - go components.MetricService.StartCleanupTask(ctx) - - // 启动聚合下采样任务 - go components.MetricService.StartAggregationTask(ctx) - // 启动指标监控任务（用于告警检测） go startMetricsMonitoring(ctx, components, app.Logger()) @@ -305,20 +299,11 @@ func autoMigrate(database *gorm.DB) error { return database.AutoMigrate( &models.Agent{}, &models.ApiKey{}, - &models.CPUMetric{}, - &models.MemoryMetric{}, - &models.DiskMetric{}, - &models.NetworkMetric{}, - &models.NetworkConnectionMetric{}, - &models.DiskIOMetric{}, - &models.GPUMetric{}, - &models.TemperatureMetric{}, - &models.HostMetric{}, + &models.HostMetric{}, // 保留主机静态信息表 &models.AuditResult{}, &models.Property{}, &models.AlertRecord{}, &models.AlertState{}, - &models.MonitorMetric{}, &models.MonitorTask{}, &models.MonitorStats{}, &models.TamperProtectConfig{}, @@ -326,17 +311,7 @@ func autoMigrate(database *gorm.DB) error { &models.TamperAlert{}, &models.DDNSConfig{}, &models.DDNSRecord{}, - // 聚合表 - &models.AggregatedCPUMetricModel{}, - &models.AggregatedMemoryMetricModel{}, - &models.AggregatedDiskMetricModel{}, - &models.AggregatedNetworkMetricModel{}, - &models.AggregatedNetworkConnectionMetricModel{}, - &models.AggregatedDiskIOMetricModel{}, - &models.AggregatedGPUMetricModel{}, - &models.AggregatedTemperatureMetricModel{}, - &models.AggregatedMonitorMetricModel{}, - &models.AggregationProgress{}, + // 指标数据已迁移到 VictoriaMetrics ) } diff --git a/internal/config/config.go b/internal/config/config.go index f0916d7..06f99d9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,11 +2,12 @@ package config // AppConfig 应用配置 type AppConfig struct { - JWT JWTConfig `json:"JWT"` - Users map[string]string `json:"Users"` // 用户名 -> bcrypt加密的密码 - OIDC *OIDCConfig `json:"OIDC"` // OIDC配置（可选） - GitHub *GitHubOAuthConfig `json:"GitHub"` // GitHub OAuth配置（可选） - GeoIP *GeoIPConfig `json:"GeoIP"` // GeoIP配置（可选） + JWT JWTConfig `json:"JWT"` + Users map[string]string `json:"Users"` // 用户名 -> bcrypt加密的密码 + OIDC *OIDCConfig `json:"OIDC"` // OIDC配置（可选） + GitHub *GitHubOAuthConfig `json:"GitHub"` // GitHub OAuth配置（可选） + GeoIP *GeoIPConfig `json:"GeoIP"` // GeoIP配置（可选） + VictoriaMetrics *VMConfig `json:"VictoriaMetrics"` // VictoriaMetrics配置（可选） } // JWTConfig JWT配置 @@ -39,3 +40,12 @@ type GeoIPConfig struct { DBPath string `json:"DBPath"` // GeoIP数据库文件路径（如：GeoLite2-City.mmdb） DBLanguage string `json:"DBLanguage"` // 数据库语言（如：zh-CN、en） } + +// VMConfig VictoriaMetrics配置 +type VMConfig struct { + Enabled bool `json:"Enabled"` // 是否启用VictoriaMetrics + URL string `json:"URL"` // VictoriaMetrics地址 + RetentionDays int `json:"RetentionDays"` // 数据保留天数（用于文档说明） + WriteTimeout int `json:"WriteTimeout"` // 写入超时（秒） + QueryTimeout int `json:"QueryTimeout"` // 查询超时（秒） +} diff --git a/internal/handler/agent_handler.go b/internal/handler/agent_handler.go index 38ffc37..1587976 100644 --- a/internal/handler/agent_handler.go +++ b/internal/handler/agent_handler.go @@ -104,8 +104,6 @@ func (h *AgentHandler) HandleWebSocket(c echo.Context) error { // 注册探针 - 使用独立的context,不依赖HTTP请求的context agent, err := h.agentService.RegisterAgent(context.Background(), c.RealIP(), ®isterReq.AgentInfo, registerReq.ApiKey) if err != nil { - h.logger.Error("failed to register agent", zap.Error(err)) - // 发送注册失败响应 h.sendRegisterError(conn, err.Error()) conn.Close() @@ -444,20 +442,13 @@ func (h *AgentHandler) GetMetrics(c echo.Context) error { } // GetMetrics 内部会自动计算最优聚合间隔 - metrics, err := h.metricService.GetMetrics(ctx, agentID, metricType, start, end, 0, interfaceName) + metrics, err := h.metricService.GetMetrics(ctx, agentID, metricType, start, end, interfaceName) if err != nil { return err } - return orz.Ok(c, orz.Map{ - "agentId": agentID, - "type": metricType, - "range": rangeParam, - "start": start, - "end": end, - "interface": interfaceName, - "metrics": metrics, - }) + // 直接返回 GetMetricsResponse，避免额外嵌套 + return orz.Ok(c, metrics) } // GetLatestMetrics 获取探针最新指标（公开接口，已登录返回全部，未登录返回公开可见） diff --git a/internal/models/metric.go b/internal/models/metric.go index e6cfc8a..29a7ed8 100644 --- a/internal/models/metric.go +++ b/internal/models/metric.go @@ -1,152 +1,6 @@ package models -// CPUMetric CPU指标 -type CPUMetric struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_cpu_agent_ts,priority:1" json:"agentId"` // 探针ID - UsagePercent float64 `json:"usagePercent"` // CPU使用率 - LogicalCores int `json:"logicalCores"` // 逻辑核心数 - PhysicalCores int `json:"physicalCores"` // 物理核心数 - ModelName string `json:"modelName"` // CPU型号 - Timestamp int64 `gorm:"index:idx_cpu_agent_ts,priority:2;index:idx_cpu_ts" json:"timestamp"` // 时间戳（毫秒） -} - -func (CPUMetric) TableName() string { - return "cpu_metrics" -} - -// MemoryMetric 内存指标 -type MemoryMetric struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_mem_agent_ts,priority:1" json:"agentId"` // 探针ID - Total uint64 `json:"total"` // 总内存(字节) - Used uint64 `json:"used"` // 已使用(字节) - Free uint64 `json:"free"` // 空闲(字节) - Available uint64 `json:"available"` // 可用内存(字节) - UsagePercent float64 `json:"usagePercent"` // 使用率 - SwapTotal uint64 `json:"swapTotal"` // 总交换空间(字节) - SwapUsed uint64 `json:"swapUsed"` // 已使用交换空间(字节) - SwapFree uint64 `json:"swapFree"` // 空闲交换空间(字节) - Timestamp int64 `gorm:"index:idx_mem_agent_ts,priority:2;index:idx_mem_ts" json:"timestamp"` // 时间戳（毫秒） -} - -func (MemoryMetric) TableName() string { - return "memory_metrics" -} - -// DiskMetric 磁盘指标 -type DiskMetric struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_disk_agent_ts,priority:1" json:"agentId"` // 探针ID - MountPoint string `json:"mountPoint"` // 挂载点 - Total uint64 `json:"total"` // 总容量(字节) - Used uint64 `json:"used"` // 已使用(字节) - Free uint64 `json:"free"` // 空闲(字节) - UsagePercent float64 `json:"usagePercent"` // 使用率 - Timestamp int64 `gorm:"index:idx_disk_agent_ts,priority:2;index:idx_disk_ts" json:"timestamp"` // 时间戳（毫秒） -} - -func (DiskMetric) TableName() string { - return "disk_metrics" -} - -// NetworkMetric 网络指标 -type NetworkMetric struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_net_agent_ts,priority:1" json:"agentId"` // 探针ID - Interface string `json:"interface"` // 网卡名称 - BytesSentRate uint64 `json:"bytesSentRate"` // 发送速率(字节/秒) - BytesRecvRate uint64 `json:"bytesRecvRate"` // 接收速率(字节/秒) - BytesSentTotal uint64 `json:"bytesSentTotal"` // 累计发送字节数 - BytesRecvTotal uint64 `json:"bytesRecvTotal"` // 累计接收字节数 - Timestamp int64 `gorm:"index:idx_net_agent_ts,priority:2;index:idx_net_ts" json:"timestamp"` // 时间戳（毫秒） -} - -func (NetworkMetric) TableName() string { - return "network_metrics" -} - -// NetworkConnectionMetric 网络连接统计指标 -type NetworkConnectionMetric struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_netconn_agent_ts,priority:1" json:"agentId"` // 探针ID - Established uint32 `json:"established"` // ESTABLISHED 状态连接数 - SynSent uint32 `json:"synSent"` // SYN_SENT 状态连接数 - SynRecv uint32 `json:"synRecv"` // SYN_RECV 状态连接数 - FinWait1 uint32 `json:"finWait1"` // FIN_WAIT1 状态连接数 - FinWait2 uint32 `json:"finWait2"` // FIN_WAIT2 状态连接数 - TimeWait uint32 `json:"timeWait"` // TIME_WAIT 状态连接数 - Close uint32 `json:"close"` // CLOSE 状态连接数 - CloseWait uint32 `json:"closeWait"` // CLOSE_WAIT 状态连接数 - LastAck uint32 `json:"lastAck"` // LAST_ACK 状态连接数 - Listen uint32 `json:"listen"` // LISTEN 状态连接数 - Closing uint32 `json:"closing"` // CLOSING 状态连接数 - Total uint32 `json:"total"` // 总连接数 - Timestamp int64 `gorm:"index:idx_netconn_agent_ts,priority:2;index:idx_netconn_ts" json:"timestamp"` // 时间戳（毫秒） -} - -func (NetworkConnectionMetric) TableName() string { - return "network_connection_metrics" -} - -// DiskIOMetric 磁盘IO指标 -type DiskIOMetric struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_diskio_agent_ts,priority:1" json:"agentId"` // 探针ID - Device string `json:"device"` // 设备名称 - ReadCount uint64 `json:"readCount"` // 读取次数 - WriteCount uint64 `json:"writeCount"` // 写入次数 - ReadBytes uint64 `json:"readBytes"` // 读取字节数 - WriteBytes uint64 `json:"writeBytes"` // 写入字节数 - ReadBytesRate uint64 `json:"readBytesRate"` // 读取速率(字节/秒) - WriteBytesRate uint64 `json:"writeBytesRate"` // 写入速率(字节/秒) - ReadTime uint64 `json:"readTime"` // 读取时间(毫秒) - WriteTime uint64 `json:"writeTime"` // 写入时间(毫秒) - IoTime uint64 `json:"ioTime"` // IO时间(毫秒) - IopsInProgress uint64 `json:"iopsInProgress"` // 正在进行的IO操作数 - Timestamp int64 `gorm:"index:idx_diskio_agent_ts,priority:2;index:idx_diskio_ts" json:"timestamp"` // 时间戳（毫秒） -} - -func (DiskIOMetric) TableName() string { - return "disk_io_metrics" -} - -// GPUMetric GPU指标 -type GPUMetric struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_gpu_agent_idx_ts,priority:1" json:"agentId"` // 探针ID - Index int `gorm:"index:idx_gpu_agent_idx_ts,priority:2" json:"index"` // GPU索引 - Name string `json:"name"` // GPU名称 - Utilization float64 `json:"utilization"` // GPU使用率(%) - MemoryTotal uint64 `json:"memoryTotal"` // 总显存(字节) - MemoryUsed uint64 `json:"memoryUsed"` // 已使用显存(字节) - MemoryFree uint64 `json:"memoryFree"` // 空闲显存(字节) - Temperature float64 `json:"temperature"` // 温度(℃) - PowerDraw float64 `json:"powerDraw"` // 功耗(瓦) - FanSpeed float64 `json:"fanSpeed"` // 风扇转速(%) - PerformanceState string `json:"performanceState"` // 性能状态 - Timestamp int64 `gorm:"index:idx_gpu_agent_idx_ts,priority:3;index:idx_gpu_ts" json:"timestamp"` // 时间戳（毫秒） -} - -func (GPUMetric) TableName() string { - return "gpu_metrics" -} - -// TemperatureMetric 温度指标 -type TemperatureMetric struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_temp_agent_sensor_ts,priority:1" json:"agentId"` // 探针ID - SensorKey string `gorm:"index:idx_temp_agent_sensor_ts,priority:2" json:"sensorKey"` // 传感器标识 - SensorLabel string `json:"sensorLabel"` // 传感器标签 - Temperature float64 `json:"temperature"` // 温度(℃) - Timestamp int64 `gorm:"index:idx_temp_agent_sensor_ts,priority:3;index:idx_temp_ts" json:"timestamp"` // 时间戳（毫秒） -} - -func (TemperatureMetric) TableName() string { - return "temperature_metrics" -} - -// HostMetric 主机信息指标 +// HostMetric 主机信息指标（静态信息，保存在 PostgreSQL） type HostMetric struct { ID uint `gorm:"primaryKey;autoIncrement" json:"id"` AgentID string `gorm:"uniqueIndex:ux_host_agent" json:"agentId"` // 探针ID（唯一约束用于 upsert） @@ -165,191 +19,5 @@ func (HostMetric) TableName() string { return "host_metrics" } -// MonitorMetric 监控指标 -type MonitorMetric struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentId string `gorm:"index:idx_mon_agent_monitor_ts,priority:1" json:"agentId"` // 探针ID - MonitorId string `gorm:"index:idx_mon_agent_monitor_ts,priority:2;index:idx_mon_type_monitor_ts,priority:2" json:"monitorId"` // 监控项ID - Type string `gorm:"index:idx_mon_type_monitor_ts,priority:1" json:"type"` // 监控类型: http, tcp - Target string `json:"target"` // 监控目标 - Status string `json:"status"` // 状态: up, down - StatusCode int `json:"statusCode"` // HTTP状态码 - ResponseTime int64 `json:"responseTime"` // 响应时间(毫秒) - Error string `json:"error"` // 错误信息 - Message string `json:"message"` // 附加信息 - ContentMatch bool `json:"contentMatch"` // 内容匹配结果 - CertExpiryTime int64 `json:"certExpiryTime"` // 证书过期时间(毫秒时间戳), 0表示无证书 - CertDaysLeft int `json:"certDaysLeft"` // 证书剩余天数 - Timestamp int64 `gorm:"index:idx_mon_agent_monitor_ts,priority:3;index:idx_mon_type_monitor_ts,priority:3;index:idx_mon_ts" json:"timestamp"` // 时间戳（毫秒） -} - -func (MonitorMetric) TableName() string { - return "monitor_metrics" -} - -// ----------- 聚合表 ----------- - -// AggregatedCPUMetricModel CPU聚合表 -type AggregatedCPUMetricModel struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_cpuagg_agent_bucket,priority:1;uniqueIndex:ux_cpuagg_bucket,priority:1" json:"agentId"` - BucketSeconds int `gorm:"index:idx_cpuagg_agent_bucket,priority:2;uniqueIndex:ux_cpuagg_bucket,priority:2" json:"bucketSeconds"` - BucketStart int64 `gorm:"index:idx_cpuagg_agent_bucket,priority:3;uniqueIndex:ux_cpuagg_bucket,priority:3" json:"bucketStart"` // 毫秒 - MaxUsage float64 `json:"maxUsage"` - LogicalCores int `json:"logicalCores"` -} - -func (AggregatedCPUMetricModel) TableName() string { - return "cpu_metrics_aggs" -} - -// AggregatedMemoryMetricModel 内存聚合表 -type AggregatedMemoryMetricModel struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_memagg_agent_bucket,priority:1;uniqueIndex:ux_memagg_bucket,priority:1" json:"agentId"` - BucketSeconds int `gorm:"index:idx_memagg_agent_bucket,priority:2;uniqueIndex:ux_memagg_bucket,priority:2" json:"bucketSeconds"` - BucketStart int64 `gorm:"index:idx_memagg_agent_bucket,priority:3;uniqueIndex:ux_memagg_bucket,priority:3" json:"bucketStart"` // 毫秒 - MaxUsage float64 `json:"maxUsage"` - Total uint64 `json:"total"` -} - -func (AggregatedMemoryMetricModel) TableName() string { - return "memory_metrics_aggs" -} - -// AggregatedDiskMetricModel 磁盘聚合表 -type AggregatedDiskMetricModel struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_diskagg_agent_bucket_mp,priority:1;uniqueIndex:ux_diskagg_bucket,priority:1" json:"agentId"` - BucketSeconds int `gorm:"index:idx_diskagg_agent_bucket_mp,priority:2;uniqueIndex:ux_diskagg_bucket,priority:2" json:"bucketSeconds"` - BucketStart int64 `gorm:"index:idx_diskagg_agent_bucket_mp,priority:3;uniqueIndex:ux_diskagg_bucket,priority:3" json:"bucketStart"` // 毫秒 - MountPoint string `gorm:"index:idx_diskagg_agent_bucket_mp,priority:4;uniqueIndex:ux_diskagg_bucket,priority:4" json:"mountPoint"` - MaxUsage float64 `json:"maxUsage"` - Total uint64 `json:"total"` -} - -func (AggregatedDiskMetricModel) TableName() string { - return "disk_metrics_aggs" -} - -// AggregatedNetworkMetricModel 网络聚合表（按网卡分组） -type AggregatedNetworkMetricModel struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_netagg_agent_bucket_iface,priority:1;uniqueIndex:ux_netagg_bucket,priority:1" json:"agentId"` - BucketSeconds int `gorm:"index:idx_netagg_agent_bucket_iface,priority:2;uniqueIndex:ux_netagg_bucket,priority:2" json:"bucketSeconds"` - BucketStart int64 `gorm:"index:idx_netagg_agent_bucket_iface,priority:3;uniqueIndex:ux_netagg_bucket,priority:3" json:"bucketStart"` // 毫秒 - Interface string `gorm:"index:idx_netagg_agent_bucket_iface,priority:4;uniqueIndex:ux_netagg_bucket,priority:4" json:"interface"` - MaxSentRate float64 `json:"maxSentRate"` - MaxRecvRate float64 `json:"maxRecvRate"` -} - -func (AggregatedNetworkMetricModel) TableName() string { - return "network_metrics_aggs" -} - -// AggregatedNetworkConnectionMetricModel 网络连接聚合表 -type AggregatedNetworkConnectionMetricModel struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_netconnagg_agent_bucket,priority:1;uniqueIndex:ux_netconnagg_bucket,priority:1" json:"agentId"` - BucketSeconds int `gorm:"index:idx_netconnagg_agent_bucket,priority:2;uniqueIndex:ux_netconnagg_bucket,priority:2" json:"bucketSeconds"` - BucketStart int64 `gorm:"index:idx_netconnagg_agent_bucket,priority:3;uniqueIndex:ux_netconnagg_bucket,priority:3" json:"bucketStart"` // 毫秒 - MaxEstablished uint32 `json:"maxEstablished"` - MaxSynSent uint32 `json:"maxSynSent"` - MaxSynRecv uint32 `json:"maxSynRecv"` - MaxFinWait1 uint32 `json:"maxFinWait1"` - MaxFinWait2 uint32 `json:"maxFinWait2"` - MaxTimeWait uint32 `json:"maxTimeWait"` - MaxClose uint32 `json:"maxClose"` - MaxCloseWait uint32 `json:"maxCloseWait"` - MaxLastAck uint32 `json:"maxLastAck"` - MaxListen uint32 `json:"maxListen"` - MaxClosing uint32 `json:"maxClosing"` - MaxTotal uint32 `json:"maxTotal"` -} - -func (AggregatedNetworkConnectionMetricModel) TableName() string { - return "network_connection_metrics_aggs" -} - -// AggregatedDiskIOMetricModel 磁盘IO聚合表（汇总所有磁盘） -type AggregatedDiskIOMetricModel struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_diskioagg_agent_bucket,priority:1;uniqueIndex:ux_diskioagg_bucket,priority:1" json:"agentId"` - BucketSeconds int `gorm:"index:idx_diskioagg_agent_bucket,priority:2;uniqueIndex:ux_diskioagg_bucket,priority:2" json:"bucketSeconds"` - BucketStart int64 `gorm:"index:idx_diskioagg_agent_bucket,priority:3;uniqueIndex:ux_diskioagg_bucket,priority:3" json:"bucketStart"` // 毫秒 - MaxReadBytesRate uint64 `json:"maxReadBytesRate"` - MaxWriteBytesRate uint64 `json:"maxWriteBytesRate"` - MaxIopsInProgress uint64 `json:"maxIopsInProgress"` -} - -func (AggregatedDiskIOMetricModel) TableName() string { - return "disk_io_metrics_aggs" -} - -// AggregatedGPUMetricModel GPU聚合表 -type AggregatedGPUMetricModel struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_gpuagg_agent_bucket_idx,priority:1;uniqueIndex:ux_gpuagg_bucket,priority:1" json:"agentId"` - BucketSeconds int `gorm:"index:idx_gpuagg_agent_bucket_idx,priority:2;uniqueIndex:ux_gpuagg_bucket,priority:2" json:"bucketSeconds"` - BucketStart int64 `gorm:"index:idx_gpuagg_agent_bucket_idx,priority:3;uniqueIndex:ux_gpuagg_bucket,priority:3" json:"bucketStart"` // 毫秒 - Index int `gorm:"index:idx_gpuagg_agent_bucket_idx,priority:4;uniqueIndex:ux_gpuagg_bucket,priority:4" json:"index"` - Name string `json:"name"` - MaxUtilization float64 `json:"maxUtilization"` - MaxMemoryUsed uint64 `json:"maxMemoryUsed"` - MaxTemperature float64 `json:"maxTemperature"` - MaxPowerDraw float64 `json:"maxPowerDraw"` - MemoryTotal uint64 `json:"memoryTotal"` -} - -func (AggregatedGPUMetricModel) TableName() string { - return "gpu_metrics_aggs" -} - -// AggregatedTemperatureMetricModel 温度聚合表 -type AggregatedTemperatureMetricModel struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - AgentID string `gorm:"index:idx_tempagg_agent_bucket_sensor,priority:1;uniqueIndex:ux_tempagg_bucket,priority:1" json:"agentId"` - BucketSeconds int `gorm:"index:idx_tempagg_agent_bucket_sensor,priority:2;uniqueIndex:ux_tempagg_bucket,priority:2" json:"bucketSeconds"` - BucketStart int64 `gorm:"index:idx_tempagg_agent_bucket_sensor,priority:3;uniqueIndex:ux_tempagg_bucket,priority:3" json:"bucketStart"` // 毫秒 - SensorKey string `gorm:"index:idx_tempagg_agent_bucket_sensor,priority:4;uniqueIndex:ux_tempagg_bucket,priority:4" json:"sensorKey"` - SensorLabel string `json:"sensorLabel"` - MaxTemperature float64 `json:"maxTemperature"` -} - -func (AggregatedTemperatureMetricModel) TableName() string { - return "temperature_metrics_aggs" -} - -// AggregationProgress 聚合进度记录 -type AggregationProgress struct { - MetricType string `gorm:"primaryKey"` // cpu/memory/disk/network - BucketSeconds int `gorm:"primaryKey"` - LastBucket int64 `json:"lastBucket"` // 已完成的最后 bucket 起始时间（毫秒） - UpdatedAt int64 `json:"updatedAt" gorm:"autoUpdateTime:milli"` - CreatedAt int64 `json:"createdAt" gorm:"autoCreateTime:milli"` -} - -func (AggregationProgress) TableName() string { - return "aggregation_progress" -} - -// AggregatedMonitorMetricModel 监控指标聚合表 -type AggregatedMonitorMetricModel struct { - ID uint `gorm:"primaryKey;autoIncrement" json:"id"` - MonitorID string `gorm:"index:idx_monitoragg_monitor_agent_bucket,priority:1;uniqueIndex:ux_monitoragg_bucket,priority:1" json:"monitorId"` - AgentID string `gorm:"index:idx_monitoragg_monitor_agent_bucket,priority:2;uniqueIndex:ux_monitoragg_bucket,priority:2" json:"agentId"` - BucketSeconds int `gorm:"index:idx_monitoragg_monitor_agent_bucket,priority:3;uniqueIndex:ux_monitoragg_bucket,priority:3" json:"bucketSeconds"` - BucketStart int64 `gorm:"index:idx_monitoragg_monitor_agent_bucket,priority:4;uniqueIndex:ux_monitoragg_bucket,priority:4" json:"bucketStart"` // 毫秒 - AvgResponse float64 `json:"avgResponse"` // 平均响应时间(ms) - MaxResponse int64 `json:"maxResponse"` // 最大响应时间(ms) - MinResponse int64 `json:"minResponse"` // 最小响应时间(ms) - SuccessCount int64 `json:"successCount"` // 成功次数 - TotalCount int64 `json:"totalCount"` // 总次数 - SuccessRate float64 `json:"successRate"` // 成功率(百分比) - LastStatus string `json:"lastStatus"` // 最后状态 - LastError string `json:"lastError"` // 最后错误信息 -} - -func (AggregatedMonitorMetricModel) TableName() string { - return "monitor_metrics_aggs" -} +// 注意：所有其他指标数据已迁移到 VictoriaMetrics +// 包括：CPU, Memory, Disk, Network, NetworkConnection, DiskIO, GPU, Temperature, Monitor diff --git a/internal/models/property.go b/internal/models/property.go index 1c628ae..ff14437 100644 --- a/internal/models/property.go +++ b/internal/models/property.go @@ -80,11 +80,6 @@ var TimeRangeOptions = []TimeRangeOption{ {Label: "7天", Value: "7d"}, } -// MetricsConfig 指标数据配置 -type MetricsConfig struct { - RetentionHours int `json:"retentionHours"` // 原始数据保留小时数（默认168小时=7天） -} - // AlertConfig 全局告警配置 type AlertConfig struct { Enabled bool `json:"enabled"` // 是否启用全局告警 diff --git a/internal/repo/metric_repo.go b/internal/repo/metric_repo.go index 1d01b31..70fdedc 100644 --- a/internal/repo/metric_repo.go +++ b/internal/repo/metric_repo.go @@ -18,41 +18,6 @@ func NewMetricRepo(db *gorm.DB) *MetricRepo { } } -// SaveCPUMetric 保存CPU指标 -func (r *MetricRepo) SaveCPUMetric(ctx context.Context, metric *models.CPUMetric) error { - return r.db.WithContext(ctx).Create(metric).Error -} - -// SaveMemoryMetric 保存内存指标 -func (r *MetricRepo) SaveMemoryMetric(ctx context.Context, metric *models.MemoryMetric) error { - return r.db.WithContext(ctx).Create(metric).Error -} - -// SaveDiskMetric 保存磁盘指标 -func (r *MetricRepo) SaveDiskMetric(ctx context.Context, metric *models.DiskMetric) error { - return r.db.WithContext(ctx).Create(metric).Error -} - -// SaveNetworkMetric 保存网络指标 -func (r *MetricRepo) SaveNetworkMetric(ctx context.Context, metric *models.NetworkMetric) error { - return r.db.WithContext(ctx).Create(metric).Error -} - -// SaveDiskIOMetric 保存磁盘IO指标 -func (r *MetricRepo) SaveDiskIOMetric(ctx context.Context, metric *models.DiskIOMetric) error { - return r.db.WithContext(ctx).Create(metric).Error -} - -// SaveGPUMetric 保存GPU指标 -func (r *MetricRepo) SaveGPUMetric(ctx context.Context, metric *models.GPUMetric) error { - return r.db.WithContext(ctx).Create(metric).Error -} - -// SaveTemperatureMetric 保存温度指标 -func (r *MetricRepo) SaveTemperatureMetric(ctx context.Context, metric *models.TemperatureMetric) error { - return r.db.WithContext(ctx).Create(metric).Error -} - // SaveHostMetric 保存主机信息指标（按 agent 覆盖，避免先删后插的空窗） func (r *MetricRepo) SaveHostMetric(ctx context.Context, metric *models.HostMetric) error { return r.db.WithContext(ctx). @@ -63,164 +28,46 @@ func (r *MetricRepo) SaveHostMetric(ctx context.Context, metric *models.HostMetr Create(metric).Error } -// SaveNetworkConnectionMetric 保存网络连接统计指标 -func (r *MetricRepo) SaveNetworkConnectionMetric(ctx context.Context, metric *models.NetworkConnectionMetric) error { - return r.db.WithContext(ctx).Create(metric).Error +// GetAvailableNetworkInterfaces 获取探针的可用网卡列表 +// TODO: 需要改为从 VictoriaMetrics 查询 labels +func (r *MetricRepo) GetAvailableNetworkInterfaces(ctx context.Context, agentID string) ([]string, error) { + // 暂时返回空列表，后续改为从 VictoriaMetrics 查询 + // 使用 GET /api/v1/label/interface/values?match[]=pika_network_sent_bytes_rate{agent_id="xxx"} + return []string{}, nil } -// AggregatedNetworkConnectionMetric 网络连接统计聚合指标 -type AggregatedNetworkConnectionMetric struct { - Timestamp int64 `json:"timestamp"` - MaxEstablished uint32 `json:"maxEstablished"` // 最大已建立连接数 - MaxSynSent uint32 `json:"maxSynSent"` // 最大 SYN_SENT 连接数 - MaxSynRecv uint32 `json:"maxSynRecv"` // 最大 SYN_RECV 连接数 - MaxFinWait1 uint32 `json:"maxFinWait1"` // 最大 FIN_WAIT1 连接数 - MaxFinWait2 uint32 `json:"maxFinWait2"` // 最大 FIN_WAIT2 连接数 - MaxTimeWait uint32 `json:"maxTimeWait"` // 最大 TIME_WAIT 连接数 - MaxClose uint32 `json:"maxClose"` // 最大 CLOSE 连接数 - MaxCloseWait uint32 `json:"maxCloseWait"` // 最大 CLOSE_WAIT 连接数 - MaxLastAck uint32 `json:"maxLastAck"` // 最大 LAST_ACK 连接数 - MaxListen uint32 `json:"maxListen"` // 最大 LISTEN 连接数 - MaxClosing uint32 `json:"maxClosing"` // 最大 CLOSING 连接数 - MaxTotal uint32 `json:"maxTotal"` // 最大总连接数 -} - -// GetNetworkConnectionMetrics 获取聚合后的网络连接统计指标 -func (r *MetricRepo) GetNetworkConnectionMetrics(ctx context.Context, agentID string, start, end int64, interval int) ([]AggregatedNetworkConnectionMetric, error) { - var metrics []AggregatedNetworkConnectionMetric - - query := ` - SELECT - CAST(FLOOR(timestamp / ?) * ? AS BIGINT) as timestamp, - MAX(established) as max_established, - MAX(syn_sent) as max_syn_sent, - MAX(syn_recv) as max_syn_recv, - MAX(fin_wait1) as max_fin_wait1, - MAX(fin_wait2) as max_fin_wait2, - MAX(time_wait) as max_time_wait, - MAX(close) as max_close, - MAX(close_wait) as max_close_wait, - MAX(last_ack) as max_last_ack, - MAX(listen) as max_listen, - MAX(closing) as max_closing, - MAX(total) as max_total - FROM network_connection_metrics - WHERE agent_id = ? AND timestamp >= ? AND timestamp <= ? - GROUP BY 1 - ORDER BY timestamp ASC - ` - - intervalMs := int64(interval * 1000) - err := r.db.WithContext(ctx). - Raw(query, intervalMs, intervalMs, agentID, start, end). - Scan(&metrics).Error - - return metrics, err -} - -// DeleteOldMetrics 删除指定时间之前的所有指标数据 -func (r *MetricRepo) DeleteOldMetrics(ctx context.Context, beforeTimestamp int64) error { - // 批量大小 - batchSize := 1000 - - // 定义要清理的表（Host 信息只保留最新的，不需要清理） - tables := []interface{}{ - &models.CPUMetric{}, - &models.MemoryMetric{}, - &models.DiskMetric{}, - &models.NetworkMetric{}, - &models.NetworkConnectionMetric{}, - &models.DiskIOMetric{}, - &models.GPUMetric{}, - &models.TemperatureMetric{}, - &models.MonitorMetric{}, +// DeleteAgentMetrics 删除指定探针的所有指标数据 +// TODO: 需要改为调用 VictoriaMetrics 的 DELETE API +func (r *MetricRepo) DeleteAgentMetrics(ctx context.Context, agentID string) error { + // 1. 删除 PostgreSQL 中的主机信息 + if err := r.db.WithContext(ctx).Where("agent_id = ?", agentID).Delete(&models.HostMetric{}).Error; err != nil { + return err } - // 对每个表进行分批删除 - for _, table := range tables { - for { - // 分批删除，避免长事务 - result := r.db.WithContext(ctx). - Where("timestamp < ?", beforeTimestamp). - Limit(batchSize). - Delete(table) - - if result.Error != nil { - return result.Error - } - - // 如果删除的行数少于批量大小，说明已经删除完毕 - if result.RowsAffected < int64(batchSize) { - break - } - } - } + // 2. TODO: 调用 VictoriaMetrics 删除时间序列数据 + // DELETE /api/v1/admin/tsdb/delete_series?match[]=pika_*{agent_id="xxx"} return nil } -// AggregatedCPUMetric CPU聚合指标（使用最大值） +// 以下是兼容性数据结构定义，用于返回查询结果 +// TODO: 后续从 VictoriaMetrics 查询时使用这些结构 + +// AggregatedCPUMetric CPU聚合指标 type AggregatedCPUMetric struct { Timestamp int64 `json:"timestamp"` MaxUsage float64 `json:"maxUsage"` LogicalCores int `json:"logicalCores"` } -// GetCPUMetrics 获取聚合后的CPU指标（取最大值） -// interval: 聚合间隔，单位秒（如：60表示1分钟） -func (r *MetricRepo) GetCPUMetrics(ctx context.Context, agentID string, start, end int64, interval int) ([]AggregatedCPUMetric, error) { - var metrics []AggregatedCPUMetric - - query := ` - SELECT - CAST(FLOOR(timestamp / ?) * ? AS BIGINT) as timestamp, - MAX(usage_percent) as max_usage, - MAX(logical_cores) as logical_cores - FROM cpu_metrics - WHERE agent_id = ? AND timestamp >= ? AND timestamp <= ? - GROUP BY 1 - ORDER BY timestamp ASC - ` - - intervalMs := int64(interval * 1000) - err := r.db.WithContext(ctx). - Raw(query, intervalMs, intervalMs, agentID, start, end). - Scan(&metrics).Error - - return metrics, err -} - -// AggregatedMemoryMetric 内存聚合指标（使用最大值） +// AggregatedMemoryMetric 内存聚合指标 type AggregatedMemoryMetric struct { Timestamp int64 `json:"timestamp"` MaxUsage float64 `json:"maxUsage"` Total uint64 `json:"total"` } -// GetMemoryMetrics 获取聚合后的内存指标（取最大值） -func (r *MetricRepo) GetMemoryMetrics(ctx context.Context, agentID string, start, end int64, interval int) ([]AggregatedMemoryMetric, error) { - var metrics []AggregatedMemoryMetric - - query := ` - SELECT - CAST(FLOOR(timestamp / ?) * ? AS BIGINT) as timestamp, - MAX(usage_percent) as max_usage, - MAX(total) as total - FROM memory_metrics - WHERE agent_id = ? AND timestamp >= ? AND timestamp <= ? - GROUP BY 1 - ORDER BY timestamp ASC - ` - - intervalMs := int64(interval * 1000) - err := r.db.WithContext(ctx). - Raw(query, intervalMs, intervalMs, agentID, start, end). - Scan(&metrics).Error - - return metrics, err -} - -// AggregatedDiskMetric 磁盘聚合指标（使用最大值） +// AggregatedDiskMetric 磁盘聚合指标 type AggregatedDiskMetric struct { Timestamp int64 `json:"timestamp"` MountPoint string `json:"mountPoint"` @@ -228,32 +75,7 @@ type AggregatedDiskMetric struct { Total uint64 `json:"total"` } -// GetDiskMetrics 获取聚合后的磁盘指标（返回预聚合的总和数据） -// 直接查询 mount_point=” 的预聚合记录 -func (r *MetricRepo) GetDiskMetrics(ctx context.Context, agentID string, start, end int64, interval int) ([]AggregatedDiskMetric, error) { - var metrics []AggregatedDiskMetric - - query := ` - SELECT - CAST(FLOOR(timestamp / ?) * ? AS BIGINT) as timestamp, - mount_point, - MAX(usage_percent) as max_usage, - MAX(total) as total - FROM disk_metrics - WHERE agent_id = ? AND timestamp >= ? AND timestamp <= ? AND mount_point = ? - GROUP BY 1, mount_point - ORDER BY timestamp ASC - ` - - intervalMs := int64(interval * 1000) - err := r.db.WithContext(ctx). - Raw(query, intervalMs, intervalMs, agentID, start, end, ""). // 空字符串查询总和记录 - Scan(&metrics).Error - - return metrics, err -} - -// AggregatedNetworkMetric 网络聚合指标（按网卡分组） +// AggregatedNetworkMetric 网络聚合指标 type AggregatedNetworkMetric struct { Timestamp int64 `json:"timestamp"` Interface string `json:"interface"` @@ -261,169 +83,34 @@ type AggregatedNetworkMetric struct { MaxRecvRate float64 `json:"maxRecvRate"` } -// AggregatedNetworkMetricByInterface 按网卡接口分组的网络聚合指标 -type AggregatedNetworkMetricByInterface struct { - Timestamp int64 `json:"timestamp"` - Interface string `json:"interface"` - MaxSentRate float64 `json:"maxSentRate"` - MaxRecvRate float64 `json:"maxRecvRate"` +// AggregatedNetworkConnectionMetric 网络连接统计聚合指标 +type AggregatedNetworkConnectionMetric struct { + Timestamp int64 `json:"timestamp"` + MaxEstablished uint32 `json:"maxEstablished"` + MaxSynSent uint32 `json:"maxSynSent"` + MaxSynRecv uint32 `json:"maxSynRecv"` + MaxFinWait1 uint32 `json:"maxFinWait1"` + MaxFinWait2 uint32 `json:"maxFinWait2"` + MaxTimeWait uint32 `json:"maxTimeWait"` + MaxClose uint32 `json:"maxClose"` + MaxCloseWait uint32 `json:"maxCloseWait"` + MaxLastAck uint32 `json:"maxLastAck"` + MaxListen uint32 `json:"maxListen"` + MaxClosing uint32 `json:"maxClosing"` + MaxTotal uint32 `json:"maxTotal"` } -// GetNetworkMetrics 获取聚合后的网络指标（可选按网卡接口过滤） -// 不指定网卡时查询 interface=” 的预聚合数据，指定网卡时只返回该网卡的数据 -func (r *MetricRepo) GetNetworkMetrics(ctx context.Context, agentID string, start, end int64, interval int, interfaceName string) ([]AggregatedNetworkMetric, error) { - var metrics []AggregatedNetworkMetric - - intervalMs := int64(interval * 1000) - - // 不管是否指定网卡，查询逻辑都一样：直接查询对应 interface 的数据 - // interfaceName 为空字符串时，会查询到预先保存的总和数据 - query := ` - SELECT - CAST(FLOOR(timestamp / ?) * ? AS BIGINT) as timestamp, - interface, - MAX(bytes_sent_rate) as max_sent_rate, - MAX(bytes_recv_rate) as max_recv_rate - FROM network_metrics - WHERE agent_id = ? AND timestamp >= ? AND timestamp <= ? AND interface = ? - GROUP BY 1, interface - ORDER BY timestamp ASC - ` - - err := r.db.WithContext(ctx). - Raw(query, intervalMs, intervalMs, agentID, start, end, interfaceName). - Scan(&metrics).Error - - return metrics, err -} - -// SaveMonitorMetric 保存监控指标 -func (r *MetricRepo) SaveMonitorMetric(ctx context.Context, metric *models.MonitorMetric) error { - return r.db.WithContext(ctx).Create(metric).Error -} - -// GetMonitorMetrics 获取监控指标列表 -func (r *MetricRepo) GetMonitorMetrics(ctx context.Context, agentID, monitorID string, start, end int64) ([]models.MonitorMetric, error) { - var metrics []models.MonitorMetric - query := r.db.WithContext(ctx). - Where("agent_id = ? AND timestamp >= ? AND timestamp <= ?", agentID, start, end) - - // 如果指定了监控项ID，则只查询该监控项 - if monitorID != "" { - query = query.Where("monitor_id = ?", monitorID) - } - - err := query.Order("timestamp ASC").Find(&metrics).Error - return metrics, err -} - -// GetMonitorMetricsByName 获取指定监控项的历史数据 -func (r *MetricRepo) GetMonitorMetricsByName(ctx context.Context, agentID, monitorID string, start, end int64, limit int) ([]models.MonitorMetric, error) { - var metrics []models.MonitorMetric - query := r.db.WithContext(ctx). - Where("agent_id = ? AND monitor_id = ? AND timestamp >= ? AND timestamp <= ?", agentID, monitorID, start, end). - Order("timestamp DESC") - - if limit > 0 { - query = query.Limit(limit) - } - - err := query.Find(&metrics).Error - return metrics, err -} - -// AggregatedMonitorMetric 聚合的监控指标 -type AggregatedMonitorMetric struct { - Timestamp int64 `json:"timestamp"` - AgentID string `json:"agentId"` - AvgResponse float64 `json:"avgResponse"` // 平均响应时间 - MaxResponse int64 `json:"maxResponse"` // 最大响应时间 - MinResponse int64 `json:"minResponse"` // 最小响应时间 - SuccessCount int64 `json:"successCount"` // 成功次数 - TotalCount int64 `json:"totalCount"` // 总次数 - SuccessRate float64 `json:"successRate"` // 成功率 - LastStatus string `json:"lastStatus"` // 最后状态 - LastErrorMsg string `json:"lastErrorMsg"` // 最后错误信息 -} - -// GetAggregatedMonitorMetrics 获取聚合后的监控指标（按探针和时间间隔聚合） -func (r *MetricRepo) GetAggregatedMonitorMetrics(ctx context.Context, monitorID string, start, end int64, interval int) ([]AggregatedMonitorMetric, error) { - var metrics []AggregatedMonitorMetric - - query := ` - WITH ranked_metrics AS ( - SELECT - agent_id, - CAST(FLOOR(timestamp / ?) * ? AS BIGINT) as time_bucket, - response_time, - status, - error, - timestamp, - ROW_NUMBER() OVER (PARTITION BY agent_id, CAST(FLOOR(timestamp / ?) * ? AS BIGINT) ORDER BY timestamp DESC) as rn - FROM monitor_metrics - WHERE monitor_id = ? AND timestamp >= ? AND timestamp <= ? - ) - SELECT - time_bucket as timestamp, - agent_id, - AVG(response_time) as avg_response, - MAX(response_time) as max_response, - MIN(response_time) as min_response, - SUM(CASE WHEN status = 'up' THEN 1 ELSE 0 END) as success_count, - COUNT(*) as total_count, - CAST(SUM(CASE WHEN status = 'up' THEN 1 ELSE 0 END) AS REAL) / CAST(COUNT(*) AS REAL) * 100 as success_rate, - MAX(CASE WHEN rn = 1 THEN status END) as last_status, - MAX(CASE WHEN rn = 1 THEN error END) as last_error_msg - FROM ranked_metrics - GROUP BY time_bucket, agent_id - ORDER BY timestamp ASC, agent_id - ` - - intervalMs := int64(interval * 1000) - err := r.db.WithContext(ctx). - Raw(query, intervalMs, intervalMs, intervalMs, intervalMs, monitorID, start, end). - Scan(&metrics).Error - - return metrics, err -} - -// AggregatedDiskIOMetric 磁盘IO聚合指标（所有磁盘汇总） +// AggregatedDiskIOMetric 磁盘IO聚合指标 type AggregatedDiskIOMetric struct { Timestamp int64 `json:"timestamp"` - MaxReadRate float64 `json:"maxReadRate"` // 最大读取速率(字节/秒) - MaxWriteRate float64 `json:"maxWriteRate"` // 最大写入速率(字节/秒) - TotalReadBytes uint64 `json:"totalReadBytes"` // 总读取字节数 - TotalWriteBytes uint64 `json:"totalWriteBytes"` // 总写入字节数 - MaxIopsInProgress uint64 `json:"maxIopsInProgress"` // 最大进行中的IO操作数 + MaxReadRate float64 `json:"maxReadRate"` + MaxWriteRate float64 `json:"maxWriteRate"` + TotalReadBytes uint64 `json:"totalReadBytes"` + TotalWriteBytes uint64 `json:"totalWriteBytes"` + MaxIopsInProgress uint64 `json:"maxIopsInProgress"` } -// GetDiskIOMetrics 获取聚合后的磁盘IO指标（已在存储时合并所有磁盘） -func (r *MetricRepo) GetDiskIOMetrics(ctx context.Context, agentID string, start, end int64, interval int) ([]AggregatedDiskIOMetric, error) { - var metrics []AggregatedDiskIOMetric - - query := ` - SELECT - CAST(FLOOR(timestamp / ?) * ? AS BIGINT) as timestamp, - MAX(read_bytes_rate) as max_read_rate, - MAX(write_bytes_rate) as max_write_rate, - MAX(read_bytes) as total_read_bytes, - MAX(write_bytes) as total_write_bytes, - MAX(iops_in_progress) as max_iops_in_progress - FROM disk_io_metrics - WHERE agent_id = ? AND timestamp >= ? AND timestamp <= ? - GROUP BY 1 - ORDER BY timestamp ASC - ` - - intervalMs := int64(interval * 1000) - err := r.db.WithContext(ctx). - Raw(query, intervalMs, intervalMs, agentID, start, end). - Scan(&metrics).Error - - return metrics, err -} - -// AggregatedGPUMetric GPU聚合指标（使用最大值） +// AggregatedGPUMetric GPU聚合指标 type AggregatedGPUMetric struct { Timestamp int64 `json:"timestamp"` Index int `json:"index"` @@ -435,32 +122,7 @@ type AggregatedGPUMetric struct { MemoryTotal uint64 `json:"memoryTotal"` } -// GetGPUMetrics 获取聚合后的GPU指标 -func (r *MetricRepo) GetGPUMetrics(ctx context.Context, agentID string, start, end int64, interval int) ([]AggregatedGPUMetric, error) { - var metrics []AggregatedGPUMetric - - query := ` - SELECT - CAST(FLOOR(timestamp / ?) * ? AS BIGINT) as timestamp, - MAX(utilization) as max_utilization, - MAX(memory_used) as max_memory_used, - MAX(temperature) as max_temperature, - MAX(power_draw) as max_power_draw - FROM gpu_metrics - WHERE agent_id = ? AND timestamp >= ? AND timestamp <= ? - GROUP BY 1 - ORDER BY timestamp ASC - ` - - intervalMs := int64(interval * 1000) - err := r.db.WithContext(ctx). - Raw(query, intervalMs, intervalMs, agentID, start, end). - Scan(&metrics).Error - - return metrics, err -} - -// AggregatedTemperatureMetric 温度聚合指标（使用最大值） +// AggregatedTemperatureMetric 温度聚合指标 type AggregatedTemperatureMetric struct { Timestamp int64 `json:"timestamp"` SensorKey string `json:"sensorKey"` @@ -468,525 +130,82 @@ type AggregatedTemperatureMetric struct { MaxTemperature float64 `json:"maxTemperature"` } -// GetTemperatureMetrics 获取聚合后的温度指标 -func (r *MetricRepo) GetTemperatureMetrics(ctx context.Context, agentID string, start, end int64, interval int) ([]AggregatedTemperatureMetric, error) { - var metrics []AggregatedTemperatureMetric - - query := ` - SELECT - CAST(FLOOR(timestamp / ?) * ? AS BIGINT) as timestamp, - sensor_key, - sensor_label, - MAX(temperature) as max_temperature - FROM temperature_metrics - WHERE agent_id = ? AND timestamp >= ? AND timestamp <= ? - GROUP BY 1, sensor_key, sensor_label - ORDER BY timestamp ASC, sensor_key - ` - - intervalMs := int64(interval * 1000) - err := r.db.WithContext(ctx). - Raw(query, intervalMs, intervalMs, agentID, start, end). - Scan(&metrics).Error - - return metrics, err +// AggregatedMonitorMetric 聚合的监控指标 +type AggregatedMonitorMetric struct { + Timestamp int64 `json:"timestamp"` + AgentID string `json:"agentId"` + AvgResponse float64 `json:"avgResponse"` + MaxResponse int64 `json:"maxResponse"` + MinResponse int64 `json:"minResponse"` + SuccessCount int64 `json:"successCount"` + TotalCount int64 `json:"totalCount"` + SuccessRate float64 `json:"successRate"` + LastStatus string `json:"lastStatus"` + LastErrorMsg string `json:"lastErrorMsg"` } -// DeleteMonitorMetrics 删除指定监控任务的所有指标数据 -func (r *MetricRepo) DeleteMonitorMetrics(ctx context.Context, monitorID string) error { - return r.db.WithContext(ctx). - Where("monitor_id = ?", monitorID). - Delete(&models.MonitorMetric{}).Error +// GetMonitorMetrics 获取监控指标历史数据 +// TODO: 需要改为从 VictoriaMetrics 查询 +func (r *MetricRepo) GetMonitorMetrics(ctx context.Context, agentID, monitorID string, start, end int64) ([]MonitorMetric, error) { + // 暂时返回空列表，后续改为从 VictoriaMetrics 查询 + return []MonitorMetric{}, nil } -// DeleteAgentMetrics 删除指定探针的所有指标数据 -func (r *MetricRepo) DeleteAgentMetrics(ctx context.Context, agentID string) error { - tables := []interface{}{ - &models.CPUMetric{}, - &models.MemoryMetric{}, - &models.DiskMetric{}, - &models.DiskIOMetric{}, - &models.NetworkMetric{}, - &models.HostMetric{}, - &models.GPUMetric{}, - &models.TemperatureMetric{}, - &models.MonitorMetric{}, - } - - return r.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - for _, table := range tables { - if err := tx.Where("agent_id = ?", agentID).Delete(table).Error; err != nil { - return err - } - } - return nil - }) +// GetMonitorMetricsByName 获取指定监控项的历史数据 +// TODO: 需要改为从 VictoriaMetrics 查询 +func (r *MetricRepo) GetMonitorMetricsByName(ctx context.Context, agentID, monitorID string, start, end int64, limit int) ([]MonitorMetric, error) { + // 暂时返回空列表，后续改为从 VictoriaMetrics 查询 + return []MonitorMetric{}, nil } // GetLatestMonitorMetricsByType 获取指定类型的最新监控指标（所有探针） -func (r *MetricRepo) GetLatestMonitorMetricsByType(ctx context.Context, monitorType string) ([]*models.MonitorMetric, error) { - var metrics []*models.MonitorMetric - - err := r.db.WithContext(ctx).Raw(` - SELECT m.* - FROM monitor_metrics m - INNER JOIN ( - SELECT monitor_id, MAX(timestamp) AS ts - FROM monitor_metrics - WHERE type = ? - GROUP BY monitor_id - ) latest ON m.monitor_id = latest.monitor_id AND m.timestamp = latest.ts - WHERE m.type = ? - ORDER BY m.monitor_id - `, monitorType, monitorType).Scan(&metrics).Error - - return metrics, err +// TODO: 需要改为从 VictoriaMetrics 查询 +func (r *MetricRepo) GetLatestMonitorMetricsByType(ctx context.Context, monitorType string) ([]*MonitorMetric, error) { + // 暂时返回空列表，后续改为从 VictoriaMetrics 查询 + // 使用 GET /api/v1/query?query=pika_monitor_status{type="xxx"} + return []*MonitorMetric{}, nil } -// GetAllLatestMonitorMetrics 获取所有最新的监控指标（所有探针的所有监控项，每个监控项的最新一条） -func (r *MetricRepo) GetAllLatestMonitorMetrics(ctx context.Context) ([]*models.MonitorMetric, error) { - var metrics []*models.MonitorMetric - - err := r.db.WithContext(ctx).Raw(` - SELECT m.* - FROM monitor_metrics m - INNER JOIN ( - SELECT monitor_id, MAX(timestamp) AS ts - FROM monitor_metrics - GROUP BY monitor_id - ) latest ON m.monitor_id = latest.monitor_id AND m.timestamp = latest.ts - ORDER BY m.monitor_id - `).Scan(&metrics).Error - - return metrics, err +// GetAllLatestMonitorMetrics 获取所有最新的监控指标（所有探针的所有监控项） +// TODO: 需要改为从 VictoriaMetrics 查询 +func (r *MetricRepo) GetAllLatestMonitorMetrics(ctx context.Context) ([]*MonitorMetric, error) { + // 暂时返回空列表，后续改为从 VictoriaMetrics 查询 + // 使用 GET /api/v1/query?query=pika_monitor_status + return []*MonitorMetric{}, nil } -// ----------- 聚合表操作 ----------- - -// AggregateCPUToAgg 将原始CPU数据聚合到聚合表 -func (r *MetricRepo) AggregateCPUToAgg(ctx context.Context, bucketSeconds int, start, end int64) error { - bucketMs := int64(bucketSeconds * 1000) - return r.db.WithContext(ctx).Exec(` - INSERT INTO cpu_metrics_aggs (agent_id, bucket_seconds, bucket_start, max_usage, logical_cores) - SELECT - agent_id, - ? as bucket_seconds, - (timestamp / ?) * ? as bucket_start, - MAX(usage_percent) as max_usage, - MAX(logical_cores) as logical_cores - FROM cpu_metrics - WHERE timestamp >= ? AND timestamp < ? - GROUP BY agent_id, bucket_start - ON CONFLICT (agent_id, bucket_seconds, bucket_start) DO UPDATE SET - max_usage = EXCLUDED.max_usage, - logical_cores = EXCLUDED.logical_cores - `, bucketSeconds, bucketMs, bucketMs, start, end).Error +// DeleteMonitorMetrics 删除指定监控任务的所有指标数据 +// TODO: 需要改为调用 VictoriaMetrics 删除 API +func (r *MetricRepo) DeleteMonitorMetrics(ctx context.Context, monitorID string) error { + // 暂时不删除数据，后续调用 VictoriaMetrics 删除 API + // DELETE /api/v1/admin/tsdb/delete_series?match[]=pika_monitor_*{monitor_id="xxx"} + return nil } -// AggregateMemoryToAgg 将原始内存数据聚合到聚合表 -func (r *MetricRepo) AggregateMemoryToAgg(ctx context.Context, bucketSeconds int, start, end int64) error { - bucketMs := int64(bucketSeconds * 1000) - return r.db.WithContext(ctx).Exec(` - INSERT INTO memory_metrics_aggs (agent_id, bucket_seconds, bucket_start, max_usage, total) - SELECT - agent_id, - ? as bucket_seconds, - (timestamp / ?) * ? as bucket_start, - MAX(usage_percent) as max_usage, - MAX(total) as total - FROM memory_metrics - WHERE timestamp >= ? AND timestamp < ? - GROUP BY agent_id, bucket_start - ON CONFLICT (agent_id, bucket_seconds, bucket_start) DO UPDATE SET - max_usage = EXCLUDED.max_usage, - total = EXCLUDED.total - `, bucketSeconds, bucketMs, bucketMs, start, end).Error -} - -// AggregateDiskToAgg 将原始磁盘数据聚合到聚合表 -func (r *MetricRepo) AggregateDiskToAgg(ctx context.Context, bucketSeconds int, start, end int64) error { - bucketMs := int64(bucketSeconds * 1000) - return r.db.WithContext(ctx).Exec(` - INSERT INTO disk_metrics_aggs (agent_id, bucket_seconds, bucket_start, mount_point, max_usage, total) - SELECT - agent_id, - ? as bucket_seconds, - (timestamp / ?) * ? as bucket_start, - mount_point, - MAX(usage_percent) as max_usage, - MAX(total) as total - FROM disk_metrics - WHERE timestamp >= ? AND timestamp < ? - GROUP BY agent_id, bucket_start, mount_point - ON CONFLICT (agent_id, bucket_seconds, bucket_start, mount_point) DO UPDATE SET - max_usage = EXCLUDED.max_usage, - total = EXCLUDED.total - `, bucketSeconds, bucketMs, bucketMs, start, end).Error -} - -// AggregateNetworkToAgg 将原始网络数据聚合到聚合表（按网卡分组） -func (r *MetricRepo) AggregateNetworkToAgg(ctx context.Context, bucketSeconds int, start, end int64) error { - bucketMs := int64(bucketSeconds * 1000) - return r.db.WithContext(ctx).Exec(` - INSERT INTO network_metrics_aggs (agent_id, bucket_seconds, bucket_start, interface, max_sent_rate, max_recv_rate) - SELECT - agent_id, - ? as bucket_seconds, - (timestamp / ?) * ? as bucket_start, - interface, - MAX(bytes_sent_rate) as max_sent_rate, - MAX(bytes_recv_rate) as max_recv_rate - FROM network_metrics - WHERE timestamp >= ? AND timestamp < ? - GROUP BY agent_id, bucket_start, interface - ON CONFLICT (agent_id, bucket_seconds, bucket_start, interface) DO UPDATE SET - max_sent_rate = EXCLUDED.max_sent_rate, - max_recv_rate = EXCLUDED.max_recv_rate - `, bucketSeconds, bucketMs, bucketMs, start, end).Error -} - -// AggregateNetworkConnectionToAgg 将原始网络连接数据聚合到聚合表 -func (r *MetricRepo) AggregateNetworkConnectionToAgg(ctx context.Context, bucketSeconds int, start, end int64) error { - bucketMs := int64(bucketSeconds * 1000) - return r.db.WithContext(ctx).Exec(` - INSERT INTO network_connection_metrics_aggs ( - agent_id, bucket_seconds, bucket_start, - max_established, max_syn_sent, max_syn_recv, max_fin_wait1, max_fin_wait2, - max_time_wait, max_close, max_close_wait, max_last_ack, max_listen, - max_closing, max_total - ) - SELECT - agent_id, - ? as bucket_seconds, - (timestamp / ?) * ? as bucket_start, - MAX(established) as max_established, - MAX(syn_sent) as max_syn_sent, - MAX(syn_recv) as max_syn_recv, - MAX(fin_wait1) as max_fin_wait1, - MAX(fin_wait2) as max_fin_wait2, - MAX(time_wait) as max_time_wait, - MAX(close) as max_close, - MAX(close_wait) as max_close_wait, - MAX(last_ack) as max_last_ack, - MAX(listen) as max_listen, - MAX(closing) as max_closing, - MAX(total) as max_total - FROM network_connection_metrics - WHERE timestamp >= ? AND timestamp < ? - GROUP BY agent_id, bucket_start - ON CONFLICT (agent_id, bucket_seconds, bucket_start) DO UPDATE SET - max_established = EXCLUDED.max_established, - max_syn_sent = EXCLUDED.max_syn_sent, - max_syn_recv = EXCLUDED.max_syn_recv, - max_fin_wait1 = EXCLUDED.max_fin_wait1, - max_fin_wait2 = EXCLUDED.max_fin_wait2, - max_time_wait = EXCLUDED.max_time_wait, - max_close = EXCLUDED.max_close, - max_close_wait = EXCLUDED.max_close_wait, - max_last_ack = EXCLUDED.max_last_ack, - max_listen = EXCLUDED.max_listen, - max_closing = EXCLUDED.max_closing, - max_total = EXCLUDED.max_total - `, bucketSeconds, bucketMs, bucketMs, start, end).Error -} - -// AggregateDiskIOToAgg 将原始磁盘IO数据聚合到聚合表（已在存储时合并所有磁盘） -func (r *MetricRepo) AggregateDiskIOToAgg(ctx context.Context, bucketSeconds int, start, end int64) error { - bucketMs := int64(bucketSeconds * 1000) - return r.db.WithContext(ctx).Exec(` - INSERT INTO disk_io_metrics_aggs ( - agent_id, bucket_seconds, bucket_start, - max_read_bytes_rate, max_write_bytes_rate, max_iops_in_progress - ) - SELECT - agent_id, - ? as bucket_seconds, - (timestamp / ?) * ? as bucket_start, - MAX(read_bytes_rate) as max_read_bytes_rate, - MAX(write_bytes_rate) as max_write_bytes_rate, - MAX(iops_in_progress) as max_iops_in_progress - FROM disk_io_metrics - WHERE timestamp >= ? AND timestamp < ? - GROUP BY agent_id, bucket_start - ON CONFLICT (agent_id, bucket_seconds, bucket_start) DO UPDATE SET - max_read_bytes_rate = EXCLUDED.max_read_bytes_rate, - max_write_bytes_rate = EXCLUDED.max_write_bytes_rate, - max_iops_in_progress = EXCLUDED.max_iops_in_progress - `, bucketSeconds, bucketMs, bucketMs, start, end).Error -} - -// AggregateGPUToAgg 将原始GPU数据聚合到聚合表 -func (r *MetricRepo) AggregateGPUToAgg(ctx context.Context, bucketSeconds int, start, end int64) error { - bucketMs := int64(bucketSeconds * 1000) - return r.db.WithContext(ctx).Exec(` - INSERT INTO gpu_metrics_aggs ( - agent_id, bucket_seconds, bucket_start, index, name, - max_utilization, max_memory_used, max_temperature, max_power_draw, memory_total - ) - SELECT - agent_id, - ? as bucket_seconds, - (timestamp / ?) * ? as bucket_start, - index, - MAX(name) as name, - MAX(utilization) as max_utilization, - MAX(memory_used) as max_memory_used, - MAX(temperature) as max_temperature, - MAX(power_draw) as max_power_draw, - MAX(memory_total) as memory_total - FROM gpu_metrics - WHERE timestamp >= ? AND timestamp < ? - GROUP BY agent_id, bucket_start, index - ON CONFLICT (agent_id, bucket_seconds, bucket_start, index) DO UPDATE SET - name = EXCLUDED.name, - max_utilization = EXCLUDED.max_utilization, - max_memory_used = EXCLUDED.max_memory_used, - max_temperature = EXCLUDED.max_temperature, - max_power_draw = EXCLUDED.max_power_draw, - memory_total = EXCLUDED.memory_total - `, bucketSeconds, bucketMs, bucketMs, start, end).Error -} - -// AggregateTemperatureToAgg 将原始温度数据聚合到聚合表 -func (r *MetricRepo) AggregateTemperatureToAgg(ctx context.Context, bucketSeconds int, start, end int64) error { - bucketMs := int64(bucketSeconds * 1000) - return r.db.WithContext(ctx).Exec(` - INSERT INTO temperature_metrics_aggs ( - agent_id, bucket_seconds, bucket_start, sensor_key, sensor_label, max_temperature - ) - SELECT - agent_id, - ? as bucket_seconds, - (timestamp / ?) * ? as bucket_start, - sensor_key, - MAX(sensor_label) as sensor_label, - MAX(temperature) as max_temperature - FROM temperature_metrics - WHERE timestamp >= ? AND timestamp < ? - GROUP BY agent_id, bucket_start, sensor_key - ON CONFLICT (agent_id, bucket_seconds, bucket_start, sensor_key) DO UPDATE SET - sensor_label = EXCLUDED.sensor_label, - max_temperature = EXCLUDED.max_temperature - `, bucketSeconds, bucketMs, bucketMs, start, end).Error -} - -// GetCPUMetricsAgg 从聚合表获取CPU指标 -func (r *MetricRepo) GetCPUMetricsAgg(ctx context.Context, agentID string, start, end int64, bucketSeconds int) ([]AggregatedCPUMetric, error) { - var metrics []AggregatedCPUMetric - err := r.db.WithContext(ctx). - Table("cpu_metrics_aggs"). - Select("bucket_start as timestamp, max_usage, logical_cores"). - Where("agent_id = ? AND bucket_seconds = ? AND bucket_start >= ? AND bucket_start <= ?", agentID, bucketSeconds, start, end). - Order("bucket_start"). - Scan(&metrics).Error - return metrics, err -} - -// GetMemoryMetricsAgg 从聚合表获取内存指标 -func (r *MetricRepo) GetMemoryMetricsAgg(ctx context.Context, agentID string, start, end int64, bucketSeconds int) ([]AggregatedMemoryMetric, error) { - var metrics []AggregatedMemoryMetric - err := r.db.WithContext(ctx). - Table("memory_metrics_aggs"). - Select("bucket_start as timestamp, max_usage, total"). - Where("agent_id = ? AND bucket_seconds = ? AND bucket_start >= ? AND bucket_start <= ?", agentID, bucketSeconds, start, end). - Order("bucket_start"). - Scan(&metrics).Error - return metrics, err -} - -// GetDiskMetricsAgg 从聚合表获取磁盘指标（返回预聚合的总和数据） -// 直接查询 mount_point=" 的预聚合记录 -func (r *MetricRepo) GetDiskMetricsAgg(ctx context.Context, agentID string, start, end int64, bucketSeconds int) ([]AggregatedDiskMetric, error) { - var metrics []AggregatedDiskMetric - err := r.db.WithContext(ctx). - Table("disk_metrics_aggs"). - Select("bucket_start as timestamp, mount_point, max_usage, total"). - Where("agent_id = ? AND bucket_seconds = ? AND bucket_start >= ? AND bucket_start <= ? AND mount_point = ?", - agentID, bucketSeconds, start, end, ""). // 空字符串查询总和记录 - Order("bucket_start"). - Scan(&metrics).Error - return metrics, err -} - -// GetNetworkMetricsAgg 从聚合表获取网络指标（可选按网卡接口过滤） -// 不指定网卡时查询 interface=" 的预聚合数据，指定网卡时只返回该网卡的数据 -func (r *MetricRepo) GetNetworkMetricsAgg(ctx context.Context, agentID string, start, end int64, bucketSeconds int, interfaceName string) ([]AggregatedNetworkMetric, error) { - var metrics []AggregatedNetworkMetric - - // 不管是否指定网卡，查询逻辑都一样：直接查询对应 interface 的数据 - // interfaceName 为空字符串时，会查询到预先保存的总和数据 - err := r.db.WithContext(ctx). - Table("network_metrics_aggs"). - Select("bucket_start as timestamp, interface, max_sent_rate, max_recv_rate"). - Where("agent_id = ? AND bucket_seconds = ? AND bucket_start >= ? AND bucket_start <= ? AND interface = ?", - agentID, bucketSeconds, start, end, interfaceName). - Order("bucket_start"). - Scan(&metrics).Error - return metrics, err -} - -// GetAvailableNetworkInterfaces 获取探针的可用网卡列表（不包括空白的总和记录） -func (r *MetricRepo) GetAvailableNetworkInterfaces(ctx context.Context, agentID string) ([]string, error) { - var interfaces []string - err := r.db.WithContext(ctx). - Table("network_metrics"). - Select("DISTINCT interface"). - Where("agent_id = ? AND interface != ?", agentID, ""). // 排除空字符串（总和记录） - Order("interface"). - Pluck("interface", &interfaces).Error - return interfaces, err -} - -// GetNetworkConnectionMetricsAgg 从聚合表获取网络连接指标 -func (r *MetricRepo) GetNetworkConnectionMetricsAgg(ctx context.Context, agentID string, start, end int64, bucketSeconds int) ([]AggregatedNetworkConnectionMetric, error) { - var metrics []AggregatedNetworkConnectionMetric - err := r.db.WithContext(ctx). - Table("network_connection_metrics_aggs"). - Select(`bucket_start as timestamp, - max_established, max_syn_sent, max_syn_recv, - max_fin_wait1, max_fin_wait2, max_time_wait, - max_close, max_close_wait, max_last_ack, - max_listen, max_closing, max_total`). - Where("agent_id = ? AND bucket_seconds = ? AND bucket_start >= ? AND bucket_start <= ?", agentID, bucketSeconds, start, end). - Order("bucket_start"). - Scan(&metrics).Error - return metrics, err -} - -// GetDiskIOMetricsAgg 从聚合表获取磁盘IO指标（汇总所有磁盘） -func (r *MetricRepo) GetDiskIOMetricsAgg(ctx context.Context, agentID string, start, end int64, bucketSeconds int) ([]AggregatedDiskIOMetric, error) { - var metrics []AggregatedDiskIOMetric - err := r.db.WithContext(ctx). - Table("disk_io_metrics_aggs"). - Select(`bucket_start as timestamp, - max_read_bytes_rate as max_read_rate, - max_write_bytes_rate as max_write_rate, - max_iops_in_progress`). - Where("agent_id = ? AND bucket_seconds = ? AND bucket_start >= ? AND bucket_start <= ?", agentID, bucketSeconds, start, end). - Order("bucket_start"). - Scan(&metrics).Error - return metrics, err -} - -// GetGPUMetricsAgg 从聚合表获取GPU指标 -func (r *MetricRepo) GetGPUMetricsAgg(ctx context.Context, agentID string, start, end int64, bucketSeconds int) ([]AggregatedGPUMetric, error) { - var metrics []AggregatedGPUMetric - err := r.db.WithContext(ctx). - Table("gpu_metrics_aggs"). - Select(`bucket_start as timestamp, index, name, - max_utilization, - max_memory_used, - max_temperature, - max_power_draw, - memory_total`). - Where("agent_id = ? AND bucket_seconds = ? AND bucket_start >= ? AND bucket_start <= ?", agentID, bucketSeconds, start, end). - Order("bucket_start, index"). - Scan(&metrics).Error - return metrics, err -} - -// GetTemperatureMetricsAgg 从聚合表获取温度指标 -func (r *MetricRepo) GetTemperatureMetricsAgg(ctx context.Context, agentID string, start, end int64, bucketSeconds int) ([]AggregatedTemperatureMetric, error) { - var metrics []AggregatedTemperatureMetric - err := r.db.WithContext(ctx). - Table("temperature_metrics_aggs"). - Select(`bucket_start as timestamp, sensor_key, sensor_label, - max_temperature`). - Where("agent_id = ? AND bucket_seconds = ? AND bucket_start >= ? AND bucket_start <= ?", agentID, bucketSeconds, start, end). - Order("bucket_start, sensor_key"). - Scan(&metrics).Error - return metrics, err -} - -// UpsertAggregationProgress 更新或插入聚合进度 -func (r *MetricRepo) UpsertAggregationProgress(ctx context.Context, metricType string, bucketSeconds int, lastBucket int64) error { - return r.db.WithContext(ctx). - Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "metric_type"}, {Name: "bucket_seconds"}}, - DoUpdates: clause.AssignmentColumns([]string{"last_bucket"}), - }). - Create(&models.AggregationProgress{ - MetricType: metricType, - BucketSeconds: bucketSeconds, - LastBucket: lastBucket, - }).Error -} - -// GetAggregationProgress 获取聚合进度 -func (r *MetricRepo) GetAggregationProgress(ctx context.Context, metricType string, bucketSeconds int) (*models.AggregationProgress, error) { - var progress models.AggregationProgress - err := r.db.WithContext(ctx). - Where("metric_type = ? AND bucket_seconds = ?", metricType, bucketSeconds). - First(&progress).Error - if err != nil { - return nil, err - } - return &progress, nil -} - -// ----------- 监控指标聚合表操作 ----------- - -// AggregateMonitorMetricsToAgg 将原始监控数据聚合到聚合表 -func (r *MetricRepo) AggregateMonitorMetricsToAgg(ctx context.Context, bucketSeconds int, start, end int64) error { - bucketMs := int64(bucketSeconds * 1000) - return r.db.WithContext(ctx).Exec(` - INSERT INTO monitor_metrics_aggs ( - monitor_id, agent_id, bucket_seconds, bucket_start, - avg_response, max_response, min_response, - success_count, total_count, success_rate, - last_status, last_error - ) - WITH ranked_metrics AS ( - SELECT - monitor_id, - agent_id, - (timestamp / ?) * ? as bucket_start, - response_time, - status, - error, - timestamp, - ROW_NUMBER() OVER (PARTITION BY monitor_id, agent_id, (timestamp / ?) * ? ORDER BY timestamp DESC) as rn - FROM monitor_metrics - WHERE timestamp >= ? AND timestamp < ? - ) - SELECT - monitor_id, - agent_id, - ? as bucket_seconds, - bucket_start, - AVG(response_time) as avg_response, - MAX(response_time) as max_response, - MIN(response_time) as min_response, - SUM(CASE WHEN status = 'up' THEN 1 ELSE 0 END) as success_count, - COUNT(*) as total_count, - CAST(SUM(CASE WHEN status = 'up' THEN 1 ELSE 0 END) AS REAL) / CAST(COUNT(*) AS REAL) * 100 as success_rate, - MAX(CASE WHEN rn = 1 THEN status END) as last_status, - MAX(CASE WHEN rn = 1 THEN error END) as last_error - FROM ranked_metrics - GROUP BY monitor_id, agent_id, bucket_start - ON CONFLICT (monitor_id, agent_id, bucket_seconds, bucket_start) DO UPDATE SET - avg_response = EXCLUDED.avg_response, - max_response = EXCLUDED.max_response, - min_response = EXCLUDED.min_response, - success_count = EXCLUDED.success_count, - total_count = EXCLUDED.total_count, - success_rate = EXCLUDED.success_rate, - last_status = EXCLUDED.last_status, - last_error = EXCLUDED.last_error - `, bucketMs, bucketMs, bucketMs, bucketMs, start, end, bucketSeconds).Error -} - -// GetMonitorMetricsAgg 从聚合表获取监控指标（按探针和时间间隔聚合） +// GetMonitorMetricsAgg 获取聚合后的监控指标 +// TODO: 需要改为从 VictoriaMetrics 查询 func (r *MetricRepo) GetMonitorMetricsAgg(ctx context.Context, monitorID string, start, end int64, bucketSeconds int) ([]AggregatedMonitorMetric, error) { - var metrics []AggregatedMonitorMetric - err := r.db.WithContext(ctx). - Table("monitor_metrics_aggs"). - Select(`bucket_start as timestamp, agent_id, - avg_response, max_response, min_response, - success_count, total_count, success_rate, - last_status, last_error as last_error_msg`). - Where("monitor_id = ? AND bucket_seconds = ? AND bucket_start >= ? AND bucket_start <= ?", monitorID, bucketSeconds, start, end). - Order("bucket_start, agent_id"). - Scan(&metrics).Error - return metrics, err + // 暂时返回空列表，后续从 VictoriaMetrics 查询 + return []AggregatedMonitorMetric{}, nil } + +// MonitorMetric 监控指标（与service层定义保持一致） +type MonitorMetric struct { + ID uint `json:"id"` + AgentId string `json:"agentId"` + MonitorId string `json:"monitorId"` + Type string `json:"type"` + Target string `json:"target"` + Status string `json:"status"` + StatusCode int `json:"statusCode"` + ResponseTime int64 `json:"responseTime"` + Error string `json:"error"` + Message string `json:"message"` + ContentMatch bool `json:"contentMatch"` + CertExpiryTime int64 `json:"certExpiryTime"` + CertDaysLeft int `json:"certDaysLeft"` + Timestamp int64 `json:"timestamp"` +} + +// 注意：所有指标数据已迁移到 VictoriaMetrics +// 查询方法已移至 MetricService，通过 VMClient 访问 diff --git a/internal/service/agent_service.go b/internal/service/agent_service.go index 710ba2b..52c7607 100644 --- a/internal/service/agent_service.go +++ b/internal/service/agent_service.go @@ -331,12 +331,12 @@ func (s *AgentService) GetStatistics(ctx context.Context) (map[string]interface{ } // GetMonitorMetrics 获取监控指标历史数据 -func (s *AgentService) GetMonitorMetrics(ctx context.Context, agentID, monitorName string, start, end int64) ([]models.MonitorMetric, error) { +func (s *AgentService) GetMonitorMetrics(ctx context.Context, agentID, monitorName string, start, end int64) ([]MonitorMetric, error) { return s.metricService.GetMonitorMetrics(ctx, agentID, monitorName, start, end) } // GetMonitorMetricsByName 获取指定监控项的历史数据 -func (s *AgentService) GetMonitorMetricsByName(ctx context.Context, agentID, monitorName string, start, end int64, limit int) ([]models.MonitorMetric, error) { +func (s *AgentService) GetMonitorMetricsByName(ctx context.Context, agentID, monitorName string, start, end int64, limit int) ([]MonitorMetric, error) { return s.metricService.GetMonitorMetricsByName(ctx, agentID, monitorName, start, end, limit) } diff --git a/internal/service/alert_service.go b/internal/service/alert_service.go index b299b5c..50a908c 100644 --- a/internal/service/alert_service.go +++ b/internal/service/alert_service.go @@ -407,7 +407,7 @@ func (s *AlertService) checkCertificateAlerts(ctx context.Context, config *model } // checkCertAlert 检查并触发证书告警 -func (s *AlertService) checkCertAlert(ctx context.Context, config *models.AlertConfig, agent *models.Agent, monitor *models.MonitorMetric, certDaysLeft float64, now int64) { +func (s *AlertService) checkCertAlert(ctx context.Context, config *models.AlertConfig, agent *models.Agent, monitor *repo.MonitorMetric, certDaysLeft float64, now int64) { stateKey := fmt.Sprintf("%s:global:cert:%s", agent.ID, monitor.MonitorId) // 从数据库加载状态 @@ -479,7 +479,7 @@ func (s *AlertService) checkCertAlert(ctx context.Context, config *models.AlertC } // resolveCertAlert 恢复证书告警 -func (s *AlertService) resolveCertAlert(ctx context.Context, config *models.AlertConfig, agent *models.Agent, monitor *models.MonitorMetric, certDaysLeft float64) { +func (s *AlertService) resolveCertAlert(ctx context.Context, config *models.AlertConfig, agent *models.Agent, monitor *repo.MonitorMetric, certDaysLeft float64) { stateKey := fmt.Sprintf("%s:global:cert:%s", agent.ID, monitor.MonitorId) state, err := s.AlertStateRepo.GetAlertState(ctx, stateKey) @@ -603,7 +603,7 @@ func (s *AlertService) checkServiceDownAlerts(ctx context.Context, config *model } // fireServiceDownAlert 触发服务下线告警 -func (s *AlertService) fireServiceDownAlert(ctx context.Context, config *models.AlertConfig, agent *models.Agent, monitor *models.MonitorMetric, state *models.AlertState, now int64) { +func (s *AlertService) fireServiceDownAlert(ctx context.Context, config *models.AlertConfig, agent *models.Agent, monitor *repo.MonitorMetric, state *models.AlertState, now int64) { s.logger.Info("触发服务下线告警", zap.String("agentId", agent.ID), zap.String("monitorId", monitor.MonitorId), @@ -641,7 +641,7 @@ func (s *AlertService) fireServiceDownAlert(ctx context.Context, config *models. } // resolveServiceDownAlert 恢复服务下线告警 -func (s *AlertService) resolveServiceDownAlert(ctx context.Context, config *models.AlertConfig, agent *models.Agent, monitor *models.MonitorMetric, state *models.AlertState) { +func (s *AlertService) resolveServiceDownAlert(ctx context.Context, config *models.AlertConfig, agent *models.Agent, monitor *repo.MonitorMetric, state *models.AlertState) { s.logger.Info("服务下线告警恢复", zap.String("agentId", agent.ID), zap.String("monitorId", monitor.MonitorId), diff --git a/internal/service/metric_converter.go b/internal/service/metric_converter.go new file mode 100644 index 0000000..f9dd528 --- /dev/null +++ b/internal/service/metric_converter.go @@ -0,0 +1,152 @@ +package service + +import ( + "fmt" + + "github.com/dushixiang/pika/internal/protocol" + "github.com/dushixiang/pika/internal/vmclient" +) + +// convertToMetrics 将指标数据转换为 VictoriaMetrics Metric 对象 +func (s *MetricService) convertToMetrics(agentID string, metricType string, data interface{}, timestamp int64) []vmclient.Metric { + var metrics []vmclient.Metric + + switch protocol.MetricType(metricType) { + case protocol.MetricTypeCPU: + cpuData := data.(*protocol.CPUData) + metrics = append(metrics, createMetric("pika_cpu_usage_percent", agentID, nil, cpuData.UsagePercent, timestamp)) + metrics = append(metrics, createMetric("pika_cpu_cores_logical", agentID, nil, float64(cpuData.LogicalCores), timestamp)) + metrics = append(metrics, createMetric("pika_cpu_cores_physical", agentID, nil, float64(cpuData.PhysicalCores), timestamp)) + + case protocol.MetricTypeMemory: + memData := data.(*protocol.MemoryData) + metrics = append(metrics, createMetric("pika_memory_usage_percent", agentID, nil, memData.UsagePercent, timestamp)) + metrics = append(metrics, createMetric("pika_memory_total_bytes", agentID, nil, float64(memData.Total), timestamp)) + metrics = append(metrics, createMetric("pika_memory_used_bytes", agentID, nil, float64(memData.Used), timestamp)) + metrics = append(metrics, createMetric("pika_memory_available_bytes", agentID, nil, float64(memData.Available), timestamp)) + metrics = append(metrics, createMetric("pika_memory_swap_total_bytes", agentID, nil, float64(memData.SwapTotal), timestamp)) + metrics = append(metrics, createMetric("pika_memory_swap_used_bytes", agentID, nil, float64(memData.SwapUsed), timestamp)) + + case protocol.MetricTypeDisk: + diskDataList := data.([]protocol.DiskData) + for _, diskData := range diskDataList { + labels := map[string]string{"mount_point": diskData.MountPoint} + metrics = append(metrics, createMetric("pika_disk_usage_percent", agentID, labels, diskData.UsagePercent, timestamp)) + metrics = append(metrics, createMetric("pika_disk_total_bytes", agentID, labels, float64(diskData.Total), timestamp)) + metrics = append(metrics, createMetric("pika_disk_used_bytes", agentID, labels, float64(diskData.Used), timestamp)) + metrics = append(metrics, createMetric("pika_disk_free_bytes", agentID, labels, float64(diskData.Free), timestamp)) + } + + case protocol.MetricTypeNetwork: + networkDataList := data.([]protocol.NetworkData) + for _, netData := range networkDataList { + labels := map[string]string{"interface": netData.Interface} + metrics = append(metrics, createMetric("pika_network_sent_bytes_rate", agentID, labels, float64(netData.BytesSentRate), timestamp)) + metrics = append(metrics, createMetric("pika_network_recv_bytes_rate", agentID, labels, float64(netData.BytesRecvRate), timestamp)) + metrics = append(metrics, createMetric("pika_network_sent_bytes_total", agentID, labels, float64(netData.BytesSentTotal), timestamp)) + metrics = append(metrics, createMetric("pika_network_recv_bytes_total", agentID, labels, float64(netData.BytesRecvTotal), timestamp)) + } + + case protocol.MetricTypeNetworkConnection: + connData := data.(*protocol.NetworkConnectionData) + metrics = append(metrics, createMetric("pika_network_conn_established", agentID, nil, float64(connData.Established), timestamp)) + metrics = append(metrics, createMetric("pika_network_conn_syn_sent", agentID, nil, float64(connData.SynSent), timestamp)) + metrics = append(metrics, createMetric("pika_network_conn_syn_recv", agentID, nil, float64(connData.SynRecv), timestamp)) + metrics = append(metrics, createMetric("pika_network_conn_time_wait", agentID, nil, float64(connData.TimeWait), timestamp)) + metrics = append(metrics, createMetric("pika_network_conn_close_wait", agentID, nil, float64(connData.CloseWait), timestamp)) + metrics = append(metrics, createMetric("pika_network_conn_total", agentID, nil, float64(connData.Total), timestamp)) + + case protocol.MetricTypeDiskIO: + diskIODataList := data.([]*protocol.DiskIOData) + // 汇总所有磁盘的 IO + var totalReadRate, totalWriteRate uint64 + for _, diskIOData := range diskIODataList { + totalReadRate += diskIOData.ReadBytesRate + totalWriteRate += diskIOData.WriteBytesRate + } + metrics = append(metrics, createMetric("pika_disk_read_bytes_rate", agentID, nil, float64(totalReadRate), timestamp)) + metrics = append(metrics, createMetric("pika_disk_write_bytes_rate", agentID, nil, float64(totalWriteRate), timestamp)) + + case protocol.MetricTypeGPU: + gpuDataList := data.([]protocol.GPUData) + for _, gpuData := range gpuDataList { + labels := map[string]string{ + "gpu_index": fmt.Sprintf("%d", gpuData.Index), + "gpu_name": gpuData.Name, + } + metrics = append(metrics, createMetric("pika_gpu_utilization_percent", agentID, labels, gpuData.Utilization, timestamp)) + metrics = append(metrics, createMetric("pika_gpu_memory_total_bytes", agentID, labels, float64(gpuData.MemoryTotal), timestamp)) + metrics = append(metrics, createMetric("pika_gpu_memory_used_bytes", agentID, labels, float64(gpuData.MemoryUsed), timestamp)) + metrics = append(metrics, createMetric("pika_gpu_temperature_celsius", agentID, labels, gpuData.Temperature, timestamp)) + metrics = append(metrics, createMetric("pika_gpu_power_draw_watts", agentID, labels, gpuData.PowerUsage, timestamp)) + } + + case protocol.MetricTypeTemperature: + tempDataList := data.([]protocol.TemperatureData) + for _, tempData := range tempDataList { + labels := map[string]string{ + "sensor_key": tempData.SensorKey, + "sensor_label": tempData.Type, + } + metrics = append(metrics, createMetric("pika_temperature_celsius", agentID, labels, tempData.Temperature, timestamp)) + } + + case protocol.MetricTypeMonitor: + monitorDataList := data.([]protocol.MonitorData) + for _, monitorData := range monitorDataList { + labels := map[string]string{ + "monitor_id": monitorData.ID, + "monitor_type": monitorData.Type, + "target": monitorData.Target, + } + metrics = append(metrics, createMetric("pika_monitor_response_time_ms", agentID, labels, float64(monitorData.ResponseTime), timestamp)) + + // 状态: up=1, down=0 + statusValue := 0.0 + if monitorData.Status == "up" { + statusValue = 1.0 + } + statusLabels := map[string]string{ + "monitor_id": monitorData.ID, + "monitor_type": monitorData.Type, + "target": monitorData.Target, + "status": monitorData.Status, + } + metrics = append(metrics, createMetric("pika_monitor_status", agentID, statusLabels, statusValue, timestamp)) + + // 状态码 + if monitorData.StatusCode > 0 { + metrics = append(metrics, createMetric("pika_monitor_status_code", agentID, labels, float64(monitorData.StatusCode), timestamp)) + } + + // 证书信息 + if monitorData.CertExpiryTime > 0 { + metrics = append(metrics, createMetric("pika_monitor_cert_expiry_timestamp_ms", agentID, labels, float64(monitorData.CertExpiryTime), timestamp)) + metrics = append(metrics, createMetric("pika_monitor_cert_days_left", agentID, labels, float64(monitorData.CertDaysLeft), timestamp)) + } + } + } + + return metrics +} + +// createMetric 创建 VictoriaMetrics Metric 对象 +func createMetric(metricName, agentID string, extraLabels map[string]string, value float64, timestamp int64) vmclient.Metric { + // 创建 metric labels，包含 __name__ 和 agent_id + labels := make(map[string]string) + labels["__name__"] = metricName + labels["agent_id"] = agentID + + // 添加额外的 labels + if extraLabels != nil { + for k, v := range extraLabels { + labels[k] = v + } + } + + return vmclient.Metric{ + Metric: labels, + Values: []float64{value}, + Timestamps: []int64{timestamp}, + } +} diff --git a/internal/service/metric_service.go b/internal/service/metric_service.go index d898a40..737bd60 100644 --- a/internal/service/metric_service.go +++ b/internal/service/metric_service.go @@ -3,40 +3,45 @@ package service import ( "context" "encoding/json" - "math" + "fmt" "time" "github.com/dushixiang/pika/internal/models" "github.com/dushixiang/pika/internal/protocol" "github.com/dushixiang/pika/internal/repo" + "github.com/dushixiang/pika/internal/vmclient" "github.com/go-orz/cache" "go.uber.org/zap" "gorm.io/gorm" ) -const ( - defaultMetricsRetentionHours = 24 * 7 // 默认保留 7 天 - defaultMaxQueryPoints = 300 // 默认最多返回 300 个点（优化前端渲染性能） -) - -var allowedIntervals = []int{ - 1, 2, 5, - 10, 15, 30, - 60, 120, 300, - 600, 900, 1800, - 3600, 7200, 14400, +// MetricDataPoint 统一的指标数据点结构 +type MetricDataPoint struct { + Timestamp int64 `json:"timestamp"` // 毫秒时间戳 + Value float64 `json:"value"` } -// 聚合任务支持的 bucket 列表（升序） -// 与前端 timeRangeOptions 对齐，优化查询效率 -var aggregationBuckets = []int{ - 60, // 1 分钟 - 适用于 15分钟、30分钟时间范围 - 300, // 5 分钟 - 适用于 1小时、3小时时间范围 - 900, // 15 分钟 - 适用于 6小时、12小时时间范围 - 1800, // 30 分钟 - 适用于 1天时间范围 - 3600, // 1 小时 - 适用于 3天时间范围 - 7200, // 2 小时 - 适用于 7天时间范围 +// MetricSeries 指标系列（支持多系列，如多网卡、多传感器） +type MetricSeries struct { + Name string `json:"name"` // 系列名称 + Labels map[string]string `json:"labels,omitempty"` // 额外标签 + Data []MetricDataPoint `json:"data"` // 数据点列表 +} + +// GetMetricsResponse 统一的查询响应格式 +type GetMetricsResponse struct { + AgentID string `json:"agentId"` + Type string `json:"type"` + Range string `json:"range"` + Series []MetricSeries `json:"series"` +} + +// QueryDefinition 查询定义（用于构建多个查询） +type QueryDefinition struct { + Name string // 系列名称 + Query string // PromQL 查询语句 + Labels map[string]string // 额外标签 } // MetricService 指标服务 @@ -46,18 +51,20 @@ type MetricService struct { monitorStatsRepo *repo.MonitorStatsRepo propertyService *PropertyService trafficService *TrafficService // 流量统计服务 + vmClient *vmclient.VMClient latestCache cache.Cache[string, *LatestMetrics] } // NewMetricService 创建指标服务 -func NewMetricService(logger *zap.Logger, db *gorm.DB, propertyService *PropertyService, trafficService *TrafficService) *MetricService { +func NewMetricService(logger *zap.Logger, db *gorm.DB, propertyService *PropertyService, trafficService *TrafficService, vmClient *vmclient.VMClient) *MetricService { return &MetricService{ logger: logger, metricRepo: repo.NewMetricRepo(db), monitorStatsRepo: repo.NewMonitorStatsRepo(db), propertyService: propertyService, trafficService: trafficService, + vmClient: vmClient, latestCache: cache.New[string, *LatestMetrics](time.Minute), } } @@ -66,20 +73,21 @@ func NewMetricService(logger *zap.Logger, db *gorm.DB, propertyService *Property func (s *MetricService) HandleMetricData(ctx context.Context, agentID string, metricType string, data json.RawMessage) error { now := time.Now().UnixMilli() + // 更新内存缓存 latestMetrics, ok := s.latestCache.Get(agentID) if !ok { latestMetrics = &LatestMetrics{} s.latestCache.Set(agentID, latestMetrics, time.Hour) } + // 解析数据并写入 VictoriaMetrics switch protocol.MetricType(metricType) { case protocol.MetricTypeCPU: - // CPU数据现在包含静态和动态信息 var cpuData protocol.CPUData if err := json.Unmarshal(data, &cpuData); err != nil { return err } - metric := &models.CPUMetric{ + metric := &CPUMetric{ AgentID: agentID, UsagePercent: cpuData.UsagePercent, LogicalCores: cpuData.LogicalCores, @@ -88,17 +96,17 @@ func (s *MetricService) HandleMetricData(ctx context.Context, agentID string, me Timestamp: now, } latestMetrics.CPU = metric - return s.metricRepo.SaveCPUMetric(ctx, metric) + metrics := s.convertToMetrics(agentID, metricType, &cpuData, now) + return s.vmClient.Write(ctx, metrics) case protocol.MetricTypeMemory: - // Memory数据现在包含静态和动态信息 var memData protocol.MemoryData if err := json.Unmarshal(data, &memData); err != nil { return err } - metric := &models.MemoryMetric{ + metric := &MemoryMetric{ AgentID: agentID, - Total: memData.Total, // 现在可以从合并后的数据获取 + Total: memData.Total, Used: memData.Used, Free: memData.Free, Available: memData.Available, @@ -109,113 +117,49 @@ func (s *MetricService) HandleMetricData(ctx context.Context, agentID string, me Timestamp: now, } latestMetrics.Memory = metric - return s.metricRepo.SaveMemoryMetric(ctx, metric) + metrics := s.convertToMetrics(agentID, metricType, &memData, now) + return s.vmClient.Write(ctx, metrics) case protocol.MetricTypeDisk: - // Disk现在是数组,需要批量处理 var diskDataList []protocol.DiskData if err := json.Unmarshal(data, &diskDataList); err != nil { return err } - - // 合并所有磁盘的数据用于保存总和 + // 计算汇总数据用于缓存 var totalTotal, totalUsed, totalFree uint64 - - // 保存每个磁盘的数据，同时累加总和 for _, diskData := range diskDataList { - // 保存单个磁盘数据 - metric := &models.DiskMetric{ - AgentID: agentID, - MountPoint: diskData.MountPoint, - Total: diskData.Total, - Used: diskData.Used, - Free: diskData.Free, - UsagePercent: diskData.UsagePercent, - Timestamp: now, - } - if err := s.metricRepo.SaveDiskMetric(ctx, metric); err != nil { - s.logger.Error("failed to save disk metric", - zap.Error(err), - zap.String("agentID", agentID), - zap.String("mountPoint", diskData.MountPoint)) - } - - // 累加所有磁盘的数据 totalTotal += diskData.Total totalUsed += diskData.Used totalFree += diskData.Free } - - // 保存合并后的总和数据（mount_point 字段设置为空字符串） var usagePercent float64 if totalTotal > 0 { usagePercent = float64(totalUsed) / float64(totalTotal) * 100 } - totalMetric := &models.DiskMetric{ - AgentID: agentID, - MountPoint: "all", + latestMetrics.Disk = &DiskSummary{ + UsagePercent: usagePercent, + TotalDisks: len(diskDataList), Total: totalTotal, Used: totalUsed, Free: totalFree, - UsagePercent: usagePercent, - Timestamp: now, } - latestMetrics.Disk = &DiskSummary{ - UsagePercent: totalMetric.UsagePercent, - TotalDisks: len(diskDataList), - Total: totalMetric.Total, - Used: totalMetric.Used, - Free: totalMetric.Free, - } - return s.metricRepo.SaveDiskMetric(ctx, totalMetric) + metrics := s.convertToMetrics(agentID, metricType, diskDataList, now) + return s.vmClient.Write(ctx, metrics) case protocol.MetricTypeNetwork: - // Network现在是数组,需要批量处理 var networkDataList []protocol.NetworkData if err := json.Unmarshal(data, &networkDataList); err != nil { return err } - - // 合并所有网卡的数据用于保存总和 + // 计算汇总数据用于缓存 var totalSentRate, totalRecvRate uint64 var totalSentTotal, totalRecvTotal uint64 - - // 保存每个网卡的数据，同时累加总和 for _, netData := range networkDataList { - // 保存单个网卡数据 - metric := &models.NetworkMetric{ - AgentID: agentID, - Interface: netData.Interface, - BytesSentRate: netData.BytesSentRate, - BytesRecvRate: netData.BytesRecvRate, - BytesSentTotal: netData.BytesSentTotal, - BytesRecvTotal: netData.BytesRecvTotal, - Timestamp: now, - } - if err := s.metricRepo.SaveNetworkMetric(ctx, metric); err != nil { - s.logger.Error("failed to save network metric", - zap.Error(err), - zap.String("agentID", agentID), - zap.String("interface", netData.Interface)) - } - - // 累加所有网卡的数据 totalSentRate += netData.BytesSentRate totalRecvRate += netData.BytesRecvRate totalSentTotal += netData.BytesSentTotal totalRecvTotal += netData.BytesRecvTotal } - - // 保存合并后的总和数据（interface 字段设置为空字符串） - totalMetric := &models.NetworkMetric{ - AgentID: agentID, - Interface: "all", // 空字符串表示所有网卡的合并数据 - BytesSentRate: totalSentRate, - BytesRecvRate: totalRecvRate, - BytesSentTotal: totalSentTotal, - BytesRecvTotal: totalRecvTotal, - Timestamp: now, - } latestMetrics.Network = &NetworkSummary{ TotalBytesSentRate: totalSentRate, TotalBytesRecvRate: totalRecvRate, @@ -223,28 +167,21 @@ func (s *MetricService) HandleMetricData(ctx context.Context, agentID string, me TotalBytesRecvTotal: totalRecvTotal, TotalInterfaces: len(networkDataList), } - - // 保存网络指标 - if err := s.metricRepo.SaveNetworkMetric(ctx, totalMetric); err != nil { - return err - } - - // 更新流量统计(仅处理汇总数据) + // 更新流量统计 if err := s.trafficService.UpdateAgentTraffic(ctx, agentID, totalRecvTotal); err != nil { s.logger.Error("更新探针流量统计失败", zap.String("agentId", agentID), zap.Error(err)) - // 流量统计失败不影响指标保存，只记录日志 } - - return nil + metrics := s.convertToMetrics(agentID, metricType, networkDataList, now) + return s.vmClient.Write(ctx, metrics) case protocol.MetricTypeNetworkConnection: var connData protocol.NetworkConnectionData if err := json.Unmarshal(data, &connData); err != nil { return err } - metric := &models.NetworkConnectionMetric{ + metric := &NetworkConnectionMetric{ AgentID: agentID, Established: connData.Established, SynSent: connData.SynSent, @@ -261,61 +198,23 @@ func (s *MetricService) HandleMetricData(ctx context.Context, agentID string, me Timestamp: now, } latestMetrics.NetworkConnection = metric - return s.metricRepo.SaveNetworkConnectionMetric(ctx, metric) + metrics := s.convertToMetrics(agentID, metricType, &connData, now) + return s.vmClient.Write(ctx, metrics) case protocol.MetricTypeDiskIO: - // DiskIO现在是数组，直接合并所有磁盘的数据存储为一条记录 var diskIODataList []*protocol.DiskIOData if err := json.Unmarshal(data, &diskIODataList); err != nil { return err } - - // 合并所有磁盘的数据 - var totalReadCount, totalWriteCount uint64 - var totalReadBytes, totalWriteBytes uint64 - var totalReadBytesRate, totalWriteBytesRate uint64 - var totalReadTime, totalWriteTime, totalIoTime uint64 - var maxIopsInProgress uint64 - - for _, diskIOData := range diskIODataList { - totalReadCount += diskIOData.ReadCount - totalWriteCount += diskIOData.WriteCount - totalReadBytes += diskIOData.ReadBytes - totalWriteBytes += diskIOData.WriteBytes - totalReadBytesRate += diskIOData.ReadBytesRate - totalWriteBytesRate += diskIOData.WriteBytesRate - totalReadTime += diskIOData.ReadTime - totalWriteTime += diskIOData.WriteTime - totalIoTime += diskIOData.IoTime - if diskIOData.IopsInProgress > maxIopsInProgress { - maxIopsInProgress = diskIOData.IopsInProgress - } - } - - // 保存合并后的数据（device 字段设置为空或 "all"） - metric := &models.DiskIOMetric{ - AgentID: agentID, - Device: "all", - ReadCount: totalReadCount, - WriteCount: totalWriteCount, - ReadBytes: totalReadBytes, - WriteBytes: totalWriteBytes, - ReadBytesRate: totalReadBytesRate, - WriteBytesRate: totalWriteBytesRate, - ReadTime: totalReadTime, - WriteTime: totalWriteTime, - IoTime: totalIoTime, - IopsInProgress: maxIopsInProgress, - Timestamp: now, - } - return s.metricRepo.SaveDiskIOMetric(ctx, metric) + metrics := s.convertToMetrics(agentID, metricType, diskIODataList, now) + return s.vmClient.Write(ctx, metrics) case protocol.MetricTypeHost: var hostData protocol.HostInfoData if err := json.Unmarshal(data, &hostData); err != nil { return err } - // 保存主机信息 + // Host 信息仍然保存到 PostgreSQL（静态信息，不频繁变化） metric := &models.HostMetric{ AgentID: agentID, OS: hostData.OS, @@ -332,15 +231,14 @@ func (s *MetricService) HandleMetricData(ctx context.Context, agentID string, me return s.metricRepo.SaveHostMetric(ctx, metric) case protocol.MetricTypeGPU: - // GPU现在是数组,需要批量处理 var gpuDataList []protocol.GPUData if err := json.Unmarshal(data, &gpuDataList); err != nil { return err } - // 保存每个GPU的数据 - var gpuMetrics []models.GPUMetric + // 更新缓存 + var gpuMetrics []GPUMetric for _, gpuData := range gpuDataList { - metric := models.GPUMetric{ + gpuMetrics = append(gpuMetrics, GPUMetric{ AgentID: agentID, Index: gpuData.Index, Name: gpuData.Name, @@ -349,86 +247,47 @@ func (s *MetricService) HandleMetricData(ctx context.Context, agentID string, me MemoryUsed: gpuData.MemoryUsed, MemoryFree: gpuData.MemoryFree, Temperature: gpuData.Temperature, - PowerDraw: gpuData.PowerUsage, // protocol 中是 PowerUsage + PowerDraw: gpuData.PowerUsage, FanSpeed: gpuData.FanSpeed, - PerformanceState: "", // protocol 中没有这个字段，留空 + PerformanceState: "", Timestamp: now, - } - gpuMetrics = append(gpuMetrics, metric) - if err := s.metricRepo.SaveGPUMetric(ctx, &metric); err != nil { - s.logger.Error("failed to save gpu metric", - zap.Error(err), - zap.String("agentID", agentID), - zap.Int("index", gpuData.Index)) - } + }) } latestMetrics.GPU = gpuMetrics - return nil + metrics := s.convertToMetrics(agentID, metricType, gpuDataList, now) + return s.vmClient.Write(ctx, metrics) case protocol.MetricTypeTemperature: - // Temperature现在是数组,需要批量处理 var tempDataList []protocol.TemperatureData if err := json.Unmarshal(data, &tempDataList); err != nil { return err } - // 保存每个温度传感器的数据 - var tempMetrics []models.TemperatureMetric + // 更新缓存 + var tempMetrics []TemperatureMetric for _, tempData := range tempDataList { - // 使用 Type 作为 SensorLabel，更易读 sensorLabel := tempData.Type if sensorLabel == "" { - sensorLabel = tempData.SensorKey // 降级使用 SensorKey + sensorLabel = tempData.SensorKey } - - metric := models.TemperatureMetric{ + tempMetrics = append(tempMetrics, TemperatureMetric{ AgentID: agentID, SensorKey: tempData.SensorKey, SensorLabel: sensorLabel, Temperature: tempData.Temperature, Timestamp: now, - } - tempMetrics = append(tempMetrics, metric) - if err := s.metricRepo.SaveTemperatureMetric(ctx, &metric); err != nil { - s.logger.Error("failed to save temperature metric", - zap.Error(err), - zap.String("agentID", agentID), - zap.String("sensor", tempData.SensorKey)) - } + }) } latestMetrics.Temp = tempMetrics - return nil + metrics := s.convertToMetrics(agentID, metricType, tempDataList, now) + return s.vmClient.Write(ctx, metrics) case protocol.MetricTypeMonitor: - // 监控数据也是数组,需要批量处理 var monitorDataList []protocol.MonitorData if err := json.Unmarshal(data, &monitorDataList); err != nil { return err } - // 保存每个监控项的数据 - for _, monitorData := range monitorDataList { - metric := &models.MonitorMetric{ - AgentId: agentID, - MonitorId: monitorData.ID, - Type: monitorData.Type, - Target: monitorData.Target, - Status: monitorData.Status, - StatusCode: monitorData.StatusCode, - ResponseTime: monitorData.ResponseTime, - Error: monitorData.Error, - Message: monitorData.Message, - ContentMatch: monitorData.ContentMatch, - CertExpiryTime: monitorData.CertExpiryTime, - CertDaysLeft: monitorData.CertDaysLeft, - Timestamp: monitorData.CheckedAt, // 使用检测时间 - } - if err := s.metricRepo.SaveMonitorMetric(ctx, metric); err != nil { - s.logger.Error("failed to save monitor metric", - zap.Error(err), - zap.String("agentID", agentID), - zap.String("MonitorId", monitorData.ID)) - } - } - return nil + metrics := s.convertToMetrics(agentID, metricType, monitorDataList, now) + return s.vmClient.Write(ctx, metrics) default: s.logger.Warn("unknown metric type", zap.String("type", metricType)) @@ -436,197 +295,42 @@ func (s *MetricService) HandleMetricData(ctx context.Context, agentID string, me } } -// GetMetrics 获取聚合指标数据（自动路由到聚合表或原始表） -// interfaceName: 网卡过滤参数（仅对 network 类型有效） -func (s *MetricService) GetMetrics(ctx context.Context, agentID, metricType string, start, end int64, interval int, interfaceName string) (interface{}, error) { - start, end = s.normalizeTimeRange(ctx, start, end) - interval = s.DetermineInterval(ctx, start, end, interval) - - // 判断是否可以使用聚合表（仅支持部分指标类型） - aggCapable := map[string]bool{ - "cpu": true, - "memory": true, - "disk": true, - "network": true, - "network_connection": true, - "disk_io": true, - "gpu": true, - "temperature": true, +// GetMetrics 获取聚合指标数据（从 VictoriaMetrics 查询） +// 返回统一的 GetMetricsResponse 格式 +func (s *MetricService) GetMetrics(ctx context.Context, agentID, metricType string, start, end int64, interfaceName string) (*GetMetricsResponse, error) { + // 构造 PromQL 查询（返回多个查询以支持多系列） + queries := s.buildPromQLQueries(agentID, metricType, interfaceName) + if len(queries) == 0 { + return nil, fmt.Errorf("unsupported metric type: %s", metricType) } - // 智能选择聚合粒度：根据查询间隔选择最合适的bucket - // 例如：查询90秒数据时使用60秒bucket，查询600秒数据时使用300秒bucket - var bucketSeconds int - useAgg := false - if aggCapable[metricType] { - bucketSeconds = chooseAggregationBucket(interval) - useAgg = bucketSeconds > 0 - } + // 执行查询并转换结果 + // step 设为 0，让 VictoriaMetrics 自动选择合适的步长 + var series []MetricSeries - // 将时间范围对齐到最终使用的 bucket，避免不同时间框架出现桶数量偏差 - bucketMs := int64(interval * 1000) - if useAgg { - bucketMs = int64(bucketSeconds * 1000) - } - start, end = alignTimeRangeToBucket(start, end, bucketMs) - - switch metricType { - case "cpu": - if useAgg { - if metrics, err := s.metricRepo.GetCPUMetricsAgg(ctx, agentID, start, end, bucketSeconds); err == nil && len(metrics) > 0 { - return metrics, nil - } + for _, q := range queries { + result, err := s.vmClient.QueryRange(ctx, q.Query, + time.UnixMilli(start), + time.UnixMilli(end), + 0) + if err != nil { + s.logger.Error("查询 VictoriaMetrics 失败", + zap.String("query", q.Query), + zap.Error(err)) + continue // 跳过失败的查询，继续处理其他查询 } - return s.metricRepo.GetCPUMetrics(ctx, agentID, start, end, interval) - case "memory": - if useAgg { - if metrics, err := s.metricRepo.GetMemoryMetricsAgg(ctx, agentID, start, end, bucketSeconds); err == nil && len(metrics) > 0 { - return metrics, nil - } - } - return s.metricRepo.GetMemoryMetrics(ctx, agentID, start, end, interval) - case "disk": - if useAgg { - if metrics, err := s.metricRepo.GetDiskMetricsAgg(ctx, agentID, start, end, bucketSeconds); err == nil && len(metrics) > 0 { - return metrics, nil - } - } - return s.metricRepo.GetDiskMetrics(ctx, agentID, start, end, interval) - case "network": - if useAgg { - if metrics, err := s.metricRepo.GetNetworkMetricsAgg(ctx, agentID, start, end, bucketSeconds, interfaceName); err == nil && len(metrics) > 0 { - return metrics, nil - } - } - return s.metricRepo.GetNetworkMetrics(ctx, agentID, start, end, interval, interfaceName) - case "network_connection": - if useAgg { - if metrics, err := s.metricRepo.GetNetworkConnectionMetricsAgg(ctx, agentID, start, end, bucketSeconds); err == nil && len(metrics) > 0 { - return metrics, nil - } - } - return s.metricRepo.GetNetworkConnectionMetrics(ctx, agentID, start, end, interval) - case "disk_io": - if useAgg { - if metrics, err := s.metricRepo.GetDiskIOMetricsAgg(ctx, agentID, start, end, bucketSeconds); err == nil && len(metrics) > 0 { - return metrics, nil - } - } - return s.metricRepo.GetDiskIOMetrics(ctx, agentID, start, end, interval) - case "gpu": - if useAgg { - if metrics, err := s.metricRepo.GetGPUMetricsAgg(ctx, agentID, start, end, bucketSeconds); err == nil && len(metrics) > 0 { - return metrics, nil - } - } - return s.metricRepo.GetGPUMetrics(ctx, agentID, start, end, interval) - case "temperature": - if useAgg { - if metrics, err := s.metricRepo.GetTemperatureMetricsAgg(ctx, agentID, start, end, bucketSeconds); err == nil && len(metrics) > 0 { - return metrics, nil - } - } - return s.metricRepo.GetTemperatureMetrics(ctx, agentID, start, end, interval) - default: - return nil, nil - } -} -// DetermineInterval 根据配置、用户请求和时间范围决定聚合粒度 -func (s *MetricService) DetermineInterval(ctx context.Context, start, end int64, requested int) int { - interval := requested - if interval <= 0 { - interval = calculateBaseInterval(start, end) + // 转换查询结果为 MetricSeries + convertedSeries := s.convertQueryResultToSeries(result, q.Name, q.Labels) + series = append(series, convertedSeries...) } - interval = adjustIntervalForMaxPoints(start, end, interval, defaultMaxQueryPoints) - return interval -} - -// normalizeTimeRange 将时间范围限制在保留周期内，避免无意义的全表扫描 -func (s *MetricService) normalizeTimeRange(ctx context.Context, start, end int64) (int64, int64) { - cfg := s.getMetricsConfig(ctx) - retentionDuration := time.Duration(cfg.RetentionHours) * time.Hour - - retentionBoundary := time.Now().Add(-retentionDuration).UnixMilli() - if start < retentionBoundary { - start = retentionBoundary - } - if end <= start { - end = start + 1000 - } - return start, end -} - -// calculateBaseInterval 根据时间范围计算基础间隔 -func calculateBaseInterval(start, end int64) int { - duration := (end - start) / 1000 // 秒 - - switch { - case duration <= 60: // 1 分钟内 - return 2 - case duration <= 5*60: // 5 分钟内 - return 5 - case duration <= 15*60: // 15 分钟内 - return 15 - case duration <= 30*60: // 30 分钟内 - return 30 - case duration <= 60*60: // 1 小时内 - return 60 - case duration <= 3*60*60: // 3 小时内 - return 180 - case duration <= 6*60*60: // 6 小时内 - return 300 - case duration <= 12*60*60: // 12 小时内 - return 600 - case duration <= 24*60*60: // 1 天内 - return 900 - case duration <= 3*24*60*60: // 3 天内 - return 1800 - case duration <= 7*24*60*60: // 7 天内 - return 3600 - case duration <= 14*24*60*60: // 14 天内 - return 7200 - default: - return 14400 // 更长时间：4 小时粒度 - } -} - -// adjustIntervalForMaxPoints 根据最大数据点限制提升聚合粒度 -func adjustIntervalForMaxPoints(start, end int64, interval int, maxPoints int) int { - if interval <= 0 { - interval = 1 - } - if maxPoints <= 0 { - return alignInterval(interval) - } - - durationSeconds := float64(end-start) / 1000 - if durationSeconds <= 0 { - return alignInterval(interval) - } - - required := int(math.Ceil(durationSeconds / float64(maxPoints))) - interval = maxInt(interval, required) - return alignInterval(interval) -} - -// alignInterval 将间隔对齐到允许的值 -func alignInterval(interval int) int { - for _, candidate := range allowedIntervals { - if interval <= candidate { - return candidate - } - } - return allowedIntervals[len(allowedIntervals)-1] -} - -// maxInt 返回两个整数的最大值 -func maxInt(a, b int) int { - if a > b { - return a - } - return b + return &GetMetricsResponse{ + AgentID: agentID, + Type: metricType, + Range: fmt.Sprintf("%d-%d", start, end), + Series: series, + }, nil } // alignTimeRangeToBucket 将时间范围对齐到桶边界，确保不同时间框架的桶数一致 @@ -643,143 +347,6 @@ func alignTimeRangeToBucket(start, end int64, bucketMs int64) (int64, int64) { return alignedStart, alignedEnd } -// chooseAggregationBucket 根据请求的 interval 选择最合适的聚合 bucket（取不小于 interval 的最小桶） -func chooseAggregationBucket(interval int) int { - if interval <= 0 { - return 0 - } - for _, bucket := range aggregationBuckets { - if interval <= bucket { - return bucket - } - } - return aggregationBuckets[len(aggregationBuckets)-1] -} - -// getMetricsConfig 获取指标配置 -func (s *MetricService) getMetricsConfig(ctx context.Context) models.MetricsConfig { - cfg := models.MetricsConfig{ - RetentionHours: defaultMetricsRetentionHours, - } - - if s.propertyService == nil { - return cfg - } - - loaded := s.propertyService.GetMetricsConfig(ctx) - if loaded.RetentionHours > 0 { - cfg.RetentionHours = loaded.RetentionHours - } - return cfg -} - -// StartAggregationTask 启动聚合下采样任务 -func (s *MetricService) StartAggregationTask(ctx context.Context) { - ticker := time.NewTicker(1 * time.Minute) - defer ticker.Stop() - - s.logger.Info("aggregation task started") - - for { - select { - case <-ctx.Done(): - s.logger.Info("aggregation task stopped") - return - case <-ticker.C: - s.runAggregation(ctx) - } - } -} - -// runAggregation 按固定 bucket 下采样存储 -func (s *MetricService) runAggregation(ctx context.Context) { - cfg := s.getMetricsConfig(ctx) - retention := time.Duration(cfg.RetentionHours) * time.Hour - - for _, bucket := range aggregationBuckets { - s.aggregateMetric(ctx, "cpu", bucket, retention, s.metricRepo.AggregateCPUToAgg) - s.aggregateMetric(ctx, "memory", bucket, retention, s.metricRepo.AggregateMemoryToAgg) - s.aggregateMetric(ctx, "disk", bucket, retention, s.metricRepo.AggregateDiskToAgg) - s.aggregateMetric(ctx, "network", bucket, retention, s.metricRepo.AggregateNetworkToAgg) - s.aggregateMetric(ctx, "network_connection", bucket, retention, s.metricRepo.AggregateNetworkConnectionToAgg) - s.aggregateMetric(ctx, "disk_io", bucket, retention, s.metricRepo.AggregateDiskIOToAgg) - s.aggregateMetric(ctx, "gpu", bucket, retention, s.metricRepo.AggregateGPUToAgg) - s.aggregateMetric(ctx, "temperature", bucket, retention, s.metricRepo.AggregateTemperatureToAgg) - s.aggregateMetric(ctx, "monitor", bucket, retention, s.metricRepo.AggregateMonitorMetricsToAgg) - } -} - -type aggregateFn func(ctx context.Context, bucketSeconds int, start, end int64) error - -// aggregateMetric 聚合指定类型的指标 -func (s *MetricService) aggregateMetric(ctx context.Context, metricType string, bucketSeconds int, retention time.Duration, fn aggregateFn) { - bucketMs := int64(bucketSeconds * 1000) - - start := s.getAggregationStart(ctx, metricType, bucketSeconds, retention, bucketMs) - endBucket := (time.Now().Add(-time.Duration(bucketSeconds)*time.Second).UnixMilli() / bucketMs) * bucketMs - - if endBucket <= start { - return - } - - end := endBucket + bucketMs - 1 - - if err := fn(ctx, bucketSeconds, start, end); err != nil { - s.logger.Error("aggregate metric failed", zap.String("metricType", metricType), zap.Int("bucketSeconds", bucketSeconds), zap.Error(err)) - return - } - - if err := s.metricRepo.UpsertAggregationProgress(ctx, metricType, bucketSeconds, endBucket); err != nil { - s.logger.Error("update aggregation progress failed", zap.String("metricType", metricType), zap.Int("bucketSeconds", bucketSeconds), zap.Error(err)) - } -} - -// getAggregationStart 获取聚合开始时间 -func (s *MetricService) getAggregationStart(ctx context.Context, metricType string, bucketSeconds int, retention time.Duration, bucketMs int64) int64 { - progress, err := s.metricRepo.GetAggregationProgress(ctx, metricType, bucketSeconds) - if err == nil && progress != nil && progress.LastBucket > 0 { - return progress.LastBucket + bucketMs - } - - // 默认从保留窗口开始 - start := time.Now().Add(-retention).UnixMilli() - return (start / bucketMs) * bucketMs -} - -// StartCleanupTask 启动数据清理任务 -func (s *MetricService) StartCleanupTask(ctx context.Context) { - ticker := time.NewTicker(1 * time.Minute) - defer ticker.Stop() - - s.logger.Info("cleanup task started") - - for { - select { - case <-ctx.Done(): - s.logger.Info("cleanup task stopped") - return - case <-ticker.C: - s.cleanupOldMetrics(ctx) - } - } -} - -// cleanupOldMetrics 清理旧数据 -func (s *MetricService) cleanupOldMetrics(ctx context.Context) { - cfg := s.getMetricsConfig(ctx) - retentionDuration := time.Duration(cfg.RetentionHours) * time.Hour - before := time.Now().Add(-retentionDuration).UnixMilli() - - s.logger.Info("starting to clean old metrics", zap.Int64("beforeTimestamp", before), zap.Int("retentionHours", cfg.RetentionHours)) - - if err := s.metricRepo.DeleteOldMetrics(ctx, before); err != nil { - s.logger.Error("failed to clean old metrics", zap.Error(err)) - return - } - - s.logger.Info("old metrics cleaned successfully") -} - // GetLatestMetrics 获取最新指标 func (s *MetricService) GetLatestMetrics(ctx context.Context, agentID string) (*LatestMetrics, error) { metrics, _ := s.latestCache.Get(agentID) @@ -787,23 +354,168 @@ func (s *MetricService) GetLatestMetrics(ctx context.Context, agentID string) (* } // GetMonitorMetrics 获取监控指标历史数据 -func (s *MetricService) GetMonitorMetrics(ctx context.Context, agentID, monitorName string, start, end int64) ([]models.MonitorMetric, error) { - return s.metricRepo.GetMonitorMetrics(ctx, agentID, monitorName, start, end) +// TODO: 需要重写为从 VictoriaMetrics 查询 +func (s *MetricService) GetMonitorMetrics(ctx context.Context, agentID, monitorName string, start, end int64) ([]MonitorMetric, error) { + // 暂时返回空数据，后续从 VictoriaMetrics 查询 + return []MonitorMetric{}, nil } // GetMonitorMetricsByName 获取指定监控项的历史数据 -func (s *MetricService) GetMonitorMetricsByName(ctx context.Context, agentID, monitorName string, start, end int64, limit int) ([]models.MonitorMetric, error) { - return s.metricRepo.GetMonitorMetricsByName(ctx, agentID, monitorName, start, end, limit) +// TODO: 需要重写为从 VictoriaMetrics 查询 +func (s *MetricService) GetMonitorMetricsByName(ctx context.Context, agentID, monitorName string, start, end int64, limit int) ([]MonitorMetric, error) { + // 暂时返回空数据，后续从 VictoriaMetrics 查询 + return []MonitorMetric{}, nil } // DeleteAgentMetrics 删除探针的所有指标数据 func (s *MetricService) DeleteAgentMetrics(ctx context.Context, agentID string) error { - return s.metricRepo.DeleteAgentMetrics(ctx, agentID) + // 1. 删除 PostgreSQL 中的主机信息 + if err := s.metricRepo.DeleteAgentMetrics(ctx, agentID); err != nil { + s.logger.Error("删除 PostgreSQL 中的探针数据失败", + zap.String("agentID", agentID), + zap.Error(err)) + // 继续删除 VictoriaMetrics 中的数据 + } + + // 2. 删除 VictoriaMetrics 中的时间序列数据 + match := []string{fmt.Sprintf(`pika_.*{agent_id="%s"}`, agentID)} + if err := s.vmClient.DeleteSeries(ctx, match); err != nil { + s.logger.Error("删除 VictoriaMetrics 中的探针数据失败", + zap.String("agentID", agentID), + zap.Error(err)) + return err + } + + s.logger.Info("成功删除探针的所有指标数据", + zap.String("agentID", agentID)) + return nil } -// GetAvailableNetworkInterfaces 获取探针的可用网卡列表 +// DeleteMonitorMetrics 删除指定监控任务的所有指标数据 +func (s *MetricService) DeleteMonitorMetrics(ctx context.Context, monitorID string) error { + // 删除 VictoriaMetrics 中的监控指标数据 + match := []string{fmt.Sprintf(`pika_monitor_.*{monitor_id="%s"}`, monitorID)} + if err := s.vmClient.DeleteSeries(ctx, match); err != nil { + s.logger.Error("删除 VictoriaMetrics 中的监控数据失败", + zap.String("monitorID", monitorID), + zap.Error(err)) + return err + } + + s.logger.Info("成功删除监控任务的所有指标数据", + zap.String("monitorID", monitorID)) + return nil +} + +// GetAvailableNetworkInterfaces 获取探针的可用网卡列表（从 VictoriaMetrics 查询） func (s *MetricService) GetAvailableNetworkInterfaces(ctx context.Context, agentID string) ([]string, error) { - return s.metricRepo.GetAvailableNetworkInterfaces(ctx, agentID) + // 查询 interface label 的所有值，排除空字符串（汇总数据） + match := []string{fmt.Sprintf(`pika_network_sent_bytes_rate{agent_id="%s"}`, agentID)} + allInterfaces, err := s.vmClient.GetLabelValues(ctx, "interface", match) + if err != nil { + s.logger.Error("查询网卡列表失败", + zap.String("agentID", agentID), + zap.Error(err)) + return []string{}, nil // 返回空列表而不是错误 + } + + // 过滤掉空字符串（汇总数据） + interfaces := make([]string, 0, len(allInterfaces)) + for _, iface := range allInterfaces { + if iface != "" { + interfaces = append(interfaces, iface) + } + } + + return interfaces, nil +} + +// ===== 内存缓存使用的本地模型定义 ===== +// 注意：这些模型仅用于内存缓存，不再保存到 PostgreSQL + +// CPUMetric CPU指标（内存缓存） +type CPUMetric struct { + AgentID string `json:"agentId"` + UsagePercent float64 `json:"usagePercent"` + LogicalCores int `json:"logicalCores"` + PhysicalCores int `json:"physicalCores"` + ModelName string `json:"modelName"` + Timestamp int64 `json:"timestamp"` +} + +// MemoryMetric 内存指标（内存缓存） +type MemoryMetric struct { + AgentID string `json:"agentId"` + Total uint64 `json:"total"` + Used uint64 `json:"used"` + Free uint64 `json:"free"` + Available uint64 `json:"available"` + UsagePercent float64 `json:"usagePercent"` + SwapTotal uint64 `json:"swapTotal"` + SwapUsed uint64 `json:"swapUsed"` + SwapFree uint64 `json:"swapFree"` + Timestamp int64 `json:"timestamp"` +} + +// NetworkConnectionMetric 网络连接统计指标（内存缓存） +type NetworkConnectionMetric struct { + AgentID string `json:"agentId"` + Established uint32 `json:"established"` + SynSent uint32 `json:"synSent"` + SynRecv uint32 `json:"synRecv"` + FinWait1 uint32 `json:"finWait1"` + FinWait2 uint32 `json:"finWait2"` + TimeWait uint32 `json:"timeWait"` + Close uint32 `json:"close"` + CloseWait uint32 `json:"closeWait"` + LastAck uint32 `json:"lastAck"` + Listen uint32 `json:"listen"` + Closing uint32 `json:"closing"` + Total uint32 `json:"total"` + Timestamp int64 `json:"timestamp"` +} + +// GPUMetric GPU指标（内存缓存） +type GPUMetric struct { + AgentID string `json:"agentId"` + Index int `json:"index"` + Name string `json:"name"` + Utilization float64 `json:"utilization"` + MemoryTotal uint64 `json:"memoryTotal"` + MemoryUsed uint64 `json:"memoryUsed"` + MemoryFree uint64 `json:"memoryFree"` + Temperature float64 `json:"temperature"` + PowerDraw float64 `json:"powerDraw"` + FanSpeed float64 `json:"fanSpeed"` + PerformanceState string `json:"performanceState"` + Timestamp int64 `json:"timestamp"` +} + +// TemperatureMetric 温度指标（内存缓存） +type TemperatureMetric struct { + AgentID string `json:"agentId"` + SensorKey string `json:"sensorKey"` + SensorLabel string `json:"sensorLabel"` + Temperature float64 `json:"temperature"` + Timestamp int64 `json:"timestamp"` +} + +// MonitorMetric 监控指标 +type MonitorMetric struct { + ID uint `json:"id"` + AgentId string `json:"agentId"` + MonitorId string `json:"monitorId"` + Type string `json:"type"` + Target string `json:"target"` + Status string `json:"status"` + StatusCode int `json:"statusCode"` + ResponseTime int64 `json:"responseTime"` + Error string `json:"error"` + Message string `json:"message"` + ContentMatch bool `json:"contentMatch"` + CertExpiryTime int64 `json:"certExpiryTime"` + CertDaysLeft int `json:"certDaysLeft"` + Timestamp int64 `json:"timestamp"` } // DiskSummary 磁盘汇总数据 @@ -826,12 +538,174 @@ type NetworkSummary struct { // LatestMetrics 最新指标数据（用于API响应） type LatestMetrics struct { - CPU *models.CPUMetric `json:"cpu,omitempty"` - Memory *models.MemoryMetric `json:"memory,omitempty"` - Disk *DiskSummary `json:"disk,omitempty"` - Network *NetworkSummary `json:"network,omitempty"` - NetworkConnection *models.NetworkConnectionMetric `json:"networkConnection,omitempty"` - Host *models.HostMetric `json:"host,omitempty"` - GPU []models.GPUMetric `json:"gpu,omitempty"` - Temp []models.TemperatureMetric `json:"temperature,omitempty"` + CPU *CPUMetric `json:"cpu,omitempty"` + Memory *MemoryMetric `json:"memory,omitempty"` + Disk *DiskSummary `json:"disk,omitempty"` + Network *NetworkSummary `json:"network,omitempty"` + NetworkConnection *NetworkConnectionMetric `json:"networkConnection,omitempty"` + Host *models.HostMetric `json:"host,omitempty"` + GPU []GPUMetric `json:"gpu,omitempty"` + Temp []TemperatureMetric `json:"temperature,omitempty"` +} + +// buildPromQLQueries 构造 PromQL 查询列表（支持多系列） +func (s *MetricService) buildPromQLQueries(agentID, metricType string, interfaceName string) []QueryDefinition { + var queries []QueryDefinition + + switch metricType { + case "cpu": + queries = []QueryDefinition{{ + Name: "usage", + Query: fmt.Sprintf(`pika_cpu_usage_percent{agent_id="%s"}`, agentID), + }} + + case "memory": + queries = []QueryDefinition{{ + Name: "usage", + Query: fmt.Sprintf(`pika_memory_usage_percent{agent_id="%s"}`, agentID), + }} + + case "disk": + queries = []QueryDefinition{{ + Name: "usage", + Query: fmt.Sprintf(`pika_disk_usage_percent{agent_id="%s",mount_point=""}`, agentID), + }} + + case "network": + // 网络流量：上行和下行 + if interfaceName != "" && interfaceName != "all" { + // 指定网卡 + queries = []QueryDefinition{ + { + Name: "upload", + Query: fmt.Sprintf(`pika_network_sent_bytes_rate{agent_id="%s",interface="%s"}`, agentID, interfaceName), + Labels: map[string]string{"interface": interfaceName}, + }, + { + Name: "download", + Query: fmt.Sprintf(`pika_network_recv_bytes_rate{agent_id="%s",interface="%s"}`, agentID, interfaceName), + Labels: map[string]string{"interface": interfaceName}, + }, + } + } else { + // 所有网卡汇总 + queries = []QueryDefinition{ + { + Name: "upload", + Query: fmt.Sprintf(`sum(pika_network_sent_bytes_rate{agent_id="%s"}) by (agent_id)`, agentID), + }, + { + Name: "download", + Query: fmt.Sprintf(`sum(pika_network_recv_bytes_rate{agent_id="%s"}) by (agent_id)`, agentID), + }, + } + } + + case "network_connection": + // 网络连接统计：多个状态 + queries = []QueryDefinition{ + {Name: "established", Query: fmt.Sprintf(`pika_network_conn_established{agent_id="%s"}`, agentID)}, + {Name: "time_wait", Query: fmt.Sprintf(`pika_network_conn_time_wait{agent_id="%s"}`, agentID)}, + {Name: "close_wait", Query: fmt.Sprintf(`pika_network_conn_close_wait{agent_id="%s"}`, agentID)}, + {Name: "listen", Query: fmt.Sprintf(`pika_network_conn_total{agent_id="%s"}`, agentID)}, + } + + case "disk_io": + // 磁盘 IO：读和写 + queries = []QueryDefinition{ + {Name: "read", Query: fmt.Sprintf(`pika_disk_read_bytes_rate{agent_id="%s"}`, agentID)}, + {Name: "write", Query: fmt.Sprintf(`pika_disk_write_bytes_rate{agent_id="%s"}`, agentID)}, + } + + case "gpu": + // GPU：利用率和温度（按 GPU 分组） + queries = []QueryDefinition{ + { + Name: "utilization", + Query: fmt.Sprintf(`pika_gpu_utilization_percent{agent_id="%s"}`, agentID), + }, + { + Name: "temperature", + Query: fmt.Sprintf(`pika_gpu_temperature_celsius{agent_id="%s"}`, agentID), + }, + } + + case "temperature": + // 温度：按传感器类型分组 + queries = []QueryDefinition{{ + Name: "temperature", + Query: fmt.Sprintf(`pika_temperature_celsius{agent_id="%s"}`, agentID), + }} + } + + return queries +} + +// convertQueryResultToSeries 将 VictoriaMetrics 查询结果转换为 MetricSeries +func (s *MetricService) convertQueryResultToSeries(result *vmclient.QueryResult, seriesName string, extraLabels map[string]string) []MetricSeries { + if result == nil || len(result.Data.Result) == 0 { + return []MetricSeries{} + } + + var allSeries []MetricSeries + + // 遍历所有时间序列 + for _, timeSeries := range result.Data.Result { + // 提取数据点 + var dataPoints []MetricDataPoint + for _, valueArray := range timeSeries.Values { + if len(valueArray) != 2 { + continue + } + + // valueArray: [timestamp(float64), value(string)] + timestamp, ok := valueArray[0].(float64) + if !ok { + continue + } + valueStr, ok := valueArray[1].(string) + if !ok { + continue + } + + var value float64 + fmt.Sscanf(valueStr, "%f", &value) + + dataPoints = append(dataPoints, MetricDataPoint{ + Timestamp: int64(timestamp * 1000), // 转换为毫秒 + Value: value, + }) + } + + // 合并标签 + labels := make(map[string]string) + for k, v := range timeSeries.Metric { + // 排除内部标签 + if k != "__name__" && k != "agent_id" { + labels[k] = v + } + } + // 添加额外标签 + for k, v := range extraLabels { + labels[k] = v + } + + // 构建系列名称（如果有特定标签如 GPU index 或 sensor_label，添加到名称中） + finalName := seriesName + if sensorLabel, ok := labels["sensor_label"]; ok { + finalName = sensorLabel + delete(labels, "sensor_label") // 已合并到名称中，从标签中删除 + } else if gpuIndex, ok := labels["gpu_index"]; ok { + finalName = fmt.Sprintf("GPU_%s", gpuIndex) + delete(labels, "gpu_index") + } + + allSeries = append(allSeries, MetricSeries{ + Name: finalName, + Labels: labels, + Data: dataPoints, + }) + } + + return allSeries } diff --git a/internal/service/monitor_service.go b/internal/service/monitor_service.go index 6db542b..af18dff 100644 --- a/internal/service/monitor_service.go +++ b/internal/service/monitor_service.go @@ -26,6 +26,7 @@ type MonitorService struct { *orz.Service agentRepo *repo.AgentRepo metricRepo *repo.MetricRepo + metricService *MetricService monitorStatsRepo *repo.MonitorStatsRepo wsManager *ws.Manager @@ -45,13 +46,14 @@ type MonitorScheduler interface { RemoveTask(monitorID string) } -func NewMonitorService(logger *zap.Logger, db *gorm.DB, wsManager *ws.Manager) *MonitorService { +func NewMonitorService(logger *zap.Logger, db *gorm.DB, metricService *MetricService, wsManager *ws.Manager) *MonitorService { return &MonitorService{ logger: logger, Service: orz.NewService(db), MonitorRepo: repo.NewMonitorRepo(db), agentRepo: repo.NewAgentRepo(db), metricRepo: repo.NewMetricRepo(db), + metricService: metricService, monitorStatsRepo: repo.NewMonitorStatsRepo(db), wsManager: wsManager, @@ -233,8 +235,8 @@ func (s *MonitorService) DeleteMonitor(ctx context.Context, id string) error { return err } - // 删除监控指标数据 - if err := s.metricRepo.DeleteMonitorMetrics(ctx, id); err != nil { + // 删除监控指标数据（从 VictoriaMetrics） + if err := s.metricService.DeleteMonitorMetrics(ctx, id); err != nil { s.logger.Error("删除监控指标数据失败", zap.String("monitorId", id), zap.Error(err)) return err } diff --git a/internal/service/property_service.go b/internal/service/property_service.go index c1354ee..43e3875 100644 --- a/internal/service/property_service.go +++ b/internal/service/property_service.go @@ -120,22 +120,6 @@ func (s *PropertyService) GetSystemConfig(ctx context.Context) (*models.SystemCo return &systemConfig, nil } -// GetMetricsConfig 获取指标配置 -func (s *PropertyService) GetMetricsConfig(ctx context.Context) models.MetricsConfig { - var config models.MetricsConfig - err := s.GetValue(ctx, PropertyIDMetricsConfig, &config) - if err != nil { - // 返回默认配置 - return models.MetricsConfig{} - } - return config -} - -// SetMetricsConfig 设置指标配置 -func (s *PropertyService) SetMetricsConfig(ctx context.Context, config models.MetricsConfig) error { - return s.Set(ctx, PropertyIDMetricsConfig, "指标数据配置", config) -} - // GetAlertConfig 获取告警配置 func (s *PropertyService) GetAlertConfig(ctx context.Context) (*models.AlertConfig, error) { var config models.AlertConfig @@ -252,13 +236,6 @@ func (s *PropertyService) InitializeDefaultConfigs(ctx context.Context) error { Name: "通知渠道配置", Value: []models.NotificationChannelConfig{}, }, - { - ID: PropertyIDMetricsConfig, - Name: "指标数据配置", - Value: models.MetricsConfig{ - RetentionHours: 168, // 默认7天 - }, - }, { ID: PropertyIDAlertConfig, Name: "告警配置", diff --git a/internal/vmclient/client.go b/internal/vmclient/client.go new file mode 100644 index 0000000..8109848 --- /dev/null +++ b/internal/vmclient/client.go @@ -0,0 +1,337 @@ +package vmclient + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" +) + +// VMClient VictoriaMetrics 客户端 +type VMClient struct { + baseURL string + httpClient *http.Client + writeTimeout time.Duration + queryTimeout time.Duration +} + +// QueryResult 查询结果 +type QueryResult struct { + Status string `json:"status"` + Data ResultData `json:"data"` +} + +// ResultData 查询结果数据 +type ResultData struct { + ResultType string `json:"resultType"` + Result []Result `json:"result"` +} + +// Result 单个时间序列结果 +type Result struct { + Metric map[string]string `json:"metric"` + Values [][]interface{} `json:"values"` // [[timestamp, value], ...] +} + +// DataPoint 数据点 +type DataPoint struct { + Timestamp int64 + Value float64 + Labels map[string]string +} + +// Metric VictoriaMetrics JSON Line Format 指标 +type Metric struct { + Metric map[string]string `json:"metric"` + Values []float64 `json:"values"` + Timestamps []int64 `json:"timestamps"` +} + +// NewVMClient 创建 VictoriaMetrics 客户端 +func NewVMClient(baseURL string, writeTimeout, queryTimeout time.Duration) *VMClient { + if writeTimeout == 0 { + writeTimeout = 30 * time.Second + } + if queryTimeout == 0 { + queryTimeout = 60 * time.Second + } + + return &VMClient{ + baseURL: baseURL, + httpClient: &http.Client{ + Timeout: writeTimeout, // 默认超时 + }, + writeTimeout: writeTimeout, + queryTimeout: queryTimeout, + } +} + +// Write 写入指标（VictoriaMetrics JSON Line Format） +func (c *VMClient) Write(ctx context.Context, metrics []Metric) error { + if len(metrics) == 0 { + return nil + } + + reqCtx, cancel := context.WithTimeout(ctx, c.writeTimeout) + defer cancel() + + // 将 Metric 数组转换为 JSON Line Format (NDJSON) + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + for _, metric := range metrics { + if err := encoder.Encode(metric); err != nil { + return fmt.Errorf("encode metric failed: %w", err) + } + } + + req, err := http.NewRequestWithContext(reqCtx, "POST", c.baseURL+"/api/v1/import", &buf) + if err != nil { + return fmt.Errorf("create request failed: %w", err) + } + + req.Header.Set("Content-Type", "application/x-ndjson") + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("write metrics failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("write metrics failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// AutoStep 根据查询范围自动生成 step（适用于 VictoriaMetrics） +func AutoStep(start, end time.Time) time.Duration { + r := end.Sub(start) + + switch { + case r <= time.Hour: + return 10 * time.Second + case r <= 2*time.Hour: + return 15 * time.Second + case r <= 6*time.Hour: + return 30 * time.Second + case r <= 12*time.Hour: + return time.Minute + case r <= 24*time.Hour: + return 2 * time.Minute + case r <= 3*24*time.Hour: + return 5 * time.Minute + case r <= 7*24*time.Hour: + return 10 * time.Minute + case r <= 30*24*time.Hour: + return 30 * time.Minute + default: + return time.Hour + } +} + +// QueryRange 范围查询 +// 如果 step 为 0，则让 VictoriaMetrics 自动选择合适的步长 +func (c *VMClient) QueryRange(ctx context.Context, query string, start, end time.Time, step time.Duration) (*QueryResult, error) { + reqCtx, cancel := context.WithTimeout(ctx, c.queryTimeout) + defer cancel() + + params := url.Values{} + params.Set("query", query) + params.Set("start", fmt.Sprintf("%d", start.Unix())) + params.Set("end", fmt.Sprintf("%d", end.Unix())) + if step > 0 { + params.Set("step", fmt.Sprintf("%ds", int(step.Seconds()))) + } else { + autoStep := AutoStep(start, end) + params.Set("step", fmt.Sprintf("%ds", int(autoStep.Seconds()))) + } + + reqURL := fmt.Sprintf("%s/api/v1/query_range?%s", c.baseURL, params.Encode()) + + req, err := http.NewRequestWithContext(reqCtx, "GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("create request failed: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("query range failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("query range failed with status %d: %s", resp.StatusCode, string(body)) + } + + var result QueryResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode response failed: %w", err) + } + + if result.Status != "success" { + return nil, fmt.Errorf("query failed with status: %s", result.Status) + } + + return &result, nil +} + +// Query 即时查询 +func (c *VMClient) Query(ctx context.Context, query string) (*QueryResult, error) { + reqCtx, cancel := context.WithTimeout(ctx, c.queryTimeout) + defer cancel() + + params := url.Values{} + params.Set("query", query) + + reqURL := fmt.Sprintf("%s/api/v1/query?%s", c.baseURL, params.Encode()) + + req, err := http.NewRequestWithContext(reqCtx, "GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("create request failed: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("query failed with status %d: %s", resp.StatusCode, string(body)) + } + + var result QueryResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode response failed: %w", err) + } + + if result.Status != "success" { + return nil, fmt.Errorf("query failed with status: %s", result.Status) + } + + return &result, nil +} + +// DeleteSeries 删除时间序列 +func (c *VMClient) DeleteSeries(ctx context.Context, match []string) error { + reqCtx, cancel := context.WithTimeout(ctx, c.writeTimeout) + defer cancel() + + params := url.Values{} + for _, m := range match { + params.Add("match[]", m) + } + + reqURL := fmt.Sprintf("%s/api/v1/admin/tsdb/delete_series?%s", c.baseURL, params.Encode()) + + req, err := http.NewRequestWithContext(reqCtx, "POST", reqURL, nil) + if err != nil { + return fmt.Errorf("create request failed: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("delete series failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("delete series failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} + +// ConvertToDataPoints 将查询结果转换为数据点列表 +func ConvertToDataPoints(result *QueryResult) []DataPoint { + if result == nil || len(result.Data.Result) == 0 { + return []DataPoint{} + } + + var points []DataPoint + for _, r := range result.Data.Result { + for _, v := range r.Values { + if len(v) < 2 { + continue + } + + // timestamp 是 float64（Unix 秒） + timestamp, ok := v[0].(float64) + if !ok { + continue + } + + // value 是 string + valueStr, ok := v[1].(string) + if !ok { + continue + } + + var value float64 + if _, err := fmt.Sscanf(valueStr, "%f", &value); err != nil { + continue + } + + points = append(points, DataPoint{ + Timestamp: int64(timestamp * 1000), // 转换为毫秒 + Value: value, + Labels: r.Metric, + }) + } + } + + return points +} + +// GetLabelValues 获取指定 label 的所有值 +func (c *VMClient) GetLabelValues(ctx context.Context, labelName string, match []string) ([]string, error) { + reqCtx, cancel := context.WithTimeout(ctx, c.queryTimeout) + defer cancel() + + params := url.Values{} + for _, m := range match { + params.Add("match[]", m) + } + + reqURL := fmt.Sprintf("%s/api/v1/label/%s/values?%s", c.baseURL, labelName, params.Encode()) + + req, err := http.NewRequestWithContext(reqCtx, "GET", reqURL, nil) + if err != nil { + return nil, fmt.Errorf("create request failed: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("get label values failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("get label values failed with status %d: %s", resp.StatusCode, string(body)) + } + + var result struct { + Status string `json:"status"` + Data []string `json:"data"` + } + + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("decode response failed: %w", err) + } + + if result.Status != "success" { + return nil, fmt.Errorf("get label values failed with status: %s", result.Status) + } + + return result.Data, nil +} diff --git a/internal/wire.go b/internal/wire.go index 7fc903a..dff2fd8 100644 --- a/internal/wire.go +++ b/internal/wire.go @@ -4,10 +4,13 @@ package internal import ( + "time" + "github.com/dushixiang/pika/internal/config" "github.com/dushixiang/pika/internal/handler" "github.com/dushixiang/pika/internal/repo" "github.com/dushixiang/pika/internal/service" + "github.com/dushixiang/pika/internal/vmclient" "github.com/dushixiang/pika/internal/websocket" "github.com/google/wire" "go.uber.org/zap" @@ -17,6 +20,9 @@ import ( // InitializeApp 初始化应用 func InitializeApp(logger *zap.Logger, db *gorm.DB, cfg *config.AppConfig) (*AppComponents, error) { wire.Build( + // VictoriaMetrics Client + provideVMClient, + service.NewAccountService, service.NewAgentService, service.NewUserService, @@ -80,4 +86,33 @@ type AppComponents struct { DDNSService *service.DDNSService WSManager *websocket.Manager + VMClient *vmclient.VMClient +} + +// provideVMClient 提供 VictoriaMetrics 客户端 +func provideVMClient(cfg *config.AppConfig, logger *zap.Logger) *vmclient.VMClient { + // 检查配置 + if cfg.VictoriaMetrics == nil || !cfg.VictoriaMetrics.Enabled { + logger.Info("VictoriaMetrics is not enabled, using default configuration") + // 返回一个默认配置的客户端（用于本地开发） + return vmclient.NewVMClient("http://localhost:8428", 30*time.Second, 60*time.Second) + } + + // 使用配置创建客户端 + writeTimeout := time.Duration(cfg.VictoriaMetrics.WriteTimeout) * time.Second + if writeTimeout == 0 { + writeTimeout = 30 * time.Second + } + + queryTimeout := time.Duration(cfg.VictoriaMetrics.QueryTimeout) * time.Second + if queryTimeout == 0 { + queryTimeout = 60 * time.Second + } + + logger.Info("VictoriaMetrics client initialized", + zap.String("url", cfg.VictoriaMetrics.URL), + zap.Duration("writeTimeout", writeTimeout), + zap.Duration("queryTimeout", queryTimeout)) + + return vmclient.NewVMClient(cfg.VictoriaMetrics.URL, writeTimeout, queryTimeout) } diff --git a/internal/wire_gen.go b/internal/wire_gen.go index 52fd4cc..81fbacc 100644 --- a/internal/wire_gen.go +++ b/internal/wire_gen.go @@ -11,9 +11,11 @@ import ( "github.com/dushixiang/pika/internal/handler" "github.com/dushixiang/pika/internal/repo" "github.com/dushixiang/pika/internal/service" + "github.com/dushixiang/pika/internal/vmclient" "github.com/dushixiang/pika/internal/websocket" "go.uber.org/zap" "gorm.io/gorm" + "time" ) // Injectors from wire.go: @@ -28,14 +30,15 @@ func InitializeApp(logger *zap.Logger, db *gorm.DB, cfg *config.AppConfig) (*App apiKeyService := service.NewApiKeyService(logger, db) propertyService := service.NewPropertyService(logger, db) trafficService := service.NewTrafficService(logger, db) - metricService := service.NewMetricService(logger, db, propertyService, trafficService) + vmClient := provideVMClient(cfg, logger) + metricService := service.NewMetricService(logger, db, propertyService, trafficService, vmClient) geoIPService, err := service.NewGeoIPService(logger, cfg) if err != nil { return nil, err } agentService := service.NewAgentService(logger, db, apiKeyService, metricService, geoIPService) manager := websocket.NewManager(logger) - monitorService := service.NewMonitorService(logger, db, manager) + monitorService := service.NewMonitorService(logger, db, metricService, manager) tamperRepo := repo.NewTamperRepo(db) tamperService := service.NewTamperService(logger, tamperRepo, manager) ddnsConfigRepo := repo.NewDDNSConfigRepo(db) @@ -70,6 +73,7 @@ func InitializeApp(logger *zap.Logger, db *gorm.DB, cfg *config.AppConfig) (*App TamperService: tamperService, DDNSService: ddnsService, WSManager: manager, + VMClient: vmClient, } return appComponents, nil } @@ -98,4 +102,29 @@ type AppComponents struct { DDNSService *service.DDNSService WSManager *websocket.Manager + VMClient *vmclient.VMClient +} + +// provideVMClient 提供 VictoriaMetrics 客户端 +func provideVMClient(cfg *config.AppConfig, logger *zap.Logger) *vmclient.VMClient { + + if cfg.VictoriaMetrics == nil || !cfg.VictoriaMetrics.Enabled { + logger.Info("VictoriaMetrics is not enabled, using default configuration") + + return vmclient.NewVMClient("http://localhost:8428", 30*time.Second, 60*time.Second) + } + + writeTimeout := time.Duration(cfg.VictoriaMetrics.WriteTimeout) * time.Second + if writeTimeout == 0 { + writeTimeout = 30 * time.Second + } + + queryTimeout := time.Duration(cfg.VictoriaMetrics.QueryTimeout) * time.Second + if queryTimeout == 0 { + queryTimeout = 60 * time.Second + } + + logger.Info("VictoriaMetrics client initialized", zap.String("url", cfg.VictoriaMetrics.URL), zap.Duration("writeTimeout", writeTimeout), zap.Duration("queryTimeout", queryTimeout)) + + return vmclient.NewVMClient(cfg.VictoriaMetrics.URL, writeTimeout, queryTimeout) } diff --git a/pkg/agent/collector/manager.go b/pkg/agent/collector/manager.go index b3314e6..6022b97 100644 --- a/pkg/agent/collector/manager.go +++ b/pkg/agent/collector/manager.go @@ -188,8 +188,6 @@ func (m *Manager) sendMetrics(conn WebSocketWriter, metricType protocol.MetricTy return err } - //dataBytes = bytes.ReplaceAll(dataBytes, []byte{0x00}, []byte(`"`)) - metrics := protocol.MetricsWrapper{ Type: metricType, Data: json.RawMessage(dataBytes), diff --git a/playground/docker-compose.yml b/playground/docker-compose.yml index f293f4b..3bff3d7 100644 --- a/playground/docker-compose.yml +++ b/playground/docker-compose.yml @@ -14,3 +14,11 @@ services: interval: 10s timeout: 5s retries: 5 + victoriametrics: + image: victoriametrics/victoria-metrics:v1.131.0 + container_name: victoriametrics + restart: always + ports: + - "8428:8428" # API, PromQL, insert, select + volumes: + - ./data/vmdata:/victoria-metrics-data \ No newline at end of file diff --git a/web/src/api/agent.ts b/web/src/api/agent.ts index 6e38d49..d7802ed 100644 --- a/web/src/api/agent.ts +++ b/web/src/api/agent.ts @@ -13,14 +13,23 @@ export interface GetAgentMetricsRequest { interface?: string; // 网卡过滤参数（仅对 network 类型有效） } +// 新的统一数据格式 +export interface MetricDataPoint { + timestamp: number; + value: number; +} + +export interface MetricSeries { + name: string; + labels?: Record; + data: MetricDataPoint[]; +} + export interface GetAgentMetricsResponse { agentId: string; type: string; range: string; - start: number; - end: number; - interface?: string; - metrics: any[]; + series: MetricSeries[]; } // 管理员接口 - 获取所有探针（需要认证） diff --git a/web/src/pages/Public/ServerDetail.tsx b/web/src/pages/Public/ServerDetail.tsx index 2c15998..092c4d4 100644 --- a/web/src/pages/Public/ServerDetail.tsx +++ b/web/src/pages/Public/ServerDetail.tsx @@ -31,18 +31,14 @@ import { getAgentLatestMetrics, getAgentMetrics, type GetAgentMetricsRequest, + type GetAgentMetricsResponse, + type MetricSeries, + type MetricDataPoint, getAvailableNetworkInterfaces, } from '@/api/agent.ts'; import {type TimeRangeOption} from '@/api/property.ts'; import type { Agent, - AggregatedCPUMetric, - AggregatedDiskIOMetric, - AggregatedGPUMetric, - AggregatedMemoryMetric, - AggregatedNetworkConnectionMetric, - AggregatedNetworkMetric, - AggregatedTemperatureMetric, LatestMetrics } from '@/types'; import dayjs from "dayjs"; @@ -246,15 +242,15 @@ const TimeRangeSelector = ({ type MetricsTooltipProps = TooltipProps & { unit?: string, label?: string, payload?: any[] }; +// 新的统一 Metrics 状态类型 type MetricsState = { - cpu: AggregatedCPUMetric[]; - memory: AggregatedMemoryMetric[]; - network: AggregatedNetworkMetric[]; - networkConnection: AggregatedNetworkConnectionMetric[]; - // disk: AggregatedDiskMetric[]; - diskIO: AggregatedDiskIOMetric[]; - gpu: AggregatedGPUMetric[]; - temperature: AggregatedTemperatureMetric[]; + cpu: MetricSeries[]; + memory: MetricSeries[]; + network: MetricSeries[]; + networkConnection: MetricSeries[]; + diskIO: MetricSeries[]; + gpu: MetricSeries[]; + temperature: MetricSeries[]; }; const createEmptyMetricsState = (): MetricsState => ({ @@ -262,7 +258,6 @@ const createEmptyMetricsState = (): MetricsState => ({ memory: [], network: [], networkConnection: [], - // disk: [], diskIO: [], gpu: [], temperature: [], @@ -370,7 +365,8 @@ const useAggregatedMetrics = (agentId: string | undefined, range: string, interf if (cancelled) return; const nextState = createEmptyMetricsState(); metricRequestConfig.forEach(({key}, index) => { - nextState[key] = responses[index].data.metrics || []; + // 新的 API 返回 series 数组 + nextState[key] = responses[index].data.series || []; }); setMetrics(nextState); } catch (error) { @@ -517,31 +513,35 @@ const ServerDetail = () => { const {agent, latestMetrics, loading} = useAgentOverview(id); const metricsData = useAggregatedMetrics(id, timeRange, selectedInterface); - const cpuChartData = useMemo( - () => - metricsData.cpu.map((item) => ({ - time: new Date(item.timestamp).toLocaleTimeString('zh-CN', { - hour: '2-digit', - minute: '2-digit', - }), - usage: Number(item.maxUsage.toFixed(2)), - timestamp: item.timestamp, - })), - [metricsData.cpu] - ); + const cpuChartData = useMemo(() => { + // CPU 数据：取第一个系列（usage） + const cpuSeries = metricsData.cpu.find(s => s.name === 'usage'); + if (!cpuSeries) return []; - const memoryChartData = useMemo( - () => - metricsData.memory.map((item) => ({ - time: new Date(item.timestamp).toLocaleTimeString('zh-CN', { - hour: '2-digit', - minute: '2-digit', - }), - usage: Number(item.maxUsage.toFixed(2)), - timestamp: item.timestamp, - })), - [metricsData.memory] - ); + return cpuSeries.data.map((point) => ({ + time: new Date(point.timestamp).toLocaleTimeString('zh-CN', { + hour: '2-digit', + minute: '2-digit', + }), + usage: Number(point.value.toFixed(2)), + timestamp: point.timestamp, + })); + }, [metricsData.cpu]); + + const memoryChartData = useMemo(() => { + // Memory 数据：取第一个系列（usage） + const memorySeries = metricsData.memory.find(s => s.name === 'usage'); + if (!memorySeries) return []; + + return memorySeries.data.map((point) => ({ + time: new Date(point.timestamp).toLocaleTimeString('zh-CN', { + hour: '2-digit', + minute: '2-digit', + }), + usage: Number(point.value.toFixed(2)), + timestamp: point.timestamp, + })); + }, [metricsData.memory]); // 获取所有可用的网卡列表（从后端接口获取） const [availableInterfaces, setAvailableInterfaces] = useState([]); @@ -587,126 +587,167 @@ const ServerDetail = () => { const networkChartData = useMemo(() => { if (metricsData.network.length === 0) return []; - // 按时间戳分组数据 - const grouped = metricsData.network.reduce((acc, item) => { - const time = new Date(item.timestamp).toLocaleTimeString('zh-CN', { + // 找到上行和下行系列 + const uploadSeries = metricsData.network.find(s => s.name === 'upload'); + const downloadSeries = metricsData.network.find(s => s.name === 'download'); + + if (!uploadSeries || !downloadSeries) return []; + + // 按时间戳对齐数据 + const timeMap = new Map(); + + uploadSeries.data.forEach(point => { + const time = new Date(point.timestamp).toLocaleTimeString('zh-CN', { hour: '2-digit', minute: '2-digit', }); - - if (!acc[time]) { - acc[time] = {time, timestamp: item.timestamp}; - } - - // 后端已经根据 interface 参数返回了对应的数据 - // 使用当前选中的 interface 作为标识（'all' 或具体网卡名） - const interfaceName = selectedInterface === 'all' ? 'total' : selectedInterface; - const uploadKey = `${interfaceName}_upload`; - const downloadKey = `${interfaceName}_download`; - // 转换为 MB/s - acc[time][uploadKey] = Number((item.maxSentRate / 1024 / 1024).toFixed(2)); - acc[time][downloadKey] = Number((item.maxRecvRate / 1024 / 1024).toFixed(2)); - - return acc; - }, {} as Record); - - return Object.values(grouped); - }, [metricsData.network, selectedInterface]); - - // Disk I/O 图表数据（汇总所有磁盘） - const diskIOChartData = useMemo(() => { - const aggregated: Record = {}; - - metricsData.diskIO.forEach((item) => { - const time = new Date(item.timestamp).toLocaleTimeString('zh-CN', { - hour: '2-digit', - minute: '2-digit', + timeMap.set(point.timestamp, { + time, + timestamp: point.timestamp, + upload: Number((point.value / 1024 / 1024).toFixed(2)), // 转换为 MB/s }); - - if (!aggregated[time]) { - aggregated[time] = {time, read: 0, write: 0, timestamp: item.timestamp}; - } - - // 转换为 MB/s - aggregated[time].read += item.maxReadRate / 1024 / 1024; - aggregated[time].write += item.maxWriteRate / 1024 / 1024; }); - return Object.values(aggregated).map((item) => ({ - ...item, - read: Number(item.read.toFixed(2)), - write: Number(item.write.toFixed(2)), - })); + downloadSeries.data.forEach(point => { + const existing = timeMap.get(point.timestamp); + if (existing) { + existing.download = Number((point.value / 1024 / 1024).toFixed(2)); + } + }); + + return Array.from(timeMap.values()); + }, [metricsData.network]); + + // Disk I/O 图表数据 + const diskIOChartData = useMemo(() => { + if (metricsData.diskIO.length === 0) return []; + + // 找到读和写系列 + const readSeries = metricsData.diskIO.find(s => s.name === 'read'); + const writeSeries = metricsData.diskIO.find(s => s.name === 'write'); + + if (!readSeries || !writeSeries) return []; + + // 按时间戳对齐数据 + const timeMap = new Map(); + + readSeries.data.forEach(point => { + const time = new Date(point.timestamp).toLocaleTimeString('zh-CN', { + hour: '2-digit', + minute: '2-digit', + }); + timeMap.set(point.timestamp, { + time, + timestamp: point.timestamp, + read: Number((point.value / 1024 / 1024).toFixed(2)), // 转换为 MB/s + }); + }); + + writeSeries.data.forEach(point => { + const existing = timeMap.get(point.timestamp); + if (existing) { + existing.write = Number((point.value / 1024 / 1024).toFixed(2)); + } + }); + + return Array.from(timeMap.values()); }, [metricsData.diskIO]); - // GPU 图表数据（汇总所有GPU的平均利用率） + // GPU 图表数据（暂不支持，需要后端返回温度系列） const gpuChartData = useMemo(() => { - const aggregated: Record = {}; + if (metricsData.gpu.length === 0) return []; - metricsData.gpu.forEach((item) => { - const time = new Date(item.timestamp).toLocaleTimeString('zh-CN', { + // 按时间戳聚合利用率和温度系列 + const timeMap = new Map(); + + const utilizationSeries = metricsData.gpu.find(s => s.name === 'utilization'); + const temperatureSeries = metricsData.gpu.find(s => s.name === 'temperature'); + + // 添加利用率数据 + utilizationSeries?.data.forEach(point => { + const time = new Date(point.timestamp).toLocaleTimeString('zh-CN', { hour: '2-digit', minute: '2-digit', }); - - if (!aggregated[time]) { - aggregated[time] = {time, utilization: 0, temperature: 0, count: 0, timestamp: item.timestamp}; - } - - aggregated[time].utilization += item.maxUtilization; - aggregated[time].temperature += item.maxTemperature; - aggregated[time].count += 1; + timeMap.set(point.timestamp, { + time, + timestamp: point.timestamp, + utilization: Number(point.value.toFixed(2)), + }); }); - return Object.values(aggregated).map((item) => ({ - time: item.time, - utilization: Number((item.utilization / item.count).toFixed(2)), - temperature: Number((item.temperature / item.count).toFixed(2)), - timestamp: item.timestamp, - })); + // 添加温度数据 + temperatureSeries?.data.forEach(point => { + const existing = timeMap.get(point.timestamp); + if (existing) { + existing.temperature = Number(point.value.toFixed(2)); + } + }); + + return Array.from(timeMap.values()); }, [metricsData.gpu]); // Temperature 图表数据（按类型分组显示各类型的温度） const temperatureChartData = useMemo(() => { - const aggregated: Record = {}; + if (metricsData.temperature.length === 0) return []; - metricsData.temperature.forEach((item) => { - const time = new Date(item.timestamp).toLocaleTimeString('zh-CN', { - hour: '2-digit', - minute: '2-digit', + // 按时间戳聚合所有温度系列 + const timeMap = new Map(); + + metricsData.temperature.forEach(series => { + const sensorName = series.name; // 使用系列名称作为传感器标识 + series.data.forEach(point => { + const time = new Date(point.timestamp).toLocaleTimeString('zh-CN', { + hour: '2-digit', + minute: '2-digit', + }); + + if (!timeMap.has(point.timestamp)) { + timeMap.set(point.timestamp, {time, timestamp: point.timestamp}); + } + + const existing = timeMap.get(point.timestamp)!; + existing[sensorName] = Number(point.value.toFixed(2)); }); - - if (!aggregated[time]) { - aggregated[time] = {time, timestamp: item.timestamp}; - } - - // 使用 sensorLabel 作为数据键（类型名称，如 CPU、GPU 等） - const label = item.sensorLabel || 'Unknown'; - // 对于同一时间点的同一类型，取最大温度 - if (!aggregated[time][label] || item.maxTemperature > aggregated[time][label]) { - aggregated[time][label] = Number(item.maxTemperature.toFixed(2)); - } }); - return Object.values(aggregated); + return Array.from(timeMap.values()); }, [metricsData.temperature]); // 提取所有唯一的温度类型（用于图表 Line 渲染和下拉选择器） const temperatureTypes = useMemo(() => { - const types = new Set(); - metricsData.temperature.forEach((item) => { - const label = item.sensorLabel || 'Unknown'; - types.add(label); - }); - return Array.from(types).sort(); + return metricsData.temperature.map(s => s.name).sort(); }, [metricsData.temperature]); + // 网络连接图表数据 + const networkConnectionChartData = useMemo(() => { + if (metricsData.networkConnection.length === 0) return []; + + // 按时间戳聚合所有连接状态系列 + const timeMap = new Map(); + + metricsData.networkConnection.forEach(series => { + const stateName = series.name; // established, time_wait, close_wait, listen + series.data.forEach(point => { + const time = new Date(point.timestamp).toLocaleTimeString('zh-CN', { + hour: '2-digit', + minute: '2-digit', + }); + + if (!timeMap.has(point.timestamp)) { + timeMap.set(point.timestamp, {time, timestamp: point.timestamp}); + } + + const existing = timeMap.get(point.timestamp)!; + // 转换为驼峰命名以匹配图表的 dataKey + const camelCaseName = stateName.replace(/_([a-z])/g, (_, letter) => letter.toUpperCase()); + existing[camelCaseName] = Number(point.value.toFixed(0)); + }); + }); + + return Array.from(timeMap.values()); + }, [metricsData.networkConnection]); + // 根据选中的类型过滤温度数据 const filteredTemperatureTypes = useMemo(() => { if (selectedTempType === 'all') { @@ -1189,30 +1230,18 @@ const ServerDetail = () => { - {(() => { - const interfaceName = selectedInterface === 'all' ? 'total' : selectedInterface; - const colorConfig = INTERFACE_COLORS[0]; - const uploadKey = `${interfaceName}_upload`; - const downloadKey = `${interfaceName}_download`; - return ( - <> - - - - - - - - - - ); - })()} + + + + + + + + @@ -1230,38 +1259,25 @@ const ServerDetail = () => { /> }/> - {/* 渲染当前选中网卡的上行和下行区域 */} - {(() => { - // 根据 selectedInterface 确定显示的网卡名称和数据 key - const interfaceName = selectedInterface === 'all' ? 'total' : selectedInterface; - const colorConfig = INTERFACE_COLORS[0]; // 使用第一组颜色 - const uploadKey = `${interfaceName}_upload`; - const downloadKey = `${interfaceName}_download`; - const displayName = selectedInterface === 'all' ? '总计' : selectedInterface; - - return ( - <> - - - - ); - })()} + {/* 渲染上行和下行区域 */} + + ) : ( @@ -1339,19 +1355,9 @@ const ServerDetail = () => { 网络连接统计 - {metricsData.networkConnection.length > 0 ? ( + {networkConnectionChartData.length > 0 ? ( - ({ - time: new Date(item.timestamp).toLocaleTimeString('zh-CN', { - hour: '2-digit', - minute: '2-digit', - }), - established: item.maxEstablished, - timeWait: item.maxTimeWait, - closeWait: item.maxCloseWait, - listen: item.maxListen, - timestamp: item.timestamp, - }))}> +