feat(model):添加渠道监控
This commit is contained in:
1 parent
bd159fb2a0
commit
6255930d0d
5 files changed
+246
-14
No files matched your search
@@ -37,6 +37,10 @@ type MonitorModelPerformanceReq struct {
|
||||
g.Meta `path:"/monitor/model-performance" method:"get" mime:"json" tags:"管理后台-监控" summary:"模型性能指标"`
|
||||
StartDate string `json:"start_date" in:"query"` // YYYY-MM-DD,缺省=近30天起点
|
||||
EndDate string `json:"end_date" in:"query"` // YYYY-MM-DD,缺省=今天
|
||||
// 可选渠道筛选:0=全部渠道(选定渠道后当天数据改从 bil_usage_logs 明细实时聚合;0 哨兵被占用,无法筛「无渠道」)
|
||||
ChannelId int64 `json:"channel_id" in:"query" d:"0"`
|
||||
// 可选模型筛选:空=全部模型(与 bil_usage_daily.model_name 同口径)
|
||||
ModelName string `json:"model_name" in:"query"`
|
||||
}
|
||||
|
||||
type MonitorModelPerformanceRes struct {
|
||||
|
||||
@@ -156,13 +156,17 @@ type perfModelAgg struct {
|
||||
cacheHitRequests int64
|
||||
}
|
||||
|
||||
// GetModelPerformance 从 bil_usage_daily 按模型聚合历史性能指标,并合并当日 Redis 实时热桶
|
||||
// (见 common.RecordModelPerfMetrics,桶粒度小时),使页面当天即可看到最新数据,不再滞后一天。
|
||||
// GetModelPerformance 从 bil_usage_daily 按模型聚合历史性能指标,并合并当日实时数据,
|
||||
// 支持可选的渠道(channelID,0=全部)与模型(modelName,空=全部)过滤:选定渠道后可查看
|
||||
// 该渠道下各模型的性能。历史段经 bil_usage_daily(唯一键含 channel_id)过滤;
|
||||
// 当天段在渠道过滤下改走 bil_usage_logs 明细实时聚合(热桶无渠道维度),两段口径一致。
|
||||
// 指标口径与 bil_usage_daily 一致:延迟按 SUM 求和、视图以 SUM/COUNT 求均值(分位数不可跨桶加,p95/p99 留后续)。
|
||||
func GetModelPerformance(ctx context.Context, startDate, endDate string) (any, error) {
|
||||
func GetModelPerformance(ctx context.Context, startDate, endDate string, channelID int64, modelName string) (any, error) {
|
||||
today := time.Now().Format("2006-01-02")
|
||||
|
||||
// 历史段:stat_date < today(当天由 Redis 热桶实时提供,避免与每日 01:00 的日聚合重复/滞后)
|
||||
// 历史段:stat_date < today(当天由实时数据提供,避免与每日 01:00 的日聚合重复/滞后)。
|
||||
// 渠道/模型过滤用「参数=哨兵值 OR 等值匹配」写法保持 SQL 常量字符串,未筛选时等价于无条件;
|
||||
// $4/$5 显式转型规避 PostgreSQL 对 OR 两侧参数的类型推断歧义。
|
||||
const query = `
|
||||
SELECT
|
||||
model_name,
|
||||
@@ -178,27 +182,131 @@ func GetModelPerformance(ctx context.Context, startDate, endDate string) (any, e
|
||||
COALESCE(SUM(cache_hit_request_count), 0) AS cache_hit_request_count
|
||||
FROM bil_usage_daily
|
||||
WHERE stat_date >= $1 AND stat_date <= $2 AND stat_date < $3 AND model_name <> 'unknown'
|
||||
AND ($4::bigint = 0 OR channel_id = $4)
|
||||
AND ($5::text = '' OR model_name = $5)
|
||||
GROUP BY model_name
|
||||
`
|
||||
var rows []modelPerfRow
|
||||
if err := g.DB().Ctx(ctx).Raw(query, startDate, endDate, today).Scan(&rows); err != nil {
|
||||
if err := g.DB().Ctx(ctx).Raw(query, startDate, endDate, today, channelID, modelName).Scan(&rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 当日 Redis 热桶(best-effort:仅当查询窗口包含今天时参与;Redis 异常则忽略当日数据,退化为历史日表)
|
||||
// 当天段(best-effort:仅当查询窗口包含今天时参与;数据源失败则忽略当日数据,退化为历史日表):
|
||||
// - 选定渠道:Redis 热桶无渠道维度,改从 bil_usage_logs 当天明细实时聚合(输出同构,合并逻辑复用)
|
||||
// - 未选渠道:保持热桶路径与现状一致;仅选模型时热桶键即模型名,按键过滤
|
||||
todayCounts := map[string]common.ModelPerfCounter{}
|
||||
if endDate == today {
|
||||
tc, err := common.GetModelPerfMetricsToday(ctx, today)
|
||||
if err != nil {
|
||||
g.Log().Warningf(ctx, "model performance today redis read failed, fallback to daily only: %v", err)
|
||||
if channelID > 0 {
|
||||
tc, err := getModelPerfTodayFromLogs(ctx, today, channelID, modelName)
|
||||
if err != nil {
|
||||
g.Log().Warningf(ctx, "model performance today logs aggregate failed, fallback to daily only: %v", err)
|
||||
} else {
|
||||
todayCounts = tc
|
||||
}
|
||||
} else {
|
||||
todayCounts = tc
|
||||
tc, err := common.GetModelPerfMetricsToday(ctx, today)
|
||||
if err != nil {
|
||||
g.Log().Warningf(ctx, "model performance today redis read failed, fallback to daily only: %v", err)
|
||||
} else {
|
||||
todayCounts = filterTodayCounts(tc, modelName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return map[string]any{"list": buildModelPerformanceList(rows, todayCounts)}, nil
|
||||
}
|
||||
|
||||
// filterTodayCounts 热桶结果按模型过滤(热桶键即模型名;modelName 空 = 原样返回)。
|
||||
// 未命中返回 nil,交给合并层按空处理。
|
||||
func filterTodayCounts(tc map[string]common.ModelPerfCounter, modelName string) map[string]common.ModelPerfCounter {
|
||||
if modelName == "" {
|
||||
return tc
|
||||
}
|
||||
if c, ok := tc[modelName]; ok {
|
||||
return map[string]common.ModelPerfCounter{modelName: c}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// modelPerfTodayRow 当天明细聚合行(bil_usage_logs 按模型分组,仅渠道筛选路径使用)。
|
||||
// 列名对齐 common.ModelPerfCounter 语义;口径与 usage_aggregation 写入 bil_usage_daily 一致:
|
||||
// ok=成功请求、ttft=Σ首Token(无首Token请求计 0)、ttft_n=有首Token样本数、chit=命中缓存的请求数。
|
||||
type modelPerfTodayRow struct {
|
||||
ModelName string `json:"model_name"`
|
||||
Req int64 `json:"req"`
|
||||
Ok int64 `json:"ok"`
|
||||
Lat int64 `json:"lat"`
|
||||
Ttft int64 `json:"ttft"`
|
||||
TtftN int64 `json:"ttft_n"`
|
||||
Tin int64 `json:"tin"`
|
||||
Tout int64 `json:"tout"`
|
||||
TotalCost float64 `json:"total_cost"`
|
||||
CacheCreation int64 `json:"cache_creation"`
|
||||
CacheRead int64 `json:"cache_read"`
|
||||
Chit int64 `json:"chit"`
|
||||
}
|
||||
|
||||
// getModelPerfTodayFromLogs 渠道筛选下的当天数据源:从 bil_usage_logs 明细按模型实时聚合。
|
||||
// Redis 热桶(perf_model:{model}:{hour})无渠道维度,选定渠道后当天必须回明细表;
|
||||
// 时间区间沿用日聚合的半开区间口径 [today 00:00, tomorrow 00:00),命中 created_at 的 BRIN 索引
|
||||
// (tomorrow 从 today 参数推导而非另取 now,避免跨午夜瞬间的边界漂移)。
|
||||
// 输出与 common.GetModelPerfMetricsToday 同构,合并逻辑复用不分叉。
|
||||
func getModelPerfTodayFromLogs(ctx context.Context, today string, channelID int64, modelName string) (map[string]common.ModelPerfCounter, error) {
|
||||
dayStart, err := time.ParseInLocation("2006-01-02", today, time.Local)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start := today + " 00:00:00"
|
||||
end := dayStart.AddDate(0, 0, 1).Format("2006-01-02") + " 00:00:00"
|
||||
const query = `
|
||||
SELECT
|
||||
model_name AS model_name,
|
||||
COUNT(*) AS req,
|
||||
COUNT(*) FILTER (WHERE status = 'success') AS ok,
|
||||
COALESCE(SUM(latency_ms), 0) AS lat,
|
||||
COALESCE(SUM(first_token_ms), 0) AS ttft,
|
||||
COUNT(*) FILTER (WHERE first_token_ms > 0) AS ttft_n,
|
||||
COALESCE(SUM(input_tokens), 0) AS tin,
|
||||
COALESCE(SUM(output_tokens), 0) AS tout,
|
||||
COALESCE(SUM(total_cost), 0) AS total_cost,
|
||||
COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation,
|
||||
COALESCE(SUM(cache_read_tokens), 0) AS cache_read,
|
||||
COUNT(*) FILTER (WHERE cache_read_tokens > 0) AS chit
|
||||
FROM bil_usage_logs
|
||||
WHERE created_at >= $1 AND created_at < $2 AND channel_id = $3
|
||||
AND ($4::text = '' OR model_name = $4)
|
||||
GROUP BY model_name
|
||||
`
|
||||
var rows []modelPerfTodayRow
|
||||
if err := g.DB().Ctx(ctx).Raw(query, start, end, channelID, modelName).Scan(&rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rowsToTodayCounters(rows), nil
|
||||
}
|
||||
|
||||
// rowsToTodayCounters 当天明细聚合行 → 热桶同构计数。
|
||||
// total_cost 以 float 聚合后转整数 micro-USD(common.microFromUSD 未导出,此处同式内联),
|
||||
// 合并侧统一经 billing.FromMicro 还原 USD,保证两条当天数据路径口径一致。
|
||||
func rowsToTodayCounters(rows []modelPerfTodayRow) map[string]common.ModelPerfCounter {
|
||||
out := make(map[string]common.ModelPerfCounter, len(rows))
|
||||
for _, r := range rows {
|
||||
out[r.ModelName] = common.ModelPerfCounter{
|
||||
Req: r.Req,
|
||||
Ok: r.Ok,
|
||||
Lat: r.Lat,
|
||||
Ttft: r.Ttft,
|
||||
TtftN: r.TtftN,
|
||||
Tin: r.Tin,
|
||||
Tout: r.Tout,
|
||||
CostMicro: int64(math.Round(r.TotalCost * 1e6)),
|
||||
CacheCreation: r.CacheCreation,
|
||||
CacheRead: r.CacheRead,
|
||||
CacheHitReq: r.Chit,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildModelPerformanceList 将历史日表聚合行与当日 Redis 热桶合并为统一的模型性能列表,
|
||||
// 统一重算派生指标(成功率/均延迟/均首Token/TPS/分级/缓存命中率)并按模型名称升序输出。
|
||||
func buildModelPerformanceList(rows []modelPerfRow, todayCounts map[string]common.ModelPerfCounter) []map[string]any {
|
||||
|
||||
@@ -137,6 +137,54 @@ func TestBuildModelPerformanceList_CacheEdgeCases(t *testing.T) {
|
||||
assertPerfField(t, list, "text-embedding-3", "cache_hit_request_rate", 0.0, 0)
|
||||
}
|
||||
|
||||
// 当天明细聚合行 → 热桶同构计数:字段映射、cost 转 micro(math.Round)、条件计数透传,并与合并层贯通。
|
||||
func TestRowsToTodayCounters(t *testing.T) {
|
||||
rows := []modelPerfTodayRow{
|
||||
{ModelName: "gpt-4o", Req: 30, Ok: 28, Lat: 15000, Ttft: 4200, TtftN: 25, Tin: 3000, Tout: 6000,
|
||||
TotalCost: 0.123456, CacheCreation: 100, CacheRead: 900, Chit: 20},
|
||||
{ModelName: "tiny", Req: 2, Ok: 2, TotalCost: 0.0000004},
|
||||
}
|
||||
m := rowsToTodayCounters(rows)
|
||||
c := m["gpt-4o"]
|
||||
if c.Req != 30 || c.Ok != 28 || c.Lat != 15000 || c.Ttft != 4200 || c.TtftN != 25 ||
|
||||
c.Tin != 3000 || c.Tout != 6000 || c.CostMicro != 123456 ||
|
||||
c.CacheCreation != 100 || c.CacheRead != 900 || c.CacheHitReq != 20 {
|
||||
t.Errorf("unexpected counter: %+v", c)
|
||||
}
|
||||
// 极小金额四舍五入到 0 micro(对齐 common.microFromUSD 的 math.Round 语义)
|
||||
if m["tiny"].CostMicro != 0 {
|
||||
t.Errorf("CostMicro = %d, want 0", m["tiny"].CostMicro)
|
||||
}
|
||||
// 与合并层贯通:明细行经转换后与热桶路径产出等价(USD 还原一致)
|
||||
list := buildModelPerformanceList(nil, m)
|
||||
assertPerfField(t, list, "gpt-4o", "request_count", 0, 30)
|
||||
assertPerfField(t, list, "gpt-4o", "total_cost", 0.123456, 0)
|
||||
assertPerfField(t, list, "gpt-4o", "cache_hit_request_count", 0, 20)
|
||||
}
|
||||
|
||||
func TestRowsToTodayCounters_Empty(t *testing.T) {
|
||||
if m := rowsToTodayCounters(nil); len(m) != 0 {
|
||||
t.Errorf("expected empty map, got %v", m)
|
||||
}
|
||||
}
|
||||
|
||||
// 热桶路径的模型过滤:命中只留一键;未命中返回空;空模型名原样返回。
|
||||
func TestFilterTodayCounts(t *testing.T) {
|
||||
tc := map[string]common.ModelPerfCounter{
|
||||
"gpt-4o": {Req: 3},
|
||||
"claude-3-5-sonnet": {Req: 5},
|
||||
}
|
||||
if got := filterTodayCounts(tc, ""); len(got) != 2 {
|
||||
t.Errorf("empty model should keep all, got %v", got)
|
||||
}
|
||||
if got := filterTodayCounts(tc, "gpt-4o"); len(got) != 1 || got["gpt-4o"].Req != 3 {
|
||||
t.Errorf("unexpected filtered map: %v", got)
|
||||
}
|
||||
if got := filterTodayCounts(tc, "no-such-model"); len(got) != 0 {
|
||||
t.Errorf("miss should be empty, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildModelPerformanceList_FilterUnknownAndEmpty(t *testing.T) {
|
||||
rows := []modelPerfRow{
|
||||
{ModelName: "unknown", RequestCount: 999, SuccessCount: 0},
|
||||
|
||||
@@ -71,7 +71,8 @@ func (s *sMonitor) ModelPerformance(ctx context.Context, req *v1.MonitorModelPer
|
||||
return nil, err
|
||||
}
|
||||
startDate, endDate := normalizeMonitorDateRange(req.StartDate, req.EndDate)
|
||||
data, err := GetModelPerformance(ctx, startDate, endDate)
|
||||
// 渠道/模型为可选过滤(channel_id=0、model_name 空 = 不过滤),透传给聚合层拼 SQL 条件
|
||||
data, err := GetModelPerformance(ctx, startDate, endDate, req.ChannelId, req.ModelName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -15,6 +15,51 @@ const dateRange = ref<[string, string] | null>(defaultRange(0))
|
||||
// 指标说明弹窗
|
||||
const showMetricsGuide = ref(false)
|
||||
|
||||
// 渠道/模型可选筛选:选定渠道后按渠道查看其下各模型性能;清空下拉 = 全部
|
||||
const filterChannelId = ref<number | undefined>(undefined)
|
||||
const filterModelName = ref<string | undefined>(undefined)
|
||||
const channelOptions = ref<{ label: string; value: number }[]>([])
|
||||
const modelOptions = ref<{ label: string; value: string }[]>([])
|
||||
|
||||
// 渠道下拉数据:渠道量级为几十,一次性拉全量(page_size 放大即可)
|
||||
async function fetchChannelOptions() {
|
||||
try {
|
||||
const res: any = await request.get('/admin/channels', { params: { page: 1, page_size: 200 } })
|
||||
channelOptions.value = (res.data?.data?.list || res.data?.list || []).map((c: any) => ({
|
||||
label: c.name ? `${c.name} (#${c.id})` : `#${c.id}`,
|
||||
value: c.id,
|
||||
}))
|
||||
} catch {
|
||||
channelOptions.value = []
|
||||
}
|
||||
}
|
||||
|
||||
// 模型下拉数据:专用不分页接口 /admin/models/options(model_id 与统计口径 model_name 同名)
|
||||
async function fetchModelOptions() {
|
||||
try {
|
||||
const res: any = await request.get('/admin/models/options')
|
||||
modelOptions.value = (res.data?.data?.list || res.data?.list || []).map((m: any) => ({
|
||||
label: m.model_name ? `${m.model_name} (${m.model_id})` : m.model_id,
|
||||
value: m.model_id,
|
||||
}))
|
||||
} catch {
|
||||
modelOptions.value = []
|
||||
}
|
||||
}
|
||||
|
||||
// 页头描述:追加当前筛选提示,避免「跨渠道」文案与实际筛选状态不符
|
||||
const pageDescription = computed(() => {
|
||||
const base =
|
||||
'各模型跨渠道的成功率 / 延迟 / 吞吐 / 缓存命中(历史数据源自 bil_usage_daily 每日聚合,当天实时统计;延迟为总延迟/总请求数均值。Token 命中率对 OpenAI 原生渠道为保守值)'
|
||||
const parts: string[] = []
|
||||
if (filterChannelId.value) {
|
||||
const ch = channelOptions.value.find((c) => c.value === filterChannelId.value)
|
||||
parts.push(`渠道:${ch ? ch.label : `#${filterChannelId.value}`}`)
|
||||
}
|
||||
if (filterModelName.value) parts.push(`模型:${filterModelName.value}`)
|
||||
return parts.length ? `${base}。当前筛选 — ${parts.join(',')}` : base
|
||||
})
|
||||
|
||||
function toDateStr(d: Date): string {
|
||||
const m = `${d.getMonth() + 1}`.padStart(2, '0')
|
||||
const day = `${d.getDate()}`.padStart(2, '0')
|
||||
@@ -196,11 +241,14 @@ const stats = computed(() => {
|
||||
async function fetchData() {
|
||||
loading.value = true
|
||||
try {
|
||||
const params: Record<string, string> = {}
|
||||
const params: Record<string, any> = {}
|
||||
if (dateRange.value && dateRange.value.length === 2) {
|
||||
params.start_date = dateRange.value[0]
|
||||
params.end_date = dateRange.value[1]
|
||||
}
|
||||
// 可选筛选:清空下拉(undefined)即不传参 = 全部
|
||||
if (filterChannelId.value) params.channel_id = filterChannelId.value
|
||||
if (filterModelName.value) params.model_name = filterModelName.value
|
||||
const res = await request.get('/admin/monitor/model-performance', { params })
|
||||
data.value = res.data?.data?.data?.list || []
|
||||
} catch {
|
||||
@@ -212,13 +260,35 @@ async function fetchData() {
|
||||
|
||||
onMounted(() => {
|
||||
fetchData()
|
||||
// 下拉数据与列表并行加载,失败仅选项为空,不影响主列表
|
||||
fetchChannelOptions()
|
||||
fetchModelOptions()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div class="page-table">
|
||||
<PageHeader title="模型性能" description="各模型跨渠道的成功率 / 延迟 / 吞吐 / 缓存命中(历史数据源自 bil_usage_daily 每日聚合,当天实时统计;延迟为总延迟/总请求数均值。Token 命中率对 OpenAI 原生渠道为保守值)">
|
||||
<PageHeader title="模型性能" :description="pageDescription">
|
||||
<template #actions>
|
||||
<!-- 渠道/模型筛选:清空 = 全部;选定渠道后当天数据由后端从请求明细实时聚合 -->
|
||||
<a-select
|
||||
v-model="filterChannelId"
|
||||
:options="channelOptions"
|
||||
placeholder="全部渠道"
|
||||
allow-clear
|
||||
allow-search
|
||||
style="width: 190px"
|
||||
@change="fetchData"
|
||||
/>
|
||||
<a-select
|
||||
v-model="filterModelName"
|
||||
:options="modelOptions"
|
||||
placeholder="全部模型"
|
||||
allow-clear
|
||||
allow-search
|
||||
style="width: 250px"
|
||||
@change="fetchData"
|
||||
/>
|
||||
<a-button
|
||||
size="small"
|
||||
v-for="q in QUICK_RANGES"
|
||||
@@ -243,7 +313,8 @@ onMounted(() => {
|
||||
<div class="metrics-guide">
|
||||
<p class="guide-intro">
|
||||
以下指标按所选时间区间逐模型聚合:历史数据(昨天及以前)源自 <code>bil_usage_daily</code> 每日聚合,
|
||||
当天数据来自 Redis 实时热桶(小时粒度),两段口径一致、合并计算。顶部概览卡为全部模型汇总。
|
||||
当天数据来自 Redis 实时热桶(小时粒度);选择渠道筛选后,当天数据改为从请求明细(<code>bil_usage_logs</code>)实时聚合。
|
||||
两段口径一致、合并计算。顶部概览卡为当前筛选范围的汇总。
|
||||
</p>
|
||||
|
||||
<div class="guide-group">基础指标</div>
|
||||
|
||||
Reference in new issue
Block a user