feat(admin): 渠道列表/详情展示最差模型，暴露被均值摊薄的单模型故障
渠道健康分是各模型 succ_ewma 的均值，单个模型全挂会被其余健康模型摊薄——10 个 健康模型 + 1 个挂掉仍能算出 90 分绿灯，而这恰是模型级熔断要隔离的典型场景。 补一个最差项：ChannelRuntimeStates 遍历目录快照时顺带取每个渠道 succ_ewma 最小的 模型，按 succ^α×100 折算为与渠道分同口径的健康分。 不落库、不加列：catalog 内存快照已逐 (渠道,模型) 持有 SuccEwma，遍历它即可，零额外 I/O；数据是调度器同源的 5 秒刷新值，比 5 分钟落盘快照更新鲜。取最小值也天然免疫冷 模型问题——无上报的模型读作满分 1.0，永远不会被选为最差项。 byModel 是 map，遍历顺序随机，同分时按模型名取小者打破平局，否则全部模型满分时 每次请求返回的模型名都会跳变。 前端：列表页健康度列在最差项低于 80（健康档）时补一行「⚠ 模型名 分数」，详情页 健康度字段同理；disabled / testing 渠道不在目录快照，不展示。
This commit is contained in:
chengqian authored and qian committed
1 parent
faddc45978
commit
dd02bf4ac3
6 files changed
+127
-8
No files matched your search
@@ -47,6 +47,8 @@ type ChannelItem struct {
|
||||
HealthScore *float64 `json:"health_score"`
|
||||
BreakerState int `json:"breaker_state"` // 调度熔断状态：0=正常 1=熔断中 2=半开探活
|
||||
BreakerModels int `json:"breaker_models"` // 处于熔断/半开的模型数量（渠道×模型级汇总）
|
||||
WorstModel string `json:"worst_model"` // 健康度最低的模型名（空=渠道不在调度目录）
|
||||
WorstModelScore float64 `json:"worst_model_score"`
|
||||
}
|
||||
|
||||
// ChannelCreateReq 创建渠道请求
|
||||
@@ -143,6 +145,8 @@ type ChannelDetailRes struct {
|
||||
HealthScore *float64 `json:"health_score"`
|
||||
BreakerState int `json:"breaker_state"` // 调度熔断状态：0=正常 1=熔断中 2=半开探活
|
||||
BreakerModels int `json:"breaker_models"` // 处于熔断/半开的模型数量（渠道×模型级汇总）
|
||||
WorstModel string `json:"worst_model"` // 健康度最低的模型名（空=渠道不在调度目录）
|
||||
WorstModelScore float64 `json:"worst_model_score"`
|
||||
KeyType string `json:"key_type"`
|
||||
KeyStatus string `json:"key_status"`
|
||||
KeyName string `json:"key_name"`
|
||||
|
||||
@@ -2,6 +2,7 @@ package dispatchadapter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
@@ -157,6 +158,13 @@ func (c *Catalog) ChannelModels() map[int64][]string {
|
||||
type ChannelRuntimeState struct {
|
||||
Breaker dispatch.BreakerState // 渠道级熔断状态（同一渠道所有模型取值相同）
|
||||
BreakerModels int // 处于 OPEN/HALF_OPEN 的模型数量（渠道×模型级汇总）
|
||||
|
||||
// WorstModel / WorstModelScore 健康度最低的模型及其健康分（succ_ewma^α×100，
|
||||
// 与渠道聚合分同口径）。渠道分是均值，单个模型全挂会被其余健康模型摊薄——10 个
|
||||
// 健康模型 + 1 个挂掉仍能算出 90 分绿灯，最差项让这种情况在列表页直接可见。
|
||||
// 取最小值天然免疫冷模型问题：无上报的模型读作满分 1.0，永远不会成为最差项。
|
||||
WorstModel string
|
||||
WorstModelScore float64
|
||||
}
|
||||
|
||||
// ChannelRuntimeStates 返回目录快照中各渠道的熔断聚合视图。
|
||||
@@ -170,19 +178,41 @@ func (c *Catalog) ChannelRuntimeStates() map[int64]ChannelRuntimeState {
|
||||
if idx == nil {
|
||||
return nil
|
||||
}
|
||||
out := make(map[int64]ChannelRuntimeState)
|
||||
for _, chans := range idx.byModel {
|
||||
type accEntry struct {
|
||||
st ChannelRuntimeState
|
||||
worstSucc float64
|
||||
found bool
|
||||
}
|
||||
acc := make(map[int64]*accEntry)
|
||||
for model, chans := range idx.byModel {
|
||||
for _, ch := range chans {
|
||||
agg, ok := out[ch.ID]
|
||||
a, ok := acc[ch.ID]
|
||||
if !ok {
|
||||
agg = ChannelRuntimeState{Breaker: ch.Breaker}
|
||||
a = &accEntry{st: ChannelRuntimeState{Breaker: ch.Breaker}}
|
||||
acc[ch.ID] = a
|
||||
}
|
||||
if ch.ModelBreaker != dispatch.BreakerClosed {
|
||||
agg.BreakerModels++
|
||||
a.st.BreakerModels++
|
||||
}
|
||||
// 同分时按模型名取小者：byModel 是 map，遍历顺序随机，不做确定性
|
||||
// 打破平局的话，全部模型同分（例如都满分）时每次请求返回的模型名都会跳变。
|
||||
if !a.found || ch.SuccEwma < a.worstSucc ||
|
||||
(ch.SuccEwma == a.worstSucc && model < a.st.WorstModel) {
|
||||
a.found = true
|
||||
a.worstSucc = ch.SuccEwma
|
||||
a.st.WorstModel = model
|
||||
}
|
||||
out[ch.ID] = agg
|
||||
}
|
||||
}
|
||||
|
||||
alpha := HealthAlpha()
|
||||
out := make(map[int64]ChannelRuntimeState, len(acc))
|
||||
for id, a := range acc {
|
||||
if a.found {
|
||||
a.st.WorstModelScore = math.Pow(clampFloat(a.worstSucc, 0, 1), alpha) * 100
|
||||
}
|
||||
out[id] = a.st
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
@@ -233,3 +233,63 @@ func TestCatalog_运行状态_未Rebuild返回空(t *testing.T) {
|
||||
c := NewCatalog(func() *dispatch.RoutingPolicy { return dispatch.DefaultRoutingPolicy() }, nil, nil)
|
||||
assert.Empty(t, c.ChannelRuntimeStates())
|
||||
}
|
||||
|
||||
// TestCatalog_最差模型_单模型故障不被均值摊薄 渠道健康分是均值，10 个健康模型 + 1 个
|
||||
// 全挂仍能算出 90 分绿灯；最差项要如实报出那个挂掉的模型。
|
||||
func TestCatalog_最差模型_单模型故障不被均值摊薄(t *testing.T) {
|
||||
rows := []catalogRow{
|
||||
{ChannelID: 1, ChannelName: "A", ModelName: "healthy-a", Weight: 10, Tier: "primary"},
|
||||
{ChannelID: 1, ChannelName: "A", ModelName: "healthy-b", Weight: 10, Tier: "primary"},
|
||||
{ChannelID: 1, ChannelName: "A", ModelName: "broken", Weight: 10, Tier: "primary"},
|
||||
}
|
||||
c := testCatalog(rows, nil, func(_ context.Context, _ int64, model string) RuntimeReadout {
|
||||
if model == "broken" {
|
||||
return RuntimeReadout{SuccEwma: 0.1, HasHealth: true}
|
||||
}
|
||||
return RuntimeReadout{SuccEwma: 1.0, HasHealth: true}
|
||||
})
|
||||
|
||||
got := c.ChannelRuntimeStates()
|
||||
require.Len(t, got, 1)
|
||||
assert.Equal(t, "broken", got[1].WorstModel)
|
||||
// α 默认 2：0.1² × 100 = 1
|
||||
assert.InDelta(t, 1.0, got[1].WorstModelScore, 1e-6)
|
||||
}
|
||||
|
||||
// TestCatalog_最差模型_冷模型不被误报 无真实上报的模型读作满分 1.0，取最小值天然
|
||||
// 不会把它选成最差项——这正是拖垮渠道均值的缺陷在 min 聚合下不存在的原因。
|
||||
func TestCatalog_最差模型_冷模型不被误报(t *testing.T) {
|
||||
rows := []catalogRow{
|
||||
{ChannelID: 1, ChannelName: "A", ModelName: "cold", Weight: 10, Tier: "primary"},
|
||||
{ChannelID: 1, ChannelName: "A", ModelName: "warm", Weight: 10, Tier: "primary"},
|
||||
}
|
||||
c := testCatalog(rows, nil, func(_ context.Context, _ int64, model string) RuntimeReadout {
|
||||
if model == "cold" {
|
||||
return RuntimeReadout{SuccEwma: 1} // HasHealth=false，从未上报
|
||||
}
|
||||
return RuntimeReadout{SuccEwma: 0.8, HasHealth: true}
|
||||
})
|
||||
|
||||
got := c.ChannelRuntimeStates()
|
||||
assert.Equal(t, "warm", got[1].WorstModel, "冷模型读作满分，不应被选为最差")
|
||||
assert.InDelta(t, 64.0, got[1].WorstModelScore, 1e-6) // 0.8² × 100
|
||||
}
|
||||
|
||||
// TestCatalog_最差模型_同分时结果稳定 byModel 是 map，遍历顺序随机；同分不做确定性
|
||||
// 打破平局的话，全部模型满分时每次请求返回的模型名都会跳变。
|
||||
func TestCatalog_最差模型_同分时结果稳定(t *testing.T) {
|
||||
rows := []catalogRow{
|
||||
{ChannelID: 1, ChannelName: "A", ModelName: "zeta", Weight: 10, Tier: "primary"},
|
||||
{ChannelID: 1, ChannelName: "A", ModelName: "alpha", Weight: 10, Tier: "primary"},
|
||||
{ChannelID: 1, ChannelName: "A", ModelName: "mid", Weight: 10, Tier: "primary"},
|
||||
}
|
||||
c := testCatalog(rows, nil, func(_ context.Context, _ int64, _ string) RuntimeReadout {
|
||||
return RuntimeReadout{SuccEwma: 1, HasHealth: true}
|
||||
})
|
||||
|
||||
for range 20 {
|
||||
got := c.ChannelRuntimeStates()
|
||||
require.Equal(t, "alpha", got[1].WorstModel, "同分必须稳定取模型名最小者")
|
||||
assert.InDelta(t, 100.0, got[1].WorstModelScore, 1e-6)
|
||||
}
|
||||
}
|
||||
@@ -149,6 +149,8 @@ func (s *sAdmin) ListChannels(ctx context.Context, req *v1.ChannelListReq) (*v1.
|
||||
if rs, ok := runtimeStates[ch.ID]; ok {
|
||||
item.BreakerState = int(rs.Breaker)
|
||||
item.BreakerModels = rs.BreakerModels
|
||||
item.WorstModel = rs.WorstModel
|
||||
item.WorstModelScore = rs.WorstModelScore
|
||||
}
|
||||
list = append(list, item)
|
||||
}
|
||||
@@ -594,6 +596,8 @@ func (s *sAdmin) GetChannelDetail(ctx context.Context, req *v1.ChannelDetailReq)
|
||||
if rs, ok := channelRuntimeStates()[req.ID]; ok {
|
||||
res.BreakerState = int(rs.Breaker)
|
||||
res.BreakerModels = rs.BreakerModels
|
||||
res.WorstModel = rs.WorstModel
|
||||
res.WorstModelScore = rs.WorstModelScore
|
||||
}
|
||||
if res.KeyType == "" {
|
||||
res.KeyType = "apikey"
|
||||
|
||||
@@ -1069,6 +1069,13 @@ function formatHeaders(headers: Record<string, string>): string {
|
||||
{{ detail.health_score.toFixed(0) }}
|
||||
</span>
|
||||
<span v-else style="color: #94a3b8">N/A</span>
|
||||
<!-- 渠道分是各模型均值，单个模型全挂会被摊薄；最差项低于健康档时点名 -->
|
||||
<span
|
||||
v-if="detail.status === 'active' && detail.worst_model && detail.worst_model_score != null && detail.worst_model_score < 80"
|
||||
:style="{ marginLeft: '8px', color: healthColor(detail.worst_model_score), fontSize: '12px' }"
|
||||
>
|
||||
⚠ 最差模型 {{ detail.worst_model }} {{ detail.worst_model_score.toFixed(0) }} 分
|
||||
</span>
|
||||
</ADescriptionsItem>
|
||||
<ADescriptionsItem label="调度状态">
|
||||
<!-- disabled / testing 渠道不在目录快照中，熔断状态无意义，置灰展示 -->
|
||||
|
||||
@@ -121,10 +121,24 @@ const columns: TableColumnData[] = [
|
||||
},
|
||||
},
|
||||
{
|
||||
title: '健康度', dataIndex: 'health_score', width: 90,
|
||||
title: '健康度', dataIndex: 'health_score', width: 140,
|
||||
render({ record }) {
|
||||
if (record.health_score === null || record.health_score === undefined) return h('span', { style: 'color:#94a3b8' }, 'N/A')
|
||||
return h('span', { style: { color: healthColor(record.health_score), fontWeight: 600 } }, record.health_score.toFixed(0))
|
||||
const score = h('span', { style: { color: healthColor(record.health_score), fontWeight: 600 } }, record.health_score.toFixed(0))
|
||||
// 渠道分是各模型均值，单个模型全挂会被其余健康模型摊薄（10 健康 + 1 全挂 ≈ 90 分绿灯）。
|
||||
// 最差模型低于健康档时补一行提示，让"是哪个模型拖后腿"在列表页直接可见。
|
||||
// disabled / testing 渠道不在目录快照，无最差项数据。
|
||||
const worst = record.worst_model_score
|
||||
if (record.status !== 'active' || !record.worst_model || worst === null || worst === undefined || worst >= 80) {
|
||||
return score
|
||||
}
|
||||
return h('div', { style: 'display:flex; flex-direction:column; gap:2px; line-height:1.3' }, [
|
||||
score,
|
||||
h('span', {
|
||||
style: `color:${healthColor(worst)}; font-size:12px; overflow:hidden; text-overflow:ellipsis; white-space:nowrap`,
|
||||
title: `最差模型 ${record.worst_model}：${worst.toFixed(0)} 分`,
|
||||
}, `⚠ ${record.worst_model} ${worst.toFixed(0)}`),
|
||||
])
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user