improve(log):优化日志输出
This commit is contained in:
1 parent
8f42091cef
commit
e8dfe9499d
5 files changed
+17
-74
No files matched your search
@@ -328,21 +328,13 @@ func (s *RedisState) processOutcome(ctx context.Context, o dispatch.Outcome) {
|
||||
if o.Success {
|
||||
success = "1"
|
||||
}
|
||||
v, err := g.Redis().Do(ctx, "EVAL", luaHealthObserve, 1, healthKey,
|
||||
success, o.LatencyMs, healthDecayFor(o.Class), now, stateKeyTTLMs)
|
||||
if err != nil {
|
||||
// 健康观察成功即静默:本函数消费每个请求的结果事件,任何逐请求日志都会在生产流量下
|
||||
// 淹没日志;健康分轨迹排查走维护轮落盘日志 + ReadRuntime 直接读值。
|
||||
if _, err := g.Redis().Do(ctx, "EVAL", luaHealthObserve, 1, healthKey,
|
||||
success, o.LatencyMs, healthDecayFor(o.Class), now, stateKeyTTLMs); err != nil {
|
||||
s.local.observe(o) // 降级:实例本地健康镜像
|
||||
return
|
||||
}
|
||||
if vals := v.Strings(); len(vals) == 4 {
|
||||
oldSucc, e1 := strconv.ParseFloat(vals[0], 64)
|
||||
newSucc, e2 := strconv.ParseFloat(vals[1], 64)
|
||||
oldLat, e3 := strconv.ParseFloat(vals[2], 64)
|
||||
newLat, e4 := strconv.ParseFloat(vals[3], 64)
|
||||
if e1 == nil && e2 == nil && e3 == nil && e4 == nil {
|
||||
logHealthChange(ctx, o, pol, oldSucc, newSucc, oldLat, newLat)
|
||||
}
|
||||
}
|
||||
|
||||
// 429 → softLimit 自动估计器(基线方案 §8.2)
|
||||
if o.Class == dispatch.ErrClassRateLimit {
|
||||
@@ -379,34 +371,6 @@ func (s *RedisState) processOutcome(ctx context.Context, o dispatch.Outcome) {
|
||||
}
|
||||
}
|
||||
|
||||
// logHealthChange 健康读值变化日志:按调度同源公式(succ^alpha 百分制,与 healthFactor
|
||||
// 及 chn_health_scores 落盘一致)折算健康分,**仅在健康分实际移动时**记 Info——稳态成功流量
|
||||
// 的 succ 恒为 1.0(只有延迟 EWMA 在抖),若每次上报都打印会在生产流量下淹没日志。
|
||||
// 健康分持平时降为 Debug,保留延迟变化的排查能力。
|
||||
func logHealthChange(ctx context.Context, o dispatch.Outcome, pol *dispatch.RoutingPolicy, oldSucc, newSucc, oldLat, newLat float64) {
|
||||
if oldSucc == newSucc && oldLat == newLat {
|
||||
return
|
||||
}
|
||||
src := "真实流量"
|
||||
if o.Probe {
|
||||
src = "探测"
|
||||
}
|
||||
result := "成功"
|
||||
if !o.Success {
|
||||
result = "失败(" + o.Class.String() + ")"
|
||||
}
|
||||
oldScore := math.Pow(oldSucc, pol.Health.Alpha) * 100
|
||||
newScore := math.Pow(newSucc, pol.Health.Alpha) * 100
|
||||
|
||||
if math.Abs(newScore-oldScore) < 0.05 {
|
||||
g.Log().Debugf(ctx, "[ChannelHealth] 渠道 %d 模型 %s 健康分持平 %.1f | lat_ewma %.1fms → %.1fms | 来源: %s %s",
|
||||
o.ChannelID, o.Model, newScore, oldLat, newLat, src, result)
|
||||
return
|
||||
}
|
||||
g.Log().Infof(ctx, "[ChannelHealth] 渠道 %d 模型 %s 健康分变化: %.1f → %.1f | succ_ewma %.4f → %.4f | lat_ewma %.1fms → %.1fms | 来源: %s %s",
|
||||
o.ChannelID, o.Model, oldScore, newScore, oldSucc, newSucc, oldLat, newLat, src, result)
|
||||
}
|
||||
|
||||
// luaBreakerManualReset 手动恢复:熔断复位为 CLOSED 并记录 recovered_ms(触发爬坡因子)。
|
||||
// ARGV: nowMs, keyTtlMs
|
||||
const luaBreakerManualReset = `
|
||||
|
||||
@@ -9,8 +9,6 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/gogf/gf/v2/frame/g"
|
||||
|
||||
v1 "github.com/qianfree/team-api/api/admin/v1"
|
||||
relaycommon "github.com/qianfree/team-api/relay/common"
|
||||
"github.com/qianfree/team-api/relay/constant"
|
||||
@@ -52,15 +50,12 @@ func doTestHTTPRequest(ctx context.Context, method, reqURL string, headers map[s
|
||||
}
|
||||
|
||||
// executeTest 封装通用的测试请求执行和结果处理逻辑
|
||||
func executeTest(ctx context.Context, provider, method, reqURL string, headers map[string]string, body []byte, useProxy bool) testResult {
|
||||
func executeTest(ctx context.Context, method, reqURL string, headers map[string]string, body []byte, useProxy bool) testResult {
|
||||
maskedHdrs := maskHeaders(headers)
|
||||
bodyStr := string(body)
|
||||
|
||||
g.Log().Infof(ctx, "[ChannelProbe]%s | 发送请求 | %s %s | 代理: %v", provider, method, reqURL, useProxy)
|
||||
|
||||
resp, err := doTestHTTPRequest(ctx, method, reqURL, headers, body, useProxy)
|
||||
if err != nil {
|
||||
g.Log().Warningf(ctx, "[ChannelProbe]%s | 请求失败 | %s | 错误: %v", provider, reqURL, err)
|
||||
return testResult{
|
||||
Error: fmt.Sprintf("请求失败: %v", err),
|
||||
Request: buildReqDetail(method, reqURL, maskedHdrs, bodyStr),
|
||||
@@ -72,7 +67,6 @@ func executeTest(ctx context.Context, provider, method, reqURL string, headers m
|
||||
respStr := string(respBody)
|
||||
|
||||
if resp.StatusCode != 200 {
|
||||
g.Log().Warningf(ctx, "[ChannelProbe]%s | 测试失败 | HTTP %d | 响应: %s", provider, resp.StatusCode, truncateStr(respStr, 200))
|
||||
return testResult{
|
||||
Success: false,
|
||||
Error: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, truncateStr(respStr, 500)),
|
||||
@@ -81,7 +75,6 @@ func executeTest(ctx context.Context, provider, method, reqURL string, headers m
|
||||
}
|
||||
}
|
||||
|
||||
g.Log().Infof(ctx, "[ChannelProbe]%s | 测试成功 | HTTP %d | 响应: %s", provider, resp.StatusCode, truncateStr(respStr, 200))
|
||||
return testResult{
|
||||
Success: true,
|
||||
Response: truncateStr(respStr, 500),
|
||||
@@ -101,7 +94,7 @@ func testOpenAI(ctx context.Context, baseURL, apiKey, modelName string, useProxy
|
||||
"Authorization": "Bearer " + apiKey,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
return executeTest(ctx, "OpenAI", "POST", reqURL, headers, bodyJSON, useProxy)
|
||||
return executeTest(ctx, "POST", reqURL, headers, bodyJSON, useProxy)
|
||||
}
|
||||
|
||||
func testClaude(ctx context.Context, baseURL, apiKey, modelName string, useProxy bool) testResult {
|
||||
@@ -117,7 +110,7 @@ func testClaude(ctx context.Context, baseURL, apiKey, modelName string, useProxy
|
||||
"anthropic-version": "2023-06-01",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
return executeTest(ctx, "Claude", "POST", reqURL, headers, bodyJSON, useProxy)
|
||||
return executeTest(ctx, "POST", reqURL, headers, bodyJSON, useProxy)
|
||||
}
|
||||
|
||||
func testGemini(ctx context.Context, baseURL, apiKey, modelName string, useProxy bool) testResult {
|
||||
@@ -135,7 +128,7 @@ func testGemini(ctx context.Context, baseURL, apiKey, modelName string, useProxy
|
||||
"x-goog-api-key": apiKey,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
return executeTest(ctx, "Gemini", "POST", reqURL, headers, bodyJSON, useProxy)
|
||||
return executeTest(ctx, "POST", reqURL, headers, bodyJSON, useProxy)
|
||||
}
|
||||
|
||||
func testZhipu(ctx context.Context, baseURL, apiKey, modelName string, useProxy bool) testResult {
|
||||
@@ -150,7 +143,7 @@ func testZhipu(ctx context.Context, baseURL, apiKey, modelName string, useProxy
|
||||
"Authorization": "Bearer " + apiKey,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
return executeTest(ctx, "Zhipu", "POST", reqURL, headers, bodyJSON, useProxy)
|
||||
return executeTest(ctx, "POST", reqURL, headers, bodyJSON, useProxy)
|
||||
}
|
||||
|
||||
func buildReqDetail(method, url string, headers map[string]string, body string) *v1.ChannelTestReqDetail {
|
||||
|
||||
@@ -108,8 +108,6 @@ func (s *sAdmin) TestChannel(ctx context.Context, req *v1.ChannelTestReq) (*v1.C
|
||||
}
|
||||
|
||||
// 发送最小测试请求
|
||||
g.Log().Infof(ctx, "[ChannelProbe] 渠道 %s (%d) 开始探测 | 模型: %s (上游: %s) | 渠道状态: %s | 代理: %v",
|
||||
ch.Name, channelID, testModel, upstreamModel, ch.Status, useProxy)
|
||||
startTime := time.Now()
|
||||
result := sendTestRequest(ctx, ch.Type, ch.BaseURL, apiKey, upstreamModel, useProxy)
|
||||
latencyMs := time.Since(startTime).Milliseconds()
|
||||
@@ -125,25 +123,16 @@ func (s *sAdmin) TestChannel(ctx context.Context, req *v1.ChannelTestReq) (*v1.C
|
||||
// "测试成功了但渠道健康度纹丝不动"(模型级能力列表直读 Redis 会立刻变,
|
||||
// 渠道健康分读的是 chn_health_scores 落盘值)
|
||||
dispatchadapter.RequestHealthSnapshot(channelID)
|
||||
if dispatchadapter.CatalogHasModel(channelID, testModel) {
|
||||
g.Log().Infof(ctx, "[ChannelProbe] 渠道 %s (%d) 探测结果已投递健康上报 | 模型: %s | success: %v | 延迟: %dms(EWMA 变化将打印 [ChannelHealth] 日志,5 分钟内落盘刷新)",
|
||||
ch.Name, channelID, testModel, result.Success, latencyMs)
|
||||
} else {
|
||||
// 探测目标不在调度目录:EWMA 照常写入 Redis,但调度不读、维护快照不落盘,
|
||||
// 管理后台健康分不会变化——探测看起来"生效了但没效果"的常见根因
|
||||
// 探测目标不在调度目录:EWMA 照常写入 Redis,但调度不读、维护快照不落盘,
|
||||
// 管理后台健康分不会变化——探测看起来"生效了但没效果"的常见根因,必须告警
|
||||
if !dispatchadapter.CatalogHasModel(channelID, testModel) {
|
||||
g.Log().Warningf(ctx, "[ChannelProbe] 渠道 %s (%d) 探测结果已投递健康上报,但模型 %s 不在调度目录(渠道无启用的模型能力行或该模型未配置能力)| 健康分不会落盘展示,渠道/模型不参与调度,请检查渠道的模型能力配置",
|
||||
ch.Name, channelID, testModel)
|
||||
}
|
||||
} else {
|
||||
g.Log().Infof(ctx, "[ChannelProbe] 渠道 %s (%d) 状态为 %s(非 active),探测结果不上报健康体系",
|
||||
ch.Name, channelID, ch.Status)
|
||||
}
|
||||
|
||||
// 记录测试结果日志
|
||||
if result.Success {
|
||||
g.Log().Infof(ctx, "[ChannelProbe] 渠道 %s (%d) 探测成功 | 模型: %s (上游: %s) | 延迟: %dms | 代理: %v",
|
||||
ch.Name, channelID, testModel, upstreamModel, latencyMs, useProxy)
|
||||
} else {
|
||||
// 仅失败需要日志留痕(成功结果已通过 API 响应返回)
|
||||
if !result.Success {
|
||||
g.Log().Warningf(ctx, "[ChannelProbe] 渠道 %s (%d) 探测失败 | 模型: %s (上游: %s) | 延迟: %dms | 代理: %v | 错误: %s",
|
||||
ch.Name, channelID, testModel, upstreamModel, latencyMs, useProxy, result.Error)
|
||||
}
|
||||
|
||||
@@ -193,8 +193,7 @@ return 1
|
||||
args := append([]any{activeSetKey}, staleIDs...)
|
||||
g.Redis().Do(ctx, "SREM", args...)
|
||||
}
|
||||
g.Log().Debugf(ctx, "[PREDEDUCT SWEEP] tenant=%d frozen recomputed: %d micro (stale members: %d)",
|
||||
tenantID, frozenMicro, len(staleIDs))
|
||||
// 心跳不打日志(每 2 分钟一条太吵),异常路径(围栏重试耗尽/Redis 故障)仍有 Warningf
|
||||
return
|
||||
}
|
||||
g.Log().Warningf(ctx, "[PREDEDUCT SWEEP] tenant=%d frozen recompute aborted after %d fenced retries (hot wallet)",
|
||||
|
||||
@@ -34,10 +34,8 @@ func AutoTestChannels(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
if len(channels) == 0 {
|
||||
g.Log().Infof(ctx, "[ChannelProbe] 自动探测开始: 无待探测渠道(活跃且配置了 test_model 的渠道为 0)")
|
||||
return
|
||||
}
|
||||
g.Log().Infof(ctx, "[ChannelProbe] 自动探测开始: 待探测活跃渠道 %d 个", len(channels))
|
||||
|
||||
successCount := 0
|
||||
failCount := 0
|
||||
@@ -48,10 +46,10 @@ func AutoTestChannels(ctx context.Context) {
|
||||
} else {
|
||||
failCount++
|
||||
}
|
||||
// 成功/失败明细日志由 TestChannel 统一打印(含模型/延迟/错误),此处只做轮次汇总
|
||||
// 失败明细由 TestChannel 以 Warning 打印(含模型/延迟/错误),此处仅 Debug 级轮次汇总
|
||||
}
|
||||
|
||||
g.Log().Infof(ctx, "[ChannelProbe] 自动探测完成: 共 %d 个渠道 | 成功 %d | 失败 %d",
|
||||
g.Log().Debugf(ctx, "[ChannelProbe] 自动探测完成: 共 %d 个渠道 | 成功 %d | 失败 %d",
|
||||
len(channels), successCount, failCount)
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user