From c3063e01a4bf8c389bcd9dbef1ef6bd92477e107 Mon Sep 17 00:00:00 2001 From: Perfecto Date: Sat, 15 Aug 2026 21:30:56 +0800 Subject: [PATCH 01/50] fix(openai): recover message-only capacity failures --- .../handler/openai_gateway_handler.go | 6 +- .../service/openai_capacity_shed_test.go | 80 ++++++++++ .../service/openai_compact_sse_keepalive.go | 27 ++-- .../openai_compact_sse_keepalive_test.go | 14 ++ .../openai_first_output_timeout_test.go | 11 +- .../service/openai_gateway_passthrough.go | 142 +++++++++++++++++- .../openai_gateway_response_handling.go | 66 +++++--- ...nai_gateway_service_codex_cli_only_test.go | 18 +++ .../service/openai_gateway_service_test.go | 10 +- .../service/openai_gateway_upstream_errors.go | 18 ++- .../service/openai_visible_ttft_test.go | 37 ++++- 11 files changed, 375 insertions(+), 54 deletions(-) diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index 1695eccc1..adecb57a7 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -567,7 +567,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { // Forward request service.SetOpsLatencyMs(c, service.OpsRoutingLatencyMsKey, time.Since(routingStart).Milliseconds()) forwardStart := time.Now() - // 用扣除 compact 心跳字节的口径快照：心跳注释不构成语义响应， + // 用扣除非语义心跳字节的口径快照：心跳注释不构成语义响应， // 不能因心跳字节变化而放弃 failover 换号（#3887）。 writerSizeBeforeForward := service.OpenAICompactKeepaliveAdjustedWrittenSize(c) // 跨 passthrough 边界的 failover：从 Kiro 等透传账号切到 Bedrock 等非透传账号前， @@ -618,7 +618,9 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { h.handleFailoverExhausted(c, failoverErr, true) return } - if failoverErr.SafeToFailoverAfterWrite && c.Writer.Written() { + // openAIForwardMayFailover 已确认写出的字节不含语义输出， + // 但重试耗尽时仍须按已提交的 SSE 响应返回流内错误。 + if c.Writer.Written() { streamStarted = true } if failoverErr.ShouldReportAccountScheduleFailure() { diff --git a/backend/internal/service/openai_capacity_shed_test.go b/backend/internal/service/openai_capacity_shed_test.go index e68a48db1..70153a0d0 100644 --- a/backend/internal/service/openai_capacity_shed_test.go +++ b/backend/internal/service/openai_capacity_shed_test.go @@ -94,6 +94,11 @@ func TestOpenAIStreamErrorFrameDoesNotStartClientOutput(t *testing.T) { {`{"type":"response.failed","response":{"error":{"code":"server_is_overloaded"}}}`, "response.failed", false}, {`{"type":"response.created","response":{"id":"resp_1"}}`, "response.created", false}, {`{"type":"response.in_progress","response":{"id":"resp_1"}}`, "response.in_progress", false}, + {`{"type":"response.output_item.added","item":{"type":"reasoning","summary":[]}}`, "response.output_item.added", false}, + {`{"type":"response.output_item.added","item":{"type":"reasoning","encrypted_content":"ciphertext"}}`, "response.output_item.added", true}, + {`{"type":"response.reasoning_summary_part.added","part":{"type":"summary_text","text":""}}`, "response.reasoning_summary_part.added", false}, + {`{"type":"response.reasoning_summary_part.added","part":{"type":"summary_text","text":"thinking"}}`, "response.reasoning_summary_part.added", true}, + {`{"type":"response.content_part.added","part":{"type":"output_text","text":""}}`, "response.content_part.added", false}, {`{"type":"response.output_text.delta","delta":"hi"}`, "response.output_text.delta", true}, {`[DONE]`, "", true}, } @@ -102,6 +107,69 @@ func TestOpenAIStreamErrorFrameDoesNotStartClientOutput(t *testing.T) { } } +func TestOpenAIStreamMetadataPreambleAndMessageOnlyOverloadFailOver(t *testing.T) { + gin.SetMode(gin.TestMode) + largeMetadata := strings.Repeat("x", 16*1024) + stream := strings.Join([]string{ + "event: response.created", + `data: {"type":"response.created","response":{"id":"resp_1","metadata":{"padding":"` + largeMetadata + `"}}}`, + "", + "event: response.output_item.added", + `data: {"type":"response.output_item.added","item":{"type":"reasoning","summary":[]}}`, + "", + "event: response.reasoning_summary_part.added", + `data: {"type":"response.reasoning_summary_part.added","part":{"type":"summary_text","text":""}}`, + "", + "event: error", + `data: {"type":"error","error":{"type":"service_unavailable_error","message":"Our servers are currently overloaded. Please try again later."}}`, + "", + }, "\n") + + tests := []struct { + name string + run func(*OpenAIGatewayService, *gin.Context, *http.Response, *Account) error + }{ + { + name: "native", + run: func(svc *OpenAIGatewayService, c *gin.Context, resp *http.Response, account *Account) error { + _, err := svc.handleStreamingResponse(c.Request.Context(), resp, c, account, time.Now(), "model", "model") + return err + }, + }, + { + name: "passthrough", + run: func(svc *OpenAIGatewayService, c *gin.Context, resp *http.Response, account *Account) error { + _, err := svc.handleStreamingResponsePassthrough(c.Request.Context(), resp, c, account, time.Now(), "model", "model") + return err + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc := &OpenAIGatewayService{cfg: &config.Config{Gateway: config.GatewayConfig{MaxLineSize: defaultMaxLineSize}}} + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/", nil) + resp := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(stream)), + Header: http.Header{"X-Request-Id": []string{"rid-message-only-overload"}}, + } + account := &Account{ID: 1, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Name: "acc"} + + err := tt.run(svc, c, resp, account) + require.Error(t, err) + var failoverErr *UpstreamFailoverError + require.ErrorAs(t, err, &failoverErr) + require.True(t, failoverErr.RetryableOnSameAccount) + require.True(t, failoverErr.RequestScopedTransient) + require.False(t, c.Writer.Written()) + require.Empty(t, rec.Body.String()) + }) + } +} + // 回归用例（真实上游降载序列）：created → in_progress → error 帧 → response.failed。 // 期望仍然走 pre-output failover（同账号重试 + 请求级瞬时标记），且不向客户端写出任何字节。 func TestOpenAIStreamCapacityShedErrorFramePrecedingFailedStillFailsOver(t *testing.T) { @@ -211,6 +279,18 @@ func TestSanitizeOpenAICapacityShedErrorCodeForClient(t *testing.T) { wantChanged: true, wantContain: `"code":"server_error"`, }, + { + name: "failed事件只有过载文案时补充code", + payload: `{"type":"response.failed","response":{"error":{"message":"Our servers are currently overloaded. Please try again later."}}}`, + wantChanged: true, + wantContain: `"code":"server_error"`, + }, + { + name: "error帧只有过载文案时补充code", + payload: `{"type":"error","error":{"message":"Server is overloaded. Please try again later."}}`, + wantChanged: true, + wantContain: `"code":"server_error"`, + }, { name: "rate_limit不改写", payload: `{"type":"response.failed","response":{"error":{"code":"rate_limit_exceeded","message":"try again in 3s"}}}`, diff --git a/backend/internal/service/openai_compact_sse_keepalive.go b/backend/internal/service/openai_compact_sse_keepalive.go index 70e6af945..360c1d55a 100644 --- a/backend/internal/service/openai_compact_sse_keepalive.go +++ b/backend/internal/service/openai_compact_sse_keepalive.go @@ -161,21 +161,28 @@ func OpenAICompactKeepaliveAdjustedWrittenSize(c *gin.Context) int { if c == nil || c.Writer == nil { return -1 } - value, ok := c.Get(openAICompactSSEKeepaliveKey) - if !ok { - return c.Writer.Size() + streamKeepaliveBytes := 0 + if value, ok := c.Get(openAIStreamKeepaliveBytesKey); ok { + streamKeepaliveBytes, _ = value.(int) } - k, ok := value.(*openAICompactSSEKeepalive) - if !ok || k == nil { - return c.Writer.Size() + size := c.Writer.Size() + compactKeepaliveBytes := 0 + if value, ok := c.Get(openAICompactSSEKeepaliveKey); ok { + if k, valid := value.(*openAICompactSSEKeepalive); valid && k != nil { + k.mu.Lock() + size = k.writer.Size() + compactKeepaliveBytes = k.bytes + k.mu.Unlock() + } } - k.mu.Lock() - defer k.mu.Unlock() - size := k.writer.Size() if size < 0 { return size } - if real := size - k.bytes; real > 0 { + keepaliveBytes := compactKeepaliveBytes + streamKeepaliveBytes + if keepaliveBytes <= 0 { + return size + } + if real := size - keepaliveBytes; real > 0 { return real } return -1 diff --git a/backend/internal/service/openai_compact_sse_keepalive_test.go b/backend/internal/service/openai_compact_sse_keepalive_test.go index 95ee4ce76..3703ea9f2 100644 --- a/backend/internal/service/openai_compact_sse_keepalive_test.go +++ b/backend/internal/service/openai_compact_sse_keepalive_test.go @@ -71,6 +71,20 @@ func TestOpenAICompactSSEKeepalive_StopBeforeFirstBeatKeepsWriterUntouched(t *te require.False(t, StopOpenAICompactSSEKeepaliveCommitted(c)) } +func TestOpenAIAdjustedWrittenSizeExcludesResponsesStreamKeepalive(t *testing.T) { + c, rec := newCompactBridgeTestContext(t, false) + n, err := c.Writer.Write([]byte(":\n\n")) + require.NoError(t, err) + recordOpenAIStreamKeepaliveBytes(c, n) + + require.Equal(t, -1, OpenAICompactKeepaliveAdjustedWrittenSize(c)) + + _, err = c.Writer.Write([]byte("data: semantic\n\n")) + require.NoError(t, err) + require.Equal(t, len("data: semantic\n\n"), OpenAICompactKeepaliveAdjustedWrittenSize(c)) + require.Equal(t, ":\n\ndata: semantic\n\n", rec.Body.String()) +} + // 心跳已提交后，2xx 桥接续写事件而不重复提交响应头。 func TestWriteOpenAICompactSSEBridge_AfterKeepaliveCommitAppendsEvents(t *testing.T) { c, rec := newCompactBridgeTestContext(t, true) diff --git a/backend/internal/service/openai_first_output_timeout_test.go b/backend/internal/service/openai_first_output_timeout_test.go index 799f7af12..6783de90d 100644 --- a/backend/internal/service/openai_first_output_timeout_test.go +++ b/backend/internal/service/openai_first_output_timeout_test.go @@ -542,7 +542,7 @@ func TestOpenAINativeFirstOutputScannerAllowsLargeEventAfterSemanticBoundary(t * require.Equal(t, "request-large-image", rec.Result().Header.Get("X-Request-Id")) } -func TestOpenAINativeFirstOutputTimeoutDisabledPreservesKeepaliveFlush(t *testing.T) { +func TestOpenAINativeFirstOutputTimeoutDisabledKeepsPreamblePrivateAcrossKeepalive(t *testing.T) { svc := &OpenAIGatewayService{cfg: &config.Config{Gateway: config.GatewayConfig{ StreamKeepaliveInterval: 1, MaxLineSize: defaultMaxLineSize, @@ -552,7 +552,7 @@ func TestOpenAINativeFirstOutputTimeoutDisabledPreservesKeepaliveFlush(t *testin defer func() { _ = pw.Close() }() _, _ = pw.Write([]byte("data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_stalled\"}}\n\n")) _, _ = pw.Write([]byte("data: {\"type\":\"response.in_progress\",\"response\":{\"id\":\"resp_stalled\"}}\n\n")) - time.Sleep(1100 * time.Millisecond) + time.Sleep(2100 * time.Millisecond) }() rec := httptest.NewRecorder() c, _ := gin.CreateTestContext(rec) @@ -561,10 +561,11 @@ func TestOpenAINativeFirstOutputTimeoutDisabledPreservesKeepaliveFlush(t *testin _, err := svc.handleStreamingResponse(c.Request.Context(), resp, c, &Account{ID: 1, Platform: PlatformOpenAI}, time.Now(), "model", "model") - require.Error(t, err) + var failoverErr *UpstreamFailoverError + require.ErrorAs(t, err, &failoverErr) require.Contains(t, rec.Body.String(), ":\n\n") - require.Contains(t, rec.Body.String(), "response.created") - require.Contains(t, rec.Body.String(), "response.in_progress") + require.NotContains(t, rec.Body.String(), "response.created") + require.NotContains(t, rec.Body.String(), "response.in_progress") } func TestOpenAINativeFirstOutputFailoverKeepsAttemptHeadersPrivateAfterKeepaliveCommit(t *testing.T) { diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index 1fac0440f..e9789d6b0 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -809,6 +809,19 @@ type openaiNonStreamingResultPassthrough struct { imageOutputSizes []string } +const openAIStreamKeepaliveBytesKey = "openai_stream_keepalive_bytes" + +func recordOpenAIStreamKeepaliveBytes(c *gin.Context, written int) { + if c == nil || written <= 0 { + return + } + current := 0 + if value, ok := c.Get(openAIStreamKeepaliveBytesKey); ok { + current, _ = value.(int) + } + c.Set(openAIStreamKeepaliveBytesKey, current+written) +} + func openAIStreamClientOutputStarted(c *gin.Context, localStarted bool) bool { if localStarted { return true @@ -831,6 +844,85 @@ func openAIStreamEventIsPreamble(eventType string) bool { } } +func openAIStreamAddedEventStartsClientOutput(payload []byte, eventType string) bool { + if len(payload) == 0 || !gjson.ValidBytes(payload) { + return true + } + + switch strings.TrimSpace(eventType) { + case "response.output_item.added": + item := gjson.GetBytes(payload, "item") + if !item.Exists() || !item.IsObject() { + return true + } + switch strings.TrimSpace(item.Get("type").String()) { + case "reasoning": + if item.Get("encrypted_content").String() != "" { + return true + } + summary := item.Get("summary") + if !summary.IsArray() { + return false + } + for _, part := range summary.Array() { + if strings.TrimSpace(part.Get("type").String()) != "summary_text" || part.Get("text").String() != "" { + return true + } + } + return false + case "message": + content := item.Get("content") + if !content.IsArray() { + return false + } + for _, part := range content.Array() { + switch strings.TrimSpace(part.Get("type").String()) { + case "output_text": + if part.Get("text").String() != "" { + return true + } + case "refusal": + if part.Get("refusal").String() != "" { + return true + } + default: + return true + } + } + return false + case "function_call": + return item.Get("arguments").String() != "" + case "custom_tool_call": + return item.Get("input").String() != "" + case "compaction": + return item.Get("encrypted_content").String() != "" + default: + return true + } + case "response.content_part.added": + part := gjson.GetBytes(payload, "part") + if !part.Exists() || !part.IsObject() { + return true + } + switch strings.TrimSpace(part.Get("type").String()) { + case "output_text": + return part.Get("text").String() != "" + case "refusal": + return part.Get("refusal").String() != "" + default: + return true + } + case "response.reasoning_summary_part.added": + part := gjson.GetBytes(payload, "part") + if !part.Exists() || !part.IsObject() || strings.TrimSpace(part.Get("type").String()) != "summary_text" { + return true + } + return part.Get("text").String() != "" + default: + return true + } +} + func openAIStreamDataStartsClientOutput(data, eventType string) bool { trimmed := strings.TrimSpace(data) if trimmed == "" { @@ -847,6 +939,8 @@ func openAIStreamDataStartsClientOutput(data, eventType string) bool { // （content_policy / invalid_request 等）维持原样转发，保留上游错误细节。 payload := []byte(trimmed) return !openAIStreamFailedEventShouldFailover(payload, extractOpenAISSEErrorMessage(payload)) + case "response.output_item.added", "response.content_part.added", "response.reasoning_summary_part.added": + return openAIStreamAddedEventStartsClientOutput([]byte(trimmed), eventType) } return !openAIStreamEventIsPreamble(eventType) } @@ -925,9 +1019,13 @@ func isOpenAIUpstreamCapacityShedEvent(payload []byte) bool { switch openAIStreamFailedEventErrorCode(payload) { case "server_is_overloaded", "slow_down": return true - default: - return false } + for _, path := range []string{"response.error.message", "error.message", "message"} { + if isOpenAICapacityShedMessage(gjson.GetBytes(payload, path).String()) { + return true + } + } + return false } // openAICapacityShedRetryableClientCode 是把上游容量降载错误转发给客户端时改写 @@ -950,9 +1048,12 @@ func sanitizeOpenAICapacityShedErrorCodeForClient(payload []byte) ([]byte, bool) updated := payload changed := false for _, path := range []string{"response.error.code", "error.code"} { - switch strings.ToLower(strings.TrimSpace(gjson.GetBytes(updated, path).String())) { - case "server_is_overloaded", "slow_down": - default: + parent := strings.TrimSuffix(path, ".code") + if !gjson.GetBytes(updated, parent).Exists() { + continue + } + code := strings.ToLower(strings.TrimSpace(gjson.GetBytes(updated, path).String())) + if code != "" && code != "server_is_overloaded" && code != "slow_down" { continue } next, err := sjson.SetBytes(updated, path, openAICapacityShedRetryableClientCode) @@ -985,7 +1086,7 @@ func openAIStreamFailedEventSemanticStatus(payload []byte, message string) int { return http.StatusUnauthorized case strings.Contains(combined, "permission") || strings.Contains(combined, "forbidden") || strings.Contains(combined, "access denied"): return http.StatusForbidden - case code == "server_is_overloaded" || code == "slow_down": + case isOpenAIUpstreamCapacityShedEvent(payload): return http.StatusServiceUnavailable default: return http.StatusBadGateway @@ -1119,6 +1220,16 @@ func openAIStreamFailedEventShouldFailover(payload []byte, message string) bool return true } +func openAIStreamErrorEventShouldFailover(payload []byte, message string) bool { + if strings.TrimSpace(gjson.GetBytes(payload, "type").String()) != "error" { + return false + } + if isOpenAIContextWindowError(message, payload) { + return false + } + return isOpenAITransientProcessingError(http.StatusBadRequest, message, payload) +} + func openAIStreamFailedEventRetryableOnSameAccount(account *Account, payload []byte, message string) bool { if account == nil { return false @@ -1347,6 +1458,25 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( } } eventType := strings.TrimSpace(gjson.Get(trimmedData, "type").String()) + if eventType == "error" && !openAIStreamClientOutputStarted(c, clientOutputStarted) { + errorMessage := extractOpenAISSEErrorMessage(dataBytes) + if status, errType, errMsg, matched := applyOpenAIStreamFailedErrorPassthroughRule(c, account.Platform, dataBytes, errorMessage); matched { + s.recordOpenAIStreamUpstreamError(c, account, true, upstreamRequestID, "http_error", dataBytes, errorMessage) + MarkResponseCommitted(c) + c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8") + c.JSON(status, gin.H{ + "error": gin.H{ + "type": errType, + "message": errMsg, + }, + }) + return resultWithUsage(), fmt.Errorf("upstream error event: passthrough rule matched message=%s", errMsg) + } + if openAIStreamErrorEventShouldFailover(dataBytes, errorMessage) { + return resultWithUsage(), + s.newOpenAIStreamFailoverError(c, account, true, upstreamRequestID, dataBytes, errorMessage, resp.Header) + } + } if eventType == "response.failed" { failedMessage = extractOpenAISSEErrorMessage(dataBytes) // response.failed 自带上游已消耗的 usage（input token 通常已扣）；必须先解析 diff --git a/backend/internal/service/openai_gateway_response_handling.go b/backend/internal/service/openai_gateway_response_handling.go index 82cd89f69..98475a4a3 100644 --- a/backend/internal/service/openai_gateway_response_handling.go +++ b/backend/internal/service/openai_gateway_response_handling.go @@ -54,8 +54,9 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. firstOutputTimeout = s.openAIFirstOutputTimeout(reasoningEffort) } guardFirstOutput := firstOutputTimeout > 0 + stageFirstOutput := account != nil && account.Platform == PlatformOpenAI var attemptResponseHeaders http.Header - if guardFirstOutput { + if stageFirstOutput { if s.responseHeaderFilter != nil { attemptResponseHeaders = responseheaders.FilterHeaders(resp.Header, s.responseHeaderFilter) } else if requestID := strings.TrimSpace(resp.Header.Get("x-request-id")); requestID != "" { @@ -66,8 +67,8 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. } // x-codex-turn-state 不在通用响应头白名单内，按 Codex 协议显式回传： // 客户端会在同回合的后续请求中回带（openai_codex_turn_state.go）。 - // 首输出守卫模式下只暂存，溯源在 applyAttemptResponseHeaders 真正提交时记录。 - if guardFirstOutput { + // OpenAI 首个语义输出前只暂存，溯源在 applyAttemptResponseHeaders 真正提交时记录。 + if stageFirstOutput { stageOpenAICodexTurnState(&attemptResponseHeaders, resp.Header) } else { s.relayOpenAICodexTurnState(c, account, resp.Header) @@ -80,12 +81,12 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. c.Header("X-Accel-Buffering", "no") // Pass through other headers - if !guardFirstOutput && resp.Header.Get("x-request-id") != "" { + if !stageFirstOutput && resp.Header.Get("x-request-id") != "" { v := resp.Header.Get("x-request-id") c.Header("x-request-id", v) } applyAttemptResponseHeaders := func() { - if !guardFirstOutput || len(attemptResponseHeaders) == 0 || c.Writer.Written() { + if !stageFirstOutput || len(attemptResponseHeaders) == 0 || c.Writer.Written() { return } for key, values := range attemptResponseHeaders { @@ -117,7 +118,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. firstOutputProgressObserved := false bufferedWriter := bufio.NewWriterSize(w, 4*1024) var firstOutputStage *openAIFirstOutputStage - if guardFirstOutput { + if stageFirstOutput { firstOutputStage = newDefaultOpenAIFirstOutputStage() defer func() { if err := firstOutputStage.Close(); err != nil { @@ -126,19 +127,19 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. }() } writePendingString := func(value string) (int, error) { - if firstOutputStage != nil && !firstOutputProgressObserved && !firstOutputStage.closed { + if firstOutputStage != nil && !firstOutputStage.closed { return firstOutputStage.WriteString(value) } return bufferedWriter.WriteString(value) } pendingBytes := func() int64 { - if firstOutputStage != nil && !firstOutputProgressObserved && !firstOutputStage.closed { + if firstOutputStage != nil && !firstOutputStage.closed { return firstOutputStage.Buffered() } return int64(bufferedWriter.Buffered()) } flushBuffered := func() error { - if firstOutputStage != nil && !firstOutputProgressObserved && !firstOutputStage.closed { + if firstOutputStage != nil && !firstOutputStage.closed { if err := firstOutputStage.CommitTo(w); err != nil { return err } @@ -155,11 +156,11 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. imageCounter := newOpenAIImageOutputCounter() responseID := "" var firstOutputScanGuard atomic.Bool - firstOutputScanGuard.Store(guardFirstOutput) + firstOutputScanGuard.Store(stageFirstOutput) scanner := bufio.NewScanner(resp.Body) scanBuf := getSSEScannerBuf64K() scanner.Buffer(scanBuf[:0], maxLineSize) - if guardFirstOutput { + if stageFirstOutput { scanner.Split(openAIFirstOutputDynamicScanLines(&firstOutputScanGuard)) } documentScanner := newOpenAISSEJSONDocumentScanner(scanner) @@ -250,7 +251,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. eventStartsVisibleOutput := false eventShouldFlush := false handlePendingWriteError := func(err error) { - if firstOutputStage != nil && !firstOutputProgressObserved && !firstOutputStage.closed { + if firstOutputStage != nil && !firstOutputStage.closed { message := "OpenAI first-output staging failed" if errors.Is(err, errOpenAIFirstOutputStageLimit) { message = "OpenAI first-output staging limit exceeded" @@ -350,7 +351,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. lastDownstreamWriteAt = time.Now() } finalizeStream := func() (*openaiStreamingResult, error) { - if guardFirstOutput && eventInProgress { + if stageFirstOutput && eventInProgress { // EOF dispatches the final SSE event even without a trailing blank line. completeGuardedEvent(true) } @@ -392,7 +393,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. failoverErr.SafeToFailoverAfterWrite = true return resultWithUsage(), failoverErr, true } - if errors.Is(scanErr, bufio.ErrTooLong) && guardFirstOutput && !firstOutputProgressObserved { + if errors.Is(scanErr, bufio.ErrTooLong) && stageFirstOutput && !firstOutputProgressObserved { logger.LegacyPrintf("service.openai_gateway", "SSE line too long before first output: account=%d max_size=%d error=%v", account.ID, maxLineSize, scanErr) failoverErr := s.newOpenAIStreamFailoverError( c, account, false, upstreamRequestID, nil, @@ -455,6 +456,26 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. responseID = extractOpenAIResponseIDFromJSONBytes(dataBytes) } forceFlushFailedEvent := false + if eventType == "error" && !openAIStreamClientOutputStarted(c, clientOutputStarted) { + errorMessage := extractOpenAISSEErrorMessage(dataBytes) + if status, errType, errMsg, matched := applyOpenAIStreamFailedErrorPassthroughRule(c, account.Platform, dataBytes, errorMessage); matched { + s.recordOpenAIStreamUpstreamError(c, account, false, upstreamRequestID, "http_error", dataBytes, errorMessage) + MarkResponseCommitted(c) + c.Writer.Header().Set("Content-Type", "application/json; charset=utf-8") + c.JSON(status, gin.H{ + "error": gin.H{ + "type": errType, + "message": errMsg, + }, + }) + streamEarlyErr = fmt.Errorf("upstream error event: passthrough rule matched message=%s", errMsg) + return + } + if openAIStreamErrorEventShouldFailover(dataBytes, errorMessage) { + streamEarlyErr = s.newOpenAIStreamFailoverError(c, account, false, upstreamRequestID, dataBytes, errorMessage, resp.Header) + return + } + } if eventType == "response.failed" { failedMessage = extractOpenAISSEErrorMessage(dataBytes) // response.failed 自带上游已消耗的 usage（input token 通常已扣）；必须先解析 @@ -559,9 +580,12 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. } startsClientOutput := forceFlushFailedEvent || openAIStreamDataStartsClientOutput(data, eventType) startsVisibleOutput := openAIStreamDataStartsVisibleOutput(data, eventType) - if guardFirstOutput { + if stageFirstOutput { eventStartsClientOutput = eventStartsClientOutput || startsClientOutput eventStartsVisibleOutput = eventStartsVisibleOutput || startsVisibleOutput + if startsClientOutput { + firstOutputScanGuard.Store(false) + } } if startsClientOutput && !openAIStreamEventTypeIsTerminal(eventType) { responsesSemanticOutputSeen = true @@ -607,7 +631,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. } // A blank line dispatches a guarded event from the attempt-local stage. - if guardFirstOutput && line == "" { + if stageFirstOutput && line == "" { if !clientDisconnected { if _, err := writePendingString("\n"); err != nil { handlePendingWriteError(err) @@ -672,7 +696,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. events := make(chan scanEvent, openAIFirstOutputEventQueueSize(guardFirstOutput)) done := make(chan struct{}) sendEvent := func(ev scanEvent) bool { - if guardFirstOutput { + if firstOutputScanGuard.Load() { ev.processed = make(chan struct{}) } select { @@ -716,7 +740,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. select { case ev, ok := <-events: if !ok { - if guardFirstOutput && eventInProgress { + if stageFirstOutput && eventInProgress { // EOF dispatches the final SSE event even without a trailing blank // line. Do not synthesize extra bytes on the downstream wire. completeGuardedEvent(true) @@ -783,10 +807,12 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. if time.Since(lastDownstreamWriteAt) < keepaliveInterval { continue } - if guardFirstOutput { + if stageFirstOutput { // Bypass attempt-local buffered frames. The stable SSE headers may be // committed here, but account headers remain private until semantic output. - if _, err := w.Write([]byte(":\n\n")); err != nil { + n, err := w.Write([]byte(":\n\n")) + recordOpenAIStreamKeepaliveBytes(c, n) + if err != nil { clientDisconnected = true logger.LegacyPrintf("service.openai_gateway", "Client disconnected during streaming, continuing to drain upstream for billing") continue diff --git a/backend/internal/service/openai_gateway_service_codex_cli_only_test.go b/backend/internal/service/openai_gateway_service_codex_cli_only_test.go index 846529828..c8acab035 100644 --- a/backend/internal/service/openai_gateway_service_codex_cli_only_test.go +++ b/backend/internal/service/openai_gateway_service_codex_cli_only_test.go @@ -287,6 +287,24 @@ func TestIsOpenAITransientProcessingError(t *testing.T) { []byte(`{"error":{"code":"slow_down","message":"Please retry later."}}`), )) + require.True(t, isOpenAITransientProcessingError( + http.StatusBadRequest, + "", + []byte(`{"error":{"message":"Our servers are currently overloaded. Please try again later."}}`), + )) + + require.True(t, isOpenAITransientProcessingError( + http.StatusServiceUnavailable, + "Server is overloaded. Please try again later.", + nil, + )) + + require.True(t, isOpenAITransientProcessingError( + http.StatusBadGateway, + "", + []byte(`{"error":{"message":"Our servers are currently overloaded. Please try again later."}}`), + )) + require.True(t, isOpenAITransientProcessingError( http.StatusBadRequest, "", diff --git a/backend/internal/service/openai_gateway_service_test.go b/backend/internal/service/openai_gateway_service_test.go index 617c07c06..c87da0955 100644 --- a/backend/internal/service/openai_gateway_service_test.go +++ b/backend/internal/service/openai_gateway_service_test.go @@ -204,17 +204,17 @@ func TestFailoverOpenAIUpstreamHTTPError_NilContextSkipsTempUnschedulablePolicy( "temp_unschedulable_enabled": true, "temp_unschedulable_rules": []any{map[string]any{ "error_code": float64(http.StatusBadRequest), - "keywords": []any{"servers are currently overloaded"}, + "keywords": []any{"custom temporary outage"}, "duration_minutes": float64(1), }}, }, } - body := []byte(`{"error":{"message":"Our servers are currently overloaded."}}`) + body := []byte(`{"error":{"message":"Custom temporary outage."}}`) resp := &http.Response{StatusCode: http.StatusBadRequest, Header: http.Header{}} got := svc.failoverOpenAIUpstreamHTTPError( context.Background(), nil, account, resp, body, - "Our servers are currently overloaded.", "gpt-5.4", + "Custom temporary outage.", "gpt-5.4", ) require.Nil(t, got) @@ -2288,7 +2288,7 @@ func TestOpenAIStreamingMissingTerminalEventReturnsIncompleteError(t *testing.T) go func() { defer func() { _ = pw.Close() }() - _, _ = pw.Write([]byte("data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\"},\"output_index\":0}\n\n")) + _, _ = pw.Write([]byte("data: {\"type\":\"response.output_text.delta\",\"delta\":\"partial\",\"output_index\":0}\n\n")) }() _, err := svc.handleStreamingResponse(c.Request.Context(), resp, c, &Account{ID: 1}, time.Now(), "model", "model") @@ -2320,7 +2320,7 @@ func TestOpenAIStreamingPassthroughMissingTerminalEventReturnsIncompleteError(t go func() { defer func() { _ = pw.Close() }() - _, _ = pw.Write([]byte("data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"message\"},\"output_index\":0}\n\n")) + _, _ = pw.Write([]byte("data: {\"type\":\"response.output_text.delta\",\"delta\":\"partial\",\"output_index\":0}\n\n")) }() _, err := svc.handleStreamingResponsePassthrough(c.Request.Context(), resp, c, &Account{ID: 1}, time.Now(), "", "") diff --git a/backend/internal/service/openai_gateway_upstream_errors.go b/backend/internal/service/openai_gateway_upstream_errors.go index 00a62f3f3..b4ecf6128 100644 --- a/backend/internal/service/openai_gateway_upstream_errors.go +++ b/backend/internal/service/openai_gateway_upstream_errors.go @@ -117,7 +117,7 @@ func isOpenAIInstructionsRequiredError(upstreamStatusCode int, upstreamMsg strin } func isOpenAITransientProcessingError(upstreamStatusCode int, upstreamMsg string, upstreamBody []byte) bool { - if upstreamStatusCode != http.StatusBadRequest && upstreamStatusCode != http.StatusServiceUnavailable { + if upstreamStatusCode < http.StatusBadRequest { return false } @@ -132,6 +132,15 @@ func isOpenAITransientProcessingError(upstreamStatusCode int, upstreamMsg string if len(upstreamBody) > 0 && hasOpenAIServerOverloadedCode(upstreamBody) { return true } + if isOpenAICapacityShedMessage(upstreamMsg) || + isOpenAICapacityShedMessage(gjson.GetBytes(upstreamBody, "error.message").String()) || + isOpenAICapacityShedMessage(gjson.GetBytes(upstreamBody, "response.error.message").String()) || + isOpenAICapacityShedMessage(string(upstreamBody)) { + return true + } + if upstreamStatusCode != http.StatusBadRequest && upstreamStatusCode != http.StatusServiceUnavailable { + return false + } if upstreamStatusCode != http.StatusBadRequest { return false } @@ -164,6 +173,13 @@ func isOpenAITransientProcessingError(upstreamStatusCode int, upstreamMsg string return match(string(upstreamBody)) } +func isOpenAICapacityShedMessage(text string) bool { + lower := strings.ToLower(strings.TrimSpace(text)) + return strings.Contains(lower, "server is overloaded") || + strings.Contains(lower, "servers are overloaded") || + strings.Contains(lower, "servers are currently overloaded") +} + func isOpenAIContextWindowError(upstreamMsg string, upstreamBody []byte) bool { match := func(text string) bool { lower := strings.ToLower(strings.TrimSpace(text)) diff --git a/backend/internal/service/openai_visible_ttft_test.go b/backend/internal/service/openai_visible_ttft_test.go index 8fd243c39..4d811dfbb 100644 --- a/backend/internal/service/openai_visible_ttft_test.go +++ b/backend/internal/service/openai_visible_ttft_test.go @@ -71,11 +71,38 @@ func TestOpenAIResponsesTTFTStartsAtCompletedImage(t *testing.T) { } } -func TestOpenAINativeProgressDisarmsTimeoutWithoutStartingTTFT(t *testing.T) { - result := runSyntheticVisibleTTFTStream(t, false, 1200*time.Millisecond, 1, - `{"type":"response.output_text.delta","delta":"test output"}`) - require.NotNil(t, result.firstTokenMs) - require.GreaterOrEqual(t, *result.firstTokenMs, 1100) +func TestOpenAINativeMetadataDoesNotDisarmFirstOutputTimeout(t *testing.T) { + gin.SetMode(gin.TestMode) + svc := &OpenAIGatewayService{cfg: &config.Config{Gateway: config.GatewayConfig{ + MaxLineSize: defaultMaxLineSize, + OpenAIFirstOutputTimeoutSeconds: 1, + }}} + reader, writer := io.Pipe() + writerDone := make(chan struct{}) + go func() { + defer close(writerDone) + defer func() { _ = writer.Close() }() + _, _ = io.WriteString(writer, "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_test\"}}\n\n") + _, _ = io.WriteString(writer, "data: {\"type\":\"response.output_item.added\",\"item\":{\"id\":\"item_test\",\"type\":\"reasoning\",\"summary\":[]}}\n\n") + time.Sleep(1200 * time.Millisecond) + }() + + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + resp := &http.Response{StatusCode: http.StatusOK, Header: http.Header{}, Body: reader} + account := &Account{ID: 1, Name: "account_test", Platform: PlatformOpenAI} + + _, err := svc.handleStreamingResponse(context.Background(), resp, c, account, time.Now(), "test-model", "test-model") + var failoverErr *UpstreamFailoverError + require.ErrorAs(t, err, &failoverErr) + require.True(t, failoverErr.SafeToFailoverAfterWrite) + require.Empty(t, recorder.Body.String()) + select { + case <-writerDone: + case <-time.After(time.Second): + t.Fatal("synthetic upstream writer did not exit") + } } func runSyntheticVisibleTTFTStream(t *testing.T, passthrough bool, visibleDelay time.Duration, timeoutSeconds int, visibleEvent string) *openaiStreamingResult { From 539064798965888f14605eb76c2733cd58851c7f Mon Sep 17 00:00:00 2001 From: Perfecto Date: Sat, 15 Aug 2026 22:54:21 +0800 Subject: [PATCH 02/50] fix(openai): complete request-scoped capacity recovery --- .../openai_account_runtime_block_fastpath.go | 5 + .../service/openai_capacity_shed_test.go | 36 +++++++ .../service/openai_gateway_passthrough.go | 29 ++++++ .../openai_gateway_response_handling.go | 8 ++ .../service/openai_gateway_service_test.go | 8 +- .../service/openai_gateway_upstream_errors.go | 11 ++- .../internal/service/openai_ws_http_bridge.go | 93 +++++++++++++----- .../service/openai_ws_http_bridge_test.go | 96 +++++++++++++++++-- 8 files changed, 250 insertions(+), 36 deletions(-) diff --git a/backend/internal/service/openai_account_runtime_block_fastpath.go b/backend/internal/service/openai_account_runtime_block_fastpath.go index ace8366a0..f8c3de54d 100644 --- a/backend/internal/service/openai_account_runtime_block_fastpath.go +++ b/backend/internal/service/openai_account_runtime_block_fastpath.go @@ -55,6 +55,11 @@ func (s *OpenAIGatewayService) handleOpenAIAccountUpstreamError(ctx context.Cont if s != nil { scheduleOllamaCloudUsageActivity(s.deferredService, account) } + // Capacity shedding describes this request, not account health. Keep the + // account schedulable while the request-local retry budget handles recovery. + if account != nil && account.Platform == PlatformOpenAI && isOpenAIRequestScopedCapacityShed("", responseBody) { + return false + } stateCtx, cancel := openAIAccountStateContext(ctx) defer cancel() diff --git a/backend/internal/service/openai_capacity_shed_test.go b/backend/internal/service/openai_capacity_shed_test.go index 70153a0d0..b529dd7e7 100644 --- a/backend/internal/service/openai_capacity_shed_test.go +++ b/backend/internal/service/openai_capacity_shed_test.go @@ -77,6 +77,37 @@ func TestStreamFailedEventCapacityShedRetriesOnSameAccount(t *testing.T) { require.False(t, openAIStreamFailedEventRetryableOnSameAccount(nonPool, other, "boom")) } +func TestOpenAIHTTPCapacityShedIsRequestScopedForOAuthAccounts(t *testing.T) { + payload := []byte(`{"error":{"type":"server_error","message":"Our servers are currently overloaded. Please try again later."}}`) + failoverErr := newOpenAIUpstreamFailoverError( + http.StatusBadRequest, + http.Header{"X-Request-Id": []string{"rid-http-capacity"}}, + payload, + "Our servers are currently overloaded. Please try again later.", + false, + ) + + require.True(t, failoverErr.RetryableOnSameAccount) + require.True(t, failoverErr.RequestScopedTransient) + + repo := &capacityShedAccountRepoStub{} + (&GatewayService{accountRepo: repo}).TempUnscheduleRetryableError(context.Background(), 1, failoverErr) + require.Zero(t, repo.tempUnschedCalls) + + rateLimitService := NewRateLimitService(repo, nil, &config.Config{}, nil, nil) + gateway := &OpenAIGatewayService{rateLimitService: rateLimitService} + account := &Account{ID: 1, Platform: PlatformOpenAI, Type: AccountTypeOAuth} + require.False(t, gateway.handleOpenAIAccountUpstreamError( + context.Background(), + account, + http.StatusBadRequest, + nil, + payload, + "gpt-5", + )) + require.Zero(t, repo.tempUnschedCalls) +} + // 上游降载的真实序列是「event: error → event: response.failed」。error 帧不算 // 客户端输出：若把它当首输出 flush，clientOutputStarted 被固化，随后的 failed // 事件就进不了 pre-output failover 分支，只能把致命错误原样转发给客户端。 @@ -217,6 +248,8 @@ func TestOpenAIStreamCapacityShedErrorFramePrecedingFailedStillFailsOver(t *test // 并终止会话，对其余错误码执行内置退避重试。消息原样保留。 func TestOpenAIStreamCapacityShedAfterOutputRewritesCodeForClient(t *testing.T) { gin.SetMode(gin.TestMode) + logSink, restore := captureStructuredLog(t) + defer restore() cfg := &config.Config{ Gateway: config.GatewayConfig{MaxLineSize: defaultMaxLineSize}, } @@ -256,6 +289,9 @@ func TestOpenAIStreamCapacityShedAfterOutputRewritesCodeForClient(t *testing.T) require.Contains(t, body, `"code":"server_error"`) require.NotContains(t, body, "server_is_overloaded") require.Contains(t, body, "Our servers are currently overloaded") + require.True(t, logSink.ContainsMessage("gateway.failover_suppressed_after_semantic_output")) + require.True(t, logSink.ContainsFieldValue("path", "native_sse")) + require.True(t, logSink.ContainsFieldValue("upstream_request_id", "rid-shed-after-output")) } // helper 单测：只有降载码被改写，其余错误码（尤其 rate_limit_exceeded，客户端 diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index e9789d6b0..9f5e6e8f5 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -1028,6 +1028,27 @@ func isOpenAIUpstreamCapacityShedEvent(payload []byte) bool { return false } +func logOpenAICapacityFailoverSuppressed( + ctx context.Context, + account *Account, + path string, + upstreamRequestID string, + eventType string, +) { + fields := []zap.Field{ + zap.String("path", path), + zap.String("event_type", strings.TrimSpace(eventType)), + zap.String("upstream_request_id", strings.TrimSpace(upstreamRequestID)), + } + if account != nil { + fields = append(fields, + zap.Int64("account_id", account.ID), + zap.String("platform", account.Platform), + ) + } + logger.FromContext(ctx).Warn("gateway.failover_suppressed_after_semantic_output", fields...) +} + // openAICapacityShedRetryableClientCode 是把上游容量降载错误转发给客户端时改写 // 使用的错误码。Codex CLI 按闭集对错误码分类：server_is_overloaded / slow_down // 被判为致命错误（客户端提示 "Selected model is at capacity. Please try a @@ -1372,6 +1393,7 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( sawTerminalEvent := false sawFailedEvent := false semanticOutputSeen := false + capacityFailoverSuppressedLogged := false failedMessage := "" clientOutputStarted := false upstreamRequestID := strings.TrimSpace(resp.Header.Get("x-request-id")) @@ -1458,6 +1480,13 @@ func (s *OpenAIGatewayService) handleStreamingResponsePassthrough( } } eventType := strings.TrimSpace(gjson.Get(trimmedData, "type").String()) + if !capacityFailoverSuppressedLogged && account != nil && account.Platform == PlatformOpenAI && + (eventType == "error" || eventType == "response.failed") && + openAIStreamClientOutputStarted(c, clientOutputStarted) && + isOpenAIUpstreamCapacityShedEvent(dataBytes) { + logOpenAICapacityFailoverSuppressed(ctx, account, "passthrough_sse", upstreamRequestID, eventType) + capacityFailoverSuppressedLogged = true + } if eventType == "error" && !openAIStreamClientOutputStarted(c, clientOutputStarted) { errorMessage := extractOpenAISSEErrorMessage(dataBytes) if status, errType, errMsg, matched := applyOpenAIStreamFailedErrorPassthroughRule(c, account.Platform, dataBytes, errorMessage); matched { diff --git a/backend/internal/service/openai_gateway_response_handling.go b/backend/internal/service/openai_gateway_response_handling.go index 98475a4a3..5577e90e4 100644 --- a/backend/internal/service/openai_gateway_response_handling.go +++ b/backend/internal/service/openai_gateway_response_handling.go @@ -242,6 +242,7 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. sawTerminalEvent := false sawFailedEvent := false responsesSemanticOutputSeen := false + capacityFailoverSuppressedLogged := false failedMessage := "" clientOutputStarted := false upstreamRequestID := strings.TrimSpace(resp.Header.Get("x-request-id")) @@ -456,6 +457,13 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. responseID = extractOpenAIResponseIDFromJSONBytes(dataBytes) } forceFlushFailedEvent := false + if !capacityFailoverSuppressedLogged && account != nil && account.Platform == PlatformOpenAI && + (eventType == "error" || eventType == "response.failed") && + openAIStreamClientOutputStarted(c, clientOutputStarted) && + isOpenAIUpstreamCapacityShedEvent(dataBytes) { + logOpenAICapacityFailoverSuppressed(ctx, account, "native_sse", upstreamRequestID, eventType) + capacityFailoverSuppressedLogged = true + } if eventType == "error" && !openAIStreamClientOutputStarted(c, clientOutputStarted) { errorMessage := extractOpenAISSEErrorMessage(dataBytes) if status, errType, errMsg, matched := applyOpenAIStreamFailedErrorPassthroughRule(c, account.Platform, dataBytes, errorMessage); matched { diff --git a/backend/internal/service/openai_gateway_service_test.go b/backend/internal/service/openai_gateway_service_test.go index c87da0955..6dfb88d80 100644 --- a/backend/internal/service/openai_gateway_service_test.go +++ b/backend/internal/service/openai_gateway_service_test.go @@ -115,7 +115,7 @@ func (r stubOpenAIAccountRepo) ListSchedulableUngroupedByPlatform(ctx context.Co return r.ListSchedulableByPlatform(ctx, platform) } -func TestOpenAIGatewayService_ForwardAsAnthropic_TempUnschedulableReturnsFailoverWithoutCommit(t *testing.T) { +func TestOpenAIGatewayService_ForwardAsAnthropic_CapacityShedReturnsRequestScopedFailoverWithoutCommit(t *testing.T) { gin.SetMode(gin.TestMode) body := []byte(`{"model":"gpt-5.4","max_tokens":32,"messages":[{"role":"user","content":"hello"}],"stream":false}`) @@ -173,8 +173,10 @@ func TestOpenAIGatewayService_ForwardAsAnthropic_TempUnschedulableReturnsFailove require.ErrorAs(t, err, &failoverErr) require.Equal(t, http.StatusBadRequest, failoverErr.StatusCode) require.True(t, failoverErr.ShouldRetryNextAccount()) - require.Equal(t, account.ID, repo.modelRateLimitAccountID, "temporary unschedulability should exclude this account from reselection") - require.Equal(t, "gpt-5.4", repo.modelRateLimitKey) + require.True(t, failoverErr.RetryableOnSameAccount) + require.True(t, failoverErr.RequestScopedTransient) + require.Zero(t, repo.modelRateLimitAccountID, "request-scoped capacity shedding must not change account health") + require.Empty(t, repo.modelRateLimitKey) require.False(t, IsResponseCommitted(c)) require.Equal(t, http.StatusOK, rec.Code) require.Empty(t, rec.Body.String()) diff --git a/backend/internal/service/openai_gateway_upstream_errors.go b/backend/internal/service/openai_gateway_upstream_errors.go index b4ecf6128..32f04b2c6 100644 --- a/backend/internal/service/openai_gateway_upstream_errors.go +++ b/backend/internal/service/openai_gateway_upstream_errors.go @@ -180,6 +180,12 @@ func isOpenAICapacityShedMessage(text string) bool { strings.Contains(lower, "servers are currently overloaded") } +func isOpenAIRequestScopedCapacityShed(upstreamMsg string, upstreamBody []byte) bool { + return isOpenAIUpstreamCapacityShedEvent(upstreamBody) || + isOpenAICapacityShedMessage(upstreamMsg) || + isOpenAICapacityShedMessage(string(upstreamBody)) +} + func isOpenAIContextWindowError(upstreamMsg string, upstreamBody []byte) bool { match := func(text string) bool { lower := strings.ToLower(strings.TrimSpace(text)) @@ -264,14 +270,17 @@ func newOpenAIUpstreamFailoverError( upstreamMsg string, retryableOnSameAccount bool, ) *UpstreamFailoverError { + requestScopedCapacity := isOpenAIRequestScopedCapacityShed(upstreamMsg, responseBody) failoverErr := &UpstreamFailoverError{ StatusCode: statusCode, ResponseBody: responseBody, ResponseHeaders: responseHeaders.Clone(), - RetryableOnSameAccount: retryableOnSameAccount, + RetryableOnSameAccount: retryableOnSameAccount || requestScopedCapacity, + RequestScopedTransient: requestScopedCapacity, } if isOpenAIRequestBodyTooLargeError(statusCode, upstreamMsg, responseBody) { failoverErr.RetryableOnSameAccount = false + failoverErr.RequestScopedTransient = false failoverErr.Scope = GatewayFailureScopeAccount failoverErr.Reason = openAIRequestBodyTooLargeReason failoverErr.NextAccountAction = NextAccountRetry diff --git a/backend/internal/service/openai_ws_http_bridge.go b/backend/internal/service/openai_ws_http_bridge.go index 87abf1ef0..69f3490f1 100644 --- a/backend/internal/service/openai_ws_http_bridge.go +++ b/backend/internal/service/openai_ws_http_bridge.go @@ -282,6 +282,9 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( upstreamTerminalEvent := "" sawDone := false wroteDownstream := false + pendingClientMessages := make([][]byte, 0, 4) + pendingClientMessageBytes := int64(0) + capacityFailoverSuppressedLogged := false clientDisconnected := false mappedModel := "" needModelReplace := false @@ -392,15 +395,20 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( replayCollector.AddEvent(eventType, upstreamMessage) var upstreamEventErr error - if eventType == "error" { - errCodeRaw, errTypeRaw, errMsgRaw := parseOpenAIWSErrorEventFields(upstreamMessage) - errMessage := strings.TrimSpace(errMsgRaw) + if eventType == "error" || eventType == "response.failed" { + errMessage := extractOpenAISSEErrorMessage(upstreamMessage) if errMessage == "" { errMessage = "upstream error event" } - statusCode := openAIWSErrorHTTPStatusFromRaw(errCodeRaw, errTypeRaw) - shouldFailover := s.shouldFailoverOpenAIUpstreamResponse(statusCode, errMessage, upstreamMessage) - if account.Platform == PlatformGrok { + statusCode := openAIStreamFailureStatus(upstreamMessage, errMessage) + shouldFailover := openAIStreamFailedEventShouldFailover(upstreamMessage, errMessage) + if eventType == "error" { + errCodeRaw, errTypeRaw, _ := parseOpenAIWSErrorEventFields(upstreamMessage) + statusCode = openAIWSErrorHTTPStatusFromRaw(errCodeRaw, errTypeRaw) + shouldFailover = s.shouldFailoverOpenAIUpstreamResponse(statusCode, errMessage, upstreamMessage) + } + requestScopedCapacity := isOpenAIUpstreamCapacityShedEvent(upstreamMessage) + if account.Platform == PlatformGrok && eventType == "error" { // SSE error events do not carry an HTTP status. The local status // mapper therefore defaults unknown xAI codes (for example // new_sensitive) to 502; classify the body as a request-scoped @@ -411,7 +419,7 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( shouldFailover = s.shouldFailoverGrokUpstreamError(statusCode, upstreamMessage) s.handleGrokAccountUpstreamError(ctx, account, statusCode, resp.Header, upstreamMessage) } - } else if shouldFailover { + } else if eventType == "error" && shouldFailover && !requestScopedCapacity { accountStatus := statusCode if transientStatus := openAIWSPayloadTransientStatus(upstreamMessage); transientStatus != 0 { accountStatus = transientStatus @@ -420,9 +428,18 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( s.handleOpenAIAccountUpstreamError(ctx, account, accountStatus, resp.Header, upstreamMessage, canonicalModel) } if turn == 1 && !wroteDownstream && shouldFailover { - return nil, newOpenAIUpstreamFailoverError(statusCode, resp.Header, upstreamMessage, errMessage, false) + if account.Platform == PlatformGrok { + return nil, newOpenAIUpstreamFailoverError(statusCode, resp.Header, upstreamMessage, errMessage, false) + } + return nil, s.newOpenAIStreamFailoverError(c, account, true, resp.Header.Get("x-request-id"), upstreamMessage, errMessage, resp.Header) + } + if wroteDownstream && requestScopedCapacity && !capacityFailoverSuppressedLogged { + logOpenAICapacityFailoverSuppressed(ctx, account, "ws_http_bridge", resp.Header.Get("x-request-id"), eventType) + capacityFailoverSuppressedLogged = true + } + if eventType == "error" { + upstreamEventErr = errors.New(errMessage) } - upstreamEventErr = errors.New(errMessage) } // 客户端写出副本改写容量降载码：Codex 对 error/response.failed 中的 @@ -436,26 +453,50 @@ func (s *OpenAIGatewayService) proxyOpenAIWSHTTPBridgeTurn( } } if !clientDisconnected { - if err := writeClientMessage(clientMessage); err != nil { - if isOpenAIWSClientDisconnectError(err) { - clientDisconnected = true - closeStatus, closeReason := summarizeOpenAIWSReadCloseError(err) - logOpenAIWSModeInfo( - "ingress_ws_http_bridge_client_disconnected_drain account_id=%d turn=%d close_status=%s close_reason=%s", - account.ID, - turn, - closeStatus, - truncateOpenAIWSLogValue(closeReason, openAIWSHeaderValueMaxLen), - ) - } else { - return nil, wrapOpenAIWSIngressTurnError( - "write_client", - fmt.Errorf("write client websocket event: %w", err), - wroteDownstream, + stageBeforeSemanticOutput := turn == 1 && account.Platform == PlatformOpenAI && !wroteDownstream + commitStagedMessages := !stageBeforeSemanticOutput || + openAIStreamDataStartsClientOutput(string(clientMessage), eventType) || + isOpenAIWSTerminalEvent(eventType) + if stageBeforeSemanticOutput && !commitStagedMessages { + if pendingClientMessageBytes+int64(len(clientMessage)) > openAIFirstOutputStageMaxBytes { + return nil, s.newOpenAIStreamFailoverError( + c, + account, + true, + resp.Header.Get("x-request-id"), + nil, + "OpenAI WS HTTP bridge first-output staging limit exceeded", + resp.Header, ) } + pendingClientMessages = append(pendingClientMessages, append([]byte(nil), clientMessage...)) + pendingClientMessageBytes += int64(len(clientMessage)) } else { - wroteDownstream = true + messages := append(pendingClientMessages, clientMessage) + pendingClientMessages = nil + pendingClientMessageBytes = 0 + for _, message := range messages { + if err := writeClientMessage(message); err != nil { + if isOpenAIWSClientDisconnectError(err) { + clientDisconnected = true + closeStatus, closeReason := summarizeOpenAIWSReadCloseError(err) + logOpenAIWSModeInfo( + "ingress_ws_http_bridge_client_disconnected_drain account_id=%d turn=%d close_status=%s close_reason=%s", + account.ID, + turn, + closeStatus, + truncateOpenAIWSLogValue(closeReason, openAIWSHeaderValueMaxLen), + ) + break + } + return nil, wrapOpenAIWSIngressTurnError( + "write_client", + fmt.Errorf("write client websocket event: %w", err), + wroteDownstream, + ) + } + wroteDownstream = true + } } } diff --git a/backend/internal/service/openai_ws_http_bridge_test.go b/backend/internal/service/openai_ws_http_bridge_test.go index 1e352f012..3c9a97c44 100644 --- a/backend/internal/service/openai_ws_http_bridge_test.go +++ b/backend/internal/service/openai_ws_http_bridge_test.go @@ -239,10 +239,9 @@ func TestProxyOpenAIWSHTTPBridgeTurnRewritesCapacityShedCodeForClient(t *testing wantErr: true, }, { - // response.failed 不走 error 事件分支：即便 turn 1 也会被当终止事件 - // 原样转发（不 failover），因此改写必须在这里同样生效。 - name: "turn1_bare_response_failed", - turn: 1, + // 后续 turn 不允许 replay，容量错误必须改写后交给客户端重试。 + name: "turn2_bare_response_failed", + turn: 2, body: "data: {\"type\":\"response.failed\",\"response\":{\"id\":\"resp_shed\",\"status\":\"failed\",\"error\":{\"code\":\"server_is_overloaded\",\"message\":\"Our servers are currently overloaded. Please try again later.\"}}}\n\n", }, } @@ -283,6 +282,91 @@ func TestProxyOpenAIWSHTTPBridgeTurnRewritesCapacityShedCodeForClient(t *testing } } +func TestProxyOpenAIWSHTTPBridgeTurnStagesMetadataBeforeCapacityFailover(t *testing.T) { + gin.SetMode(gin.TestMode) + body := strings.Join([]string{ + `data: {"type":"response.created","response":{"id":"resp_shed"}}`, + "", + `data: {"type":"response.in_progress","response":{"id":"resp_shed"}}`, + "", + `data: {"type":"response.failed","response":{"id":"resp_shed","status":"failed","error":{"message":"Our servers are currently overloaded. Please try again later."}}}`, + "", + }, "\n") + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"X-Request-Id": []string{"rid-ws-bridge-capacity"}}, + Body: io.NopCloser(strings.NewReader(body)), + }} + svc := &OpenAIGatewayService{cfg: &config.Config{}, httpUpstream: upstream} + account := &Account{ID: 12, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Concurrency: 1} + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil) + payload := []byte(`{"type":"response.create","model":"gpt-5","input":"hi"}`) + var writes [][]byte + + result, err := svc.proxyOpenAIWSHTTPBridgeTurn( + context.Background(), c, account, "sk-test", payload, len(payload), + "gpt-5", "", "", "", "", 1, + func(message []byte) error { + writes = append(writes, append([]byte(nil), message...)) + return nil + }, + ) + + require.Nil(t, result) + var failoverErr *UpstreamFailoverError + require.ErrorAs(t, err, &failoverErr) + require.True(t, failoverErr.RetryableOnSameAccount) + require.True(t, failoverErr.RequestScopedTransient) + require.Empty(t, writes) +} + +func TestProxyOpenAIWSHTTPBridgeTurnDoesNotReplayCapacityAfterSemanticOutput(t *testing.T) { + gin.SetMode(gin.TestMode) + logSink, restore := captureStructuredLog(t) + defer restore() + body := strings.Join([]string{ + `data: {"type":"response.created","response":{"id":"resp_partial"}}`, + "", + `data: {"type":"response.output_text.delta","delta":"partial"}`, + "", + `data: {"type":"response.failed","response":{"id":"resp_partial","status":"failed","error":{"code":"server_is_overloaded","message":"Our servers are currently overloaded. Please try again later."}}}`, + "", + }, "\n") + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"X-Request-Id": []string{"rid-ws-bridge-post-output"}}, + Body: io.NopCloser(strings.NewReader(body)), + }} + svc := &OpenAIGatewayService{cfg: &config.Config{}, httpUpstream: upstream} + account := &Account{ID: 13, Platform: PlatformOpenAI, Type: AccountTypeOAuth, Concurrency: 1} + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil) + payload := []byte(`{"type":"response.create","model":"gpt-5","input":"hi"}`) + var writes [][]byte + + result, err := svc.proxyOpenAIWSHTTPBridgeTurn( + context.Background(), c, account, "sk-test", payload, len(payload), + "gpt-5", "", "", "", "", 1, + func(message []byte) error { + writes = append(writes, append([]byte(nil), message...)) + return nil + }, + ) + + require.NotNil(t, result) + require.NoError(t, err) + require.Len(t, writes, 3) + var failoverErr *UpstreamFailoverError + require.False(t, errors.As(err, &failoverErr)) + require.Contains(t, string(writes[2]), `"code":"server_error"`) + require.NotContains(t, string(writes[2]), "server_is_overloaded") + require.True(t, logSink.ContainsMessage("gateway.failover_suppressed_after_semantic_output")) + require.True(t, logSink.ContainsFieldValue("path", "ws_http_bridge")) +} + func TestProxyOpenAIWSHTTPBridgeTurnRequiresTerminalEvent(t *testing.T) { gin.SetMode(gin.TestMode) @@ -294,10 +378,10 @@ func TestProxyOpenAIWSHTTPBridgeTurnRequiresTerminalEvent(t *testing.T) { }{ {name: "done_without_events_fails_over", body: "data: [DONE]\n\n", wantFailover: true}, { - name: "created_then_done_is_truncated_not_success", + name: "created_then_done_fails_over_before_semantic_output", body: "data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_truncated\"}}\n\n" + "data: [DONE]\n\n", - wantWrites: 1, + wantFailover: true, }, } for _, tt := range tests { From 612436a5a7cbb7ab9d4ce4174fcdff948fab31fb Mon Sep 17 00:00:00 2001 From: lbyxiaolizi Date: Mon, 17 Aug 2026 16:48:36 +0800 Subject: [PATCH 03/50] =?UTF-8?q?fix(openai-compat):=20Responses=E2=86=92C?= =?UTF-8?q?hat=20=E6=A1=A5=E6=8E=A5=E6=8C=89=20reasoning=20item=20id=20?= =?UTF-8?q?=E7=BC=93=E5=AD=98=E5=9B=9E=E6=B3=A8=20reasoning=5Fcontent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复 #5520：Codex 经 force_chat_completions 桥接到 DeepSeek thinking 上游时， 历史中的 encrypted-only reasoning item（summary 为空 + 不透明 encrypted_content， 远程 compaction / 跨会话恢复后常见）取不出明文，后续 assistant 消息缺 reasoning_content，DeepSeek 400 "The reasoning_content in the thinking mode must be passed back to the API"，客户端仅看到通用 502。 reasoning item 的 id 一定会被客户端回传，以其为 key 做服务端缓存： - GatewayCache 新增 Set/GetReasoningContent（Redis，默认 TTL 7 天） - 响应侧：流式扫 response.output_item.done、非流式扫 output，把 reasoning 全文按 item id 写缓存；客户端断连后 drain 期间用 detached ctx 仍会写完 - 请求侧：apicompat 新增 ResponsesToChatCompletionsRequestWithOptions 与 ReasoningContentByID 钩子，encrypted-only item 查缓存补回 pendingReasoning； 缓存 miss/出错一律 fail-open 维持原行为 - 自愈：历史里带明文 summary 的 reasoning item 顺手刷新缓存，覆盖 Redis flush / 跨实例漂移 测试：apicompat（命中恢复/miss 保持原样/明文优先）、service 端到端（流式 写缓存、请求侧回注+自愈）、repository miniredis 存取。 --- .../chatcompletions_responses_bridge.go | 63 +++++++- ...pletions_responses_reasoning_cache_test.go | 115 ++++++++++++++ backend/internal/repository/gateway_cache.go | 42 ++++++ .../gateway_cache_reasoning_test.go | 41 +++++ .../gateway_hotpath_optimization_test.go | 7 + .../service/gateway_multiplatform_test.go | 7 + backend/internal/service/gateway_service.go | 14 ++ .../service/gemini_multiplatform_test.go | 7 + .../service/openai_account_scheduler_test.go | 7 + .../openai_cyber_session_block_test.go | 7 + .../openai_gateway_responses_chat_fallback.go | 115 +++++++++++++- ...ai_gateway_responses_chat_fallback_test.go | 142 ++++++++++++++++++ .../service/openai_gateway_service_test.go | 7 + .../service/openai_ws_state_store_test.go | 7 + backend/internal/testutil/stubs.go | 7 + 15 files changed, 582 insertions(+), 6 deletions(-) create mode 100644 backend/internal/pkg/apicompat/chatcompletions_responses_reasoning_cache_test.go create mode 100644 backend/internal/repository/gateway_cache_reasoning_test.go diff --git a/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go b/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go index 70dee035d..05eb95af6 100644 --- a/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go +++ b/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go @@ -17,15 +17,37 @@ const ( type toolOutputMediaByCallID map[string][]ChatContentPart +// ResponsesToChatOptions carries optional hooks for +// ResponsesToChatCompletionsRequestWithOptions. All fields are optional; a nil +// *ResponsesToChatOptions behaves exactly like ResponsesToChatCompletionsRequest. +type ResponsesToChatOptions struct { + // ReasoningContentByID looks up the cached reasoning text for a reasoning + // item id. Codex histories may carry reasoning items with no plaintext + // summary (empty summary + opaque encrypted_content, e.g. after remote + // compaction); DeepSeek's thinking mode rejects such histories with 400 + // "The `reasoning_content` in the thinking mode must be passed back to the + // API". The gateway caches the reasoning text it streamed under the item + // id, so the lookup restores the reasoning_content the client can no + // longer provide. Return "" on a miss. A nil lookup keeps the original + // behavior. + ReasoningContentByID func(itemID string) string +} + // ResponsesToChatCompletionsRequest converts a Responses API request into a // Chat Completions request for upstreams that only implement // /v1/chat/completions. func ResponsesToChatCompletionsRequest(req *ResponsesRequest) (*ChatCompletionsRequest, error) { + return ResponsesToChatCompletionsRequestWithOptions(req, nil) +} + +// ResponsesToChatCompletionsRequestWithOptions is ResponsesToChatCompletionsRequest +// with optional hooks (see ResponsesToChatOptions). +func ResponsesToChatCompletionsRequestWithOptions(req *ResponsesRequest, opts *ResponsesToChatOptions) (*ChatCompletionsRequest, error) { if req == nil { return nil, fmt.Errorf("responses request is nil") } - messages, err := responsesInputToChatMessages(req.Instructions, req.Input) + messages, err := responsesInputToChatMessagesWithOptions(req.Instructions, req.Input, opts) if err != nil { return nil, err } @@ -202,6 +224,12 @@ func HasToolSearchTool(tools []ResponsesTool) bool { // scattered across per-item cases, and makes unknown future codex item types // fail safe instead of leaking into the upstream request. func responsesInputToChatMessages(instructions string, inputRaw json.RawMessage) ([]ChatMessage, error) { + return responsesInputToChatMessagesWithOptions(instructions, inputRaw, nil) +} + +// responsesInputToChatMessagesWithOptions is responsesInputToChatMessages with +// optional hooks (see ResponsesToChatOptions). +func responsesInputToChatMessagesWithOptions(instructions string, inputRaw json.RawMessage, opts *ResponsesToChatOptions) ([]ChatMessage, error) { var messages []ChatMessage if strings.TrimSpace(instructions) != "" { content, _ := json.Marshal(instructions) @@ -226,7 +254,7 @@ func responsesInputToChatMessages(instructions string, inputRaw json.RawMessage) return nil, fmt.Errorf("parse responses input: %w", err) } - built, mediaByCallID, err := buildChatMessagesFromItems(messages, rawItems) + built, mediaByCallID, err := buildChatMessagesFromItems(messages, rawItems, opts) if err != nil { return nil, err } @@ -235,7 +263,7 @@ func responsesInputToChatMessages(instructions string, inputRaw json.RawMessage) // buildChatMessagesFromItems walks the Responses input items and appends the // corresponding Chat messages. -func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessage) ([]ChatMessage, toolOutputMediaByCallID, error) { +func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessage, opts *ResponsesToChatOptions) ([]ChatMessage, toolOutputMediaByCallID, error) { // pendingReasoning holds the reasoning text from a reasoning item until the // assistant message it belongs to is emitted. DeepSeek's thinking mode // requires the reasoning_content that produced a tool call to be passed back @@ -269,6 +297,15 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa case "reasoning": if txt := extractResponsesReasoningText(item); txt != "" { pendingReasoning = txt + } else if opts != nil && opts.ReasoningContentByID != nil { + // No plaintext summary (encrypted-only reasoning, e.g. after codex + // remote compaction): fall back to the gateway-side cache keyed + // by the reasoning item id, which always round-trips in history. + if id := rawString(item["id"]); id != "" { + if cached := opts.ReasoningContentByID(id); cached != "" { + pendingReasoning = cached + } + } } continue case "function_call": @@ -687,6 +724,26 @@ func extractResponsesReasoningText(item map[string]json.RawMessage) string { return strings.Join(parts, "\n") } +// ExtractResponsesReasoningItem parses a raw Responses input item and, when it +// is a reasoning item, returns its id and extractable plaintext (summary +// preferred, content fallback). ok is false for non-reasoning items. It exists +// for the gateway-side reasoning cache: items with plaintext get (re)cached so +// later encrypted-only replicas of the same item id can be restored. +func ExtractResponsesReasoningItem(raw json.RawMessage) (id string, text string, ok bool) { + raw = bytesTrimSpace(raw) + if len(raw) == 0 || string(raw) == "null" { + return "", "", false + } + var item map[string]json.RawMessage + if err := json.Unmarshal(raw, &item); err != nil { + return "", "", false + } + if rawString(item["type"]) != "reasoning" { + return "", "", false + } + return rawString(item["id"]), extractResponsesReasoningText(item), true +} + func chatCompletionsBridgeRole(role string) string { trimmed := strings.TrimSpace(role) if trimmed == "" { diff --git a/backend/internal/pkg/apicompat/chatcompletions_responses_reasoning_cache_test.go b/backend/internal/pkg/apicompat/chatcompletions_responses_reasoning_cache_test.go new file mode 100644 index 000000000..33816f7ab --- /dev/null +++ b/backend/internal/pkg/apicompat/chatcompletions_responses_reasoning_cache_test.go @@ -0,0 +1,115 @@ +package apicompat + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// Encrypted-only reasoning items (empty summary + opaque encrypted_content, +// e.g. after codex remote compaction) carry no plaintext the bridge can map to +// reasoning_content. The gateway-side cache keyed by reasoning item id restores +// it; without the restore, DeepSeek thinking mode rejects the history with 400 +// "The `reasoning_content` in the thinking mode must be passed back to the API". +func TestResponsesToChat_ReasoningCacheLookup_RestoresEncryptedOnlyItem(t *testing.T) { + req := &ResponsesRequest{ + Model: "deepseek-reasoner", + Input: json.RawMessage(`[ + {"type":"reasoning","id":"item_enc1","summary":[],"encrypted_content":"opaque"}, + {"type":"function_call","call_id":"call_1","name":"get_value","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_1","output":"ok"}, + {"type":"message","role":"user","content":[{"type":"input_text","text":"go on"}]} + ]`), + } + + out, err := ResponsesToChatCompletionsRequestWithOptions(req, &ResponsesToChatOptions{ + ReasoningContentByID: func(itemID string) string { + if itemID == "item_enc1" { + return "cached thinking" + } + return "" + }, + }) + require.NoError(t, err) + require.Len(t, out.Messages, 3) + require.Equal(t, "assistant", out.Messages[0].Role) + require.Equal(t, "cached thinking", out.Messages[0].ReasoningContent) + require.Len(t, out.Messages[0].ToolCalls, 1) + require.Equal(t, "call_1", out.Messages[0].ToolCalls[0].ID) + require.Equal(t, "tool", out.Messages[1].Role) + require.Equal(t, "user", out.Messages[2].Role) +} + +// A cache miss keeps the original behavior: no reasoning_content, no error. +func TestResponsesToChat_ReasoningCacheLookup_MissKeepsOriginalBehavior(t *testing.T) { + req := &ResponsesRequest{ + Model: "deepseek-reasoner", + Input: json.RawMessage(`[ + {"type":"reasoning","id":"item_unknown","summary":[],"encrypted_content":"opaque"}, + {"type":"function_call","call_id":"call_1","name":"get_value","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_1","output":"ok"}, + {"type":"message","role":"user","content":[{"type":"input_text","text":"go on"}]} + ]`), + } + + out, err := ResponsesToChatCompletionsRequestWithOptions(req, &ResponsesToChatOptions{ + ReasoningContentByID: func(string) string { return "" }, + }) + require.NoError(t, err) + require.Len(t, out.Messages, 3) + require.Empty(t, out.Messages[0].ReasoningContent) + + // Nil options (legacy path) behaves identically. + legacy, err := ResponsesToChatCompletionsRequest(req) + require.NoError(t, err) + require.Equal(t, out.Messages, legacy.Messages) +} + +// Plaintext summary wins and the cache lookup is not consulted. +func TestResponsesToChat_ReasoningCacheLookup_PlaintextPreferred(t *testing.T) { + req := &ResponsesRequest{ + Model: "deepseek-reasoner", + Input: json.RawMessage(`[ + {"type":"reasoning","id":"item_plain","summary":[{"type":"summary_text","text":"plain thinking"}]}, + {"type":"function_call","call_id":"call_1","name":"get_value","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_1","output":"ok"}, + {"type":"message","role":"user","content":[{"type":"input_text","text":"go on"}]} + ]`), + } + + lookupCalled := false + out, err := ResponsesToChatCompletionsRequestWithOptions(req, &ResponsesToChatOptions{ + ReasoningContentByID: func(string) string { + lookupCalled = true + return "cached thinking" + }, + }) + require.NoError(t, err) + require.Len(t, out.Messages, 3) + require.Equal(t, "plain thinking", out.Messages[0].ReasoningContent) + require.False(t, lookupCalled, "plaintext summary present → cache lookup must not run") +} + +func TestExtractResponsesReasoningItem(t *testing.T) { + id, text, ok := ExtractResponsesReasoningItem(json.RawMessage( + `{"type":"reasoning","id":"item_a","summary":[{"type":"summary_text","text":"think"}]}`)) + require.True(t, ok) + require.Equal(t, "item_a", id) + require.Equal(t, "think", text) + + // Encrypted-only item: ok with id but empty text. + id, text, ok = ExtractResponsesReasoningItem(json.RawMessage( + `{"type":"reasoning","id":"item_b","summary":[],"encrypted_content":"opaque"}`)) + require.True(t, ok) + require.Equal(t, "item_b", id) + require.Empty(t, text) + + // Non-reasoning items are skipped. + _, _, ok = ExtractResponsesReasoningItem(json.RawMessage( + `{"type":"message","role":"user","content":[{"type":"input_text","text":"hi"}]}`)) + require.False(t, ok) + + _, _, ok = ExtractResponsesReasoningItem(json.RawMessage(`"bare string"`)) + require.False(t, ok) +} diff --git a/backend/internal/repository/gateway_cache.go b/backend/internal/repository/gateway_cache.go index dc3d4a980..c05cbded4 100644 --- a/backend/internal/repository/gateway_cache.go +++ b/backend/internal/repository/gateway_cache.go @@ -131,6 +131,48 @@ func (c *gatewayCache) ReleaseGrokVideoBilled(ctx context.Context, key string) e var _ service.CyberSessionBlockStore = (*gatewayCache)(nil) var _ service.LiveCallStore = (*gatewayCache)(nil) +const reasoningContentPrefix = "reasoning_content:" + +// reasoningContentDefaultTTL 是 reasoning 缓存的默认过期时间。Codex 会话可能 +// 跨多天恢复，取 7 天；调用方传入非正 TTL 时兜底。 +const reasoningContentDefaultTTL = 7 * 24 * time.Hour + +// SetReasoningContent 按 reasoning item id 缓存 reasoning 全文。 +// itemID 或 content 为空时直接返回 nil（无可缓存内容，属正常情况而非错误）。 +func (c *gatewayCache) SetReasoningContent(ctx context.Context, itemID string, content string, ttl time.Duration) error { + if c == nil || c.rdb == nil { + return errors.New("gateway cache unavailable") + } + itemID = strings.TrimSpace(itemID) + if itemID == "" || content == "" { + return nil + } + if ttl <= 0 { + ttl = reasoningContentDefaultTTL + } + return c.rdb.Set(ctx, reasoningContentPrefix+itemID, content, ttl).Err() +} + +// GetReasoningContent 返回缓存的 reasoning 全文；未命中返回 +// service.ErrReasoningContentNotFound。 +func (c *gatewayCache) GetReasoningContent(ctx context.Context, itemID string) (string, error) { + if c == nil || c.rdb == nil { + return "", errors.New("gateway cache unavailable") + } + itemID = strings.TrimSpace(itemID) + if itemID == "" { + return "", service.ErrReasoningContentNotFound + } + val, err := c.rdb.Get(ctx, reasoningContentPrefix+itemID).Result() + if err != nil { + if errors.Is(err, redis.Nil) { + return "", service.ErrReasoningContentNotFound + } + return "", err + } + return val, nil +} + const cyberSessionBlockPrefix = "cyber_session_block:" // SetCyberSessionBlocked 把被 cyber_policy 命中的会话写入屏蔽表（TTL 自动过期）。 diff --git a/backend/internal/repository/gateway_cache_reasoning_test.go b/backend/internal/repository/gateway_cache_reasoning_test.go new file mode 100644 index 000000000..f284da51b --- /dev/null +++ b/backend/internal/repository/gateway_cache_reasoning_test.go @@ -0,0 +1,41 @@ +package repository + +import ( + "context" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/require" +) + +func TestGatewayCacheReasoningContent(t *testing.T) { + mr := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: mr.Addr()}) + cache := NewGatewayCache(client) + ctx := context.Background() + + // 未命中返回哨兵错误，区别于真实读取失败。 + _, err := cache.GetReasoningContent(ctx, "item_missing") + require.ErrorIs(t, err, service.ErrReasoningContentNotFound) + + // 写入后可读回。 + require.NoError(t, cache.SetReasoningContent(ctx, "item_abc", "think hard", time.Minute)) + got, err := cache.GetReasoningContent(ctx, "item_abc") + require.NoError(t, err) + require.Equal(t, "think hard", got) + + // ttl<=0 时兜底为默认 7 天。 + require.NoError(t, cache.SetReasoningContent(ctx, "item_ttl", "x", 0)) + ttl := mr.TTL(reasoningContentPrefix + "item_ttl") + require.Greater(t, ttl, 6*24*time.Hour) + require.LessOrEqual(t, ttl, reasoningContentDefaultTTL) + + // 空 itemID / 空 content 是 no-op（无可缓存内容不算错误）。 + require.NoError(t, cache.SetReasoningContent(ctx, "", "x", time.Minute)) + require.NoError(t, cache.SetReasoningContent(ctx, "item_empty", "", time.Minute)) + _, err = cache.GetReasoningContent(ctx, "item_empty") + require.ErrorIs(t, err, service.ErrReasoningContentNotFound) +} diff --git a/backend/internal/service/gateway_hotpath_optimization_test.go b/backend/internal/service/gateway_hotpath_optimization_test.go index 48cfc358a..c8ca1acf0 100644 --- a/backend/internal/service/gateway_hotpath_optimization_test.go +++ b/backend/internal/service/gateway_hotpath_optimization_test.go @@ -158,6 +158,13 @@ func (s *stickyGatewayCacheHotpathStub) ReleaseGrokVideoBilled(_ context.Context return nil } +func (s *stickyGatewayCacheHotpathStub) SetReasoningContent(_ context.Context, _ string, _ string, _ time.Duration) error { + return nil +} +func (s *stickyGatewayCacheHotpathStub) GetReasoningContent(_ context.Context, _ string) (string, error) { + return "", ErrReasoningContentNotFound +} + func (s *modelsListAccountRepoStub) ListSchedulableByGroupID(ctx context.Context, groupID int64) ([]Account, error) { s.listByGroupCalls.Add(1) if s.err != nil { diff --git a/backend/internal/service/gateway_multiplatform_test.go b/backend/internal/service/gateway_multiplatform_test.go index ac734a364..664b6bf08 100644 --- a/backend/internal/service/gateway_multiplatform_test.go +++ b/backend/internal/service/gateway_multiplatform_test.go @@ -291,6 +291,13 @@ func (m *mockGatewayCacheForPlatform) ReleaseGrokVideoBilled(_ context.Context, return nil } +func (m *mockGatewayCacheForPlatform) SetReasoningContent(_ context.Context, _ string, _ string, _ time.Duration) error { + return nil +} +func (m *mockGatewayCacheForPlatform) GetReasoningContent(_ context.Context, _ string) (string, error) { + return "", ErrReasoningContentNotFound +} + type mockGroupRepoForGateway struct { groups map[int64]*Group getByIDCalls int diff --git a/backend/internal/service/gateway_service.go b/backend/internal/service/gateway_service.go index 0f3415fab..bdc028bb4 100644 --- a/backend/internal/service/gateway_service.go +++ b/backend/internal/service/gateway_service.go @@ -452,6 +452,10 @@ var allowedHeaders = map[string]bool{ // cache implementation (e.g. redis.Nil), mirroring ErrRefreshTokenNotFound. var ErrStickySessionNotFound = errors.New("sticky session not found") +// ErrReasoningContentNotFound is returned by GatewayCache.GetReasoningContent +// when no cached reasoning content exists for the reasoning item ID. +var ErrReasoningContentNotFound = errors.New("reasoning content not found") + // GatewayCache 定义网关服务的缓存操作接口。 // 提供粘性会话（Sticky Session）的存储、查询、刷新和删除功能。 // @@ -486,6 +490,16 @@ type GatewayCache interface { ClaimGrokVideoBilled(ctx context.Context, key string, ttl time.Duration) (bool, error) // ReleaseGrokVideoBilled clears a claim so a failed RecordUsage can retry billing. ReleaseGrokVideoBilled(ctx context.Context, key string) error + + // Reasoning content cache (Responses→Chat Completions 桥接）。 + // SetReasoningContent 按 reasoning item id 缓存 reasoning 全文，供后续请求 + // 在客户端不回传明文 summary 时回注 reasoning_content（DeepSeek thinking + // mode 要求回传，否则 400）。 + SetReasoningContent(ctx context.Context, itemID string, content string, ttl time.Duration) error + // GetReasoningContent 返回缓存的 reasoning 全文；未命中返回 + // ErrReasoningContentNotFound，使 service 层无需依赖具体缓存实现即可 + // 区分"未缓存"与真实读取失败。 + GetReasoningContent(ctx context.Context, itemID string) (string, error) } // derefGroupID safely dereferences *int64 to int64, returning 0 if nil diff --git a/backend/internal/service/gemini_multiplatform_test.go b/backend/internal/service/gemini_multiplatform_test.go index 63ebc929b..9a89998d5 100644 --- a/backend/internal/service/gemini_multiplatform_test.go +++ b/backend/internal/service/gemini_multiplatform_test.go @@ -319,6 +319,13 @@ func (m *mockGatewayCacheForGemini) ReleaseGrokVideoBilled(_ context.Context, _ return nil } +func (m *mockGatewayCacheForGemini) SetReasoningContent(_ context.Context, _ string, _ string, _ time.Duration) error { + return nil +} +func (m *mockGatewayCacheForGemini) GetReasoningContent(_ context.Context, _ string) (string, error) { + return "", ErrReasoningContentNotFound +} + // TestGeminiMessagesCompatService_SelectAccountForModelWithExclusions_GeminiPlatform 测试 Gemini 单平台选择 func TestGeminiMessagesCompatService_SelectAccountForModelWithExclusions_GeminiPlatform(t *testing.T) { ctx := context.Background() diff --git a/backend/internal/service/openai_account_scheduler_test.go b/backend/internal/service/openai_account_scheduler_test.go index 3b3a64856..cd914da0f 100644 --- a/backend/internal/service/openai_account_scheduler_test.go +++ b/backend/internal/service/openai_account_scheduler_test.go @@ -196,6 +196,13 @@ func (c *schedulerTestGatewayCache) ReleaseGrokVideoBilled(_ context.Context, _ return nil } +func (c *schedulerTestGatewayCache) SetReasoningContent(_ context.Context, _ string, _ string, _ time.Duration) error { + return nil +} +func (c *schedulerTestGatewayCache) GetReasoningContent(_ context.Context, _ string) (string, error) { + return "", ErrReasoningContentNotFound +} + func newSchedulerTestOpenAIWSV2Config() *config.Config { cfg := &config.Config{} cfg.Gateway.OpenAIWS.Enabled = true diff --git a/backend/internal/service/openai_cyber_session_block_test.go b/backend/internal/service/openai_cyber_session_block_test.go index 6d6ad8bad..66ee99fa7 100644 --- a/backend/internal/service/openai_cyber_session_block_test.go +++ b/backend/internal/service/openai_cyber_session_block_test.go @@ -148,6 +148,13 @@ func (c *comboCacheAndStore) ReleaseGrokVideoBilled(_ context.Context, _ string) return nil } +func (c *comboCacheAndStore) SetReasoningContent(_ context.Context, _ string, _ string, _ time.Duration) error { + return nil +} +func (c *comboCacheAndStore) GetReasoningContent(_ context.Context, _ string) (string, error) { + return "", ErrReasoningContentNotFound +} + func (c *comboCacheAndStore) SetCyberSessionBlocked(ctx context.Context, key string, ttl time.Duration) error { return c.store.SetCyberSessionBlocked(ctx, key, ttl) } diff --git a/backend/internal/service/openai_gateway_responses_chat_fallback.go b/backend/internal/service/openai_gateway_responses_chat_fallback.go index 1082b020d..084f9e502 100644 --- a/backend/internal/service/openai_gateway_responses_chat_fallback.go +++ b/backend/internal/service/openai_gateway_responses_chat_fallback.go @@ -1,6 +1,7 @@ package service import ( + "bytes" "context" "encoding/json" "errors" @@ -52,7 +53,13 @@ func (s *OpenAIGatewayService) forwardResponsesViaRawChatCompletions( toolSearch := apicompat.HasToolSearchTool(effectiveTools) namespaceTools := apicompat.NamespaceToolNames(effectiveTools) - chatReq, err := apicompat.ResponsesToChatCompletionsRequest(&responsesReq) + // 自愈回写：历史里带明文 summary 的 reasoning item 刷新进缓存，覆盖 Redis + // 被 flush / 跨实例漂移后同 id 的 encrypted-only 副本无法再取明文的情况。 + s.recacheReasoningItemsFromInput(responsesReq.Input) + + chatReq, err := apicompat.ResponsesToChatCompletionsRequestWithOptions(&responsesReq, &apicompat.ResponsesToChatOptions{ + ReasoningContentByID: s.reasoningContentByID, + }) if err != nil { writeOpenAIResponsesFallbackError(c, http.StatusBadRequest, "invalid_request_error", err.Error()) return nil, fmt.Errorf("convert responses to chat completions: %w", err) @@ -136,6 +143,7 @@ func (s *OpenAIGatewayService) bufferChatCompletionsAsResponses( return nil, err } responsesResp := apicompat.ChatCompletionsResponseToResponses(ccResp, originalModel, customTools, toolSearch, namespaceTools) + s.cacheReasoningItemsFromOutput(responsesResp.Output) if s.responseHeaderFilter != nil { responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter) @@ -204,7 +212,9 @@ func (s *OpenAIGatewayService) streamChatCompletionsAsResponses( } scan := s.scanCCStream(resp, "openai responses chat fallback", requestID, startTime, func(chunk *apicompat.ChatCompletionsChunk) { - writeEvents(apicompat.ChatCompletionsChunkToResponsesEvents(chunk, state)) + events := apicompat.ChatCompletionsChunkToResponsesEvents(chunk, state) + s.cacheReasoningItemsFromEvents(events) + writeEvents(events) }) if scan.Err != nil { @@ -222,7 +232,9 @@ func (s *OpenAIGatewayService) streamChatCompletionsAsResponses( }, fmt.Errorf("stream usage incomplete: %w", scan.Err) } - writeEvents(apicompat.FinalizeChatCompletionsResponsesStream(state)) + finalEvents := apicompat.FinalizeChatCompletionsResponsesStream(state) + s.cacheReasoningItemsFromEvents(finalEvents) + writeEvents(finalEvents) if !clientDisconnected { writeStreamHeaders() if _, err := fmt.Fprint(c.Writer, "data: [DONE]\n\n"); err != nil { @@ -261,3 +273,100 @@ func chatChunkStartsResponsesOutput(chunk *apicompat.ChatCompletionsChunk) bool } return false } + +// responsesReasoningCacheTTL 是 reasoning 缓存（按 reasoning item id）的过期时间。 +// Codex 会话可能跨多天恢复历史，取 7 天。 +const responsesReasoningCacheTTL = 7 * 24 * time.Hour + +// reasoningContentByID 按 reasoning item id 回查缓存的 reasoning 全文，供 +// Responses→CC 桥接在客户端不回传明文 summary（encrypted-only reasoning +// item）时回注 reasoning_content。任何失败都 fail-open 返回 ""（维持桥接原 +// 行为），因为缓存只是优化而非正确性前提。 +func (s *OpenAIGatewayService) reasoningContentByID(itemID string) string { + if s == nil || s.cache == nil { + return "" + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + content, err := s.cache.GetReasoningContent(ctx, itemID) + if err != nil { + return "" + } + return content +} + +// recacheReasoningItemsFromInput 把请求历史里带明文 summary 的 reasoning item +// 重新写入缓存（best-effort）。Codex 多数时候会原样回传明文 summary，借机 +// 刷新 TTL 并自愈 Redis 被 flush / 跨实例漂移造成的缓存缺失。 +func (s *OpenAIGatewayService) recacheReasoningItemsFromInput(inputRaw json.RawMessage) { + if s == nil || s.cache == nil { + return + } + inputRaw = bytes.TrimSpace(inputRaw) + if len(inputRaw) == 0 || inputRaw[0] != '[' { + return + } + var items []json.RawMessage + if err := json.Unmarshal(inputRaw, &items); err != nil { + return + } + for _, raw := range items { + id, text, ok := apicompat.ExtractResponsesReasoningItem(raw) + if !ok || id == "" || text == "" { + continue + } + s.setReasoningContent(id, text) + } +} + +// cacheReasoningItemsFromEvents 从 Responses 流事件里提取完成的 reasoning +// item 写入缓存（覆盖一个流中的多个 reasoning item）。 +func (s *OpenAIGatewayService) cacheReasoningItemsFromEvents(events []apicompat.ResponsesStreamEvent) { + for _, event := range events { + if event.Type != "response.output_item.done" || event.Item == nil { + continue + } + s.cacheReasoningItem(event.Item) + } +} + +// cacheReasoningItemsFromOutput 从非流式 Responses 响应的 output 里提取 +// reasoning item 写入缓存。 +func (s *OpenAIGatewayService) cacheReasoningItemsFromOutput(output []apicompat.ResponsesOutput) { + for i := range output { + s.cacheReasoningItem(&output[i]) + } +} + +func (s *OpenAIGatewayService) cacheReasoningItem(item *apicompat.ResponsesOutput) { + if item == nil || item.Type != "reasoning" || item.ID == "" { + return + } + var parts []string + for _, sum := range item.Summary { + if t := strings.TrimSpace(sum.Text); t != "" { + parts = append(parts, t) + } + } + if len(parts) == 0 { + return + } + s.setReasoningContent(item.ID, strings.Join(parts, "\n")) +} + +// setReasoningContent 写入缓存，使用 detached ctx：客户端断连后仍在 drain +// 上游流（计费需要），此时的 reasoning 也是后续轮次回注所依赖的，不能随 +// 请求 ctx 一起取消。失败仅记日志，不影响转发。 +func (s *OpenAIGatewayService) setReasoningContent(itemID, content string) { + if s == nil || s.cache == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := s.cache.SetReasoningContent(ctx, itemID, content, responsesReasoningCacheTTL); err != nil { + logger.L().Warn("openai responses chat fallback: cache reasoning content failed", + zap.Error(err), + zap.String("item_id", itemID), + ) + } +} diff --git a/backend/internal/service/openai_gateway_responses_chat_fallback_test.go b/backend/internal/service/openai_gateway_responses_chat_fallback_test.go index 3682335c1..15952f3a0 100644 --- a/backend/internal/service/openai_gateway_responses_chat_fallback_test.go +++ b/backend/internal/service/openai_gateway_responses_chat_fallback_test.go @@ -9,7 +9,9 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" + "time" "github.com/Wei-Shaw/sub2api/internal/pkg/openai_compat" "github.com/gin-gonic/gin" @@ -227,3 +229,143 @@ func forceChatResponsesFallbackAccount() *Account { } return account } + +// reasoningRecordingCache 记录 reasoning 缓存写入、并按需响应回查。 +type reasoningRecordingCache struct { + stubGatewayCache + mu sync.Mutex + sets map[string]string + getResp map[string]string +} + +func (c *reasoningRecordingCache) SetReasoningContent(_ context.Context, itemID string, content string, _ time.Duration) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.sets == nil { + c.sets = make(map[string]string) + } + c.sets[itemID] = content + return nil +} + +func (c *reasoningRecordingCache) GetReasoningContent(_ context.Context, itemID string) (string, error) { + if v, ok := c.getResp[itemID]; ok { + return v, nil + } + return "", ErrReasoningContentNotFound +} + +func (c *reasoningRecordingCache) snapshotSets() map[string]string { + c.mu.Lock() + defer c.mu.Unlock() + out := make(map[string]string, len(c.sets)) + for k, v := range c.sets { + out[k] = v + } + return out +} + +// 流式响应里的 reasoning_content 应按 reasoning item id 写入缓存，供后续轮次 +// 客户端不回传明文 summary 时回注（DeepSeek thinking mode 400 修复的写入侧）。 +func TestForwardResponses_ChatFallbackCachesStreamedReasoning(t *testing.T) { + gin.SetMode(gin.TestMode) + + body := []byte(`{"model":"deepseek-reasoner","input":"hello","stream":true}`) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + + upstreamBody := strings.Join([]string{ + `data: {"id":"chatcmpl_rc","object":"chat.completion.chunk","model":"deepseek-reasoner","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}`, + "", + `data: {"id":"chatcmpl_rc","object":"chat.completion.chunk","model":"deepseek-reasoner","choices":[{"index":0,"delta":{"reasoning_content":"think "},"finish_reason":null}]}`, + "", + `data: {"id":"chatcmpl_rc","object":"chat.completion.chunk","model":"deepseek-reasoner","choices":[{"index":0,"delta":{"reasoning_content":"first"},"finish_reason":null}]}`, + "", + `data: {"id":"chatcmpl_rc","object":"chat.completion.chunk","model":"deepseek-reasoner","choices":[{"index":0,"delta":{"content":"answer"},"finish_reason":"stop"}]}`, + "", + `data: {"id":"chatcmpl_rc","object":"chat.completion.chunk","model":"deepseek-reasoner","choices":[],"usage":{"prompt_tokens":4,"completion_tokens":3,"total_tokens":7}}`, + "", + "data: [DONE]", + "", + }, "\n") + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}, "x-request-id": []string{"rid_reasoning_cache_stream"}}, + Body: io.NopCloser(strings.NewReader(upstreamBody)), + }} + cache := &reasoningRecordingCache{} + svc := &OpenAIGatewayService{ + cfg: rawChatCompletionsTestConfig(), + httpUpstream: upstream, + cache: cache, + } + + result, err := svc.Forward(context.Background(), c, forceChatResponsesFallbackAccount(), body) + require.NoError(t, err) + require.NotNil(t, result) + + sets := cache.snapshotSets() + require.Len(t, sets, 1, "应恰好缓存一个 reasoning item") + for itemID, content := range sets { + require.NotEmpty(t, itemID) + require.Equal(t, "think first", content) + } +} + +// 请求侧：encrypted-only reasoning item（无明文 summary）经缓存回查补回 +// reasoning_content；带明文 summary 的 item 顺手回写缓存（自愈）。 +func TestForwardResponses_ChatFallbackRestoresReasoningFromCache(t *testing.T) { + gin.SetMode(gin.TestMode) + + body := []byte(`{ + "model":"deepseek-reasoner", + "stream":false, + "input":[ + {"type":"reasoning","id":"item_plain","summary":[{"type":"summary_text","text":"plain thinking"}]}, + {"type":"function_call","call_id":"call_0","name":"get_value","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_0","output":"ok"}, + {"type":"reasoning","id":"item_enc1","summary":[],"encrypted_content":"opaque"}, + {"type":"function_call","call_id":"call_1","name":"get_value","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_1","output":"ok"}, + {"type":"message","role":"user","content":[{"type":"input_text","text":"go on"}]} + ] + }`) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", bytes.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}, "x-request-id": []string{"rid_reasoning_cache_restore"}}, + Body: io.NopCloser(strings.NewReader( + `{"id":"chatcmpl_restore","object":"chat.completion","model":"deepseek-reasoner","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":2,"total_tokens":5}}`, + )), + }} + cache := &reasoningRecordingCache{ + getResp: map[string]string{"item_enc1": "cached thinking"}, + } + svc := &OpenAIGatewayService{ + cfg: rawChatCompletionsTestConfig(), + httpUpstream: upstream, + cache: cache, + } + + result, err := svc.Forward(context.Background(), c, forceChatResponsesFallbackAccount(), body) + require.NoError(t, err) + require.NotNil(t, result) + + // 明文 summary 的 assistant 工具调用消息：reasoning_content 来自 summary 本身。 + require.Equal(t, "plain thinking", gjson.GetBytes(upstream.lastBody, "messages.0.reasoning_content").String()) + require.Equal(t, "call_0", gjson.GetBytes(upstream.lastBody, "messages.0.tool_calls.0.id").String()) + require.Equal(t, "tool", gjson.GetBytes(upstream.lastBody, "messages.1.role").String()) + // encrypted-only 的 assistant 工具调用消息：reasoning_content 来自缓存回查。 + require.Equal(t, "cached thinking", gjson.GetBytes(upstream.lastBody, "messages.2.reasoning_content").String()) + require.Equal(t, "call_1", gjson.GetBytes(upstream.lastBody, "messages.2.tool_calls.0.id").String()) + require.Equal(t, "tool", gjson.GetBytes(upstream.lastBody, "messages.3.role").String()) + + // 明文 summary 的 item 被回写进缓存（自愈）。 + require.Equal(t, "plain thinking", cache.snapshotSets()["item_plain"]) +} diff --git a/backend/internal/service/openai_gateway_service_test.go b/backend/internal/service/openai_gateway_service_test.go index 617c07c06..36e38c659 100644 --- a/backend/internal/service/openai_gateway_service_test.go +++ b/backend/internal/service/openai_gateway_service_test.go @@ -710,6 +710,13 @@ func (c *stubGatewayCache) ReleaseGrokVideoBilled(_ context.Context, _ string) e return nil } +func (c *stubGatewayCache) SetReasoningContent(_ context.Context, _ string, _ string, _ time.Duration) error { + return nil +} +func (c *stubGatewayCache) GetReasoningContent(_ context.Context, _ string) (string, error) { + return "", ErrReasoningContentNotFound +} + func TestOpenAISelectAccountWithLoadAwareness_FiltersUnschedulable(t *testing.T) { now := time.Now() resetAt := now.Add(10 * time.Minute) diff --git a/backend/internal/service/openai_ws_state_store_test.go b/backend/internal/service/openai_ws_state_store_test.go index 9c98a1ea8..869e5e0fd 100644 --- a/backend/internal/service/openai_ws_state_store_test.go +++ b/backend/internal/service/openai_ws_state_store_test.go @@ -207,6 +207,13 @@ func (c *openAIWSStateStoreTimeoutProbeCache) ReleaseGrokVideoBilled(_ context.C return nil } +func (c *openAIWSStateStoreTimeoutProbeCache) SetReasoningContent(_ context.Context, _ string, _ string, _ time.Duration) error { + return nil +} +func (c *openAIWSStateStoreTimeoutProbeCache) GetReasoningContent(_ context.Context, _ string) (string, error) { + return "", ErrReasoningContentNotFound +} + func TestOpenAIWSStateStore_RedisOpsUseShortTimeout(t *testing.T) { probe := &openAIWSStateStoreTimeoutProbeCache{} store := NewOpenAIWSStateStore(probe) diff --git a/backend/internal/testutil/stubs.go b/backend/internal/testutil/stubs.go index 9890ec14a..339d57d86 100644 --- a/backend/internal/testutil/stubs.go +++ b/backend/internal/testutil/stubs.go @@ -118,6 +118,13 @@ func (c StubGatewayCache) ReleaseGrokVideoBilled(_ context.Context, _ string) er return nil } +func (c StubGatewayCache) SetReasoningContent(_ context.Context, _ string, _ string, _ time.Duration) error { + return nil +} +func (c StubGatewayCache) GetReasoningContent(_ context.Context, _ string) (string, error) { + return "", service.ErrReasoningContentNotFound +} + // ============================================================ // StubSessionLimitCache — service.SessionLimitCache 的空实现 // ============================================================ From 401dd43b4b27883ae60d27c8270942a5fdbf2d07 Mon Sep 17 00:00:00 2001 From: lbyxiaolizi Date: Mon, 17 Aug 2026 18:47:18 +0800 Subject: [PATCH 04/50] =?UTF-8?q?fix(apicompat):=20=E9=93=BE=E5=BC=8F?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E8=B0=83=E7=94=A8=E5=9B=9E=E6=94=BE=E6=9C=AC?= =?UTF-8?q?=E8=BD=AE=20reasoning=5Fcontent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codex 0.147.0 真实 resume 历史复现：DeepSeek 每轮只在开头流式产生一次 reasoning，reasoning → call A → output A → call B 的链式调用中 call B 前 没有 reasoning item，其 assistant 消息缺 reasoning_content，DeepSeek thinking mode 400 整个历史。 buildChatMessagesFromItems 新增 lastTurnReasoning：记录本轮最近一次 reasoning，跨 tool output 存活，仅被 user 侧 item 清除；assistant 消息 （文本或 tool_calls）在 pendingReasoning 为空时回放本轮 reasoning。 --- .../chatcompletions_responses_bridge.go | 43 ++++++++++++++--- ...pletions_responses_reasoning_cache_test.go | 46 +++++++++++++++++++ 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go b/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go index 05eb95af6..2b9d69eae 100644 --- a/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go +++ b/backend/internal/pkg/apicompat/chatcompletions_responses_bridge.go @@ -271,8 +271,22 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa // across an assistant message (so a following tool call in the same turn // still receives it); any other role ends the thinking span. var pendingReasoning string + // lastTurnReasoning is the most recent reasoning text of the current turn, + // surviving tool outputs. DeepSeek emits reasoning only once per turn, so + // chained tool calls (reasoning → call A → output A → call B) leave call B's + // assistant message without reasoning_content and DeepSeek 400s the history; + // replaying the turn's reasoning on B's message satisfies the contract. Only + // a user-side item ends the turn and clears it. + var lastTurnReasoning string mediaByCallID := make(toolOutputMediaByCallID) + reasoningForAssistant := func() string { + if pendingReasoning != "" { + return pendingReasoning + } + return lastTurnReasoning + } + for _, raw := range rawItems { raw = bytesTrimSpace(raw) if len(raw) == 0 || string(raw) == "null" { @@ -286,6 +300,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa content, _ := json.Marshal(text) messages = append(messages, ChatMessage{Role: "user", Content: content}) pendingReasoning = "" + lastTurnReasoning = "" continue } return nil, nil, fmt.Errorf("parse responses input item: %w", err) @@ -307,6 +322,9 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa } } } + if pendingReasoning != "" { + lastTurnReasoning = pendingReasoning + } continue case "function_call": arguments := rawString(item["arguments"]) @@ -327,7 +345,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa Arguments: arguments, }, } - messages = appendAssistantToolCall(messages, toolCall, pendingReasoning) + messages = appendAssistantToolCall(messages, toolCall, reasoningForAssistant()) pendingReasoning = "" continue case "tool_search_call": @@ -348,7 +366,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa Arguments: arguments, }, } - messages = appendAssistantToolCall(messages, toolCall, pendingReasoning) + messages = appendAssistantToolCall(messages, toolCall, reasoningForAssistant()) pendingReasoning = "" continue case "custom_tool_call": @@ -364,7 +382,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa Arguments: string(arguments), }, } - messages = appendAssistantToolCall(messages, toolCall, pendingReasoning) + messages = appendAssistantToolCall(messages, toolCall, reasoningForAssistant()) pendingReasoning = "" continue case "function_call_output", "custom_tool_call_output", "tool_search_output": @@ -396,6 +414,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa content, _ := json.Marshal(rawString(item["text"])) messages = append(messages, ChatMessage{Role: "user", Content: content}) pendingReasoning = "" + lastTurnReasoning = "" continue case "input_image": content, err := chatContentFromSingleResponsesPart(itemType, item) @@ -404,6 +423,7 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa } messages = append(messages, ChatMessage{Role: "user", Content: content}) pendingReasoning = "" + lastTurnReasoning = "" continue } @@ -428,11 +448,22 @@ func buildChatMessagesFromItems(messages []ChatMessage, rawItems []json.RawMessa if err != nil { return nil, nil, err } - messages = append(messages, ChatMessage{Role: role, Content: chatContent}) - // Reasoning only survives across an assistant text message. - if role != "assistant" { + msg := ChatMessage{Role: role, Content: chatContent} + // DeepSeek thinking mode requires the reasoning_content from a prior + // reasoning-only / plain-text assistant turn to be passed back on its + // assistant message; dropping it yields 400 "The `reasoning_content` in + // the thinking mode must be passed back to the API" on the next turn. + // A following function_call in the same turn still receives it because + // appendAssistantToolCall merges into this message and only fills + // ReasoningContent when it is still empty. + if role == "assistant" { + msg.ReasoningContent = reasoningForAssistant() pendingReasoning = "" + } else { + pendingReasoning = "" + lastTurnReasoning = "" } + messages = append(messages, msg) } return messages, mediaByCallID, nil diff --git a/backend/internal/pkg/apicompat/chatcompletions_responses_reasoning_cache_test.go b/backend/internal/pkg/apicompat/chatcompletions_responses_reasoning_cache_test.go index 33816f7ab..dacedfc61 100644 --- a/backend/internal/pkg/apicompat/chatcompletions_responses_reasoning_cache_test.go +++ b/backend/internal/pkg/apicompat/chatcompletions_responses_reasoning_cache_test.go @@ -91,6 +91,52 @@ func TestResponsesToChat_ReasoningCacheLookup_PlaintextPreferred(t *testing.T) { require.False(t, lookupCalled, "plaintext summary present → cache lookup must not run") } +// DeepSeek emits reasoning only once per turn; chained tool calls +// (reasoning → call A → output A → call B) have no reasoning item before call +// B. The turn's reasoning must be replayed on B's assistant message, otherwise +// DeepSeek thinking mode 400s the history ("reasoning_content ... must be +// passed back"). Reproduced from a real codex 0.147.0 resume history. +func TestResponsesToChat_ChainedToolCallsReplayTurnReasoning(t *testing.T) { + req := &ResponsesRequest{ + Model: "deepseek-reasoner", + Input: json.RawMessage(`[ + {"type":"reasoning","id":"item_r1","summary":[{"type":"summary_text","text":"turn thinking"}]}, + {"type":"message","role":"assistant","content":[{"type":"output_text","text":"\n\n"}]}, + {"type":"function_call","call_id":"call_a","name":"exec_command","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_a","output":"ok"}, + {"type":"function_call","call_id":"call_b","name":"exec_command","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_b","output":"ok"}, + {"type":"message","role":"user","content":[{"type":"input_text","text":"next"}]}, + {"type":"reasoning","id":"item_r2","summary":[{"type":"summary_text","text":"second turn"}]}, + {"type":"function_call","call_id":"call_c","name":"exec_command","arguments":"{}"}, + {"type":"function_call_output","call_id":"call_c","output":"ok"} + ]`), + } + + out, err := ResponsesToChatCompletionsRequest(req) + require.NoError(t, err) + + byCallID := map[string]ChatMessage{} + for _, m := range out.Messages { + for _, tc := range m.ToolCalls { + byCallID[tc.ID] = m + } + } + require.Len(t, byCallID, 3) + require.Equal(t, "turn thinking", byCallID["call_a"].ReasoningContent) + require.Equal(t, "turn thinking", byCallID["call_b"].ReasoningContent, + "链式第二个工具调用必须回放本轮 reasoning") + require.Equal(t, "second turn", byCallID["call_c"].ReasoningContent, + "user 消息后开启新轮次，不得沿用上一轮 reasoning") + + // 每一条 assistant 消息都必须带 reasoning_content（DeepSeek 契约）。 + for i, m := range out.Messages { + if m.Role == "assistant" { + require.NotEmpty(t, m.ReasoningContent, "messages[%d] 缺 reasoning_content", i) + } + } +} + func TestExtractResponsesReasoningItem(t *testing.T) { id, text, ok := ExtractResponsesReasoningItem(json.RawMessage( `{"type":"reasoning","id":"item_a","summary":[{"type":"summary_text","text":"think"}]}`)) From 7e45634df955ddf8a2ae014f2895bd6139544f41 Mon Sep 17 00:00:00 2001 From: Randark Date: Tue, 18 Aug 2026 00:51:30 +0000 Subject: [PATCH 05/50] chore: remove leftover Sora references after platform removal PR #1463 removed the Sora platform, but some references survived: - README/README_CN/README_JA kept the 'Sora status (temporarily unavailable)' sections and gateway.sora_* docs that the removal PR never touched. - deploy/config.example.yaml still documented ~130 lines of sora_* gateway keys, the top-level sora: direct-client/storage block, and token_refresh.sync_linked_sora_accounts - none of which map to any field in the config structs anymore. - The OIDC login PR (02a66a01c, branched off pre-removal main and merged 4 days after #1463) re-added the dead PublicSettings.SoraClientEnabled field, which no code ever sets. - A release sync (748a84d87) re-introduced sora i18n keys that the later i18n split (d9e514f98) faithfully carried into locales/{zh,en}/admin/{overview,settings}.ts. No component references any of these keys. This drops all of the above. Pure deletions, no behavior change. --- README.md | 6 - README_CN.md | 27 ---- README_JA.md | 6 - .../public_settings_injection_schema_test.go | 2 - backend/internal/handler/dto/settings.go | 1 - deploy/config.example.yaml | 130 ------------------ .../src/i18n/locales/en/admin/overview.ts | 2 - .../src/i18n/locales/en/admin/settings.ts | 98 ------------- .../src/i18n/locales/zh/admin/overview.ts | 2 - .../src/i18n/locales/zh/admin/settings.ts | 98 ------------- 10 files changed, 372 deletions(-) diff --git a/README.md b/README.md index f4ad2fa6b..39ce7da0e 100644 --- a/README.md +++ b/README.md @@ -550,12 +550,6 @@ default: rate_multiplier: 1.0 ``` -### Sora Status (Temporarily Unavailable) - -> ⚠️ Sora-related features are temporarily unavailable due to technical issues in upstream integration and media delivery. -> Please do not rely on Sora in production at this time. -> Existing `gateway.sora_*` configuration keys are reserved and may not take effect until these issues are resolved. - Additional security-related options are available in `config.yaml`: - `cors.allowed_origins` for CORS allowlist diff --git a/README_CN.md b/README_CN.md index 2d2a57a9f..3fa4fa2e3 100644 --- a/README_CN.md +++ b/README_CN.md @@ -563,33 +563,6 @@ default: rate_multiplier: 1.0 ``` -### Sora 功能状态（暂不可用） - -> ⚠️ 当前 Sora 相关功能因上游接入与媒体链路存在技术问题，暂时不可用。 -> 现阶段请勿在生产环境依赖 Sora 能力。 -> 文档中的 `gateway.sora_*` 配置仅作预留，待技术问题修复后再恢复可用。 - -### Sora 媒体签名 URL（功能恢复后可选） - -当配置 `gateway.sora_media_signing_key` 且 `gateway.sora_media_signed_url_ttl_seconds > 0` 时，网关会将 Sora 输出的媒体地址改写为临时签名 URL（`/sora/media-signed/...`）。这样无需 API Key 即可在浏览器中直接访问，且具备过期控制与防篡改能力（签名包含 path + query）。 - -```yaml -gateway: - # /sora/media 是否强制要求 API Key（默认 false） - sora_media_require_api_key: false - # 媒体临时签名密钥（为空则禁用签名） - sora_media_signing_key: "your-signing-key" - # 临时签名 URL 有效期（秒） - sora_media_signed_url_ttl_seconds: 900 -``` - -> 若未配置签名密钥，`/sora/media-signed` 将返回 503。 -> 如需更严格的访问控制，可将 `sora_media_require_api_key` 设为 true，仅允许携带 API Key 的 `/sora/media` 访问。 - -访问策略说明： -- `/sora/media`：内部调用或客户端携带 API Key 才能下载 -- `/sora/media-signed`：外部可访问，但有签名 + 过期控制 - `config.yaml` 还支持以下安全相关配置： - `cors.allowed_origins` 配置 CORS 白名单 diff --git a/README_JA.md b/README_JA.md index 09aa77ee4..c8ed0adc6 100644 --- a/README_JA.md +++ b/README_JA.md @@ -550,12 +550,6 @@ default: rate_multiplier: 1.0 ``` -### Sora ステータス（一時的に利用不可） - -> ⚠️ Sora 関連の機能は、上流統合およびメディア配信の技術的問題により一時的に利用できません。 -> 現時点では本番環境で Sora に依存しないでください。 -> 既存の `gateway.sora_*` 設定キーは予約されていますが、これらの問題が解決されるまで有効にならない場合があります。 - `config.yaml` では追加のセキュリティ関連オプションも利用できます: - `cors.allowed_origins` - CORS 許可リスト diff --git a/backend/internal/handler/dto/public_settings_injection_schema_test.go b/backend/internal/handler/dto/public_settings_injection_schema_test.go index 428fed3d8..6a35c8867 100644 --- a/backend/internal/handler/dto/public_settings_injection_schema_test.go +++ b/backend/internal/handler/dto/public_settings_injection_schema_test.go @@ -29,8 +29,6 @@ func TestPublicSettingsInjectionPayload_SchemaDoesNotDrift(t *testing.T) { // Fields that legitimately live only on the DTO. Keep tiny; document each. dtoOnlyFields := map[string]string{ - // sora_client_enabled is an upstream-only field the fork does not surface. - "sora_client_enabled": "upstream-only field, not used on this fork", // force_email_on_third_party_signup lives on the DTO but is not injected via SSR. "force_email_on_third_party_signup": "auth-source default, not a feature flag", } diff --git a/backend/internal/handler/dto/settings.go b/backend/internal/handler/dto/settings.go index 5ab5eaa68..a2fc17845 100644 --- a/backend/internal/handler/dto/settings.go +++ b/backend/internal/handler/dto/settings.go @@ -397,7 +397,6 @@ type PublicSettings struct { OIDCOAuthProviderName string `json:"oidc_oauth_provider_name"` GitHubOAuthEnabled bool `json:"github_oauth_enabled"` GoogleOAuthEnabled bool `json:"google_oauth_enabled"` - SoraClientEnabled bool `json:"sora_client_enabled"` BackendModeEnabled bool `json:"backend_mode_enabled"` PaymentEnabled bool `json:"payment_enabled"` Version string `json:"version"` diff --git a/deploy/config.example.yaml b/deploy/config.example.yaml index 095c55099..e8c1c84b7 100644 --- a/deploy/config.example.yaml +++ b/deploy/config.example.yaml @@ -228,33 +228,6 @@ gateway: # Enable Gemini upstream response header debug logs (default: false) # 是否开启 Gemini 上游响应头调试日志（默认 false） gemini_debug_response_headers: false - # Sora max request body size in bytes (0=use max_body_size) - # Sora 请求体最大字节数（0=使用 max_body_size） - sora_max_body_size: 268435456 - # Sora stream timeout (seconds, 0=disable) - # Sora 流式请求总超时（秒，0=禁用） - sora_stream_timeout_seconds: 900 - # Sora non-stream timeout (seconds, 0=disable) - # Sora 非流式请求超时（秒，0=禁用） - sora_request_timeout_seconds: 180 - # Sora stream enforcement mode: force/error - # Sora stream 强制策略：force/error - sora_stream_mode: "force" - # Sora model filters - # Sora 模型过滤配置 - sora_model_filters: - # Hide prompt-enhance models by default - # 默认隐藏 prompt-enhance 模型 - hide_prompt_enhance: true - # Require API key for /sora/media proxy (default: false) - # /sora/media 是否强制要求 API Key（默认 true） - sora_media_require_api_key: true - # Sora media temporary signing key (empty disables signed URL) - # Sora 媒体临时签名密钥（为空则禁用签名） - sora_media_signing_key: "" - # Signed URL TTL seconds (<=0 disables) - # 临时签名 URL 有效期（秒，<=0 表示禁用） - sora_media_signed_url_ttl_seconds: 900 # Connection pool isolation strategy: # 连接池隔离策略： # - proxy: Isolate by proxy, same proxy shares connection pool (suitable for few proxies, many accounts) @@ -649,112 +622,9 @@ log: # 之后每 N 条保留 1 条 thereafter: 100 -# ============================================================================= -# Sora Direct Client Configuration -# Sora 直连配置 -# ============================================================================= -sora: - client: - # Sora backend base URL - # Sora 上游 Base URL - base_url: "https://sora.chatgpt.com/backend" - # Request timeout (seconds) - # 请求超时（秒） - timeout_seconds: 120 - # Max retries for upstream requests - # 上游请求最大重试次数 - max_retries: 3 - # Account+proxy cooldown window after Cloudflare challenge (seconds, 0 to disable) - # Cloudflare challenge 后按账号+代理冷却窗口（秒，0 表示关闭） - cloudflare_challenge_cooldown_seconds: 900 - # Poll interval (seconds) - # 轮询间隔（秒） - poll_interval_seconds: 2 - # Max poll attempts - # 最大轮询次数 - max_poll_attempts: 600 - # Recent task query limit (image) - # 最近任务查询数量（图片轮询） - recent_task_limit: 50 - # Recent task query max limit (fallback) - # 最近任务查询最大数量（回退） - recent_task_limit_max: 200 - # Enable debug logs for Sora upstream requests - # 启用 Sora 直连调试日志 - # 调试日志会输出上游请求尝试、重试、响应摘要；Authorization/openai-sentinel-token 等敏感头会自动脱敏 - debug: false - # Allow Sora client to fetch token via OpenAI token provider - # 是否允许 Sora 客户端通过 OpenAI token provider 取 token（默认 false，避免误走 OpenAI 刷新链路） - use_openai_token_provider: false - # Optional custom headers (key-value) - # 额外请求头（键值对） - headers: {} - # Default User-Agent for Sora requests - # Sora 默认 User-Agent - user_agent: "Sora/1.2026.007 (Android 15; 24122RKC7C; build 2600700)" - # Disable TLS fingerprint for Sora upstream - # 关闭 Sora 上游 TLS 指纹伪装 - disable_tls_fingerprint: false - # curl_cffi sidecar for Sora only (required) - # 仅 Sora 链路使用的 curl_cffi sidecar（必需） - curl_cffi_sidecar: - # Sora 强制通过 sidecar 请求，必须启用 - # Sora is forced to use sidecar only; keep enabled=true - enabled: true - # Sidecar base URL (default endpoint: /request) - # sidecar 基础地址（默认请求端点：/request） - base_url: "http://sora-curl-cffi-sidecar:8080" - # curl_cffi impersonate profile, e.g. chrome131/chrome124/safari18_0 - # curl_cffi 指纹伪装 profile，例如 chrome131/chrome124/safari18_0 - impersonate: "chrome131" - # Sidecar request timeout (seconds) - # sidecar 请求超时（秒） - timeout_seconds: 60 - # Reuse session key per account+proxy to let sidecar persist cookies/session - # 按账号+代理复用 session key，让 sidecar 持久化 cookies/session - session_reuse_enabled: true - # Session TTL in sidecar (seconds) - # sidecar 会话 TTL（秒） - session_ttl_seconds: 3600 - storage: - # Storage type (local only for now) - # 存储类型（首发仅支持 local） - type: "local" - # Local base path; empty uses /app/data/sora - # 本地存储基础路径；为空使用 /app/data/sora - local_path: "" - # Fallback to upstream URL when download fails - # 下载失败时回退到上游 URL - fallback_to_upstream: true - # Max concurrent downloads - # 并发下载上限 - max_concurrent_downloads: 4 - # Download timeout (seconds) - # 下载超时（秒） - download_timeout_seconds: 120 - # Max download bytes - # 最大下载字节数 - max_download_bytes: 209715200 - # Enable debug logs for media storage - # 启用媒体存储调试日志 - debug: false - cleanup: - # Enable cleanup task - # 启用清理任务 - enabled: true - # Retention days - # 保留天数 - retention_days: 7 - # Cron schedule - # Cron 调度表达式 - schedule: "0 3 * * *" - # Token refresh behavior # token 刷新行为控制 token_refresh: - # Whether OpenAI refresh flow is allowed to sync linked Sora accounts - # 是否允许 OpenAI 刷新流程同步覆盖 linked_openai_account_id 关联的 Sora 账号 token - sync_linked_sora_accounts: false # Candidate accounts loaded per cursor page (maximum 1000) # 每个游标分页加载的候选账号数量（最大 1000） candidate_page_size: 200 diff --git a/frontend/src/i18n/locales/en/admin/overview.ts b/frontend/src/i18n/locales/en/admin/overview.ts index b10dc0d7d..d8ea00c62 100644 --- a/frontend/src/i18n/locales/en/admin/overview.ts +++ b/frontend/src/i18n/locales/en/admin/overview.ts @@ -578,8 +578,6 @@ export default { failedToLoadApiKeys: 'Failed to load user API keys', emailRequired: 'Please enter email', concurrencyMin: 'Concurrency must be at least 1', - soraStorageQuota: 'Sora Storage Quota', - soraStorageQuotaHint: 'In GB, 0 means use group or system default quota', amountRequired: 'Please enter a valid amount', insufficientBalance: 'Insufficient balance', adjustBalance: 'Adjust Balance', diff --git a/frontend/src/i18n/locales/en/admin/settings.ts b/frontend/src/i18n/locales/en/admin/settings.ts index 9aff76d25..84ede270d 100644 --- a/frontend/src/i18n/locales/en/admin/settings.ts +++ b/frontend/src/i18n/locales/en/admin/settings.ts @@ -653,12 +653,6 @@ export default { integrationDoc: 'Payment Integration Docs', integrationDocHint: 'Covers endpoint specs, idempotency semantics, and code samples' }, - soraClient: { - title: 'Sora Client', - description: 'Control whether to show the Sora client entry in the sidebar', - enabled: 'Enable Sora Client', - enabledHint: 'When enabled, the Sora entry will be shown in the sidebar for users to access Sora features' - }, customMenu: { title: 'Custom Menu Pages', description: 'Add custom iframe pages to the sidebar navigation. Each page can be visible to regular users or administrators.', @@ -981,98 +975,6 @@ export default { securityWarning: 'Warning: This key provides full admin access. Keep it secure.', usage: 'Usage: Add to request header - x-api-key: ' }, - soraS3: { - title: 'Sora Storage', - description: 'Manage Sora media storage profiles with S3 and Google Drive support', - newProfile: 'New Profile', - reloadProfiles: 'Reload Profiles', - empty: 'No storage profiles yet, create one first', - createTitle: 'Create Storage Profile', - editTitle: 'Edit Storage Profile', - selectProvider: 'Select Storage Type', - providerS3Desc: 'S3-compatible object storage', - providerGDriveDesc: 'Google Drive cloud storage', - profileID: 'Profile ID', - profileName: 'Profile Name', - setActive: 'Set as active after creation', - saveProfile: 'Save Profile', - activateProfile: 'Activate', - profileCreated: 'Storage profile created', - profileSaved: 'Storage profile saved', - profileDeleted: 'Storage profile deleted', - profileActivated: 'Active storage profile switched', - profileIDRequired: 'Profile ID is required', - profileNameRequired: 'Profile name is required', - profileSelectRequired: 'Please select a profile first', - endpointRequired: 'S3 endpoint is required when enabled', - bucketRequired: 'Bucket is required when enabled', - accessKeyRequired: 'Access Key ID is required when enabled', - deleteConfirm: 'Delete storage profile {profileID}?', - columns: { - profile: 'Profile', - profileId: 'Profile ID', - name: 'Name', - provider: 'Type', - active: 'Active', - endpoint: 'Endpoint', - bucket: 'Bucket', - storagePath: 'Storage Path', - capacityUsage: 'Capacity / Used', - capacityUnlimited: 'Unlimited', - videoCount: 'Videos', - videoCompleted: 'completed', - videoInProgress: 'in progress', - quota: 'Default Quota', - updatedAt: 'Updated At', - actions: 'Actions', - rootFolder: 'Root folder', - testInTable: 'Test', - testingInTable: 'Testing...', - testTimeout: 'Test timed out (15s)' - }, - enabled: 'Enable Storage', - enabledHint: 'When enabled, Sora generated media files will be automatically uploaded', - endpoint: 'S3 Endpoint', - region: 'Region', - bucket: 'Bucket', - prefix: 'Object Prefix', - accessKeyId: 'Access Key ID', - secretAccessKey: 'Secret Access Key', - secretConfigured: '(Configured, leave blank to keep)', - cdnUrl: 'CDN URL', - cdnUrlHint: 'Optional. When configured, files are accessed via CDN URL', - forcePathStyle: 'Force Path Style', - defaultQuota: 'Default Storage Quota', - defaultQuotaHint: 'Default quota when not specified at user or group level. 0 means unlimited', - testConnection: 'Test Connection', - testing: 'Testing...', - testSuccess: 'Connection test successful', - testFailed: 'Connection test failed', - saved: 'Storage settings saved successfully', - saveFailed: 'Failed to save storage settings', - gdrive: { - authType: 'Authentication Method', - serviceAccount: 'Service Account', - clientId: 'Client ID', - clientSecret: 'Client Secret', - clientSecretConfigured: '(Configured, leave blank to keep)', - refreshToken: 'Refresh Token', - refreshTokenConfigured: '(Configured, leave blank to keep)', - serviceAccountJson: 'Service Account JSON', - serviceAccountConfigured: '(Configured, leave blank to keep)', - folderId: 'Folder ID (optional)', - authorize: 'Authorize Google Drive', - authorizeHint: 'Get Refresh Token via OAuth2', - oauthFieldsRequired: 'Please fill in Client ID and Client Secret first', - oauthSuccess: 'Google Drive authorization successful', - oauthFailed: 'Google Drive authorization failed', - closeWindow: 'This window will close automatically', - processing: 'Processing authorization...', - testStorage: 'Test Storage', - testSuccess: 'Google Drive storage test passed (upload, access, delete all OK)', - testFailed: 'Google Drive storage test failed' - } - }, overloadCooldown: { title: '529 Overload Cooldown', description: 'Configure account scheduling pause strategy when upstream returns 529 (overloaded)', diff --git a/frontend/src/i18n/locales/zh/admin/overview.ts b/frontend/src/i18n/locales/zh/admin/overview.ts index e32cadc64..fdfbc8fda 100644 --- a/frontend/src/i18n/locales/zh/admin/overview.ts +++ b/frontend/src/i18n/locales/zh/admin/overview.ts @@ -599,8 +599,6 @@ export default { failedToAdjust: '调整失败', emailRequired: '请输入邮箱', concurrencyMin: '并发数不能小于1', - soraStorageQuota: 'Sora 存储配额', - soraStorageQuotaHint: '单位 GB，0 表示使用分组或系统默认配额', amountRequired: '请输入有效金额', insufficientBalance: '余额不足', setAllowedGroups: '设置允许分组', diff --git a/frontend/src/i18n/locales/zh/admin/settings.ts b/frontend/src/i18n/locales/zh/admin/settings.ts index 82a932e56..f2f0435df 100644 --- a/frontend/src/i18n/locales/zh/admin/settings.ts +++ b/frontend/src/i18n/locales/zh/admin/settings.ts @@ -648,12 +648,6 @@ export default { integrationDoc: '支付集成文档', integrationDocHint: '包含接口说明、幂等语义及示例代码' }, - soraClient: { - title: 'Sora 客户端', - description: '控制是否在侧边栏展示 Sora 客户端入口', - enabled: '启用 Sora 客户端', - enabledHint: '开启后，侧边栏将显示 Sora 入口，用户可访问 Sora 功能' - }, customMenu: { title: '自定义菜单页面', description: '添加自定义 iframe 页面到侧边栏导航。每个页面可以设置为普通用户或管理员可见。', @@ -975,98 +969,6 @@ export default { securityWarning: '警告：此密钥拥有完整的管理员权限，请妥善保管。', usage: '使用方法：在请求头中添加 x-api-key: ' }, - soraS3: { - title: 'Sora 存储配置', - description: '以多配置列表管理 Sora 媒体存储，支持 S3 和 Google Drive', - newProfile: '新建配置', - reloadProfiles: '刷新列表', - empty: '暂无存储配置，请先创建', - createTitle: '新建存储配置', - editTitle: '编辑存储配置', - selectProvider: '选择存储类型', - providerS3Desc: 'S3 兼容对象存储', - providerGDriveDesc: 'Google Drive 云盘', - profileID: '配置 ID', - profileName: '配置名称', - setActive: '创建后设为生效', - saveProfile: '保存配置', - activateProfile: '设为生效', - profileCreated: '存储配置创建成功', - profileSaved: '存储配置保存成功', - profileDeleted: '存储配置删除成功', - profileActivated: '生效配置已切换', - profileIDRequired: '请填写配置 ID', - profileNameRequired: '请填写配置名称', - profileSelectRequired: '请先选择配置', - endpointRequired: '启用时必须填写 S3 端点', - bucketRequired: '启用时必须填写存储桶', - accessKeyRequired: '启用时必须填写 Access Key ID', - deleteConfirm: '确定删除存储配置 {profileID} 吗？', - columns: { - profile: '配置', - profileId: 'Profile ID', - name: '名称', - provider: '存储类型', - active: '生效状态', - endpoint: '端点', - bucket: '存储桶', - storagePath: '存储路径', - capacityUsage: '容量 / 已用', - capacityUnlimited: '无限制', - videoCount: '视频数', - videoCompleted: '完成', - videoInProgress: '进行中', - quota: '默认配额', - updatedAt: '更新时间', - actions: '操作', - rootFolder: '根目录', - testInTable: '测试', - testingInTable: '测试中...', - testTimeout: '测试超时（15秒）' - }, - enabled: '启用存储', - enabledHint: '启用后，Sora 生成的媒体文件将自动上传到存储', - endpoint: 'S3 端点', - region: '区域', - bucket: '存储桶', - prefix: '对象前缀', - accessKeyId: 'Access Key ID', - secretAccessKey: 'Secret Access Key', - secretConfigured: '(已配置，留空保持不变)', - cdnUrl: 'CDN URL', - cdnUrlHint: '可选，配置后使用 CDN URL 访问文件', - forcePathStyle: '强制路径风格（Path Style）', - defaultQuota: '默认存储配额', - defaultQuotaHint: '未在用户或分组级别指定配额时的默认值，0 表示无限制', - testConnection: '测试连接', - testing: '测试中...', - testSuccess: '连接测试成功', - testFailed: '连接测试失败', - saved: '存储设置保存成功', - saveFailed: '保存存储设置失败', - gdrive: { - authType: '认证方式', - serviceAccount: '服务账号', - clientId: 'Client ID', - clientSecret: 'Client Secret', - clientSecretConfigured: '(已配置，留空保持不变)', - refreshToken: 'Refresh Token', - refreshTokenConfigured: '(已配置，留空保持不变)', - serviceAccountJson: '服务账号 JSON', - serviceAccountConfigured: '(已配置，留空保持不变)', - folderId: 'Folder ID（可选）', - authorize: '授权 Google Drive', - authorizeHint: '通过 OAuth2 获取 Refresh Token', - oauthFieldsRequired: '请先填写 Client ID 和 Client Secret', - oauthSuccess: 'Google Drive 授权成功', - oauthFailed: 'Google Drive 授权失败', - closeWindow: '此窗口将自动关闭', - processing: '正在处理授权...', - testStorage: '测试存储', - testSuccess: 'Google Drive 存储测试成功（上传、访问、删除均正常）', - testFailed: 'Google Drive 存储测试失败' - } - }, overloadCooldown: { title: '529 过载冷却', description: '配置上游返回 529（过载）时的账号调度暂停策略', From a9514a68d20bddc971f31f956e682b50edfdc183 Mon Sep 17 00:00:00 2001 From: jaxxjj Date: Tue, 18 Aug 2026 14:26:19 +0800 Subject: [PATCH 06/50] perf(usage): aggregate stats in one scan Reduce filtered admin usage statistics from four scans to one GROUPING SETS query so every breakdown shares the exact same filters. Add concurrent expression indexes for requested and upstream model filters on large usage_logs tables. --- .../internal/repository/migrations_runner.go | 10 + .../repository/migrations_runner_notx_test.go | 41 +++ .../usage_log_repo_request_type_test.go | 91 +++--- .../repository/usage_log_repo_stats.go | 268 +++++++----------- .../usage_log_repo_stats_integration_test.go | 54 +++- ...usage_log_effective_model_indexes_notx.sql | 13 + .../usage_log_effective_model_index_test.go | 19 ++ 7 files changed, 273 insertions(+), 223 deletions(-) create mode 100644 backend/migrations/226_add_usage_log_effective_model_indexes_notx.sql create mode 100644 backend/migrations/usage_log_effective_model_index_test.go diff --git a/backend/internal/repository/migrations_runner.go b/backend/internal/repository/migrations_runner.go index 169a01675..0b786616a 100644 --- a/backend/internal/repository/migrations_runner.go +++ b/backend/internal/repository/migrations_runner.go @@ -59,6 +59,9 @@ const latestAPIKeyIPIndexMigration = "174_add_usage_logs_api_key_latest_ip_index const latestAPIKeyIPIndex = "idx_usage_logs_api_key_latest_ip" const usageLogsUpstreamModelMismatchIndexMigration = "195_add_usage_log_upstream_model_mismatch_index_notx.sql" const usageLogsUpstreamModelMismatchIndex = "idx_usage_logs_upstream_model_mismatch_created_at" +const usageLogsEffectiveModelIndexesMigration = "226_add_usage_log_effective_model_indexes_notx.sql" +const usageLogsEffectiveRequestedModelIndex = "idx_usage_logs_effective_requested_model_created" +const usageLogsEffectiveUpstreamModelIndex = "idx_usage_logs_effective_upstream_model_created" type migrationChecksumCompatibilityRule struct { fileChecksum string @@ -295,6 +298,13 @@ func prepareNonTransactionalMigration(ctx context.Context, db migrationConnectio return dropInvalidIndexIfPresent(ctx, db, latestAPIKeyIPIndex) case usageLogsUpstreamModelMismatchIndexMigration: return dropInvalidIndexIfPresent(ctx, db, usageLogsUpstreamModelMismatchIndex) + case usageLogsEffectiveModelIndexesMigration: + for _, indexName := range []string{usageLogsEffectiveRequestedModelIndex, usageLogsEffectiveUpstreamModelIndex} { + if err := dropInvalidIndexIfPresent(ctx, db, indexName); err != nil { + return err + } + } + return nil default: return nil } diff --git a/backend/internal/repository/migrations_runner_notx_test.go b/backend/internal/repository/migrations_runner_notx_test.go index 55588bb04..8f9882e6d 100644 --- a/backend/internal/repository/migrations_runner_notx_test.go +++ b/backend/internal/repository/migrations_runner_notx_test.go @@ -191,6 +191,47 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_logs_upstream_model_mismatch_c require.NoError(t, mock.ExpectationsWereMet()) } +func TestApplyMigrationsFS_NonTransactionalMigration_EffectiveModelIndexesDropInvalidIndexesBeforeRetry(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer func() { _ = db.Close() }() + + prepareMigrationsBootstrapExpectations(mock) + mock.ExpectQuery("SELECT checksum FROM schema_migrations WHERE filename = \\$1"). + WithArgs(usageLogsEffectiveModelIndexesMigration). + WillReturnError(sql.ErrNoRows) + for _, indexName := range []string{usageLogsEffectiveRequestedModelIndex, usageLogsEffectiveUpstreamModelIndex} { + mock.ExpectQuery("SELECT EXISTS \\("). + WithArgs(indexName). + WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(true)) + mock.ExpectExec("DROP INDEX CONCURRENTLY IF EXISTS " + indexName). + WillReturnResult(sqlmock.NewResult(0, 0)) + } + mock.ExpectExec("CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_logs_effective_requested_model_created"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_logs_effective_upstream_model_created"). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("INSERT INTO schema_migrations \\(filename, checksum\\) VALUES \\(\\$1, \\$2\\)"). + WithArgs(usageLogsEffectiveModelIndexesMigration, sqlmock.AnyArg()). + WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec("SELECT pg_advisory_unlock\\(\\$1\\)"). + WithArgs(migrationsAdvisoryLockID). + WillReturnResult(sqlmock.NewResult(0, 1)) + + fsys := fstest.MapFS{ + usageLogsEffectiveModelIndexesMigration: &fstest.MapFile{Data: []byte(` +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_logs_effective_requested_model_created + ON usage_logs ((COALESCE(NULLIF(BTRIM(requested_model), ''), model)), created_at DESC, id DESC); +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_logs_effective_upstream_model_created + ON usage_logs ((COALESCE(NULLIF(BTRIM(upstream_model), ''), model)), created_at DESC, id DESC); +`)}, + } + + err = applyMigrationsFS(context.Background(), db, fsys) + require.NoError(t, err) + require.NoError(t, mock.ExpectationsWereMet()) +} + func TestApplyMigrationsFS_PaymentOrdersOutTradeNoUniqueMigration_FailsFastOnDuplicatePrecheck(t *testing.T) { db, mock, err := sqlmock.New() require.NoError(t, err) diff --git a/backend/internal/repository/usage_log_repo_request_type_test.go b/backend/internal/repository/usage_log_repo_request_type_test.go index a2b82dfa6..c69f374d4 100644 --- a/backend/internal/repository/usage_log_repo_request_type_test.go +++ b/backend/internal/repository/usage_log_repo_request_type_test.go @@ -505,33 +505,33 @@ func TestUsageLogRepositoryGetStatsWithFiltersRequestedModelSource(t *testing.T) ModelFilterSource: usagestats.ModelSourceRequested, } - mock.ExpectQuery("FROM usage_logs\\s+WHERE COALESCE\\(NULLIF\\(TRIM\\(requested_model\\), ''\\), model\\) = \\$1"). + mock.ExpectQuery("(?s)FROM usage_logs\\s+WHERE COALESCE\\(NULLIF\\(TRIM\\(requested_model\\), ''\\), model\\) = \\$1.*GROUP BY GROUPING SETS"). WithArgs("gpt-5"). WillReturnRows(sqlmock.NewRows([]string{ - "total_requests", - "total_input_tokens", - "total_output_tokens", - "total_cache_tokens", - "total_cache_creation_tokens", - "total_cache_read_tokens", - "total_cost", - "total_actual_cost", - "total_account_cost", + "inbound_grouped", + "upstream_grouped", + "inbound_endpoint", + "upstream_endpoint", + "requests", + "input_tokens", + "output_tokens", + "cache_creation_tokens", + "cache_read_tokens", + "cost", + "actual_cost", + "account_cost", "avg_duration_ms", - }).AddRow(int64(1), int64(2), int64(3), int64(4), int64(1), int64(3), 1.2, 1.0, 1.2, 20.0)) - mock.ExpectQuery("SELECT COALESCE\\(NULLIF\\(TRIM\\(inbound_endpoint\\), ''\\), 'unknown'\\) AS endpoint"). - WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), "gpt-5"). - WillReturnRows(sqlmock.NewRows([]string{"endpoint", "requests", "total_tokens", "cost", "actual_cost"})) - mock.ExpectQuery("SELECT COALESCE\\(NULLIF\\(TRIM\\(upstream_endpoint\\), ''\\), 'unknown'\\) AS endpoint"). - WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), "gpt-5"). - WillReturnRows(sqlmock.NewRows([]string{"endpoint", "requests", "total_tokens", "cost", "actual_cost"})) - mock.ExpectQuery("SELECT CONCAT\\("). - WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), "gpt-5"). - WillReturnRows(sqlmock.NewRows([]string{"endpoint", "requests", "total_tokens", "cost", "actual_cost"})) + }). + AddRow(1, 1, nil, nil, int64(1), int64(2), int64(3), int64(1), int64(3), 1.2, 1.0, 1.2, 20.0). + AddRow(0, 1, "/v1/responses", nil, int64(1), int64(2), int64(3), int64(1), int64(3), 1.2, 1.0, 1.2, 20.0). + AddRow(1, 0, nil, "/v1/responses", int64(1), int64(2), int64(3), int64(1), int64(3), 1.2, 1.0, 1.2, 20.0). + AddRow(0, 0, "/v1/responses", "/v1/responses", int64(1), int64(2), int64(3), int64(1), int64(3), 1.2, 1.0, 1.2, 20.0)) stats, err := repo.GetStatsWithFilters(context.Background(), filters) require.NoError(t, err) require.Equal(t, int64(1), stats.TotalRequests) + require.Equal(t, "/v1/responses", stats.Endpoints[0].Endpoint) + require.Equal(t, "/v1/responses -> /v1/responses", stats.EndpointPaths[0].Endpoint) require.NoError(t, mock.ExpectationsWereMet()) } @@ -546,29 +546,23 @@ func TestUsageLogRepositoryGetStatsWithFiltersRequestTypePriority(t *testing.T) Stream: &stream, } - mock.ExpectQuery("FROM usage_logs\\s+WHERE \\(request_type = \\$1 OR \\(request_type = 0 AND stream = FALSE AND openai_ws_mode = FALSE\\)\\)"). + mock.ExpectQuery("(?s)FROM usage_logs\\s+WHERE \\(request_type = \\$1 OR \\(request_type = 0 AND stream = FALSE AND openai_ws_mode = FALSE\\)\\).*GROUP BY GROUPING SETS"). WithArgs(requestType). WillReturnRows(sqlmock.NewRows([]string{ - "total_requests", - "total_input_tokens", - "total_output_tokens", - "total_cache_tokens", - "total_cache_creation_tokens", - "total_cache_read_tokens", - "total_cost", - "total_actual_cost", - "total_account_cost", + "inbound_grouped", + "upstream_grouped", + "inbound_endpoint", + "upstream_endpoint", + "requests", + "input_tokens", + "output_tokens", + "cache_creation_tokens", + "cache_read_tokens", + "cost", + "actual_cost", + "account_cost", "avg_duration_ms", - }).AddRow(int64(1), int64(2), int64(3), int64(4), int64(1), int64(3), 1.2, 1.0, 1.2, 20.0)) - mock.ExpectQuery("SELECT COALESCE\\(NULLIF\\(TRIM\\(inbound_endpoint\\), ''\\), 'unknown'\\) AS endpoint"). - WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), requestType). - WillReturnRows(sqlmock.NewRows([]string{"endpoint", "requests", "total_tokens", "cost", "actual_cost"})) - mock.ExpectQuery("SELECT COALESCE\\(NULLIF\\(TRIM\\(upstream_endpoint\\), ''\\), 'unknown'\\) AS endpoint"). - WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), requestType). - WillReturnRows(sqlmock.NewRows([]string{"endpoint", "requests", "total_tokens", "cost", "actual_cost"})) - mock.ExpectQuery("SELECT CONCAT\\("). - WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), requestType). - WillReturnRows(sqlmock.NewRows([]string{"endpoint", "requests", "total_tokens", "cost", "actual_cost"})) + }).AddRow(1, 1, nil, nil, int64(1), int64(2), int64(3), int64(1), int64(3), 1.2, 1.0, 1.2, 20.0)) stats, err := repo.GetStatsWithFilters(context.Background(), filters) require.NoError(t, err) @@ -689,19 +683,12 @@ func TestUsageLogRepositoryGetStatsWithFiltersAlwaysReturnsAccountCost(t *testin // No AccountID filter set - TotalAccountCost should still be returned filters := usagestats.UsageLogFilters{} - mock.ExpectQuery("FROM usage_logs"). + mock.ExpectQuery("(?s)FROM usage_logs.*GROUP BY GROUPING SETS"). WillReturnRows(sqlmock.NewRows([]string{ - "total_requests", "total_input_tokens", "total_output_tokens", - "total_cache_tokens", "total_cache_creation_tokens", "total_cache_read_tokens", - "total_cost", "total_actual_cost", - "total_account_cost", "avg_duration_ms", - }).AddRow(int64(50), int64(1000), int64(2000), int64(100), int64(60), int64(40), 15.0, 12.5, 11.0, 100.0)) - mock.ExpectQuery("SELECT COALESCE\\(NULLIF\\(TRIM\\(inbound_endpoint\\)"). - WillReturnRows(sqlmock.NewRows([]string{"endpoint", "requests", "total_tokens", "cost", "actual_cost"})) - mock.ExpectQuery("SELECT COALESCE\\(NULLIF\\(TRIM\\(upstream_endpoint\\)"). - WillReturnRows(sqlmock.NewRows([]string{"endpoint", "requests", "total_tokens", "cost", "actual_cost"})) - mock.ExpectQuery("SELECT CONCAT\\("). - WillReturnRows(sqlmock.NewRows([]string{"endpoint", "requests", "total_tokens", "cost", "actual_cost"})) + "inbound_grouped", "upstream_grouped", "inbound_endpoint", "upstream_endpoint", + "requests", "input_tokens", "output_tokens", "cache_creation_tokens", "cache_read_tokens", + "cost", "actual_cost", "account_cost", "avg_duration_ms", + }).AddRow(1, 1, nil, nil, int64(50), int64(1000), int64(2000), int64(60), int64(40), 15.0, 12.5, 11.0, 100.0)) stats, err := repo.GetStatsWithFilters(context.Background(), filters) require.NoError(t, err) diff --git a/backend/internal/repository/usage_log_repo_stats.go b/backend/internal/repository/usage_log_repo_stats.go index 53d60b7f4..9385900a7 100644 --- a/backend/internal/repository/usage_log_repo_stats.go +++ b/backend/internal/repository/usage_log_repo_stats.go @@ -3,9 +3,9 @@ package repository import ( "context" "database/sql" - "errors" "fmt" "os" + "sort" "strings" "time" @@ -14,7 +14,6 @@ import ( "github.com/Wei-Shaw/sub2api/internal/pkg/usagestats" "github.com/Wei-Shaw/sub2api/internal/service" "github.com/lib/pq" - "golang.org/x/sync/errgroup" ) // GetUserStatsAggregated returns aggregated usage statistics for a user using database-level aggregation @@ -696,108 +695,131 @@ func (r *usageLogRepository) GetStatsWithFilters(ctx context.Context, filters Us } query := fmt.Sprintf(` + WITH scoped AS ( + SELECT + COALESCE(NULLIF(TRIM(inbound_endpoint), ''), 'unknown') AS inbound_endpoint, + COALESCE(NULLIF(TRIM(upstream_endpoint), ''), 'unknown') AS upstream_endpoint, + input_tokens, + output_tokens, + cache_creation_tokens, + cache_read_tokens, + total_cost, + actual_cost, + COALESCE(account_stats_cost, total_cost) * COALESCE(account_rate_multiplier, 1) AS account_cost, + duration_ms + FROM usage_logs + %s + ) SELECT - COUNT(*) as total_requests, - COALESCE(SUM(input_tokens), 0) as total_input_tokens, - COALESCE(SUM(output_tokens), 0) as total_output_tokens, - COALESCE(SUM(cache_creation_tokens + cache_read_tokens), 0) as total_cache_tokens, - COALESCE(SUM(cache_creation_tokens), 0) as total_cache_creation_tokens, - COALESCE(SUM(cache_read_tokens), 0) as total_cache_read_tokens, - COALESCE(SUM(total_cost), 0) as total_cost, - COALESCE(SUM(actual_cost), 0) as total_actual_cost, - COALESCE(SUM(COALESCE(account_stats_cost, total_cost) * COALESCE(account_rate_multiplier, 1)), 0) as total_account_cost, - COALESCE(AVG(duration_ms), 0) as avg_duration_ms - FROM usage_logs - %s + GROUPING(inbound_endpoint) AS inbound_grouped, + GROUPING(upstream_endpoint) AS upstream_grouped, + inbound_endpoint, + upstream_endpoint, + COUNT(*) AS requests, + COALESCE(SUM(input_tokens), 0) AS input_tokens, + COALESCE(SUM(output_tokens), 0) AS output_tokens, + COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens, + COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens, + COALESCE(SUM(total_cost), 0) AS cost, + COALESCE(SUM(actual_cost), 0) AS actual_cost, + COALESCE(SUM(account_cost), 0) AS account_cost, + COALESCE(AVG(duration_ms), 0) AS avg_duration_ms + FROM scoped + GROUP BY GROUPING SETS ( + (), + (inbound_endpoint), + (upstream_endpoint), + (inbound_endpoint, upstream_endpoint) + ) `, buildWhere(conditions)) stats := &UsageStats{} var totalAccountCost float64 - - start := time.Unix(0, 0).UTC() - if filters.StartTime != nil { - start = *filters.StartTime - } - end := time.Now().UTC() - if filters.EndTime != nil { - end = *filters.EndTime + useAccountCostForEndpoint := filters.AccountID > 0 && filters.UserID == 0 && filters.APIKeyID == 0 + rows, err := r.sql.QueryContext(ctx, query, args...) + if err != nil { + return nil, err } + defer func() { _ = rows.Close() }() - var endpoints, upstreamEndpoints, endpointPaths []EndpointStat - - // 汇总查询:失败即致命。 - runSummary := func(c context.Context) error { - return scanSingleRow( - c, r.sql, query, args, - &stats.TotalRequests, - &stats.TotalInputTokens, - &stats.TotalOutputTokens, - &stats.TotalCacheTokens, - &stats.TotalCacheCreationTokens, - &stats.TotalCacheReadTokens, - &stats.TotalCost, - &stats.TotalActualCost, - &totalAccountCost, - &stats.AverageDurationMs, + for rows.Next() { + var ( + inboundGrouped, upstreamGrouped int + inboundEndpoint, upstreamEndpoint sql.NullString + requests, inputTokens, outputTokens, cacheCreationTokens, cacheReads int64 + cost, actualCost, accountCost, averageDurationMs float64 ) - } - // endpoint 明细:best-effort(失败 log + 返空),不致命。 - runEndpoints := func(c context.Context) { - res, err := r.getEndpointStatsByColumnWithFilters(c, "inbound_endpoint", start, end, filters.UserID, filters.APIKeyID, filters.AccountID, filters.GroupID, filters.Model, filters.ModelFilterSource, filters.RequestType, filters.Stream, filters.BillingType, filters.BillingMode) - if err != nil { - if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - logger.LegacyPrintf("repository.usage_log", "GetEndpointStatsWithFilters failed in GetStatsWithFilters: %v", err) - } - res = []EndpointStat{} + if err := rows.Scan( + &inboundGrouped, + &upstreamGrouped, + &inboundEndpoint, + &upstreamEndpoint, + &requests, + &inputTokens, + &outputTokens, + &cacheCreationTokens, + &cacheReads, + &cost, + &actualCost, + &accountCost, + &averageDurationMs, + ); err != nil { + return nil, err } - endpoints = res - } - runUpstream := func(c context.Context) { - res, err := r.getEndpointStatsByColumnWithFilters(c, "upstream_endpoint", start, end, filters.UserID, filters.APIKeyID, filters.AccountID, filters.GroupID, filters.Model, filters.ModelFilterSource, filters.RequestType, filters.Stream, filters.BillingType, filters.BillingMode) - if err != nil { - if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - logger.LegacyPrintf("repository.usage_log", "GetUpstreamEndpointStatsWithFilters failed in GetStatsWithFilters: %v", err) - } - res = []EndpointStat{} + + totalTokens := inputTokens + outputTokens + cacheCreationTokens + cacheReads + endpointActualCost := actualCost + if useAccountCostForEndpoint { + endpointActualCost = accountCost } - upstreamEndpoints = res - } - runPaths := func(c context.Context) { - res, err := r.getEndpointPathStatsWithFilters(c, start, end, filters.UserID, filters.APIKeyID, filters.AccountID, filters.GroupID, filters.Model, filters.ModelFilterSource, filters.RequestType, filters.Stream, filters.BillingType, filters.BillingMode) - if err != nil { - if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { - logger.LegacyPrintf("repository.usage_log", "getEndpointPathStatsWithFilters failed in GetStatsWithFilters: %v", err) - } - res = []EndpointStat{} + + switch { + case inboundGrouped == 1 && upstreamGrouped == 1: + stats.TotalRequests = requests + stats.TotalInputTokens = inputTokens + stats.TotalOutputTokens = outputTokens + stats.TotalCacheCreationTokens = cacheCreationTokens + stats.TotalCacheReadTokens = cacheReads + stats.TotalCacheTokens = cacheCreationTokens + cacheReads + stats.TotalCost = cost + stats.TotalActualCost = actualCost + totalAccountCost = accountCost + stats.AverageDurationMs = averageDurationMs + case inboundGrouped == 0 && upstreamGrouped == 1: + stats.Endpoints = append(stats.Endpoints, EndpointStat{ + Endpoint: inboundEndpoint.String, Requests: requests, TotalTokens: totalTokens, + Cost: cost, ActualCost: endpointActualCost, + }) + case inboundGrouped == 1 && upstreamGrouped == 0: + stats.UpstreamEndpoints = append(stats.UpstreamEndpoints, EndpointStat{ + Endpoint: upstreamEndpoint.String, Requests: requests, TotalTokens: totalTokens, + Cost: cost, ActualCost: endpointActualCost, + }) + case inboundGrouped == 0 && upstreamGrouped == 0: + stats.EndpointPaths = append(stats.EndpointPaths, EndpointStat{ + Endpoint: inboundEndpoint.String + " -> " + upstreamEndpoint.String, + Requests: requests, TotalTokens: totalTokens, Cost: cost, ActualCost: endpointActualCost, + }) } - endpointPaths = res + } + if err := rows.Err(); err != nil { + return nil, err } - if r.db != nil { - // 生产路径:r.sql 是 *sql.DB 连接池,可并发。4 条查询并行,延迟取最大值。 - g, gctx := errgroup.WithContext(ctx) - g.Go(func() error { return runSummary(gctx) }) - g.Go(func() error { runEndpoints(gctx); return nil }) - g.Go(func() error { runUpstream(gctx); return nil }) - g.Go(func() error { runPaths(gctx); return nil }) - if err := g.Wait(); err != nil { - return nil, err - } - } else { - // 事务路径(ent.Tx 不能并发查询):顺序执行,行为与重构前一致。 - if err := runSummary(ctx); err != nil { - return nil, err - } - runEndpoints(ctx) - runUpstream(ctx) - runPaths(ctx) + sortEndpointStats := func(values []EndpointStat) { + sort.Slice(values, func(i, j int) bool { + if values[i].Requests != values[j].Requests { + return values[i].Requests > values[j].Requests + } + return values[i].Endpoint < values[j].Endpoint + }) } + sortEndpointStats(stats.Endpoints) + sortEndpointStats(stats.UpstreamEndpoints) + sortEndpointStats(stats.EndpointPaths) stats.TotalAccountCost = &totalAccountCost stats.TotalTokens = stats.TotalInputTokens + stats.TotalOutputTokens + stats.TotalCacheTokens - stats.Endpoints = endpoints - stats.UpstreamEndpoints = upstreamEndpoints - stats.EndpointPaths = endpointPaths return stats, nil } @@ -882,78 +904,6 @@ func (r *usageLogRepository) getEndpointStatsByColumnWithFilters(ctx context.Con return results, nil } -func (r *usageLogRepository) getEndpointPathStatsWithFilters(ctx context.Context, startTime, endTime time.Time, userID, apiKeyID, accountID, groupID int64, model string, modelSource string, requestType *int16, stream *bool, billingType *int8, billingMode string) (results []EndpointStat, err error) { - actualCostExpr := "COALESCE(SUM(actual_cost), 0) as actual_cost" - if accountID > 0 && userID == 0 && apiKeyID == 0 { - actualCostExpr = "COALESCE(SUM(COALESCE(account_stats_cost, total_cost) * COALESCE(account_rate_multiplier, 1)), 0) as actual_cost" - } - - query := fmt.Sprintf(` - SELECT - CONCAT( - COALESCE(NULLIF(TRIM(inbound_endpoint), ''), 'unknown'), - ' -> ', - COALESCE(NULLIF(TRIM(upstream_endpoint), ''), 'unknown') - ) AS endpoint, - COUNT(*) AS requests, - COALESCE(SUM(input_tokens + output_tokens + cache_creation_tokens + cache_read_tokens), 0) AS total_tokens, - COALESCE(SUM(total_cost), 0) as cost, - %s - FROM usage_logs - WHERE created_at >= $1 AND created_at < $2 - `, actualCostExpr) - - args := []any{startTime, endTime} - if userID > 0 { - query += fmt.Sprintf(" AND user_id = $%d", len(args)+1) - args = append(args, userID) - } - if apiKeyID > 0 { - query += fmt.Sprintf(" AND api_key_id = $%d", len(args)+1) - args = append(args, apiKeyID) - } - if accountID > 0 { - query += fmt.Sprintf(" AND account_id = $%d", len(args)+1) - args = append(args, accountID) - } - if groupID > 0 { - query += fmt.Sprintf(" AND group_id = $%d", len(args)+1) - args = append(args, groupID) - } - query, args = appendUsageLogModelQueryFilter(query, args, model, modelSource) - query, args = appendRequestTypeOrStreamQueryFilter(query, args, requestType, stream) - if billingType != nil { - query += fmt.Sprintf(" AND billing_type = $%d", len(args)+1) - args = append(args, int16(*billingType)) - } - query, args = appendUsageLogBillingModeQueryFilter(query, args, billingMode, "") - query += " GROUP BY endpoint ORDER BY requests DESC" - - rows, err := r.sql.QueryContext(ctx, query, args...) - if err != nil { - return nil, err - } - defer func() { - if closeErr := rows.Close(); closeErr != nil && err == nil { - err = closeErr - results = nil - } - }() - - results = make([]EndpointStat, 0) - for rows.Next() { - var row EndpointStat - if err := rows.Scan(&row.Endpoint, &row.Requests, &row.TotalTokens, &row.Cost, &row.ActualCost); err != nil { - return nil, err - } - results = append(results, row) - } - if err := rows.Err(); err != nil { - return nil, err - } - return results, nil -} - // GetEndpointStatsWithFilters returns inbound endpoint statistics with optional filters. func (r *usageLogRepository) GetEndpointStatsWithFilters(ctx context.Context, startTime, endTime time.Time, userID, apiKeyID, accountID, groupID int64, model string, requestType *int16, stream *bool, billingType *int8) ([]EndpointStat, error) { return r.getEndpointStatsByColumnWithFilters(ctx, "inbound_endpoint", startTime, endTime, userID, apiKeyID, accountID, groupID, model, "", requestType, stream, billingType, "") diff --git a/backend/internal/repository/usage_log_repo_stats_integration_test.go b/backend/internal/repository/usage_log_repo_stats_integration_test.go index e9025b1bd..754ea32a9 100644 --- a/backend/internal/repository/usage_log_repo_stats_integration_test.go +++ b/backend/internal/repository/usage_log_repo_stats_integration_test.go @@ -43,6 +43,15 @@ func TestUsageLog_UpstreamModelMismatchFilterAndPartialIndex(t *testing.T) { }) require.NoError(t, err) require.Equal(t, int64(1), stats.TotalRequests) + require.Equal(t, []usagestats.EndpointStat{{ + Endpoint: "unknown", Requests: 1, TotalTokens: 2, + }}, stats.Endpoints) + require.Equal(t, []usagestats.EndpointStat{{ + Endpoint: "unknown", Requests: 1, TotalTokens: 2, + }}, stats.UpstreamEndpoints) + require.Equal(t, []usagestats.EndpointStat{{ + Endpoint: "unknown -> unknown", Requests: 1, TotalTokens: 2, + }}, stats.EndpointPaths) trend, err := repo.GetUsageTrendWithUsageFilters(ctx, start, end, "hour", usagestats.UsageLogFilters{ UserID: user.ID, UpstreamModelMismatch: &trueValue, @@ -53,24 +62,45 @@ func TestUsageLog_UpstreamModelMismatchFilterAndPartialIndex(t *testing.T) { _, err = tx.ExecContext(ctx, "SET LOCAL enable_seqscan = off") require.NoError(t, err) - rows, err := tx.QueryContext(ctx, ` + assertPlanUsesIndex := func(query, indexName string, args ...any) { + rows, queryErr := tx.QueryContext(ctx, query, args...) + require.NoError(t, queryErr) + var planLines []string + for rows.Next() { + var line string + require.NoError(t, rows.Scan(&line)) + planLines = append(planLines, line) + } + require.NoError(t, rows.Err()) + require.NoError(t, rows.Close()) + require.Contains(t, strings.Join(planLines, "\n"), indexName) + } + assertPlanUsesIndex(` EXPLAIN (COSTS OFF) SELECT id FROM usage_logs WHERE upstream_model_mismatch IS TRUE ORDER BY created_at DESC, id DESC LIMIT 100 -`) - require.NoError(t, err) - defer func() { require.NoError(t, rows.Close()) }() - var planLines []string - for rows.Next() { - var line string - require.NoError(t, rows.Scan(&line)) - planLines = append(planLines, line) - } - require.NoError(t, rows.Err()) - require.Contains(t, strings.Join(planLines, "\n"), usageLogsUpstreamModelMismatchIndex) +`, usageLogsUpstreamModelMismatchIndex) + assertPlanUsesIndex(` +EXPLAIN (COSTS OFF) +SELECT id +FROM usage_logs +WHERE COALESCE(NULLIF(TRIM(requested_model), ''), model) = $1 + AND created_at >= $2 AND created_at < $3 +ORDER BY created_at DESC, id DESC +LIMIT 100 +`, usageLogsEffectiveRequestedModelIndex, "gpt-5.5", start, end) + assertPlanUsesIndex(` +EXPLAIN (COSTS OFF) +SELECT id +FROM usage_logs +WHERE COALESCE(NULLIF(TRIM(upstream_model), ''), model) = $1 + AND created_at >= $2 AND created_at < $3 +ORDER BY created_at DESC, id DESC +LIMIT 100 +`, usageLogsEffectiveUpstreamModelIndex, "gpt-5.5", start, end) } func TestUsageLog_GetStatsWithFilters_AggregatesAndEndpoints(t *testing.T) { diff --git a/backend/migrations/226_add_usage_log_effective_model_indexes_notx.sql b/backend/migrations/226_add_usage_log_effective_model_indexes_notx.sql new file mode 100644 index 000000000..7bb6250fb --- /dev/null +++ b/backend/migrations/226_add_usage_log_effective_model_indexes_notx.sql @@ -0,0 +1,13 @@ +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_logs_effective_requested_model_created + ON usage_logs ( + (COALESCE(NULLIF(BTRIM(requested_model), ''), model)), + created_at DESC, + id DESC + ); + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_logs_effective_upstream_model_created + ON usage_logs ( + (COALESCE(NULLIF(BTRIM(upstream_model), ''), model)), + created_at DESC, + id DESC + ); diff --git a/backend/migrations/usage_log_effective_model_index_test.go b/backend/migrations/usage_log_effective_model_index_test.go new file mode 100644 index 000000000..9505889dc --- /dev/null +++ b/backend/migrations/usage_log_effective_model_index_test.go @@ -0,0 +1,19 @@ +package migrations + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestUsageLogEffectiveModelIndexesMigration(t *testing.T) { + content, err := FS.ReadFile("226_add_usage_log_effective_model_indexes_notx.sql") + require.NoError(t, err) + + sql := strings.Join(strings.Fields(string(content)), " ") + require.Contains(t, sql, "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_logs_effective_requested_model_created") + require.Contains(t, sql, "(COALESCE(NULLIF(BTRIM(requested_model), ''), model)), created_at DESC, id DESC") + require.Contains(t, sql, "CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_usage_logs_effective_upstream_model_created") + require.Contains(t, sql, "(COALESCE(NULLIF(BTRIM(upstream_model), ''), model)), created_at DESC, id DESC") +} From 3d21d61609f7f7db4d4f99182372a1f1baa4f648 Mon Sep 17 00:00:00 2001 From: lbyxiaolizi Date: Tue, 18 Aug 2026 18:20:09 +0800 Subject: [PATCH 07/50] =?UTF-8?q?chore:=20retrigger=20CI=EF=BC=88=E4=B8=8A?= =?UTF-8?q?=E6=B8=B8=20flaky=20=E6=B5=8B=E8=AF=95=20TestApplyCodexFingerpr?= =?UTF-8?q?intClientMetadataRaw=5FMatchesMapVariant=20=E6=AF=AB=E7=A7=92?= =?UTF-8?q?=E8=BE=B9=E7=95=8C=E8=AF=AF=E6=8A=A5=EF=BC=8C=E4=B8=8E=E6=9C=AC?= =?UTF-8?q?=20PR=20=E6=97=A0=E5=85=B3=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 1128df2592d9a505df22c9df953374899b3eee70 Mon Sep 17 00:00:00 2001 From: Randark Date: Tue, 18 Aug 2026 09:46:37 +0000 Subject: [PATCH 08/50] fix(monitor): align quota-fetcher credential/balance semantics with scheduler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit P2-1/P2-3 from review: - fetchCNQuota: credential-invalid now judged by StatusCode 401/403 (aligned with fetchCNBalance) instead of `!Success && !CredentialValid` — CN quota service only sets CredentialValid=true on the success path, so 500/429/zhipu business errors were all misclassified as failed instead of error. - fetchCNBalance: snapshot carries new BalanceLow flag computed with the scheduler's exact criterion (`!Available || allCNBalancesBelowThreshold`) against Gateway.CNProviders.BalanceThreshold (ctor now takes cfg; wire regenerated). quotaDegradedHint reports "balance low" instead of the old `<=0` check, so an account already paused by the scheduler (balance 5 / threshold 10) no longer shows green in the monitor. - threshold helper falls back to viper default 0.5 for nil/<=0 config to avoid a zero-threshold regression where balance=0 stops alerting. Tests: CN quota status-code matrix (rewrites the test that cemented the old behavior), balance-low matrix (below-threshold / unavailable / multi-currency healthy), threshold-from-config; PayG stubs now set Available explicitly (zero-value trap). --- backend/cmd/server/wire_gen.go | 2 +- .../internal/domain/channel_monitor_quota.go | 4 + .../service/channel_monitor_quota_fetcher.go | 38 ++++- .../channel_monitor_quota_fetcher_test.go | 152 +++++++++++++++--- 4 files changed, 168 insertions(+), 28 deletions(-) diff --git a/backend/cmd/server/wire_gen.go b/backend/cmd/server/wire_gen.go index e68a6bd7d..6432afcfb 100644 --- a/backend/cmd/server/wire_gen.go +++ b/backend/cmd/server/wire_gen.go @@ -337,7 +337,7 @@ func initializeApplication(buildInfo handler.BuildInfo) (*Application, error) { batchImageWorkerRuntime := service.ProvideBatchImageWorkerRuntime(batchImageRepository, accountRepository, batchImageQueue, usageBillingRepository, usageLogRepository, batchImageModelPricingResolver, apiKeyAuthCacheInvalidator, configConfig) scheduledTestRunnerService := service.ProvideScheduledTestRunnerService(scheduledTestPlanRepository, scheduledTestService, accountTestService, rateLimitService, configConfig) paymentOrderExpiryService := service.ProvidePaymentOrderExpiryService(paymentService, leaderLockCache, db) - channelMonitorQuotaFetcher := service.NewChannelMonitorQuotaFetcher(accountUsageService, cnProviderQuotaService, cnProviderBalanceService, accountRepository) + channelMonitorQuotaFetcher := service.NewChannelMonitorQuotaFetcher(accountUsageService, cnProviderQuotaService, cnProviderBalanceService, accountRepository, configConfig) channelMonitorRunner := service.ProvideChannelMonitorRunner(channelMonitorService, settingService, channelMonitorQuotaFetcher) channelMonitorV2Aggregator := service.ProvideChannelMonitorV2Aggregator(channelMonitorV2Repository, db, settingService) userPlatformQuotaUsageFlusher := service.ProvideUserPlatformQuotaUsageFlusher(configConfig, billingCache, serviceUserPlatformQuotaRepository, timingWheelService) diff --git a/backend/internal/domain/channel_monitor_quota.go b/backend/internal/domain/channel_monitor_quota.go index 72c989c38..1b93cd394 100644 --- a/backend/internal/domain/channel_monitor_quota.go +++ b/backend/internal/domain/channel_monitor_quota.go @@ -50,6 +50,10 @@ type MonitorQuotaSnapshot struct { Balances []MonitorBalance `json:"balances,omitempty"` // 多币种余额（如 DeepSeek CNY+USD） Currency string `json:"currency,omitempty"` // 主余额币种 PlanLevel string `json:"plan_level,omitempty"` // 套餐等级（如智谱 level） + // BalanceLow 余额低于阈值或账号被上游标记不可用（仅 cn_balance 来源）。 + // 抓取器按 Gateway.CNProviders.BalanceThreshold 判定，口径与账号停调 + // （CNProviderBalanceCheckService.checkOne）一致：任一币种达标即健康。 + BalanceLow bool `json:"balance_low,omitempty"` // CredentialInvalid 上游 401/403 鉴权失败（区别于网络/解析错误）， // 检测状态据此推导 failed 而非 error。 CredentialInvalid bool `json:"credential_invalid,omitempty"` diff --git a/backend/internal/service/channel_monitor_quota_fetcher.go b/backend/internal/service/channel_monitor_quota_fetcher.go index a4fa992c3..2b6d54b68 100644 --- a/backend/internal/service/channel_monitor_quota_fetcher.go +++ b/backend/internal/service/channel_monitor_quota_fetcher.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/Wei-Shaw/sub2api/internal/config" "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/pkg/xai" "golang.org/x/sync/singleflight" @@ -58,6 +59,8 @@ type ChannelMonitorQuotaFetcher struct { cnQuota monitorCNQuotaSource cnBalance monitorCNBalanceSource accounts monitorAccountSource + // balanceThreshold cn_balance 余额告警阈值（与账号停调共用配置，见 monitorBalanceThreshold）。 + balanceThreshold float64 mu sync.Mutex cache map[int64]monitorQuotaCacheEntry @@ -76,8 +79,12 @@ func NewChannelMonitorQuotaFetcher( cnQuota *CNProviderQuotaService, cnBalance *CNProviderBalanceService, accounts AccountRepository, + cfg *config.Config, ) *ChannelMonitorQuotaFetcher { - f := &ChannelMonitorQuotaFetcher{cache: make(map[int64]monitorQuotaCacheEntry)} + f := &ChannelMonitorQuotaFetcher{ + cache: make(map[int64]monitorQuotaCacheEntry), + balanceThreshold: monitorBalanceThreshold(cfg), + } if usage != nil { f.usage = usage } @@ -93,6 +100,17 @@ func NewChannelMonitorQuotaFetcher( return f } +// monitorBalanceThreshold 余额告警阈值，与账号停调（CNProviderBalanceCheckService） +// 共用 gateway.cn_providers.balance_threshold，保证监控 degraded 与调度器停调 +// 口径一致（任一币种达标即健康）。未配置/非正值时回退 viper 默认 0.5（config.go）， +// 避免 0 阈值下「余额=0 也不告警」相对旧 `<=0` 判定的回归。 +func monitorBalanceThreshold(cfg *config.Config) float64 { + if cfg != nil && cfg.Gateway.CNProviders.BalanceThreshold > 0 { + return cfg.Gateway.CNProviders.BalanceThreshold + } + return 0.5 +} + // LoadAccount 加载账号（不走缓存）。供 Create/Update 时校验 // provider 与 account.platform 一致；账号不存在时返回错误。 func (f *ChannelMonitorQuotaFetcher) LoadAccount(ctx context.Context, id int64) (*Account, error) { @@ -341,7 +359,10 @@ func (f *ChannelMonitorQuotaFetcher) fetchCNQuota(ctx context.Context, accountID Error: result.Error, FetchedAt: now, } - if !result.Success && !result.CredentialValid { + // 只有 401/403 判凭据失效（与 fetchCNBalance 口径一致）：CN quota 服务的 + // CredentialValid 仅在成功路径置 true，若按 `!Success && !CredentialValid` + // 推导，500/429/智谱业务错误全会被误判为 failed。 + if !result.Success && (result.StatusCode == 401 || result.StatusCode == 403) { snapshot.CredentialInvalid = true } if len(result.Tiers) > 0 { @@ -386,6 +407,10 @@ func (f *ChannelMonitorQuotaFetcher) fetchCNBalance(ctx context.Context, account if result.Success { balance := result.Balance snapshot.Balance = &balance + // 与账号停调（checkOne）同口径：上游标记不可用或全部币种低于阈值 + // 才告警，任一币种达标即健康（余额 5 元/阈值 10 元的账号调度器已 + // 停调，监控不能仍绿灯）。 + snapshot.BalanceLow = !result.Available || allCNBalancesBelowThreshold(result, f.balanceThreshold) } else if result.StatusCode == 401 || result.StatusCode == 403 { snapshot.CredentialInvalid = true } @@ -454,7 +479,7 @@ func usageFailureInfo(usage *UsageInfo) (failed, credentialInvalid bool, msg str // deriveQuotaCheckResult 把配额快照推导为检测状态（复用既有 status 枚举， // 时间线/可用率机制自动生效）： // - 查询成功且无告警 → operational -// - 任一窗口使用率 >= 阈值或余额耗尽 → degraded +// - 任一窗口使用率 >= 阈值或余额低于阈值/不可用 → degraded // - 账号未关联（配置问题） → degraded // - 凭据失效（401/403） → failed // - 网络/解析等其他错误 → error @@ -499,8 +524,11 @@ func quotaDegradedHint(snapshot *domain.MonitorQuotaSnapshot) string { return fmt.Sprintf("quota high: %s at %s%%", name, strconv.FormatFloat(tier.UsedPercent, 'f', 1, 64)) } } - if snapshot.Balance != nil && *snapshot.Balance <= 0 { - return fmt.Sprintf("balance depleted (%s)", firstNonEmpty(snapshot.Currency, "?")) + if snapshot.BalanceLow { + if snapshot.Balance != nil { + return fmt.Sprintf("balance low: %s %s", strconv.FormatFloat(*snapshot.Balance, 'f', -1, 64), firstNonEmpty(snapshot.Currency, "?")) + } + return fmt.Sprintf("balance low (%s)", firstNonEmpty(snapshot.Currency, "?")) } return "" } diff --git a/backend/internal/service/channel_monitor_quota_fetcher_test.go b/backend/internal/service/channel_monitor_quota_fetcher_test.go index f2e6ff998..d4f4d38a4 100644 --- a/backend/internal/service/channel_monitor_quota_fetcher_test.go +++ b/backend/internal/service/channel_monitor_quota_fetcher_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/Wei-Shaw/sub2api/internal/config" "github.com/Wei-Shaw/sub2api/internal/domain" "github.com/Wei-Shaw/sub2api/internal/pkg/xai" "github.com/stretchr/testify/require" @@ -87,11 +88,12 @@ func newQuotaFetcherTestSetup(t *testing.T) (*ChannelMonitorQuotaFetcher, *stubM cnBalance := &stubMonitorCNBalanceSource{} accounts := &stubMonitorAccountSource{accounts: make(map[int64]*Account)} fetcher := &ChannelMonitorQuotaFetcher{ - usage: usage, - cnQuota: cnQuota, - cnBalance: cnBalance, - accounts: accounts, - cache: make(map[int64]monitorQuotaCacheEntry), + usage: usage, + cnQuota: cnQuota, + cnBalance: cnBalance, + accounts: accounts, + balanceThreshold: monitorBalanceThreshold(nil), + cache: make(map[int64]monitorQuotaCacheEntry), } return fetcher, usage, cnQuota, cnBalance, accounts } @@ -167,9 +169,10 @@ func TestQuotaFetcher_PayGAccountUsesCNBalance(t *testing.T) { Credentials: map[string]any{"account_mode": AccountModePayG}, } cnBalance.result = &CNProviderBalanceResult{ - Success: true, - Balance: 12.34, - Currency: "CNY", + Success: true, + Available: true, + Balance: 12.34, + Currency: "CNY", Balances: []CNProviderBalanceEntry{ {Currency: "CNY", Balance: 12.34}, {Currency: "USD", Balance: 1.5}, @@ -185,6 +188,7 @@ func TestQuotaFetcher_PayGAccountUsesCNBalance(t *testing.T) { require.Equal(t, "CNY", snapshot.Currency) require.Len(t, snapshot.Balances, 2) require.Equal(t, "USD", snapshot.Balances[1].Currency) + require.False(t, snapshot.BalanceLow) require.Empty(t, snapshot.Error) } @@ -278,20 +282,44 @@ func TestUsageFailureInfo_ClassificationMatrix(t *testing.T) { } } -func TestQuotaFetcher_CNQuotaCredentialInvalidFlagPropagates(t *testing.T) { - fetcher, _, cnQuota, _, accounts := newQuotaFetcherTestSetup(t) - accounts.accounts[5] = &Account{ - ID: 5, - Platform: domain.PlatformZhipu, - Credentials: map[string]any{"account_mode": AccountModeCoding}, +// 凭据失效只认 401/403（与 fetchCNBalance 口径一致）：CN quota 服务的 +// CredentialValid 仅成功路径置 true，500/429/智谱业务错误须推导为 error 而非 failed。 +func TestQuotaFetcher_CNQuotaCredentialInvalidByStatusCode(t *testing.T) { + cases := []struct { + name string + accountID int64 + statusCode int + credentialBad bool + expectedStatus string + }{ + {name: "401 unauthorized", accountID: 5, statusCode: 401, credentialBad: true, expectedStatus: MonitorStatusFailed}, + {name: "403 forbidden", accountID: 15, statusCode: 403, credentialBad: true, expectedStatus: MonitorStatusFailed}, + {name: "500 server error", accountID: 16, statusCode: 500, expectedStatus: MonitorStatusError}, + {name: "429 rate limited", accountID: 17, statusCode: 429, expectedStatus: MonitorStatusError}, + // 智谱 2xx 但业务级失败：StatusCode=200，非凭据问题。 + {name: "200 business error", accountID: 18, statusCode: 200, expectedStatus: MonitorStatusError}, } - cnQuota.result = &CNProviderQuotaProbeResult{Success: false, CredentialValid: false, Error: "api key expired"} + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fetcher, _, cnQuota, _, accounts := newQuotaFetcherTestSetup(t) + accounts.accounts[tc.accountID] = &Account{ + ID: tc.accountID, + Platform: domain.PlatformZhipu, + Credentials: map[string]any{"account_mode": AccountModeCoding}, + } + cnQuota.result = &CNProviderQuotaProbeResult{ + Success: false, + StatusCode: tc.statusCode, + Error: "api key expired", + } - snapshot := fetcher.Fetch(context.Background(), 5) + snapshot := fetcher.Fetch(context.Background(), tc.accountID) - require.False(t, snapshot.Success) - require.True(t, snapshot.CredentialInvalid) - require.Equal(t, "api key expired", snapshot.Error) + require.False(t, snapshot.Success) + require.Equal(t, tc.credentialBad, snapshot.CredentialInvalid) + require.Equal(t, tc.expectedStatus, deriveQuotaCheckResult(snapshot, "quota", time.Now()).Status) + }) + } } func TestQuotaFetcher_CNBalanceHTTP403MarksCredentialInvalid(t *testing.T) { @@ -305,6 +333,86 @@ func TestQuotaFetcher_CNBalanceHTTP403MarksCredentialInvalid(t *testing.T) { require.True(t, snapshot.CredentialInvalid) } +// 余额告警口径与账号停调（CNProviderBalanceCheckService.checkOne）一致： +// 上游标记不可用或全部币种低于阈值 → BalanceLow → degraded；任一币种达标即健康。 +func TestQuotaFetcher_CNBalanceLowMarksDegraded(t *testing.T) { + cases := []struct { + name string + accountID int64 + result *CNProviderBalanceResult + balanceLow bool + wantStatus string + wantMessage string + }{ + { + // 审查例：余额 5/阈值 10 的账号调度器已停调，监控不能仍绿灯。 + name: "balance below threshold", + accountID: 21, + result: &CNProviderBalanceResult{Success: true, Available: true, Balance: 5, Currency: "CNY"}, + balanceLow: true, + wantStatus: MonitorStatusDegraded, wantMessage: "balance low: 5 CNY", + }, + { + name: "upstream marked unavailable", + accountID: 22, + result: &CNProviderBalanceResult{Success: true, Available: false, Balance: 20, Currency: "CNY"}, + balanceLow: true, + wantStatus: MonitorStatusDegraded, wantMessage: "balance low: 20 CNY", + }, + { + // deepseek 双币种：任一币种（USD 20）达标即健康。 + name: "any currency above threshold is healthy", + accountID: 23, + result: &CNProviderBalanceResult{ + Success: true, Available: true, Balance: 5, Currency: "CNY", + Balances: []CNProviderBalanceEntry{{Currency: "CNY", Balance: 5}, {Currency: "USD", Balance: 20}}, + }, + wantStatus: MonitorStatusOperational, + }, + { + name: "single currency above threshold", + accountID: 24, + result: &CNProviderBalanceResult{Success: true, Available: true, Balance: 20, Currency: "CNY"}, + wantStatus: MonitorStatusOperational, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fetcher, _, _, cnBalance, accounts := newQuotaFetcherTestSetup(t) + fetcher.balanceThreshold = 10 + accounts.accounts[tc.accountID] = &Account{ + ID: tc.accountID, + Platform: domain.PlatformKimi, + Credentials: map[string]any{"account_mode": AccountModePayG}, + } + cnBalance.result = tc.result + + snapshot := fetcher.Fetch(context.Background(), tc.accountID) + + require.True(t, snapshot.Success) + require.Equal(t, tc.balanceLow, snapshot.BalanceLow) + res := deriveQuotaCheckResult(snapshot, "quota", time.Now()) + require.Equal(t, tc.wantStatus, res.Status) + if tc.wantMessage != "" { + require.Contains(t, res.Message, tc.wantMessage) + } else { + require.Empty(t, res.Message) + } + }) + } +} + +func TestNewChannelMonitorQuotaFetcher_ThresholdFromConfig(t *testing.T) { + require.InDelta(t, 0.5, NewChannelMonitorQuotaFetcher(nil, nil, nil, nil, nil).balanceThreshold, 0.0001) + + cfg10 := &config.Config{Gateway: config.GatewayConfig{CNProviders: config.GatewayCNProvidersConfig{BalanceThreshold: 10}}} + require.InDelta(t, 10, NewChannelMonitorQuotaFetcher(nil, nil, nil, nil, cfg10).balanceThreshold, 0.0001) + + // 非正值（含显式 0）回退默认，避免 0 阈值下「余额=0 也不告警」。 + cfg0 := &config.Config{Gateway: config.GatewayConfig{CNProviders: config.GatewayCNProvidersConfig{BalanceThreshold: 0}}} + require.InDelta(t, 0.5, NewChannelMonitorQuotaFetcher(nil, nil, nil, nil, cfg0).balanceThreshold, 0.0001) +} + func TestQuotaFetcher_NilDependenciesProduceErrorSnapshots(t *testing.T) { // fetcher 本体为 nil：直接降级为错误快照，不 panic。 var nilFetcher *ChannelMonitorQuotaFetcher @@ -495,10 +603,10 @@ func TestDeriveQuotaCheckResult_StatusMatrix(t *testing.T) { require.Contains(t, res.Message, "95.0%") balance := -0.5 - depleted := &domain.MonitorQuotaSnapshot{Success: true, Balance: &balance, Currency: "CNY"} - res = deriveQuotaCheckResult(depleted, "quota", now) + lowBalance := &domain.MonitorQuotaSnapshot{Success: true, BalanceLow: true, Balance: &balance, Currency: "CNY"} + res = deriveQuotaCheckResult(lowBalance, "quota", now) require.Equal(t, MonitorStatusDegraded, res.Status) - require.Contains(t, res.Message, "balance depleted") + require.Contains(t, res.Message, "balance low") invalid := &domain.MonitorQuotaSnapshot{Success: false, CredentialInvalid: true, Error: "401 unauthorized"} res = deriveQuotaCheckResult(invalid, "quota", now) From c41ae19e529e0297df1d3212627410d731cb27ea Mon Sep 17 00:00:00 2001 From: Randark Date: Tue, 18 Aug 2026 09:54:33 +0000 Subject: [PATCH 09/50] fix(monitor): reject unusable quota data sources and invalid mode combos at write time P2-2/P2-4/P2-5 from review: - validateLinkedAccount/revalidateLinkedAccount now check monitorAccountQuotaCapability after the platform match, blocking combos that would permanently error at runtime: deepseek coding and custom-domain kimi coding (no quota endpoint), zhipu payg (no balance endpoint), anthropic/openai API-Key accounts (usage query requires oauth; anthropic setup-token still allowed via local estimation). gemini/grok/antigravity stay permissive. Quota mode surfaces CHANNEL_MONITOR_ACCOUNT_NOT_SUPPORTABLE on edit; probe mode silently unbinds (same as platform mismatch). - applyMonitorUpdate re-runs validateCheckMode on the effective provider+check_mode whenever either field is patched, so a provider-only update can no longer persist antigravity+probe (the check is conditioned on provider/check_mode presence so legacy illegal rows can still be renamed/disabled). - normalizeMonitorPrimaryModel only substitutes the "quota" placeholder for pure quota mode; quota_probe with an empty model now fails with MissingPrimaryModel instead of probing model="quota". The quota branch also moves ahead of the grok default, so grok+quota now gets "quota" rather than "grok-4.5" (review L-item). Tests: capability matrix (15 cases incl. anti-over-blocking rows), linked-account wiring, revalidate quota/probe split, provider-only update bypass + legacy-row rename, quota_probe missing-model create, normalize matrix. --- .../internal/service/channel_monitor_const.go | 3 + .../channel_monitor_quota_mode_test.go | 163 +++++++++++++++++- .../service/channel_monitor_service.go | 25 ++- .../service/channel_monitor_validate.go | 54 +++++- 4 files changed, 232 insertions(+), 13 deletions(-) diff --git a/backend/internal/service/channel_monitor_const.go b/backend/internal/service/channel_monitor_const.go index bf7676ed5..6a2b9b92e 100644 --- a/backend/internal/service/channel_monitor_const.go +++ b/backend/internal/service/channel_monitor_const.go @@ -162,6 +162,9 @@ var ( ErrChannelMonitorProviderIncompatible = infraerrors.BadRequest( "CHANNEL_MONITOR_PROVIDER_INCOMPATIBLE", "monitor provider must match the linked account platform", ) + ErrChannelMonitorAccountNotSupportable = infraerrors.BadRequest( + "CHANNEL_MONITOR_ACCOUNT_NOT_SUPPORTABLE", "linked account cannot serve as a quota data source (cn coding plan must be kimi/zhipu, cn payg must be kimi/deepseek, openai requires an oauth account, anthropic requires oauth or setup-token)", + ) ErrChannelMonitorInvalidAPIMode = infraerrors.BadRequest( "CHANNEL_MONITOR_INVALID_API_MODE", "api_mode must be chat_completions or responses; responses is only supported for openai", ) diff --git a/backend/internal/service/channel_monitor_quota_mode_test.go b/backend/internal/service/channel_monitor_quota_mode_test.go index 0e4b3ea7d..7309d2cf2 100644 --- a/backend/internal/service/channel_monitor_quota_mode_test.go +++ b/backend/internal/service/channel_monitor_quota_mode_test.go @@ -325,6 +325,16 @@ func TestValidateCreateParams_CheckModeMatrix(t *testing.T) { }, wantErr: ErrChannelMonitorInvalidCheckMode, }, + { + // quota_probe 仍要打真实探活请求：空模型必须报错，不再用 "quota" 占位。 + name: "quota_probe requires primary model", + params: ChannelMonitorCreateParams{ + Provider: MonitorProviderKimi, CheckMode: MonitorCheckModeQuotaProbe, + Endpoint: "https://api.kimi.com", APIKey: "sk", + IntervalSeconds: 60, AccountID: &accountID, + }, + wantErr: ErrChannelMonitorMissingPrimaryModel, + }, } for _, tc := range cases { @@ -342,8 +352,15 @@ func TestValidateCreateParams_CheckModeMatrix(t *testing.T) { func TestNormalizeMonitorPrimaryModel_QuotaDefault(t *testing.T) { require.Equal(t, "quota", normalizeMonitorPrimaryModel(MonitorProviderKimi, MonitorCheckModeQuota, "")) require.Equal(t, "quota", normalizeMonitorPrimaryModel(MonitorProviderAntigravity, MonitorCheckModeQuota, " ")) - // 探活模式沿用原语义：grok 默认模型，其余必填（空串报错在 validateCreateParams）。 + // quota_probe 仍要打真实探活请求：空模型返回 ""（由上层报 MissingPrimaryModel）， + // 不再用 "quota" 占位打 model="quota" 的请求。 + require.Equal(t, "", normalizeMonitorPrimaryModel(MonitorProviderKimi, MonitorCheckModeQuotaProbe, "")) + // grok 分支在纯 quota 占位之后：grok+quota 占位 "quota"， + // grok 探活（probe/quota_probe）默认轻量测活模型。 + require.Equal(t, "quota", normalizeMonitorPrimaryModel(MonitorProviderGrok, MonitorCheckModeQuota, "")) require.Equal(t, MonitorDefaultGrokModel, normalizeMonitorPrimaryModel(MonitorProviderGrok, MonitorCheckModeProbe, "")) + require.Equal(t, MonitorDefaultGrokModel, normalizeMonitorPrimaryModel(MonitorProviderGrok, MonitorCheckModeQuotaProbe, "")) + // 探活模式沿用原语义：其余必填（空串报错在 validateCreateParams）。 require.Equal(t, "kimi-k2", normalizeMonitorPrimaryModel(MonitorProviderKimi, MonitorCheckModeQuotaProbe, "kimi-k2")) } @@ -414,6 +431,150 @@ func TestRevalidateLinkedAccount_PlatformMismatch(t *testing.T) { require.Nil(t, probe.AccountID) } +// 能力矩阵：与 fetchUncached 路由一一对应，创建期拦截注定运行期永久 error 的组合。 +func TestMonitorAccountQuotaCapability_Matrix(t *testing.T) { + cases := []struct { + name string + account *Account + wantErr error + }{ + { + name: "deepseek coding has no quota endpoint", + account: &Account{ID: 1, Platform: domain.PlatformDeepseek, Credentials: map[string]any{"account_mode": AccountModeCoding}}, + wantErr: ErrChannelMonitorAccountNotSupportable, + }, + { + // 自定义域名 kimi coding：GetCodingPlanProvider 识别不到 → 无额度端点。 + name: "custom-domain kimi coding unsupported", + account: &Account{ID: 2, Platform: domain.PlatformKimi, Type: AccountTypeAPIKey, + Credentials: map[string]any{"account_mode": AccountModeCoding, "base_url": "https://cw.example.com"}}, + wantErr: ErrChannelMonitorAccountNotSupportable, + }, + { + name: "kimi coding default endpoint ok", + account: &Account{ID: 3, Platform: domain.PlatformKimi, Credentials: map[string]any{"account_mode": AccountModeCoding}}, + }, + { + name: "zhipu coding default endpoint ok", + account: &Account{ID: 4, Platform: domain.PlatformZhipu, Credentials: map[string]any{"account_mode": AccountModeCoding}}, + }, + { + name: "zhipu payg has no balance endpoint", + account: &Account{ID: 5, Platform: domain.PlatformZhipu}, + wantErr: ErrChannelMonitorAccountNotSupportable, + }, + { + name: "kimi payg ok", + account: &Account{ID: 6, Platform: domain.PlatformKimi}, + }, + { + name: "deepseek payg ok", + account: &Account{ID: 7, Platform: domain.PlatformDeepseek}, + }, + { + name: "anthropic api key cannot query usage", + account: &Account{ID: 8, Platform: domain.PlatformAnthropic, Type: AccountTypeAPIKey}, + wantErr: ErrChannelMonitorAccountNotSupportable, + }, + { + name: "anthropic oauth ok", + account: &Account{ID: 9, Platform: domain.PlatformAnthropic, Type: AccountTypeOAuth}, + }, + { + name: "anthropic setup token ok (local estimation)", + account: &Account{ID: 10, Platform: domain.PlatformAnthropic, Type: AccountTypeSetupToken}, + }, + { + name: "openai api key cannot query usage", + account: &Account{ID: 11, Platform: domain.PlatformOpenAI, Type: AccountTypeAPIKey}, + wantErr: ErrChannelMonitorAccountNotSupportable, + }, + { + name: "openai oauth ok", + account: &Account{ID: 12, Platform: domain.PlatformOpenAI, Type: AccountTypeOAuth}, + }, + { + // 防过度拦截：gemini/grok/antigravity 走本地统计/值通道降级，不会永久 error。 + name: "gemini api key ok", + account: &Account{ID: 13, Platform: domain.PlatformGemini, Type: AccountTypeAPIKey}, + }, + { + name: "grok ok", + account: &Account{ID: 14, Platform: domain.PlatformGrok}, + }, + { + name: "antigravity ok", + account: &Account{ID: 15, Platform: domain.PlatformAntigravity}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := monitorAccountQuotaCapability(tc.account) + if tc.wantErr == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, tc.wantErr) + } + }) + } +} + +func TestValidateLinkedAccount_CapabilityRejected(t *testing.T) { + svc := NewChannelMonitorService(nil, nil) + svc.SetQuotaFetcher(newQuotaModeFetcher(map[int64]*Account{ + 1: {ID: 1, Platform: domain.PlatformDeepseek, Credentials: map[string]any{"account_mode": AccountModeCoding}}, + }, nil)) + + err := svc.validateLinkedAccount(context.Background(), MonitorProviderDeepseek, int64Ptr(1)) + require.ErrorIs(t, err, ErrChannelMonitorAccountNotSupportable) +} + +func TestRevalidateLinkedAccount_Capability(t *testing.T) { + svc := NewChannelMonitorService(nil, nil) + svc.SetQuotaFetcher(newQuotaModeFetcher(map[int64]*Account{ + 2: {ID: 2, Platform: domain.PlatformDeepseek, Credentials: map[string]any{"account_mode": AccountModeCoding}}, + }, nil)) + + quota := &ChannelMonitor{Provider: MonitorProviderDeepseek, CheckMode: MonitorCheckModeQuota, AccountID: int64Ptr(2)} + require.ErrorIs(t, svc.revalidateLinkedAccount(context.Background(), quota), ErrChannelMonitorAccountNotSupportable) + require.NotNil(t, quota.AccountID, "quota mode keeps the binding for the admin to fix") + + probe := &ChannelMonitor{Provider: MonitorProviderDeepseek, CheckMode: MonitorCheckModeProbe, AccountID: int64Ptr(2)} + require.NoError(t, svc.revalidateLinkedAccount(context.Background(), probe)) + require.Nil(t, probe.AccountID, "probe mode should silently unbind unusable account") +} + +// provider-only 更新不得绕过 provider × check_mode 组合校验。 +func TestApplyMonitorUpdate_ProviderOnlyRevalidatesCheckMode(t *testing.T) { + probeKimi := func() *ChannelMonitor { + return &ChannelMonitor{ + Provider: MonitorProviderKimi, APIMode: MonitorAPIModeChatCompletions, + Endpoint: "https://api.kimi.com", PrimaryModel: "kimi-k2", + CheckMode: MonitorCheckModeProbe, + } + } + + provider := MonitorProviderAntigravity + err := applyMonitorUpdate(probeKimi(), ChannelMonitorUpdateParams{Provider: &provider}) + require.ErrorIs(t, err, ErrChannelMonitorInvalidCheckMode) + + // 带上 check_mode/account_id 的完整切换合法。 + accountID := int64(3) + err = applyMonitorUpdate(probeKimi(), ChannelMonitorUpdateParams{ + Provider: &provider, CheckMode: strPtr(MonitorCheckModeQuota), AccountID: &accountID, + }) + require.NoError(t, err) + + // 存量非法行（antigravity+probe）仅改名/停用不被砖化。 + legacy := &ChannelMonitor{ + Provider: MonitorProviderAntigravity, APIMode: MonitorAPIModeChatCompletions, + Endpoint: "https://example.com", PrimaryModel: "gemini-3-pro", + CheckMode: MonitorCheckModeProbe, + } + newName := "renamed" + require.NoError(t, applyMonitorUpdate(legacy, ChannelMonitorUpdateParams{Name: &newName})) +} + // --- quota → probe 切换的 key 管控（validateProbeAPIKey） --- func TestValidateProbeAPIKey_QuotaToProbeRequiresFreshKey(t *testing.T) { diff --git a/backend/internal/service/channel_monitor_service.go b/backend/internal/service/channel_monitor_service.go index 9b0bb0763..b08a623ce 100644 --- a/backend/internal/service/channel_monitor_service.go +++ b/backend/internal/service/channel_monitor_service.go @@ -406,7 +406,8 @@ func validateCreateParams(p ChannelMonitorCreateParams) error { return nil } -// validateLinkedAccount 校验关联账号存在且平台与监控 provider 一致。 +// validateLinkedAccount 校验关联账号存在、平台与监控 provider 一致、且能充当 +// 配额数据源（能力拦截，见 monitorAccountQuotaCapability）。 // fetcher 未注入时 fail-closed（拒绝创建配额监控，而不是创建后静默坏）。 func (s *ChannelMonitorService) validateLinkedAccount(ctx context.Context, provider string, accountID *int64) error { if accountID == nil || *accountID <= 0 { @@ -422,7 +423,7 @@ func (s *ChannelMonitorService) validateLinkedAccount(ctx context.Context, provi if account.Platform != provider { return ErrChannelMonitorProviderIncompatible } - return nil + return monitorAccountQuotaCapability(account) } // Update 更新监控。APIKey 字段：nil 或空字符串 = 不修改；非空 = 加密后覆盖。 @@ -533,6 +534,15 @@ func (s *ChannelMonitorService) revalidateLinkedAccount(ctx context.Context, m * m.AccountID = nil return nil } + // 能力失配（如 deepseek coding / zhipu payg / API-Key 型海外账号）： + // quota 模式显式报错（有该类存量监控时编辑会被拦，出路是换账号或切 probe）， + // probe 模式账号无用途，静默解绑。 + if err := monitorAccountQuotaCapability(account); err != nil { + if usesQuota { + return err + } + m.AccountID = nil + } return nil } @@ -886,11 +896,16 @@ func applyMonitorUpdate(existing *ChannelMonitor, p ChannelMonitorUpdateParams) existing.Provider = *p.Provider } if p.CheckMode != nil { - mode := defaultCheckMode(*p.CheckMode) - if err := validateCheckMode(existing.Provider, mode); err != nil { + existing.CheckMode = defaultCheckMode(*p.CheckMode) + } + // provider 与 check_mode 任一变化后统一复核组合矩阵：provider-only 更新 + // （如把 probe 监控的 provider 改成 antigravity）也不得落库非法组合，否则 + // 运行期恒 error。条件限定避免把存量非法行的 name/enabled-only 更新也判死 + // （否则连改名/停用都无法操作）。 + if p.Provider != nil || p.CheckMode != nil { + if err := validateCheckMode(existing.Provider, defaultCheckMode(existing.CheckMode)); err != nil { return err } - existing.CheckMode = mode } if p.AccountID != nil { if *p.AccountID > 0 { diff --git a/backend/internal/service/channel_monitor_validate.go b/backend/internal/service/channel_monitor_validate.go index faf7247fb..6b3b9fd81 100644 --- a/backend/internal/service/channel_monitor_validate.go +++ b/backend/internal/service/channel_monitor_validate.go @@ -190,21 +190,61 @@ func normalizeModels(in []string) []string { return out } -// normalizeMonitorPrimaryModel applies provider/check_mode defaults while -// preserving the existing required-model behavior: -// - Grok 探活默认轻量测活模型 -// - quota 模式占位 "quota"（primary_model 列 NotEmpty；历史行/时间线机制无需特判） +// normalizeMonitorPrimaryModel applies provider/check_mode defaults: +// - pure quota mode never sends requests: placeholder "quota" keeps +// primary_model NOT NULL (history rows / timeline need no special-casing) +// - quota_probe still sends a real probe request: empty model returns "" +// so validateCreateParams / applyMonitorUpdate report +// ErrChannelMonitorMissingPrimaryModel instead of probing model="quota" +// - Grok probing (probe/quota_probe) defaults to the lightweight check model func normalizeMonitorPrimaryModel(provider, checkMode, model string) string { model = strings.TrimSpace(model) + if model == "" && defaultCheckMode(checkMode) == MonitorCheckModeQuota { + return MonitorDefaultQuotaModel + } if model == "" && provider == MonitorProviderGrok { return MonitorDefaultGrokModel } - if model == "" && monitorCheckModeUsesQuota(defaultCheckMode(checkMode)) { - return MonitorDefaultQuotaModel - } return model } +// monitorAccountQuotaCapability 校验关联账号能否充当配额数据源，与 +// fetchUncached 的路由一一对应（coding→CN 额度端点 / payg→CN 余额端点 / +// 其余→AccountUsageService）。在创建/更新期拦截注定运行期永久 error 的组合： +// - kimi/zhipu/deepseek coding：GetCodingPlanProvider 须识别为 kimi/zhipu +// （deepseek coding、自定义域名 kimi coding 无法路由额度端点） +// - kimi/zhipu/deepseek payg：仅 kimi/deepseek 有公开余额端点（zhipu payg 无） +// - anthropic：OAuth / Setup Token（API-Key 型无 usage 通道，永久 error） +// - openai：OAuth（API-Key 型无 usage 通道） +// - gemini/grok/antigravity：本地统计/值通道降级，不会永久 error，放行 +func monitorAccountQuotaCapability(account *Account) error { + switch account.Platform { + case PlatformKimi, PlatformZhipu, PlatformDeepseek: + if account.IsCodingPlan() { + if p := account.GetCodingPlanProvider(); p != PlatformKimi && p != PlatformZhipu { + return ErrChannelMonitorAccountNotSupportable + } + return nil + } + if account.Platform == PlatformZhipu { + return ErrChannelMonitorAccountNotSupportable + } + return nil + case PlatformAnthropic: + if account.Type == AccountTypeOAuth || account.Type == AccountTypeSetupToken { + return nil + } + return ErrChannelMonitorAccountNotSupportable + case PlatformOpenAI: + if account.Type == AccountTypeOAuth { + return nil + } + return ErrChannelMonitorAccountNotSupportable + default: + return nil + } +} + // defaultAPIMode 空串归一为 chat_completions，保证历史数据与旧客户端兼容。 func defaultAPIMode(apiMode string) string { if strings.TrimSpace(apiMode) == "" { From e2dfb3b8cb6e7c8e217f1551850d7b67e2f147f1 Mon Sep 17 00:00:00 2001 From: Randark Date: Tue, 18 Aug 2026 10:04:23 +0000 Subject: [PATCH 10/50] refactor(monitor): pass loaded account through quota sources (single load per fetch) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 配额 fetcher 缓存未命中时账号被加载两次：fetchUncached GetByID 一次， 下游数据源（GetUsage / CN QueryUsage / CN QueryBalance）按 ID 再各自 GetByID 一次——每次 GetByID 含 proxies/groups 联查，纯属重复劳动。 统一改为「路由前加载一次、指针直传」： - fetcher 三个数据源接口签名改收 *Account： GetUsageForAccount / QueryUsageForAccount / QueryBalanceForAccount - AccountUsageService 暴露 GetUsageForAccount 直通（getUsageForAccount 既有逻辑零改动） - CN 两侧服务抽 validateCodingPlanAccount / validatePayGAccount （加载后校验原样），新 ForAccount 入口 = NOT_CONFIGURED 守卫 → 校验 → singleflight → 探测；ID 入口 = load + 委托，语义不变 - singleflight key 保持 "cn_quota:" / "cn_balance:"：ForAccount 与 admin handler 的 ID 入口并发探测仍按账号合并 - cn_provider_balance_check_service.checkOne 刻意留在 ID 入口：调度器 路径按 ID 取的是最新凭据，不复用可能已过期的已加载账号 校验移到 flight 之外且 ForAccount 复用同一套校验，直传无法绕过 平台/模式检查；专项测试固化（无效账号零出站请求、单次加载指针直传 require.Same）。 --- .../internal/service/account_usage_service.go | 7 + .../service/channel_monitor_quota_fetcher.go | 44 +++--- .../channel_monitor_quota_fetcher_test.go | 83 ++++++++++-- .../service/cn_provider_balance_service.go | 40 ++++-- .../service/cn_provider_foraccount_test.go | 125 ++++++++++++++++++ .../service/cn_provider_quota_service.go | 49 ++++--- 6 files changed, 288 insertions(+), 60 deletions(-) create mode 100644 backend/internal/service/cn_provider_foraccount_test.go diff --git a/backend/internal/service/account_usage_service.go b/backend/internal/service/account_usage_service.go index 4088408c7..1661eca7c 100644 --- a/backend/internal/service/account_usage_service.go +++ b/backend/internal/service/account_usage_service.go @@ -503,6 +503,13 @@ func (s *AccountUsageService) GetUsage(ctx context.Context, accountID int64, for return s.getUsageForAccount(ctx, account, forceProbe) } +// GetUsageForAccount 已加载账号的使用量直通入口（配额监控 fetcher 复用， +// 避免缓存未命中时账号被加载两次——每次 GetByID 含 proxies/groups 联查）。 +func (s *AccountUsageService) GetUsageForAccount(ctx context.Context, account *Account, force ...bool) (*UsageInfo, error) { + forceProbe := len(force) > 0 && force[0] + return s.getUsageForAccount(ctx, account, forceProbe) +} + // GetUsageBatch 批量获取账号使用量。 // Anthropic OAuth/SetupToken 统一走 passive 链路，其他账号复用现有主动查询逻辑。 // 单个账号失败不会中断整批请求，错误会按账号返回。 diff --git a/backend/internal/service/channel_monitor_quota_fetcher.go b/backend/internal/service/channel_monitor_quota_fetcher.go index 2b6d54b68..479762ae8 100644 --- a/backend/internal/service/channel_monitor_quota_fetcher.go +++ b/backend/internal/service/channel_monitor_quota_fetcher.go @@ -19,10 +19,12 @@ import ( // 渠道监控「配额模式」的配额抓取器。 // // 不直接对接上游，而是把账号侧现成的用量服务归一成 domain.MonitorQuotaSnapshot： -// - 海外 5 家（anthropic/openai/gemini/antigravity/grok）→ AccountUsageService.GetUsage -// - 国产 coding plan（kimi/zhipu/deepseek）→ CNProviderQuotaService.QueryUsage -// - 国产 payg（kimi/deepseek）→ CNProviderBalanceService.QueryBalance -// （zhipu payg 无公开余额端点，QueryBalance 会返回该错误，原样透出） +// - 海外 5 家（anthropic/openai/gemini/antigravity/grok）→ AccountUsageService.GetUsageForAccount +// - 国产 coding plan（kimi/zhipu/deepseek）→ CNProviderQuotaService.QueryUsageForAccount +// - 国产 payg（kimi/deepseek）→ CNProviderBalanceService.QueryBalanceForAccount +// （zhipu payg 无公开余额端点，探测会返回该错误，原样透出） +// 数据源统一接受已加载的 *Account：fetchUncached 路由前 GetByID 一次并传下去， +// 下游服务不再各自重载（每次 GetByID 含 proxies/groups 联查）。 // // Fetch 永不返回 error：所有失败都降级为 Success=false 的快照照常入库， // 由 deriveQuotaCheckResult 推导为 failed/error 状态。 @@ -33,18 +35,19 @@ import ( // 抓取由 singleflight 合并为一次上游查询。 // monitorUsageSource 海外平台账号用量查询（AccountUsageService 天然满足）。 +// 传已加载的 *Account：fetchUncached 只 GetByID 一次，下游不再重复加载。 type monitorUsageSource interface { - GetUsage(ctx context.Context, accountID int64, force ...bool) (*UsageInfo, error) + GetUsageForAccount(ctx context.Context, account *Account, force ...bool) (*UsageInfo, error) } // monitorCNQuotaSource 国产 coding plan 滚动窗口额度探测（CNProviderQuotaService 天然满足）。 type monitorCNQuotaSource interface { - QueryUsage(ctx context.Context, accountID int64) (*CNProviderQuotaProbeResult, error) + QueryUsageForAccount(ctx context.Context, account *Account) (*CNProviderQuotaProbeResult, error) } // monitorCNBalanceSource 国产 payg 余额探测（CNProviderBalanceService 天然满足）。 type monitorCNBalanceSource interface { - QueryBalance(ctx context.Context, accountID int64) (*CNProviderBalanceResult, error) + QueryBalanceForAccount(ctx context.Context, account *Account) (*CNProviderBalanceResult, error) } // monitorAccountSource 账号加载（AccountRepository 天然满足）。 @@ -191,23 +194,26 @@ func (f *ChannelMonitorQuotaFetcher) fetchUncached(ctx context.Context, accountI return quotaErrorSnapshot("usage", "linked account not found", now) } + // 账号只在路由前加载这一次；已加载的 account 直接传给数据源 + // （GetUsageForAccount / QueryUsageForAccount / QueryBalanceForAccount）， + // 下游服务不再各自 GetByID（每次含 proxies/groups 联查）。 switch account.Platform { case domain.PlatformKimi, domain.PlatformZhipu, domain.PlatformDeepseek: if account.IsCodingPlan() { - return f.fetchCNQuota(ctx, accountID, now) + return f.fetchCNQuota(ctx, account, now) } - return f.fetchCNBalance(ctx, accountID, now) + return f.fetchCNBalance(ctx, account, now) default: - return f.fetchUsage(ctx, accountID, now) + return f.fetchUsage(ctx, account, now) } } -// fetchUsage 海外平台：AccountUsageService.GetUsage → 快照。 -func (f *ChannelMonitorQuotaFetcher) fetchUsage(ctx context.Context, accountID int64, now time.Time) *domain.MonitorQuotaSnapshot { +// fetchUsage 海外平台：AccountUsageService.GetUsageForAccount → 快照。 +func (f *ChannelMonitorQuotaFetcher) fetchUsage(ctx context.Context, account *Account, now time.Time) *domain.MonitorQuotaSnapshot { if f.usage == nil { return quotaErrorSnapshot("usage", "usage service is not configured", now) } - usage, err := f.usage.GetUsage(ctx, accountID) + usage, err := f.usage.GetUsageForAccount(ctx, account) if err != nil { msg := truncateMessage(sanitizeErrorMessage(err.Error())) return &domain.MonitorQuotaSnapshot{ @@ -336,12 +342,12 @@ func sortedQuotaModelNames(quotas map[string]*AntigravityModelQuota) []string { return names } -// fetchCNQuota 国产 coding plan：CNProviderQuotaService.QueryUsage → 快照。 -func (f *ChannelMonitorQuotaFetcher) fetchCNQuota(ctx context.Context, accountID int64, now time.Time) *domain.MonitorQuotaSnapshot { +// fetchCNQuota 国产 coding plan：CNProviderQuotaService.QueryUsageForAccount → 快照。 +func (f *ChannelMonitorQuotaFetcher) fetchCNQuota(ctx context.Context, account *Account, now time.Time) *domain.MonitorQuotaSnapshot { if f.cnQuota == nil { return quotaErrorSnapshot("cn_quota", "cn quota service is not configured", now) } - result, err := f.cnQuota.QueryUsage(ctx, accountID) + result, err := f.cnQuota.QueryUsageForAccount(ctx, account) if err != nil { msg := truncateMessage(sanitizeErrorMessage(err.Error())) return &domain.MonitorQuotaSnapshot{ @@ -381,12 +387,12 @@ func (f *ChannelMonitorQuotaFetcher) fetchCNQuota(ctx context.Context, accountID return snapshot } -// fetchCNBalance 国产 payg：CNProviderBalanceService.QueryBalance → 快照。 -func (f *ChannelMonitorQuotaFetcher) fetchCNBalance(ctx context.Context, accountID int64, now time.Time) *domain.MonitorQuotaSnapshot { +// fetchCNBalance 国产 payg：CNProviderBalanceService.QueryBalanceForAccount → 快照。 +func (f *ChannelMonitorQuotaFetcher) fetchCNBalance(ctx context.Context, account *Account, now time.Time) *domain.MonitorQuotaSnapshot { if f.cnBalance == nil { return quotaErrorSnapshot("cn_balance", "cn balance service is not configured", now) } - result, err := f.cnBalance.QueryBalance(ctx, accountID) + result, err := f.cnBalance.QueryBalanceForAccount(ctx, account) if err != nil { msg := truncateMessage(sanitizeErrorMessage(err.Error())) return &domain.MonitorQuotaSnapshot{ diff --git a/backend/internal/service/channel_monitor_quota_fetcher_test.go b/backend/internal/service/channel_monitor_quota_fetcher_test.go index d4f4d38a4..6c5b06559 100644 --- a/backend/internal/service/channel_monitor_quota_fetcher_test.go +++ b/backend/internal/service/channel_monitor_quota_fetcher_test.go @@ -20,18 +20,20 @@ import ( type stubMonitorUsageSource struct { usage *UsageInfo err error - // block 非 nil 时 GetUsage 阻塞在该 channel 上，用于并发/singleflight 测试。 + // block 非 nil 时 GetUsageForAccount 阻塞在该 channel 上，用于并发/singleflight 测试。 block chan struct{} - mu sync.Mutex - calls int - lastCtx context.Context + mu sync.Mutex + calls int + lastCtx context.Context + lastAccount *Account } -func (s *stubMonitorUsageSource) GetUsage(ctx context.Context, accountID int64, force ...bool) (*UsageInfo, error) { +func (s *stubMonitorUsageSource) GetUsageForAccount(ctx context.Context, account *Account, force ...bool) (*UsageInfo, error) { s.mu.Lock() s.calls++ s.lastCtx = ctx + s.lastAccount = account s.mu.Unlock() if s.block != nil { <-s.block @@ -45,25 +47,35 @@ func (s *stubMonitorUsageSource) getCalls() int { return s.calls } -type stubMonitorCNQuotaSource struct { - result *CNProviderQuotaProbeResult - err error - calls int +func (s *stubMonitorUsageSource) getLastAccount() *Account { + s.mu.Lock() + defer s.mu.Unlock() + return s.lastAccount } -func (s *stubMonitorCNQuotaSource) QueryUsage(ctx context.Context, accountID int64) (*CNProviderQuotaProbeResult, error) { +type stubMonitorCNQuotaSource struct { + result *CNProviderQuotaProbeResult + err error + calls int + lastAccount *Account +} + +func (s *stubMonitorCNQuotaSource) QueryUsageForAccount(ctx context.Context, account *Account) (*CNProviderQuotaProbeResult, error) { s.calls++ + s.lastAccount = account return s.result, s.err } type stubMonitorCNBalanceSource struct { - result *CNProviderBalanceResult - err error - calls int + result *CNProviderBalanceResult + err error + calls int + lastAccount *Account } -func (s *stubMonitorCNBalanceSource) QueryBalance(ctx context.Context, accountID int64) (*CNProviderBalanceResult, error) { +func (s *stubMonitorCNBalanceSource) QueryBalanceForAccount(ctx context.Context, account *Account) (*CNProviderBalanceResult, error) { s.calls++ + s.lastAccount = account return s.result, s.err } @@ -192,6 +204,49 @@ func TestQuotaFetcher_PayGAccountUsesCNBalance(t *testing.T) { require.Empty(t, snapshot.Error) } +// P2-6：fetchUncached 只 GetByID 一次，已加载的 account 指针直传数据源， +// 三条路由都不能让下游重载账号。 +func TestQuotaFetcher_LoadsAccountOnceAndPassesItThrough(t *testing.T) { + t.Run("overseas usage", func(t *testing.T) { + fetcher, usage, _, _, accounts := newQuotaFetcherTestSetup(t) + acc := &Account{ID: 21, Platform: domain.PlatformAnthropic} + accounts.accounts[21] = acc + usage.usage = &UsageInfo{} + + fetcher.Fetch(context.Background(), 21) + + require.Equal(t, 1, accounts.calls) + require.Same(t, acc, usage.getLastAccount()) + require.Equal(t, 1, usage.getCalls()) + }) + + t.Run("cn coding plan", func(t *testing.T) { + fetcher, _, cnQuota, _, accounts := newQuotaFetcherTestSetup(t) + acc := &Account{ID: 22, Platform: domain.PlatformKimi, Credentials: map[string]any{"account_mode": AccountModeCoding}} + accounts.accounts[22] = acc + cnQuota.result = &CNProviderQuotaProbeResult{Success: true} + + fetcher.Fetch(context.Background(), 22) + + require.Equal(t, 1, accounts.calls) + require.Same(t, acc, cnQuota.lastAccount) + require.Equal(t, 1, cnQuota.calls) + }) + + t.Run("cn payg", func(t *testing.T) { + fetcher, _, _, cnBalance, accounts := newQuotaFetcherTestSetup(t) + acc := &Account{ID: 23, Platform: domain.PlatformDeepseek, Credentials: map[string]any{"account_mode": AccountModePayG}} + accounts.accounts[23] = acc + cnBalance.result = &CNProviderBalanceResult{Success: true, Available: true, Balance: 1, Currency: "CNY"} + + fetcher.Fetch(context.Background(), 23) + + require.Equal(t, 1, accounts.calls) + require.Same(t, acc, cnBalance.lastAccount) + require.Equal(t, 1, cnBalance.calls) + }) +} + // --- 失败路径（Fetch 永不返回 error） --- func TestQuotaFetcher_AccountMissingYieldsLinkedAccountSnapshot(t *testing.T) { diff --git a/backend/internal/service/cn_provider_balance_service.go b/backend/internal/service/cn_provider_balance_service.go index d01c9b6d0..06bab8ae0 100644 --- a/backend/internal/service/cn_provider_balance_service.go +++ b/backend/internal/service/cn_provider_balance_service.go @@ -84,14 +84,27 @@ func NewCNProviderBalanceService( // QueryBalance 探测指定 payg 账号的余额并落 Extra 快照。 func (s *CNProviderBalanceService) QueryBalance(ctx context.Context, accountID int64) (*CNProviderBalanceResult, error) { + account, err := s.loadPayGAccount(ctx, accountID) + if err != nil { + return nil, err + } + return s.QueryBalanceForAccount(ctx, account) +} + +// QueryBalanceForAccount 探测已加载账号（配额监控 fetcher / 周期余额检测复用， +// 避免二次 GetByID）。singleflight key 与 QueryBalance 相同，按账号 ID 合并。 +func (s *CNProviderBalanceService) QueryBalanceForAccount(ctx context.Context, account *Account) (*CNProviderBalanceResult, error) { if s == nil || s.accountRepo == nil || s.httpUpstream == nil { return nil, infraerrors.New(http.StatusInternalServerError, "CN_BALANCE_NOT_CONFIGURED", "cn provider balance service is not configured") } - key := "cn_balance:" + strconv.FormatInt(accountID, 10) + if err := validatePayGAccount(account); err != nil { + return nil, err + } + key := "cn_balance:" + strconv.FormatInt(account.ID, 10) resultCh := s.flight.DoChan(key, func() (any, error) { probeCtx, cancel := context.WithTimeout(context.Background(), cnBalanceUpstreamTimeout+5*time.Second) defer cancel() - return s.queryBalance(probeCtx, accountID) + return s.queryBalanceForAccount(probeCtx, account) }) select { case <-ctx.Done(): @@ -109,11 +122,7 @@ func (s *CNProviderBalanceService) QueryBalance(ctx context.Context, accountID i } } -func (s *CNProviderBalanceService) queryBalance(ctx context.Context, accountID int64) (*CNProviderBalanceResult, error) { - account, err := s.loadPayGAccount(ctx, accountID) - if err != nil { - return nil, err - } +func (s *CNProviderBalanceService) queryBalanceForAccount(ctx context.Context, account *Account) (*CNProviderBalanceResult, error) { provider := account.Platform if provider != PlatformKimi && provider != PlatformDeepseek { return nil, infraerrors.New(http.StatusBadRequest, "CN_BALANCE_NO_ENDPOINT", "account provider has no balance endpoint") @@ -230,17 +239,26 @@ func (s *CNProviderBalanceService) loadPayGAccount(ctx context.Context, accountI if err != nil { return nil, infraerrors.Newf(http.StatusNotFound, "CN_BALANCE_ACCOUNT_NOT_FOUND", "account not found: %v", err) } + if err := validatePayGAccount(account); err != nil { + return nil, err + } + return account, nil +} + +// validatePayGAccount 加载后的非 DB 校验（ForAccount 入口同样复用， +// 保证直传 account 也不绕过平台/模式检查）。 +func validatePayGAccount(account *Account) error { if account == nil { - return nil, infraerrors.New(http.StatusNotFound, "CN_BALANCE_ACCOUNT_NOT_FOUND", "account not found") + return infraerrors.New(http.StatusNotFound, "CN_BALANCE_ACCOUNT_NOT_FOUND", "account not found") } if !account.IsCNProvider() { - return nil, infraerrors.New(http.StatusBadRequest, "CN_BALANCE_INVALID_PLATFORM", "account is not a CN provider account") + return infraerrors.New(http.StatusBadRequest, "CN_BALANCE_INVALID_PLATFORM", "account is not a CN provider account") } // coding 账号走额度探测，余额端点不适用。 if account.IsCodingPlan() { - return nil, infraerrors.New(http.StatusBadRequest, "CN_BALANCE_CODING_PLAN", "coding plan account has no balance endpoint; use quota probe") + return infraerrors.New(http.StatusBadRequest, "CN_BALANCE_CODING_PLAN", "coding plan account has no balance endpoint; use quota probe") } - return account, nil + return nil } func (s *CNProviderBalanceService) resolveProxyURL(ctx context.Context, account *Account) string { diff --git a/backend/internal/service/cn_provider_foraccount_test.go b/backend/internal/service/cn_provider_foraccount_test.go new file mode 100644 index 000000000..664da505f --- /dev/null +++ b/backend/internal/service/cn_provider_foraccount_test.go @@ -0,0 +1,125 @@ +package service + +// ForAccount 直传入口（P2-6）的校验回归测试： +// QueryUsageForAccount / QueryBalanceForAccount 接受已加载的 *Account， +// 但必须复用与 ID 入口相同的加载后校验——直传不能绕过平台/模式检查， +// 且校验在 singleflight 之前完成（无效账号不得发起任何上游请求）。 + +import ( + "context" + "testing" + + infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" + "github.com/stretchr/testify/require" +) + +func codingAccount(platform string) *Account { + return &Account{ + ID: 1, Platform: platform, Type: AccountTypeAPIKey, Status: StatusActive, + Credentials: map[string]any{"account_mode": AccountModeCoding, "api_key": "sk-test"}, + } +} + +func paygAccount(platform string) *Account { + return &Account{ + ID: 2, Platform: platform, Type: AccountTypeAPIKey, Status: StatusActive, + Credentials: map[string]any{"account_mode": AccountModePayG, "api_key": "sk-test"}, + } +} + +func requireReason(t *testing.T, err error, reason string) { + t.Helper() + require.Error(t, err) + var appErr *infraerrors.ApplicationError + require.ErrorAs(t, err, &appErr) + require.Equal(t, reason, appErr.Reason) +} + +func TestValidateCodingPlanAccount_Matrix(t *testing.T) { + cases := []struct { + name string + account *Account + wantReason string + }{ + {name: "nil", account: nil, wantReason: "CN_QUOTA_ACCOUNT_NOT_FOUND"}, + {name: "non cn provider", account: &Account{ID: 3, Platform: PlatformAnthropic}, wantReason: "CN_QUOTA_INVALID_PLATFORM"}, + {name: "payg has no quota endpoint", account: paygAccount(PlatformKimi), wantReason: "CN_QUOTA_NOT_CODING_PLAN"}, + {name: "kimi coding ok", account: codingAccount(PlatformKimi)}, + {name: "zhipu coding ok", account: codingAccount(PlatformZhipu)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := validateCodingPlanAccount(tc.account) + if tc.wantReason == "" { + require.NoError(t, err) + return + } + requireReason(t, err, tc.wantReason) + }) + } +} + +func TestValidatePayGAccount_Matrix(t *testing.T) { + cases := []struct { + name string + account *Account + wantReason string + }{ + {name: "nil", account: nil, wantReason: "CN_BALANCE_ACCOUNT_NOT_FOUND"}, + {name: "non cn provider", account: &Account{ID: 3, Platform: PlatformAnthropic}, wantReason: "CN_BALANCE_INVALID_PLATFORM"}, + {name: "coding has no balance endpoint", account: codingAccount(PlatformKimi), wantReason: "CN_BALANCE_CODING_PLAN"}, + {name: "kimi payg ok", account: paygAccount(PlatformKimi)}, + {name: "deepseek payg ok", account: paygAccount(PlatformDeepseek)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := validatePayGAccount(tc.account) + if tc.wantReason == "" { + require.NoError(t, err) + return + } + requireReason(t, err, tc.wantReason) + }) + } +} + +// 直传入口的校验在 singleflight/上游请求之前：无效账号必须零出站请求。 +func TestCNProviderQuotaService_QueryUsageForAccount_RejectsInvalidAccount(t *testing.T) { + repo := &fakeCNProbeAccountRepo{} + upstream := &recordingHTTPUpstream{} + svc := NewCNProviderQuotaService(repo, nil, upstream, nil) + + _, err := svc.QueryUsageForAccount(context.Background(), paygAccount(PlatformKimi)) + requireReason(t, err, "CN_QUOTA_NOT_CODING_PLAN") + require.Zero(t, upstream.calls) + + _, err = svc.QueryUsageForAccount(context.Background(), nil) + requireReason(t, err, "CN_QUOTA_ACCOUNT_NOT_FOUND") + require.Zero(t, upstream.calls) +} + +func TestCNProviderBalanceService_QueryBalanceForAccount_RejectsInvalidAccount(t *testing.T) { + repo := &fakeCNProbeAccountRepo{} + upstream := &recordingHTTPUpstream{} + svc := NewCNProviderBalanceService(repo, nil, upstream, nil) + + _, err := svc.QueryBalanceForAccount(context.Background(), codingAccount(PlatformKimi)) + requireReason(t, err, "CN_BALANCE_CODING_PLAN") + require.Zero(t, upstream.calls) + + _, err = svc.QueryBalanceForAccount(context.Background(), &Account{ID: 9, Platform: PlatformAnthropic}) + requireReason(t, err, "CN_BALANCE_INVALID_PLATFORM") + require.Zero(t, upstream.calls) +} + +// ID 入口与 ForAccount 入口对同一账号的行为一致（loadCodingPlanAccount 的 +// 加载后校验 = validateCodingPlanAccount；余额侧对称）。 +func TestCNProviderServices_IDEntryAppliesSameValidation(t *testing.T) { + repo := &fakeCNProbeAccountRepo{account: paygAccount(PlatformKimi)} + upstream := &recordingHTTPUpstream{} + svc := NewCNProviderQuotaService(repo, nil, upstream, nil) + + _, err := svc.QueryUsage(context.Background(), 2) + requireReason(t, err, "CN_QUOTA_NOT_CODING_PLAN") + require.Zero(t, upstream.calls) +} diff --git a/backend/internal/service/cn_provider_quota_service.go b/backend/internal/service/cn_provider_quota_service.go index f4cda8d6d..271b18afe 100644 --- a/backend/internal/service/cn_provider_quota_service.go +++ b/backend/internal/service/cn_provider_quota_service.go @@ -89,14 +89,27 @@ func NewCNProviderQuotaService( // QueryUsage 探测指定账号的 Coding Plan 滚动窗口用量并落 Extra 快照。 // 同一账号的并发探测会被 singleflight 合并。 func (s *CNProviderQuotaService) QueryUsage(ctx context.Context, accountID int64) (*CNProviderQuotaProbeResult, error) { + account, err := s.loadCodingPlanAccount(ctx, accountID) + if err != nil { + return nil, err + } + return s.QueryUsageForAccount(ctx, account) +} + +// QueryUsageForAccount 探测已加载账号（配额监控 fetcher 复用，避免二次 GetByID）。 +// singleflight key 与 QueryUsage 相同，按账号 ID 与 admin 侧并发探测合并。 +func (s *CNProviderQuotaService) QueryUsageForAccount(ctx context.Context, account *Account) (*CNProviderQuotaProbeResult, error) { if s == nil || s.accountRepo == nil || s.httpUpstream == nil { return nil, infraerrors.New(http.StatusInternalServerError, "CN_QUOTA_NOT_CONFIGURED", "cn provider quota service is not configured") } - key := "cn_quota:" + strconv.FormatInt(accountID, 10) + if err := validateCodingPlanAccount(account); err != nil { + return nil, err + } + key := "cn_quota:" + strconv.FormatInt(account.ID, 10) resultCh := s.flight.DoChan(key, func() (any, error) { probeCtx, cancel := context.WithTimeout(context.Background(), cnQuotaUpstreamTimeout+5*time.Second) defer cancel() - return s.queryUsage(probeCtx, accountID) + return s.queryUsageForAccount(probeCtx, account) }) select { case <-ctx.Done(): @@ -114,12 +127,7 @@ func (s *CNProviderQuotaService) QueryUsage(ctx context.Context, accountID int64 } } -func (s *CNProviderQuotaService) queryUsage(ctx context.Context, accountID int64) (*CNProviderQuotaProbeResult, error) { - account, err := s.loadCodingPlanAccount(ctx, accountID) - if err != nil { - return nil, err - } - +func (s *CNProviderQuotaService) queryUsageForAccount(ctx context.Context, account *Account) (*CNProviderQuotaProbeResult, error) { provider := account.GetCodingPlanProvider() if provider != PlatformKimi && provider != PlatformZhipu { return nil, infraerrors.New(http.StatusBadRequest, "CN_QUOTA_NOT_CODING_PLAN", "account is not a kimi/zhipu coding plan account") @@ -231,18 +239,27 @@ func (s *CNProviderQuotaService) loadCodingPlanAccount(ctx context.Context, acco if err != nil { return nil, infraerrors.Newf(http.StatusNotFound, "CN_QUOTA_ACCOUNT_NOT_FOUND", "account not found: %v", err) } - if account == nil { - return nil, infraerrors.New(http.StatusNotFound, "CN_QUOTA_ACCOUNT_NOT_FOUND", "account not found") - } - if !account.IsCNProvider() { - return nil, infraerrors.New(http.StatusBadRequest, "CN_QUOTA_INVALID_PLATFORM", "account is not a CN provider account") - } - if !account.IsCodingPlan() { - return nil, infraerrors.New(http.StatusBadRequest, "CN_QUOTA_NOT_CODING_PLAN", "account is not a coding plan account") + if err := validateCodingPlanAccount(account); err != nil { + return nil, err } return account, nil } +// validateCodingPlanAccount 加载后的非 DB 校验（ForAccount 入口同样复用， +// 保证直传 account 也不绕过平台/模式检查）。 +func validateCodingPlanAccount(account *Account) error { + if account == nil { + return infraerrors.New(http.StatusNotFound, "CN_QUOTA_ACCOUNT_NOT_FOUND", "account not found") + } + if !account.IsCNProvider() { + return infraerrors.New(http.StatusBadRequest, "CN_QUOTA_INVALID_PLATFORM", "account is not a CN provider account") + } + if !account.IsCodingPlan() { + return infraerrors.New(http.StatusBadRequest, "CN_QUOTA_NOT_CODING_PLAN", "account is not a coding plan account") + } + return nil +} + func (s *CNProviderQuotaService) resolveProxyURL(ctx context.Context, account *Account) string { if account == nil || account.ProxyID == nil { return "" From c9effc45603f1051d9dcad7fbfd1cc209c227c7a Mon Sep 17 00:00:00 2001 From: Randark Date: Tue, 18 Aug 2026 10:08:01 +0000 Subject: [PATCH 11/50] fix(frontend): monitor form check-mode restore, account unbinding and mode badge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 审计 P2-7 三处前端收尾： - 切离 antigravity 时对称还原 check_mode：antigravity 强制 quota，但反向 切走没有还原——编辑存量 antigravity 监控换平台后停留在 quota，目标平台 未必支持。现切走时还原为 probe 并清掉占位模型 'quota'（回填值），否则 probe 模式拿 'quota' 当探活模型（与离开 grok 清 DEFAULT_GROK_MODEL 同一先例）。 - update 切回 probe 显式发 account_id=0：后端语义 0=解绑、null=不动， 此前 probe 模式 payload 带 null，存量关联永远清不掉。仅 update—— create 发 0 会落 &0 触发外键违约（API 类型注释同步写明）。 - 管理列表 provider 徽章旁并列 check_mode 徽章（checkModeLabel 此前 定义了却无处使用）：quota 系=蓝、probe=中性灰，三种模式一眼可分； i18n 复用既有 monitorCommon.checkMode.*，零新增键。 --- frontend/src/api/admin/channelMonitor.ts | 7 +- .../admin/monitor/MonitorFormDialog.vue | 11 ++ .../composables/useChannelMonitorFormat.ts | 20 +++ .../src/views/admin/ChannelMonitorView.vue | 6 + .../ChannelMonitorView.checkModeBadge.spec.ts | 159 ++++++++++++++++++ .../MonitorFormDialog.accountSelector.spec.ts | 65 +++++++ 6 files changed, 266 insertions(+), 2 deletions(-) create mode 100644 frontend/src/views/admin/__tests__/ChannelMonitorView.checkModeBadge.spec.ts diff --git a/frontend/src/api/admin/channelMonitor.ts b/frontend/src/api/admin/channelMonitor.ts index f3353ebd4..bf9e1c88f 100644 --- a/frontend/src/api/admin/channelMonitor.ts +++ b/frontend/src/api/admin/channelMonitor.ts @@ -134,7 +134,9 @@ export interface CreateParams { api_key: string /** 缺省 probe；antigravity 仅支持 quota */ check_mode?: CheckMode - /** 配额模式必填：数据源账号（provider 需与账号平台一致） */ + /** 配额模式必填：数据源账号（provider 需与账号平台一致）。 + * update 语义：>0=换绑，0=解绑（切回 probe 模式时前端发 0 清空存量关联）； + * create 绝不发 0——后端会把 0 存成 &0 触发外键违约。 */ account_id?: number | null primary_model: string extra_models?: string[] @@ -148,7 +150,8 @@ export interface CreateParams { body_override?: Record | null } -// Update request: api_key 空串 = 不修改；clear_template=true 时把 template_id 置空 +// Update request: api_key 空串 = 不修改；clear_template=true 时把 template_id 置空； +// account_id=0 显式解绑关联账号（null = 不动，见 CreateParams 注释） export type UpdateParams = Partial & { clear_template?: boolean } diff --git a/frontend/src/components/admin/monitor/MonitorFormDialog.vue b/frontend/src/components/admin/monitor/MonitorFormDialog.vue index ea73b929e..fb89f1a0c 100644 --- a/frontend/src/components/admin/monitor/MonitorFormDialog.vue +++ b/frontend/src/components/admin/monitor/MonitorFormDialog.vue @@ -675,6 +675,14 @@ function selectProvider(provider: Provider) { if (provider === PROVIDER_ANTIGRAVITY && form.check_mode !== CHECK_MODE_QUOTA) { form.check_mode = CHECK_MODE_QUOTA } + // 对称还原：从 antigravity 切走时撤掉强制 quota，否则编辑存量 antigravity + // 监控换平台后仍停留在 quota（目标平台未必支持），update 会携带残留配置。 + // 同步清掉 quota 占位模型（loadFromMonitor 回填的 'quota'），否则切回 + // probe 后拿 'quota' 当探活模型（与离开 grok 清 DEFAULT_GROK_MODEL 同理）。 + if (previousProvider === PROVIDER_ANTIGRAVITY && form.check_mode === CHECK_MODE_QUOTA) { + form.check_mode = CHECK_MODE_PROBE + if (form.primary_model.trim() === 'quota') form.primary_model = '' + } if (provider === PROVIDER_GROK) { if (!form.endpoint.trim()) form.endpoint = DEFAULT_GROK_ENDPOINT if (!form.primary_model.trim()) form.primary_model = DEFAULT_GROK_MODEL @@ -849,6 +857,9 @@ async function handleSubmit() { req.clear_template = true delete req.template_id } + // account_id 同理：probe 模式不带账号，update 发 0 显式解绑存量关联 + // （后端 0=清空、null=不动）。仅 update——create 发 0 会落 &0 触发 FK 违约。 + if (!usesQuotaMode.value) req.account_id = 0 await adminAPI.channelMonitor.update(target.id, req) appStore.showSuccess(t('admin.channelMonitor.updateSuccess')) } else { diff --git a/frontend/src/composables/useChannelMonitorFormat.ts b/frontend/src/composables/useChannelMonitorFormat.ts index 04fc0546f..67c0f197f 100644 --- a/frontend/src/composables/useChannelMonitorFormat.ts +++ b/frontend/src/composables/useChannelMonitorFormat.ts @@ -26,6 +26,9 @@ import { STATUS_DEGRADED, STATUS_FAILED, STATUS_ERROR, + CHECK_MODE_PROBE, + CHECK_MODE_QUOTA, + CHECK_MODE_QUOTA_PROBE, } from '@/constants/channelMonitor' const NEUTRAL_BADGE = 'bg-gray-100 text-gray-800 dark:bg-dark-700 dark:text-gray-300' @@ -101,6 +104,22 @@ export function useChannelMonitorFormat() { } } + /** + * Tailwind class for the check-mode badge shown next to the provider badge + * in the admin monitor list. Quota-bearing modes = blue (数据源是账号配额), + * plain probe = neutral grey. + */ + function checkModeBadgeClass(m: CheckMode | string): string { + switch (m) { + case CHECK_MODE_QUOTA: + case CHECK_MODE_QUOTA_PROBE: + return 'bg-blue-100 text-blue-700 dark:bg-blue-500/15 dark:text-blue-300' + case CHECK_MODE_PROBE: + default: + return NEUTRAL_BADGE + } + } + /** * Tailwind class for a provider radio-button-style picker (active/inactive state). * Reuses the same emerald/orange/sky palette as providerBadgeClass to keep @@ -182,6 +201,7 @@ export function useChannelMonitorFormat() { providerLabel, checkModeLabel, providerBadgeClass, + checkModeBadgeClass, providerPickerClass, formatLatency, formatPercent, diff --git a/frontend/src/views/admin/ChannelMonitorView.vue b/frontend/src/views/admin/ChannelMonitorView.vue index d1bb148e8..ae154f713 100644 --- a/frontend/src/views/admin/ChannelMonitorView.vue +++ b/frontend/src/views/admin/ChannelMonitorView.vue @@ -78,6 +78,10 @@ {{ providerLabel(row.provider) }} + + + {{ checkModeLabel(row.check_mode) }} +