diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index 21a4fea79..6c4a028c9 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -32,7 +32,7 @@ jobs: cache-dependency-path: backend/go.sum - name: Verify Go version run: | - go version | grep -q 'go1.26.5' + go version | grep -q 'go1.26.6' - name: Unit tests working-directory: backend run: make test-unit @@ -72,7 +72,7 @@ jobs: cache-dependency-path: backend/go.sum - name: Verify Go version run: | - go version | grep -q 'go1.26.5' + go version | grep -q 'go1.26.6' - name: golangci-lint uses: golangci/golangci-lint-action@v9 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2ba01833c..4ecc8a26b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -115,7 +115,7 @@ jobs: - name: Verify Go version run: | - go version | grep -q 'go1.26.5' + go version | grep -q 'go1.26.6' # Docker setup for GoReleaser - name: Set up QEMU diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 96a7ae4ed..0e68d950a 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -23,7 +23,7 @@ jobs: cache-dependency-path: backend/go.sum - name: Verify Go version run: | - go version | grep -q 'go1.26.5' + go version | grep -q 'go1.26.6' - name: Run govulncheck working-directory: backend run: | diff --git a/CODEX_OVERDRAFT_DEPLOYMENT_CN.md b/CODEX_OVERDRAFT_DEPLOYMENT_CN.md index d3b682534..7b22e9592 100644 --- a/CODEX_OVERDRAFT_DEPLOYMENT_CN.md +++ b/CODEX_OVERDRAFT_DEPLOYMENT_CN.md @@ -387,13 +387,13 @@ docker compose \ 如果服务器目录是 `/opt/sub2api`，只需替换第一条路径。升级完成后重新执行“判断功能是否生效”中的镜像、环境变量和日志检查。 -更新后可在管理后台点击刷新，版本应显示为类似 `v0.1.176-overdraft.1`，更新方式应提示源码构建使用 `git pull`。也可以直接检查容器内二进制版本： +更新后可在管理后台点击刷新，版本应显示为类似 `v0.1.177-overdraft.1`，更新方式应提示源码构建使用 `git pull`。也可以直接检查容器内二进制版本： ```bash docker exec sub2api /app/sub2api -version ``` -维护者发布新的源码版本时必须递增 `FORK_VERSION`，例如从 `0.1.176-overdraft.1` 改为 `0.1.176-overdraft.2`。同步到新的上游 Sub2API 版本时，使用 `0.1.177-overdraft.1` 这样的版本号。 +维护者发布新的源码版本时必须递增 `FORK_VERSION`，例如从 `0.1.177-overdraft.1` 改为 `0.1.177-overdraft.2`。同步到新的上游 Sub2API 版本时，使用 `0.1.178-overdraft.1` 这样的版本号。 ## 合并 Sub2API 官方更新 diff --git a/CODEX_QUOTA_OVERDRAFT_CUSTOMIZATION.md b/CODEX_QUOTA_OVERDRAFT_CUSTOMIZATION.md index ae1cd6c60..6ff5d812d 100644 --- a/CODEX_QUOTA_OVERDRAFT_CUSTOMIZATION.md +++ b/CODEX_QUOTA_OVERDRAFT_CUSTOMIZATION.md @@ -5,7 +5,7 @@ ## 基本信息 -- 实施基线：`fbfdcef81` +- 当前实施基线：Sub2API `v0.1.177`，提交 `baeac1f3d` - 参考实现： - 功能开关：`gateway.codex_quota_overdraft_enabled` - 代码默认值：关闭；`deploy/config.example.yaml` 部署示例默认开启 @@ -16,7 +16,9 @@ 源码 Docker 构建会读取根目录的 `FORK_VERSION`，并以 `BuildType=source` 写入二进制。后台更新服务通过 GitHub Contents API 读取 Fork 分支上的同名文件，使用语义化版本比较判断是否有新版本。Redis 缓存同时记录仓库和构建类型，因此旧的官方更新缓存不会继续生效。 -源码构建仅显示 `git pull` 更新提示，`PerformUpdate`、指定版本回退和在线回退列表均被禁用，防止官方二进制覆盖透支功能。维护者每次发布 Fork 更新都必须递增 `FORK_VERSION`；普通补丁使用 `0.1.176-overdraft.2`，同步上游版本后使用 `0.1.177-overdraft.1`。 +源码构建仅显示 `git pull` 更新提示，`PerformUpdate`、指定版本回退和在线回退列表均被禁用，防止官方二进制覆盖透支功能。维护者每次发布 Fork 更新都必须递增 `FORK_VERSION`；普通补丁使用 `0.1.177-overdraft.2`，同步下一上游版本后使用 `0.1.178-overdraft.1`。 + +Sub2API `v0.1.177` 将原生 remote compaction v2 保留在 `/responses` 路径。本定制同时检查旧 Compact 路径和原生 v2 请求信号，二者都不会开启额度透支调度或注入透支请求形态。 ## 后台测试连接与残留限流修复 diff --git a/FORK_VERSION b/FORK_VERSION index 7770a2f75..fb7f732ab 100644 --- a/FORK_VERSION +++ b/FORK_VERSION @@ -1 +1 @@ -0.1.176-overdraft.1 +0.1.177-overdraft.1 diff --git a/backend/cmd/server/VERSION b/backend/cmd/server/VERSION index e8be64cc0..dfb8b6438 100644 --- a/backend/cmd/server/VERSION +++ b/backend/cmd/server/VERSION @@ -1 +1 @@ -0.1.176 +0.1.177 diff --git a/backend/go.mod b/backend/go.mod index d6f1c10ed..d5188b3ca 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,6 +1,6 @@ module github.com/Wei-Shaw/sub2api -go 1.26.5 +go 1.26.6 require ( entgo.io/ent v0.14.5 diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go index 81b257b8a..12faf126a 100644 --- a/backend/internal/config/config.go +++ b/backend/internal/config/config.go @@ -1706,6 +1706,10 @@ func load(allowMissingJWTSecret bool) (*Config, error) { // 环境变量支持 viper.AutomaticEnv() viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + if tz, ok := os.LookupEnv("TZ"); ok && strings.TrimSpace(tz) != "" { + // AutomaticEnv 会先把 timezone 映射到 TIMEZONE；显式 Set 保证标准 TZ 变量优先。 + viper.Set("timezone", strings.TrimSpace(tz)) + } if err := viper.BindEnv("server.enable_server_timing", "ENABLE_SERVER_TIMING"); err != nil { return nil, fmt.Errorf("bind ENABLE_SERVER_TIMING: %w", err) } diff --git a/backend/internal/config/config_test.go b/backend/internal/config/config_test.go index 718913a04..d8cfe820b 100644 --- a/backend/internal/config/config_test.go +++ b/backend/internal/config/config_test.go @@ -23,6 +23,38 @@ func resetViperWithJWTSecret(t *testing.T) { t.Setenv("JWT_SECRET", strings.Repeat("x", 32)) } +func TestLoadTimezonePrecedence(t *testing.T) { + tests := []struct { + name string + fileTimezone string + timezoneEnv string + tzEnv string + want string + }{ + {name: "default", want: "Asia/Shanghai"}, + {name: "config_file", fileTimezone: "Europe/London", want: "Europe/London"}, + {name: "timezone_env", fileTimezone: "Europe/London", timezoneEnv: "UTC", want: "UTC"}, + {name: "tz_env", fileTimezone: "Europe/London", timezoneEnv: "UTC", tzEnv: "America/New_York", want: "America/New_York"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetViperWithJWTSecret(t) + t.Setenv("TIMEZONE", tt.timezoneEnv) + t.Setenv("TZ", tt.tzEnv) + if tt.fileTimezone != "" { + configFile := filepath.Join(t.TempDir(), "config.yaml") + require.NoError(t, os.WriteFile(configFile, []byte("timezone: "+tt.fileTimezone+"\n"), 0o600)) + t.Setenv("CONFIG_FILE", configFile) + } + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, tt.want, cfg.Timezone) + }) + } +} + func TestLoadServerTimingConfig(t *testing.T) { t.Run("disabled by default", func(t *testing.T) { resetViperWithJWTSecret(t) diff --git a/backend/internal/handler/admin/group_handler.go b/backend/internal/handler/admin/group_handler.go index b8b9587f7..dfbd4eba6 100644 --- a/backend/internal/handler/admin/group_handler.go +++ b/backend/internal/handler/admin/group_handler.go @@ -8,11 +8,11 @@ import ( "log/slog" "strconv" "strings" + "time" "github.com/Wei-Shaw/sub2api/internal/handler/dto" infraerrors "github.com/Wei-Shaw/sub2api/internal/pkg/errors" "github.com/Wei-Shaw/sub2api/internal/pkg/response" - "github.com/Wei-Shaw/sub2api/internal/pkg/timezone" "github.com/Wei-Shaw/sub2api/internal/platform/liveattestation" "github.com/Wei-Shaw/sub2api/internal/service" @@ -734,12 +734,10 @@ func (h *GroupHandler) GetStats(c *gin.Context) { _ = groupID // TODO: implement actual stats } -// GetUsageSummary returns today's and cumulative cost for all groups. -// GET /api/v1/admin/groups/usage-summary?timezone=Asia/Shanghai +// GetUsageSummary returns today's, yesterday's, and cumulative cost for all groups. +// GET /api/v1/admin/groups/usage-summary func (h *GroupHandler) GetUsageSummary(c *gin.Context) { - userTZ := c.Query("timezone") - now := timezone.NowInUserLocation(userTZ) - todayStart := timezone.StartOfDayInUserLocation(now, userTZ) + todayStart := service.GroupUsageTodayStart(time.Now()) results, err := h.dashboardService.GetGroupUsageSummary(c.Request.Context(), todayStart) if err != nil { diff --git a/backend/internal/handler/openai_gateway_compact_body_signal_test.go b/backend/internal/handler/openai_gateway_compact_body_signal_test.go index a44d47c85..ca1f94a35 100644 --- a/backend/internal/handler/openai_gateway_compact_body_signal_test.go +++ b/backend/internal/handler/openai_gateway_compact_body_signal_test.go @@ -25,54 +25,236 @@ func newCompactBodySignalTestContext(t *testing.T, path string, body []byte) *gi func TestNormalizeOpenAIResponsesCompactRequest_RemoteV2StaysOnResponses(t *testing.T) { h := &OpenAIGatewayHandler{} - body := []byte(`{ - "model":"gpt-5.6-sol", - "stream":true, - "store":true, - "prompt_cache_key":"pck-signal-1", - "reasoning":{"effort":"max","context":"all_turns"}, - "input":[ - {"type":"message","role":"user","content":"hello"}, - {"type":"compaction_trigger"} - ] - }`) - c := newCompactBodySignalTestContext(t, "/v1/responses", body) - c.Request.Header.Set("x-codex-beta-features", "responses_websockets_v2, remote_compaction_v2, another_feature") + tests := []struct { + name string + betaHeader string + userAgent string + }{ + {name: "headerless"}, + {name: "unrelated_header", betaHeader: "responses_websockets_v2"}, + {name: "wrong_case_header", betaHeader: "REMOTE_COMPACTION_V2"}, + {name: "declared_header", betaHeader: "remote_compaction_v2"}, + {name: "codex_cli_user_agent", userAgent: "codex_cli_rs/0.144.1 (Ubuntu 22.4.0; x86_64) xterm-256color"}, + {name: "codex_desktop_user_agent", userAgent: "Codex Desktop/0.139.0 (Mac OS X 14; arm64) unknown"}, + } - normalized, ok := h.normalizeOpenAIResponsesCompactRequest(c, zap.NewNop(), body) - require.True(t, ok) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := []byte(`{ + "model":"gpt-5.6-sol", + "stream":true, + "store":true, + "prompt_cache_key":"pck-signal-1", + "reasoning":{"effort":"max","context":"all_turns"}, + "input":[ + {"type":"message","role":"user","content":"hello"}, + {"type":"compaction_trigger"} + ] + }`) + c := newCompactBodySignalTestContext(t, "/v1/responses", body) + if tt.betaHeader != "" { + c.Request.Header.Set("x-codex-beta-features", tt.betaHeader) + } + if tt.userAgent != "" { + c.Request.Header.Set("User-Agent", tt.userAgent) + } - require.Equal(t, "/v1/responses", c.Request.URL.Path) - require.False(t, isOpenAIRemoteCompactPath(c)) - require.Equal(t, body, normalized) - require.True(t, gjson.GetBytes(normalized, "stream").Bool()) - require.True(t, gjson.GetBytes(normalized, "store").Bool()) - require.Equal(t, "pck-signal-1", gjson.GetBytes(normalized, "prompt_cache_key").String()) - require.Equal(t, "max", gjson.GetBytes(normalized, "reasoning.effort").String()) - require.Equal(t, "all_turns", gjson.GetBytes(normalized, "reasoning.context").String()) + normalized, ok := h.normalizeOpenAIResponsesCompactRequest(c, zap.NewNop(), body) + require.True(t, ok) - reqStream, streamOK := parseOpenAICompatibleStream(normalized) - require.True(t, streamOK) - require.True(t, reqStream) + require.Equal(t, "/v1/responses", c.Request.URL.Path) + require.False(t, isOpenAILegacyCompactPath(c)) + require.Equal(t, body, normalized) + require.True(t, gjson.GetBytes(normalized, "stream").Bool()) + require.True(t, gjson.GetBytes(normalized, "store").Bool()) + require.Equal(t, "pck-signal-1", gjson.GetBytes(normalized, "prompt_cache_key").String()) + require.Equal(t, "max", gjson.GetBytes(normalized, "reasoning.effort").String()) + require.Equal(t, "all_turns", gjson.GetBytes(normalized, "reasoning.context").String()) + legacyCompact := service.IsOpenAIResponsesCompactPath(c) + nativeV2 := isBareOpenAIResponsesPath(c) && isOpenAIRemoteCompactionV2Request(normalized) + require.False(t, legacyCompact) + require.True(t, nativeV2) + require.Equal(t, service.OpenAIEndpointCapabilityResponses, + openAIResponsesRequiredCapabilityForRequest(false, nativeV2 || legacyCompact, service.PlatformOpenAI)) - _, seedExists := c.Get(service.OpenAICompactSessionSeedKeyForTest()) - require.False(t, seedExists) - _, streamMarkerExists := c.Get(service.OpenAICompactClientStreamKeyForTest()) - require.False(t, streamMarkerExists) + reqStream, streamOK := parseOpenAICompatibleStream(normalized) + require.True(t, streamOK) + require.True(t, reqStream) + + _, seedExists := c.Get(service.OpenAICompactSessionSeedKeyForTest()) + require.False(t, seedExists) + _, streamMarkerExists := c.Get(service.OpenAICompactClientStreamKeyForTest()) + require.False(t, streamMarkerExists) + }) + } } func TestNormalizeOpenAIResponsesCompactRequest_RemoteV2PathAliasesStayOnResponses(t *testing.T) { h := &OpenAIGatewayHandler{} body := []byte(`{"model":"gpt-5.6-sol","stream":true,"input":[{"type":"compaction_trigger"}]}`) - for _, path := range []string{"/v1/responses/", "/backend-api/codex/responses"} { + for _, path := range []string{ + "/v1/responses/", + "/openai/v1/responses", + "/responses", + "/backend-api/codex/responses", + } { t.Run(path, func(t *testing.T) { c := newCompactBodySignalTestContext(t, path, body) - c.Request.Header.Set("x-codex-beta-features", "remote_compaction_v2") normalized, ok := h.normalizeOpenAIResponsesCompactRequest(c, zap.NewNop(), body) require.True(t, ok) require.Equal(t, path, c.Request.URL.Path) require.Equal(t, body, normalized) + legacyCompact := service.IsOpenAIResponsesCompactPath(c) + nativeV2 := isBareOpenAIResponsesPath(c) && isOpenAIRemoteCompactionV2Request(normalized) + require.False(t, legacyCompact) + require.True(t, nativeV2) + require.Equal(t, service.OpenAIEndpointCapabilityResponses, + openAIResponsesRequiredCapabilityForRequest(false, nativeV2 || legacyCompact, service.PlatformOpenAI)) + }) + } +} + +func TestShouldEnableCodexQuotaOverdraftForResponsesExcludesCompactAndImage(t *testing.T) { + tests := []struct { + name string + legacyCompact bool + nativeV2 bool + imageIntent bool + want bool + }{ + {name: "ordinary_responses", want: true}, + {name: "legacy_compact", legacyCompact: true}, + {name: "native_compact_v2", nativeV2: true}, + {name: "image_generation", imageIntent: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, shouldEnableCodexQuotaOverdraftForResponses( + tt.legacyCompact, + tt.nativeV2, + tt.imageIntent, + )) + }) + } +} + +func TestOpenAIResponsesCompactionRoutingFlags(t *testing.T) { + h := &OpenAIGatewayHandler{} + tests := []struct { + name string + body []byte + path string + wantLegacyBefore bool + wantNativeBefore bool + wantLegacyAfter bool + wantNativeAfter bool + wantCapabilityAfter service.OpenAIEndpointCapability + wantPathAfter string + wantBodyUnchanged bool + }{ + { + name: "native_v2_stream_trigger", + body: []byte(`{"model":"gpt-5.6-sol","stream":true,"input":[{"type":"compaction_trigger"}]}`), + path: "/v1/responses", + wantLegacyBefore: false, + wantNativeBefore: true, + wantLegacyAfter: false, + wantNativeAfter: true, + wantCapabilityAfter: service.OpenAIEndpointCapabilityResponses, + wantPathAfter: "/v1/responses", + wantBodyUnchanged: true, + }, + { + name: "native_v2_without_trigger", + body: []byte(`{"model":"gpt-5.6-sol","stream":true,"input":[{"type":"message","role":"user","content":"hello"}]}`), + path: "/v1/responses", + wantLegacyBefore: false, + wantNativeBefore: false, + wantLegacyAfter: false, + wantNativeAfter: false, + wantCapabilityAfter: service.OpenAIEndpointCapabilityChatCompletions, + wantPathAfter: "/v1/responses", + wantBodyUnchanged: true, + }, + { + name: "explicit_compact", + body: []byte(`{"model":"gpt-5.6-sol","stream":true,"input":[{"type":"compaction_trigger"}]}`), + path: "/v1/responses/compact", + wantLegacyBefore: true, + wantNativeBefore: false, + wantLegacyAfter: true, + wantNativeAfter: false, + wantCapabilityAfter: service.OpenAIEndpointCapabilityResponses, + wantPathAfter: "/v1/responses/compact", + }, + { + name: "nested_compact", + body: []byte(`{"model":"gpt-5.6-sol","stream":true,"input":[{"type":"compaction_trigger"}]}`), + path: "/v1/responses/compact/detail", + wantLegacyBefore: true, + wantNativeBefore: false, + wantLegacyAfter: true, + wantNativeAfter: false, + wantCapabilityAfter: service.OpenAIEndpointCapabilityResponses, + wantPathAfter: "/v1/responses/compact/detail", + }, + { + name: "responses_subpath_with_native_signal", + body: []byte(`{"model":"gpt-5.6-sol","stream":true,"input":[{"type":"compaction_trigger"}]}`), + path: "/v1/responses/resp_123/responses", + wantLegacyBefore: false, + wantNativeBefore: false, + wantLegacyAfter: false, + wantNativeAfter: false, + wantCapabilityAfter: service.OpenAIEndpointCapabilityChatCompletions, + wantPathAfter: "/v1/responses/resp_123/responses", + wantBodyUnchanged: true, + }, + { + name: "stream_false_promotes", + body: []byte(`{"model":"gpt-5.6-sol","stream":false,"input":[{"type":"compaction_trigger"}]}`), + path: "/v1/responses", + wantLegacyBefore: false, + wantNativeBefore: false, + wantLegacyAfter: true, + wantNativeAfter: false, + wantCapabilityAfter: service.OpenAIEndpointCapabilityResponses, + wantPathAfter: "/v1/responses/compact", + }, + { + name: "stream_absent_promotes", + body: []byte(`{"model":"gpt-5.6-sol","input":[{"type":"compaction_trigger"}]}`), + path: "/v1/responses", + wantLegacyBefore: false, + wantNativeBefore: false, + wantLegacyAfter: true, + wantNativeAfter: false, + wantCapabilityAfter: service.OpenAIEndpointCapabilityResponses, + wantPathAfter: "/v1/responses/compact", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newCompactBodySignalTestContext(t, tt.path, tt.body) + legacyBefore := service.IsOpenAIResponsesCompactPath(c) + nativeBefore := isBareOpenAIResponsesPath(c) && isOpenAIRemoteCompactionV2Request(tt.body) + require.Equal(t, tt.wantLegacyBefore, legacyBefore) + require.Equal(t, tt.wantNativeBefore, nativeBefore) + normalized, ok := h.normalizeOpenAIResponsesCompactRequest(c, zap.NewNop(), tt.body) + require.True(t, ok) + require.Equal(t, tt.wantPathAfter, c.Request.URL.Path) + legacyAfter := service.IsOpenAIResponsesCompactPath(c) + nativeAfter := isBareOpenAIResponsesPath(c) && isOpenAIRemoteCompactionV2Request(normalized) + require.Equal(t, tt.wantLegacyAfter, legacyAfter) + require.Equal(t, tt.wantNativeAfter, nativeAfter) + require.Equal(t, tt.wantCapabilityAfter, + openAIResponsesRequiredCapabilityForRequest(false, nativeAfter || legacyAfter, service.PlatformOpenAI)) + if tt.wantBodyUnchanged { + require.Equal(t, tt.body, normalized) + } }) } } @@ -106,31 +288,22 @@ func TestNormalizeOpenAIResponsesCompactRequest_NonRemoteV2BodySignalPromoted(t wantMarked bool }{ { - name: "no_header", - body: []byte(`{"model":"gpt-5.5","stream":true,"input":[{"type":"compaction_trigger"}]}`), - wantMarked: true, + name: "stream_false_headerless", + body: []byte(`{"model":"gpt-5.5","stream":false,"input":[{"type":"compaction_trigger"}]}`), }, { - name: "unrelated_header", - body: []byte(`{"model":"gpt-5.5","stream":true,"input":[{"type":"compaction_trigger"}]}`), - betaHeader: "responses_websockets_v2", - wantMarked: true, + name: "stream_absent_headerless", + body: []byte(`{"model":"gpt-5.5","input":[{"type":"compaction_trigger"}]}`), }, { - name: "wrong_case_header", - body: []byte(`{"model":"gpt-5.5","stream":true,"input":[{"type":"compaction_trigger"}]}`), - betaHeader: "REMOTE_COMPACTION_V2", - wantMarked: true, - }, - { - name: "stream_false", + name: "stream_false_declared_header", body: []byte(`{"model":"gpt-5.5","stream":false,"input":[{"type":"compaction_trigger"}]}`), betaHeader: "remote_compaction_v2", }, { - name: "stream_absent", + name: "stream_absent_wrong_case_header", body: []byte(`{"model":"gpt-5.5","input":[{"type":"compaction_trigger"}]}`), - betaHeader: "remote_compaction_v2", + betaHeader: "REMOTE_COMPACTION_V2", }, } @@ -163,7 +336,7 @@ func TestNormalizeOpenAIResponsesCompactRequest_NoTriggerUntouched(t *testing.T) normalized, ok := h.normalizeOpenAIResponsesCompactRequest(c, zap.NewNop(), body) require.True(t, ok) require.Equal(t, "/v1/responses", c.Request.URL.Path) - require.False(t, isOpenAIRemoteCompactPath(c)) + require.False(t, isOpenAILegacyCompactPath(c)) require.Equal(t, body, normalized) require.True(t, gjson.GetBytes(normalized, "stream").Bool()) } @@ -172,7 +345,6 @@ func TestNormalizeOpenAIResponsesCompactRequest_PathBasedNoDoubleSuffix(t *testi h := &OpenAIGatewayHandler{} body := []byte(`{"model":"gpt-5.5","stream":true,"store":true,"input":[{"type":"message","role":"user","content":"hello"}]}`) c := newCompactBodySignalTestContext(t, "/v1/responses/compact", body) - c.Request.Header.Set("x-codex-beta-features", "remote_compaction_v2") normalized, ok := h.normalizeOpenAIResponsesCompactRequest(c, zap.NewNop(), body) require.True(t, ok) diff --git a/backend/internal/handler/openai_gateway_compact_log_test.go b/backend/internal/handler/openai_gateway_compact_log_test.go index 16afd30c3..0d1140366 100644 --- a/backend/internal/handler/openai_gateway_compact_log_test.go +++ b/backend/internal/handler/openai_gateway_compact_log_test.go @@ -105,21 +105,29 @@ func captureHandlerStructuredLog(t *testing.T) (*handlerInMemoryLogSink, func()) } } -func TestIsOpenAIRemoteCompactPath(t *testing.T) { - require.False(t, isOpenAIRemoteCompactPath(nil)) +func TestIsOpenAILegacyCompactPath(t *testing.T) { + require.False(t, isOpenAILegacyCompactPath(nil)) gin.SetMode(gin.TestMode) rec := httptest.NewRecorder() c, _ := gin.CreateTestContext(rec) - c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses/compact", nil) - require.True(t, isOpenAIRemoteCompactPath(c)) - - c.Request = httptest.NewRequest(http.MethodPost, "/responses/compact/", nil) - require.True(t, isOpenAIRemoteCompactPath(c)) - - c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) - require.False(t, isOpenAIRemoteCompactPath(c)) + for _, test := range []struct { + path string + want bool + }{ + {path: "/v1/responses/compact", want: true}, + {path: "/v1/responses/compact/detail", want: true}, + {path: "/responses/compact/", want: true}, + {path: "/v1/responses", want: false}, + {path: "/openai/v1/responses", want: false}, + {path: "/responses", want: false}, + {path: "/backend-api/codex/responses", want: false}, + {path: "/v1/responses/resp_123/cancel", want: false}, + } { + c.Request = httptest.NewRequest(http.MethodPost, test.path, nil) + require.Equal(t, test.want, isOpenAILegacyCompactPath(c), test.path) + } } func TestLogOpenAIRemoteCompactOutcome_Succeeded(t *testing.T) { diff --git a/backend/internal/handler/openai_gateway_handler.go b/backend/internal/handler/openai_gateway_handler.go index b71abfe10..84f3711a8 100644 --- a/backend/internal/handler/openai_gateway_handler.go +++ b/backend/internal/handler/openai_gateway_handler.go @@ -182,6 +182,20 @@ func openAIResponsesRequiredCapability(imageIntent bool, platform string) servic return service.OpenAIEndpointCapabilityChatCompletions } +// openAIResponsesRequiredCapabilityForRequest returns the endpoint capability +// required by an image or Responses request. needsResponses includes both the +// legacy /responses/compact endpoint and native remote compaction v2. +func openAIResponsesRequiredCapabilityForRequest(imageIntent bool, needsResponses bool, platform string) service.OpenAIEndpointCapability { + if needsResponses && platform == service.PlatformOpenAI { + return service.OpenAIEndpointCapabilityResponses + } + return openAIResponsesRequiredCapability(imageIntent, platform) +} + +func shouldEnableCodexQuotaOverdraftForResponses(legacyCompact, nativeV2, imageIntent bool) bool { + return !legacyCompact && !nativeV2 && !imageIntent +} + func allowOpenAICompatibleMessagesDispatch(apiKey *service.APIKey) bool { if apiKey == nil || apiKey.Group == nil { return true @@ -288,6 +302,13 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { if !ok { return } + legacyCompact := service.IsOpenAIResponsesCompactPath(c) + nativeV2 := isBareOpenAIResponsesPath(c) && isOpenAIRemoteCompactionV2Request(body) + if nativeV2 { + // 原生 v2 压缩出站前补注 x-codex-beta-features: remote_compaction_v2， + // 与真实 Codex 线型一致（网关链剥头后本级负责恢复，#5586）。 + service.MarkOpenAINativeCompactionV2(c) + } // body-signal compact：上游 unary 等待期间向下游发 SSE 注释行心跳，防止 // 反向代理空闲超时掐断长压缩连接（#3887）。首拍延迟一个心跳间隔，快速 // 失败仍走 JSON+状态码链路；未标记客户端流式或间隔为 0 时是 no-op。 @@ -377,6 +398,15 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { channelMapping, _ := h.gatewayService.ResolveChannelMappingAndRestrict(c.Request.Context(), apiKey.GroupID, reqModel) forwardBody := openAIModelMappedBody(body, channelMapping.Mapped, channelMapping.MappedModel, h.gatewayService.ReplaceModelInBody) seedOpenAIForwardImageIntentHint(c, channelMapping.Mapped, imageIntent) + forwardModel := reqModel + if channelMapping.Mapped { + forwardModel = channelMapping.MappedModel + } + c.Request = c.Request.WithContext(service.WithOpenAIForwardModel( + c.Request.Context(), + forwardModel, + legacyCompact, + )) // 提前校验 function_call_output 是否具备可关联上下文，避免上游 400。 if !h.validateFunctionCallOutputRequest(c, body, reqLog) { @@ -420,7 +450,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { if h.rejectIfCyberSessionBlocked(c, apiKey, sessionHashBody, reqModel, cyberBlockFormatResponses) { return } - requireCompact := isOpenAIRemoteCompactPath(c) + requireCompact := legacyCompact maxAccountSwitches := h.maxAccountSwitches switchCount := 0 @@ -437,14 +467,15 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { // 仅对 OpenAI 平台生效：Grok 生图走独立的 forwardGrokResponses 路径，不应被过滤。 // 复用前置权限与并发阶段在未修改 body 上确认的显式生图意图，避免大 tools 请求重复扫描。 // 该判断已排除 Codex 被动 image_gen namespace，避免 CC-only 账号被误过滤（#4476）。 - requiredCapability := openAIResponsesRequiredCapability(imageIntent, requestPlatform) + needsResponses := nativeV2 || legacyCompact + requiredCapability := openAIResponsesRequiredCapabilityForRequest(imageIntent, needsResponses, requestPlatform) // 分组利润控制：请求级装配定价上下文——pricingAt 固定本请求的 // D 与计费高峰因子，选号、槽位终检与全部 failover 重入共用同一门与阈值。 // 生图意图只影响能力路由与图片计费，不关门：混合 /v1/responses 请求的 // token 计费部分仍受利润门保护，独立图片/视频端点才在门外。 requestCtx := c.Request.Context() - if !requireCompact && !imageIntent { + if shouldEnableCodexQuotaOverdraftForResponses(legacyCompact, nativeV2, imageIntent) { requestCtx = service.WithCodexQuotaOverdraftScheduling(requestCtx) } pricingCtx, pricingAt := h.gatewayService.WithOpenAIRequestPricingContext(requestCtx, apiKey.GroupID) @@ -483,7 +514,7 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { zap.Int("excluded_account_count", len(failedAccountIDs)), ) if len(failedAccountIDs) == 0 { - if errors.Is(err, service.ErrNoAvailableCompactAccounts) { + if legacyCompact && errors.Is(err, service.ErrNoAvailableCompactAccounts) { markOpsRoutingCapacityLimitedIfNoAvailable(c, err) h.handleStreamingAwareError(c, http.StatusServiceUnavailable, "compact_not_supported", "No available accounts support /responses/compact", streamStarted) return @@ -739,12 +770,8 @@ func (h *OpenAIGatewayHandler) Responses(c *gin.Context) { } } -func isOpenAIRemoteCompactPath(c *gin.Context) bool { - if c == nil || c.Request == nil || c.Request.URL == nil { - return false - } - normalizedPath := strings.TrimRight(strings.TrimSpace(c.Request.URL.Path), "/") - return strings.HasSuffix(normalizedPath, "/responses/compact") +func isOpenAILegacyCompactPath(c *gin.Context) bool { + return service.IsOpenAIResponsesCompactPath(c) } // isBareOpenAIResponsesPath 仅匹配裸 /responses 端点（无 /compact 等子路径）， @@ -754,32 +781,27 @@ func isBareOpenAIResponsesPath(c *gin.Context) bool { return false } normalizedPath := strings.TrimRight(strings.TrimSpace(c.Request.URL.Path), "/") - return strings.HasSuffix(normalizedPath, "/responses") -} - -func isOpenAIRemoteCompactionV2Request(c *gin.Context, body []byte) bool { - stream, valid := parseOpenAICompatibleStream(body) - if !valid || !stream || c == nil || c.Request == nil { + switch normalizedPath { + case EndpointResponses, "/openai/v1/responses", "/responses", "/backend-api/codex/responses": + return true + default: return false } - for _, header := range c.Request.Header.Values("x-codex-beta-features") { - for _, feature := range strings.Split(header, ",") { - if strings.TrimSpace(feature) == "remote_compaction_v2" { - return true - } - } - } - return false +} + +func isOpenAIRemoteCompactionV2Request(body []byte) bool { + stream, valid := parseOpenAICompatibleStream(body) + return valid && stream && service.HasCompactionTriggerInInput(body) } // normalizeOpenAIResponsesCompactRequest keeps Codex remote compaction v2 on // its native streaming /responses wire and preserves the legacy body-signal -// promotion for clients that do not explicitly advertise that protocol. +// promotion for non-streaming requests. // 返回归一化后的 body；ok=false 表示错误响应已写出，调用方应直接 return。 func (h *OpenAIGatewayHandler) normalizeOpenAIResponsesCompactRequest(c *gin.Context, reqLog *zap.Logger, body []byte) ([]byte, bool) { - isCompactRequest := service.IsOpenAIResponsesCompactPathForTest(c) + isCompactRequest := isOpenAILegacyCompactPath(c) if !isCompactRequest && isBareOpenAIResponsesPath(c) && service.HasCompactionTriggerInInput(body) { - if isOpenAIRemoteCompactionV2Request(c, body) { + if isOpenAIRemoteCompactionV2Request(body) { return body, true } c.Request.URL.Path = strings.TrimRight(c.Request.URL.Path, "/") + "/compact" @@ -808,7 +830,7 @@ func (h *OpenAIGatewayHandler) normalizeOpenAIResponsesCompactRequest(c *gin.Con } func (h *OpenAIGatewayHandler) logOpenAIRemoteCompactOutcome(c *gin.Context, startedAt time.Time) { - if !isOpenAIRemoteCompactPath(c) { + if !isOpenAILegacyCompactPath(c) { return } diff --git a/backend/internal/handler/openai_profit_slot_recheck_test.go b/backend/internal/handler/openai_profit_slot_recheck_test.go index b40fbc4e2..31e251c08 100644 --- a/backend/internal/handler/openai_profit_slot_recheck_test.go +++ b/backend/internal/handler/openai_profit_slot_recheck_test.go @@ -5,8 +5,7 @@ package handler // 槽位终检与生图跳门回归（handler 半程）： // - 槽位获取成功后的利润终检：越线账号释放槽位并要求调用方排除重选， // 不写响应、不绑定粘连； -// - openAIResponsesRequiredCapability 的生图意图映射钉死（scheduler 的 -// 跳门条件依赖 CapabilityResponses ⇔ 显式生图意图这一耦合）。 +// - openAIResponsesRequiredCapability 的请求能力映射覆盖生图与原生远程压缩。 import ( "context" @@ -141,10 +140,10 @@ func TestAcquireResponsesAccountSlotProfitRecheck(t *testing.T) { }) } -// scheduler 跳门条件依赖"CapabilityResponses 仅在显式生图意图时被要求"这一 -// 映射；后续若扩展该 capability 的用途，本测试失败提示同步收窄跳门条件。 -func TestOpenAIResponsesRequiredCapabilityPinsImageIntentMapping(t *testing.T) { +func TestOpenAIResponsesRequiredCapabilityForRequest(t *testing.T) { require.Equal(t, service.OpenAIEndpointCapabilityResponses, openAIResponsesRequiredCapability(true, service.PlatformOpenAI)) require.Equal(t, service.OpenAIEndpointCapabilityChatCompletions, openAIResponsesRequiredCapability(false, service.PlatformOpenAI)) require.Equal(t, service.OpenAIEndpointCapabilityChatCompletions, openAIResponsesRequiredCapability(true, service.PlatformGrok)) + require.Equal(t, service.OpenAIEndpointCapabilityResponses, openAIResponsesRequiredCapabilityForRequest(false, true, service.PlatformOpenAI)) + require.Equal(t, service.OpenAIEndpointCapabilityChatCompletions, openAIResponsesRequiredCapabilityForRequest(false, true, service.PlatformGrok)) } diff --git a/backend/internal/pkg/usagestats/usage_log_types.go b/backend/internal/pkg/usagestats/usage_log_types.go index a100b2cca..ad97026e2 100644 --- a/backend/internal/pkg/usagestats/usage_log_types.go +++ b/backend/internal/pkg/usagestats/usage_log_types.go @@ -115,11 +115,12 @@ type EndpointStat struct { ActualCost float64 `json:"actual_cost"` // 实际扣除 } -// GroupUsageSummary represents today's and cumulative cost for a single group. +// GroupUsageSummary represents today's, yesterday's, and cumulative cost for a single group. type GroupUsageSummary struct { - GroupID int64 `json:"group_id"` - TodayCost float64 `json:"today_cost"` - TotalCost float64 `json:"total_cost"` + GroupID int64 `json:"group_id"` + TodayCost float64 `json:"today_cost"` + YesterdayCost float64 `json:"yesterday_cost"` + TotalCost float64 `json:"total_cost"` } // GroupStat represents usage statistics for a single group diff --git a/backend/internal/repository/custom_group_usage_rollup_repo.go b/backend/internal/repository/custom_group_usage_rollup_repo.go new file mode 100644 index 000000000..ac30edfc1 --- /dev/null +++ b/backend/internal/repository/custom_group_usage_rollup_repo.go @@ -0,0 +1,259 @@ +package repository + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/Wei-Shaw/sub2api/internal/pkg/usagestats" + "github.com/Wei-Shaw/sub2api/internal/service" +) + +func (r *usageLogRepository) getAllGroupUsageSummaryFromRollups(ctx context.Context, todayStart time.Time) (results []usagestats.GroupUsageSummary, err error) { + todayStart = service.GroupUsageTodayStart(todayStart) + yesterdayStart := service.GroupUsageYesterdayStart(todayStart) + timezoneName := service.GroupUsageTimezoneName() + todayDate := service.GroupUsageDate(todayStart) + yesterdayDate := service.GroupUsageDate(yesterdayStart) + + const query = ` + WITH state_values AS ( + SELECT + COUNT(*) = 1 + AND MAX(timezone_name) = $3 + AND MAX(closed_before) <= $4::date AS valid, + MAX(closed_before) AS closed_before, + MAX(retained_from) AS retained_from + FROM usage_group_rollup_state + WHERE id = 1 + ), + state AS ( + SELECT + CASE WHEN valid THEN closed_before ELSE DATE '1970-01-01' END AS closed_before, + CASE WHEN valid THEN retained_from ELSE TIMESTAMPTZ '1970-01-01 00:00:00+00' END AS retained_from, + CASE + WHEN valid THEN closed_before::timestamp AT TIME ZONE $3::text + ELSE TIMESTAMPTZ '1970-01-01 00:00:00+00' + END AS tail_start, + valid + FROM state_values + ), + historical AS ( + SELECT + rollup.group_id, + COALESCE(SUM(rollup.actual_cost), 0) AS actual_cost, + COALESCE(SUM(rollup.actual_cost) FILTER ( + WHERE rollup.bucket_date = $5::date + ), 0) AS yesterday_cost + FROM usage_group_daily_rollups rollup + CROSS JOIN state + WHERE state.valid + AND rollup.bucket_date >= (state.retained_from AT TIME ZONE $3::text)::date + AND rollup.bucket_date < state.closed_before + GROUP BY rollup.group_id + ), + tail AS ( + SELECT + ul.group_id, + COALESCE(SUM(ul.actual_cost), 0) AS actual_cost, + COALESCE(SUM(ul.actual_cost) FILTER (WHERE ul.created_at >= $1), 0) AS today_cost, + COALESCE(SUM(ul.actual_cost) FILTER ( + WHERE ul.created_at >= $2 + AND ul.created_at < $1 + ), 0) AS yesterday_cost + FROM usage_logs ul + CROSS JOIN state + WHERE ul.created_at >= state.tail_start + GROUP BY ul.group_id + ) + SELECT + g.id AS group_id, + COALESCE(historical.actual_cost, 0) + COALESCE(tail.actual_cost, 0) AS total_cost, + COALESCE(tail.today_cost, 0) AS today_cost, + COALESCE(historical.yesterday_cost, 0) + COALESCE(tail.yesterday_cost, 0) AS yesterday_cost + FROM groups g + LEFT JOIN historical ON historical.group_id = g.id + LEFT JOIN tail ON tail.group_id = g.id + ORDER BY g.id + ` + + rows, err := r.sql.QueryContext( + ctx, + query, + todayStart, + yesterdayStart, + timezoneName, + todayDate, + yesterdayDate, + ) + if err != nil { + return nil, err + } + defer func() { + if closeErr := rows.Close(); closeErr != nil && err == nil { + err = closeErr + results = nil + } + }() + + results = make([]usagestats.GroupUsageSummary, 0) + for rows.Next() { + var row usagestats.GroupUsageSummary + if err := rows.Scan(&row.GroupID, &row.TotalCost, &row.TodayCost, &row.YesterdayCost); err != nil { + return nil, err + } + results = append(results, row) + } + if err := rows.Err(); err != nil { + return nil, err + } + return results, nil +} + +// SyncGroupUsageRollups 将服务端配置时区今日以前的用量发布为分组日桶。 +func (r *dashboardAggregationRepository) SyncGroupUsageRollups(ctx context.Context, todayStart time.Time) error { + if r == nil || r.sql == nil { + return nil + } + todayStart = service.GroupUsageTodayStart(todayStart) + if db, ok := r.sql.(*sql.DB); ok { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + txRepo := newDashboardAggregationRepositoryWithSQL(tx) + if err := txRepo.syncGroupUsageRollupsInTx(ctx, todayStart); err != nil { + _ = tx.Rollback() + return err + } + return tx.Commit() + } + return r.syncGroupUsageRollupsInTx(ctx, todayStart) +} + +func (r *dashboardAggregationRepository) syncGroupUsageRollupsInTx(ctx context.Context, todayStart time.Time) error { + var closedBefore string + var previousRetainedFrom time.Time + var stateTimezoneName string + if err := scanSingleRow(ctx, r.sql, ` + SELECT closed_before::text, retained_from, timezone_name + FROM usage_group_rollup_state + WHERE id = 1 + FOR UPDATE + `, nil, &closedBefore, &previousRetainedFrom, &stateTimezoneName); err != nil { + return fmt.Errorf("读取分组用量汇总水位: %w", err) + } + + todayDate := service.GroupUsageDate(todayStart) + timezoneName := service.GroupUsageTimezoneName() + timezoneChanged := stateTimezoneName != timezoneName + var closedTime time.Time + if !timezoneChanged { + var err error + closedTime, err = service.ParseGroupUsageDate(closedBefore) + if err != nil { + return fmt.Errorf("解析分组用量汇总水位 %q: %w", closedBefore, err) + } + todayDateTime, err := service.ParseGroupUsageDate(todayDate) + if err != nil { + return err + } + if closedTime.After(todayDateTime) { + return fmt.Errorf("分组用量汇总水位位于未来: %s", closedBefore) + } + if closedBefore == todayDate { + return nil + } + } + + var earliest sql.NullTime + if err := scanSingleRow(ctx, r.sql, "SELECT MIN(created_at) FROM usage_logs", nil, &earliest); err != nil { + return fmt.Errorf("读取最早用量记录: %w", err) + } + retainedFrom := todayStart + if earliest.Valid { + retainedFrom = earliest.Time.UTC() + } + retainedDate := service.GroupUsageDate(retainedFrom) + retainedDateTime, err := service.ParseGroupUsageDate(retainedDate) + if err != nil { + return err + } + rebuildStartDate := retainedDate + if !timezoneChanged && closedTime.After(retainedDateTime) { + rebuildStartDate = closedBefore + } + rebuildStart, err := service.ParseGroupUsageDate(rebuildStartDate) + if err != nil { + return err + } + + if _, err := r.sql.ExecContext(ctx, ` + DELETE FROM usage_group_daily_rollups + WHERE bucket_date < $1::date + OR (bucket_date >= $2::date AND bucket_date < $3::date) + OR bucket_date >= $3::date + `, retainedDate, rebuildStartDate, todayDate); err != nil { + return fmt.Errorf("清理分组用量日桶: %w", err) + } + + if _, err := r.sql.ExecContext(ctx, ` + INSERT INTO usage_group_daily_rollups (bucket_date, group_id, actual_cost, computed_at) + SELECT + (created_at AT TIME ZONE $3::text)::date AS bucket_date, + group_id, + COALESCE(SUM(actual_cost), 0) AS actual_cost, + NOW() + FROM usage_logs + WHERE group_id IS NOT NULL + AND created_at >= $1 + AND created_at < $2 + GROUP BY 1, 2 + ON CONFLICT (bucket_date, group_id) + DO UPDATE SET + actual_cost = EXCLUDED.actual_cost, + computed_at = EXCLUDED.computed_at + `, rebuildStart.UTC(), todayStart.UTC(), timezoneName); err != nil { + return fmt.Errorf("重建分组用量日桶: %w", err) + } + + if _, err := r.sql.ExecContext(ctx, ` + UPDATE usage_group_rollup_state + SET closed_before = $1::date, + retained_from = $2, + timezone_name = $3, + updated_at = NOW() + WHERE id = 1 + `, todayDate, retainedFrom, timezoneName); err != nil { + return fmt.Errorf("更新分组用量汇总水位: %w", err) + } + return nil +} + +func lockGroupUsageRollupState(ctx context.Context, tx *sql.Tx) error { + var id int16 + if err := tx.QueryRowContext(ctx, ` + SELECT id + FROM usage_group_rollup_state + WHERE id = 1 + FOR UPDATE + `).Scan(&id); err != nil { + return fmt.Errorf("锁定分组用量汇总水位: %w", err) + } + return nil +} + +func invalidateGroupUsageRollupsAt(ctx context.Context, tx *sql.Tx, affectedAt time.Time) error { + timezoneName := service.GroupUsageTimezoneName() + _, err := tx.ExecContext(ctx, ` + UPDATE usage_group_rollup_state + SET closed_before = LEAST( + closed_before, + ($1::timestamptz AT TIME ZONE $2::text)::date + ), + updated_at = NOW() + WHERE id = 1 + `, affectedAt.UTC(), timezoneName) + return err +} diff --git a/backend/internal/repository/custom_group_usage_timezone_test.go b/backend/internal/repository/custom_group_usage_timezone_test.go new file mode 100644 index 000000000..c77c5b41b --- /dev/null +++ b/backend/internal/repository/custom_group_usage_timezone_test.go @@ -0,0 +1,16 @@ +package repository + +import ( + "testing" + + appTimezone "github.com/Wei-Shaw/sub2api/internal/pkg/timezone" + "github.com/stretchr/testify/require" +) + +func useGroupUsageRepositoryTestTimezone(t *testing.T, name string) { + t.Helper() + + previousName := appTimezone.Name() + require.NoError(t, appTimezone.Init(name)) + t.Cleanup(func() { require.NoError(t, appTimezone.Init(previousName)) }) +} diff --git a/backend/internal/repository/dashboard_aggregation_group_usage_test.go b/backend/internal/repository/dashboard_aggregation_group_usage_test.go new file mode 100644 index 000000000..cbb5d3c6b --- /dev/null +++ b/backend/internal/repository/dashboard_aggregation_group_usage_test.go @@ -0,0 +1,319 @@ +//go:build unit + +package repository + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/Wei-Shaw/sub2api/internal/service" + "github.com/stretchr/testify/require" +) + +func TestDashboardAggregationRepositorySyncGroupUsageRollupsNoopsAtCurrentDate(t *testing.T) { + setGroupUsageRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC) + + mock.ExpectBegin() + mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}). + AddRow("2026-08-14", time.Unix(0, 0).UTC(), "Asia/Shanghai")) + mock.ExpectCommit() + + require.NoError(t, repo.SyncGroupUsageRollups(context.Background(), todayStart)) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDashboardAggregationRepositorySyncGroupUsageRollupsRebuildsWhenTimezoneChanges(t *testing.T) { + useGroupUsageRepositoryTestTimezone(t, "America/New_York") + + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + todayStart := time.Date(2026, 3, 9, 4, 0, 0, 0, time.UTC) + retainedFrom := time.Date(2026, 3, 1, 5, 0, 0, 0, time.UTC) + + mock.ExpectBegin() + mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}). + AddRow("2026-03-09", time.Unix(0, 0).UTC(), "Asia/Shanghai")) + mock.ExpectQuery(`SELECT MIN\(created_at\) FROM usage_logs`). + WillReturnRows(sqlmock.NewRows([]string{"min"}).AddRow(retainedFrom)) + mock.ExpectExec(`DELETE FROM usage_group_daily_rollups`). + WithArgs("2026-03-01", "2026-03-01", "2026-03-09"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO usage_group_daily_rollups`). + WithArgs(retainedFrom, todayStart, "America/New_York"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs("2026-03-09", retainedFrom, "America/New_York"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + require.NoError(t, repo.SyncGroupUsageRollups(context.Background(), todayStart)) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDashboardAggregationRepositorySyncGroupUsageRollupsPublishesWatermarkLast(t *testing.T) { + setGroupUsageRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC) + retainedFrom := time.Date(2026, 5, 1, 3, 0, 0, 0, time.UTC) + rebuildStart := time.Date(2026, 8, 12, 16, 0, 0, 0, time.UTC) + + mock.ExpectBegin() + mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}). + AddRow("2026-08-13", time.Unix(0, 0).UTC(), "Asia/Shanghai")) + mock.ExpectQuery(`SELECT MIN\(created_at\) FROM usage_logs`). + WillReturnRows(sqlmock.NewRows([]string{"min"}).AddRow(retainedFrom)) + mock.ExpectExec(`DELETE FROM usage_group_daily_rollups`). + WithArgs("2026-05-01", "2026-08-13", "2026-08-14"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO usage_group_daily_rollups`). + WithArgs(rebuildStart, todayStart, "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 2)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs("2026-08-14", retainedFrom, "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + require.NoError(t, repo.SyncGroupUsageRollups(context.Background(), todayStart)) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDashboardAggregationRepositorySyncGroupUsageRollupsRejectsFutureWatermark(t *testing.T) { + setGroupUsageRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC) + + mock.ExpectBegin() + mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}). + AddRow("2026-08-15", time.Unix(0, 0).UTC(), "Asia/Shanghai")) + mock.ExpectRollback() + + err := repo.SyncGroupUsageRollups(context.Background(), todayStart) + require.ErrorContains(t, err, "未来") + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDashboardAggregationRepositoryRecomputeRangeInvalidatesGroupRollupsBeforeDashboardRebuild(t *testing.T) { + setGroupUsageRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + start := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC) + end := start.Add(time.Hour) + + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(start, "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`DELETE FROM usage_dashboard_hourly`). + WillReturnError(sql.ErrConnDone) + mock.ExpectRollback() + + err := repo.RecomputeRange(context.Background(), start, end) + require.ErrorIs(t, err, sql.ErrConnDone) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDashboardAggregationRepositoryRecomputeRangeRebuildsGroupRollupsBeforeCommit(t *testing.T) { + setGroupUsageRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + start := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC) + end := start.Add(time.Hour) + fixedNow := time.Date(2026, 8, 14, 8, 0, 0, 0, time.UTC) + repo.clock = func() time.Time { return fixedNow } + todayStart := service.GroupUsageTodayStart(fixedNow) + startDate := service.GroupUsageDate(start) + rebuildStart, err := service.ParseGroupUsageDate(startDate) + require.NoError(t, err) + + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(start, "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + for _, query := range []string{ + `DELETE FROM usage_dashboard_hourly WHERE`, + `DELETE FROM usage_dashboard_hourly_users WHERE`, + `DELETE FROM usage_dashboard_daily WHERE`, + `DELETE FROM usage_dashboard_daily_users WHERE`, + `INSERT INTO usage_dashboard_hourly_users`, + `INSERT INTO usage_dashboard_daily_users`, + `INSERT INTO usage_dashboard_hourly`, + `INSERT INTO usage_dashboard_daily`, + } { + mock.ExpectExec(query).WillReturnResult(sqlmock.NewResult(0, 1)) + } + mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}). + AddRow(startDate, time.Unix(0, 0).UTC(), "Asia/Shanghai")) + mock.ExpectQuery(`SELECT MIN\(created_at\) FROM usage_logs`). + WillReturnRows(sqlmock.NewRows([]string{"min"}).AddRow(start)) + mock.ExpectExec(`DELETE FROM usage_group_daily_rollups`). + WithArgs(startDate, startDate, service.GroupUsageDate(todayStart)). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`INSERT INTO usage_group_daily_rollups`). + WithArgs(rebuildStart.UTC(), todayStart.UTC(), "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(service.GroupUsageDate(todayStart), start, "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + require.NoError(t, repo.RecomputeRange(context.Background(), start, end)) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDashboardAggregationRepositoryCleanupUsageLogsNonPartitionedInvalidatesEachBatchAndSyncs(t *testing.T) { + setGroupUsageRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + cutoff := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + earliestDeletedAt := time.Date(2026, 5, 3, 2, 0, 0, 0, time.UTC) + fixedNow := time.Date(2026, 8, 14, 8, 0, 0, 0, time.UTC) + repo.clock = func() time.Time { return fixedNow } + todayStart := service.GroupUsageTodayStart(fixedNow) + + mock.ExpectQuery(`SELECT EXISTS`). + WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(false)) + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + mock.ExpectQuery(`(?s)DELETE FROM usage_logs.*RETURNING created_at`). + WithArgs(cutoff, usageLogsCleanupBatchSize). + WillReturnRows(sqlmock.NewRows([]string{"created_at"}). + AddRow(earliestDeletedAt.Add(time.Hour)). + AddRow(earliestDeletedAt)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(earliestDeletedAt, "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + mock.ExpectBegin() + mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}). + AddRow(service.GroupUsageDate(todayStart), time.Unix(0, 0).UTC(), "Asia/Shanghai")) + mock.ExpectCommit() + + require.NoError(t, repo.CleanupUsageLogs(context.Background(), cutoff)) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDashboardAggregationRepositoryCleanupUsageLogsPartitionedSortsAndInvalidatesEachDropBeforeSync(t *testing.T) { + setGroupUsageRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + cutoff := time.Date(2026, 7, 18, 0, 0, 0, 0, time.UTC) + aprilStart := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC) + juneStart := time.Date(2026, 6, 1, 0, 0, 0, 0, time.UTC) + fixedNow := time.Date(2026, 8, 14, 8, 0, 0, 0, time.UTC) + repo.clock = func() time.Time { return fixedNow } + todayStart := service.GroupUsageTodayStart(fixedNow) + + mock.ExpectQuery(`SELECT EXISTS`). + WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(true)) + mock.ExpectQuery(`SELECT c.relname`). + WillReturnRows(sqlmock.NewRows([]string{"relname"}). + AddRow("usage_logs_202606"). + AddRow("usage_logs_invalid"). + AddRow("usage_logs_202604"). + AddRow("usage_logs_202607")) + + for _, partition := range []struct { + name string + start time.Time + }{ + {name: "usage_logs_202604", start: aprilStart}, + {name: "usage_logs_202606", start: juneStart}, + } { + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(partition.start, "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`DROP TABLE IF EXISTS "` + partition.name + `"`). + WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectCommit() + } + mock.ExpectBegin() + mock.ExpectQuery(`SELECT closed_before::text, retained_from.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"closed_before", "retained_from", "timezone_name"}). + AddRow(service.GroupUsageDate(todayStart), time.Unix(0, 0).UTC(), "Asia/Shanghai")) + mock.ExpectCommit() + + require.NoError(t, repo.CleanupUsageLogs(context.Background(), cutoff)) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDashboardAggregationRepositoryCleanupUsageLogsNonPartitionedFailureRollsBackWithoutSync(t *testing.T) { + setGroupUsageRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + cutoff := time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) + deletedAt := time.Date(2026, 5, 3, 2, 0, 0, 0, time.UTC) + + mock.ExpectQuery(`SELECT EXISTS`). + WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(false)) + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + mock.ExpectQuery(`(?s)SELECT ctid.*ORDER BY created_at ASC, id ASC.*DELETE FROM usage_logs.*RETURNING created_at`). + WithArgs(cutoff, usageLogsCleanupBatchSize). + WillReturnRows(sqlmock.NewRows([]string{"created_at"}).AddRow(deletedAt)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(deletedAt, "Asia/Shanghai"). + WillReturnError(sql.ErrConnDone) + mock.ExpectRollback() + + err := repo.CleanupUsageLogs(context.Background(), cutoff) + require.ErrorIs(t, err, sql.ErrConnDone) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestDashboardAggregationRepositoryCleanupUsageLogsPartitionFailureRollsBackAndStops(t *testing.T) { + setGroupUsageRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := newDashboardAggregationRepositoryWithSQL(db) + cutoff := time.Date(2026, 7, 18, 0, 0, 0, 0, time.UTC) + aprilStart := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC) + dropErr := errors.New("drop partition failed") + + mock.ExpectQuery(`SELECT EXISTS`). + WillReturnRows(sqlmock.NewRows([]string{"exists"}).AddRow(true)) + mock.ExpectQuery(`SELECT c.relname`). + WillReturnRows(sqlmock.NewRows([]string{"relname"}). + AddRow("usage_logs_202606"). + AddRow("usage_logs_202604")) + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(aprilStart, "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectExec(`DROP TABLE IF EXISTS "usage_logs_202604"`). + WillReturnError(dropErr) + mock.ExpectRollback() + + err := repo.CleanupUsageLogs(context.Background(), cutoff) + require.ErrorIs(t, err, dropErr) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func setGroupUsageRollupTestTimezone(t *testing.T) { + t.Helper() + useGroupUsageRepositoryTestTimezone(t, "Asia/Shanghai") +} diff --git a/backend/internal/repository/dashboard_aggregation_repo.go b/backend/internal/repository/dashboard_aggregation_repo.go index 5e09e75d4..6701a3c99 100644 --- a/backend/internal/repository/dashboard_aggregation_repo.go +++ b/backend/internal/repository/dashboard_aggregation_repo.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "log" + "sort" "strings" "time" @@ -14,7 +15,8 @@ import ( ) type dashboardAggregationRepository struct { - sql sqlExecutor + sql sqlExecutor + clock func() time.Time } const usageLogsCleanupBatchSize = 10000 @@ -33,7 +35,14 @@ func NewDashboardAggregationRepository(sqlDB *sql.DB) service.DashboardAggregati } func newDashboardAggregationRepositoryWithSQL(sqlq sqlExecutor) *dashboardAggregationRepository { - return &dashboardAggregationRepository{sql: sqlq} + return &dashboardAggregationRepository{sql: sqlq, clock: time.Now} +} + +func (r *dashboardAggregationRepository) now() time.Time { + if r.clock != nil { + return r.clock() + } + return time.Now() } func isPostgresDriver(db *sql.DB) bool { @@ -128,11 +137,23 @@ func (r *dashboardAggregationRepository) RecomputeRange(ctx context.Context, sta if err != nil { return err } + if err := lockGroupUsageRollupState(ctx, tx); err != nil { + _ = tx.Rollback() + return err + } + if err := invalidateGroupUsageRollupsAt(ctx, tx, start); err != nil { + _ = tx.Rollback() + return err + } txRepo := newDashboardAggregationRepositoryWithSQL(tx) if err := txRepo.recomputeRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd); err != nil { _ = tx.Rollback() return err } + if err := txRepo.syncGroupUsageRollupsInTx(ctx, service.GroupUsageTodayStart(r.now())); err != nil { + _ = tx.Rollback() + return err + } return tx.Commit() } return r.recomputeRangeInTx(ctx, hourStart, hourEnd, dayStart, dayEnd) @@ -215,14 +236,35 @@ func (r *dashboardAggregationRepository) CleanupUsageLogs(ctx context.Context, c return err } if isPartitioned { - return r.dropUsageLogsPartitions(ctx, cutoff) + if err := r.dropUsageLogsPartitions(ctx, cutoff); err != nil { + return err + } + } else if err := r.cleanupUsageLogsBatches(ctx, cutoff); err != nil { + return err } + return r.SyncGroupUsageRollups(ctx, service.GroupUsageTodayStart(r.now())) +} + +func (r *dashboardAggregationRepository) cleanupUsageLogsBatches(ctx context.Context, cutoff time.Time) error { + db, transactional := r.sql.(*sql.DB) for { + if transactional { + affected, err := cleanupUsageLogsBatchWithRollupInvalidation(ctx, db, cutoff) + if err != nil { + return err + } + if affected < usageLogsCleanupBatchSize { + return nil + } + continue + } + res, err := r.sql.ExecContext(ctx, ` WITH victims AS ( SELECT ctid FROM usage_logs WHERE created_at < $1 + ORDER BY created_at ASC, id ASC LIMIT $2 ) DELETE FROM usage_logs @@ -241,6 +283,66 @@ func (r *dashboardAggregationRepository) CleanupUsageLogs(ctx context.Context, c } } +func cleanupUsageLogsBatchWithRollupInvalidation(ctx context.Context, db *sql.DB, cutoff time.Time) (int64, error) { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + rollback := func(err error) (int64, error) { + _ = tx.Rollback() + return 0, err + } + + if err := lockGroupUsageRollupState(ctx, tx); err != nil { + return rollback(err) + } + rows, err := tx.QueryContext(ctx, ` + WITH victims AS ( + SELECT ctid + FROM usage_logs + WHERE created_at < $1 + ORDER BY created_at ASC, id ASC + LIMIT $2 + ) + DELETE FROM usage_logs + WHERE ctid IN (SELECT ctid FROM victims) + RETURNING created_at + `, cutoff.UTC(), usageLogsCleanupBatchSize) + if err != nil { + return rollback(err) + } + + var affected int64 + var earliestDeletedAt time.Time + for rows.Next() { + var deletedAt time.Time + if err := rows.Scan(&deletedAt); err != nil { + _ = rows.Close() + return rollback(err) + } + affected++ + if earliestDeletedAt.IsZero() || deletedAt.Before(earliestDeletedAt) { + earliestDeletedAt = deletedAt + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return rollback(err) + } + if err := rows.Close(); err != nil { + return rollback(err) + } + if affected > 0 { + if err := invalidateGroupUsageRollupsAt(ctx, tx, earliestDeletedAt); err != nil { + return rollback(err) + } + } + if err := tx.Commit(); err != nil { + return 0, err + } + return affected, nil +} + func (r *dashboardAggregationRepository) CleanupUsageBillingDedup(ctx context.Context, cutoff time.Time) error { for { res, err := r.sql.ExecContext(ctx, ` @@ -489,14 +591,16 @@ func (r *dashboardAggregationRepository) dropUsageLogsPartitions(ctx context.Con if err != nil { return err } - defer func() { - _ = rows.Close() - }() - cutoffMonth := truncateToMonthUTC(cutoff) + type usageLogsPartition struct { + name string + month time.Time + } + partitions := make([]usageLogsPartition, 0) for rows.Next() { var name string if err := rows.Scan(&name); err != nil { + _ = rows.Close() return err } if !strings.HasPrefix(name, "usage_logs_") { @@ -509,12 +613,56 @@ func (r *dashboardAggregationRepository) dropUsageLogsPartitions(ctx context.Con } month = month.UTC() if month.Before(cutoffMonth) { - if _, err := r.sql.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", pq.QuoteIdentifier(name))); err != nil { + partitions = append(partitions, usageLogsPartition{name: name, month: month}) + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return err + } + if err := rows.Close(); err != nil { + return err + } + + sort.Slice(partitions, func(i, j int) bool { + return partitions[i].month.Before(partitions[j].month) + }) + if db, ok := r.sql.(*sql.DB); ok { + for _, partition := range partitions { + if err := dropUsageLogsPartitionWithRollupInvalidation(ctx, db, partition.name, partition.month); err != nil { return err } } + return nil } - return rows.Err() + for _, partition := range partitions { + if _, err := r.sql.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", pq.QuoteIdentifier(partition.name))); err != nil { + return err + } + } + return nil +} + +func dropUsageLogsPartitionWithRollupInvalidation(ctx context.Context, db *sql.DB, name string, monthStart time.Time) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + rollback := func(err error) error { + _ = tx.Rollback() + return err + } + + if err := lockGroupUsageRollupState(ctx, tx); err != nil { + return rollback(err) + } + if err := invalidateGroupUsageRollupsAt(ctx, tx, monthStart); err != nil { + return rollback(err) + } + if _, err := tx.ExecContext(ctx, fmt.Sprintf("DROP TABLE IF EXISTS %s", pq.QuoteIdentifier(name))); err != nil { + return rollback(err) + } + return tx.Commit() } func (r *dashboardAggregationRepository) createUsageLogsPartition(ctx context.Context, month time.Time) error { diff --git a/backend/internal/repository/group_usage_rollup_trigger_integration_test.go b/backend/internal/repository/group_usage_rollup_trigger_integration_test.go new file mode 100644 index 000000000..db182be49 --- /dev/null +++ b/backend/internal/repository/group_usage_rollup_trigger_integration_test.go @@ -0,0 +1,511 @@ +//go:build integration + +package repository + +import ( + "context" + "database/sql" + "fmt" + "testing" + "time" + + "github.com/Wei-Shaw/sub2api/migrations" + "github.com/lib/pq" + "github.com/stretchr/testify/require" +) + +func TestGroupUsageRollupTriggerInvalidatesCascadedHistoricalDelete(t *testing.T) { + for _, partitioned := range []bool{false, true} { + name := "ordinary" + if partitioned { + name = "partitioned" + } + t.Run(name, func(t *testing.T) { + ctx := context.Background() + schema := createGroupUsageRollupTriggerTestSchema(t, ctx, partitioned) + tx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = tx.Rollback() }() + + _, err := tx.ExecContext(ctx, ` + INSERT INTO groups (id) VALUES (10); + INSERT INTO users (id) VALUES (1); + INSERT INTO usage_logs (id, user_id, group_id, actual_cost, created_at) + VALUES (1, 1, 10, 1.25, TIMESTAMPTZ '2020-01-02 08:00:00+08'); + UPDATE usage_group_rollup_state + SET closed_before = (CURRENT_TIMESTAMP AT TIME ZONE 'Asia/Shanghai')::date + WHERE id = 1; + DELETE FROM users WHERE id = 1; + `) + require.NoError(t, err) + + var closedBefore string + err = tx.QueryRowContext(ctx, ` + SELECT closed_before::text + FROM usage_group_rollup_state + WHERE id = 1 + `).Scan(&closedBefore) + require.NoError(t, err) + require.Equal(t, "2020-01-02", closedBefore) + }) + } +} + +func TestGroupUsageRollupTriggerSerializesLateHistoricalInsertWithPublish(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + schema := createGroupUsageRollupTriggerTestSchema(t, ctx, false) + seedTx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + _, err := seedTx.ExecContext(ctx, ` + INSERT INTO groups (id) VALUES (10); + INSERT INTO users (id) VALUES (1); + UPDATE usage_group_rollup_state + SET closed_before = DATE '2020-01-02' + WHERE id = 1; + `) + require.NoError(t, err) + require.NoError(t, seedTx.Commit()) + + syncTx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = syncTx.Rollback() }() + var stateID int16 + require.NoError(t, syncTx.QueryRowContext(ctx, ` + SELECT id + FROM usage_group_rollup_state + WHERE id = 1 + FOR UPDATE + `).Scan(&stateID)) + + lateTx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = lateTx.Rollback() }() + var lateBackendPID int + require.NoError(t, lateTx.QueryRowContext(ctx, "SELECT pg_backend_pid()").Scan(&lateBackendPID)) + + insertResult := make(chan error, 1) + go func() { + _, insertErr := lateTx.ExecContext(ctx, ` + INSERT INTO usage_logs (id, user_id, group_id, actual_cost, created_at) + VALUES (1, 1, 10, 1.25, TIMESTAMPTZ '2020-01-02 09:00:00+08') + `) + insertResult <- insertErr + }() + + blocked, err := waitForGroupUsageRollupStateLock(ctx, lateBackendPID, insertResult) + if err != nil || !blocked { + _ = syncTx.Rollback() + _ = lateTx.Rollback() + require.NoError(t, err) + require.True(t, blocked, "迟到写入必须等待正在发布水位的事务") + } + + _, err = syncTx.ExecContext(ctx, ` + UPDATE usage_group_rollup_state + SET closed_before = (CURRENT_TIMESTAMP AT TIME ZONE 'Asia/Shanghai')::date + WHERE id = 1 + `) + require.NoError(t, err) + require.NoError(t, syncTx.Commit()) + + select { + case err = <-insertResult: + require.NoError(t, err) + case <-ctx.Done(): + t.Fatal("等待迟到写入完成超时") + } + require.NoError(t, lateTx.Commit()) + + var closedBefore string + err = integrationDB.QueryRowContext(ctx, fmt.Sprintf( + "SELECT closed_before::text FROM %s.usage_group_rollup_state WHERE id = 1", + pq.QuoteIdentifier(schema), + )).Scan(&closedBefore) + require.NoError(t, err) + require.Equal(t, "2020-01-02", closedBefore) +} + +func TestGroupUsageRollupTriggerSerializesInsertTransactionAcrossMidnight(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + schema := createGroupUsageRollupTriggerTestSchema(t, ctx, false) + seedTx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + _, err := seedTx.ExecContext(ctx, ` + INSERT INTO groups (id) VALUES (10); + INSERT INTO users (id) VALUES (1); + UPDATE usage_group_rollup_state + SET closed_before = (CURRENT_TIMESTAMP AT TIME ZONE 'Asia/Shanghai')::date + WHERE id = 1; + `) + require.NoError(t, err) + require.NoError(t, seedTx.Commit()) + + syncTx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = syncTx.Rollback() }() + var stateID int16 + require.NoError(t, syncTx.QueryRowContext(ctx, ` + SELECT id + FROM usage_group_rollup_state + WHERE id = 1 + FOR UPDATE + `).Scan(&stateID)) + + insertTx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = insertTx.Rollback() }() + var insertBackendPID int + require.NoError(t, insertTx.QueryRowContext(ctx, "SELECT pg_backend_pid()").Scan(&insertBackendPID)) + + insertResult := make(chan error, 1) + go func() { + _, insertErr := insertTx.ExecContext(ctx, ` + INSERT INTO usage_logs (id, user_id, group_id, actual_cost, created_at) + VALUES (1, 1, 10, 1.25, CURRENT_TIMESTAMP) + `) + insertResult <- insertErr + }() + + blocked, err := waitForGroupUsageRollupStateLock(ctx, insertBackendPID, insertResult) + if err != nil || !blocked { + _ = syncTx.Rollback() + _ = insertTx.Rollback() + require.NoError(t, err) + require.True(t, blocked, "跨越零点的在途写入必须与水位发布串行化") + } + + _, err = syncTx.ExecContext(ctx, ` + UPDATE usage_group_rollup_state + SET closed_before = (CURRENT_TIMESTAMP AT TIME ZONE 'Asia/Shanghai')::date + 1 + WHERE id = 1 + `) + require.NoError(t, err) + require.NoError(t, syncTx.Commit()) + + select { + case err = <-insertResult: + require.NoError(t, err) + case <-ctx.Done(): + t.Fatal("等待跨零点写入完成超时") + } + require.NoError(t, insertTx.Commit()) + + var currentDate string + require.NoError(t, integrationDB.QueryRowContext(ctx, ` + SELECT (CURRENT_TIMESTAMP AT TIME ZONE 'Asia/Shanghai')::date::text + `).Scan(¤tDate)) + var closedBefore string + err = integrationDB.QueryRowContext(ctx, fmt.Sprintf( + "SELECT closed_before::text FROM %s.usage_group_rollup_state WHERE id = 1", + pq.QuoteIdentifier(schema), + )).Scan(&closedBefore) + require.NoError(t, err) + require.Equal(t, currentDate, closedBefore) +} + +func TestGroupUsageRollupTriggerKeepsWatermarkForTodayInsert(t *testing.T) { + ctx := context.Background() + schema := createGroupUsageRollupTriggerTestSchema(t, ctx, false) + + tx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = tx.Rollback() }() + _, err := tx.ExecContext(ctx, ` + INSERT INTO groups (id) VALUES (10); + INSERT INTO users (id) VALUES (1); + UPDATE usage_group_rollup_state + SET closed_before = (CURRENT_TIMESTAMP AT TIME ZONE 'Asia/Shanghai')::date + WHERE id = 1; + INSERT INTO usage_logs (id, user_id, group_id, actual_cost, created_at) + VALUES (1, 1, 10, 1.25, CURRENT_TIMESTAMP); + `) + require.NoError(t, err) + + var unchanged bool + err = tx.QueryRowContext(ctx, ` + SELECT closed_before = (CURRENT_TIMESTAMP AT TIME ZONE 'Asia/Shanghai')::date + FROM usage_group_rollup_state + WHERE id = 1 + `).Scan(&unchanged) + require.NoError(t, err) + require.True(t, unchanged) +} + +func TestGroupUsageRollupTriggerUsesSessionTimezoneAcrossDST(t *testing.T) { + ctx := context.Background() + schema := createGroupUsageRollupTriggerTestSchema(t, ctx, false) + + tx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = tx.Rollback() }() + _, err := tx.ExecContext(ctx, ` + SET LOCAL TIME ZONE 'America/New_York'; + INSERT INTO groups (id) VALUES (10); + INSERT INTO users (id) VALUES (1); + UPDATE usage_group_rollup_state + SET closed_before = DATE '2026-03-09', + timezone_name = 'America/New_York' + WHERE id = 1; + INSERT INTO usage_logs (id, user_id, group_id, actual_cost, created_at) + VALUES (1, 1, 10, 1.25, TIMESTAMPTZ '2026-03-08 04:30:00+00'); + `) + require.NoError(t, err) + + var closedBefore string + err = tx.QueryRowContext(ctx, ` + SELECT closed_before::text + FROM usage_group_rollup_state + WHERE id = 1 + `).Scan(&closedBefore) + require.NoError(t, err) + require.Equal(t, "2026-03-07", closedBefore) +} + +func TestGroupUsageSummaryIncludesYesterdayAcrossWatermark(t *testing.T) { + ctx := context.Background() + useGroupUsageRepositoryTestTimezone(t, "Asia/Shanghai") + todayStart := time.Date(2026, 8, 13, 16, 0, 0, 0, time.UTC) + + tests := []struct { + name string + closedBefore string + includeYesterday bool + }{ + {name: "closed_rollup", closedBefore: "2026-08-14", includeYesterday: true}, + {name: "raw_tail", closedBefore: "2026-08-13", includeYesterday: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + schema := createGroupUsageRollupTriggerTestSchema(t, ctx, false) + tx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = tx.Rollback() }() + + _, err := tx.ExecContext(ctx, ` + INSERT INTO groups (id) VALUES (10); + INSERT INTO users (id) VALUES (1); + INSERT INTO usage_logs (id, user_id, group_id, actual_cost, created_at) VALUES + (1, 1, 10, 2, TIMESTAMPTZ '2026-08-12 12:00:00+08'), + (2, 1, 10, 3, TIMESTAMPTZ '2026-08-13 12:00:00+08'), + (3, 1, 10, 4, TIMESTAMPTZ '2026-08-14 12:00:00+08'); + INSERT INTO usage_group_daily_rollups (bucket_date, group_id, actual_cost, computed_at) + VALUES (DATE '2026-08-12', 10, 2, NOW()); + `) + require.NoError(t, err) + if tt.includeYesterday { + _, err = tx.ExecContext(ctx, ` + INSERT INTO usage_group_daily_rollups (bucket_date, group_id, actual_cost, computed_at) + VALUES (DATE '2026-08-13', 10, 3, NOW()) + `) + require.NoError(t, err) + } + _, err = tx.ExecContext(ctx, ` + UPDATE usage_group_rollup_state + SET closed_before = $1::date, + retained_from = TIMESTAMPTZ '2026-08-12 00:00:00+08' + WHERE id = 1 + `, tt.closedBefore) + require.NoError(t, err) + + repo := newUsageLogRepositoryWithSQL(nil, tx) + result, err := repo.GetAllGroupUsageSummary(ctx, todayStart) + require.NoError(t, err) + require.Len(t, result, 1) + require.InDelta(t, 9, result[0].TotalCost, 0.0000001) + require.InDelta(t, 4, result[0].TodayCost, 0.0000001) + require.InDelta(t, 3, result[0].YesterdayCost, 0.0000001) + }) + } +} + +func TestGroupUsageRollupSyncRebuildsAfterTimezoneChange(t *testing.T) { + ctx := context.Background() + useGroupUsageRepositoryTestTimezone(t, "America/New_York") + todayStart := time.Date(2026, 3, 9, 4, 0, 0, 0, time.UTC) + schema := createGroupUsageRollupTriggerTestSchema(t, ctx, false) + tx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = tx.Rollback() }() + + _, err := tx.ExecContext(ctx, ` + SET LOCAL TIME ZONE 'UTC'; + INSERT INTO groups (id) VALUES (10); + INSERT INTO users (id) VALUES (1); + INSERT INTO usage_logs (id, user_id, group_id, actual_cost, created_at) VALUES + (1, 1, 10, 3, TIMESTAMPTZ '2026-03-08 05:30:00+00'), + (2, 1, 10, 5, TIMESTAMPTZ '2026-03-09 04:30:00+00'); + INSERT INTO usage_group_daily_rollups (bucket_date, group_id, actual_cost, computed_at) + VALUES (DATE '2026-03-08', 10, 99, NOW()); + UPDATE usage_group_rollup_state + SET closed_before = DATE '2026-03-09', + retained_from = TIMESTAMPTZ '2026-03-08 05:30:00+00', + timezone_name = 'Asia/Shanghai' + WHERE id = 1; + `) + require.NoError(t, err) + + repo := newDashboardAggregationRepositoryWithSQL(tx) + require.NoError(t, repo.SyncGroupUsageRollups(ctx, todayStart)) + + var stateTimezone string + var closedBefore string + require.NoError(t, tx.QueryRowContext(ctx, ` + SELECT timezone_name, closed_before::text + FROM usage_group_rollup_state + WHERE id = 1 + `).Scan(&stateTimezone, &closedBefore)) + require.Equal(t, "America/New_York", stateTimezone) + require.Equal(t, "2026-03-09", closedBefore) + + var rollupCost float64 + require.NoError(t, tx.QueryRowContext(ctx, ` + SELECT actual_cost + FROM usage_group_daily_rollups + WHERE bucket_date = DATE '2026-03-08' AND group_id = 10 + `).Scan(&rollupCost)) + require.InDelta(t, 3, rollupCost, 0.0000001) + + usageRepo := newUsageLogRepositoryWithSQL(nil, tx) + result, err := usageRepo.GetAllGroupUsageSummary(ctx, todayStart) + require.NoError(t, err) + require.Len(t, result, 1) + require.InDelta(t, 8, result[0].TotalCost, 0.0000001) + require.InDelta(t, 5, result[0].TodayCost, 0.0000001) + require.InDelta(t, 3, result[0].YesterdayCost, 0.0000001) +} + +func TestGroupUsageSummaryUsesConfiguredDSTBoundaries(t *testing.T) { + ctx := context.Background() + useGroupUsageRepositoryTestTimezone(t, "America/New_York") + todayStart := time.Date(2026, 3, 9, 4, 0, 0, 0, time.UTC) + schema := createGroupUsageRollupTriggerTestSchema(t, ctx, false) + tx := beginGroupUsageRollupTriggerTestTx(t, ctx, schema) + defer func() { _ = tx.Rollback() }() + + _, err := tx.ExecContext(ctx, ` + SET LOCAL TIME ZONE 'UTC'; + INSERT INTO groups (id) VALUES (10); + INSERT INTO users (id) VALUES (1); + INSERT INTO usage_logs (id, user_id, group_id, actual_cost, created_at) VALUES + (1, 1, 10, 100, TIMESTAMPTZ '2026-03-08 04:30:00+00'), + (2, 1, 10, 3, TIMESTAMPTZ '2026-03-08 05:30:00+00'), + (3, 1, 10, 4, TIMESTAMPTZ '2026-03-09 03:30:00+00'), + (4, 1, 10, 5, TIMESTAMPTZ '2026-03-09 04:30:00+00'); + UPDATE usage_group_rollup_state + SET closed_before = DATE '1970-01-01', + retained_from = TIMESTAMPTZ '1970-01-01 00:00:00+00', + timezone_name = 'America/New_York' + WHERE id = 1; + `) + require.NoError(t, err) + + repo := newUsageLogRepositoryWithSQL(nil, tx) + result, err := repo.GetAllGroupUsageSummary(ctx, todayStart) + require.NoError(t, err) + require.Len(t, result, 1) + require.InDelta(t, 112, result[0].TotalCost, 0.0000001) + require.InDelta(t, 5, result[0].TodayCost, 0.0000001) + require.InDelta(t, 7, result[0].YesterdayCost, 0.0000001) +} + +func createGroupUsageRollupTriggerTestSchema(t *testing.T, ctx context.Context, partitioned bool) string { + t.Helper() + + schema := fmt.Sprintf("group_usage_rollup_trigger_%d", time.Now().UnixNano()) + quotedSchema := pq.QuoteIdentifier(schema) + _, err := integrationDB.ExecContext(ctx, "CREATE SCHEMA "+quotedSchema) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = integrationDB.ExecContext(context.Background(), "DROP SCHEMA IF EXISTS "+quotedSchema+" CASCADE") + }) + + tx, err := integrationDB.BeginTx(ctx, nil) + require.NoError(t, err) + defer func() { _ = tx.Rollback() }() + require.NoError(t, setGroupUsageRollupTriggerSearchPath(ctx, tx, quotedSchema)) + + usageLogsDDL := ` + CREATE TABLE usage_logs ( + id BIGINT PRIMARY KEY, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + group_id BIGINT REFERENCES groups(id) ON DELETE SET NULL, + actual_cost NUMERIC(20, 10) NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ); + ` + if partitioned { + usageLogsDDL = ` + CREATE TABLE usage_logs ( + id BIGINT NOT NULL, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + group_id BIGINT REFERENCES groups(id) ON DELETE SET NULL, + actual_cost NUMERIC(20, 10) NOT NULL, + created_at TIMESTAMPTZ NOT NULL + ) PARTITION BY RANGE (created_at); + CREATE TABLE usage_logs_default PARTITION OF usage_logs DEFAULT; + ` + } + + _, err = tx.ExecContext(ctx, ` + CREATE TABLE users (id BIGINT PRIMARY KEY); + CREATE TABLE groups (id BIGINT PRIMARY KEY); + `+usageLogsDDL) + require.NoError(t, err) + + for _, migrationName := range []string{ + "222_group_usage_daily_rollups.sql", + "223_group_usage_rollup_timezone.sql", + } { + migrationSQL, readErr := migrations.FS.ReadFile(migrationName) + require.NoError(t, readErr) + for range 2 { + _, err = tx.ExecContext(ctx, string(migrationSQL)) + require.NoError(t, err) + } + } + require.NoError(t, tx.Commit()) + + return schema +} + +func beginGroupUsageRollupTriggerTestTx(t *testing.T, ctx context.Context, schema string) *sql.Tx { + t.Helper() + + tx, err := integrationDB.BeginTx(ctx, nil) + require.NoError(t, err) + require.NoError(t, setGroupUsageRollupTriggerSearchPath(ctx, tx, pq.QuoteIdentifier(schema))) + return tx +} + +func setGroupUsageRollupTriggerSearchPath(ctx context.Context, tx *sql.Tx, quotedSchema string) error { + _, err := tx.ExecContext(ctx, "SET LOCAL search_path TO "+quotedSchema) + return err +} + +func waitForGroupUsageRollupStateLock( + ctx context.Context, + backendPID int, + insertResult <-chan error, +) (bool, error) { + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + + for { + select { + case err := <-insertResult: + if err != nil { + return false, err + } + return false, nil + case <-ticker.C: + var waitEventType sql.NullString + err := integrationDB.QueryRowContext(ctx, ` + SELECT wait_event_type + FROM pg_stat_activity + WHERE pid = $1 + `, backendPID).Scan(&waitEventType) + if err != nil { + return false, err + } + if waitEventType.Valid && waitEventType.String == "Lock" { + return true, nil + } + case <-ctx.Done(): + return false, ctx.Err() + } + } +} diff --git a/backend/internal/repository/usage_cleanup_repo.go b/backend/internal/repository/usage_cleanup_repo.go index 1a25696e4..f8aa67477 100644 --- a/backend/internal/repository/usage_cleanup_repo.go +++ b/backend/internal/repository/usage_cleanup_repo.go @@ -291,6 +291,9 @@ func (r *usageCleanupRepository) DeleteUsageLogsBatch(ctx context.Context, filte return 0, fmt.Errorf("cleanup filters missing time range") } args = append(args, limit) + if db, ok := r.sql.(*sql.DB); ok { + return r.deleteUsageLogsBatchWithRollupInvalidation(ctx, db, whereClause, args) + } query := fmt.Sprintf(` WITH target AS ( SELECT id @@ -301,7 +304,7 @@ func (r *usageCleanupRepository) DeleteUsageLogsBatch(ctx context.Context, filte ) DELETE FROM usage_logs WHERE id IN (SELECT id FROM target) - RETURNING id + RETURNING created_at `, whereClause, len(args)) rows, err := r.sql.QueryContext(ctx, query, args...) @@ -320,6 +323,68 @@ func (r *usageCleanupRepository) DeleteUsageLogsBatch(ctx context.Context, filte return deleted, nil } +func (r *usageCleanupRepository) deleteUsageLogsBatchWithRollupInvalidation(ctx context.Context, db *sql.DB, whereClause string, args []any) (int64, error) { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return 0, err + } + rollback := func(err error) (int64, error) { + _ = tx.Rollback() + return 0, err + } + + if err := lockGroupUsageRollupState(ctx, tx); err != nil { + return rollback(err) + } + query := fmt.Sprintf(` + WITH target AS ( + SELECT id + FROM usage_logs + WHERE %s + ORDER BY created_at ASC, id ASC + LIMIT $%d + ) + DELETE FROM usage_logs + WHERE id IN (SELECT id FROM target) + RETURNING created_at + `, whereClause, len(args)) + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return rollback(err) + } + + var deleted int64 + var earliestDeletedAt time.Time + for rows.Next() { + var deletedAt time.Time + if err := rows.Scan(&deletedAt); err != nil { + _ = rows.Close() + return rollback(err) + } + deleted++ + if earliestDeletedAt.IsZero() || deletedAt.Before(earliestDeletedAt) { + earliestDeletedAt = deletedAt + } + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return rollback(err) + } + if err := rows.Close(); err != nil { + return rollback(err) + } + + if deleted > 0 { + if err := invalidateGroupUsageRollupsAt(ctx, tx, earliestDeletedAt); err != nil { + return rollback(err) + } + } + if err := tx.Commit(); err != nil { + return 0, err + } + return deleted, nil +} + func buildUsageCleanupWhere(filters service.UsageCleanupFilters) (string, []any) { conditions := make([]string, 0, 8) args := make([]any, 0, 8) diff --git a/backend/internal/repository/usage_cleanup_repo_test.go b/backend/internal/repository/usage_cleanup_repo_test.go index 1ac7cca56..b147d03a4 100644 --- a/backend/internal/repository/usage_cleanup_repo_test.go +++ b/backend/internal/repository/usage_cleanup_repo_test.go @@ -21,6 +21,11 @@ func newSQLMock(t *testing.T) (*sql.DB, sqlmock.Sqlmock) { return db, mock } +func setUsageCleanupRollupTestTimezone(t *testing.T) { + t.Helper() + useGroupUsageRepositoryTestTimezone(t, "Asia/Shanghai") +} + func TestNewUsageCleanupRepository(t *testing.T) { db, _ := newSQLMock(t) repo := NewUsageCleanupRepository(nil, db) @@ -398,6 +403,7 @@ func TestUsageCleanupRepositoryDeleteUsageLogsBatchMissingRange(t *testing.T) { } func TestUsageCleanupRepositoryDeleteUsageLogsBatch(t *testing.T) { + setUsageCleanupRollupTestTimezone(t) db, mock := newSQLMock(t) repo := &usageCleanupRepository{sql: db} @@ -412,9 +418,16 @@ func TestUsageCleanupRepositoryDeleteUsageLogsBatch(t *testing.T) { Model: &model, } + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) mock.ExpectQuery("DELETE FROM usage_logs"). WithArgs(start, end, userID, "gpt-4", 2). - WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(int64(1)).AddRow(int64(2))) + WillReturnRows(sqlmock.NewRows([]string{"created_at"}).AddRow(start.Add(time.Hour)).AddRow(start.Add(2 * time.Hour))) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(start.Add(time.Hour), "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() deleted, err := repo.DeleteUsageLogsBatch(context.Background(), filters, 2) require.NoError(t, err) @@ -422,6 +435,62 @@ func TestUsageCleanupRepositoryDeleteUsageLogsBatch(t *testing.T) { require.NoError(t, mock.ExpectationsWereMet()) } +func TestUsageCleanupRepositoryDeleteUsageLogsBatchAtomicallyInvalidatesGroupRollups(t *testing.T) { + setUsageCleanupRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := &usageCleanupRepository{sql: db} + + start := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + end := start.Add(24 * time.Hour) + firstDeletedAt := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC) + secondDeletedAt := firstDeletedAt.Add(time.Hour) + filters := service.UsageCleanupFilters{StartTime: start, EndTime: end} + + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + mock.ExpectQuery(`(?s)DELETE FROM usage_logs.*RETURNING created_at`). + WithArgs(start, end, 2). + WillReturnRows(sqlmock.NewRows([]string{"created_at"}). + AddRow(firstDeletedAt). + AddRow(secondDeletedAt)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(firstDeletedAt, "Asia/Shanghai"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectCommit() + + deleted, err := repo.DeleteUsageLogsBatch(context.Background(), filters, 2) + require.NoError(t, err) + require.Equal(t, int64(2), deleted) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestUsageCleanupRepositoryDeleteUsageLogsBatchRollsBackWhenInvalidationFails(t *testing.T) { + setUsageCleanupRollupTestTimezone(t) + db, mock := newSQLMock(t) + repo := &usageCleanupRepository{sql: db} + + start := time.Date(2026, 8, 1, 0, 0, 0, 0, time.UTC) + end := start.Add(24 * time.Hour) + deletedAt := time.Date(2026, 8, 1, 3, 0, 0, 0, time.UTC) + filters := service.UsageCleanupFilters{StartTime: start, EndTime: end} + + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) + mock.ExpectQuery(`(?s)DELETE FROM usage_logs.*RETURNING created_at`). + WithArgs(start, end, 1). + WillReturnRows(sqlmock.NewRows([]string{"created_at"}).AddRow(deletedAt)) + mock.ExpectExec(`UPDATE usage_group_rollup_state`). + WithArgs(deletedAt, "Asia/Shanghai"). + WillReturnError(sql.ErrConnDone) + mock.ExpectRollback() + + _, err := repo.DeleteUsageLogsBatch(context.Background(), filters, 1) + require.ErrorIs(t, err, sql.ErrConnDone) + require.NoError(t, mock.ExpectationsWereMet()) +} + func TestUsageCleanupRepositoryDeleteUsageLogsBatchQueryError(t *testing.T) { db, mock := newSQLMock(t) repo := &usageCleanupRepository{sql: db} @@ -430,9 +499,13 @@ func TestUsageCleanupRepositoryDeleteUsageLogsBatchQueryError(t *testing.T) { end := start.Add(24 * time.Hour) filters := service.UsageCleanupFilters{StartTime: start, EndTime: end} + mock.ExpectBegin() + mock.ExpectQuery(`SELECT id FROM usage_group_rollup_state.*FOR UPDATE`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(1)) mock.ExpectQuery("DELETE FROM usage_logs"). WithArgs(start, end, 5). WillReturnError(sql.ErrConnDone) + mock.ExpectRollback() _, err := repo.DeleteUsageLogsBatch(context.Background(), filters, 5) require.Error(t, err) diff --git a/backend/internal/repository/usage_log_repo_group_summary_test.go b/backend/internal/repository/usage_log_repo_group_summary_test.go new file mode 100644 index 000000000..f67facb4b --- /dev/null +++ b/backend/internal/repository/usage_log_repo_group_summary_test.go @@ -0,0 +1,33 @@ +//go:build unit + +package repository + +import ( + "context" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/stretchr/testify/require" +) + +func TestUsageLogRepositoryGetAllGroupUsageSummaryUsesRollupTail(t *testing.T) { + db, mock := newSQLMock(t) + repo := newUsageLogRepositoryWithSQL(nil, db) + useGroupUsageRepositoryTestTimezone(t, "America/New_York") + todayStart := time.Date(2026, 3, 9, 4, 0, 0, 0, time.UTC) + yesterdayStart := time.Date(2026, 3, 8, 5, 0, 0, 0, time.UTC) + + mock.ExpectQuery(`(?s)usage_group_rollup_state.*usage_group_daily_rollups.*created_at >= state\.tail_start`). + WithArgs(todayStart, yesterdayStart, "America/New_York", "2026-03-09", "2026-03-08"). + WillReturnRows(sqlmock.NewRows([]string{"group_id", "total_cost", "today_cost", "yesterday_cost"}). + AddRow(int64(7), 12.5, 1.25, 2.5)) + + result, err := repo.GetAllGroupUsageSummary(context.Background(), todayStart) + require.NoError(t, err) + require.Equal(t, int64(7), result[0].GroupID) + require.InDelta(t, 12.5, result[0].TotalCost, 0.0000001) + require.InDelta(t, 1.25, result[0].TodayCost, 0.0000001) + require.InDelta(t, 2.5, result[0].YesterdayCost, 0.0000001) + require.NoError(t, mock.ExpectationsWereMet()) +} diff --git a/backend/internal/repository/usage_log_repo_trend.go b/backend/internal/repository/usage_log_repo_trend.go index 6d8bd79ea..3ed3aa51b 100644 --- a/backend/internal/repository/usage_log_repo_trend.go +++ b/backend/internal/repository/usage_log_repo_trend.go @@ -715,39 +715,9 @@ func (r *usageLogRepository) GetUserBreakdownStats(ctx context.Context, startTim return results, nil } -// GetAllGroupUsageSummary returns today's and cumulative actual_cost for every group. -// todayStart is the start-of-day in the caller's timezone (UTC-based). -// TODO(perf): This query scans ALL usage_logs rows for total_cost aggregation. -// When usage_logs exceeds ~1M rows, consider adding a short-lived cache (30s) -// or a materialized view / pre-aggregation table for cumulative costs. +// GetAllGroupUsageSummary 返回所有分组在服务端配置时区内的今日、昨日与当前保留记录累计金额。 func (r *usageLogRepository) GetAllGroupUsageSummary(ctx context.Context, todayStart time.Time) ([]usagestats.GroupUsageSummary, error) { - query := ` - SELECT - g.id AS group_id, - COALESCE(SUM(ul.actual_cost), 0) AS total_cost, - COALESCE(SUM(CASE WHEN ul.created_at >= $1 THEN ul.actual_cost ELSE 0 END), 0) AS today_cost - FROM groups g - LEFT JOIN usage_logs ul ON ul.group_id = g.id - GROUP BY g.id - ` - - rows, err := r.sql.QueryContext(ctx, query, todayStart) - if err != nil { - return nil, err - } - defer func() { _ = rows.Close() }() - var results []usagestats.GroupUsageSummary - for rows.Next() { - var row usagestats.GroupUsageSummary - if err := rows.Scan(&row.GroupID, &row.TotalCost, &row.TodayCost); err != nil { - return nil, err - } - results = append(results, row) - } - if err := rows.Err(); err != nil { - return nil, err - } - return results, nil + return r.getAllGroupUsageSummaryFromRollups(ctx, todayStart) } // resolveModelDimensionExpression maps model source type to a safe SQL expression. diff --git a/backend/internal/service/account_test_service.go b/backend/internal/service/account_test_service.go index 450af91c8..6a1b94249 100644 --- a/backend/internal/service/account_test_service.go +++ b/backend/internal/service/account_test_service.go @@ -622,10 +622,11 @@ func (s *AccountTestService) testOpenAIAccountConnection(c *gin.Context, account } // Align test routing with gateway behavior: OpenAI accounts apply normal - // account model mapping, and compact mode applies compact-only mapping on top. + // account model mapping. Native remote compaction v2 rides the ordinary + // /responses wire and does NOT apply the legacy compact-only mapping + // (post-#5641 semantics: compact_model_mapping is /responses/compact-only). testModelID = account.GetMappedModel(testModelID) if mode == AccountTestModeCompact { - testModelID = resolveOpenAICompactForwardModel(account, testModelID) return s.testOpenAICompactConnection(c, account, testModelID) } @@ -2007,8 +2008,10 @@ func (s *AccountTestService) testOpenAIChatCompletionsConnection( return s.processOpenAIChatCompletionsStream(c, resp.Body) } -// testOpenAICompactConnection probes /responses/compact and persists the -// resulting capability state on the account. +// testOpenAICompactConnection probes native remote compaction v2 (streaming +// /responses with a compaction_trigger input item) and persists the resulting +// capability state on the account. The legacy unary /responses/compact +// endpoint has been sunset upstream (404, #5598/#5624) and is no longer probed. func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account *Account, testModelID string) error { ctx := c.Request.Context() credentialAccount := account @@ -2033,7 +2036,7 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account if authToken == "" && !credentialAccount.IsOpenAIAgentIdentity() { return s.sendErrorAndEnd(c, "No access token available") } - apiURL = chatgptCodexAPIURL + "/compact" + apiURL = chatgptCodexAPIURL case account.Type == AccountTypeAPIKey: authToken = account.GetOpenAIApiKey() if authToken == "" { @@ -2047,7 +2050,7 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account if err != nil { return s.sendErrorAndEnd(c, fmt.Sprintf("Invalid base URL: %s", err.Error())) } - apiURL = appendOpenAIResponsesRequestPathSuffix(buildOpenAIResponsesURL(normalizedBaseURL), "/compact") + apiURL = buildOpenAIResponsesURL(normalizedBaseURL) default: return s.sendErrorAndEnd(c, fmt.Sprintf("Unsupported account type: %s", account.Type)) } @@ -2058,7 +2061,11 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account c.Writer.Header().Set("X-Accel-Buffering", "no") c.Writer.Flush() - payloadBytes, _ := json.Marshal(createOpenAICompactProbePayload(testModelID)) + // 原生 v2 走普通 /responses 线：OAuth 与真实转发一致做上游模型归一化。 + if isOAuth { + testModelID = normalizeOpenAIModelForUpstream(credentialAccount, testModelID) + } + payloadBytes, _ := json.Marshal(createOpenAICompactProbePayload(testModelID, isOAuth)) if !agentIdentityTaskRecoveryWasTried(ctx) { s.sendEvent(c, TestEvent{Type: "test_start", Model: testModelID}) } @@ -2070,7 +2077,9 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account req = req.WithContext(WithHTTPUpstreamProfile(req.Context(), HTTPUpstreamProfileOpenAI)) req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") + // v2 探测是流式请求；同时补注协商头，与真实 codex 出站线型一致。 + req.Header.Set("Accept", "text/event-stream") + ensureOpenAIRemoteCompactionV2BetaFeature(req.Header) if credentialAccount.IsOpenAIAgentIdentity() { authHeaders, authErr := buildAgentIdentityAuthenticationHeaders(ctx, s.accountRepo, s.agentIdentityWS, &s.agentIdentityTaskMu, credentialAccount) if authErr != nil { @@ -2092,6 +2101,12 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account if isOAuth { req.Host = "chatgpt.com" setOpenAIChatGPTAccountHeaders(req.Header, credentialAccount) + // 指纹收敛：探测与真实转发走同一个 /responses 端点，身份也必须同构， + // 否则探测流量会以「缺 x-codex-installation-id + 非收敛 session」的 + // 形态暴露在上游眼里。账号关闭收敛（off）时返回 nil，探测保持原样。 + if fpIDs := resolveCodexFingerprintIDsFromRequest(account, req.Header); fpIDs != nil { + applyCodexFingerprintHeaders(req.Header, fpIDs) + } } // 账号级请求头覆写：测试请求与真实转发保持一致的最终头 @@ -2105,7 +2120,7 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account resp, err := s.httpUpstream.DoWithTLS(req, proxyURL, account.ID, account.Concurrency, s.tlsFPProfileService.ResolveTLSProfile(account)) if err != nil { if s.accountRepo != nil { - updates := buildOpenAICompactProbeExtraUpdates(nil, nil, err, time.Now()) + updates := buildOpenAICompactProbeExtraUpdates(nil, nil, err, false, time.Now()) _ = s.accountRepo.UpdateExtra(ctx, account.ID, updates) mergeAccountExtra(account, updates) } @@ -2124,8 +2139,9 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account return s.testOpenAICompactConnection(c, account, testModelID) } + compactionFound := openAICompactProbeFoundCompactionItem(body) if s.accountRepo != nil { - updates := buildOpenAICompactProbeExtraUpdates(resp, body, nil, time.Now()) + updates := buildOpenAICompactProbeExtraUpdates(resp, body, nil, compactionFound, time.Now()) if codexUpdates, err := extractOpenAICodexProbeUpdates(resp); err == nil && len(codexUpdates) > 0 { updates = mergeExtraUpdates(updates, codexUpdates) } @@ -2147,7 +2163,11 @@ func (s *AccountTestService) testOpenAICompactConnection(c *gin.Context, account return s.sendErrorAndEnd(c, fmt.Sprintf("API returned %d: %s", resp.StatusCode, string(body))) } - s.sendEvent(c, TestEvent{Type: "content", Text: "Compact probe succeeded"}) + if !compactionFound { + return s.sendErrorAndEnd(c, "Upstream returned 2xx without a compaction output item (native remote compaction v2 unsupported on this chain)") + } + + s.sendEvent(c, TestEvent{Type: "content", Text: "Compact probe succeeded (native remote compaction v2)"}) s.sendEvent(c, TestEvent{Type: "test_complete", Success: true}) return nil } diff --git a/backend/internal/service/account_test_service_openai_compact_test.go b/backend/internal/service/account_test_service_openai_compact_test.go index c5cb7b9f8..8d1f1d752 100644 --- a/backend/internal/service/account_test_service_openai_compact_test.go +++ b/backend/internal/service/account_test_service_openai_compact_test.go @@ -10,10 +10,16 @@ import ( "github.com/Wei-Shaw/sub2api/internal/config" "github.com/gin-gonic/gin" + "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/tidwall/gjson" ) +// compactProbeSSESuccessBody 是原生 v2 压缩成功的最小 SSE 形态： +// output_item.done 携带 compaction item + response.completed。 +const compactProbeSSESuccessBody = "data: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"compaction\",\"id\":\"cmp_probe\",\"encrypted_content\":\"blob\"}}\n\n" + + "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_probe\",\"output\":[]}}\n\n" + func TestAccountTestService_TestAccountConnection_OpenAICompactOAuthSuccessPersistsSupport(t *testing.T) { gin.SetMode(gin.TestMode) @@ -38,8 +44,8 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactOAuthSuccessPersi } upstream := &httpUpstreamRecorder{resp: &http.Response{ StatusCode: http.StatusOK, - Header: http.Header{"Content-Type": []string{"application/json"}, "x-request-id": []string{"rid-probe"}}, - Body: io.NopCloser(strings.NewReader(`{"id":"cmp_probe","status":"completed"}`)), + Header: http.Header{"Content-Type": []string{"text/event-stream"}, "x-request-id": []string{"rid-probe"}}, + Body: io.NopCloser(strings.NewReader(compactProbeSSESuccessBody)), }} svc := &AccountTestService{ accountRepo: repo, @@ -54,17 +60,22 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactOAuthSuccessPersi err := svc.TestAccountConnection(c, account.ID, "gpt-5.4", "", AccountTestModeCompact) require.NoError(t, err) - require.Equal(t, chatgptCodexAPIURL+"/compact", upstream.lastReq.URL.String()) + // 原生 v2：探测普通 /responses 线，不再打已下线的 /responses/compact。 + require.Equal(t, chatgptCodexAPIURL, upstream.lastReq.URL.String()) require.Equal(t, "chatgpt.com", upstream.lastReq.Host) - require.Equal(t, "application/json", upstream.lastReq.Header.Get("Accept")) - require.Equal(t, codexCLIVersion, upstream.lastReq.Header.Get("Version")) + require.Equal(t, "text/event-stream", upstream.lastReq.Header.Get("Accept")) + require.Contains(t, upstream.lastReq.Header.Get("x-codex-beta-features"), "remote_compaction_v2") require.NotEmpty(t, upstream.lastReq.Header.Get("Session_Id")) require.Equal(t, HTTPUpstreamProfileOpenAI, HTTPUpstreamProfileFromContext(upstream.lastReq.Context())) require.Equal(t, codexCLIUserAgent, upstream.lastReq.Header.Get("User-Agent")) require.Equal(t, "chatgpt-acc", upstream.lastReq.Header.Get("chatgpt-account-id")) require.Equal(t, "true", upstream.lastReq.Header.Get("x-openai-fedramp")) require.Equal(t, "gpt-5.4", gjson.GetBytes(upstream.lastBody, "model").String()) - require.False(t, gjson.GetBytes(upstream.lastBody, "input.1").Exists()) + require.True(t, gjson.GetBytes(upstream.lastBody, "stream").Bool()) + require.False(t, gjson.GetBytes(upstream.lastBody, "store").Bool()) + inputItems := gjson.GetBytes(upstream.lastBody, "input").Array() + require.NotEmpty(t, inputItems) + require.Equal(t, "compaction_trigger", inputItems[len(inputItems)-1].Get("type").String()) updates := <-updateCalls require.Equal(t, true, updates["openai_compact_supported"]) @@ -116,7 +127,7 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactOAuth404MarksUnsu require.Contains(t, rec.Body.String(), `"type":"error"`) } -func TestAccountTestService_TestAccountConnection_OpenAICompactAPIKeyUsesCompactPath(t *testing.T) { +func TestAccountTestService_TestAccountConnection_OpenAICompactAPIKeyUsesNativeResponsesPath(t *testing.T) { gin.SetMode(gin.TestMode) updateCalls := make(chan map[string]any, 1) @@ -129,8 +140,10 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactAPIKeyUsesCompact Schedulable: true, Concurrency: 1, Credentials: map[string]any{ - "api_key": "sk-test", - "base_url": "https://example.com/v1", + "api_key": "sk-test", + "base_url": "https://example.com/v1", + // post-#5641：compact_model_mapping 仅作用于 legacy /responses/compact， + // 原生 v2 探测不应用它。 "compact_model_mapping": map[string]any{"gpt-5.4": "gpt-5.4-openai-compact"}, }, } @@ -140,8 +153,8 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactAPIKeyUsesCompact } upstream := &httpUpstreamRecorder{resp: &http.Response{ StatusCode: http.StatusOK, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"id":"cmp_probe_apikey","status":"completed"}`)), + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(compactProbeSSESuccessBody)), }} svc := &AccountTestService{ accountRepo: repo, @@ -156,14 +169,16 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactAPIKeyUsesCompact err := svc.TestAccountConnection(c, account.ID, "gpt-5.4", "", AccountTestModeCompact) require.NoError(t, err) - require.Equal(t, "https://example.com/v1/responses/compact", upstream.lastReq.URL.String()) + require.Equal(t, "https://example.com/v1/responses", upstream.lastReq.URL.String()) requireOpenAICodexProbeHeaders(t, upstream.lastReq.Header) - require.Equal(t, "gpt-5.4-openai-compact", gjson.GetBytes(upstream.lastBody, "model").String()) + require.Contains(t, upstream.lastReq.Header.Get("x-codex-beta-features"), "remote_compaction_v2") + require.Equal(t, "gpt-5.4", gjson.GetBytes(upstream.lastBody, "model").String(), + "原生 v2 探测不应用 compact_model_mapping") updates := <-updateCalls require.Equal(t, true, updates["openai_compact_supported"]) } -func TestAccountTestService_TestAccountConnection_OpenAICompactAPIKeyDefaultBaseURLUsesV1Path(t *testing.T) { +func TestAccountTestService_TestAccountConnection_OpenAICompactAPIKeyDefaultBaseURLUsesResponsesPath(t *testing.T) { gin.SetMode(gin.TestMode) updateCalls := make(chan map[string]any, 1) @@ -185,8 +200,8 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactAPIKeyDefaultBase } upstream := &httpUpstreamRecorder{resp: &http.Response{ StatusCode: http.StatusOK, - Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"id":"cmp_probe_apikey_default","status":"completed"}`)), + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(compactProbeSSESuccessBody)), }} svc := &AccountTestService{ accountRepo: repo, @@ -200,6 +215,112 @@ func TestAccountTestService_TestAccountConnection_OpenAICompactAPIKeyDefaultBase err := svc.TestAccountConnection(c, account.ID, "gpt-5.4", "", AccountTestModeCompact) require.NoError(t, err) - require.Equal(t, "https://api.openai.com/v1/responses/compact", upstream.lastReq.URL.String()) + require.Equal(t, "https://api.openai.com/v1/responses", upstream.lastReq.URL.String()) <-updateCalls } + +func TestAccountTestService_TestAccountConnection_OpenAICompact2xxWithoutItemMarksUnsupported(t *testing.T) { + gin.SetMode(gin.TestMode) + + updateCalls := make(chan map[string]any, 1) + account := Account{ + ID: 5, + Name: "openai-oauth-no-item", + Platform: PlatformOpenAI, + Type: AccountTypeOAuth, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Credentials: map[string]any{ + "access_token": "oauth-token", + "chatgpt_account_id": "chatgpt-acc", + }, + } + repo := &snapshotUpdateAccountRepo{ + stubOpenAIAccountRepo: stubOpenAIAccountRepo{accounts: []Account{account}}, + updateExtraCalls: updateCalls, + } + // 200 但流里没有 compaction item：链路吞掉了 compaction_trigger 的形态 + //（#5478 的 "got 0 items"），必须判定为不支持。 + noItemBody := "data: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"message\",\"id\":\"msg_1\"}}\n\n" + + "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_x\",\"output\":[]}}\n\n" + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(noItemBody)), + }} + svc := &AccountTestService{ + accountRepo: repo, + httpUpstream: upstream, + } + + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/admin/accounts/5/test", bytes.NewReader(nil)) + + err := svc.TestAccountConnection(c, account.ID, "gpt-5.4", "", AccountTestModeCompact) + require.Error(t, err) + + updates := <-updateCalls + require.Equal(t, false, updates["openai_compact_supported"]) + require.Contains(t, rec.Body.String(), `"type":"error"`) +} + +// 探测与真实转发走同一 /responses 端点，出站身份必须与真实 Codex 同构： +// session/thread 为 UUID、携带 x-codex-installation-id（收敛账号用收敛值）。 +func TestAccountTestService_TestAccountConnection_OpenAICompactProbeIdentityMatchesRealTraffic(t *testing.T) { + gin.SetMode(gin.TestMode) + + updateCalls := make(chan map[string]any, 1) + account := Account{ + ID: 6, + Name: "openai-oauth-identity", + Platform: PlatformOpenAI, + Type: AccountTypeOAuth, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Credentials: map[string]any{ + "access_token": "oauth-token", + "chatgpt_account_id": "chatgpt-acc", + }, + // 收敛是显式 opt-in（#5610），这里显式开启以验证探测身份与真实流量同构。 + Extra: map[string]any{"codex_fingerprint_mode": "session"}, + } + repo := &snapshotUpdateAccountRepo{ + stubOpenAIAccountRepo: stubOpenAIAccountRepo{accounts: []Account{account}}, + updateExtraCalls: updateCalls, + } + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(compactProbeSSESuccessBody)), + }} + svc := &AccountTestService{accountRepo: repo, httpUpstream: upstream} + + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/api/v1/admin/accounts/6/test", bytes.NewReader(nil)) + + require.NoError(t, svc.TestAccountConnection(c, account.ID, "gpt-5.4", "", AccountTestModeCompact)) + + // 显式 session 收敛模式：出站身份 = 账号级收敛值 + converged := resolveConvergedSessionID(&account) + require.Equal(t, converged, upstream.lastReq.Header.Get("session-id")) + require.Equal(t, converged, upstream.lastReq.Header.Get("session_id")) + require.Equal(t, resolveConvergedInstallationID(&account), upstream.lastReq.Header.Get("x-codex-installation-id"), + "真实 Codex 每个请求必带 installation-id，探测不得缺失") + require.NotContains(t, upstream.lastReq.Header.Get("session-id"), "probe_compact", + "探测标识不得是可被上游一眼识别的字面量") + <-updateCalls +} + +func TestCompactProbeSessionID_IsUUIDShaped(t *testing.T) { + for _, id := range []int64{0, 1, 987654} { + got := compactProbeSessionID(id) + _, err := uuid.Parse(got) + require.NoError(t, err, "探测会话标识必须是 UUID 形态: %s", got) + } + require.Equal(t, compactProbeSessionID(7), compactProbeSessionID(7), "同账号应稳定复用同一会话") + require.NotEqual(t, compactProbeSessionID(7), compactProbeSessionID(8)) +} diff --git a/backend/internal/service/custom_group_usage_rollup.go b/backend/internal/service/custom_group_usage_rollup.go new file mode 100644 index 000000000..d14dc9bfd --- /dev/null +++ b/backend/internal/service/custom_group_usage_rollup.go @@ -0,0 +1,40 @@ +package service + +import ( + "context" + "time" + + appTimezone "github.com/Wei-Shaw/sub2api/internal/pkg/timezone" +) + +const groupUsageDateFormat = "2006-01-02" + +// GroupUsageRollupRepository 是分组日汇总的可选持久化能力。 +type GroupUsageRollupRepository interface { + SyncGroupUsageRollups(ctx context.Context, todayStart time.Time) error +} + +// GroupUsageTimezoneName 返回服务端配置的时区名称。 +func GroupUsageTimezoneName() string { + return appTimezone.Location().String() +} + +// GroupUsageTodayStart 返回指定时刻在服务端配置时区内的自然日 UTC 起点。 +func GroupUsageTodayStart(at time.Time) time.Time { + return appTimezone.StartOfDay(at).UTC() +} + +// GroupUsageYesterdayStart 返回指定时刻前一个本地日历日的 UTC 起点。 +func GroupUsageYesterdayStart(at time.Time) time.Time { + return appTimezone.StartOfDay(at).AddDate(0, 0, -1).UTC() +} + +// GroupUsageDate 返回指定时刻在服务端配置时区内的日期。 +func GroupUsageDate(at time.Time) string { + return at.In(appTimezone.Location()).Format(groupUsageDateFormat) +} + +// ParseGroupUsageDate 解析服务端配置时区内的日期并返回其零点。 +func ParseGroupUsageDate(value string) (time.Time, error) { + return appTimezone.ParseInLocation(groupUsageDateFormat, value) +} diff --git a/backend/internal/service/custom_group_usage_rollup_test.go b/backend/internal/service/custom_group_usage_rollup_test.go new file mode 100644 index 000000000..35c568a45 --- /dev/null +++ b/backend/internal/service/custom_group_usage_rollup_test.go @@ -0,0 +1,50 @@ +//go:build unit + +package service + +import ( + "testing" + "time" + + appTimezone "github.com/Wei-Shaw/sub2api/internal/pkg/timezone" + "github.com/stretchr/testify/require" +) + +func useGroupUsageTestTimezone(t *testing.T, name string) { + t.Helper() + + previousName := appTimezone.Name() + require.NoError(t, appTimezone.Init(name)) + t.Cleanup(func() { require.NoError(t, appTimezone.Init(previousName)) }) +} + +func TestGroupUsageDateUsesConfiguredTimezoneBoundary(t *testing.T) { + useGroupUsageTestTimezone(t, "America/New_York") + + beforeMidnight := time.Date(2026, 3, 9, 3, 59, 59, 0, time.UTC) + atMidnight := time.Date(2026, 3, 9, 4, 0, 0, 0, time.UTC) + + require.Equal(t, "2026-03-08", GroupUsageDate(beforeMidnight)) + require.Equal(t, "2026-03-09", GroupUsageDate(atMidnight)) + require.Equal(t, atMidnight, GroupUsageTodayStart(atMidnight)) +} + +func TestGroupUsageParseDateUsesConfiguredTimezone(t *testing.T) { + useGroupUsageTestTimezone(t, "America/New_York") + + parsed, err := ParseGroupUsageDate("2026-03-08") + require.NoError(t, err) + require.Equal(t, time.Date(2026, 3, 8, 5, 0, 0, 0, time.UTC), parsed.UTC()) + require.Equal(t, "America/New_York", parsed.Location().String()) +} + +func TestGroupUsageYesterdayStartHandlesDST(t *testing.T) { + useGroupUsageTestTimezone(t, "America/New_York") + + todayStart := time.Date(2026, 3, 9, 4, 0, 0, 0, time.UTC) + yesterdayStart := GroupUsageYesterdayStart(todayStart) + + require.Equal(t, time.Date(2026, 3, 8, 5, 0, 0, 0, time.UTC), yesterdayStart) + require.Equal(t, 23*time.Hour, todayStart.Sub(yesterdayStart)) + require.Equal(t, "America/New_York", GroupUsageTimezoneName()) +} diff --git a/backend/internal/service/dashboard_aggregation_service.go b/backend/internal/service/dashboard_aggregation_service.go index 5dfa02a82..6e351d75d 100644 --- a/backend/internal/service/dashboard_aggregation_service.go +++ b/backend/internal/service/dashboard_aggregation_service.go @@ -18,12 +18,14 @@ const ( defaultDashboardAggregationBackfillTimeout = 30 * time.Minute dashboardAggregationRetentionInterval = 6 * time.Hour - // dashboardAggregationLeaderLockKey gates the periodic scheduled aggregation so - // that only one instance runs it per cycle in a multi-replica deployment. + // dashboardAggregationLeaderLockKey 保证多副本部署中每个周期只有一个实例执行聚合。 dashboardAggregationLeaderLockKey = "dashboard:aggregation:leader" - // dashboardAggregationLeaderLockTTL must exceed the job's worst-case runtime - // (defaultDashboardAggregationTimeout) so the lock never expires mid-run. + // TTL 必须覆盖 dashboard 聚合与分组日汇总两个有界阶段，避免任务中途失锁。 dashboardAggregationLeaderLockTTL = 5 * time.Minute + + // 启动回填耗时可能远长于周期聚合，因此使用独立锁并让 TTL 严格大于回填超时。 + dashboardAggregationGroupUsageBackfillLeaderLockKey = "dashboard:aggregation:group-usage-backfill:leader" + dashboardAggregationGroupUsageBackfillLeaderLockTTL = defaultDashboardAggregationBackfillTimeout + time.Minute ) var ( @@ -95,6 +97,7 @@ func (s *DashboardAggregationService) Start() { logger.LegacyPrintf("service.dashboard_aggregation", "[DashboardAggregation] 聚合作业已禁用") return } + go s.runStartupGroupUsageSync() interval := time.Duration(s.cfg.IntervalSeconds) * time.Second if interval <= 0 { @@ -229,6 +232,7 @@ func (s *DashboardAggregationService) runScheduledAggregation() { return } defer release() + defer s.runScheduledGroupUsageSync() now := time.Now().UTC() last, err := s.repo.GetAggregationWatermark(ctx) @@ -269,6 +273,35 @@ func (s *DashboardAggregationService) runScheduledAggregation() { s.maybeCleanupRetention(ctx, now) } +func (s *DashboardAggregationService) runScheduledGroupUsageSync() { + ctx, cancel := context.WithTimeout(context.Background(), defaultDashboardAggregationTimeout) + defer cancel() + if err := s.syncGroupUsageRollups(ctx, time.Now().UTC()); err != nil { + logger.LegacyPrintf("service.dashboard_aggregation", "[DashboardAggregation] 分组用量日汇总失败: %v", err) + } +} + +func (s *DashboardAggregationService) runStartupGroupUsageSync() { + ctx, cancel := context.WithTimeout(context.Background(), defaultDashboardAggregationBackfillTimeout) + defer cancel() + release, ok := tryAcquireSingletonLeaderLock(ctx, s.lockCache, s.db, dashboardAggregationGroupUsageBackfillLeaderLockKey, s.instanceID, dashboardAggregationGroupUsageBackfillLeaderLockTTL) + if !ok { + return + } + defer release() + if err := s.syncGroupUsageRollups(ctx, time.Now().UTC()); err != nil { + logger.LegacyPrintf("service.dashboard_aggregation", "[DashboardAggregation] 启动分组用量回填失败: %v", err) + } +} + +func (s *DashboardAggregationService) syncGroupUsageRollups(ctx context.Context, now time.Time) error { + repo, ok := s.repo.(GroupUsageRollupRepository) + if !ok { + return nil + } + return repo.SyncGroupUsageRollups(ctx, GroupUsageTodayStart(now)) +} + func (s *DashboardAggregationService) backfillRange(ctx context.Context, start, end time.Time) error { if !atomic.CompareAndSwapInt32(&s.running, 0, 1) { return errDashboardAggregationRunning diff --git a/backend/internal/service/dashboard_aggregation_service_test.go b/backend/internal/service/dashboard_aggregation_service_test.go index fbb671bb6..d16451933 100644 --- a/backend/internal/service/dashboard_aggregation_service_test.go +++ b/backend/internal/service/dashboard_aggregation_service_test.go @@ -24,12 +24,36 @@ type dashboardAggregationRepoTestStub struct { cleanupUsageErr error cleanupDedupErr error ensurePartitionErr error + aggregateCtx context.Context + events *[]string +} + +type dashboardAggregationRollupRepoTestStub struct { + *dashboardAggregationRepoTestStub + groupRollupCalls int + groupRollupAt time.Time + groupRollupErr error + groupRollupCtx context.Context +} + +func (s *dashboardAggregationRollupRepoTestStub) SyncGroupUsageRollups(ctx context.Context, todayStart time.Time) error { + s.groupRollupCalls++ + s.groupRollupAt = todayStart + s.groupRollupCtx = ctx + if s.events != nil { + *s.events = append(*s.events, "group_rollup") + } + return s.groupRollupErr } func (s *dashboardAggregationRepoTestStub) AggregateRange(ctx context.Context, start, end time.Time) error { s.aggregateCalls++ + s.aggregateCtx = ctx s.lastStart = start s.lastEnd = end + if s.events != nil { + *s.events = append(*s.events, "dashboard_aggregation") + } return s.aggregateErr } @@ -88,6 +112,103 @@ func TestDashboardAggregationService_RunScheduledAggregation_EpochUsesRetentionS require.Equal(t, truncateToDayUTC(repo.lastEnd.AddDate(0, 0, -1)), repo.lastStart) } +func TestDashboardAggregationService_RunScheduledAggregationSyncsGroupUsageRollups(t *testing.T) { + baseRepo := &dashboardAggregationRepoTestStub{watermark: time.Now().UTC()} + repo := &dashboardAggregationRollupRepoTestStub{dashboardAggregationRepoTestStub: baseRepo} + svc := &DashboardAggregationService{ + repo: repo, + cfg: config.DashboardAggregationConfig{ + Enabled: true, + IntervalSeconds: 60, + LookbackSeconds: 120, + Retention: config.DashboardAggregationRetentionConfig{ + UsageLogsDays: 1, + UsageBillingDedupDays: 2, + HourlyDays: 1, + DailyDays: 1, + }, + }, + } + + before := GroupUsageTodayStart(time.Now()) + svc.runScheduledAggregation() + after := GroupUsageTodayStart(time.Now()) + + require.Equal(t, 1, repo.groupRollupCalls) + require.Contains(t, []time.Time{before, after}, repo.groupRollupAt) +} + +func TestDashboardAggregationService_RunScheduledAggregationSyncsGroupAfterDashboardEarlyReturn(t *testing.T) { + events := make([]string, 0, 2) + baseRepo := &dashboardAggregationRepoTestStub{ + watermark: time.Now().UTC(), + aggregateErr: errors.New("dashboard aggregation failed"), + events: &events, + } + repo := &dashboardAggregationRollupRepoTestStub{ + dashboardAggregationRepoTestStub: baseRepo, + groupRollupErr: errors.New("group rollup failed"), + } + svc := &DashboardAggregationService{ + repo: repo, + cfg: config.DashboardAggregationConfig{ + LookbackSeconds: 120, + Retention: config.DashboardAggregationRetentionConfig{ + UsageLogsDays: 1, + }, + }, + } + + svc.runScheduledAggregation() + + require.Equal(t, []string{"dashboard_aggregation", "group_rollup"}, events) + require.NotNil(t, repo.aggregateCtx) + require.NotNil(t, repo.groupRollupCtx) + if repo.aggregateCtx == repo.groupRollupCtx { + t.Fatal("分组日汇总必须使用独立于 dashboard 聚合的 context") + } + groupDeadline, ok := repo.groupRollupCtx.Deadline() + require.True(t, ok, "group rollup context must be bounded") + require.LessOrEqual(t, time.Until(groupDeadline), defaultDashboardAggregationTimeout) +} + +type dashboardAggregationLeaderLockRecordingCache struct { + delegate *fakeLeaderLockCache + acquireKeys []string + acquireTTLs []time.Duration +} + +func (c *dashboardAggregationLeaderLockRecordingCache) TryAcquireLeaderLock(ctx context.Context, key, owner string, ttl time.Duration) (bool, error) { + c.acquireKeys = append(c.acquireKeys, key) + c.acquireTTLs = append(c.acquireTTLs, ttl) + return c.delegate.TryAcquireLeaderLock(ctx, key, owner, ttl) +} + +func (c *dashboardAggregationLeaderLockRecordingCache) ReleaseLeaderLock(ctx context.Context, key, owner string) error { + return c.delegate.ReleaseLeaderLock(ctx, key, owner) +} + +func TestDashboardAggregationService_StartupGroupSyncUsesIndependentLongLivedLeaderLock(t *testing.T) { + delegate := &fakeLeaderLockCache{} + _, err := delegate.TryAcquireLeaderLock(context.Background(), dashboardAggregationLeaderLockKey, "periodic-peer", time.Hour) + require.NoError(t, err) + cache := &dashboardAggregationLeaderLockRecordingCache{delegate: delegate} + repo := &dashboardAggregationRollupRepoTestStub{dashboardAggregationRepoTestStub: &dashboardAggregationRepoTestStub{}} + svc := &DashboardAggregationService{ + repo: repo, + lockCache: cache, + instanceID: "startup-instance", + } + + svc.runStartupGroupUsageSync() + + require.Len(t, cache.acquireKeys, 1) + require.NotEqual(t, dashboardAggregationLeaderLockKey, cache.acquireKeys[0]) + require.Len(t, cache.acquireTTLs, 1) + require.Greater(t, cache.acquireTTLs[0], defaultDashboardAggregationBackfillTimeout) + require.Equal(t, 1, repo.groupRollupCalls) +} + func TestDashboardAggregationService_CleanupRetentionFailure_DoesNotRecord(t *testing.T) { repo := &dashboardAggregationRepoTestStub{cleanupAggregatesErr: errors.New("清理失败")} svc := &DashboardAggregationService{ diff --git a/backend/internal/service/dashboard_service.go b/backend/internal/service/dashboard_service.go index d815ed6ef..3c656b61f 100644 --- a/backend/internal/service/dashboard_service.go +++ b/backend/internal/service/dashboard_service.go @@ -212,7 +212,7 @@ func (s *DashboardService) GetGroupStatsWithUsageFilters(ctx context.Context, st return s.GetGroupStatsWithFilters(ctx, startTime, endTime, filters.UserID, filters.APIKeyID, filters.AccountID, filters.GroupID, filters.RequestType, filters.Stream, filters.BillingType) } -// GetGroupUsageSummary returns today's and cumulative cost for all groups. +// GetGroupUsageSummary returns today's, yesterday's, and cumulative cost for all groups. func (s *DashboardService) GetGroupUsageSummary(ctx context.Context, todayStart time.Time) ([]usagestats.GroupUsageSummary, error) { results, err := s.usageRepo.GetAllGroupUsageSummary(ctx, todayStart) if err != nil { diff --git a/backend/internal/service/openai_account_scheduler.go b/backend/internal/service/openai_account_scheduler.go index 154dd1dc6..6b4513753 100644 --- a/backend/internal/service/openai_account_scheduler.go +++ b/backend/internal/service/openai_account_scheduler.go @@ -82,8 +82,10 @@ type OpenAIAccountScheduleRequest struct { RequiredTransport OpenAIUpstreamTransport RequiredCapability OpenAIEndpointCapability RequiredImageCapability OpenAIImagesCapability - RequireCompact bool - ExcludedIDs map[int64]struct{} + // RequireCompact is only for legacy /responses/compact capability filtering + // and compact_model_mapping; native remote compaction v2 leaves it false. + RequireCompact bool + ExcludedIDs map[int64]struct{} } type OpenAIAccountScheduleDecision struct { @@ -2154,11 +2156,9 @@ func (s *OpenAIGatewayService) selectAccountWithSchedulerOnce( // 入口已在请求开始经 WithOpenAIRequestPricingContext 装门并固定 pricingAt， // 此处对同分组门直接复用（failover 重入阈值稳定），仅为不经 handler 装配的 // 内部调用兜底。图片/视频调度不在利润门范围：requiredImageCapability 非空的 - // Images 调度不装门；requiredCapability == OpenAIEndpointCapabilityResponses - // 当前仅显式生图意图的 /v1/responses 设置（HTTP openAIResponsesRequiredCapability - // 与 WS 桥同款判定），同样不装门——若未来把该 capability 用于非生图流量， - // 需要同步收窄本条件（有测试钉死该映射）。 - if requiredImageCapability == "" && requiredCapability != OpenAIEndpointCapabilityResponses { + // Images 调度不装门；其他使用 Responses 能力的文本请求（包括原生远程压缩） + // 仍须装门。其余媒体路径通过 WithOpenAIProfitControlSuppressed 显式跳过。 + if requiredImageCapability == "" { ctx = s.withOpenAIProfitControlGate(ctx, groupID) } platform = normalizeOpenAICompatiblePlatform(platform) diff --git a/backend/internal/service/openai_account_scheduler_compact_test.go b/backend/internal/service/openai_account_scheduler_compact_test.go index b98760038..87a5b2281 100644 --- a/backend/internal/service/openai_account_scheduler_compact_test.go +++ b/backend/internal/service/openai_account_scheduler_compact_test.go @@ -116,6 +116,262 @@ func TestOpenAIGatewayService_SelectAccountWithScheduler_CompactRejectsExplicitl require.Nil(t, selection) } +func newOpenAICompactionSchedulerTestService(accounts []Account, advanced bool) *OpenAIGatewayService { + cfg := &config.Config{} + cfg.Gateway.Scheduling.LoadBatchEnabled = false + svc := &OpenAIGatewayService{ + accountRepo: schedulerTestOpenAIAccountRepo{accounts: accounts}, + cache: &schedulerTestGatewayCache{}, + cfg: cfg, + concurrencyService: NewConcurrencyService(schedulerTestConcurrencyCache{}), + } + if advanced { + svc.rateLimitService = newOpenAIAdvancedSchedulerRateLimitService("true") + } + return svc +} + +func selectOpenAICompactionSchedulerTestAccount(t *testing.T, svc *OpenAIGatewayService, groupID int64, requireCompact bool) (*AccountSelectionResult, error) { + t.Helper() + selection, _, err := svc.SelectAccountWithSchedulerForCapability( + context.Background(), + &groupID, + "", + "", + "gpt-5.6-sol", + nil, + OpenAIUpstreamTransportAny, + OpenAIEndpointCapabilityResponses, + requireCompact, + false, + false, + ) + return selection, err +} + +func TestOpenAIGatewayService_SelectAccountWithScheduler_NativeCompactionIgnoresLegacyCompactProbe(t *testing.T) { + for _, advanced := range []bool{false, true} { + t.Run(map[bool]string{false: "legacy_scheduler", true: "advanced_scheduler"}[advanced], func(t *testing.T) { + resetOpenAIAdvancedSchedulerSettingCacheForTest() + svc := newOpenAICompactionSchedulerTestService([]Account{{ + ID: 71012, + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Extra: map[string]any{ + "openai_compact_supported": false, + "openai_responses_supported": true, + }, + }}, advanced) + + selection, err := selectOpenAICompactionSchedulerTestAccount(t, svc, 91007, false) + require.NoError(t, err) + require.NotNil(t, selection) + require.Equal(t, int64(71012), selection.Account.ID) + }) + } +} + +func TestOpenAIGatewayService_SelectAccountWithScheduler_NativeCompactionAllowsForceOff(t *testing.T) { + for _, advanced := range []bool{false, true} { + t.Run(map[bool]string{false: "legacy_scheduler", true: "advanced_scheduler"}[advanced], func(t *testing.T) { + resetOpenAIAdvancedSchedulerSettingCacheForTest() + svc := newOpenAICompactionSchedulerTestService([]Account{{ + ID: 71013, + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Extra: map[string]any{ + "openai_compact_mode": OpenAICompactModeForceOff, + "openai_responses_supported": true, + }, + }}, advanced) + + selection, err := selectOpenAICompactionSchedulerTestAccount(t, svc, 91008, false) + require.NoError(t, err) + require.NotNil(t, selection) + require.Equal(t, int64(71013), selection.Account.ID) + }) + } +} + +func TestOpenAIGatewayService_SelectAccountWithScheduler_NativeCompactionRequiresResponsesCapability(t *testing.T) { + for _, advanced := range []bool{false, true} { + t.Run(map[bool]string{false: "legacy_scheduler", true: "advanced_scheduler"}[advanced], func(t *testing.T) { + resetOpenAIAdvancedSchedulerSettingCacheForTest() + svc := newOpenAICompactionSchedulerTestService([]Account{{ + ID: 71014, + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Extra: map[string]any{ + "openai_compact_mode": OpenAICompactModeForceOn, + "openai_responses_supported": false, + }, + }}, advanced) + + selection, err := selectOpenAICompactionSchedulerTestAccount(t, svc, 91009, false) + require.ErrorIs(t, err, ErrNoAvailableAccounts) + require.NotErrorIs(t, err, ErrNoAvailableCompactAccounts) + require.NotContains(t, err.Error(), "/responses/compact") + require.Nil(t, selection) + }) + } +} + +func TestOpenAIGatewayService_SelectAccountWithScheduler_LegacyCompactionKeepsCompactEligibility(t *testing.T) { + for _, advanced := range []bool{false, true} { + t.Run(map[bool]string{false: "legacy_scheduler", true: "advanced_scheduler"}[advanced], func(t *testing.T) { + resetOpenAIAdvancedSchedulerSettingCacheForTest() + svc := newOpenAICompactionSchedulerTestService([]Account{ + { + ID: 71015, + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Extra: map[string]any{ + "openai_compact_supported": false, + "openai_responses_supported": true, + }, + }, + { + ID: 71016, + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Extra: map[string]any{ + "openai_compact_mode": OpenAICompactModeForceOff, + "openai_responses_supported": true, + }, + }, + }, advanced) + + selection, err := selectOpenAICompactionSchedulerTestAccount(t, svc, 91010, true) + require.ErrorIs(t, err, ErrNoAvailableCompactAccounts) + require.Contains(t, err.Error(), "/responses/compact") + require.Nil(t, selection) + }) + } +} + +// TestOpenAIGatewayService_SelectAccountWithScheduler_CompactRequiresResponsesCapability +// prevents a confirmed Chat Completions-only API key from silently dropping the +// compaction trigger through the Responses-to-Chat fallback. +func TestOpenAIGatewayService_SelectAccountWithScheduler_CompactRequiresResponsesCapability(t *testing.T) { + resetOpenAIAdvancedSchedulerSettingCacheForTest() + + ctx := context.Background() + groupID := int64(91005) + accounts := []Account{{ + ID: 71050, + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Extra: map[string]any{ + "openai_compact_supported": true, + "openai_responses_supported": false, + }, + }} + cfg := &config.Config{} + cfg.Gateway.Scheduling.LoadBatchEnabled = false + svc := &OpenAIGatewayService{ + accountRepo: schedulerTestOpenAIAccountRepo{accounts: accounts}, + cache: &schedulerTestGatewayCache{}, + cfg: cfg, + concurrencyService: NewConcurrencyService(schedulerTestConcurrencyCache{}), + } + + selection, _, err := svc.SelectAccountWithSchedulerForCapability( + ctx, + &groupID, + "", + "", + "gpt-5.4", + nil, + OpenAIUpstreamTransportAny, + OpenAIEndpointCapabilityResponses, + true, + false, + false, + ) + require.Error(t, err) + require.True(t, errors.Is(err, ErrNoAvailableAccounts)) + require.Nil(t, selection) +} + +func TestOpenAIGatewayService_SelectAccountWithScheduler_CompactSkipsChatOnlyAccount(t *testing.T) { + resetOpenAIAdvancedSchedulerSettingCacheForTest() + + ctx := context.Background() + groupID := int64(91006) + accounts := []Account{ + { + ID: 71060, + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Priority: 10, + Extra: map[string]any{ + "openai_compact_supported": true, + "openai_responses_supported": false, + }, + }, + { + ID: 71061, + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Status: StatusActive, + Schedulable: true, + Concurrency: 1, + Priority: 0, + Extra: map[string]any{ + "openai_compact_supported": true, + "openai_responses_supported": true, + }, + }, + } + cfg := &config.Config{} + cfg.Gateway.Scheduling.LoadBatchEnabled = false + svc := &OpenAIGatewayService{ + accountRepo: schedulerTestOpenAIAccountRepo{accounts: accounts}, + cache: &schedulerTestGatewayCache{}, + cfg: cfg, + concurrencyService: NewConcurrencyService(schedulerTestConcurrencyCache{}), + } + + selection, _, err := svc.SelectAccountWithSchedulerForCapability( + ctx, + &groupID, + "", + "", + "gpt-5.6-sol", + nil, + OpenAIUpstreamTransportAny, + OpenAIEndpointCapabilityResponses, + true, + false, + false, + ) + require.NoError(t, err) + require.NotNil(t, selection) + require.NotNil(t, selection.Account) + require.Equal(t, int64(71061), selection.Account.ID) +} + // TestOpenAIGatewayService_SelectAccountWithScheduler_CompactFallsBackToUnknown // 验证当没有"已知支持"账号时，compact 请求会回退到"未探测"账号。 func TestOpenAIGatewayService_SelectAccountWithScheduler_CompactFallsBackToUnknown(t *testing.T) { diff --git a/backend/internal/service/openai_agent_identity_compat_test.go b/backend/internal/service/openai_agent_identity_compat_test.go index 5dad35887..a8f234f58 100644 --- a/backend/internal/service/openai_agent_identity_compat_test.go +++ b/backend/internal/service/openai_agent_identity_compat_test.go @@ -40,7 +40,7 @@ func TestAccountTestServiceOpenAICompactAgentIdentityUsesFreshAssertion(t *testi upstream := &httpUpstreamRecorder{resp: &http.Response{ StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, - Body: io.NopCloser(strings.NewReader(`{"id":"compact-agent","status":"completed"}`)), + Body: io.NopCloser(strings.NewReader(`{"id":"compact-agent","status":"completed","output":[{"type":"compaction","id":"cmp_agent_fresh","encrypted_content":"blob"}]}`)), }} svc := &AccountTestService{accountRepo: repo, httpUpstream: upstream} @@ -87,7 +87,7 @@ func TestAccountTestServiceOpenAICompactAgentIdentityRecoversInvalidTaskOnce(t * upstream := &httpUpstreamRecorder{responses: []*http.Response{ {StatusCode: http.StatusUnauthorized, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"error":{"code":"invalid_task_id"}}`))}, - {StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"id":"compact-agent","status":"completed"}`))}, + {StatusCode: http.StatusOK, Header: http.Header{"Content-Type": []string{"application/json"}}, Body: io.NopCloser(strings.NewReader(`{"id":"compact-agent","status":"completed","output":[{"type":"compaction","id":"cmp_agent","encrypted_content":"blob"}]}`))}, }} invalidator := &agentIdentityWSInvalidationRecorder{} svc := &AccountTestService{accountRepo: repo, httpUpstream: upstream, agentIdentityWS: invalidator} diff --git a/backend/internal/service/openai_channel_restriction_test.go b/backend/internal/service/openai_channel_restriction_test.go index c9dbceabe..098f053c3 100644 --- a/backend/internal/service/openai_channel_restriction_test.go +++ b/backend/internal/service/openai_channel_restriction_test.go @@ -86,6 +86,226 @@ func TestOpenAISelectAccountForModelWithExclusions_UpstreamRestrictionSkipsDisal require.Equal(t, int64(2), account.ID) } +func TestIsUpstreamModelRestrictedByChannel_CompactMappingMatchesForwardPath(t *testing.T) { + t.Parallel() + + account := &Account{ + Platform: PlatformOpenAI, + Credentials: map[string]any{ + "model_mapping": map[string]any{"gpt-5.4-channel": "gpt-5.4-account"}, + "compact_model_mapping": map[string]any{"gpt-5.4-account": "gpt-5.4-compact"}, + }, + } + tests := []struct { + name string + allowedUpstreamModel string + useCompactModelMapping bool + }{ + { + name: "legacy compact applies compact mapping after channel and account mapping", + allowedUpstreamModel: "gpt-5.4-compact", + useCompactModelMapping: true, + }, + { + name: "native v2 stops after channel and account mapping", + allowedUpstreamModel: "gpt-5.4-account", + useCompactModelMapping: false, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + channelSvc := newTestChannelService(makeStandardRepo(Channel{ + ID: 1, + Status: StatusActive, + GroupIDs: []int64{10}, + RestrictModels: true, + BillingModelSource: BillingModelSourceUpstream, + ModelPricing: []ChannelModelPricing{ + {Platform: PlatformOpenAI, Models: []string{tt.allowedUpstreamModel}}, + }, + ModelMapping: map[string]map[string]string{ + PlatformOpenAI: {"gpt-5.4": "gpt-5.4-channel"}, + }, + }, map[int64]string{10: PlatformOpenAI})) + svc := &OpenAIGatewayService{channelService: channelSvc} + mapping := channelSvc.ResolveChannelMapping(context.Background(), 10, "gpt-5.4") + require.True(t, mapping.Mapped) + require.Equal(t, "gpt-5.4-channel", mapping.MappedModel) + + ctx := WithOpenAIForwardModel( + context.Background(), + mapping.MappedModel, + tt.useCompactModelMapping, + ) + require.False(t, svc.isUpstreamModelRestrictedByChannel( + ctx, 10, account, "gpt-5.4", true, + )) + require.True(t, svc.isUpstreamModelRestrictedByChannel( + context.Background(), 10, account, "gpt-5.4", true, + ), "without the forward-model context the restriction check follows a different chain") + }) + } +} + +func TestIsUpstreamModelRestrictedByChannel_PassthroughMatchesForwardPath(t *testing.T) { + t.Parallel() + + account := &Account{ + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "model_mapping": map[string]any{"gpt-5.4-channel": "gpt-5.4-account"}, + "compact_model_mapping": map[string]any{ + "gpt-5.4-channel": "gpt-5.4-compact", + }, + }, + Extra: map[string]any{"openai_passthrough": true}, + } + tests := []struct { + name string + allowedUpstreamModel string + useCompactModelMapping bool + }{ + { + name: "native v2 keeps channel-mapped model and ignores normal account mapping", + allowedUpstreamModel: "gpt-5.4-channel", + useCompactModelMapping: false, + }, + { + name: "legacy compact applies compact mapping to channel-mapped model", + allowedUpstreamModel: "gpt-5.4-compact", + useCompactModelMapping: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + channelSvc := newTestChannelService(makeStandardRepo(Channel{ + ID: 1, + Status: StatusActive, + GroupIDs: []int64{10}, + RestrictModels: true, + BillingModelSource: BillingModelSourceUpstream, + ModelPricing: []ChannelModelPricing{ + {Platform: PlatformOpenAI, Models: []string{tt.allowedUpstreamModel}}, + }, + ModelMapping: map[string]map[string]string{ + PlatformOpenAI: {"gpt-5.4": "gpt-5.4-channel"}, + }, + }, map[int64]string{10: PlatformOpenAI})) + svc := &OpenAIGatewayService{channelService: channelSvc} + mapping := channelSvc.ResolveChannelMapping(context.Background(), 10, "gpt-5.4") + require.True(t, mapping.Mapped) + require.Equal(t, "gpt-5.4-channel", mapping.MappedModel) + + ctx := WithOpenAIForwardModel( + context.Background(), + mapping.MappedModel, + tt.useCompactModelMapping, + ) + require.False(t, svc.isUpstreamModelRestrictedByChannel( + ctx, 10, account, "gpt-5.4", true, + )) + }) + } +} + +func TestIsUpstreamModelRestrictedByChannel_PassthroughFlagWithRawChatFallbackMatchesForwardPath(t *testing.T) { + t.Parallel() + + account := &Account{ + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "model_mapping": map[string]any{"gpt-5.4-channel": "gpt-5.4-account"}, + "compact_model_mapping": map[string]any{ + "gpt-5.4-account": "gpt-5.4-compact", + }, + }, + Extra: map[string]any{ + "openai_passthrough": true, + "openai_responses_supported": false, + }, + } + + for _, useCompactModelMapping := range []bool{false, true} { + useCompactModelMapping := useCompactModelMapping + name := "native v2" + if useCompactModelMapping { + name = "legacy compact" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + + channelSvc := newTestChannelService(makeStandardRepo(Channel{ + ID: 1, + Status: StatusActive, + GroupIDs: []int64{10}, + RestrictModels: true, + BillingModelSource: BillingModelSourceUpstream, + ModelPricing: []ChannelModelPricing{ + {Platform: PlatformOpenAI, Models: []string{"gpt-5.4-account"}}, + }, + ModelMapping: map[string]map[string]string{ + PlatformOpenAI: {"gpt-5.4": "gpt-5.4-channel"}, + }, + }, map[int64]string{10: PlatformOpenAI})) + svc := &OpenAIGatewayService{channelService: channelSvc} + ctx := WithOpenAIForwardModel( + context.Background(), + "gpt-5.4-channel", + useCompactModelMapping, + ) + + require.False(t, svc.isUpstreamModelRestrictedByChannel( + ctx, 10, account, "gpt-5.4", true, + )) + }) + } +} + +func TestIsUpstreamModelRestrictedByChannel_ForwardModelContextMatchesNormalForwardPath(t *testing.T) { + t.Parallel() + + account := &Account{ + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Credentials: map[string]any{ + "model_mapping": map[string]any{"gpt-5.4-channel": "gpt-5.4-account"}, + }, + Extra: map[string]any{ + "openai_passthrough": true, + "openai_responses_supported": false, + }, + } + channelSvc := newTestChannelService(makeStandardRepo(Channel{ + ID: 1, + Status: StatusActive, + GroupIDs: []int64{10}, + RestrictModels: true, + BillingModelSource: BillingModelSourceUpstream, + ModelPricing: []ChannelModelPricing{ + {Platform: PlatformOpenAI, Models: []string{"gpt-5.4-account"}}, + }, + ModelMapping: map[string]map[string]string{ + PlatformOpenAI: {"gpt-5.4": "gpt-5.4-channel"}, + }, + }, map[int64]string{10: PlatformOpenAI})) + svc := &OpenAIGatewayService{channelService: channelSvc} + ctx := WithOpenAIForwardModel(context.Background(), "gpt-5.4-channel", false) + + require.False(t, svc.isUpstreamModelRestrictedByChannel( + ctx, 10, account, "gpt-5.4", false, + )) +} + func TestOpenAISelectAccountForModelWithExclusions_StickyRestrictedUpstreamFallsBack(t *testing.T) { t.Parallel() diff --git a/backend/internal/service/openai_codex_fingerprint.go b/backend/internal/service/openai_codex_fingerprint.go index 55564c534..c74669c00 100644 --- a/backend/internal/service/openai_codex_fingerprint.go +++ b/backend/internal/service/openai_codex_fingerprint.go @@ -9,9 +9,43 @@ import ( "strings" "time" + "github.com/gin-gonic/gin" "github.com/google/uuid" + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" ) +// codexFingerprintIDsContextKey 是暂存在 gin context 的收敛 ID 集合键。 +// 由 Forward（非透传）或 forwardOpenAIPassthrough（透传）解析后写入，请求 +// 构造器读取用于出站头改写——请求体与出站头必须共享同一份 IDs，保证 +// turn_id 等随机字段一致。 +const codexFingerprintIDsContextKey = "codex_fingerprint_ids" + +// stageCodexFingerprintIDs 将本 attempt 解析出的收敛 ID 暂存到 gin context。 +// 必须无条件覆写（含 nil）：failover 从收敛账号切到 off 账号时，上一账号的 +// IDs 不得残留并被误应用到新账号的出站头（typed-nil 由应用侧 nil 守卫吸收）。 +func stageCodexFingerprintIDs(c *gin.Context, ids *codexFingerprintIDs) { + if c != nil { + c.Set(codexFingerprintIDsContextKey, ids) + } +} + +// applyStagedCodexFingerprintHeaders 读取 context 暂存的收敛 ID 并改写出站头。 +// 非透传与透传两个请求构造器共用本函数，防止应用语义漂移。仅 OAuth 账号 +// 生效（stale 键在账号类型混合 failover 下由该门挡住）。 +func applyStagedCodexFingerprintHeaders(c *gin.Context, account *Account, h http.Header) { + if c == nil || account == nil || account.Type != AccountTypeOAuth { + return + } + value, ok := c.Get(codexFingerprintIDsContextKey) + if !ok { + return + } + if ids, ok := value.(*codexFingerprintIDs); ok { + applyCodexFingerprintHeaders(h, ids) + } +} + // codexFingerprintMode 控制 OAuth 账号出站请求的设备指纹收敛强度。 // 多人共享同一 OAuth 账号时，每个用户的 Codex 客户端会携带各自不同的 // installation_id / session_id / thread_id，上游据此判定设备数和会话数。 @@ -19,7 +53,8 @@ import ( type codexFingerprintMode string const ( - // codexFingerprintOff 不做任何收敛，原样透传客户端标识（默认行为）。 + // codexFingerprintOff 不做任何收敛，原样透传客户端标识。 + // 这是默认值：收敛是显式 opt-in 的（见 GetCodexFingerprintMode）。 codexFingerprintOff codexFingerprintMode = "off" // codexFingerprintDevice 仅收敛 installation_id 为账号级恒定值。 // 上游看到 1 台设备 + 多会话（每用户各自的 session）。 @@ -36,7 +71,16 @@ const ( const codexFingerprintModeExtraKey = "codex_fingerprint_mode" // GetCodexFingerprintMode 从账号 extra JSON 读取指纹收敛模式。 -// 未设置时默认 session（设备+会话收敛），显式设为 "off" 才关闭。 +// +// **收敛是显式 opt-in**：未设置、空值或非法值一律按 off 处理，只有管理员 +// 明确配置 device / session / full 才收敛。 +// +// 历史：v0.1.175（#5553）把缺省值当作 session，导致升级后存量 OAuth 账号 +// （普遍没有这个 extra 键）的每个非透传请求都被静默改写 installation / +// session / thread / turn / window 五类标识；#5555、#5556、#5582 报告的额度 +// 缩水都卡在该版本边界，并有"回退 v0.1.173 即恢复"与"新账号开收敛后降额" +// 的 A/B 实测。上游的配额判定策略不可观测，因此这里取兼容安全的一侧： +// 不显式 opt-in 就保持 v0.1.175 之前的客户端身份（#5610）。 func (a *Account) GetCodexFingerprintMode() codexFingerprintMode { if a == nil || !a.IsOpenAIOAuth() { return codexFingerprintOff @@ -46,7 +90,7 @@ func (a *Account) GetCodexFingerprintMode() codexFingerprintMode { case codexFingerprintOff, codexFingerprintDevice, codexFingerprintSession, codexFingerprintFull: return codexFingerprintMode(raw) default: - return codexFingerprintSession + return codexFingerprintOff } } @@ -245,6 +289,21 @@ func applyCodexFingerprintClientMetadata(reqBody map[string]any, ids *codexFinge existing = make(map[string]any) } + if !applyCodexFingerprintToClientMetadataMap(existing, ids) { + return false + } + reqBody["client_metadata"] = existing + return true +} + +// applyCodexFingerprintToClientMetadataMap 是 client_metadata 改写的共享核心， +// map 版（非透传，body 已解码）与 raw 字节版（透传热路径）都经由它，保证两条 +// 路径的收敛语义永不漂移。 +func applyCodexFingerprintToClientMetadataMap(existing map[string]any, ids *codexFingerprintIDs) bool { + if existing == nil || ids == nil { + return false + } + modified := false if ids.installationID != "" { @@ -256,9 +315,6 @@ func applyCodexFingerprintClientMetadata(reqBody map[string]any, ids *codexFinge rewriteClientMetadataEmbeddedTurnMetadata(existing, map[string]any{ "installation_id": ids.installationID, }) - if modified { - reqBody["client_metadata"] = existing - } return modified } @@ -276,11 +332,47 @@ func applyCodexFingerprintClientMetadata(reqBody map[string]any, ids *codexFinge "window_id": ids.windowID, "turn_started_at_unix_ms": time.Now().UnixMilli(), }) - - reqBody["client_metadata"] = existing return true } +// applyCodexFingerprintClientMetadataRaw 在原始 JSON 字节上改写 client_metadata， +// 供透传路径使用——透传是热路径，禁止对可能高达数十 MB 的 body 做全量 +// Unmarshal（见 forwardOpenAIPassthrough 的轻量提取注释）。实现为：gjson 提取 +// client_metadata 小对象单独解码，经共享核心改写后 sjson 一次性拼回，body +// 其余字节原样保留。语义与 applyCodexFingerprintClientMetadata 逐点一致 +// （含"非对象值整体替换为收敛集合"的行为）。 +func applyCodexFingerprintClientMetadataRaw(body []byte, ids *codexFingerprintIDs) ([]byte, bool, error) { + if len(body) == 0 || ids == nil { + return body, false, nil + } + // 非 JSON 对象的 body（数组/标量/畸形）没有 client_metadata 语义， + // sjson 在这类根上写字段会改写整体结构，直接放行保持原样。 + if !gjson.ParseBytes(body).IsObject() { + return body, false, nil + } + + existing := map[string]any{} + if cm := gjson.GetBytes(body, "client_metadata"); cm.IsObject() { + if err := json.Unmarshal([]byte(cm.Raw), &existing); err != nil { + return body, false, fmt.Errorf("decode client_metadata for fingerprint: %w", err) + } + } + + if !applyCodexFingerprintToClientMetadataMap(existing, ids) { + return body, false, nil + } + + raw, err := json.Marshal(existing) + if err != nil { + return body, false, fmt.Errorf("encode converged client_metadata: %w", err) + } + next, err := sjson.SetRawBytes(body, "client_metadata", raw) + if err != nil { + return body, false, fmt.Errorf("splice converged client_metadata: %w", err) + } + return next, true, nil +} + // rewriteClientMetadataEmbeddedTurnMetadata 改写 client_metadata 中内嵌的 // x-codex-turn-metadata JSON 字符串里的指定字段。 func rewriteClientMetadataEmbeddedTurnMetadata(clientMetadata map[string]any, fields map[string]any) { diff --git a/backend/internal/service/openai_codex_fingerprint_test.go b/backend/internal/service/openai_codex_fingerprint_test.go index 039f843c2..8e74bfa8b 100644 --- a/backend/internal/service/openai_codex_fingerprint_test.go +++ b/backend/internal/service/openai_codex_fingerprint_test.go @@ -1,10 +1,13 @@ package service import ( + "context" "encoding/json" "net/http" + "net/http/httptest" "testing" + "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -51,9 +54,11 @@ func TestGetCodexFingerprintMode(t *testing.T) { }{ {"nil 账号", nil, codexFingerprintOff}, {"非 OAuth 账号", &Account{Platform: PlatformOpenAI, Type: "api_key"}, codexFingerprintOff}, - {"无 extra 默认 session", newTestOAuthAccount(1, nil), codexFingerprintSession}, - {"空值默认 session", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: ""}), codexFingerprintSession}, - {"非法值默认 session", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "invalid"}), codexFingerprintSession}, + // 收敛是显式 opt-in：缺省/空/非法一律 off（#5610）。存量账号普遍没有这个 + // extra 键，升级不得把它们静默切进收敛。 + {"无 extra 默认 off", newTestOAuthAccount(1, nil), codexFingerprintOff}, + {"空值默认 off", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: ""}), codexFingerprintOff}, + {"非法值默认 off", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "invalid"}), codexFingerprintOff}, {"显式 off", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "off"}), codexFingerprintOff}, {"device", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "device"}), codexFingerprintDevice}, {"session", newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: "session"}), codexFingerprintSession}, @@ -116,13 +121,24 @@ func TestResolveCodexFingerprintIDsFromRequest_ExplicitOff(t *testing.T) { assert.Nil(t, ids, "显式 off 模式应返回 nil") } -func TestResolveCodexFingerprintIDsFromRequest_DefaultIsSession(t *testing.T) { +// 未显式配置的存量账号不得被收敛（#5610）：默认返回 nil，出站身份保持 +// v0.1.175 之前的客户端原值。 +func TestResolveCodexFingerprintIDsFromRequest_DefaultIsOff(t *testing.T) { account := newTestOAuthAccount(1, nil) - ids := resolveCodexFingerprintIDsFromRequest(account, nil) - require.NotNil(t, ids, "无 extra 默认 session 模式，应返回非 nil") - assert.Equal(t, codexFingerprintSession, ids.mode) - assert.NotEmpty(t, ids.sessionID) - assert.NotEmpty(t, ids.turnID) + assert.Nil(t, resolveCodexFingerprintIDsFromRequest(account, nil), "无 extra 应视为 off") +} + +// 管理员显式 opt-in 的账号行为不变。 +func TestResolveCodexFingerprintIDsFromRequest_ExplicitOptInHonored(t *testing.T) { + for _, mode := range []string{"device", "session", "full"} { + t.Run(mode, func(t *testing.T) { + account := newTestOAuthAccount(1, map[string]any{codexFingerprintModeExtraKey: mode}) + ids := resolveCodexFingerprintIDsFromRequest(account, nil) + require.NotNil(t, ids, "显式配置必须生效") + assert.Equal(t, codexFingerprintMode(mode), ids.mode) + assert.NotEmpty(t, ids.installationID) + }) + } } // --- applyCodexFingerprintHeaders: off 模式 --- @@ -458,3 +474,188 @@ func TestExtractClientSessionID(t *testing.T) { }) } } + +// --- 透传路径：raw 字节版 client_metadata 改写 --- + +// rawVsMapClientMetadata 用同一份 ids 分别跑 map 版与 raw 字节版， +// 返回两侧最终的 client_metadata 解码结果。 +func rawVsMapClientMetadata(t *testing.T, body []byte, ids *codexFingerprintIDs) (map[string]any, map[string]any) { + t.Helper() + + var decoded map[string]any + require.NoError(t, json.Unmarshal(body, &decoded)) + applyCodexFingerprintClientMetadata(decoded, ids) + mapCM, _ := decoded["client_metadata"].(map[string]any) + + rawBody, changed, err := applyCodexFingerprintClientMetadataRaw(body, ids) + require.NoError(t, err) + require.True(t, changed) + var rawDecoded map[string]any + require.NoError(t, json.Unmarshal(rawBody, &rawDecoded)) + rawCM, _ := rawDecoded["client_metadata"].(map[string]any) + return mapCM, rawCM +} + +func TestApplyCodexFingerprintClientMetadataRaw_MatchesMapVariant(t *testing.T) { + embedded := `{\"installation_id\":\"real-install\",\"session_id\":\"real-session\",\"sandbox\":\"seatbelt\"}` + bodies := map[string]string{ + "no_client_metadata": `{"model":"gpt-5.6-sol","input":[],"stream":true}`, + "object_with_extras": `{"model":"gpt-5.6-sol","client_metadata":{"session_id":"client-session","traceparent":"00-abc-def-01","x-codex-turn-metadata":"` + embedded + `"},"stream":true}`, + "non_object_value": `{"model":"gpt-5.6-sol","client_metadata":"bogus","stream":true}`, + } + for _, mode := range []codexFingerprintMode{codexFingerprintDevice, codexFingerprintSession, codexFingerprintFull} { + account := newTestOAuthAccount(4242, nil) + ids := resolveCodexFingerprintIDs(account, "client-sess-raw", mode) + require.NotNil(t, ids) + for name, body := range bodies { + t.Run(string(mode)+"/"+name, func(t *testing.T) { + mapCM, rawCM := rawVsMapClientMetadata(t, []byte(body), ids) + assert.Equal(t, mapCM, rawCM, "raw 字节版与 map 版的 client_metadata 结果必须逐点一致") + }) + } + } +} + +func TestApplyCodexFingerprintClientMetadataRaw_PreservesUnrelatedFields(t *testing.T) { + account := newTestOAuthAccount(4243, nil) + ids := resolveCodexFingerprintIDs(account, "client-sess-preserve", codexFingerprintSession) + require.NotNil(t, ids) + + body := []byte(`{"model":"gpt-5.6-sol","input":[{"type":"message","role":"user","content":"hi"}],"stream":true,"prompt_cache_key":"pck-1"}`) + out, changed, err := applyCodexFingerprintClientMetadataRaw(body, ids) + require.NoError(t, err) + require.True(t, changed) + + var decoded map[string]any + require.NoError(t, json.Unmarshal(out, &decoded)) + assert.Equal(t, "gpt-5.6-sol", decoded["model"]) + assert.Equal(t, "pck-1", decoded["prompt_cache_key"]) + assert.Equal(t, true, decoded["stream"]) + cm, _ := decoded["client_metadata"].(map[string]any) + require.NotNil(t, cm) + assert.Equal(t, ids.sessionID, cm["session_id"]) + assert.Equal(t, ids.turnID, cm["turn_id"]) +} + +func TestApplyCodexFingerprintClientMetadataRaw_Noop(t *testing.T) { + body := []byte(`{"model":"gpt-5.6-sol"}`) + out, changed, err := applyCodexFingerprintClientMetadataRaw(body, nil) + require.NoError(t, err) + assert.False(t, changed) + assert.Equal(t, body, out) + + out, changed, err = applyCodexFingerprintClientMetadataRaw(nil, &codexFingerprintIDs{mode: codexFingerprintSession, installationID: "x"}) + require.NoError(t, err) + assert.False(t, changed) + assert.Nil(t, out) +} + +// --- context 暂存与出站头应用（透传/非透传共用 seam）--- + +func newFingerprintStageTestContext(t *testing.T) *gin.Context { + t.Helper() + gin.SetMode(gin.TestMode) + c, _ := gin.CreateTestContext(httptest.NewRecorder()) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + return c +} + +func TestStageCodexFingerprintIDs_NilOverwritesPreviousAccount(t *testing.T) { + c := newFingerprintStageTestContext(t) + accountA := newTestOAuthAccount(1001, nil) + idsA := resolveCodexFingerprintIDs(accountA, "sess-x", codexFingerprintSession) + require.NotNil(t, idsA) + stageCodexFingerprintIDs(c, idsA) + + // failover 切到 off 模式账号：无条件覆写为 nil，上一账号 IDs 不得残留 + stageCodexFingerprintIDs(c, nil) + + h := http.Header{} + h.Set("session_id", "isolated-session") + accountB := newTestOAuthAccount(1002, map[string]any{"codex_fingerprint_mode": "off"}) + applyStagedCodexFingerprintHeaders(c, accountB, h) + assert.Equal(t, "isolated-session", h.Get("session_id"), "off 账号不得应用上一账号的收敛 ID") + assert.Empty(t, h.Get("x-codex-installation-id")) +} + +func TestApplyStagedCodexFingerprintHeaders_SkipsNonOAuthAccount(t *testing.T) { + c := newFingerprintStageTestContext(t) + oauthIDs := resolveCodexFingerprintIDs(newTestOAuthAccount(1003, nil), "sess-y", codexFingerprintSession) + require.NotNil(t, oauthIDs) + stageCodexFingerprintIDs(c, oauthIDs) + + h := http.Header{} + apiKeyAccount := &Account{ID: 1004, Platform: PlatformOpenAI, Type: AccountTypeAPIKey} + applyStagedCodexFingerprintHeaders(c, apiKeyAccount, h) + assert.Empty(t, h.Get("x-codex-installation-id"), "stale 收敛 ID 不得应用到非 OAuth 账号") +} + +func TestBuildUpstreamRequestOpenAIPassthrough_AppliesStagedFingerprint(t *testing.T) { + svc := &OpenAIGatewayService{} + // 收敛是显式 opt-in（#5610）：显式开启后验证透传路径的出站头收敛。 + account := newTestOAuthAccount(2001, map[string]any{ + "openai_oauth_passthrough": true, + "codex_fingerprint_mode": "session", + }) + + c := newFingerprintStageTestContext(t) + c.Request.Header.Set("session_id", "real-client-session") + c.Request.Header.Set("User-Agent", "codex_cli_rs/0.144.1 (Ubuntu 22.4.0; x86_64) xterm-256color") + c.Request.Header.Set("originator", "codex_cli_rs") + c.Request.Header.Set("x-codex-turn-metadata", `{"installation_id":"real-install","session_id":"real-session","sandbox":"seatbelt"}`) + + // 复刻 forwardOpenAIPassthrough 的解析+暂存 seam（默认 session 模式） + ids := resolveCodexFingerprintIDsFromRequest(account, c.Request.Header) + require.NotNil(t, ids) + stageCodexFingerprintIDs(c, ids) + + body := []byte(`{"model":"gpt-5.6-sol","input":[],"stream":true}`) + req, err := svc.buildUpstreamRequestOpenAIPassthrough(context.Background(), c, account, body, "test-token") + require.NoError(t, err) + + assert.Equal(t, ids.sessionID, req.Header.Get("session_id"), "session 模式下出站 session_id 应为账号级收敛值") + assert.Equal(t, ids.installationID, req.Header.Get("x-codex-installation-id")) + assert.Equal(t, ids.windowID, req.Header.Get("x-codex-window-id")) + assert.Equal(t, ids.threadID, req.Header.Get("x-client-request-id")) + turnMetadata := req.Header.Get("x-codex-turn-metadata") + require.NotEmpty(t, turnMetadata) + assert.Contains(t, turnMetadata, ids.sessionID, "turn-metadata JSON 中的 session_id 应被收敛") + assert.Contains(t, turnMetadata, `"sandbox":"seatbelt"`, "turn-metadata 未指定字段应原样保留") +} + +func TestBuildUpstreamRequestOpenAIPassthrough_OffModeKeepsIsolatedSession(t *testing.T) { + svc := &OpenAIGatewayService{} + account := newTestOAuthAccount(2002, map[string]any{ + "openai_oauth_passthrough": true, + "codex_fingerprint_mode": "off", + }) + + c := newFingerprintStageTestContext(t) + c.Request.Header.Set("session_id", "real-client-session") + c.Request.Header.Set("originator", "codex_cli_rs") + + ids := resolveCodexFingerprintIDsFromRequest(account, c.Request.Header) + require.Nil(t, ids) + stageCodexFingerprintIDs(c, ids) + + body := []byte(`{"model":"gpt-5.6-sol","input":[],"stream":true}`) + req, err := svc.buildUpstreamRequestOpenAIPassthrough(context.Background(), c, account, body, "test-token") + require.NoError(t, err) + + assert.NotEmpty(t, req.Header.Get("session_id")) + assert.NotEqual(t, resolveConvergedSessionID(account), req.Header.Get("session_id"), "off 模式不得收敛 session_id") + assert.Empty(t, req.Header.Get("x-codex-window-id")) +} + +func TestApplyCodexFingerprintClientMetadataRaw_NonObjectBodyUntouched(t *testing.T) { + account := newTestOAuthAccount(4244, nil) + ids := resolveCodexFingerprintIDs(account, "client-sess-nonobj", codexFingerprintSession) + require.NotNil(t, ids) + + for _, body := range []string{`[1,2,3]`, `"plain string"`, `not json at all`} { + out, changed, err := applyCodexFingerprintClientMetadataRaw([]byte(body), ids) + require.NoError(t, err) + assert.False(t, changed, "非 JSON 对象 body 不应被改写: %s", body) + assert.Equal(t, []byte(body), out) + } +} diff --git a/backend/internal/service/openai_codex_turn_state.go b/backend/internal/service/openai_codex_turn_state.go new file mode 100644 index 000000000..d5598c3d3 --- /dev/null +++ b/backend/internal/service/openai_codex_turn_state.go @@ -0,0 +1,165 @@ +package service + +import ( + "net/http" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" +) + +// openAICodexTurnStateHeader 是 Codex 的回合状态头。上游在响应头中铸造该 +// 不透明 blob，客户端在同一回合的后续请求中原样回带（codex-rs 侧从 +// /responses SSE、/responses/compact JSON 与 WS 握手三种响应中捕获，见 +// codex-api/src/sse/responses.rs 与 endpoint/compact.rs）。 +const openAICodexTurnStateHeader = "x-codex-turn-state" + +// turn-state blob 是上游在"出站身份"（含 #5553 指纹收敛改写后的 +// installation/session/thread 标识）下铸造的，同账号回放自洽；跨账号回放 +// （failover 换号后客户端仍回带旧账号的 blob）是代理链独有、真实 Codex +// 永远不会产生的矛盾信号。溯源表记录每个下游会话最近一次铸造该 blob 的 +// 账号，出站守卫据此剥离已知异账号的回带值。 +type openAICodexTurnStateOrigin struct { + accountID int64 + expiresAt time.Time +} + +// openAICodexTurnStateSeed 返回溯源表键：API Key + 客户端原始会话标识。 +// 客户端会话标识取自请求头（与指纹收敛的 thread 派生同源，见 +// extractClientSessionID），确保同一下游会话的记录/守卫两侧使用同一键。 +// 无会话标识时返回空串，表示不做跟踪（保持透传现状）。 +func openAICodexTurnStateSeed(c *gin.Context) string { + if c == nil || c.Request == nil { + return "" + } + sessionID := extractClientSessionID(c.Request.Header) + if sessionID == "" { + return "" + } + return strconv.FormatInt(getAPIKeyIDFromContext(c), 10) + "\x00" + sessionID +} + +// relayOpenAICodexTurnState 将上游响应中的 turn-state 显式写入下游响应头， +// 并记录铸造账号。必须在响应头提交点调用（WriteHeader 之前、且确认本次 +// 上游响应就是将要写回客户端的响应之后）。上游无该头时主动清除 writer 上 +// 可能残留的上一 failover attempt 的值——否则换号后旧账号的 blob 会粘到 +// 新账号的响应上，这正是本文件要防止的跨账号矛盾。 +func (s *OpenAIGatewayService) relayOpenAICodexTurnState(c *gin.Context, account *Account, upstream http.Header) { + if c == nil || c.Writer == nil { + return + } + canonical := http.CanonicalHeaderKey(openAICodexTurnStateHeader) + state := extractOpenAICodexTurnState(upstream) + if state == "" { + c.Writer.Header().Del(canonical) + return + } + c.Writer.Header().Set(canonical, state) + s.noteOpenAICodexTurnStateProvenance(c, account) +} + +// stageOpenAICodexTurnState 将上游 turn-state 暂存到延迟提交的响应头集合 +// （首输出守卫路径先缓存头、见到首个输出事件才提交）。此处**不**记录铸造 +// 账号：该 attempt 仍可能在首输出超时后 failover，暂存头会被整体丢弃， +// 客户端从未收到该 blob。溯源必须在真正提交时记录，见 +// noteStagedOpenAICodexTurnStateCommitted。 +func stageOpenAICodexTurnState(dst *http.Header, upstream http.Header) { + if dst == nil { + return + } + canonical := http.CanonicalHeaderKey(openAICodexTurnStateHeader) + state := extractOpenAICodexTurnState(upstream) + if state == "" { + if *dst != nil { + dst.Del(canonical) + } + return + } + if *dst == nil { + *dst = http.Header{} + } + dst.Set(canonical, state) +} + +// noteStagedOpenAICodexTurnStateCommitted 在暂存响应头真正写入下游时记录 +// 铸造账号——只有此刻客户端才确定收到了该 blob，溯源表才与客户端持有的 +// 值一致（否则被 failover 丢弃的 attempt 会污染溯源，导致后续误剥离）。 +func (s *OpenAIGatewayService) noteStagedOpenAICodexTurnStateCommitted(c *gin.Context, account *Account, staged http.Header) { + if staged == nil || strings.TrimSpace(staged.Get(openAICodexTurnStateHeader)) == "" { + return + } + s.noteOpenAICodexTurnStateProvenance(c, account) +} + +func extractOpenAICodexTurnState(upstream http.Header) string { + if upstream == nil { + return "" + } + return strings.TrimSpace(upstream.Get(openAICodexTurnStateHeader)) +} + +// noteOpenAICodexTurnStateProvenance 记录（下游会话 → 铸造账号）。 +func (s *OpenAIGatewayService) noteOpenAICodexTurnStateProvenance(c *gin.Context, account *Account) { + if s == nil || account == nil || account.ID <= 0 { + return + } + seed := openAICodexTurnStateSeed(c) + if seed == "" { + return + } + s.openaiCodexTurnStateOrigins.Store(seed, openAICodexTurnStateOrigin{ + accountID: account.ID, + expiresAt: time.Now().Add(s.openAIWSSessionStickyTTL()), + }) + s.sweepOpenAICodexTurnStateOrigins() +} + +// guardOpenAICodexTurnStateEcho 出站守卫：客户端回带的 turn-state 若已知由 +// 其他账号铸造则剥离，同账号或无溯源记录时保持原样。只剥离、不注入—— +// /responses 路径的客户端是真实 Codex，会按自身回合语义自行回带；服务端 +// 注入是 Claude 兼容桥（无法回带的客户端）的专属行为。 +func (s *OpenAIGatewayService) guardOpenAICodexTurnStateEcho(c *gin.Context, account *Account, h http.Header) { + if s == nil || h == nil || account == nil { + return + } + if strings.TrimSpace(h.Get(openAICodexTurnStateHeader)) == "" { + return + } + seed := openAICodexTurnStateSeed(c) + if seed == "" { + return + } + raw, ok := s.openaiCodexTurnStateOrigins.Load(seed) + if !ok { + return + } + origin, ok := raw.(openAICodexTurnStateOrigin) + if !ok { + s.openaiCodexTurnStateOrigins.Delete(seed) + return + } + if !origin.expiresAt.IsZero() && time.Now().After(origin.expiresAt) { + s.openaiCodexTurnStateOrigins.Delete(seed) + return + } + if origin.accountID != account.ID { + h.Del(openAICodexTurnStateHeader) + } +} + +// sweepOpenAICodexTurnStateOrigins 机会式清扫过期溯源记录：每 256 次写入 +// 全量遍历一轮，防止仅靠读侧惰性删除导致的慢泄漏（会话键无上界）。 +func (s *OpenAIGatewayService) sweepOpenAICodexTurnStateOrigins() { + if s.openaiCodexTurnStateWrites.Add(1)%256 != 0 { + return + } + now := time.Now() + s.openaiCodexTurnStateOrigins.Range(func(key, value any) bool { + origin, ok := value.(openAICodexTurnStateOrigin) + if !ok || (!origin.expiresAt.IsZero() && now.After(origin.expiresAt)) { + s.openaiCodexTurnStateOrigins.Delete(key) + } + return true + }) +} diff --git a/backend/internal/service/openai_codex_turn_state_test.go b/backend/internal/service/openai_codex_turn_state_test.go new file mode 100644 index 000000000..a58e16629 --- /dev/null +++ b/backend/internal/service/openai_codex_turn_state_test.go @@ -0,0 +1,381 @@ +package service + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" +) + +func newTurnStateTestContext(t *testing.T, apiKeyID int64, sessionID string) (*gin.Context, *httptest.ResponseRecorder) { + t.Helper() + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) + if sessionID != "" { + c.Request.Header.Set("session_id", sessionID) + } + if apiKeyID > 0 { + c.Set("api_key", &APIKey{ID: apiKeyID}) + } + return c, rec +} + +func TestOpenAICodexTurnStateSeed(t *testing.T) { + c, _ := newTurnStateTestContext(t, 7, "sess-1") + require.Equal(t, "7\x00sess-1", openAICodexTurnStateSeed(c)) + + // 连字符形式优先（Codex CLI 标准头） + c.Request.Header.Set("session-id", "sess-hyphen") + require.Equal(t, "7\x00sess-hyphen", openAICodexTurnStateSeed(c)) + + // 无会话标识 → 不跟踪 + cNoSession, _ := newTurnStateTestContext(t, 7, "") + require.Empty(t, openAICodexTurnStateSeed(cNoSession)) + + require.Empty(t, openAICodexTurnStateSeed(nil)) +} + +func TestRelayOpenAICodexTurnState_SetsHeaderAndRecordsProvenance(t *testing.T) { + svc := &OpenAIGatewayService{} + account := &Account{ID: 42} + c, _ := newTurnStateTestContext(t, 7, "sess-relay") + + upstream := http.Header{} + upstream.Set("x-codex-turn-state", "blob-A") + svc.relayOpenAICodexTurnState(c, account, upstream) + + require.Equal(t, "blob-A", c.Writer.Header().Get("X-Codex-Turn-State")) + + raw, ok := svc.openaiCodexTurnStateOrigins.Load("7\x00sess-relay") + require.True(t, ok) + origin, ok := raw.(openAICodexTurnStateOrigin) + require.True(t, ok) + require.Equal(t, int64(42), origin.accountID) + require.True(t, origin.expiresAt.After(time.Now())) +} + +func TestRelayOpenAICodexTurnState_ClearsStaleValueWhenUpstreamAbsent(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 7, "sess-stale") + // 模拟上一 failover attempt 残留的值 + c.Writer.Header().Set("X-Codex-Turn-State", "blob-old") + + svc.relayOpenAICodexTurnState(c, &Account{ID: 43}, http.Header{}) + + require.Empty(t, c.Writer.Header().Get("X-Codex-Turn-State")) + _, ok := svc.openaiCodexTurnStateOrigins.Load("7\x00sess-stale") + require.False(t, ok) +} + +func TestStageOpenAICodexTurnState_StagedHeaders(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 9, "sess-staged") + + // nil 集合 + 上游有值 → 创建集合并写入，但此刻还不记录溯源 + var staged http.Header + upstream := http.Header{} + upstream.Set("x-codex-turn-state", "blob-B") + stageOpenAICodexTurnState(&staged, upstream) + require.NotNil(t, staged) + require.Equal(t, "blob-B", staged.Get("X-Codex-Turn-State")) + _, noted := svc.openaiCodexTurnStateOrigins.Load("9\x00sess-staged") + require.False(t, noted, "暂存阶段不得记录溯源：该 attempt 仍可能 failover 丢弃") + + // 真正提交时才记录 + svc.noteStagedOpenAICodexTurnStateCommitted(c, &Account{ID: 44}, staged) + raw, ok := svc.openaiCodexTurnStateOrigins.Load("9\x00sess-staged") + require.True(t, ok) + origin, ok := raw.(openAICodexTurnStateOrigin) + require.True(t, ok) + require.Equal(t, int64(44), origin.accountID) + + // 上游无值 → 清除已暂存的值；nil 集合保持 nil + stageOpenAICodexTurnState(&staged, http.Header{}) + require.Empty(t, staged.Get("X-Codex-Turn-State")) + var nilStaged http.Header + stageOpenAICodexTurnState(&nilStaged, http.Header{}) + require.Nil(t, nilStaged) +} + +// 首输出超时导致 attempt 被丢弃时，溯源不得被该 attempt 污染——否则后续 +// 请求会把客户端持有的合法 blob 误判成跨账号回带而剥离。 +func TestStagedTurnState_AbandonedAttemptDoesNotPoisonProvenance(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 11, "sess-abandoned") + + // 账号 A 的 attempt 暂存了 blob，但从未提交（首输出超时 → failover） + var staged http.Header + upstreamA := http.Header{} + upstreamA.Set("x-codex-turn-state", "blob-A") + stageOpenAICodexTurnState(&staged, upstreamA) + + // 账号 B 接手并真正提交 + svc.relayOpenAICodexTurnState(c, &Account{ID: 52}, upstreamA) + + // 客户端回带的 blob 来自 B，出站到 B 时不得被剥离 + h := http.Header{} + h.Set("x-codex-turn-state", "blob-A") + svc.guardOpenAICodexTurnStateEcho(c, &Account{ID: 52}, h) + require.Equal(t, "blob-A", h.Get("x-codex-turn-state")) + + raw, ok := svc.openaiCodexTurnStateOrigins.Load("11\x00sess-abandoned") + require.True(t, ok) + origin, ok := raw.(openAICodexTurnStateOrigin) + require.True(t, ok) + require.Equal(t, int64(52), origin.accountID) +} + +func TestNoteStagedOpenAICodexTurnStateCommitted_NoopWithoutState(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 12, "sess-nostate") + + svc.noteStagedOpenAICodexTurnStateCommitted(c, &Account{ID: 60}, nil) + svc.noteStagedOpenAICodexTurnStateCommitted(c, &Account{ID: 60}, http.Header{"X-Request-Id": []string{"rid"}}) + + _, ok := svc.openaiCodexTurnStateOrigins.Load("12\x00sess-nostate") + require.False(t, ok) +} + +func TestGuardOpenAICodexTurnStateEcho(t *testing.T) { + newOutbound := func(state string) http.Header { + h := http.Header{} + if state != "" { + h.Set("x-codex-turn-state", state) + } + return h + } + + t.Run("same_account_keeps_echo", func(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 7, "sess-g1") + upstream := http.Header{} + upstream.Set("x-codex-turn-state", "blob-A") + svc.relayOpenAICodexTurnState(c, &Account{ID: 42}, upstream) + + h := newOutbound("blob-A") + svc.guardOpenAICodexTurnStateEcho(c, &Account{ID: 42}, h) + require.Equal(t, "blob-A", h.Get("x-codex-turn-state")) + }) + + t.Run("foreign_account_strips_echo", func(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 7, "sess-g2") + upstream := http.Header{} + upstream.Set("x-codex-turn-state", "blob-A") + svc.relayOpenAICodexTurnState(c, &Account{ID: 42}, upstream) + + // failover 换到账号 43：blob 由 42 铸造，必须剥离 + h := newOutbound("blob-A") + svc.guardOpenAICodexTurnStateEcho(c, &Account{ID: 43}, h) + require.Empty(t, h.Get("x-codex-turn-state")) + }) + + t.Run("no_provenance_passthrough", func(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 7, "sess-g3") + h := newOutbound("blob-unknown") + svc.guardOpenAICodexTurnStateEcho(c, &Account{ID: 43}, h) + require.Equal(t, "blob-unknown", h.Get("x-codex-turn-state")) + }) + + t.Run("expired_provenance_passthrough_and_pruned", func(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 7, "sess-g4") + svc.openaiCodexTurnStateOrigins.Store("7\x00sess-g4", openAICodexTurnStateOrigin{ + accountID: 42, + expiresAt: time.Now().Add(-time.Minute), + }) + h := newOutbound("blob-A") + svc.guardOpenAICodexTurnStateEcho(c, &Account{ID: 43}, h) + require.Equal(t, "blob-A", h.Get("x-codex-turn-state")) + _, ok := svc.openaiCodexTurnStateOrigins.Load("7\x00sess-g4") + require.False(t, ok) + }) + + t.Run("no_session_seed_noop", func(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 7, "") + h := newOutbound("blob-A") + svc.guardOpenAICodexTurnStateEcho(c, &Account{ID: 43}, h) + require.Equal(t, "blob-A", h.Get("x-codex-turn-state")) + }) + + t.Run("no_echo_noop", func(t *testing.T) { + svc := &OpenAIGatewayService{} + c, _ := newTurnStateTestContext(t, 7, "sess-g5") + h := newOutbound("") + svc.guardOpenAICodexTurnStateEcho(c, &Account{ID: 43}, h) + require.Empty(t, h.Get("x-codex-turn-state")) + }) +} + +func TestSweepOpenAICodexTurnStateOrigins_PrunesExpiredEntries(t *testing.T) { + svc := &OpenAIGatewayService{} + svc.openaiCodexTurnStateOrigins.Store("expired", openAICodexTurnStateOrigin{ + accountID: 1, + expiresAt: time.Now().Add(-time.Minute), + }) + svc.openaiCodexTurnStateOrigins.Store("alive", openAICodexTurnStateOrigin{ + accountID: 2, + expiresAt: time.Now().Add(time.Hour), + }) + + // 计数器推进到触发清扫的边界 + svc.openaiCodexTurnStateWrites.Store(255) + svc.sweepOpenAICodexTurnStateOrigins() + + _, expiredOK := svc.openaiCodexTurnStateOrigins.Load("expired") + require.False(t, expiredOK) + _, aliveOK := svc.openaiCodexTurnStateOrigins.Load("alive") + require.True(t, aliveOK) +} + +func TestWriteOpenAIPassthroughResponseHeaders_RelaysAndClearsTurnState(t *testing.T) { + // filter=nil 走 content-type 兜底分支；turn-state 强制放行不依赖 filter。 + dst := http.Header{} + src := http.Header{} + src.Set("X-Codex-Turn-State", "blob-P") + writeOpenAIPassthroughResponseHeaders(dst, src, nil) + require.Equal(t, "blob-P", dst.Get("X-Codex-Turn-State")) + + // 上游缺失时清除残留（failover 换号防串扰） + writeOpenAIPassthroughResponseHeaders(dst, http.Header{"Content-Type": []string{"application/json"}}, nil) + require.Empty(t, dst.Get("X-Codex-Turn-State")) +} + +func TestEnsureOpenAIRemoteCompactionV2BetaFeature(t *testing.T) { + t.Run("absent_sets_feature", func(t *testing.T) { + h := http.Header{} + ensureOpenAIRemoteCompactionV2BetaFeature(h) + require.Equal(t, "remote_compaction_v2", h.Get("x-codex-beta-features")) + }) + + t.Run("present_unchanged", func(t *testing.T) { + h := http.Header{} + h.Set("x-codex-beta-features", "responses_websockets_v2, remote_compaction_v2") + ensureOpenAIRemoteCompactionV2BetaFeature(h) + require.Equal(t, "responses_websockets_v2, remote_compaction_v2", h.Get("x-codex-beta-features")) + }) + + t.Run("other_tokens_merged", func(t *testing.T) { + h := http.Header{} + h.Set("x-codex-beta-features", "responses_websockets_v2") + ensureOpenAIRemoteCompactionV2BetaFeature(h) + require.Equal(t, "responses_websockets_v2,remote_compaction_v2", h.Get("x-codex-beta-features")) + }) + + t.Run("multi_line_values_merged_single_line", func(t *testing.T) { + h := http.Header{} + h.Add("x-codex-beta-features", "feature_a") + h.Add("x-codex-beta-features", "feature_b") + ensureOpenAIRemoteCompactionV2BetaFeature(h) + require.Equal(t, []string{"feature_a,feature_b,remote_compaction_v2"}, h.Values("x-codex-beta-features")) + }) +} + +// 对齐真实 Codex：该头是会话级常量，挂在 OAuth 的每个请求上，而不是只在 +// 压缩回合出现（codex-rs build_model_client_beta_features_header）。 +func TestApplyOpenAICodexBetaFeatures(t *testing.T) { + oauthAccount := &Account{ID: 1, Platform: PlatformOpenAI, Type: AccountTypeOAuth} + apiKeyAccount := &Account{ID: 2, Platform: PlatformOpenAI, Type: AccountTypeAPIKey} + + t.Run("oauth_plain_request_gets_default_codex_shape", func(t *testing.T) { + c, _ := newTurnStateTestContext(t, 7, "sess-beta") + h := http.Header{} + applyOpenAICodexBetaFeatures(c, oauthAccount, h) + require.Equal(t, "remote_compaction_v2", h.Get("x-codex-beta-features"), + "OAuth 的普通请求也必须带会话级 beta 头") + }) + + t.Run("client_declared_header_preserved", func(t *testing.T) { + c, _ := newTurnStateTestContext(t, 7, "sess-beta") + h := http.Header{} + h.Set("x-codex-beta-features", "some_other_feature") + applyOpenAICodexBetaFeatures(c, oauthAccount, h) + require.Equal(t, "some_other_feature", h.Get("x-codex-beta-features"), + "客户端显式声明的能力集不得被网关改写（非空即视为用户已关闭 v2）") + }) + + t.Run("native_v2_forces_feature_even_when_client_trimmed_it", func(t *testing.T) { + c, _ := newTurnStateTestContext(t, 7, "sess-beta") + MarkOpenAINativeCompactionV2(c) + h := http.Header{} + h.Set("x-codex-beta-features", "some_other_feature") + applyOpenAICodexBetaFeatures(c, oauthAccount, h) + require.Contains(t, h.Get("x-codex-beta-features"), "remote_compaction_v2", + "body 带 compaction_trigger 是实锤，必须确保 v2 在列") + require.Contains(t, h.Get("x-codex-beta-features"), "some_other_feature") + }) + + t.Run("native_v2_applies_to_non_oauth_too", func(t *testing.T) { + c, _ := newTurnStateTestContext(t, 7, "sess-beta") + MarkOpenAINativeCompactionV2(c) + h := http.Header{} + applyOpenAICodexBetaFeatures(c, apiKeyAccount, h) + require.Equal(t, "remote_compaction_v2", h.Get("x-codex-beta-features")) + }) + + t.Run("non_oauth_plain_request_untouched", func(t *testing.T) { + c, _ := newTurnStateTestContext(t, 7, "sess-beta") + h := http.Header{} + applyOpenAICodexBetaFeatures(c, apiKeyAccount, h) + require.Empty(t, h.Get("x-codex-beta-features"), + "非 Codex 后端不做会话级注入") + }) + + t.Run("nil_account_plain_request_untouched", func(t *testing.T) { + c, _ := newTurnStateTestContext(t, 7, "sess-beta") + h := http.Header{} + applyOpenAICodexBetaFeatures(c, nil, h) + require.Empty(t, h.Get("x-codex-beta-features")) + }) +} + +// WS 握手与 HTTP 出站必须给出同一份会话级 beta 头：真实 Codex 的 +// build_websocket_headers 复用 build_responses_headers（client.rs）， +// 两侧不一致还会让预热连接与实际请求落进不同的连接池兼容分桶。 +func TestBuildOpenAIWSHeaders_CarriesSessionBetaFeatures(t *testing.T) { + gin.SetMode(gin.TestMode) + svc := &OpenAIGatewayService{} + decision := OpenAIWSProtocolDecision{Transport: OpenAIUpstreamTransportResponsesWebsocketV2} + + build := func(t *testing.T, account *Account, clientBeta string) http.Header { + t.Helper() + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodGet, "/v1/responses", nil) + if clientBeta != "" { + c.Request.Header.Set("x-codex-beta-features", clientBeta) + } + headers, _, err := svc.buildOpenAIWSHeaders( + context.Background(), c, account, "test-token", decision, + true, "", "", "", "gpt-5.6-codex", "", + ) + require.NoError(t, err) + return headers + } + + oauthAccount := &Account{ + Platform: PlatformOpenAI, + Type: AccountTypeOAuth, + Credentials: map[string]any{"chatgpt_account_id": "test-account"}, + } + + headers := build(t, oauthAccount, "") + require.Equal(t, "remote_compaction_v2", headers.Get("x-codex-beta-features"), + "WS 握手也必须带会话级 beta 头") + + declared := build(t, oauthAccount, "some_other_feature") + require.Equal(t, []string{"some_other_feature"}, declared.Values("x-codex-beta-features"), + "客户端已声明时原样保留") + + apiKeyHeaders := build(t, &Account{Platform: PlatformOpenAI, Type: AccountTypeAPIKey}, "") + require.Empty(t, apiKeyHeaders.Get("x-codex-beta-features"), + "非 Codex 后端不注入") +} diff --git a/backend/internal/service/openai_compact_body_signal.go b/backend/internal/service/openai_compact_body_signal.go index ce561b0c5..83266e6c0 100644 --- a/backend/internal/service/openai_compact_body_signal.go +++ b/backend/internal/service/openai_compact_body_signal.go @@ -1,11 +1,119 @@ package service -import "github.com/tidwall/gjson" +import ( + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/tidwall/gjson" +) + +// openAINativeCompactionV2Key 标记本请求是原生 remote compaction v2 +// （裸 /responses + stream:true + compaction_trigger），由 handler 在判定后 +// 写入，供上游请求构造时补注协商头。 +const openAINativeCompactionV2Key = "openai_native_compaction_v2" + +const openAIRemoteCompactionV2Feature = "remote_compaction_v2" + +// MarkOpenAINativeCompactionV2 由 handler 在识别出原生 v2 压缩请求时调用。 +func MarkOpenAINativeCompactionV2(c *gin.Context) { + if c != nil { + c.Set(openAINativeCompactionV2Key, true) + } +} + +func isOpenAINativeCompactionV2(c *gin.Context) bool { + if c == nil { + return false + } + return c.GetBool(openAINativeCompactionV2Key) +} + +// ensureOpenAIRemoteCompactionV2BetaFeature 确保出站 x-codex-beta-features +// 头包含 remote_compaction_v2。真实 Codex 发送 compaction_trigger 时总会同时 +// 携带该协商头（codex-rs build_model_client_beta_features_header 对该 feature +// 特判 advertise）；上游或下游网关链剥掉它后，请求会在依赖该头做门控的 +// 环节被降级（#5586）。这里在原生 v2 请求出站前补齐，使线型与真实 Codex +// 一致。已存在时保持原样，不重复追加。 +func ensureOpenAIRemoteCompactionV2BetaFeature(h http.Header) { + if h == nil { + return + } + tokens := make([]string, 0, 4) + for _, value := range h.Values("x-codex-beta-features") { + for _, token := range strings.Split(value, ",") { + token = strings.TrimSpace(token) + if token == "" { + continue + } + if token == openAIRemoteCompactionV2Feature { + return + } + tokens = append(tokens, token) + } + } + tokens = append(tokens, openAIRemoteCompactionV2Feature) + h.Set("x-codex-beta-features", strings.Join(tokens, ",")) +} + +// hasOpenAICodexBetaFeaturesHeader 报告出站头里是否已存在非空的 +// x-codex-beta-features（即客户端自己声明过能力集）。 +func hasOpenAICodexBetaFeaturesHeader(h http.Header) bool { + if h == nil { + return false + } + for _, value := range h.Values("x-codex-beta-features") { + if strings.TrimSpace(value) != "" { + return true + } + } + return false +} + +// applyOpenAICodexBetaFeatures 按真实 Codex 的会话级行为补注 +// x-codex-beta-features。 +// +// codex 侧规则（codex-rs：session/mod.rs build_model_client_beta_features_header +// 组装、client.rs build_responses_headers 附加）：该头是**会话级常量**，挂在 +// /responses、WS 握手、/responses/compact 三处的**每一个**请求上，而不是只挂 +// 压缩回合。其内容是"已启用且需要 advertise 的 feature 列表"，RemoteCompactionV2 +// 被特判 advertise；实测默认安装下没有任何 Experimental 特性默认开启， +// 因此默认 Codex 的头值恰好就是单个 "remote_compaction_v2"。 +// +// 网关据此对齐： +// - 原生 v2 压缩回合（body 带 compaction_trigger 实锤）：无论账号类型都确保 +// v2 在列，覆盖中间网关裁剪 token 的情形（#5586）； +// - ChatGPT codex 上游（OAuth）的其余请求：客户端**未**声明该头时补成默认 +// Codex 形态，消除"仅压缩回合才带该头"这种真实 Codex 不会产生的模式； +// - 客户端已声明该头：原样保留。非空但不含 v2 表示用户显式关闭了该特性， +// 网关不得替其改写能力声明； +// - 非 OAuth 上游（API Key/第三方兼容网关）：不做会话级注入，只保留压缩回合 +// 的那一条，避免向非 Codex 后端撒 Codex 专属头。 +// +// 已知无解的歧义：用户关掉 v2 且无其他特性时，真实 Codex 同样不发该头，与"老 +// 客户端"在线型上不可区分，此时按默认形态补注。该用户的 legacy 压缩端点本就 +// 已被上游下线（404），不存在可回退的正确行为。 +func applyOpenAICodexBetaFeatures(c *gin.Context, account *Account, h http.Header) { + if h == nil { + return + } + if isOpenAINativeCompactionV2(c) { + ensureOpenAIRemoteCompactionV2BetaFeature(h) + return + } + if account == nil || !account.IsOpenAIOAuth() { + return + } + if hasOpenAICodexBetaFeaturesHeader(h) { + return + } + h.Set("x-codex-beta-features", openAIRemoteCompactionV2Feature) +} // HasCompactionTriggerInInput detects an input item with // type="compaction_trigger". The handler combines this body signal with the -// request path, stream flag, and Codex beta feature header to distinguish the -// native remote compaction v2 wire from the legacy /responses/compact bridge. +// request path and stream flag to distinguish the native remote compaction v2 +// wire from the legacy /responses/compact bridge. func HasCompactionTriggerInInput(body []byte) bool { if len(body) == 0 { return false diff --git a/backend/internal/service/openai_compact_probe.go b/backend/internal/service/openai_compact_probe.go index e8deff2d6..9d5e8ccad 100644 --- a/backend/internal/service/openai_compact_probe.go +++ b/backend/internal/service/openai_compact_probe.go @@ -10,7 +10,8 @@ import ( const ( // AccountTestModeDefault drives the standard /responses connection test. AccountTestModeDefault = "default" - // AccountTestModeCompact drives the /responses/compact compact-probe test. + // AccountTestModeCompact drives the remote-compaction probe test + // (native v2: streaming /responses with a compaction_trigger input item). AccountTestModeCompact = "compact" ) @@ -23,8 +24,12 @@ func normalizeAccountTestMode(mode string) string { } } -func createOpenAICompactProbePayload(model string) map[string]any { - return map[string]any{ +// createOpenAICompactProbePayload 构造原生 remote compaction v2 探测载荷： +// 流式 /responses + input 末尾 {"type":"compaction_trigger"}。上游已下线 +// legacy unary /responses/compact（v1 形态恒 404，#5598/#5624），现行 codex +// 默认协议即 v2（RemoteCompactionV2 Stable + default_enabled）。 +func createOpenAICompactProbePayload(model string, isOAuth bool) map[string]any { + payload := map[string]any{ "model": strings.TrimSpace(model), "instructions": "You are a helpful coding assistant.", "input": []any{ @@ -33,8 +38,35 @@ func createOpenAICompactProbePayload(model string) map[string]any { "role": "user", "content": "Respond with OK.", }, + map[string]any{"type": "compaction_trigger"}, }, + "stream": true, } + // ChatGPT internal API 要求 store: false，与真实转发一致。 + if isOAuth { + payload["store"] = false + } + return payload +} + +// openAICompactProbeFoundCompactionItem 判定探测响应是否产出了 compaction +// 输出 item——v2 契约的核心（codex 缺它即 fatal "got 0 items"）。三种形态都 +// 认：① SSE 的 output_item.done/added（原生 v2 主形态，codex 只从这里收集 +// item）；② SSE 终态 response.completed 的 response.output[]（部分上游只在 +// 终态给出 item）；③ 整体 JSON 的 output[]（老网关链把请求降级成 unary）。 +func openAICompactProbeFoundCompactionItem(body []byte) bool { + if len(body) == 0 { + return false + } + bodyText := string(body) + if _, found := findRawCompactionItemFromSSE(bodyText); found { + return true + } + if finalResponse, ok := extractCodexFinalResponse(bodyText); ok && + responsesOutputHasCompactionItem(finalResponse) { + return true + } + return responsesOutputHasCompactionItem(body) } func shouldMarkOpenAICompactUnsupported(status int, body []byte) bool { @@ -60,7 +92,12 @@ func shouldMarkOpenAICompactUnsupported(status int, body []byte) bool { return false } -func buildOpenAICompactProbeExtraUpdates(resp *http.Response, body []byte, probeErr error, now time.Time) map[string]any { +// buildOpenAICompactProbeExtraUpdates 计算探测结果的账号 extra 更新。 +// compactionFound 是 v2 契约判据：HTTP 2xx 但响应无 compaction item 时同样 +// 记为不支持（链路把 compaction_trigger 吞掉的形态，等价 codex 的 "got 0 +// items" fatal，#5478/#5648）。极端场景（上游链只支持 legacy unary compact） +// 可用账号级 openai_compact_mode=force_on 人工覆盖。 +func buildOpenAICompactProbeExtraUpdates(resp *http.Response, body []byte, probeErr error, compactionFound bool, now time.Time) map[string]any { updates := map[string]any{ "openai_compact_checked_at": now.Format(time.RFC3339), "openai_compact_last_status": nil, @@ -84,10 +121,14 @@ func buildOpenAICompactProbeExtraUpdates(resp *http.Response, body []byte, probe errMsg = "HTTP " + strconv.Itoa(resp.StatusCode) } errMsg = truncateString(sanitizeUpstreamErrorMessage(errMsg), 2048) - if resp.StatusCode >= 200 && resp.StatusCode < 300 { + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300 && compactionFound: updates["openai_compact_supported"] = true updates["openai_compact_last_error"] = "" - } else { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + updates["openai_compact_supported"] = false + updates["openai_compact_last_error"] = "upstream returned 2xx without a compaction output item (native remote compaction v2 unsupported)" + default: if shouldMarkOpenAICompactUnsupported(resp.StatusCode, body) { updates["openai_compact_supported"] = false } @@ -112,9 +153,14 @@ func mergeExtraUpdates(base map[string]any, more map[string]any) map[string]any return out } +// compactProbeSessionID 返回探测请求使用的会话标识。真实 Codex 的 +// session-id / thread-id 恒为 UUID（codex-protocol ThreadId 是 UUIDv7）， +// 探测既然与真实流量走同一个 /responses 端点，标识形态就必须同构—— +// 否则上游能凭 "probe_compact_5" 这类字面量一眼区分出探测流量。 +// 账号级稳定派生：重复探测复用同一会话，而不是每次新开一个。 func compactProbeSessionID(accountID int64) string { if accountID <= 0 { - return "probe_compact" + return deriveStableUUIDv4("sub2api:codex-compact-probe:v1:anonymous") } - return "probe_compact_" + strconv.FormatInt(accountID, 10) + return deriveStableUUIDv4("sub2api:codex-compact-probe:v1:" + strconv.FormatInt(accountID, 10)) } diff --git a/backend/internal/service/openai_compact_probe_test.go b/backend/internal/service/openai_compact_probe_test.go index fe3ba0e8d..c273152e3 100644 --- a/backend/internal/service/openai_compact_probe_test.go +++ b/backend/internal/service/openai_compact_probe_test.go @@ -28,7 +28,7 @@ func TestNormalizeAccountTestMode(t *testing.T) { func TestBuildOpenAICompactProbeExtraUpdates_SuccessMarksSupported(t *testing.T) { now := time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC) - updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusOK}, []byte(`{"id":"cmp_1"}`), nil, now) + updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusOK}, []byte(`{"id":"cmp_1"}`), nil, true, now) if got := updates["openai_compact_supported"]; got != true { t.Fatalf("openai_compact_supported = %v, want true", got) @@ -47,7 +47,7 @@ func TestBuildOpenAICompactProbeExtraUpdates_SuccessMarksSupported(t *testing.T) func TestBuildOpenAICompactProbeExtraUpdates_404MarksUnsupported(t *testing.T) { now := time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC) body := []byte(`404 page not found`) - updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusNotFound}, body, nil, now) + updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusNotFound}, body, nil, false, now) if got := updates["openai_compact_supported"]; got != false { t.Fatalf("openai_compact_supported = %v, want false", got) @@ -59,7 +59,7 @@ func TestBuildOpenAICompactProbeExtraUpdates_404MarksUnsupported(t *testing.T) { func TestBuildOpenAICompactProbeExtraUpdates_502DoesNotMarkUnsupported(t *testing.T) { now := time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC) - updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusBadGateway}, []byte(`Upstream request failed`), nil, now) + updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusBadGateway}, []byte(`Upstream request failed`), nil, false, now) if _, exists := updates["openai_compact_supported"]; exists { t.Fatalf("did not expect openai_compact_supported for 502 response") @@ -71,7 +71,7 @@ func TestBuildOpenAICompactProbeExtraUpdates_502DoesNotMarkUnsupported(t *testin func TestBuildOpenAICompactProbeExtraUpdates_RequestErrorDoesNotMarkUnsupported(t *testing.T) { now := time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC) - updates := buildOpenAICompactProbeExtraUpdates(nil, nil, errors.New("dial tcp timeout"), now) + updates := buildOpenAICompactProbeExtraUpdates(nil, nil, errors.New("dial tcp timeout"), false, now) if _, exists := updates["openai_compact_supported"]; exists { t.Fatalf("did not expect openai_compact_supported for request error") @@ -86,7 +86,7 @@ func TestBuildOpenAICompactProbeExtraUpdates_RequestErrorDoesNotMarkUnsupported( func TestBuildOpenAICompactProbeExtraUpdates_NoResponseClearsLastStatus(t *testing.T) { now := time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC) - updates := buildOpenAICompactProbeExtraUpdates(nil, nil, nil, now) + updates := buildOpenAICompactProbeExtraUpdates(nil, nil, nil, false, now) if got, exists := updates["openai_compact_last_status"]; !exists || got != nil { t.Fatalf("openai_compact_last_status = %v, want nil key", got) @@ -99,7 +99,7 @@ func TestBuildOpenAICompactProbeExtraUpdates_NoResponseClearsLastStatus(t *testi func TestBuildOpenAICompactProbeExtraUpdates_UnknownModelDoesNotMarkUnsupported(t *testing.T) { now := time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC) body := []byte(`{"error":{"message":"unknown model gpt-5.4-openai-compact"}}`) - updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusBadRequest}, body, nil, now) + updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusBadRequest}, body, nil, false, now) if _, exists := updates["openai_compact_supported"]; exists { t.Fatalf("did not expect openai_compact_supported for unknown-model diagnostics") @@ -111,7 +111,7 @@ func TestBuildOpenAICompactProbeExtraUpdates_UnknownModelDoesNotMarkUnsupported( func TestBuildOpenAICompactProbeExtraUpdates_EmptyFailureBodyFallsBackToHTTPStatus(t *testing.T) { now := time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC) - updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusServiceUnavailable}, nil, nil, now) + updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusServiceUnavailable}, nil, nil, false, now) if got := updates["openai_compact_last_status"]; got != http.StatusServiceUnavailable { t.Fatalf("openai_compact_last_status = %v, want %d", got, http.StatusServiceUnavailable) @@ -120,3 +120,79 @@ func TestBuildOpenAICompactProbeExtraUpdates_EmptyFailureBodyFallsBackToHTTPStat t.Fatalf("openai_compact_last_error = %v, want HTTP 503", got) } } + +func TestBuildOpenAICompactProbeExtraUpdates_2xxWithoutCompactionItemMarksUnsupported(t *testing.T) { + now := time.Date(2026, 4, 10, 10, 0, 0, 0, time.UTC) + updates := buildOpenAICompactProbeExtraUpdates(&http.Response{StatusCode: http.StatusOK}, []byte(`{"id":"resp_1","output":[]}`), nil, false, now) + + if got := updates["openai_compact_supported"]; got != false { + t.Fatalf("openai_compact_supported = %v, want false（2xx 无 compaction item = v2 不可用）", got) + } + if got := updates["openai_compact_last_error"]; got == "" { + t.Fatalf("expected openai_compact_last_error to explain the missing compaction item") + } +} + +func TestCreateOpenAICompactProbePayload_NativeV2Shape(t *testing.T) { + payload := createOpenAICompactProbePayload("gpt-5.6-sol", true) + if payload["stream"] != true { + t.Fatalf("v2 probe payload must be streaming") + } + if payload["store"] != false { + t.Fatalf("OAuth probe payload must carry store:false") + } + input, ok := payload["input"].([]any) + if !ok || len(input) != 2 { + t.Fatalf("expected 2 input items, got %v", payload["input"]) + } + last, ok := input[len(input)-1].(map[string]any) + if !ok || last["type"] != "compaction_trigger" { + t.Fatalf("last input item must be compaction_trigger, got %v", input[len(input)-1]) + } + + apiKeyPayload := createOpenAICompactProbePayload("gpt-5.6-sol", false) + if _, has := apiKeyPayload["store"]; has { + t.Fatalf("API-key probe payload must not force store") + } +} + +func TestOpenAICompactProbeFoundCompactionItem(t *testing.T) { + sseWithItem := []byte("event: response.output_item.done\ndata: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"compaction\",\"id\":\"cmp_1\",\"encrypted_content\":\"blob\"}}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"output\":[]}}\n\n") + if !openAICompactProbeFoundCompactionItem(sseWithItem) { + t.Fatalf("SSE output_item.done 携带 compaction item 应判定为支持") + } + + sseAlias := []byte("data: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"compaction_summary\",\"id\":\"cmp_2\"}}\n\n") + if !openAICompactProbeFoundCompactionItem(sseAlias) { + t.Fatalf("compaction_summary 别名应判定为支持") + } + + sseNoItem := []byte("data: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"message\",\"id\":\"msg_1\"}}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_2\",\"output\":[]}}\n\n") + if openAICompactProbeFoundCompactionItem(sseNoItem) { + t.Fatalf("无 compaction item 的流不应判定为支持") + } + + jsonWithItem := []byte(`{"id":"resp_3","output":[{"type":"compaction","id":"cmp_3"}]}`) + if !openAICompactProbeFoundCompactionItem(jsonWithItem) { + t.Fatalf("JSON output[] 携带 compaction item 应判定为支持（降级链兜底）") + } + + if openAICompactProbeFoundCompactionItem(nil) { + t.Fatalf("空响应不应判定为支持") + } +} + +func TestOpenAICompactProbeFoundCompactionItem_TerminalResponseOutput(t *testing.T) { + // 部分上游只在终态 response.completed 的 output[] 给出 compaction item， + // 事件流里没有 output_item.done——探针同样必须判定为支持。 + sseTerminalOnly := []byte("data: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"message\",\"id\":\"msg_1\"}}\n\n" + + "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_t\",\"output\":[{\"type\":\"compaction\",\"id\":\"cmp_t\"}]}}\n\n") + if !openAICompactProbeFoundCompactionItem(sseTerminalOnly) { + t.Fatalf("终态 response.output 中的 compaction item 应判定为支持") + } + + sseTerminalEmpty := []byte("data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_e\",\"output\":[]}}\n\n") + if openAICompactProbeFoundCompactionItem(sseTerminalEmpty) { + t.Fatalf("终态 output 为空且无 item 事件时不应判定为支持") + } +} diff --git a/backend/internal/service/openai_compaction_context.go b/backend/internal/service/openai_compaction_context.go new file mode 100644 index 000000000..360ec4ecd --- /dev/null +++ b/backend/internal/service/openai_compaction_context.go @@ -0,0 +1,32 @@ +package service + +import "context" + +type openAIForwardModelContextKey struct{} + +type openAIForwardModel struct { + model string + useCompactModelMapping bool +} + +// WithOpenAIForwardModel records the model present in the forwarded request +// body after channel mapping and whether the legacy /responses/compact-only +// model mapping applies. Native remote compaction v2 keeps this false, so +// channel restriction checks follow the same model chain used by Forward. +func WithOpenAIForwardModel(ctx context.Context, forwardModel string, useCompactModelMapping bool) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, openAIForwardModelContextKey{}, openAIForwardModel{ + model: forwardModel, + useCompactModelMapping: useCompactModelMapping, + }) +} + +func openAIForwardModelFromContext(ctx context.Context) (openAIForwardModel, bool) { + if ctx == nil { + return openAIForwardModel{}, false + } + forwardModel, ok := ctx.Value(openAIForwardModelContextKey{}).(openAIForwardModel) + return forwardModel, ok +} diff --git a/backend/internal/service/openai_gateway_forward.go b/backend/internal/service/openai_gateway_forward.go index dfb29d1aa..a6919bf58 100644 --- a/backend/internal/service/openai_gateway_forward.go +++ b/backend/internal/service/openai_gateway_forward.go @@ -108,7 +108,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco return s.forwardGrokResponses(ctx, c, account, body, originalModel, reqStream, startTime) } - if account.Type == AccountTypeAPIKey && !openai_compat.ShouldUseResponsesAPI(account.Extra) { + if shouldForwardOpenAIResponsesViaRawChatCompletions(account) { return s.forwardResponsesViaRawChatCompletions(ctx, c, account, body) } if account.Platform == PlatformOpenAI && account.Type == AccountTypeAPIKey { @@ -427,10 +427,10 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco markDecodedModified() } } - // 将 fpIDs 存入 gin context，供 buildUpstreamRequest 中头改写使用 - if c != nil && fpIDs != nil { - c.Set("codex_fingerprint_ids", fpIDs) - } + // 将 fpIDs 存入 gin context，供 buildUpstreamRequest 中头改写使用。 + // 无条件覆写（含 nil）：failover 从收敛账号切到 off 账号时，上一 + // 账号的 IDs 不得残留（stageCodexFingerprintIDs 注释）。 + stageCodexFingerprintIDs(c, fpIDs) } if codexResult.NormalizedModel != "" { upstreamModel = codexResult.NormalizedModel @@ -1028,6 +1028,12 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco } } +func shouldForwardOpenAIResponsesViaRawChatCompletions(account *Account) bool { + return account != nil && + account.Type == AccountTypeAPIKey && + !openai_compat.ShouldUseResponsesAPI(account.Extra) +} + func (s *OpenAIGatewayService) buildUpstreamRequest(ctx context.Context, c *gin.Context, account *Account, body []byte, token string, isStream bool, promptCacheKey string, isCodexCLI bool) (*http.Request, error) { body = s.prepareCodexQuotaOverdraftBody(ctx, account, isOpenAIResponsesCompactPath(c), body) // Determine target URL based on account type @@ -1089,6 +1095,9 @@ func (s *OpenAIGatewayService) buildUpstreamRequest(ctx context.Context, c *gin. } } } + // 客户端回带的 x-codex-turn-state 若已知由其他账号铸造（failover 换号）， + // 剥离后再出站——异账号 blob 与本账号的（指纹收敛后）出站身份自相矛盾。 + s.guardOpenAICodexTurnStateEcho(c, account, req.Header) if account.Type == AccountTypeOAuth { compatMessagesBridge := isOpenAICompatMessagesBridgeContext(c) || isOpenAICompatMessagesBridgeBody(body) // 清除客户端透传的 session 头，后续用隔离后的值重新设置，防止跨用户会话碰撞。 @@ -1139,13 +1148,7 @@ func (s *OpenAIGatewayService) buildUpstreamRequest(ctx context.Context, c *gin. } // 指纹收敛：使用 Forward() 中预计算的收敛 ID 改写出站头，与请求体使用同一份 IDs。 - if account.Type == AccountTypeOAuth && c != nil { - if fpIDs, ok := c.Get("codex_fingerprint_ids"); ok { - if ids, ok := fpIDs.(*codexFingerprintIDs); ok { - applyCodexFingerprintHeaders(req.Header, ids) - } - } - } + applyStagedCodexFingerprintHeaders(c, account, req.Header) // 终态收口：强制统一 OAuth 出站身份（User-Agent / originator / version 同源自洽）。 // 客户端自报身份不参与构造，浏览器型 UA 也因此不会再到达上游（原浏览器 UA 兜底已被吸收）。 @@ -1160,6 +1163,9 @@ func (s *OpenAIGatewayService) buildUpstreamRequest(ctx context.Context, c *gin. // 账号级请求头覆写（仅 openai api_key 账号启用时生效；OAuth 路径 no-op） account.ApplyHeaderOverrides(req.Header) + // x-codex-beta-features：按真实 Codex 的会话级行为补注（在账号级覆写之后， + // 保证不被覆盖丢失）。 + applyOpenAICodexBetaFeatures(c, account, req.Header) setOpenAICodexRoutingHintFromBody(req.Header, account, body) logOpenAIRoutingDiagnosticsFromBody(ctx, account, "http", req.Header, body, "not_applicable") diff --git a/backend/internal/service/openai_gateway_passthrough.go b/backend/internal/service/openai_gateway_passthrough.go index d712ba25a..36a96c1fe 100644 --- a/backend/internal/service/openai_gateway_passthrough.go +++ b/backend/internal/service/openai_gateway_passthrough.go @@ -80,6 +80,28 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough( body = normalizedBody } reqStream = gjson.GetBytes(body, "stream").Bool() + + // 指纹收敛：与非透传路径同门控（仅 OAuth、legacy compact 形态跳过）。 + // 一次性解析收敛 ID：请求体 client_metadata 在此改写（raw 字节外科 + // 手术，透传热路径禁全量 Unmarshal），出站头改写由请求构造器读取 + // context 中的同一份 IDs 完成（turn_id 等随机字段两侧必须一致）。 + if !isOpenAIResponsesCompactPath(c) { + var clientHeaders http.Header + if c != nil && c.Request != nil { + clientHeaders = c.Request.Header + } + fpIDs := resolveCodexFingerprintIDsFromRequest(account, clientHeaders) + if fpIDs != nil { + fpBody, fpChanged, fpErr := applyCodexFingerprintClientMetadataRaw(body, fpIDs) + if fpErr != nil { + return nil, fpErr + } + if fpChanged { + body = fpBody + } + } + stageCodexFingerprintIDs(c, fpIDs) + } } sanitizedBody, sanitized, err := sanitizeEmptyBase64InputImagesInOpenAIBody(body) @@ -239,6 +261,13 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough( serviceTier := extractOpenAIServiceTierFromBody(body) + // x-codex-turn-state 溯源：下游回传由 writeOpenAIPassthroughResponseHeaders + // 在各 handler 的写头点强制放行，铸造账号在此统一记录，供出站守卫剥离 + // failover 换号后的跨账号回带（openai_codex_turn_state.go）。 + if extractOpenAICodexTurnState(resp.Header) != "" { + s.noteOpenAICodexTurnStateProvenance(c, account) + } + var usage *OpenAIUsage var firstTokenMs *int responseID := "" @@ -377,6 +406,10 @@ func (s *OpenAIGatewayService) buildUpstreamRequestOpenAIPassthrough( } } + // 客户端回带的 x-codex-turn-state 若已知由其他账号铸造（failover 换号）， + // 剥离后再出站（openai_codex_turn_state.go）。 + s.guardOpenAICodexTurnStateEcho(c, account, req.Header) + // 覆盖入站鉴权残留，并注入上游认证 req.Header.Del("authorization") req.Header.Del("x-api-key") @@ -448,6 +481,10 @@ func (s *OpenAIGatewayService) buildUpstreamRequestOpenAIPassthrough( if s.cfg != nil && s.cfg.Gateway.ForceCodexCLI { req.Header.Set("user-agent", codexCLIUserAgent) } + // 指纹收敛：使用 forwardOpenAIPassthrough 中预计算的收敛 ID 改写出站头， + // 与请求体 client_metadata 共享同一份 IDs（与非透传路径相同的相对位置： + // 会话隔离之后、终态身份收口之前）。 + applyStagedCodexFingerprintHeaders(c, account, req.Header) // 终态收口：透传路径的 OAuth 与非透传完全一致，同样强制统一出站身份 // （User-Agent / originator / version 同源自洽），客户端自报身份不会到达上游。 if account.Type == AccountTypeOAuth { @@ -460,6 +497,9 @@ func (s *OpenAIGatewayService) buildUpstreamRequestOpenAIPassthrough( // 账号级请求头覆写（仅 openai api_key 账号启用时生效；OAuth 路径 no-op） account.ApplyHeaderOverrides(req.Header) + // x-codex-beta-features：按真实 Codex 的会话级行为补注（在账号级覆写之后， + // 保证不被覆盖丢失）。 + applyOpenAICodexBetaFeatures(c, account, req.Header) setOpenAICodexRoutingHintFromBody(req.Header, account, body) logOpenAIRoutingDiagnosticsFromBody(ctx, account, "http_passthrough", req.Header, body, "not_applicable") @@ -1651,4 +1691,13 @@ func writeOpenAIPassthroughResponseHeaders(dst http.Header, src http.Header, fil dst.Add(key, v) } } + + // x-codex-turn-state：Codex 回合状态头，客户端会在同回合后续请求回带。 + // 与上面的用量头不同，这里在上游缺失时也主动清除——failover 换号后残留 + // 上一账号的 blob 会构成跨账号矛盾（openai_codex_turn_state.go）。 + turnStateKey := http.CanonicalHeaderKey(openAICodexTurnStateHeader) + dst.Del(turnStateKey) + for _, v := range getCaseInsensitiveValues(src, openAICodexTurnStateHeader) { + dst.Add(turnStateKey, v) + } } diff --git a/backend/internal/service/openai_gateway_request_body.go b/backend/internal/service/openai_gateway_request_body.go index 04b3e7e05..8be2d0cf2 100644 --- a/backend/internal/service/openai_gateway_request_body.go +++ b/backend/internal/service/openai_gateway_request_body.go @@ -276,7 +276,9 @@ func isOpenAIEncryptedReasoningInputItem(item any) bool { return has } -func IsOpenAIResponsesCompactPathForTest(c *gin.Context) bool { +// IsOpenAIResponsesCompactPath reports whether the request targets the legacy +// /responses/compact endpoint, including its forwardable subpaths. +func IsOpenAIResponsesCompactPath(c *gin.Context) bool { return isOpenAIResponsesCompactPath(c) } diff --git a/backend/internal/service/openai_gateway_response_handling.go b/backend/internal/service/openai_gateway_response_handling.go index cab620b0d..82cd89f69 100644 --- a/backend/internal/service/openai_gateway_response_handling.go +++ b/backend/internal/service/openai_gateway_response_handling.go @@ -64,6 +64,14 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. } else if s.responseHeaderFilter != nil { responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter) } + // x-codex-turn-state 不在通用响应头白名单内，按 Codex 协议显式回传： + // 客户端会在同回合的后续请求中回带（openai_codex_turn_state.go）。 + // 首输出守卫模式下只暂存，溯源在 applyAttemptResponseHeaders 真正提交时记录。 + if guardFirstOutput { + stageOpenAICodexTurnState(&attemptResponseHeaders, resp.Header) + } else { + s.relayOpenAICodexTurnState(c, account, resp.Header) + } // Set SSE response headers c.Header("Content-Type", "text/event-stream") @@ -85,6 +93,9 @@ func (s *OpenAIGatewayService) handleStreamingResponseWithReasoning(ctx context. c.Writer.Header().Add(key, value) } } + // 暂存头此刻才真正写给客户端：turn-state 溯源在这里记录（见 + // noteStagedOpenAICodexTurnStateCommitted 的 failover 说明）。 + s.noteStagedOpenAICodexTurnStateCommitted(c, account, attemptResponseHeaders) // These headers describe this gateway's SSE stream and are stable across // account attempts. Keep them authoritative over upstream values. c.Header("Content-Type", "text/event-stream") @@ -1219,7 +1230,7 @@ func (s *OpenAIGatewayService) handleNonStreamingResponse(ctx context.Context, r // Some OpenAI-compatible upstreams (including other sub2api instances) // may return SSE even when stream=false was requested. if isEventStreamResponse(resp.Header) { - return s.handleSSEToJSON(resp, c, body, originalModel, mappedModel) + return s.handleSSEToJSON(resp, c, account, body, originalModel, mappedModel) } // bodyLooksLikeSSE is a line-level heuristic: real SSE framing requires // "data:"/"event:" field names at the very start of a physical line. A @@ -1235,7 +1246,7 @@ func (s *OpenAIGatewayService) handleNonStreamingResponse(ctx context.Context, r // positives on JSON responses that coincidentally contain "data:" or // "event:" in their text content. if account.Type == AccountTypeOAuth && bodyLooksLikeSSE { - return s.handleSSEToJSON(resp, c, body, originalModel, mappedModel) + return s.handleSSEToJSON(resp, c, account, body, originalModel, mappedModel) } if account != nil && account.IsGrok() && isOpenAIResponsesCompactPath(c) { body, err = convertGrokResponseToOpenAICompact(body) @@ -1247,7 +1258,7 @@ func (s *OpenAIGatewayService) handleNonStreamingResponse(ctx context.Context, r usageValue, usageOK := extractOpenAIUsageFromJSONBytes(body) if !usageOK { if bodyLooksLikeSSE { - return s.handleSSEToJSON(resp, c, body, originalModel, mappedModel) + return s.handleSSEToJSON(resp, c, account, body, originalModel, mappedModel) } return nil, fmt.Errorf("parse response: invalid json response") } @@ -1266,6 +1277,9 @@ func (s *OpenAIGatewayService) handleNonStreamingResponse(ctx context.Context, r return nil, fmt.Errorf("restore OpenAI namespace response: %w", err) } responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter) + // Codex 协议要求 /responses/compact JSON 响应携带 x-codex-turn-state + // （codex-api/src/endpoint/compact.rs 从响应头捕获），显式回传。 + s.relayOpenAICodexTurnState(c, account, resp.Header) contentType := "application/json" if s.cfg != nil && !s.cfg.Security.ResponseHeaders.Enabled { @@ -1309,7 +1323,7 @@ func bodyHasSSEFraming(body []byte) bool { return false } -func (s *OpenAIGatewayService) handleSSEToJSON(resp *http.Response, c *gin.Context, body []byte, originalModel, mappedModel string) (*openaiNonStreamingResult, error) { +func (s *OpenAIGatewayService) handleSSEToJSON(resp *http.Response, c *gin.Context, account *Account, body []byte, originalModel, mappedModel string) (*openaiNonStreamingResult, error) { bodyText := string(body) finalResponse, ok := extractCodexFinalResponse(bodyText) @@ -1361,6 +1375,7 @@ func (s *OpenAIGatewayService) handleSSEToJSON(resp *http.Response, c *gin.Conte } responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter) + s.relayOpenAICodexTurnState(c, account, resp.Header) contentType := "application/json; charset=utf-8" if !ok { diff --git a/backend/internal/service/openai_gateway_responses_chat_fallback_test.go b/backend/internal/service/openai_gateway_responses_chat_fallback_test.go index ee484001f..3682335c1 100644 --- a/backend/internal/service/openai_gateway_responses_chat_fallback_test.go +++ b/backend/internal/service/openai_gateway_responses_chat_fallback_test.go @@ -53,6 +53,50 @@ func TestForwardResponses_ForceChatCompletionsRoutesNonStreamingToChatCompletion require.False(t, result.Stream) } +func TestForwardResponses_PassthroughFlagWithUnsupportedResponsesUsesAccountMapping(t *testing.T) { + gin.SetMode(gin.TestMode) + + for _, path := range []string{"/v1/responses", "/v1/responses/compact"} { + path := path + t.Run(path, func(t *testing.T) { + body := []byte(`{"model":"gpt-5.4-channel","input":"hello","stream":false}`) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodPost, path, bytes.NewReader(body)) + c.Request.Header.Set("Content-Type", "application/json") + + upstream := &httpUpstreamRecorder{resp: &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"application/json"}}, + Body: io.NopCloser(strings.NewReader( + `{"id":"chatcmpl_mapping","object":"chat.completion","model":"gpt-5.4-account","choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":1,"completion_tokens":1,"total_tokens":2}}`, + )), + }} + svc := &OpenAIGatewayService{ + cfg: rawChatCompletionsTestConfig(), + httpUpstream: upstream, + } + account := rawChatCompletionsTestAccount() + account.Credentials["model_mapping"] = map[string]any{ + "gpt-5.4-channel": "gpt-5.4-account", + } + account.Credentials["compact_model_mapping"] = map[string]any{ + "gpt-5.4-account": "gpt-5.4-compact", + } + account.Extra = map[string]any{ + "openai_passthrough": true, + openai_compat.ExtraKeyResponsesSupported: false, + } + + result, err := svc.Forward(context.Background(), c, account, body) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, "http://upstream.example/v1/chat/completions", upstream.lastReq.URL.String()) + require.Equal(t, "gpt-5.4-account", gjson.GetBytes(upstream.lastBody, "model").String()) + }) + } +} + func TestForwardResponses_ForceChatCompletionsRoutesStreamingToChatCompletions(t *testing.T) { gin.SetMode(gin.TestMode) diff --git a/backend/internal/service/openai_gateway_scheduling.go b/backend/internal/service/openai_gateway_scheduling.go index 301ba997d..ffd0e4fc7 100644 --- a/backend/internal/service/openai_gateway_scheduling.go +++ b/backend/internal/service/openai_gateway_scheduling.go @@ -244,7 +244,7 @@ func (s *OpenAIGatewayService) SelectAccountForModelWithExclusions(ctx context.C } // noAvailableOpenAISelectionError builds the standard "no account available" error -// while preserving the compact-specific error when applicable. +// while preserving the legacy /responses/compact error when applicable. func normalizeOpenAICompatiblePlatform(platform string) string { if platform == PlatformGrok { return PlatformGrok @@ -689,17 +689,44 @@ func prioritizeOpenAICompactAccounts(accounts []*Account) []*Account { } // resolveOpenAIAccountUpstreamModelForRequest resolves the upstream model that -// would be sent for a given request, honouring compact-only mappings when the -// caller is on the /responses/compact path. +// would be sent for a given request, honoring the legacy compact-only mapping +// when the caller is on the /responses/compact path. func resolveOpenAIAccountUpstreamModelForRequest(account *Account, requestedModel string, requireCompact bool) string { + // Forward checks the raw Chat Completions fallback before passthrough. + // These API-key accounts therefore apply normal account model_mapping and + // upstream normalization, but never compact_model_mapping. + if shouldForwardOpenAIResponsesViaRawChatCompletions(account) { + upstreamModel := resolveOpenAIForwardModel(account, requestedModel, "") + return normalizeOpenAIModelForUpstream(account, upstreamModel) + } + + // Passthrough accounts only replace authentication. Their Forward path + // keeps the channel-mapped model in the request body and does not apply the + // account's normal model_mapping. Legacy /responses/compact is the one + // exception: forwardOpenAIPassthrough applies compact_model_mapping + // directly to that channel-mapped model. + if account != nil && account.IsOpenAIPassthroughEnabled() { + upstreamModel := strings.TrimSpace(requestedModel) + if upstreamModel == "" { + return "" + } + if requireCompact { + return resolveOpenAICompactForwardModel(account, upstreamModel) + } + return upstreamModel + } + upstreamModel := resolveOpenAIForwardModel(account, requestedModel, "") if upstreamModel == "" { return "" } if requireCompact { - return resolveOpenAICompactForwardModel(account, upstreamModel) + compactModel := resolveOpenAICompactForwardModel(account, upstreamModel) + if compactModel != upstreamModel { + return compactModel + } } - return upstreamModel + return normalizeOpenAIModelForUpstream(account, upstreamModel) } func (s *OpenAIGatewayService) selectAccountForModelWithExclusions(ctx context.Context, groupID *int64, platform string, sessionHash string, requestedModel string, excludedIDs map[int64]struct{}, requireCompact bool, stickyAccountID int64, requiredCapability OpenAIEndpointCapability, preferLowUpstreamRate bool) (*Account, error) { @@ -819,7 +846,8 @@ func (s *OpenAIGatewayService) tryStickySessionHit(ctx context.Context, groupID // selectBestAccount selects the best account from candidates (priority + LRU). // Returns nil if no available account. The second return reports whether at // least one candidate was filtered out solely because it lacks compact support -// (only meaningful when requireCompact=true); the third contains deterministic +// (only meaningful when the legacy /responses/compact requireCompact flag is +// true); the third contains deterministic // exclusion diagnostics for the evaluated snapshot. func (s *OpenAIGatewayService) selectBestAccount(ctx context.Context, groupID *int64, platform string, accounts []Account, requestedModel string, excludedIDs map[int64]struct{}, requireCompact bool, requiredCapability OpenAIEndpointCapability, preferLowUpstreamRate bool) (*Account, bool, openAISelectionFilterStats) { platform = normalizeOpenAICompatiblePlatform(platform) diff --git a/backend/internal/service/openai_gateway_service.go b/backend/internal/service/openai_gateway_service.go index 1f0893afa..0c4e18a12 100644 --- a/backend/internal/service/openai_gateway_service.go +++ b/backend/internal/service/openai_gateway_service.go @@ -398,8 +398,8 @@ func (t *accountWriteThrottle) Allow(id int64, now time.Time) bool { var defaultOpenAICodexSnapshotPersistThrottle = newAccountWriteThrottle(openAICodexSnapshotPersistMinInterval) -// ErrNoAvailableCompactAccounts indicates the request needs /responses/compact -// support but no compatible account is available. +// ErrNoAvailableCompactAccounts indicates a legacy /responses/compact request +// needs compact support but no compatible account is available. var ErrNoAvailableCompactAccounts = errors.New("no available accounts support /responses/compact") // OpenAIGatewayService handles OpenAI API gateway operations @@ -463,6 +463,11 @@ type OpenAIGatewayService struct { codexModelsManifestCache codexModelsManifestCache openaiCompatSessionResponses sync.Map openaiCompatAnthropicDigestSessions sync.Map + // openaiCodexTurnStateOrigins: 下游会话 seed → openAICodexTurnStateOrigin， + // 记录最近一次向该会话下发 x-codex-turn-state 的铸造账号，供出站守卫 + // 剥离跨账号回带（openai_codex_turn_state.go）。 + openaiCodexTurnStateOrigins sync.Map + openaiCodexTurnStateWrites atomic.Uint64 } func (s *OpenAIGatewayService) SetCodexQuotaOverdraftCoordinator(coordinator *CodexQuotaOverdraftCoordinator) { @@ -606,6 +611,10 @@ func (s *OpenAIGatewayService) isUpstreamModelRestrictedByChannel(ctx context.Co if s.channelService == nil { return false } + if compactForwardModel, ok := openAIForwardModelFromContext(ctx); ok { + requestedModel = compactForwardModel.model + requireCompact = compactForwardModel.useCompactModelMapping + } upstreamModel := resolveOpenAIAccountUpstreamModelForRequest(account, requestedModel, requireCompact) if upstreamModel == "" { return false diff --git a/backend/internal/service/openai_gateway_service_test.go b/backend/internal/service/openai_gateway_service_test.go index e7ddaca80..617c07c06 100644 --- a/backend/internal/service/openai_gateway_service_test.go +++ b/backend/internal/service/openai_gateway_service_test.go @@ -3475,7 +3475,7 @@ func TestHandleSSEToJSON_CompletedEventReturnsJSON(t *testing.T) { `data: [DONE]`, }, "\n")) - usage, err := svc.handleSSEToJSON(resp, c, body, "gpt-4o", "gpt-4o") + usage, err := svc.handleSSEToJSON(resp, c, nil, body, "gpt-4o", "gpt-4o") require.NoError(t, err) require.NotNil(t, usage) require.Equal(t, 7, usage.InputTokens) @@ -3594,7 +3594,7 @@ func TestHandleSSEToJSON_ReconstructsImageGenerationOutputItemDone(t *testing.T) `data: [DONE]`, }, "\n")) - usage, err := svc.handleSSEToJSON(resp, c, body, "gpt-5.4", "gpt-5.4") + usage, err := svc.handleSSEToJSON(resp, c, nil, body, "gpt-5.4", "gpt-5.4") require.NoError(t, err) require.NotNil(t, usage) require.Equal(t, 4, usage.ImageOutputTokens) @@ -3621,7 +3621,7 @@ func TestHandleSSEToJSON_NoFinalResponseKeepsSSEBody(t *testing.T) { `data: [DONE]`, }, "\n")) - usage, err := svc.handleSSEToJSON(resp, c, body, "gpt-4o", "gpt-4o") + usage, err := svc.handleSSEToJSON(resp, c, nil, body, "gpt-4o", "gpt-4o") require.NoError(t, err) require.NotNil(t, usage) require.Equal(t, 0, usage.InputTokens) @@ -3645,7 +3645,7 @@ func TestHandleSSEToJSON_ResponseFailedReturnsProtocolError(t *testing.T) { `data: [DONE]`, }, "\n")) - usage, err := svc.handleSSEToJSON(resp, c, body, "gpt-4o", "gpt-4o") + usage, err := svc.handleSSEToJSON(resp, c, nil, body, "gpt-4o", "gpt-4o") require.Nil(t, usage) require.Error(t, err) require.Equal(t, http.StatusBadGateway, rec.Code) diff --git a/backend/internal/service/openai_oauth_passthrough_test.go b/backend/internal/service/openai_oauth_passthrough_test.go index 2b3b00176..d2c030c5c 100644 --- a/backend/internal/service/openai_oauth_passthrough_test.go +++ b/backend/internal/service/openai_oauth_passthrough_test.go @@ -735,12 +735,17 @@ func TestOpenAIGatewayService_OAuthPassthrough_CompactUsesJSONAndKeepsNonStreami } account := &Account{ - ID: 123, - Name: "acc", - Platform: PlatformOpenAI, - Type: AccountTypeOAuth, - Concurrency: 1, - Credentials: map[string]any{"access_token": "oauth-token", "chatgpt_account_id": "chatgpt-acc"}, + ID: 123, + Name: "acc", + Platform: PlatformOpenAI, + Type: AccountTypeOAuth, + Concurrency: 1, + Credentials: map[string]any{ + "access_token": "oauth-token", + "chatgpt_account_id": "chatgpt-acc", + "model_mapping": map[string]any{"gpt-5.1-codex": "gpt-5.1-account"}, + "compact_model_mapping": map[string]any{"gpt-5.1-codex": "gpt-5.1-compact"}, + }, Extra: map[string]any{"openai_passthrough": true}, Status: StatusActive, Schedulable: true, @@ -754,7 +759,7 @@ func TestOpenAIGatewayService_OAuthPassthrough_CompactUsesJSONAndKeepsNonStreami require.False(t, gjson.GetBytes(upstream.lastBody, "store").Exists()) require.False(t, gjson.GetBytes(upstream.lastBody, "stream").Exists()) - require.Equal(t, "gpt-5.1-codex", gjson.GetBytes(upstream.lastBody, "model").String()) + require.Equal(t, "gpt-5.1-compact", gjson.GetBytes(upstream.lastBody, "model").String()) require.Equal(t, "compact me", gjson.GetBytes(upstream.lastBody, "input.0.text").String()) require.Equal(t, "local-test-instructions", strings.TrimSpace(gjson.GetBytes(upstream.lastBody, "instructions").String())) require.Equal(t, "application/json", upstream.lastReq.Header.Get("Accept")) @@ -2172,12 +2177,16 @@ func TestOpenAIGatewayService_APIKeyPassthrough_PreservesBodyAndUsesResponsesEnd } account := &Account{ - ID: 456, - Name: "apikey-acc", - Platform: PlatformOpenAI, - Type: AccountTypeAPIKey, - Concurrency: 1, - Credentials: map[string]any{"api_key": "sk-api-key", "base_url": "https://api.openai.com"}, + ID: 456, + Name: "apikey-acc", + Platform: PlatformOpenAI, + Type: AccountTypeAPIKey, + Concurrency: 1, + Credentials: map[string]any{ + "api_key": "sk-api-key", + "base_url": "https://api.openai.com", + "model_mapping": map[string]any{"gpt-5.2": "gpt-5.2-account"}, + }, Extra: map[string]any{"openai_passthrough": true}, Status: StatusActive, Schedulable: true, diff --git a/backend/internal/service/openai_profit_control_pricing_test.go b/backend/internal/service/openai_profit_control_pricing_test.go index 03077d2a8..379789f20 100644 --- a/backend/internal/service/openai_profit_control_pricing_test.go +++ b/backend/internal/service/openai_profit_control_pricing_test.go @@ -1,7 +1,7 @@ package service // 请求级定价与利润门回归：请求级 pricingAt 定价上下文、门复用（failover 阈值稳定）、 -// 生图意图跳门、U 使用账号倍率且与探测新鲜度解耦、 +// Responses 文本能力利润门、U 使用账号倍率且与探测新鲜度解耦、 // 用量记录定价时刻取值。 import ( @@ -109,8 +109,9 @@ func TestProfitControl_UsesAccountRateInsteadOfProbeSnapshot(t *testing.T) { require.Equal(t, openAIProfitFilterReasonThreshold, reason) } -// 显式生图意图（requiredCapability=Responses）在唯一调度入口跳门（图片边界不装门）。 -func TestProfitControl_ResponsesImageIntentSkipsGateAtScheduler(t *testing.T) { +// Responses 是端点能力，不代表媒体请求；原生远程压缩同样要求该能力， +// 因此唯一文本调度入口必须照常安装利润门。 +func TestProfitControl_ResponsesCapabilityUsesTextGateAtScheduler(t *testing.T) { now := time.Now() expensive := upstreamCostTestAccount(51, UpstreamBillingProbeStatusOK, 0.8, now.Add(-time.Minute), 30*time.Minute) expensive.Status = StatusActive @@ -129,12 +130,8 @@ func TestProfitControl_ResponsesImageIntentSkipsGateAtScheduler(t *testing.T) { require.ErrorIs(t, err, ErrNoAvailableAccounts, "文本能力必须过利润门") selection, _, err := svc.SelectAccountWithSchedulerForCapability(ctx, &groupID, "", "", "gpt-test", nil, OpenAIUpstreamTransportAny, OpenAIEndpointCapabilityResponses, false, false, true) - require.NoError(t, err, "生图意图不装门，保持既有调度") - require.NotNil(t, selection) - require.Equal(t, expensive.ID, selection.Account.ID) - if selection.ReleaseFunc != nil { - selection.ReleaseFunc() - } + require.ErrorIs(t, err, ErrNoAvailableAccounts, "Responses 文本能力不得绕过利润门") + require.Nil(t, selection) } // 账号倍率缺失一律视为非法保守拒绝；手工或同步维护了倍率的任意账号类型都按 diff --git a/backend/internal/service/openai_ws_forwarder_payload.go b/backend/internal/service/openai_ws_forwarder_payload.go index c53d9e824..4c02cb30f 100644 --- a/backend/internal/service/openai_ws_forwarder_payload.go +++ b/backend/internal/service/openai_ws_forwarder_payload.go @@ -99,6 +99,12 @@ func (s *OpenAIGatewayService) buildOpenAIWSHeaders( } } } + // 真实 Codex 的 WS 握手同样携带会话级 x-codex-beta-features + // （client.rs build_websocket_headers 复用 build_responses_headers）， + // 客户端未声明时补成默认形态，与 HTTP 出站保持一致。放在客户端头拷贝 + // 之外：该头是账号/会话级属性，不依赖入站请求是否存在，也避免预热与 + // 实际请求因头差异落进不同的连接池兼容分桶。 + applyOpenAICodexBetaFeatures(c, account, headers) // OAuth 账号：将 apiKeyID 混入 session 标识符，防止跨用户会话碰撞。 if account != nil && account.Type == AccountTypeOAuth { apiKeyID := getAPIKeyIDFromContext(c) diff --git a/backend/internal/service/upstream_path_guard_test.go b/backend/internal/service/upstream_path_guard_test.go index 3ea88da28..e04781e6a 100644 --- a/backend/internal/service/upstream_path_guard_test.go +++ b/backend/internal/service/upstream_path_guard_test.go @@ -145,6 +145,34 @@ func TestOpenAIResponsesRequestPathSuffixRejectsNonConformingSubpaths(t *testing } } +func TestIsOpenAIResponsesCompactPathUsesLegacyEndpointShape(t *testing.T) { + legacyPaths := []string{ + "/v1/responses/compact", + "/v1/responses/compact/detail", + "/responses/compact/", + } + for _, path := range legacyPaths { + t.Run("legacy_"+path, func(t *testing.T) { + c := newResponsesSuffixTestContext(t, path) + require.True(t, IsOpenAIResponsesCompactPath(c)) + }) + } + + nonLegacyPaths := []string{ + "/v1/responses", + "/openai/v1/responses", + "/responses", + "/backend-api/codex/responses", + "/v1/responses/resp_123/cancel", + } + for _, path := range nonLegacyPaths { + t.Run("non_legacy_"+path, func(t *testing.T) { + c := newResponsesSuffixTestContext(t, path) + require.False(t, IsOpenAIResponsesCompactPath(c)) + }) + } +} + func TestAppendOpenAIResponsesRequestPathSuffixRefusesUnsafeSuffix(t *testing.T) { // 调用方漏了校验时，拼接函数本身也不得把不合规片段带进上游 URL。 require.Equal(t, chatgptCodexURL, appendOpenAIResponsesRequestPathSuffix(chatgptCodexURL, "/../../x")) diff --git a/backend/migrations/222_group_usage_daily_rollups.sql b/backend/migrations/222_group_usage_daily_rollups.sql new file mode 100644 index 000000000..c16b69b4a --- /dev/null +++ b/backend/migrations/222_group_usage_daily_rollups.sql @@ -0,0 +1,138 @@ +-- /admin/groups 分组用量日汇总。 +-- 迁移创建结构与源表失效触发器，历史数据由后台聚合作业按持久水位回填。 + +CREATE TABLE IF NOT EXISTS usage_group_daily_rollups ( + bucket_date DATE NOT NULL, + group_id BIGINT NOT NULL, + actual_cost DECIMAL(20, 10) NOT NULL DEFAULT 0, + computed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (bucket_date, group_id) +); + +COMMENT ON TABLE usage_group_daily_rollups IS '按北京时间自然日聚合的分组实际费用。'; +COMMENT ON COLUMN usage_group_daily_rollups.bucket_date IS '北京时间自然日。'; + +CREATE TABLE IF NOT EXISTS usage_group_rollup_state ( + id SMALLINT PRIMARY KEY CHECK (id = 1), + closed_before DATE NOT NULL DEFAULT DATE '1970-01-01', + retained_from TIMESTAMPTZ NOT NULL DEFAULT TIMESTAMPTZ '1970-01-01 00:00:00+00', + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE usage_group_rollup_state IS '分组日汇总的单行发布水位。'; +COMMENT ON COLUMN usage_group_rollup_state.closed_before IS '已完整发布日桶的北京时间日期排他上界。'; + +INSERT INTO usage_group_rollup_state (id, closed_before, retained_from) +VALUES (1, DATE '1970-01-01', TIMESTAMPTZ '1970-01-01 00:00:00+00') +ON CONFLICT (id) DO NOTHING; + +-- 已发布范围的源记录发生变化时，必须在同一事务内后退发布水位。 +-- DELETE/UPDATE 使用行级触发器，以覆盖外键级联、分区表和直接分区写入。 +CREATE OR REPLACE FUNCTION invalidate_group_usage_rollup_state() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +DECLARE + affected_date DATE; + published_before DATE; +BEGIN + IF TG_OP = 'DELETE' THEN + affected_date := (OLD.created_at AT TIME ZONE 'Asia/Shanghai')::date; + ELSE + IF OLD.group_id IS NULL THEN + affected_date := (NEW.created_at AT TIME ZONE 'Asia/Shanghai')::date; + ELSIF NEW.group_id IS NULL THEN + affected_date := (OLD.created_at AT TIME ZONE 'Asia/Shanghai')::date; + ELSE + affected_date := LEAST( + (OLD.created_at AT TIME ZONE 'Asia/Shanghai')::date, + (NEW.created_at AT TIME ZONE 'Asia/Shanghai')::date + ); + END IF; + END IF; + + -- 即使当前已发布水位尚未越过受影响日期，也必须先锁行。 + -- 否则并发关闭作业可能在本事务之后把水位推进，覆盖本次失效。 + SELECT closed_before + INTO published_before + FROM usage_group_rollup_state + WHERE id = 1 + FOR UPDATE; + + IF published_before > affected_date THEN + UPDATE usage_group_rollup_state + SET closed_before = LEAST(closed_before, affected_date), + updated_at = NOW() + WHERE id = 1; + END IF; + + IF TG_OP = 'DELETE' THEN + RETURN OLD; + END IF; + RETURN NEW; +END; +$$; + +-- INSERT 是网关高频路径。transition table 让每个批量 INSERT 只锁一次状态行； +-- KEY SHARE 在普通写入之间兼容，但会与关闭作业的 FOR UPDATE 串行化。 +CREATE OR REPLACE FUNCTION invalidate_group_usage_rollup_state_after_insert() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +DECLARE + affected_date DATE; + published_before DATE; +BEGIN + SELECT MIN((created_at AT TIME ZONE 'Asia/Shanghai')::date) + INTO affected_date + FROM inserted_usage_logs + WHERE group_id IS NOT NULL; + + IF affected_date IS NULL THEN + RETURN NULL; + END IF; + + SELECT closed_before + INTO published_before + FROM usage_group_rollup_state + WHERE id = 1 + FOR KEY SHARE; + + IF published_before > affected_date THEN + UPDATE usage_group_rollup_state + SET closed_before = LEAST(closed_before, affected_date), + updated_at = NOW() + WHERE id = 1; + END IF; + + RETURN NULL; +END; +$$; + +DROP TRIGGER IF EXISTS usage_logs_group_rollup_invalidate_insert ON usage_logs; +CREATE TRIGGER usage_logs_group_rollup_invalidate_insert +AFTER INSERT ON usage_logs +REFERENCING NEW TABLE AS inserted_usage_logs +FOR EACH STATEMENT +EXECUTE FUNCTION invalidate_group_usage_rollup_state_after_insert(); + +DROP TRIGGER IF EXISTS usage_logs_group_rollup_invalidate_delete ON usage_logs; +CREATE TRIGGER usage_logs_group_rollup_invalidate_delete +AFTER DELETE ON usage_logs +FOR EACH ROW +WHEN (OLD.group_id IS NOT NULL) +EXECUTE FUNCTION invalidate_group_usage_rollup_state(); + +DROP TRIGGER IF EXISTS usage_logs_group_rollup_invalidate_update ON usage_logs; +CREATE TRIGGER usage_logs_group_rollup_invalidate_update +AFTER UPDATE OF created_at, group_id, actual_cost ON usage_logs +FOR EACH ROW +WHEN ( + ( + OLD.created_at IS DISTINCT FROM NEW.created_at + OR OLD.group_id IS DISTINCT FROM NEW.group_id + OR OLD.actual_cost IS DISTINCT FROM NEW.actual_cost + ) + AND (OLD.group_id IS NOT NULL OR NEW.group_id IS NOT NULL) +) +EXECUTE FUNCTION invalidate_group_usage_rollup_state(); diff --git a/backend/migrations/223_group_usage_rollup_timezone.sql b/backend/migrations/223_group_usage_rollup_timezone.sql new file mode 100644 index 000000000..8de2bdff9 --- /dev/null +++ b/backend/migrations/223_group_usage_rollup_timezone.sql @@ -0,0 +1,89 @@ +-- 让 /admin/groups 分组日汇总跟随服务端配置时区。 +-- 222 迁移生成的存量日桶均为北京时间，因此新增状态默认标记为 Asia/Shanghai； +-- 服务启动后若当前 TZ 不同，后台同步会检测到不一致并重建日桶。 + +ALTER TABLE usage_group_rollup_state + ADD COLUMN IF NOT EXISTS timezone_name TEXT NOT NULL DEFAULT 'Asia/Shanghai'; + +COMMENT ON COLUMN usage_group_rollup_state.timezone_name IS '当前分组日桶采用的 IANA 时区名称。'; +COMMENT ON COLUMN usage_group_rollup_state.closed_before IS '已完整发布日桶的配置时区日期排他上界。'; +COMMENT ON COLUMN usage_group_daily_rollups.bucket_date IS 'timezone_name 对应时区的自然日。'; + +CREATE OR REPLACE FUNCTION invalidate_group_usage_rollup_state() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +DECLARE + affected_date DATE; + published_before DATE; + configured_timezone TEXT := current_setting('TimeZone'); +BEGIN + IF TG_OP = 'DELETE' THEN + affected_date := (OLD.created_at AT TIME ZONE configured_timezone)::date; + ELSE + IF OLD.group_id IS NULL THEN + affected_date := (NEW.created_at AT TIME ZONE configured_timezone)::date; + ELSIF NEW.group_id IS NULL THEN + affected_date := (OLD.created_at AT TIME ZONE configured_timezone)::date; + ELSE + affected_date := LEAST( + (OLD.created_at AT TIME ZONE configured_timezone)::date, + (NEW.created_at AT TIME ZONE configured_timezone)::date + ); + END IF; + END IF; + + SELECT closed_before + INTO published_before + FROM usage_group_rollup_state + WHERE id = 1 + FOR UPDATE; + + IF published_before > affected_date THEN + UPDATE usage_group_rollup_state + SET closed_before = LEAST(closed_before, affected_date), + updated_at = NOW() + WHERE id = 1; + END IF; + + IF TG_OP = 'DELETE' THEN + RETURN OLD; + END IF; + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION invalidate_group_usage_rollup_state_after_insert() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +DECLARE + affected_date DATE; + published_before DATE; + configured_timezone TEXT := current_setting('TimeZone'); +BEGIN + SELECT MIN((created_at AT TIME ZONE configured_timezone)::date) + INTO affected_date + FROM inserted_usage_logs + WHERE group_id IS NOT NULL; + + IF affected_date IS NULL THEN + RETURN NULL; + END IF; + + SELECT closed_before + INTO published_before + FROM usage_group_rollup_state + WHERE id = 1 + FOR KEY SHARE; + + IF published_before > affected_date THEN + UPDATE usage_group_rollup_state + SET closed_before = LEAST(closed_before, affected_date), + updated_at = NOW() + WHERE id = 1; + END IF; + + RETURN NULL; +END; +$$; diff --git a/backend/migrations/group_usage_rollup_migration_test.go b/backend/migrations/group_usage_rollup_migration_test.go new file mode 100644 index 000000000..f2e9a5d0a --- /dev/null +++ b/backend/migrations/group_usage_rollup_migration_test.go @@ -0,0 +1,52 @@ +//go:build unit + +package migrations + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMigration222CreatesGroupUsageRollups(t *testing.T) { + content, err := FS.ReadFile("222_group_usage_daily_rollups.sql") + require.NoError(t, err) + + sql := string(content) + require.Contains(t, sql, "CREATE TABLE IF NOT EXISTS usage_group_daily_rollups") + require.Contains(t, sql, "actual_cost DECIMAL(20, 10)") + require.Contains(t, sql, "PRIMARY KEY (bucket_date, group_id)") + require.Contains(t, sql, "CREATE TABLE IF NOT EXISTS usage_group_rollup_state") + require.Contains(t, sql, "CHECK (id = 1)") + require.Contains(t, sql, "TIMESTAMPTZ '1970-01-01 00:00:00+00'") + require.Contains(t, sql, "ON CONFLICT (id) DO NOTHING") +} + +func TestMigration222InvalidatesClosedBucketsWhenUsageLogsChange(t *testing.T) { + content, err := FS.ReadFile("222_group_usage_daily_rollups.sql") + require.NoError(t, err) + + sql := string(content) + require.Contains(t, sql, "CREATE OR REPLACE FUNCTION invalidate_group_usage_rollup_state") + require.Contains(t, sql, "SELECT closed_before") + require.Contains(t, sql, "FOR UPDATE") + require.Contains(t, sql, "FOR KEY SHARE") + require.Contains(t, sql, "REFERENCING NEW TABLE AS inserted_usage_logs") + require.Contains(t, sql, "closed_before = LEAST(closed_before, affected_date)") + require.Contains(t, sql, "CREATE TRIGGER usage_logs_group_rollup_invalidate_insert") + require.Contains(t, sql, "CREATE TRIGGER usage_logs_group_rollup_invalidate_delete") + require.Contains(t, sql, "CREATE TRIGGER usage_logs_group_rollup_invalidate_update") + require.Contains(t, sql, "AFTER UPDATE OF created_at, group_id, actual_cost") +} + +func TestMigration223TracksConfiguredTimezone(t *testing.T) { + content, err := FS.ReadFile("223_group_usage_rollup_timezone.sql") + require.NoError(t, err) + + sql := string(content) + require.Contains(t, sql, "ADD COLUMN IF NOT EXISTS timezone_name TEXT") + require.Contains(t, sql, "DEFAULT 'Asia/Shanghai'") + require.Contains(t, sql, "current_setting('TimeZone')") + require.Contains(t, sql, "CREATE OR REPLACE FUNCTION invalidate_group_usage_rollup_state") + require.Contains(t, sql, "CREATE OR REPLACE FUNCTION invalidate_group_usage_rollup_state_after_insert") +} diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index e17969c9c..4c86452cc 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -3322,8 +3322,8 @@ packages: mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} - nanoid@3.3.17: - resolution: {integrity: sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g==} + nanoid@3.3.18: + resolution: {integrity: sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true @@ -8443,7 +8443,7 @@ snapshots: object-assign: 4.1.1 thenify-all: 1.6.0 - nanoid@3.3.17: {} + nanoid@3.3.18: {} natural-compare@1.4.0: {} @@ -8641,7 +8641,7 @@ snapshots: postcss@8.5.23: dependencies: - nanoid: 3.3.17 + nanoid: 3.3.18 picocolors: 1.1.1 source-map-js: 1.2.1 diff --git a/frontend/src/api/__tests__/admin.groups.usage-summary.spec.ts b/frontend/src/api/__tests__/admin.groups.usage-summary.spec.ts new file mode 100644 index 000000000..c0b2c4bff --- /dev/null +++ b/frontend/src/api/__tests__/admin.groups.usage-summary.spec.ts @@ -0,0 +1,29 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { get } = vi.hoisted(() => ({ + get: vi.fn(), +})) + +vi.mock('@/api/client', () => ({ + apiClient: { get }, +})) + +import { getUsageSummary } from '@/api/admin/groups' + +describe('admin group usage summary API', () => { + beforeEach(() => { + get.mockReset() + get.mockResolvedValue({ data: [] }) + }) + + it('does not send browser timezone parameters', async () => { + const summary = [ + { group_id: 1, today_cost: 1.25, yesterday_cost: 2.5, total_cost: 9.75 }, + ] + get.mockResolvedValue({ data: summary }) + + await expect(getUsageSummary()).resolves.toEqual(summary) + + expect(get).toHaveBeenCalledWith('/admin/groups/usage-summary') + }) +}) diff --git a/frontend/src/api/admin/groups.ts b/frontend/src/api/admin/groups.ts index 3828bf9d7..2c9c80f5e 100644 --- a/frontend/src/api/admin/groups.ts +++ b/frontend/src/api/admin/groups.ts @@ -446,18 +446,15 @@ export async function clearGroupRPMOverrides(id: number): Promise<{ message: str } /** - * Get usage summary (today + cumulative cost) for all groups - * @param timezone - IANA timezone string (e.g. "Asia/Shanghai") + * Get usage summary (today + yesterday + cumulative cost) for all groups * @returns Array of group usage summaries */ -export async function getUsageSummary( - timezone?: string -): Promise<{ group_id: number; today_cost: number; total_cost: number }[]> { +export async function getUsageSummary(): Promise< + { group_id: number; today_cost: number; yesterday_cost: number; total_cost: number }[] +> { const { data } = await apiClient.get< - { group_id: number; today_cost: number; total_cost: number }[] - >('/admin/groups/usage-summary', { - params: timezone ? { timezone } : undefined - }) + { group_id: number; today_cost: number; yesterday_cost: number; total_cost: number }[] + >('/admin/groups/usage-summary') return data } diff --git a/frontend/src/components/account/BulkEditAccountModal.vue b/frontend/src/components/account/BulkEditAccountModal.vue index e7d251792..86379b6bb 100644 --- a/frontend/src/components/account/BulkEditAccountModal.vue +++ b/frontend/src/components/account/BulkEditAccountModal.vue @@ -1535,7 +1535,7 @@ const codexCLIOnlyEnabled = ref(false) const codexCLIOnlyAppServerEnabled = ref(false) type CodexFingerprintMode = 'off' | 'device' | 'session' | 'full' const enableCodexFingerprintMode = ref(false) -const codexFingerprintMode = ref('session') +const codexFingerprintMode = ref('off') const codexFingerprintModeOptions = computed(() => [ { value: 'off' as CodexFingerprintMode, label: t('admin.accounts.openai.codexFingerprintOff') }, { value: 'device' as CodexFingerprintMode, label: t('admin.accounts.openai.codexFingerprintDevice') }, @@ -1829,7 +1829,8 @@ const buildUpdatePayload = (): Record | null => { if (enableCodexFingerprintMode.value) { const extra = ensureExtra() - if (codexFingerprintMode.value !== 'session') { + // off = 默认值，清键即可；device/session/full 是显式 opt-in，必须落键（#5610）。 + if (codexFingerprintMode.value !== 'off') { extra.codex_fingerprint_mode = codexFingerprintMode.value } else { delete extra.codex_fingerprint_mode @@ -2082,7 +2083,7 @@ watch( enableCodexCLIOnly.value = false enableCodexCLIOnlyAppServer.value = false enableCodexFingerprintMode.value = false - codexFingerprintMode.value = 'session' + codexFingerprintMode.value = 'off' enableOpenAICompactMode.value = false enableOpenAICompactModelMapping.value = false enableRpmLimit.value = false diff --git a/frontend/src/components/account/CreateAccountModal.vue b/frontend/src/components/account/CreateAccountModal.vue index f231df8d5..f93b71e57 100644 --- a/frontend/src/components/account/CreateAccountModal.vue +++ b/frontend/src/components/account/CreateAccountModal.vue @@ -3855,7 +3855,7 @@ const openaiAPIKeyResponsesWebSocketV2Mode = ref(OPENAI_WS_MODE_OF const codexCLIOnlyEnabled = ref(false) const codexCLIOnlyAppServerEnabled = ref(false) type CodexFingerprintMode = 'off' | 'device' | 'session' | 'full' -const codexFingerprintMode = ref('session') +const codexFingerprintMode = ref('off') const codexFingerprintModeOptions = computed(() => [ { value: 'off' as CodexFingerprintMode, label: t('admin.accounts.openai.codexFingerprintOff') }, { value: 'device' as CodexFingerprintMode, label: t('admin.accounts.openai.codexFingerprintDevice') }, @@ -4741,7 +4741,7 @@ const resetForm = () => { openaiAPIKeyResponsesWebSocketV2Mode.value = OPENAI_WS_MODE_OFF codexCLIOnlyEnabled.value = false codexCLIOnlyAppServerEnabled.value = false - codexFingerprintMode.value = 'session' + codexFingerprintMode.value = 'off' anthropicPassthroughEnabled.value = false anthropicAPIKeyAuthScheme.value = 'x_api_key' webSearchEmulationMode.value = 'default' @@ -4840,7 +4840,9 @@ const buildOpenAIExtra = (base?: Record): Record(OPENAI_WS_MODE_OF const codexCLIOnlyEnabled = ref(false) const codexCLIOnlyAppServerEnabled = ref(false) type CodexFingerprintMode = 'off' | 'device' | 'session' | 'full' -const codexFingerprintMode = ref('session') +const codexFingerprintMode = ref('off') type CodexImageToolMode = 'inherit' | 'enabled' | 'disabled' | 'block' const codexImageToolMode = ref('inherit') type AnthropicAPIKeyAuthScheme = 'x_api_key' | 'authorization_bearer' @@ -3440,7 +3440,7 @@ const syncFormFromAccount = (newAccount: Account | null) => { openaiAPIKeyResponsesWebSocketV2Mode.value = OPENAI_WS_MODE_OFF codexCLIOnlyEnabled.value = false codexCLIOnlyAppServerEnabled.value = false - codexFingerprintMode.value = 'session' + codexFingerprintMode.value = 'off' codexImageToolMode.value = 'inherit' anthropicPassthroughEnabled.value = false anthropicAPIKeyAuthScheme.value = 'x_api_key' @@ -3494,9 +3494,10 @@ const syncFormFromAccount = (newAccount: Account | null) => { } if (newAccount.type === 'oauth') { const fpMode = extra?.codex_fingerprint_mode as string | undefined + // 缺省/非法值按 off 呈现，与后端 GetCodexFingerprintMode 的 opt-in 语义一致（#5610） codexFingerprintMode.value = (['off', 'device', 'session', 'full'].includes(fpMode || '') ? fpMode as CodexFingerprintMode - : 'session') + : 'off') } const credentials = newAccount.credentials as Record | undefined const compactMappings = credentials?.compact_model_mapping as Record | undefined @@ -4843,9 +4844,10 @@ const handleSubmit = async () => { } } - // 指纹收敛模式：默认 session，不写入；非默认值显式写入（包括 off） + // 指纹收敛模式：默认 off（不写入）；device/session/full 是显式 opt-in， + // 必须落键，否则管理员的选择会被后端当作"未设置"而回落到 off（#5610）。 if (props.account.type === 'oauth') { - if (codexFingerprintMode.value !== 'session') { + if (codexFingerprintMode.value !== 'off') { newExtra.codex_fingerprint_mode = codexFingerprintMode.value } else { delete newExtra.codex_fingerprint_mode diff --git a/frontend/src/i18n/locales/en/admin/accounts.ts b/frontend/src/i18n/locales/en/admin/accounts.ts index 5de4a2fcc..0eb245033 100644 --- a/frontend/src/i18n/locales/en/admin/accounts.ts +++ b/frontend/src/i18n/locales/en/admin/accounts.ts @@ -574,10 +574,10 @@ export default { codexCLIOnlyAppServerDesc: "Effective only when the switch above is on. When enabled, this account also allows third-party clients that embed the Codex engine over the app-server protocol (e.g. Claude Code's codex plugin); they still pass the global engine-fingerprint gate. OR-combined with the global app-server toggle.", codexFingerprintMode: 'Codex fingerprint convergence', - codexFingerprintModeDesc: 'When multiple users share the same OAuth account, converge device/session identifiers to account-level stable values to reduce upstream-visible device and session count. Off = pass through client identifiers as-is.', - codexFingerprintOff: 'Off (passthrough)', + codexFingerprintModeDesc: 'When multiple users share the same OAuth account, converge device/session identifiers to account-level stable values to reduce upstream-visible device and session count. Off by default (client identifiers pass through as-is); opt in explicitly when needed. Some accounts reported quota shrinkage after enabling convergence, so choose based on your own measurements.', + codexFingerprintOff: 'Off (passthrough, default)', codexFingerprintDevice: 'Device only', - codexFingerprintSession: 'Device + Session (recommended)', + codexFingerprintSession: 'Device + Session', codexFingerprintFull: 'Full convergence', codexImageTool: 'Codex image bridge policy', codexImageToolDesc: diff --git a/frontend/src/i18n/locales/en/admin/overview.ts b/frontend/src/i18n/locales/en/admin/overview.ts index 1fcc7bf0e..225865db9 100644 --- a/frontend/src/i18n/locales/en/admin/overview.ts +++ b/frontend/src/i18n/locales/en/admin/overview.ts @@ -821,6 +821,7 @@ export default { userStatus: 'Status' }, usageToday: 'Today', + usageYesterday: 'Yesterday', usageTotal: 'Total', accountsAvailable: 'Avail:', accountsRateLimited: 'Limited:', diff --git a/frontend/src/i18n/locales/zh/admin/accounts.ts b/frontend/src/i18n/locales/zh/admin/accounts.ts index 1359fb0ee..1c6d000c3 100644 --- a/frontend/src/i18n/locales/zh/admin/accounts.ts +++ b/frontend/src/i18n/locales/zh/admin/accounts.ts @@ -644,10 +644,10 @@ export default { codexCLIOnlyAppServer: '允许 Codex app-server 客户端', codexCLIOnlyAppServerDesc: '仅在上方开关开启时生效。开启后本账号额外放行内嵌 Codex 引擎、经 app-server 协议接入的第三方客户端（如 Claude Code 的 codex 插件），仍需通过全局引擎指纹门；与全局 app-server 开关取 OR（任一开即放行）。', codexFingerprintMode: 'Codex 指纹收敛', - codexFingerprintModeDesc: '多人共享同一 OAuth 账号时，将各用户的设备/会话标识收敛为账号级恒定值，减少上游可见的设备数和会话数。关闭时原样透传客户端标识。', - codexFingerprintOff: '关闭（透传）', + codexFingerprintModeDesc: '多人共享同一 OAuth 账号时，将各用户的设备/会话标识收敛为账号级恒定值，减少上游可见的设备数和会话数。默认关闭（原样透传客户端标识），需要时再显式开启；部分账号开启收敛后出现过额度缩水，请按自己的实测结果选择。', + codexFingerprintOff: '关闭（透传，默认）', codexFingerprintDevice: '仅设备', - codexFingerprintSession: '设备+会话（推荐）', + codexFingerprintSession: '设备+会话', codexFingerprintFull: '完全收敛', codexImageTool: 'Codex 图片桥接策略', codexImageToolDesc: diff --git a/frontend/src/i18n/locales/zh/admin/overview.ts b/frontend/src/i18n/locales/zh/admin/overview.ts index f4b02a9ca..200bb6c54 100644 --- a/frontend/src/i18n/locales/zh/admin/overview.ts +++ b/frontend/src/i18n/locales/zh/admin/overview.ts @@ -814,6 +814,7 @@ export default { userStatus: '状态' }, usageToday: '今日', + usageYesterday: '昨日', usageTotal: '累计', accountsAvailable: '可用:', accountsRateLimited: '限流:', diff --git a/frontend/src/views/admin/GroupsView.vue b/frontend/src/views/admin/GroupsView.vue index 5cb036f7b..904b8926d 100644 --- a/frontend/src/views/admin/GroupsView.vue +++ b/frontend/src/views/admin/GroupsView.vue @@ -333,6 +333,16 @@ }} +
+ {{ + t("admin.groups.usageYesterday") + }} + ${{ + formatCost(usageMap.get(row.id)?.yesterday_cost ?? 0) + }} +
{{ t("admin.groups.usageTotal") @@ -4894,6 +4904,7 @@ const groups = ref([]); const loading = ref(false); type GroupUsageSummary = { today_cost: number; + yesterday_cost: number; total_cost: number; }; @@ -5739,12 +5750,12 @@ const loadUsageSummary = async () => { } usageLoading.value = true; try { - const tz = Intl.DateTimeFormat().resolvedOptions().timeZone; - const data = await adminAPI.groups.getUsageSummary(tz); + const data = await adminAPI.groups.getUsageSummary(); const map = new Map(); for (const item of data) { map.set(item.group_id, { today_cost: item.today_cost, + yesterday_cost: item.yesterday_cost, total_cost: item.total_cost, }); } diff --git a/frontend/src/views/admin/__tests__/GroupsView.columnSettings.spec.ts b/frontend/src/views/admin/__tests__/GroupsView.columnSettings.spec.ts index b69cc6bd5..8d4f8df74 100644 --- a/frontend/src/views/admin/__tests__/GroupsView.columnSettings.spec.ts +++ b/frontend/src/views/admin/__tests__/GroupsView.columnSettings.spec.ts @@ -43,6 +43,9 @@ const messages: Record = { 'admin.groups.columns.usage': 'Usage', 'admin.groups.columns.status': 'Status', 'admin.groups.columns.actions': 'Actions', + 'admin.groups.usageToday': 'Today', + 'admin.groups.usageYesterday': 'Yesterday', + 'admin.groups.usageTotal': 'Total', } vi.mock('@/api/admin', () => ({ @@ -151,6 +154,9 @@ const DataTableStub = {
{{ columns.map((col) => col.key).join(',') }}
{{ data.map((row) => row.name).join(',') }}
+
+ +
`, } @@ -378,10 +384,26 @@ describe('admin GroupsView column settings', () => { await openColumnSettings(wrapper) await clickColumnToggle(wrapper, 'Usage') expect(getUsageSummary).toHaveBeenCalledTimes(1) + expect(getUsageSummary).toHaveBeenCalledWith() expect(getCapacitySummary).not.toHaveBeenCalled() await clickColumnToggle(wrapper, 'Capacity') expect(getUsageSummary).toHaveBeenCalledTimes(1) expect(getCapacitySummary).toHaveBeenCalledTimes(1) }) + + it('renders yesterday usage between today and total', async () => { + getUsageSummary.mockResolvedValue([ + { group_id: 1, today_cost: 1.25, yesterday_cost: 2.5, total_cost: 9.75 }, + ]) + + const wrapper = await mountView() + const text = wrapper.get('[data-test="usage-cell"]').text() + + expect(text).toContain('Today$1.25') + expect(text).toContain('Yesterday$2.50') + expect(text).toContain('Total$9.75') + expect(text.indexOf('Today')).toBeLessThan(text.indexOf('Yesterday')) + expect(text.indexOf('Yesterday')).toBeLessThan(text.indexOf('Total')) + }) })