fix(billing-probe): 收口探测资格放宽后的抑制清单与调度信任面
#4896 把上游计费探测从 OpenAI API-key 放宽到全部 API-key 账号后, 遗留了四处需要收口的问题: - M1 官方域抑制清单补 ollama.com。Ollama Cloud 是本仓一等支持配置 (platform openai/anthropic + type apikey + base_url https://ollama.com/v1),放宽后 anthropic 侧这类账号会每个探测周期 拿 Ollama Key 请求 ollama.com/v1/sub2api/billing 并恒定落空,正是该 守卫注释声明要防的行为。后缀匹配已覆盖 www.ollama.com 等子域, notollama.com 等形似域不受影响。 - M2 legacy 低倍率优先排序补平台门控。newOpenAILegacyUpstreamRateOrder 遍历全部候选且无平台门控,与 openAIUpstreamCostFactors 的门控不对称; 放宽后 grok 账号的上游自报倍率开始影响 legacy 调度排序,而实际结算走 本地倍率,中转方自报低价即可吸流量。现补上同一道门控,使调度侧信任面 回到 PR 前状态(探测资格的放宽保持不变)。 - L1 修正 IsUpstreamBillingProbeIdentity 注释。原注释称类型限制的依据是 "OAuth/Bedrock 没有静态 API key",但 AccountTypeUpstream(antigravity 中转账号)同样是 base_url + 静态 api_key 却也被排除。仅改注释如实说明 取舍,不改行为。 - L2 批量探测空选文案去掉 OpenAI 限定(en/zh 成对)。 - L3 unsupported 状态改用加长退避(interval 的 8 倍,仍按 24h 封顶)。 放宽后大量官方域账号会落 unsupported 并按常规 interval 重排,占满每周期 20 个名额,把真正接入 sub2api 的中转账号挤到后面。封顶保证上游后来接入时 最迟一天内会被重新发现;Retry-After 更长时原样保留不被缩短;手动探测不受 退避影响。 测试:ollama.com 官方域行为级与 host 匹配矩阵用例、legacy 排序平台门控 (含混合候选集)用例、unsupported 退避上下界与 runner 跳过/手动探测放行 用例;同步更新既有 unsupported 的 next_probe_at 断言。
This commit is contained in:
1 parent
f3a3d86845
commit
56f3d3c9b0
7 files changed
+166
-9
No files matched your search
@@ -2653,6 +2653,15 @@ func newOpenAILegacyUpstreamRateOrder(accounts []*Account, now time.Time, oauthS
|
||||
var first float64
|
||||
distinct := false
|
||||
for _, account := range accounts {
|
||||
if account == nil {
|
||||
continue
|
||||
}
|
||||
// 与 openAIUpstreamCostFactors 使用同一道平台门控:只有 OpenAI 平台账号
|
||||
// 的倍率参与 legacy 低倍率优先排序。上游自报倍率来自中转方,不能让它对
|
||||
// 其他平台的调度产生影响——否则自报低价即可吸走流量,而实际结算走本地倍率。
|
||||
if !account.IsOpenAIApiKey() && !account.IsOpenAIOAuth() {
|
||||
continue
|
||||
}
|
||||
rate, ok := openAISchedulingRate(account, now, oauthSchedulingRateMultiplier)
|
||||
if !ok {
|
||||
continue
|
||||
|
||||
@@ -491,6 +491,46 @@ func TestOpenAILegacyUpstreamRateOrderRequiresComparableRates(t *testing.T) {
|
||||
require.Negative(t, distinct.compare(&Account{ID: 2}, &Account{ID: 3}))
|
||||
}
|
||||
|
||||
// 探测资格已放宽到全部 API-key 平台,但调度侧的信任面没有跟着扩大:
|
||||
// 只有 OpenAI 平台账号的上游自报倍率参与 legacy 低倍率优先排序,
|
||||
// 否则中转方自报低价即可吸走流量,而实际结算走本地倍率。
|
||||
// 本用例钉死 newOpenAILegacyUpstreamRateOrder 与 openAIUpstreamCostFactors
|
||||
// 使用同一道平台门控。
|
||||
func TestOpenAILegacyUpstreamRateOrderIgnoresNonOpenAIPlatforms(t *testing.T) {
|
||||
now := time.Now()
|
||||
nonOpenAI := func(id int64, platform string, rate float64) *Account {
|
||||
account := upstreamCostTestAccount(id, UpstreamBillingProbeStatusOK, rate, now.Add(-time.Minute), 30*time.Minute)
|
||||
account.Platform = platform
|
||||
return account
|
||||
}
|
||||
grokCheap := nonOpenAI(1, PlatformGrok, 0.01)
|
||||
anthropicExpensive := nonOpenAI(2, PlatformAnthropic, 0.9)
|
||||
|
||||
order := newOpenAILegacyUpstreamRateOrder([]*Account{grokCheap, anthropicExpensive, nil}, now, defaultOpenAIOAuthSchedulingRateMultiplier)
|
||||
require.False(t, order.enabled)
|
||||
require.Empty(t, order.rates)
|
||||
require.Zero(t, order.compare(grokCheap, anthropicExpensive))
|
||||
|
||||
factors := openAIUpstreamCostFactors([]*Account{grokCheap, anthropicExpensive}, now, defaultOpenAIOAuthSchedulingRateMultiplier)
|
||||
require.Equal(t, openAIUpstreamCostNeutralFactor, factors[grokCheap.ID])
|
||||
require.Equal(t, openAIUpstreamCostNeutralFactor, factors[anthropicExpensive.ID])
|
||||
|
||||
// 混合候选集里,非 OpenAI 账号既不进 rates 也不影响 OpenAI 账号之间的排序。
|
||||
openAICheap := upstreamCostTestAccount(3, UpstreamBillingProbeStatusOK, 0.02, now.Add(-time.Minute), 30*time.Minute)
|
||||
openAIExpensive := upstreamCostTestAccount(4, UpstreamBillingProbeStatusOK, 0.12, now.Add(-time.Minute), 30*time.Minute)
|
||||
mixed := newOpenAILegacyUpstreamRateOrder(
|
||||
[]*Account{grokCheap, openAICheap, anthropicExpensive, openAIExpensive},
|
||||
now, defaultOpenAIOAuthSchedulingRateMultiplier,
|
||||
)
|
||||
require.True(t, mixed.enabled)
|
||||
require.Len(t, mixed.rates, 2)
|
||||
require.NotContains(t, mixed.rates, grokCheap.ID)
|
||||
require.NotContains(t, mixed.rates, anthropicExpensive.ID)
|
||||
require.Negative(t, mixed.compare(openAICheap, openAIExpensive))
|
||||
// 自报 0.01 的 grok 账号没有已知倍率,排在有倍率的 OpenAI 账号之后。
|
||||
require.Positive(t, mixed.compare(grokCheap, openAIExpensive))
|
||||
}
|
||||
|
||||
func TestOpenAISchedulingRatePlacesOAuthAtConfiguredReference(t *testing.T) {
|
||||
now := time.Now()
|
||||
cheap := upstreamCostTestAccount(1, UpstreamBillingProbeStatusOK, 0.02, now.Add(-time.Minute), 30*time.Minute)
|
||||
|
||||
@@ -40,6 +40,10 @@ const (
|
||||
upstreamBillingProbeMaxPerCycle = 20
|
||||
upstreamBillingProbeConcurrency = 4
|
||||
upstreamBillingProbeMaxDelay = 24 * time.Hour
|
||||
// unsupported 账号的重探间隔倍数:上游不是 sub2api 中转就不会突然长出
|
||||
// /v1/sub2api/billing,按常规 interval 重排只会持续占满每周期
|
||||
// upstreamBillingProbeMaxPerCycle 个名额。
|
||||
upstreamBillingProbeUnsupportedDelayFactor = 8
|
||||
upstreamBillingProbeLeaderLockKey = "upstream:billing:probe:leader"
|
||||
upstreamBillingProbeLeaderLockTTL = 2 * time.Minute
|
||||
)
|
||||
@@ -661,13 +665,15 @@ func (s *UpstreamBillingProbeService) persistProbeFailure(
|
||||
failureCount = previous.FailureCount + 1
|
||||
}
|
||||
status := UpstreamBillingProbeStatusFailed
|
||||
delay := nextProbeDelay(intervalMinutes, retryAfterDuration)
|
||||
if reason == "unsupported" {
|
||||
status = UpstreamBillingProbeStatusUnsupported
|
||||
delay = unsupportedProbeDelay(intervalMinutes, retryAfterDuration)
|
||||
}
|
||||
snapshot := &UpstreamBillingProbeSnapshot{
|
||||
Status: status,
|
||||
LastAttemptAt: now,
|
||||
NextProbeAt: now.Add(nextProbeDelay(intervalMinutes, retryAfterDuration)),
|
||||
NextProbeAt: now.Add(delay),
|
||||
FailureCount: failureCount,
|
||||
HTTPStatus: statusCode,
|
||||
LastError: reason,
|
||||
@@ -863,8 +869,12 @@ func decodeUpstreamBillingProbeSnapshot(extra map[string]any) *UpstreamBillingPr
|
||||
// sub2api-compatible upstream answers it regardless of the account platform,
|
||||
// so eligibility is not restricted to OpenAI.
|
||||
// Non-sub2api upstreams return 404 and the snapshot records "unsupported".
|
||||
// OAuth/Bedrock credentials carry no static API key to present, hence the
|
||||
// type restriction.
|
||||
// Only AccountTypeAPIKey is in scope. OAuth/Bedrock hold no static API key to
|
||||
// present at all; AccountTypeUpstream (antigravity relay accounts) does carry
|
||||
// a base_url plus a static api_key, but it is deliberately left out of the
|
||||
// current supported set. New antigravity relay accounts are created with
|
||||
// type=apikey by the admin form, so only pre-existing type=upstream rows
|
||||
// cannot turn the probe on.
|
||||
func IsUpstreamBillingProbeIdentity(platform, accountType string) bool {
|
||||
return platform != "" && accountType == AccountTypeAPIKey
|
||||
}
|
||||
@@ -883,12 +893,16 @@ func isUpstreamBillingProbeAccount(account *Account) bool {
|
||||
// relays (the only targets that can answer /v1/sub2api/billing) always do
|
||||
// probe. OpenAI-platform accounts never reach this check: they keep the
|
||||
// upstream-official behavior of probing api.openai.com.
|
||||
// ollama.com is a first-class configuration here (Ollama Cloud accounts are
|
||||
// platform openai/anthropic with base_url https://ollama.com/v1), and it is
|
||||
// an official provider API just like the rest, so it belongs on this list.
|
||||
var upstreamBillingProbeOfficialAPIDomains = []string{
|
||||
"anthropic.com",
|
||||
"googleapis.com",
|
||||
"x.ai",
|
||||
"grok.com",
|
||||
"openai.com",
|
||||
"ollama.com",
|
||||
}
|
||||
|
||||
func upstreamBillingProbeTargetIsOfficialAPI(baseURL string) bool {
|
||||
@@ -953,6 +967,23 @@ func nextProbeDelay(intervalMinutes int, retryAfterDuration time.Duration) time.
|
||||
return interval
|
||||
}
|
||||
|
||||
// unsupportedProbeDelay 拉长 unsupported 账号的重探间隔,让无效候选自然退出
|
||||
// 热队列,不再和真正接入 sub2api 的中转账号抢每周期的探测名额。
|
||||
// 仍按 upstreamBillingProbeMaxDelay 封顶,保证上游后来接入 sub2api 时最迟一天
|
||||
// 内会被重新发现;base 本身已达上限(例如 Retry-After 明确要求更久)时原样返回,
|
||||
// 不缩短上游指令。
|
||||
func unsupportedProbeDelay(intervalMinutes int, retryAfterDuration time.Duration) time.Duration {
|
||||
base := nextProbeDelay(intervalMinutes, retryAfterDuration)
|
||||
if base >= upstreamBillingProbeMaxDelay {
|
||||
return base
|
||||
}
|
||||
stretched := base * upstreamBillingProbeUnsupportedDelayFactor
|
||||
if stretched > upstreamBillingProbeMaxDelay {
|
||||
return upstreamBillingProbeMaxDelay
|
||||
}
|
||||
return stretched
|
||||
}
|
||||
|
||||
func retryAfter(header http.Header, now time.Time) time.Duration {
|
||||
value := strings.TrimSpace(header.Get("Retry-After"))
|
||||
if value == "" {
|
||||
|
||||
@@ -118,6 +118,11 @@ func TestUpstreamBillingProbeOfficialAPIBaseURLIsUnsupportedWithoutRequest(t *te
|
||||
// 跨平台官方域同样必无该端点,一并拦截。
|
||||
{PlatformAnthropic, "https://api.x.ai/v1"},
|
||||
{PlatformGrok, "https://api.openai.com"},
|
||||
// Ollama Cloud 是本仓一等支持配置(platform openai/anthropic +
|
||||
// base_url https://ollama.com/v1),同为官方 API,不能拿 Key 去空探。
|
||||
{PlatformAnthropic, "https://ollama.com/v1"},
|
||||
{PlatformAnthropic, "https://ollama.com"},
|
||||
{PlatformAnthropic, "https://www.ollama.com/v1"},
|
||||
}
|
||||
for i, tc := range cases {
|
||||
account := &Account{
|
||||
@@ -149,11 +154,19 @@ func TestUpstreamBillingProbeOfficialAPIHostMatchingIsNormalized(t *testing.T) {
|
||||
require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://x.ai"))
|
||||
// openai 官方域在全集内;openai 平台账号不经过本判定(行为级测试钉死照探)。
|
||||
require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://api.openai.com"))
|
||||
// Ollama Cloud 官方域及其子域(Ollama Cloud 账号的 base_url 允许带 www.)。
|
||||
require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://ollama.com/v1"))
|
||||
require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://ollama.com:443/v1"))
|
||||
require.True(t, upstreamBillingProbeTargetIsOfficialAPI("https://www.ollama.com/v1"))
|
||||
require.True(t, upstreamBillingProbeTargetIsOfficialAPI("HTTPS://OLLAMA.COM./v1"))
|
||||
// 相似但不同的注册域不拦:中转完全可能叫 *-x.ai 之外的任何名字。
|
||||
require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://relay.example/v1"))
|
||||
require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://notx.ai"))
|
||||
require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://anthropic.com.evil.example"))
|
||||
require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://api.relay-station.example"))
|
||||
require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://notollama.com/v1"))
|
||||
require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://ollama.com.evil.example/v1"))
|
||||
require.False(t, upstreamBillingProbeTargetIsOfficialAPI("https://ollama.example/v1"))
|
||||
}
|
||||
|
||||
// OpenAI 语义保持不变:无自定义 base 时仍探官方域,且沿用 openai 传输画像。
|
||||
|
||||
@@ -499,6 +499,69 @@ func TestUpstreamBillingProbeRetryAfterIsNotShortened(t *testing.T) {
|
||||
require.Equal(t, 48*time.Hour, delay)
|
||||
}
|
||||
|
||||
// unsupported 的重探间隔明显长于普通失败,但始终有上界:上游后来接入 sub2api
|
||||
// 时最迟一天内会被重新发现,且不会缩短上游 Retry-After 指令。
|
||||
func TestUpstreamBillingProbeUnsupportedDelayIsStretchedAndBounded(t *testing.T) {
|
||||
// 默认 30 分钟 interval:普通失败 24~36 分钟,unsupported 为其 8 倍。
|
||||
stretched := unsupportedProbeDelay(30, 0)
|
||||
require.Greater(t, stretched, 36*time.Minute)
|
||||
require.GreaterOrEqual(t, stretched, 192*time.Minute)
|
||||
require.LessOrEqual(t, stretched, 288*time.Minute)
|
||||
|
||||
// 永不超过封顶值,因此 unsupported 账号不会被永久排除在重探之外。
|
||||
require.LessOrEqual(t, unsupportedProbeDelay(upstreamBillingProbeMaxIntervalMinutes, 0), upstreamBillingProbeMaxDelay)
|
||||
require.Positive(t, unsupportedProbeDelay(upstreamBillingProbeMinIntervalMinutes, 0))
|
||||
|
||||
// Retry-After 更长时原样保留,不被封顶缩短;更短时至少不早于该指令。
|
||||
require.Equal(t, 48*time.Hour, unsupportedProbeDelay(30, 48*time.Hour))
|
||||
require.GreaterOrEqual(t, unsupportedProbeDelay(30, time.Hour), time.Hour)
|
||||
}
|
||||
|
||||
// 加长退避只把 unsupported 账号移出周期性热队列,手动探测不受影响。
|
||||
func TestUpstreamBillingProbeUnsupportedBackoffDefersRunnerButNotManualProbe(t *testing.T) {
|
||||
// Ollama Cloud 形态:官方域,不发请求直接落 unsupported。
|
||||
account := &Account{
|
||||
ID: 31,
|
||||
Platform: PlatformAnthropic,
|
||||
Type: AccountTypeAPIKey,
|
||||
Status: StatusActive,
|
||||
Concurrency: 1,
|
||||
Credentials: map[string]any{"api_key": "sk-ollama", "base_url": "https://ollama.com/v1"},
|
||||
Extra: map[string]any{UpstreamBillingProbeEnabledExtraKey: true},
|
||||
}
|
||||
repo := &upstreamBillingProbeAccountRepo{accounts: map[int64]*Account{account.ID: account}}
|
||||
upstream := &upstreamBillingProbeHTTPStub{}
|
||||
settingsRepo := &upstreamBillingProbeSettingRepo{values: map[string]string{
|
||||
SettingKeyUpstreamBillingProbeSettings: `{"enabled":true,"interval_minutes":30}`,
|
||||
}}
|
||||
svc := newUpstreamBillingProbeTestService(repo, upstream, settingsRepo)
|
||||
start := time.Date(2026, time.July, 26, 2, 0, 0, 0, time.UTC)
|
||||
now := start
|
||||
svc.now = func() time.Time { return now }
|
||||
|
||||
require.NoError(t, svc.RunDue(context.Background()))
|
||||
first := decodeUpstreamBillingProbeSnapshot(account.Extra)
|
||||
require.NotNil(t, first)
|
||||
require.Equal(t, UpstreamBillingProbeStatusUnsupported, first.Status)
|
||||
require.Equal(t, start, first.LastAttemptAt)
|
||||
require.False(t, first.NextProbeAt.Before(start.Add(192*time.Minute)))
|
||||
require.Zero(t, upstream.calls.Load())
|
||||
|
||||
// 一个普通失败早就该重探的时间点(远超 36 分钟),runner 仍跳过该账号。
|
||||
now = start.Add(90 * time.Minute)
|
||||
require.NoError(t, svc.RunDue(context.Background()))
|
||||
deferred := decodeUpstreamBillingProbeSnapshot(account.Extra)
|
||||
require.NotNil(t, deferred)
|
||||
require.Equal(t, first.LastAttemptAt, deferred.LastAttemptAt)
|
||||
|
||||
// 手动探测无视退避窗口,管理员随时可以重试。
|
||||
manual, err := svc.ProbeAccount(context.Background(), account.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, UpstreamBillingProbeStatusUnsupported, manual.Status)
|
||||
require.Equal(t, now, manual.LastAttemptAt)
|
||||
require.Equal(t, 2, manual.FailureCount)
|
||||
}
|
||||
|
||||
func TestUpstreamBillingProbeEmptyResponseIsPersistedAsFailure(t *testing.T) {
|
||||
account := &Account{
|
||||
ID: 21,
|
||||
@@ -552,14 +615,15 @@ func TestUpstreamBillingProbeUnsupportedAndAccountToggle(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, UpstreamBillingProbeStatusUnsupported, snapshot.Status)
|
||||
require.Equal(t, "unsupported", snapshot.LastError)
|
||||
require.False(t, snapshot.NextProbeAt.Before(fixedNow.Add(24*time.Minute)))
|
||||
require.False(t, snapshot.NextProbeAt.After(fixedNow.Add(36*time.Minute)))
|
||||
// unsupported 走加长退避:默认 30 分钟 interval ⇒ (24~36) * 8 分钟。
|
||||
require.False(t, snapshot.NextProbeAt.Before(fixedNow.Add(192*time.Minute)))
|
||||
require.False(t, snapshot.NextProbeAt.After(fixedNow.Add(288*time.Minute)))
|
||||
|
||||
snapshot, err = svc.ProbeAccount(context.Background(), account.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, snapshot.FailureCount)
|
||||
require.False(t, snapshot.NextProbeAt.Before(fixedNow.Add(24*time.Minute)))
|
||||
require.False(t, snapshot.NextProbeAt.After(fixedNow.Add(36*time.Minute)))
|
||||
require.False(t, snapshot.NextProbeAt.Before(fixedNow.Add(192*time.Minute)))
|
||||
require.False(t, snapshot.NextProbeAt.After(fixedNow.Add(288*time.Minute)))
|
||||
|
||||
invalid := &Account{ID: 20, Platform: PlatformOpenAI, Type: AccountTypeOAuth}
|
||||
repo.accounts[invalid.ID] = invalid
|
||||
|
||||
@@ -250,7 +250,7 @@ export default {
|
||||
enabled: 'On',
|
||||
disabled: 'Off',
|
||||
probeFailed: 'Failed to probe upstream rate',
|
||||
noEligibleAccounts: 'Select OpenAI API key accounts',
|
||||
noEligibleAccounts: 'Select API key accounts',
|
||||
batchLimit: 'A batch can probe at most 20 accounts',
|
||||
batchCompleted: 'Probed {count} account(s)',
|
||||
batchPartial: 'Probe partially completed: {success} succeeded, {failed} failed'
|
||||
|
||||
@@ -203,7 +203,7 @@ export default {
|
||||
enabled: '打开',
|
||||
disabled: '关闭',
|
||||
probeFailed: '探测上游倍率失败',
|
||||
noEligibleAccounts: '请选择 OpenAI API Key 账号',
|
||||
noEligibleAccounts: '请选择 API Key 账号',
|
||||
batchLimit: '每次最多探测 20 个账号',
|
||||
batchCompleted: '已完成 {count} 个账号的倍率探测',
|
||||
batchPartial: '倍率探测部分完成:成功 {success} 个,失败 {failed} 个'
|
||||
|
||||
Reference in new issue
Block a user