From 532be86e9379ac895a0484740c5cfd7380b444e7 Mon Sep 17 00:00:00 2001 From: Cita Date: Wed, 1 Apr 2026 16:41:25 +0800 Subject: [PATCH] [codex] Add route cooldown controls (#365) * Add route cooldown controls * Address route cooldown review comments --- src/server/config.ts | 13 + src/server/index.ts | 12 +- src/server/routes/api/settings.events.test.ts | 47 +++ src/server/routes/api/settings.ts | 22 +- .../routes/api/tokens.cooldown-clear.test.ts | 281 ++++++++++++++++++ src/server/routes/api/tokens.ts | 14 +- src/server/services/routeCooldownService.ts | 142 +++++++++ src/server/services/tokenRouter.cache.test.ts | 73 +++++ .../services/tokenRouter.selection.test.ts | 82 +++++ src/server/services/tokenRouter.ts | 124 +++++++- src/web/api.ts | 2 + src/web/pages/Settings.tsx | 92 ++++++ src/web/pages/TokenRoutes.tsx | 46 +++ .../pages/settings.route-cooldown.test.tsx | 184 ++++++++++++ src/web/pages/token-routes/RouteCard.test.tsx | 51 ++++ src/web/pages/token-routes/RouteCard.tsx | 14 + 16 files changed, 1182 insertions(+), 17 deletions(-) create mode 100644 src/server/routes/api/tokens.cooldown-clear.test.ts create mode 100644 src/server/services/routeCooldownService.ts create mode 100644 src/web/pages/settings.route-cooldown.test.tsx diff --git a/src/server/config.ts b/src/server/config.ts index ff3fa34..36ff57f 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -7,6 +7,7 @@ const DEFAULT_CODEX_CLIENT_ID = 'app_EMoamEEZ73f0CkXaXp7hrann'; const DEFAULT_CLAUDE_CLIENT_ID = '9d1c250a-e61b-44d9-88ed-5944d1962f5e'; const DEFAULT_GEMINI_CLI_CLIENT_ID = '681255809395-oo8ft2oprdrnp9e3aqf6av3hmdib135j.apps.googleusercontent.com'; const DEFAULT_GEMINI_CLI_CLIENT_SECRET = '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'; +export const TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING = 30 * 24 * 60 * 60; function parseBoolean(value: string | undefined, fallback = false): boolean { if (value === undefined) return fallback; @@ -49,6 +50,15 @@ function parseDbType(value: string | undefined): 'sqlite' | 'mysql' | 'postgres' return 'sqlite'; } +export function normalizeTokenRouterFailureCooldownMaxSec(value: unknown): number | null { + const normalized = Number(value); + if (!Number.isFinite(normalized) || normalized <= 0) return null; + return Math.min( + TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING, + Math.max(1, Math.trunc(normalized)), + ); +} + function parseListenHost(env: NodeJS.ProcessEnv): string { return (env.HOST || '0.0.0.0').trim() || '0.0.0.0'; } @@ -108,6 +118,9 @@ export function buildConfig(env: NodeJS.ProcessEnv) { dbSsl: parseBoolean(env.DB_SSL, false), requestBodyLimit: DEFAULT_REQUEST_BODY_LIMIT, routingFallbackUnitCost: Math.max(1e-6, parseNumber(env.ROUTING_FALLBACK_UNIT_COST, 1)), + tokenRouterFailureCooldownMaxSec: normalizeTokenRouterFailureCooldownMaxSec( + parseNumber(env.TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC, TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING), + ) ?? TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING, tokenRouterCacheTtlMs: Math.max(100, Math.trunc(parseNumber(env.TOKEN_ROUTER_CACHE_TTL_MS, 1_500))), proxyMaxChannelAttempts: Math.max(1, Math.trunc(parseNumber(env.PROXY_MAX_CHANNEL_ATTEMPTS, 3))), proxyStickySessionEnabled: parseBoolean(env.PROXY_STICKY_SESSION_ENABLED, true), diff --git a/src/server/index.ts b/src/server/index.ts index af9bf9a..46dbaae 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -1,7 +1,11 @@ import Fastify from 'fastify'; import cors from '@fastify/cors'; import fastifyStatic from '@fastify/static'; -import { buildFastifyOptions, config } from './config.js'; +import { + buildFastifyOptions, + config, + normalizeTokenRouterFailureCooldownMaxSec, +} from './config.js'; import { normalizePayloadRulesConfig } from './services/payloadRules.js'; import { authMiddleware } from './middleware/auth.js'; import { sitesRoutes } from './routes/api/sites.js'; @@ -287,6 +291,12 @@ function applyRuntimeSettings(settingsMap: Map) { config.routingFallbackUnitCost = Math.max(1e-6, routingFallbackUnitCost); } + const tokenRouterFailureCooldownMaxSec = parseSettingFromMap(settingsMap, 'token_router_failure_cooldown_max_sec'); + const normalizedFailureCooldownMaxSec = normalizeTokenRouterFailureCooldownMaxSec(tokenRouterFailureCooldownMaxSec); + if (normalizedFailureCooldownMaxSec != null) { + config.tokenRouterFailureCooldownMaxSec = normalizedFailureCooldownMaxSec; + } + const webhookUrl = parseSettingFromMap(settingsMap, 'webhook_url'); if (typeof webhookUrl === 'string') config.webhookUrl = webhookUrl; diff --git a/src/server/routes/api/settings.events.test.ts b/src/server/routes/api/settings.events.test.ts index 2c16956..b155600 100644 --- a/src/server/routes/api/settings.events.test.ts +++ b/src/server/routes/api/settings.events.test.ts @@ -65,6 +65,7 @@ describe('settings and auth events', () => { (config as any).proxyDebugRetentionHours = 24; (config as any).proxyDebugMaxBodyBytes = 262144; config.routingFallbackUnitCost = 1; + (config as any).tokenRouterFailureCooldownMaxSec = 30 * 24 * 60 * 60; (config as any).disableCrossProtocolFallback = false; (config as any).telegramEnabled = false; (config as any).telegramApiBaseUrl = 'https://api.telegram.org'; @@ -461,6 +462,52 @@ describe('settings and auth events', () => { expect(runtime.routingFallbackUnitCost).toBe(0.25); }); + it('persists and returns token router failure cooldown cap from runtime settings', async () => { + const updateResponse = await app.inject({ + method: 'PUT', + url: '/api/settings/runtime', + payload: { + tokenRouterFailureCooldownMaxSec: 2 * 24 * 60 * 60, + }, + }); + + expect(updateResponse.statusCode).toBe(200); + const updated = updateResponse.json() as { tokenRouterFailureCooldownMaxSec?: number }; + expect(updated.tokenRouterFailureCooldownMaxSec).toBe(2 * 24 * 60 * 60); + expect((config as any).tokenRouterFailureCooldownMaxSec).toBe(2 * 24 * 60 * 60); + + const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'token_router_failure_cooldown_max_sec')).get(); + expect(saved?.value).toBe(JSON.stringify(2 * 24 * 60 * 60)); + + const getResponse = await app.inject({ + method: 'GET', + url: '/api/settings/runtime', + }); + expect(getResponse.statusCode).toBe(200); + const runtime = getResponse.json() as { tokenRouterFailureCooldownMaxSec?: number }; + expect(runtime.tokenRouterFailureCooldownMaxSec).toBe(2 * 24 * 60 * 60); + }); + + it('clamps token router failure cooldown cap to the supported ceiling', async () => { + const ninetyDaysSec = 90 * 24 * 60 * 60; + const thirtyDaysSec = 30 * 24 * 60 * 60; + const updateResponse = await app.inject({ + method: 'PUT', + url: '/api/settings/runtime', + payload: { + tokenRouterFailureCooldownMaxSec: ninetyDaysSec, + }, + }); + + expect(updateResponse.statusCode).toBe(200); + const updated = updateResponse.json() as { tokenRouterFailureCooldownMaxSec?: number }; + expect(updated.tokenRouterFailureCooldownMaxSec).toBe(thirtyDaysSec); + expect((config as any).tokenRouterFailureCooldownMaxSec).toBe(thirtyDaysSec); + + const saved = await db.select().from(schema.settings).where(eq(schema.settings.key, 'token_router_failure_cooldown_max_sec')).get(); + expect(saved?.value).toBe(JSON.stringify(thirtyDaysSec)); + }); + it('persists and returns disable cross protocol fallback from runtime settings', async () => { const updateResponse = await app.inject({ method: 'PUT', diff --git a/src/server/routes/api/settings.ts b/src/server/routes/api/settings.ts index 0fa246d..62b8b18 100644 --- a/src/server/routes/api/settings.ts +++ b/src/server/routes/api/settings.ts @@ -1,7 +1,7 @@ import { FastifyInstance } from 'fastify'; import cron from 'node-cron'; import { fetch } from 'undici'; -import { config } from '../../config.js'; +import { config, normalizeTokenRouterFailureCooldownMaxSec } from '../../config.js'; import { db, runtimeDbDialect, schema } from '../../db/index.js'; import { upsertSetting } from '../../db/upsertSetting.js'; import * as routeRefreshWorkflow from '../../services/routeRefreshWorkflow.js'; @@ -90,6 +90,7 @@ interface RuntimeSettingsBody { notifyCooldownSec?: number; adminIpAllowlist?: string[] | string; routingFallbackUnitCost?: number; + tokenRouterFailureCooldownMaxSec?: number; routingWeights?: Partial; proxyErrorKeywords?: string[] | string; proxyEmptyContentFailEnabled?: boolean; @@ -665,6 +666,12 @@ function applyImportedSettingToRuntime(key: string, value: unknown) { config.routingFallbackUnitCost = Math.max(1e-6, n); return; } + case 'token_router_failure_cooldown_max_sec': { + const normalized = normalizeTokenRouterFailureCooldownMaxSec(value); + if (normalized == null) return; + config.tokenRouterFailureCooldownMaxSec = normalized; + return; + } default: return; } @@ -694,6 +701,7 @@ function getRuntimeSettingsResponse(currentAdminIp = '') { proxyDebugRetentionHours: config.proxyDebugRetentionHours, proxyDebugMaxBodyBytes: config.proxyDebugMaxBodyBytes, routingFallbackUnitCost: config.routingFallbackUnitCost, + tokenRouterFailureCooldownMaxSec: config.tokenRouterFailureCooldownMaxSec, routingWeights: config.routingWeights, webhookUrl: config.webhookUrl, barkUrl: config.barkUrl, @@ -1585,6 +1593,18 @@ export async function settingsRoutes(app: FastifyInstance) { upsertSetting('routing_fallback_unit_cost', normalized); } + if (body.tokenRouterFailureCooldownMaxSec !== undefined) { + const normalized = normalizeTokenRouterFailureCooldownMaxSec(body.tokenRouterFailureCooldownMaxSec); + if (normalized == null) { + return reply.code(400).send({ success: false, message: '路由失败冷却上限必须是大于 0 的数字（秒）' }); + } + if (normalized !== config.tokenRouterFailureCooldownMaxSec) { + changedLabels.push(`路由失败冷却上限（${config.tokenRouterFailureCooldownMaxSec}s -> ${normalized}s）`); + } + config.tokenRouterFailureCooldownMaxSec = normalized; + upsertSetting('token_router_failure_cooldown_max_sec', normalized); + } + if (changedLabels.length > 0) { let eventType: 'checkin' | 'balance' | 'proxy' | 'status' | 'token' = 'status'; if (changedLabels.length === 1) { diff --git a/src/server/routes/api/tokens.cooldown-clear.test.ts b/src/server/routes/api/tokens.cooldown-clear.test.ts new file mode 100644 index 0000000..35c379c --- /dev/null +++ b/src/server/routes/api/tokens.cooldown-clear.test.ts @@ -0,0 +1,281 @@ +import Fastify, { type FastifyInstance } from 'fastify'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { eq } from 'drizzle-orm'; + +type DbModule = typeof import('../../db/index.js'); + +describe('POST /api/routes/:id/cooldown/clear', () => { + let app: FastifyInstance; + let db: DbModule['db']; + let schema: DbModule['schema']; + let dataDir = ''; + let seedId = 0; + + const nextId = () => { + seedId += 1; + return seedId; + }; + + const seedAccountWithToken = async () => { + const id = nextId(); + const site = await db.insert(schema.sites).values({ + name: `cooldown-site-${id}`, + url: `https://cooldown-site-${id}.example.com`, + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: `cooldown-user-${id}`, + accessToken: `cooldown-access-token-${id}`, + status: 'active', + }).returning().get(); + + const token = await db.insert(schema.accountTokens).values({ + accountId: account.id, + name: `cooldown-token-${id}`, + token: `sk-cooldown-token-${id}`, + enabled: true, + isDefault: true, + }).returning().get(); + + return { site, account, token }; + }; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-route-cooldown-clear-')); + process.env.DATA_DIR = dataDir; + + await import('../../db/migrate.js'); + const dbModule = await import('../../db/index.js'); + const routesModule = await import('./tokens.js'); + + db = dbModule.db; + schema = dbModule.schema; + + app = Fastify(); + await app.register(routesModule.tokensRoutes); + }); + + beforeEach(async () => { + await db.delete(schema.routeGroupSources).run(); + await db.delete(schema.routeChannels).run(); + await db.delete(schema.tokenRoutes).run(); + await db.delete(schema.accountTokens).run(); + await db.delete(schema.accounts).run(); + await db.delete(schema.sites).run(); + seedId = 0; + }); + + afterAll(async () => { + await app.close(); + delete process.env.DATA_DIR; + }); + + it('clears cooldown and failure counters for a direct route', async () => { + const seeded = await seedAccountWithToken(); + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-4o-mini', + enabled: true, + }).returning().get(); + + const channel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: seeded.account.id, + tokenId: seeded.token.id, + priority: 0, + weight: 10, + enabled: true, + failCount: 8, + lastFailAt: '2026-04-01T00:00:00.000Z', + consecutiveFailCount: 2, + cooldownLevel: 3, + cooldownUntil: '2099-01-01T00:00:00.000Z', + }).returning().get(); + + const response = await app.inject({ + method: 'POST', + url: `/api/routes/${route.id}/cooldown/clear`, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + success: true, + clearedChannels: 1, + }); + + const refreshed = await db.select().from(schema.routeChannels) + .where(eq(schema.routeChannels.id, channel.id)) + .get(); + + expect(refreshed).toMatchObject({ + failCount: 0, + lastFailAt: null, + consecutiveFailCount: 0, + cooldownLevel: 0, + cooldownUntil: null, + }); + }); + + it('clears cooldown for source-route channels exposed by explicit groups', async () => { + const seeded = await seedAccountWithToken(); + const sourceRoute = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'claude-sonnet-4-5', + enabled: true, + }).returning().get(); + + const groupRoute = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'claude-opus-4-6', + displayName: 'claude-opus-4-6', + routeMode: 'explicit_group', + enabled: true, + }).returning().get(); + + await db.insert(schema.routeGroupSources).values({ + groupRouteId: groupRoute.id, + sourceRouteId: sourceRoute.id, + }).run(); + + const channel = await db.insert(schema.routeChannels).values({ + routeId: sourceRoute.id, + accountId: seeded.account.id, + tokenId: seeded.token.id, + sourceModel: 'claude-sonnet-4-5', + priority: 0, + weight: 10, + enabled: true, + failCount: 5, + lastFailAt: '2026-04-01T00:00:00.000Z', + consecutiveFailCount: 1, + cooldownLevel: 2, + cooldownUntil: '2099-01-01T00:00:00.000Z', + }).returning().get(); + + const response = await app.inject({ + method: 'POST', + url: `/api/routes/${groupRoute.id}/cooldown/clear`, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + success: true, + clearedChannels: 1, + }); + + const refreshed = await db.select().from(schema.routeChannels) + .where(eq(schema.routeChannels.id, channel.id)) + .get(); + + expect(refreshed).toMatchObject({ + failCount: 0, + lastFailAt: null, + consecutiveFailCount: 0, + cooldownLevel: 0, + cooldownUntil: null, + }); + }); + + it('only clears cooldown for explicit-group source routes that are enabled exact routes', async () => { + const seeded = await seedAccountWithToken(); + const visibleSourceRoute = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-5.4', + enabled: true, + }).returning().get(); + const disabledSourceRoute = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-4.1', + enabled: false, + }).returning().get(); + const wildcardSourceRoute = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-*', + enabled: true, + }).returning().get(); + + const groupRoute = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-clear-group', + displayName: 'gpt-clear-group', + routeMode: 'explicit_group', + enabled: true, + }).returning().get(); + + await db.insert(schema.routeGroupSources).values([ + { groupRouteId: groupRoute.id, sourceRouteId: visibleSourceRoute.id }, + { groupRouteId: groupRoute.id, sourceRouteId: disabledSourceRoute.id }, + { groupRouteId: groupRoute.id, sourceRouteId: wildcardSourceRoute.id }, + ]).run(); + + const visibleChannel = await db.insert(schema.routeChannels).values({ + routeId: visibleSourceRoute.id, + accountId: seeded.account.id, + tokenId: seeded.token.id, + priority: 0, + weight: 10, + enabled: true, + failCount: 4, + lastFailAt: '2026-04-01T00:00:00.000Z', + consecutiveFailCount: 1, + cooldownLevel: 2, + cooldownUntil: '2099-01-01T00:00:00.000Z', + }).returning().get(); + const disabledChannel = await db.insert(schema.routeChannels).values({ + routeId: disabledSourceRoute.id, + accountId: seeded.account.id, + tokenId: seeded.token.id, + priority: 0, + weight: 10, + enabled: true, + failCount: 6, + lastFailAt: '2026-04-01T00:00:00.000Z', + consecutiveFailCount: 2, + cooldownLevel: 3, + cooldownUntil: '2099-01-01T00:00:00.000Z', + }).returning().get(); + const wildcardChannel = await db.insert(schema.routeChannels).values({ + routeId: wildcardSourceRoute.id, + accountId: seeded.account.id, + tokenId: seeded.token.id, + priority: 0, + weight: 10, + enabled: true, + failCount: 7, + lastFailAt: '2026-04-01T00:00:00.000Z', + consecutiveFailCount: 2, + cooldownLevel: 3, + cooldownUntil: '2099-01-01T00:00:00.000Z', + }).returning().get(); + + const response = await app.inject({ + method: 'POST', + url: `/api/routes/${groupRoute.id}/cooldown/clear`, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + success: true, + clearedChannels: 1, + }); + + const refreshedVisible = await db.select().from(schema.routeChannels) + .where(eq(schema.routeChannels.id, visibleChannel.id)) + .get(); + const refreshedDisabled = await db.select().from(schema.routeChannels) + .where(eq(schema.routeChannels.id, disabledChannel.id)) + .get(); + const refreshedWildcard = await db.select().from(schema.routeChannels) + .where(eq(schema.routeChannels.id, wildcardChannel.id)) + .get(); + + expect(refreshedVisible).toMatchObject({ + failCount: 0, + lastFailAt: null, + consecutiveFailCount: 0, + cooldownLevel: 0, + cooldownUntil: null, + }); + expect(refreshedDisabled?.cooldownUntil).toBe('2099-01-01T00:00:00.000Z'); + expect(refreshedWildcard?.cooldownUntil).toBe('2099-01-01T00:00:00.000Z'); + }); +}); diff --git a/src/server/routes/api/tokens.ts b/src/server/routes/api/tokens.ts index 86e197c..911b032 100644 --- a/src/server/routes/api/tokens.ts +++ b/src/server/routes/api/tokens.ts @@ -20,6 +20,7 @@ import { parseRouteDecisionSnapshot, saveRouteDecisionSnapshots, } from '../../services/routeDecisionSnapshotStore.js'; +import { clearRouteCooldown } from '../../services/routeCooldownService.js'; import { normalizeTokenRouteMode, type RouteMode } from '../../../shared/tokenRouteContract.js'; import { parseRouteChannelBatchCreatePayload, @@ -768,8 +769,17 @@ export async function tokensRoutes(app: FastifyInstance) { const channelsByRoute = await fetchChannelsForRouteRows([route]); return channelsByRoute.get(routeId) || []; }); - - // Batch add channels to a route + + app.post<{ Params: { id: string } }>('/api/routes/:id/cooldown/clear', async (request, reply) => { + const routeId = parseInt(request.params.id, 10); + const result = await clearRouteCooldown(routeId); + if (!result) { + return reply.code(404).send({ success: false, message: '路由不存在' }); + } + return result; + }); + + // Batch add channels to a route app.post<{ Params: { id: string }; Body: unknown }>('/api/routes/:id/channels/batch', async (request, reply) => { const parsedBody = parseRouteChannelBatchCreatePayload(request.body); if (!parsedBody.success) { diff --git a/src/server/services/routeCooldownService.ts b/src/server/services/routeCooldownService.ts new file mode 100644 index 0000000..9e2e90b --- /dev/null +++ b/src/server/services/routeCooldownService.ts @@ -0,0 +1,142 @@ +import { eq, inArray } from 'drizzle-orm'; +import { db, schema } from '../db/index.js'; +import { + clearRouteDecisionSnapshot, + clearRouteDecisionSnapshots, +} from './routeDecisionSnapshotStore.js'; +import { tokenRouter } from './tokenRouter.js'; +import { normalizeTokenRouteMode, type RouteMode } from '../../shared/tokenRouteContract.js'; + +type RouteRow = typeof schema.tokenRoutes.$inferSelect & { + routeMode: RouteMode; + sourceRouteIds: number[]; +}; + +function isExactModelPattern(modelPattern: string): boolean { + const normalized = modelPattern.trim(); + if (!normalized) return false; + if (normalized.toLowerCase().startsWith('re:')) return false; + return !/[\*\?]/.test(normalized); +} + +function isExplicitGroupRoute(route: Pick | Pick): boolean { + return normalizeTokenRouteMode(route.routeMode) === 'explicit_group'; +} + +function normalizeSourceRouteIds(sourceRouteIds: number[]): number[] { + return Array.from(new Set( + sourceRouteIds + .filter((routeId): routeId is number => Number.isFinite(routeId) && routeId > 0) + .map((routeId) => Math.trunc(routeId)), + )); +} + +async function loadRouteSourceIdsMap(routeIds: number[]): Promise> { + const normalizedRouteIds = normalizeSourceRouteIds(routeIds); + if (normalizedRouteIds.length === 0) return new Map(); + + const rows = await db.select().from(schema.routeGroupSources) + .where(inArray(schema.routeGroupSources.groupRouteId, normalizedRouteIds)) + .all(); + const sourceRouteIdsByRouteId = new Map(); + for (const row of rows) { + if (!sourceRouteIdsByRouteId.has(row.groupRouteId)) { + sourceRouteIdsByRouteId.set(row.groupRouteId, []); + } + sourceRouteIdsByRouteId.get(row.groupRouteId)!.push(row.sourceRouteId); + } + for (const [routeId, sourceRouteIds] of sourceRouteIdsByRouteId.entries()) { + sourceRouteIdsByRouteId.set(routeId, normalizeSourceRouteIds(sourceRouteIds)); + } + return sourceRouteIdsByRouteId; +} + +function decorateRoutesWithSources( + routes: Array, + sourceRouteIdsByRouteId: Map, +): RouteRow[] { + return routes.map((route) => ({ + ...route, + routeMode: normalizeTokenRouteMode(route.routeMode), + sourceRouteIds: sourceRouteIdsByRouteId.get(route.id) ?? [], + })); +} + +async function getRouteWithSources(routeId: number): Promise { + const route = await db.select().from(schema.tokenRoutes).where(eq(schema.tokenRoutes.id, routeId)).get(); + if (!route) return null; + const sourceRouteIdsByRouteId = await loadRouteSourceIdsMap([routeId]); + return decorateRoutesWithSources([route], sourceRouteIdsByRouteId)[0] ?? null; +} + +async function resolveCooldownClearRouteIds(route: RouteRow): Promise { + if (!isExplicitGroupRoute(route)) { + return [route.id]; + } + + const sourceRouteIds = normalizeSourceRouteIds(route.sourceRouteIds); + if (sourceRouteIds.length === 0) return []; + + const sourceRoutes = await db.select({ + id: schema.tokenRoutes.id, + modelPattern: schema.tokenRoutes.modelPattern, + routeMode: schema.tokenRoutes.routeMode, + enabled: schema.tokenRoutes.enabled, + }).from(schema.tokenRoutes) + .where(inArray(schema.tokenRoutes.id, sourceRouteIds)) + .all(); + + return sourceRoutes + .filter((sourceRoute) => ( + sourceRoute.enabled + && normalizeTokenRouteMode(sourceRoute.routeMode) !== 'explicit_group' + && isExactModelPattern(sourceRoute.modelPattern) + )) + .map((sourceRoute) => sourceRoute.id); +} + +async function clearDependentExplicitGroupSnapshotsBySourceRouteIds(sourceRouteIds: number[]): Promise { + const normalizedSourceRouteIds = normalizeSourceRouteIds(sourceRouteIds); + if (normalizedSourceRouteIds.length === 0) return; + + const rows = await db.select({ groupRouteId: schema.routeGroupSources.groupRouteId }) + .from(schema.routeGroupSources) + .where(inArray(schema.routeGroupSources.sourceRouteId, normalizedSourceRouteIds)) + .all(); + const dependentRouteIds: number[] = Array.from(new Set( + rows + .map((row) => row.groupRouteId) + .filter((routeId): routeId is number => Number.isFinite(routeId) && routeId > 0), + )); + if (dependentRouteIds.length === 0) return; + await clearRouteDecisionSnapshots(dependentRouteIds); +} + +export async function clearRouteCooldown(routeId: number): Promise<{ success: true; clearedChannels: number } | null> { + const route = await getRouteWithSources(routeId); + if (!route) return null; + + const actualRouteIds = await resolveCooldownClearRouteIds(route); + const channelRows: Array<{ id: number; routeId: number }> = actualRouteIds.length > 0 + ? await db.select({ + id: schema.routeChannels.id, + routeId: schema.routeChannels.routeId, + }).from(schema.routeChannels) + .where(inArray(schema.routeChannels.routeId, actualRouteIds)) + .all() + : []; + + const affectedRouteIds = Array.from(new Set(channelRows.map((row) => row.routeId))); + const clearedChannels = await tokenRouter.clearChannelFailureState(channelRows.map((row) => row.id)); + + await clearRouteDecisionSnapshot(route.id); + if (affectedRouteIds.length > 0) { + await clearRouteDecisionSnapshots(affectedRouteIds); + await clearDependentExplicitGroupSnapshotsBySourceRouteIds(affectedRouteIds); + } + + return { + success: true, + clearedChannels, + }; +} diff --git a/src/server/services/tokenRouter.cache.test.ts b/src/server/services/tokenRouter.cache.test.ts index 1731181..b66ff17 100644 --- a/src/server/services/tokenRouter.cache.test.ts +++ b/src/server/services/tokenRouter.cache.test.ts @@ -13,10 +13,12 @@ describe('TokenRouter runtime cache', () => { let schema: DbModule['schema']; let TokenRouter: TokenRouterModule['TokenRouter']; let invalidateTokenRouterCache: TokenRouterModule['invalidateTokenRouterCache']; + let isChannelRecentlyFailed: TokenRouterModule['isChannelRecentlyFailed']; let resetSiteRuntimeHealthState: TokenRouterModule['resetSiteRuntimeHealthState']; let config: ConfigModule['config']; let dataDir = ''; let originalCacheTtlMs = 0; + let originalFailureCooldownMaxSec = 0; beforeAll(async () => { dataDir = mkdtempSync(join(tmpdir(), 'metapi-token-router-cache-')); @@ -30,9 +32,11 @@ describe('TokenRouter runtime cache', () => { schema = dbModule.schema; TokenRouter = tokenRouterModule.TokenRouter; invalidateTokenRouterCache = tokenRouterModule.invalidateTokenRouterCache; + isChannelRecentlyFailed = tokenRouterModule.isChannelRecentlyFailed; resetSiteRuntimeHealthState = tokenRouterModule.resetSiteRuntimeHealthState; config = configModule.config; originalCacheTtlMs = config.tokenRouterCacheTtlMs; + originalFailureCooldownMaxSec = config.tokenRouterFailureCooldownMaxSec; }); beforeEach(async () => { @@ -43,12 +47,14 @@ describe('TokenRouter runtime cache', () => { await db.delete(schema.accounts).run(); await db.delete(schema.sites).run(); config.tokenRouterCacheTtlMs = 60_000; + config.tokenRouterFailureCooldownMaxSec = originalFailureCooldownMaxSec; invalidateTokenRouterCache(); resetSiteRuntimeHealthState(); }); afterAll(() => { config.tokenRouterCacheTtlMs = originalCacheTtlMs; + config.tokenRouterFailureCooldownMaxSec = originalFailureCooldownMaxSec; invalidateTokenRouterCache(); resetSiteRuntimeHealthState(); delete process.env.DATA_DIR; @@ -175,6 +181,73 @@ describe('TokenRouter runtime cache', () => { expect(thirdCooldownMs).toBeLessThanOrEqual(35_000); }); + it('caps generic failure cooldowns at the configured maximum', async () => { + config.tokenRouterFailureCooldownMaxSec = 20; + + const site = await db.insert(schema.sites).values({ + name: 'capped-cooldown-site', + url: 'https://capped-cooldown.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'capped-cooldown-user', + accessToken: 'capped-cooldown-access-token', + apiToken: 'capped-cooldown-api-token', + status: 'active', + }).returning().get(); + + const token = await db.insert(schema.accountTokens).values({ + accountId: account.id, + name: 'capped-cooldown-token', + token: 'sk-capped-cooldown-token', + enabled: true, + isDefault: true, + }).returning().get(); + + const route = await db.insert(schema.tokenRoutes).values({ + modelPattern: 'gpt-4o-mini', + routingStrategy: 'weighted', + enabled: true, + }).returning().get(); + + const channel = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: account.id, + tokenId: token.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const router = new TokenRouter(); + + await router.recordFailure(channel.id); + await router.recordFailure(channel.id); + + const startedAt = Date.now(); + await router.recordFailure(channel.id); + + const record = await db.select().from(schema.routeChannels) + .where(eq(schema.routeChannels.id, channel.id)) + .get(); + const cooldownMs = Date.parse(String(record?.cooldownUntil || '')) - startedAt; + + expect(cooldownMs).toBeGreaterThanOrEqual(17_000); + expect(cooldownMs).toBeLessThanOrEqual(23_000); + const recentFailureCheckAt = Date.now(); + expect(isChannelRecentlyFailed({ + failCount: 3, + lastFailAt: new Date(recentFailureCheckAt - 19_000).toISOString(), + }, recentFailureCheckAt)).toBe(true); + expect(isChannelRecentlyFailed({ + failCount: 3, + lastFailAt: new Date(recentFailureCheckAt - 21_000).toISOString(), + }, recentFailureCheckAt)).toBe(false); + }); + it('uses codex oauth reset hints for usage-limit cooldowns', async () => { const site = await db.insert(schema.sites).values({ name: 'codex-oauth-site', diff --git a/src/server/services/tokenRouter.selection.test.ts b/src/server/services/tokenRouter.selection.test.ts index 9909e84..038d2c3 100644 --- a/src/server/services/tokenRouter.selection.test.ts +++ b/src/server/services/tokenRouter.selection.test.ts @@ -594,6 +594,88 @@ describe('TokenRouter selection scoring', () => { expect((recoveredCandidateB?.probability || 0)).toBeLessThan(70); }); + it('clears persisted runtime breaker state when channel cooldown is manually cleared', async () => { + config.routingWeights = { + baseWeightFactor: 1, + valueScoreFactor: 0, + costWeight: 0, + balanceWeight: 0, + usageWeight: 0, + }; + + const route = await createRoute('gpt-5.4'); + + const siteA = await createSite('clear-breaker-a'); + const accountA = await createAccount(siteA.id, 'clear-breaker-user-a'); + const tokenA = await createToken(accountA.id, 'clear-breaker-token-a'); + const channelA = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountA.id, + tokenId: tokenA.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const siteB = await createSite('clear-breaker-b'); + const accountB = await createAccount(siteB.id, 'clear-breaker-user-b'); + const tokenB = await createToken(accountB.id, 'clear-breaker-token-b'); + const channelB = await db.insert(schema.routeChannels).values({ + routeId: route.id, + accountId: accountB.id, + tokenId: tokenB.id, + priority: 0, + weight: 10, + enabled: true, + }).returning().get(); + + const router = new TokenRouter(); + for (let index = 0; index < 3; index += 1) { + await router.recordFailure(channelA.id, { + status: 502, + errorText: 'Bad gateway', + modelName: 'gpt-5.4', + }); + } + await db.update(schema.routeChannels).set({ + cooldownUntil: null, + lastFailAt: null, + failCount: 0, + consecutiveFailCount: 0, + cooldownLevel: 0, + }).where(eq(schema.routeChannels.id, channelA.id)).run(); + invalidateTokenRouterCache(); + + let decision = await router.explainSelection('gpt-5.4'); + const breakerCandidateA = decision.candidates.find((candidate) => candidate.channelId === channelA.id); + const breakerCandidateB = decision.candidates.find((candidate) => candidate.channelId === channelB.id); + expect(breakerCandidateA?.reason || '').toContain('熔断'); + expect((breakerCandidateA?.probability || 0)).toBe(0); + expect((breakerCandidateB?.probability || 0)).toBe(100); + + await router.clearChannelFailureState([channelA.id]); + resetSiteRuntimeHealthState(); + invalidateTokenRouterCache(); + + const refreshedChannel = await db.select().from(schema.routeChannels) + .where(eq(schema.routeChannels.id, channelA.id)) + .get(); + expect(refreshedChannel).toMatchObject({ + failCount: 0, + lastFailAt: null, + consecutiveFailCount: 0, + cooldownLevel: 0, + cooldownUntil: null, + }); + + decision = await router.explainSelection('gpt-5.4'); + const recoveredCandidateA = decision.candidates.find((candidate) => candidate.channelId === channelA.id); + const recoveredCandidateB = decision.candidates.find((candidate) => candidate.channelId === channelB.id); + expect(recoveredCandidateA?.reason || '').not.toContain('熔断'); + expect((recoveredCandidateA?.probability || 0)).toBeGreaterThan(30); + expect((recoveredCandidateB?.probability || 0)).toBeLessThan(70); + }); + it('does not open a site breaker for repeated timeout validation errors', async () => { config.routingWeights = { baseWeightFactor: 1, diff --git a/src/server/services/tokenRouter.ts b/src/server/services/tokenRouter.ts index bc35ade..6872992 100644 --- a/src/server/services/tokenRouter.ts +++ b/src/server/services/tokenRouter.ts @@ -1,7 +1,11 @@ import { and, eq, inArray, isNull } from 'drizzle-orm'; import { db, schema } from '../db/index.js'; import { upsertSetting } from '../db/upsertSetting.js'; -import { config } from '../config.js'; +import { + config, + normalizeTokenRouterFailureCooldownMaxSec, + TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING, +} from '../config.js'; import { getCachedModelRoutingReferenceCost, refreshModelPricingCatalog } from './modelPricingService.js'; import { RETRYABLE_TIMEOUT_PATTERNS } from './proxyRetryPolicy.js'; import { @@ -203,6 +207,21 @@ function resolveFailureBackoffSec(failCount?: number | null): number { return Math.min(FAILURE_BACKOFF_BASE_SEC * fibonacciNumber(normalizedFailCount), MAX_FAILURE_BACKOFF_SEC); } +function resolveConfiguredFailureCooldownMaxMs(): number { + const normalized = normalizeTokenRouterFailureCooldownMaxSec(config.tokenRouterFailureCooldownMaxSec) + ?? TOKEN_ROUTER_FAILURE_COOLDOWN_MAX_SEC_CEILING; + return Math.max(1_000, normalized * 1000); +} + +function clampFailureCooldownMs(cooldownMs: number): number { + const normalized = Math.max(0, Math.trunc(cooldownMs)); + return Math.min(normalized, resolveConfiguredFailureCooldownMaxMs()); +} + +function resolveEffectiveFailureCooldownMs(failCount?: number | null): number { + return clampFailureCooldownMs(resolveFailureBackoffSec(failCount) * 1000); +} + function resolveRoundRobinCooldownSec(level: number): number { const normalizedLevel = Math.max(0, Math.min(ROUND_ROBIN_COOLDOWN_LEVELS_SEC.length - 1, Math.trunc(level))); return ROUND_ROBIN_COOLDOWN_LEVELS_SEC[normalizedLevel] ?? 0; @@ -695,6 +714,45 @@ export async function flushSiteRuntimeHealthPersistence(): Promise { } } +function clearRuntimeHealthStatesForChannels(rows: Array<{ + siteId: number; + sourceModel: string | null; + routeModelPattern: string; +}>): boolean { + let changed = false; + const modelKeysBySiteId = new Map>(); + + for (const row of rows) { + if (siteRuntimeHealthStates.delete(row.siteId)) { + changed = true; + } + + const resolvedModelName = normalizeChannelSourceModel(row.sourceModel) + || (isExactRouteModelPattern(row.routeModelPattern) ? row.routeModelPattern.trim() : ''); + const modelKey = normalizeModelAlias(resolvedModelName); + if (!modelKey) continue; + if (!modelKeysBySiteId.has(row.siteId)) { + modelKeysBySiteId.set(row.siteId, new Set()); + } + modelKeysBySiteId.get(row.siteId)!.add(modelKey); + } + + for (const [siteId, modelKeys] of modelKeysBySiteId.entries()) { + const modelStates = siteModelRuntimeHealthStates.get(siteId); + if (!modelStates) continue; + for (const modelKey of modelKeys) { + if (modelStates.delete(modelKey)) { + changed = true; + } + } + if (modelStates.size === 0) { + siteModelRuntimeHealthStates.delete(siteId); + } + } + + return changed; +} + export function getSiteRuntimeHealthMultiplier(siteId: number, nowMs = Date.now()): number { const state = siteRuntimeHealthStates.get(siteId); return getRuntimeHealthMultiplier(state, nowMs); @@ -919,15 +977,16 @@ export function isChannelRecentlyFailed( nowMs = Date.now(), avoidSec = resolveFailureBackoffSec(channel.failCount), ): boolean { - if (avoidSec <= 0) return false; - if ((channel.failCount ?? 0) <= 0) return false; - if (!channel.lastFailAt) return false; - - const failTs = Date.parse(channel.lastFailAt); - if (Number.isNaN(failTs)) return false; - - return nowMs - failTs < avoidSec * 1000; -} + const avoidMs = clampFailureCooldownMs(avoidSec * 1000); + if (avoidMs <= 0) return false; + if ((channel.failCount ?? 0) <= 0) return false; + if (!channel.lastFailAt) return false; + + const failTs = Date.parse(channel.lastFailAt); + if (Number.isNaN(failTs)) return false; + + return nowMs - failTs < avoidMs; +} export function filterRecentlyFailedCandidates( candidates: T[], @@ -1771,6 +1830,44 @@ export class TokenRouter { recordSiteRuntimeSuccess(account.siteId, latencyMs, modelName); } + /** + * Clear persisted failure and cooldown state for the given channels. + */ + async clearChannelFailureState(channelIds: number[]): Promise { + const normalizedChannelIds = Array.from(new Set( + channelIds + .filter((channelId): channelId is number => Number.isFinite(channelId) && channelId > 0) + .map((channelId) => Math.trunc(channelId)), + )); + if (normalizedChannelIds.length === 0) return 0; + + await ensureSiteRuntimeHealthStateLoaded(); + const runtimeHealthRows = await db.select({ + siteId: schema.accounts.siteId, + sourceModel: schema.routeChannels.sourceModel, + routeModelPattern: schema.tokenRoutes.modelPattern, + }).from(schema.routeChannels) + .innerJoin(schema.accounts, eq(schema.routeChannels.accountId, schema.accounts.id)) + .innerJoin(schema.tokenRoutes, eq(schema.routeChannels.routeId, schema.tokenRoutes.id)) + .where(inArray(schema.routeChannels.id, normalizedChannelIds)) + .all(); + + const result = await db.update(schema.routeChannels).set({ + failCount: 0, + lastFailAt: null, + consecutiveFailCount: 0, + cooldownLevel: 0, + cooldownUntil: null, + }).where(inArray(schema.routeChannels.id, normalizedChannelIds)).run(); + + if (clearRuntimeHealthStatesForChannels(runtimeHealthRows)) { + await persistSiteRuntimeHealthState(); + } + + invalidateTokenRouterCache(); + return Number(result?.changes || normalizedChannelIds.length); + } + /** * Record failure and set cooldown. */ @@ -1810,12 +1907,13 @@ export class TokenRouter { if (consecutiveFailCount >= ROUND_ROBIN_FAILURE_THRESHOLD) { cooldownLevel = Math.min(cooldownLevel + 1, ROUND_ROBIN_COOLDOWN_LEVELS_SEC.length - 1); const cooldownSec = resolveRoundRobinCooldownSec(cooldownLevel); - cooldownUntil = cooldownSec > 0 ? new Date(nowMs + cooldownSec * 1000).toISOString() : null; + cooldownUntil = cooldownSec > 0 + ? new Date(nowMs + clampFailureCooldownMs(cooldownSec * 1000)).toISOString() + : null; consecutiveFailCount = 0; } } else { - const cooldownSec = resolveFailureBackoffSec(failCount); - cooldownUntil = new Date(nowMs + cooldownSec * 1000).toISOString(); + cooldownUntil = new Date(nowMs + resolveEffectiveFailureCooldownMs(failCount)).toISOString(); consecutiveFailCount = 0; cooldownLevel = 0; } diff --git a/src/web/api.ts b/src/web/api.ts index 3f4d883..0d93a97 100644 --- a/src/web/api.ts +++ b/src/web/api.ts @@ -386,6 +386,7 @@ export type RuntimeSettingsPayload = { notifyCooldownSec?: number; adminIpAllowlist?: string[] | string; routingFallbackUnitCost?: number; + tokenRouterFailureCooldownMaxSec?: number; routingWeights?: RuntimeRoutingWeightsPayload; proxyErrorKeywords?: string[] | string; proxyEmptyContentFailEnabled?: boolean; @@ -717,6 +718,7 @@ export const api = { addRoute: (data: any) => request('/api/routes', { method: 'POST', body: JSON.stringify(data) }), updateRoute: (id: number, data: any) => request(`/api/routes/${id}`, { method: 'PUT', body: JSON.stringify(data) }), deleteRoute: (id: number) => request(`/api/routes/${id}`, { method: 'DELETE' }), + clearRouteCooldown: (id: number) => request(`/api/routes/${id}/cooldown/clear`, { method: 'POST' }), batchUpdateRoutes: (data: { ids: number[]; action: 'enable' | 'disable' }) => request('/api/routes/batch', { method: 'POST', body: JSON.stringify(data) }), addChannel: (routeId: number, data: any) => request(`/api/routes/${routeId}/channels`, { method: 'POST', body: JSON.stringify(data) }), diff --git a/src/web/pages/Settings.tsx b/src/web/pages/Settings.tsx index 6b925a8..c0dcbbc 100644 --- a/src/web/pages/Settings.tsx +++ b/src/web/pages/Settings.tsx @@ -28,6 +28,13 @@ import { generateDownstreamSkKey } from './helpers/generateDownstreamSkKey.js'; const PROXY_TOKEN_PREFIX = 'sk-'; const FACTORY_RESET_ADMIN_TOKEN = 'change-me-admin-token'; const FACTORY_RESET_CONFIRM_SECONDS = 3; +const SECONDS_PER_DAY = 24 * 60 * 60; +const ROUTE_COOLDOWN_UNIT_OPTIONS = [ + { value: 'second', label: '秒', multiplierSec: 1 }, + { value: 'minute', label: '分钟', multiplierSec: 60 }, + { value: 'hour', label: '小时', multiplierSec: 60 * 60 }, + { value: 'day', label: '天', multiplierSec: SECONDS_PER_DAY }, +] as const; const CHECKIN_SCHEDULE_MODE_OPTIONS = [ { value: 'cron', label: 'Cron' }, { value: 'interval', label: '间隔签到' }, @@ -40,6 +47,7 @@ const CHECKIN_INTERVAL_OPTIONS = Array.from({ length: 24 }, (_, index) => { }; }); type DbDialect = 'sqlite' | 'mysql' | 'postgres'; +type RouteCooldownUnit = typeof ROUTE_COOLDOWN_UNIT_OPTIONS[number]['value']; type RuntimeSettings = { checkinCron: string; @@ -55,6 +63,8 @@ type RuntimeSettings = { proxySessionChannelConcurrencyLimit: number; proxySessionChannelQueueWaitMs: number; routingFallbackUnitCost: number; + routeFailureCooldownMaxValue: number; + routeFailureCooldownMaxUnit: RouteCooldownUnit; routingWeights: RoutingWeights; systemProxyUrl: string; proxyErrorKeywords: string[]; @@ -184,6 +194,35 @@ function inferUrlDialect(connectionString: string): 'mysql' | 'postgres' | null return null; } +function resolveRouteCooldownInput(seconds: number | null | undefined): { + value: number; + unit: RouteCooldownUnit; +} { + const normalizedSeconds = Number.isFinite(Number(seconds)) && Number(seconds) > 0 + ? Math.max(1, Math.trunc(Number(seconds))) + : 30 * SECONDS_PER_DAY; + + for (const option of [...ROUTE_COOLDOWN_UNIT_OPTIONS].reverse()) { + if (normalizedSeconds % option.multiplierSec === 0) { + return { + value: normalizedSeconds / option.multiplierSec, + unit: option.value, + }; + } + } + + return { + value: normalizedSeconds, + unit: 'second', + }; +} + +function toRouteCooldownSeconds(value: number, unit: RouteCooldownUnit): number { + const normalizedValue = Number.isFinite(value) && value > 0 ? Math.max(1, Math.trunc(value)) : 1; + const unitConfig = ROUTE_COOLDOWN_UNIT_OPTIONS.find((option) => option.value === unit) || ROUTE_COOLDOWN_UNIT_OPTIONS[0]; + return normalizedValue * unitConfig.multiplierSec; +} + export default function Settings() { const navigate = useNavigate(); const isMobile = useIsMobile(); @@ -201,6 +240,8 @@ export default function Settings() { proxySessionChannelConcurrencyLimit: 2, proxySessionChannelQueueWaitMs: 1500, routingFallbackUnitCost: 1, + routeFailureCooldownMaxValue: 30, + routeFailureCooldownMaxUnit: 'day', routingWeights: defaultWeights, systemProxyUrl: '', proxyErrorKeywords: [], @@ -414,6 +455,7 @@ export default function Settings() { api.getRuntimeDatabaseConfig(), ]); setMaskedToken(authInfo.masked || '****'); + const routeCooldownInput = resolveRouteCooldownInput(runtimeInfo.tokenRouterFailureCooldownMaxSec); setRuntime({ checkinCron: runtimeInfo.checkinCron || '0 8 * * *', checkinScheduleMode: runtimeInfo.checkinScheduleMode === 'interval' ? 'interval' : 'cron', @@ -438,6 +480,8 @@ export default function Settings() { routingFallbackUnitCost: Number(runtimeInfo.routingFallbackUnitCost) > 0 ? Number(runtimeInfo.routingFallbackUnitCost) : 1, + routeFailureCooldownMaxValue: routeCooldownInput.value, + routeFailureCooldownMaxUnit: routeCooldownInput.unit, routingWeights: { ...defaultWeights, ...(runtimeInfo.routingWeights || {}), @@ -845,6 +889,10 @@ export default function Settings() { await api.updateRuntimeSettings({ routingWeights: runtime.routingWeights, routingFallbackUnitCost: runtime.routingFallbackUnitCost, + tokenRouterFailureCooldownMaxSec: toRouteCooldownSeconds( + runtime.routeFailureCooldownMaxValue, + runtime.routeFailureCooldownMaxUnit, + ), disableCrossProtocolFallback: runtime.disableCrossProtocolFallback, }); toast.success('Routing weights saved'); @@ -1498,6 +1546,50 @@ export default function Settings() { style={inputStyle} /> +
+
+ 普通失败冷却上限 +
+
+ { + const nextValue = Number(e.target.value); + setRuntime((prev) => ({ + ...prev, + routeFailureCooldownMaxValue: Number.isFinite(nextValue) && nextValue > 0 + ? Math.max(1, Math.trunc(nextValue)) + : prev.routeFailureCooldownMaxValue, + })); + }} + style={{ ...inputStyle, flex: '1 1 180px', marginBottom: 0 }} + /> +
+ { + setRuntime((prev) => ({ + ...prev, + routeFailureCooldownMaxUnit: nextValue as RouteCooldownUnit, + })); + }} + options={ROUTE_COOLDOWN_UNIT_OPTIONS.map((option) => ({ + value: option.value, + label: option.label, + }))} + placeholder="选择单位" + /> +
+
+
+ 支持秒、分钟、小时、天。只封顶普通失败与轮询分级冷却；429 限额类冷却仍优先遵循上游 reset 提示，避免过早重试。 +
+
+ ); + }; // Collapsed card if (!expanded) { @@ -325,6 +337,7 @@ function RouteCardInner({
+ {renderClearCooldownButton()} {!readOnlyRoute && (explicitGroupRoute || !exactRoute) && ( )} @@ -358,6 +371,7 @@ function RouteCardInner({
{!readOnlyRoute && (
+ {renderClearCooldownButton()} {!exactRoute && ( )}