From 76c85d0a1345d9aafa537d40d43c0f3d2a56b81d Mon Sep 17 00:00:00 2001 From: Cita Date: Wed, 1 Apr 2026 20:58:37 +0800 Subject: [PATCH] [codex] Split site API endpoint pool from panel host (#373) * feat: add site api endpoint schema scaffolding * feat: wire site api endpoints into backup migration * feat: add site api endpoint selection service * feat: add site api endpoint contracts * feat: route discovery through site api endpoints * feat: route proxy traffic through site api endpoints * feat: add site api endpoint pool editor * fix: keep proxy video tasks on site endpoint pool * fix: restore core test compatibility for site endpoint pool * fix: address endpoint pool review findings --- drizzle/0018_site_api_endpoints.sql | 18 + drizzle/meta/0018_snapshot.json | 2760 +++++++++++++++++ drizzle/meta/_journal.json | 7 + src/server/db/generated/mysql.bootstrap.sql | 4 + src/server/db/generated/mysql.upgrade.sql | 12 +- .../db/generated/postgres.bootstrap.sql | 4 + src/server/db/generated/postgres.upgrade.sql | 12 +- src/server/db/generated/schemaContract.json | 117 + src/server/db/schema.ts | 18 + src/server/db/siteSchemaCompatibility.test.ts | 27 + src/server/db/siteSchemaCompatibility.ts | 25 + src/server/proxy-core/surfaces/chatSurface.ts | 551 ++-- .../surfaces/openAiResponsesSurface.ts | 407 +-- .../proxy-core/surfaces/sharedSurface.ts | 5 +- .../routes/api/accounts.apiSite.test.ts | 271 ++ src/server/routes/api/accounts.ts | 100 +- .../routes/api/sites.api-endpoints.test.ts | 226 ++ src/server/routes/api/sites.ts | 235 +- .../routes/proxy/chat.codex-oauth.test.ts | 26 +- .../routes/proxy/chat.count-tokens.test.ts | 26 +- .../routes/proxy/chat.siteApiEndpoint.test.ts | 239 ++ src/server/routes/proxy/chat.stream.test.ts | 26 +- .../proxy/completions.siteApiEndpoint.test.ts | 213 ++ src/server/routes/proxy/completions.ts | 121 +- .../downstreamClientContext.routes.test.ts | 26 +- src/server/routes/proxy/embeddings.ts | 133 +- src/server/routes/proxy/images.edits.test.ts | 26 +- src/server/routes/proxy/images.ts | 218 +- .../proxy/responses.codex-oauth.test.ts | 26 +- .../proxy/responses.compact-upstream.test.ts | 26 +- .../routes/proxy/responses.websocket.test.ts | 34 +- src/server/routes/proxy/search.test.ts | 26 +- src/server/routes/proxy/search.ts | 94 +- src/server/routes/proxy/videos.test.ts | 240 +- src/server/routes/proxy/videos.ts | 202 +- src/server/services/backupService.test.ts | 91 +- src/server/services/backupService.ts | 31 + .../services/databaseMigrationService.test.ts | 76 + .../services/databaseMigrationService.ts | 38 + .../services/modelService.discovery.test.ts | 263 ++ src/server/services/modelService.ts | 29 +- .../services/platformDiscoveryRegistry.ts | 122 +- src/server/services/proxyVideoTaskStore.ts | 12 + .../services/siteApiEndpointService.test.ts | 340 ++ src/server/services/siteApiEndpointService.ts | 287 ++ src/web/pages/Sites.tsx | 225 +- src/web/pages/helpers/sitesEditor.test.ts | 55 + src/web/pages/helpers/sitesEditor.ts | 113 +- src/web/pages/sites.create-redirect.test.tsx | 29 +- 49 files changed, 7226 insertions(+), 986 deletions(-) create mode 100644 drizzle/0018_site_api_endpoints.sql create mode 100644 drizzle/meta/0018_snapshot.json create mode 100644 src/server/routes/api/accounts.apiSite.test.ts create mode 100644 src/server/routes/api/sites.api-endpoints.test.ts create mode 100644 src/server/routes/proxy/chat.siteApiEndpoint.test.ts create mode 100644 src/server/routes/proxy/completions.siteApiEndpoint.test.ts create mode 100644 src/server/services/siteApiEndpointService.test.ts create mode 100644 src/server/services/siteApiEndpointService.ts diff --git a/drizzle/0018_site_api_endpoints.sql b/drizzle/0018_site_api_endpoints.sql new file mode 100644 index 0000000..36667ad --- /dev/null +++ b/drizzle/0018_site_api_endpoints.sql @@ -0,0 +1,18 @@ +CREATE TABLE `site_api_endpoints` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `site_id` integer NOT NULL, + `url` text NOT NULL, + `enabled` integer DEFAULT true, + `sort_order` integer DEFAULT 0, + `cooldown_until` text, + `last_selected_at` text, + `last_failed_at` text, + `last_failure_reason` text, + `created_at` text DEFAULT (datetime('now')), + `updated_at` text DEFAULT (datetime('now')), + FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`,`url`);--> statement-breakpoint +CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`,`enabled`,`sort_order`);--> statement-breakpoint +CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`,`cooldown_until`); \ No newline at end of file diff --git a/drizzle/meta/0018_snapshot.json b/drizzle/meta/0018_snapshot.json new file mode 100644 index 0000000..f5e3657 --- /dev/null +++ b/drizzle/meta/0018_snapshot.json @@ -0,0 +1,2760 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "04823136-cf24-48e5-a75f-bc269de435c6", + "prevId": "c9629972-818c-430a-a406-d7c283cbb6ff", + "tables": { + "account_tokens": { + "name": "account_tokens", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_group": { + "name": "token_group", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "value_status": { + "name": "value_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'ready'" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'manual'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "account_tokens_account_id_idx": { + "name": "account_tokens_account_id_idx", + "columns": [ + "account_id" + ], + "isUnique": false + }, + "account_tokens_account_enabled_idx": { + "name": "account_tokens_account_enabled_idx", + "columns": [ + "account_id", + "enabled" + ], + "isUnique": false + }, + "account_tokens_enabled_idx": { + "name": "account_tokens_enabled_idx", + "columns": [ + "enabled" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_tokens_account_id_accounts_id_fk": { + "name": "account_tokens_account_id_accounts_id_fk", + "tableFrom": "account_tokens", + "tableTo": "accounts", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "site_id": { + "name": "site_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_token": { + "name": "api_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "balance": { + "name": "balance", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "balance_used": { + "name": "balance_used", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "quota": { + "name": "quota", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "unit_cost": { + "name": "unit_cost", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "value_score": { + "name": "value_score", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'active'" + }, + "is_pinned": { + "name": "is_pinned", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "checkin_enabled": { + "name": "checkin_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "last_checkin_at": { + "name": "last_checkin_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_balance_refresh": { + "name": "last_balance_refresh", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "oauth_provider": { + "name": "oauth_provider", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "oauth_account_key": { + "name": "oauth_account_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "oauth_project_id": { + "name": "oauth_project_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "extra_config": { + "name": "extra_config", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "accounts_site_id_idx": { + "name": "accounts_site_id_idx", + "columns": [ + "site_id" + ], + "isUnique": false + }, + "accounts_status_idx": { + "name": "accounts_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "accounts_site_status_idx": { + "name": "accounts_site_status_idx", + "columns": [ + "site_id", + "status" + ], + "isUnique": false + }, + "accounts_oauth_provider_idx": { + "name": "accounts_oauth_provider_idx", + "columns": [ + "oauth_provider" + ], + "isUnique": false + }, + "accounts_oauth_identity_idx": { + "name": "accounts_oauth_identity_idx", + "columns": [ + "oauth_provider", + "oauth_account_key", + "oauth_project_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "accounts_site_id_sites_id_fk": { + "name": "accounts_site_id_sites_id_fk", + "tableFrom": "accounts", + "tableTo": "sites", + "columnsFrom": [ + "site_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "checkin_logs": { + "name": "checkin_logs", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reward": { + "name": "reward", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "checkin_logs_account_created_at_idx": { + "name": "checkin_logs_account_created_at_idx", + "columns": [ + "account_id", + "created_at" + ], + "isUnique": false + }, + "checkin_logs_created_at_idx": { + "name": "checkin_logs_created_at_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "checkin_logs_status_idx": { + "name": "checkin_logs_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "checkin_logs_account_id_accounts_id_fk": { + "name": "checkin_logs_account_id_accounts_id_fk", + "tableFrom": "checkin_logs", + "tableTo": "accounts", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "downstream_api_keys": { + "name": "downstream_api_keys", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "group_name": { + "name": "group_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "expires_at": { + "name": "expires_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_cost": { + "name": "max_cost", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "used_cost": { + "name": "used_cost", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "max_requests": { + "name": "max_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "used_requests": { + "name": "used_requests", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "supported_models": { + "name": "supported_models", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allowed_route_ids": { + "name": "allowed_route_ids", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "site_weight_multipliers": { + "name": "site_weight_multipliers", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "downstream_api_keys_key_unique": { + "name": "downstream_api_keys_key_unique", + "columns": [ + "key" + ], + "isUnique": true + }, + "downstream_api_keys_name_idx": { + "name": "downstream_api_keys_name_idx", + "columns": [ + "name" + ], + "isUnique": false + }, + "downstream_api_keys_enabled_idx": { + "name": "downstream_api_keys_enabled_idx", + "columns": [ + "enabled" + ], + "isUnique": false + }, + "downstream_api_keys_expires_at_idx": { + "name": "downstream_api_keys_expires_at_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "events": { + "name": "events", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'info'" + }, + "read": { + "name": "read", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "related_id": { + "name": "related_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "related_type": { + "name": "related_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "events_read_created_at_idx": { + "name": "events_read_created_at_idx", + "columns": [ + "read", + "created_at" + ], + "isUnique": false + }, + "events_type_created_at_idx": { + "name": "events_type_created_at_idx", + "columns": [ + "type", + "created_at" + ], + "isUnique": false + }, + "events_created_at_idx": { + "name": "events_created_at_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "model_availability": { + "name": "model_availability", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_name": { + "name": "model_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "available": { + "name": "available", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_manual": { + "name": "is_manual", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "model_availability_account_model_unique": { + "name": "model_availability_account_model_unique", + "columns": [ + "account_id", + "model_name" + ], + "isUnique": true + }, + "model_availability_account_available_idx": { + "name": "model_availability_account_available_idx", + "columns": [ + "account_id", + "available" + ], + "isUnique": false + }, + "model_availability_model_name_idx": { + "name": "model_availability_model_name_idx", + "columns": [ + "model_name" + ], + "isUnique": false + } + }, + "foreignKeys": { + "model_availability_account_id_accounts_id_fk": { + "name": "model_availability_account_id_accounts_id_fk", + "tableFrom": "model_availability", + "tableTo": "accounts", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "proxy_debug_attempts": { + "name": "proxy_debug_attempts", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "trace_id": { + "name": "trace_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempt_index": { + "name": "attempt_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "request_path": { + "name": "request_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_url": { + "name": "target_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "runtime_executor": { + "name": "runtime_executor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "request_headers_json": { + "name": "request_headers_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "request_body_json": { + "name": "request_body_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response_status": { + "name": "response_status", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response_headers_json": { + "name": "response_headers_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response_body_json": { + "name": "response_body_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_error_text": { + "name": "raw_error_text", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "recover_applied": { + "name": "recover_applied", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "downgrade_decision": { + "name": "downgrade_decision", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "downgrade_reason": { + "name": "downgrade_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "memory_write_json": { + "name": "memory_write_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "proxy_debug_attempts_trace_attempt_unique": { + "name": "proxy_debug_attempts_trace_attempt_unique", + "columns": [ + "trace_id", + "attempt_index" + ], + "isUnique": true + }, + "proxy_debug_attempts_trace_created_at_idx": { + "name": "proxy_debug_attempts_trace_created_at_idx", + "columns": [ + "trace_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "proxy_debug_attempts_trace_id_proxy_debug_traces_id_fk": { + "name": "proxy_debug_attempts_trace_id_proxy_debug_traces_id_fk", + "tableFrom": "proxy_debug_attempts", + "tableTo": "proxy_debug_traces", + "columnsFrom": [ + "trace_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "proxy_debug_traces": { + "name": "proxy_debug_traces", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "downstream_path": { + "name": "downstream_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "client_kind": { + "name": "client_kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "trace_hint": { + "name": "trace_hint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requested_model": { + "name": "requested_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "downstream_api_key_id": { + "name": "downstream_api_key_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "request_headers_json": { + "name": "request_headers_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "request_body_json": { + "name": "request_body_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sticky_session_key": { + "name": "sticky_session_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "sticky_hit_channel_id": { + "name": "sticky_hit_channel_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_channel_id": { + "name": "selected_channel_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_route_id": { + "name": "selected_route_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_account_id": { + "name": "selected_account_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_site_id": { + "name": "selected_site_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "selected_site_platform": { + "name": "selected_site_platform", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "endpoint_candidates_json": { + "name": "endpoint_candidates_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "endpoint_runtime_state_json": { + "name": "endpoint_runtime_state_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "decision_summary_json": { + "name": "decision_summary_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "final_status": { + "name": "final_status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "final_http_status": { + "name": "final_http_status", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "final_upstream_path": { + "name": "final_upstream_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "final_response_headers_json": { + "name": "final_response_headers_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "final_response_body_json": { + "name": "final_response_body_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "proxy_debug_traces_created_at_idx": { + "name": "proxy_debug_traces_created_at_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "proxy_debug_traces_session_created_at_idx": { + "name": "proxy_debug_traces_session_created_at_idx", + "columns": [ + "session_id", + "created_at" + ], + "isUnique": false + }, + "proxy_debug_traces_model_created_at_idx": { + "name": "proxy_debug_traces_model_created_at_idx", + "columns": [ + "requested_model", + "created_at" + ], + "isUnique": false + }, + "proxy_debug_traces_final_status_created_at_idx": { + "name": "proxy_debug_traces_final_status_created_at_idx", + "columns": [ + "final_status", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "proxy_files": { + "name": "proxy_files", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "public_id": { + "name": "public_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_type": { + "name": "owner_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "filename": { + "name": "filename", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "purpose": { + "name": "purpose", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "byte_size": { + "name": "byte_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content_base64": { + "name": "content_base64", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "deleted_at": { + "name": "deleted_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "proxy_files_public_id_unique": { + "name": "proxy_files_public_id_unique", + "columns": [ + "public_id" + ], + "isUnique": true + }, + "proxy_files_owner_lookup_idx": { + "name": "proxy_files_owner_lookup_idx", + "columns": [ + "owner_type", + "owner_id", + "deleted_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "proxy_logs": { + "name": "proxy_logs", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "route_id": { + "name": "route_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "channel_id": { + "name": "channel_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "downstream_api_key_id": { + "name": "downstream_api_key_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_requested": { + "name": "model_requested", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_actual": { + "name": "model_actual", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "http_status": { + "name": "http_status", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prompt_tokens": { + "name": "prompt_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completion_tokens": { + "name": "completion_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_tokens": { + "name": "total_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "estimated_cost": { + "name": "estimated_cost", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "billing_details": { + "name": "billing_details", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_family": { + "name": "client_family", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_app_id": { + "name": "client_app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_app_name": { + "name": "client_app_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_confidence": { + "name": "client_confidence", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retry_count": { + "name": "retry_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "proxy_logs_created_at_idx": { + "name": "proxy_logs_created_at_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "proxy_logs_account_created_at_idx": { + "name": "proxy_logs_account_created_at_idx", + "columns": [ + "account_id", + "created_at" + ], + "isUnique": false + }, + "proxy_logs_status_created_at_idx": { + "name": "proxy_logs_status_created_at_idx", + "columns": [ + "status", + "created_at" + ], + "isUnique": false + }, + "proxy_logs_model_actual_created_at_idx": { + "name": "proxy_logs_model_actual_created_at_idx", + "columns": [ + "model_actual", + "created_at" + ], + "isUnique": false + }, + "proxy_logs_downstream_api_key_created_at_idx": { + "name": "proxy_logs_downstream_api_key_created_at_idx", + "columns": [ + "downstream_api_key_id", + "created_at" + ], + "isUnique": false + }, + "proxy_logs_client_app_id_created_at_idx": { + "name": "proxy_logs_client_app_id_created_at_idx", + "columns": [ + "client_app_id", + "created_at" + ], + "isUnique": false + }, + "proxy_logs_client_family_created_at_idx": { + "name": "proxy_logs_client_family_created_at_idx", + "columns": [ + "client_family", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "proxy_video_tasks": { + "name": "proxy_video_tasks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "public_id": { + "name": "public_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "upstream_video_id": { + "name": "upstream_video_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "site_url": { + "name": "site_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_value": { + "name": "token_value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requested_model": { + "name": "requested_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actual_model": { + "name": "actual_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "channel_id": { + "name": "channel_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status_snapshot": { + "name": "status_snapshot", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upstream_response_meta": { + "name": "upstream_response_meta", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_upstream_status": { + "name": "last_upstream_status", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_polled_at": { + "name": "last_polled_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "proxy_video_tasks_public_id_unique": { + "name": "proxy_video_tasks_public_id_unique", + "columns": [ + "public_id" + ], + "isUnique": true + }, + "proxy_video_tasks_upstream_video_id_idx": { + "name": "proxy_video_tasks_upstream_video_id_idx", + "columns": [ + "upstream_video_id" + ], + "isUnique": false + }, + "proxy_video_tasks_created_at_idx": { + "name": "proxy_video_tasks_created_at_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "route_channels": { + "name": "route_channels", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "route_id": { + "name": "route_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_id": { + "name": "token_id", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source_model": { + "name": "source_model", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "weight": { + "name": "weight", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "manual_override": { + "name": "manual_override", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "success_count": { + "name": "success_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "fail_count": { + "name": "fail_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "total_latency_ms": { + "name": "total_latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "total_cost": { + "name": "total_cost", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "last_used_at": { + "name": "last_used_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_selected_at": { + "name": "last_selected_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_fail_at": { + "name": "last_fail_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "consecutive_fail_count": { + "name": "consecutive_fail_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cooldown_level": { + "name": "cooldown_level", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cooldown_until": { + "name": "cooldown_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "route_channels_route_id_idx": { + "name": "route_channels_route_id_idx", + "columns": [ + "route_id" + ], + "isUnique": false + }, + "route_channels_account_id_idx": { + "name": "route_channels_account_id_idx", + "columns": [ + "account_id" + ], + "isUnique": false + }, + "route_channels_token_id_idx": { + "name": "route_channels_token_id_idx", + "columns": [ + "token_id" + ], + "isUnique": false + }, + "route_channels_route_enabled_idx": { + "name": "route_channels_route_enabled_idx", + "columns": [ + "route_id", + "enabled" + ], + "isUnique": false + }, + "route_channels_route_token_idx": { + "name": "route_channels_route_token_idx", + "columns": [ + "route_id", + "token_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "route_channels_route_id_token_routes_id_fk": { + "name": "route_channels_route_id_token_routes_id_fk", + "tableFrom": "route_channels", + "tableTo": "token_routes", + "columnsFrom": [ + "route_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "route_channels_account_id_accounts_id_fk": { + "name": "route_channels_account_id_accounts_id_fk", + "tableFrom": "route_channels", + "tableTo": "accounts", + "columnsFrom": [ + "account_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "route_channels_token_id_account_tokens_id_fk": { + "name": "route_channels_token_id_account_tokens_id_fk", + "tableFrom": "route_channels", + "tableTo": "account_tokens", + "columnsFrom": [ + "token_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "route_group_sources": { + "name": "route_group_sources", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "group_route_id": { + "name": "group_route_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_route_id": { + "name": "source_route_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "route_group_sources_group_source_unique": { + "name": "route_group_sources_group_source_unique", + "columns": [ + "group_route_id", + "source_route_id" + ], + "isUnique": true + }, + "route_group_sources_source_route_id_idx": { + "name": "route_group_sources_source_route_id_idx", + "columns": [ + "source_route_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "route_group_sources_group_route_id_token_routes_id_fk": { + "name": "route_group_sources_group_route_id_token_routes_id_fk", + "tableFrom": "route_group_sources", + "tableTo": "token_routes", + "columnsFrom": [ + "group_route_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "route_group_sources_source_route_id_token_routes_id_fk": { + "name": "route_group_sources_source_route_id_token_routes_id_fk", + "tableFrom": "route_group_sources", + "tableTo": "token_routes", + "columnsFrom": [ + "source_route_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "settings": { + "name": "settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "site_announcements": { + "name": "site_announcements", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "site_id": { + "name": "site_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'info'" + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "starts_at": { + "name": "starts_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ends_at": { + "name": "ends_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upstream_created_at": { + "name": "upstream_created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "upstream_updated_at": { + "name": "upstream_updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "read_at": { + "name": "read_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dismissed_at": { + "name": "dismissed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_payload": { + "name": "raw_payload", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "site_announcements_site_source_key_unique": { + "name": "site_announcements_site_source_key_unique", + "columns": [ + "site_id", + "source_key" + ], + "isUnique": true + }, + "site_announcements_site_id_first_seen_at_idx": { + "name": "site_announcements_site_id_first_seen_at_idx", + "columns": [ + "site_id", + "first_seen_at" + ], + "isUnique": false + }, + "site_announcements_read_at_idx": { + "name": "site_announcements_read_at_idx", + "columns": [ + "read_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "site_announcements_site_id_sites_id_fk": { + "name": "site_announcements_site_id_sites_id_fk", + "tableFrom": "site_announcements", + "tableTo": "sites", + "columnsFrom": [ + "site_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "site_api_endpoints": { + "name": "site_api_endpoints", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "site_id": { + "name": "site_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "cooldown_until": { + "name": "cooldown_until", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_selected_at": { + "name": "last_selected_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failed_at": { + "name": "last_failed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_failure_reason": { + "name": "last_failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "site_api_endpoints_site_url_unique": { + "name": "site_api_endpoints_site_url_unique", + "columns": [ + "site_id", + "url" + ], + "isUnique": true + }, + "site_api_endpoints_site_enabled_sort_idx": { + "name": "site_api_endpoints_site_enabled_sort_idx", + "columns": [ + "site_id", + "enabled", + "sort_order" + ], + "isUnique": false + }, + "site_api_endpoints_site_cooldown_idx": { + "name": "site_api_endpoints_site_cooldown_idx", + "columns": [ + "site_id", + "cooldown_until" + ], + "isUnique": false + } + }, + "foreignKeys": { + "site_api_endpoints_site_id_sites_id_fk": { + "name": "site_api_endpoints_site_id_sites_id_fk", + "tableFrom": "site_api_endpoints", + "tableTo": "sites", + "columnsFrom": [ + "site_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "site_disabled_models": { + "name": "site_disabled_models", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "site_id": { + "name": "site_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_name": { + "name": "model_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "site_disabled_models_site_model_unique": { + "name": "site_disabled_models_site_model_unique", + "columns": [ + "site_id", + "model_name" + ], + "isUnique": true + }, + "site_disabled_models_site_id_idx": { + "name": "site_disabled_models_site_id_idx", + "columns": [ + "site_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "site_disabled_models_site_id_sites_id_fk": { + "name": "site_disabled_models_site_id_sites_id_fk", + "tableFrom": "site_disabled_models", + "tableTo": "sites", + "columnsFrom": [ + "site_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "sites": { + "name": "sites", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "external_checkin_url": { + "name": "external_checkin_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "proxy_url": { + "name": "proxy_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "use_system_proxy": { + "name": "use_system_proxy", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "custom_headers": { + "name": "custom_headers", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "is_pinned": { + "name": "is_pinned", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "global_weight": { + "name": "global_weight", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 1 + }, + "api_key": { + "name": "api_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "sites_status_idx": { + "name": "sites_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "sites_platform_url_unique": { + "name": "sites_platform_url_unique", + "columns": [ + "platform", + "url" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "token_model_availability": { + "name": "token_model_availability", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "token_id": { + "name": "token_id", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_name": { + "name": "model_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "available": { + "name": "available", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "latency_ms": { + "name": "latency_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "token_model_availability_token_model_unique": { + "name": "token_model_availability_token_model_unique", + "columns": [ + "token_id", + "model_name" + ], + "isUnique": true + }, + "token_model_availability_token_available_idx": { + "name": "token_model_availability_token_available_idx", + "columns": [ + "token_id", + "available" + ], + "isUnique": false + }, + "token_model_availability_model_name_idx": { + "name": "token_model_availability_model_name_idx", + "columns": [ + "model_name" + ], + "isUnique": false + }, + "token_model_availability_available_idx": { + "name": "token_model_availability_available_idx", + "columns": [ + "available" + ], + "isUnique": false + } + }, + "foreignKeys": { + "token_model_availability_token_id_account_tokens_id_fk": { + "name": "token_model_availability_token_id_account_tokens_id_fk", + "tableFrom": "token_model_availability", + "tableTo": "account_tokens", + "columnsFrom": [ + "token_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "token_routes": { + "name": "token_routes", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "model_pattern": { + "name": "model_pattern", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "display_icon": { + "name": "display_icon", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "route_mode": { + "name": "route_mode", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'pattern'" + }, + "model_mapping": { + "name": "model_mapping", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "decision_snapshot": { + "name": "decision_snapshot", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "decision_refreshed_at": { + "name": "decision_refreshed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "routing_strategy": { + "name": "routing_strategy", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "'weighted'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + }, + "updated_at": { + "name": "updated_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(datetime('now'))" + } + }, + "indexes": { + "token_routes_model_pattern_idx": { + "name": "token_routes_model_pattern_idx", + "columns": [ + "model_pattern" + ], + "isUnique": false + }, + "token_routes_enabled_idx": { + "name": "token_routes_enabled_idx", + "columns": [ + "enabled" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index fce734d..608b977 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -120,6 +120,13 @@ "when": 1774692431588, "tag": "0017_proxy_debug_traces", "breakpoints": true + }, + { + "idx": 17, + "version": "6", + "when": 1774959227309, + "tag": "0018_site_api_endpoints", + "breakpoints": true } ] } diff --git a/src/server/db/generated/mysql.bootstrap.sql b/src/server/db/generated/mysql.bootstrap.sql index 6f1c74d..d07535b 100644 --- a/src/server/db/generated/mysql.bootstrap.sql +++ b/src/server/db/generated/mysql.bootstrap.sql @@ -15,6 +15,7 @@ CREATE TABLE IF NOT EXISTS `route_channels` (`id` INT AUTO_INCREMENT NOT NULL PR CREATE TABLE IF NOT EXISTS `route_group_sources` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `group_route_id` INT NOT NULL, `source_route_id` INT NOT NULL, FOREIGN KEY (`group_route_id`) REFERENCES `token_routes`(`id`) ON DELETE CASCADE, FOREIGN KEY (`source_route_id`) REFERENCES `token_routes`(`id`) ON DELETE CASCADE); CREATE TABLE IF NOT EXISTS `settings` (`key` VARCHAR(191) NOT NULL PRIMARY KEY, `value` TEXT); CREATE TABLE IF NOT EXISTS `site_announcements` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `platform` TEXT NOT NULL, `source_key` TEXT NOT NULL, `title` TEXT NOT NULL, `content` TEXT NOT NULL, `level` VARCHAR(191) NOT NULL DEFAULT 'info', `source_url` TEXT, `starts_at` VARCHAR(191), `ends_at` VARCHAR(191), `upstream_created_at` VARCHAR(191), `upstream_updated_at` VARCHAR(191), `first_seen_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `last_seen_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `read_at` VARCHAR(191), `dismissed_at` VARCHAR(191), `raw_payload` TEXT, FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE); +CREATE TABLE IF NOT EXISTS `site_api_endpoints` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `url` TEXT NOT NULL, `enabled` BOOLEAN DEFAULT true, `sort_order` INT DEFAULT 0, `cooldown_until` VARCHAR(191), `last_selected_at` VARCHAR(191), `last_failed_at` VARCHAR(191), `last_failure_reason` TEXT, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE); CREATE TABLE IF NOT EXISTS `site_disabled_models` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `model_name` TEXT NOT NULL, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE); CREATE TABLE IF NOT EXISTS `token_model_availability` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `token_id` INT NOT NULL, `model_name` TEXT NOT NULL, `available` BOOLEAN, `latency_ms` INT, `checked_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`token_id`) REFERENCES `account_tokens`(`id`) ON DELETE CASCADE); CREATE UNIQUE INDEX `downstream_api_keys_key_unique` ON `downstream_api_keys` (`key`(191)); @@ -24,6 +25,7 @@ CREATE UNIQUE INDEX `proxy_files_public_id_unique` ON `proxy_files` (`public_id` CREATE UNIQUE INDEX `proxy_video_tasks_public_id_unique` ON `proxy_video_tasks` (`public_id`(191)); CREATE UNIQUE INDEX `route_group_sources_group_source_unique` ON `route_group_sources` (`group_route_id`, `source_route_id`); CREATE UNIQUE INDEX `site_announcements_site_source_key_unique` ON `site_announcements` (`site_id`, `source_key`(191)); +CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`, `url`(191)); CREATE UNIQUE INDEX `site_disabled_models_site_model_unique` ON `site_disabled_models` (`site_id`, `model_name`(191)); CREATE UNIQUE INDEX `sites_platform_url_unique` ON `sites` (`platform`(191), `url`(191)); CREATE UNIQUE INDEX `token_model_availability_token_model_unique` ON `token_model_availability` (`token_id`, `model_name`(191)); @@ -69,6 +71,8 @@ CREATE INDEX `route_channels_token_id_idx` ON `route_channels` (`token_id`); CREATE INDEX `route_group_sources_source_route_id_idx` ON `route_group_sources` (`source_route_id`); CREATE INDEX `site_announcements_read_at_idx` ON `site_announcements` (`read_at`); CREATE INDEX `site_announcements_site_id_first_seen_at_idx` ON `site_announcements` (`site_id`, `first_seen_at`); +CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`, `cooldown_until`); +CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`, `enabled`, `sort_order`); CREATE INDEX `site_disabled_models_site_id_idx` ON `site_disabled_models` (`site_id`); CREATE INDEX `sites_status_idx` ON `sites` (`status`(191)); CREATE INDEX `token_model_availability_available_idx` ON `token_model_availability` (`available`); diff --git a/src/server/db/generated/mysql.upgrade.sql b/src/server/db/generated/mysql.upgrade.sql index 34444e2..0a3afd7 100644 --- a/src/server/db/generated/mysql.upgrade.sql +++ b/src/server/db/generated/mysql.upgrade.sql @@ -1,8 +1,4 @@ -CREATE TABLE IF NOT EXISTS `proxy_debug_traces` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `downstream_path` TEXT NOT NULL, `client_kind` TEXT, `session_id` TEXT, `trace_hint` TEXT, `requested_model` TEXT, `downstream_api_key_id` INT, `request_headers_json` JSON, `request_body_json` JSON, `sticky_session_key` TEXT, `sticky_hit_channel_id` INT, `selected_channel_id` INT, `selected_route_id` INT, `selected_account_id` INT, `selected_site_id` INT, `selected_site_platform` TEXT, `endpoint_candidates_json` JSON, `endpoint_runtime_state_json` JSON, `decision_summary_json` JSON, `final_status` TEXT, `final_http_status` INT, `final_upstream_path` TEXT, `final_response_headers_json` JSON, `final_response_body_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s'))); -CREATE TABLE IF NOT EXISTS `proxy_debug_attempts` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `trace_id` INT NOT NULL, `attempt_index` INT NOT NULL, `endpoint` TEXT NOT NULL, `request_path` TEXT NOT NULL, `target_url` TEXT NOT NULL, `runtime_executor` TEXT, `request_headers_json` JSON, `request_body_json` JSON, `response_status` INT, `response_headers_json` JSON, `response_body_json` JSON, `raw_error_text` TEXT, `recover_applied` BOOLEAN DEFAULT false, `downgrade_decision` BOOLEAN DEFAULT false, `downgrade_reason` TEXT, `memory_write_json` JSON, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`trace_id`) REFERENCES `proxy_debug_traces`(`id`) ON DELETE CASCADE); -CREATE UNIQUE INDEX `proxy_debug_attempts_trace_attempt_unique` ON `proxy_debug_attempts` (`trace_id`, `attempt_index`); -CREATE INDEX `proxy_debug_attempts_trace_created_at_idx` ON `proxy_debug_attempts` (`trace_id`, `created_at`); -CREATE INDEX `proxy_debug_traces_created_at_idx` ON `proxy_debug_traces` (`created_at`); -CREATE INDEX `proxy_debug_traces_final_status_created_at_idx` ON `proxy_debug_traces` (`final_status`(191), `created_at`); -CREATE INDEX `proxy_debug_traces_model_created_at_idx` ON `proxy_debug_traces` (`requested_model`(191), `created_at`); -CREATE INDEX `proxy_debug_traces_session_created_at_idx` ON `proxy_debug_traces` (`session_id`(191), `created_at`); +CREATE TABLE IF NOT EXISTS `site_api_endpoints` (`id` INT AUTO_INCREMENT NOT NULL PRIMARY KEY, `site_id` INT NOT NULL, `url` TEXT NOT NULL, `enabled` BOOLEAN DEFAULT true, `sort_order` INT DEFAULT 0, `cooldown_until` VARCHAR(191), `last_selected_at` VARCHAR(191), `last_failed_at` VARCHAR(191), `last_failure_reason` TEXT, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), '%Y-%m-%d %H:%i:%s')), FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE); +CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`, `url`(191)); +CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`, `cooldown_until`); +CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`, `enabled`, `sort_order`); diff --git a/src/server/db/generated/postgres.bootstrap.sql b/src/server/db/generated/postgres.bootstrap.sql index 984c613..6c52e58 100644 --- a/src/server/db/generated/postgres.bootstrap.sql +++ b/src/server/db/generated/postgres.bootstrap.sql @@ -15,6 +15,7 @@ CREATE TABLE IF NOT EXISTS "route_channels" ("id" INTEGER GENERATED BY DEFAULT A CREATE TABLE IF NOT EXISTS "route_group_sources" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "group_route_id" INTEGER NOT NULL, "source_route_id" INTEGER NOT NULL, FOREIGN KEY ("group_route_id") REFERENCES "token_routes"("id") ON DELETE CASCADE, FOREIGN KEY ("source_route_id") REFERENCES "token_routes"("id") ON DELETE CASCADE); CREATE TABLE IF NOT EXISTS "settings" ("key" TEXT NOT NULL PRIMARY KEY, "value" TEXT); CREATE TABLE IF NOT EXISTS "site_announcements" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "platform" TEXT NOT NULL, "source_key" TEXT NOT NULL, "title" TEXT NOT NULL, "content" TEXT NOT NULL, "level" TEXT NOT NULL DEFAULT 'info', "source_url" TEXT, "starts_at" TEXT, "ends_at" TEXT, "upstream_created_at" TEXT, "upstream_updated_at" TEXT, "first_seen_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "last_seen_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "read_at" TEXT, "dismissed_at" TEXT, "raw_payload" TEXT, FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE); +CREATE TABLE IF NOT EXISTS "site_api_endpoints" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "url" TEXT NOT NULL, "enabled" BOOLEAN DEFAULT true, "sort_order" INTEGER DEFAULT 0, "cooldown_until" TEXT, "last_selected_at" TEXT, "last_failed_at" TEXT, "last_failure_reason" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE); CREATE TABLE IF NOT EXISTS "site_disabled_models" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE); CREATE TABLE IF NOT EXISTS "token_model_availability" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "token_id" INTEGER NOT NULL, "model_name" TEXT NOT NULL, "available" BOOLEAN, "latency_ms" INTEGER, "checked_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("token_id") REFERENCES "account_tokens"("id") ON DELETE CASCADE); CREATE UNIQUE INDEX "downstream_api_keys_key_unique" ON "downstream_api_keys" ("key"); @@ -24,6 +25,7 @@ CREATE UNIQUE INDEX "proxy_files_public_id_unique" ON "proxy_files" ("public_id" CREATE UNIQUE INDEX "proxy_video_tasks_public_id_unique" ON "proxy_video_tasks" ("public_id"); CREATE UNIQUE INDEX "route_group_sources_group_source_unique" ON "route_group_sources" ("group_route_id", "source_route_id"); CREATE UNIQUE INDEX "site_announcements_site_source_key_unique" ON "site_announcements" ("site_id", "source_key"); +CREATE UNIQUE INDEX "site_api_endpoints_site_url_unique" ON "site_api_endpoints" ("site_id", "url"); CREATE UNIQUE INDEX "site_disabled_models_site_model_unique" ON "site_disabled_models" ("site_id", "model_name"); CREATE UNIQUE INDEX "sites_platform_url_unique" ON "sites" ("platform", "url"); CREATE UNIQUE INDEX "token_model_availability_token_model_unique" ON "token_model_availability" ("token_id", "model_name"); @@ -69,6 +71,8 @@ CREATE INDEX "route_channels_token_id_idx" ON "route_channels" ("token_id"); CREATE INDEX "route_group_sources_source_route_id_idx" ON "route_group_sources" ("source_route_id"); CREATE INDEX "site_announcements_read_at_idx" ON "site_announcements" ("read_at"); CREATE INDEX "site_announcements_site_id_first_seen_at_idx" ON "site_announcements" ("site_id", "first_seen_at"); +CREATE INDEX "site_api_endpoints_site_cooldown_idx" ON "site_api_endpoints" ("site_id", "cooldown_until"); +CREATE INDEX "site_api_endpoints_site_enabled_sort_idx" ON "site_api_endpoints" ("site_id", "enabled", "sort_order"); CREATE INDEX "site_disabled_models_site_id_idx" ON "site_disabled_models" ("site_id"); CREATE INDEX "sites_status_idx" ON "sites" ("status"); CREATE INDEX "token_model_availability_available_idx" ON "token_model_availability" ("available"); diff --git a/src/server/db/generated/postgres.upgrade.sql b/src/server/db/generated/postgres.upgrade.sql index 0f3f6fb..4a17438 100644 --- a/src/server/db/generated/postgres.upgrade.sql +++ b/src/server/db/generated/postgres.upgrade.sql @@ -1,8 +1,4 @@ -CREATE TABLE IF NOT EXISTS "proxy_debug_traces" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "downstream_path" TEXT NOT NULL, "client_kind" TEXT, "session_id" TEXT, "trace_hint" TEXT, "requested_model" TEXT, "downstream_api_key_id" INTEGER, "request_headers_json" JSONB, "request_body_json" JSONB, "sticky_session_key" TEXT, "sticky_hit_channel_id" INTEGER, "selected_channel_id" INTEGER, "selected_route_id" INTEGER, "selected_account_id" INTEGER, "selected_site_id" INTEGER, "selected_site_platform" TEXT, "endpoint_candidates_json" JSONB, "endpoint_runtime_state_json" JSONB, "decision_summary_json" JSONB, "final_status" TEXT, "final_http_status" INTEGER, "final_upstream_path" TEXT, "final_response_headers_json" JSONB, "final_response_body_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS')); -CREATE TABLE IF NOT EXISTS "proxy_debug_attempts" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "trace_id" INTEGER NOT NULL, "attempt_index" INTEGER NOT NULL, "endpoint" TEXT NOT NULL, "request_path" TEXT NOT NULL, "target_url" TEXT NOT NULL, "runtime_executor" TEXT, "request_headers_json" JSONB, "request_body_json" JSONB, "response_status" INTEGER, "response_headers_json" JSONB, "response_body_json" JSONB, "raw_error_text" TEXT, "recover_applied" BOOLEAN DEFAULT false, "downgrade_decision" BOOLEAN DEFAULT false, "downgrade_reason" TEXT, "memory_write_json" JSONB, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("trace_id") REFERENCES "proxy_debug_traces"("id") ON DELETE CASCADE); -CREATE UNIQUE INDEX "proxy_debug_attempts_trace_attempt_unique" ON "proxy_debug_attempts" ("trace_id", "attempt_index"); -CREATE INDEX "proxy_debug_attempts_trace_created_at_idx" ON "proxy_debug_attempts" ("trace_id", "created_at"); -CREATE INDEX "proxy_debug_traces_created_at_idx" ON "proxy_debug_traces" ("created_at"); -CREATE INDEX "proxy_debug_traces_final_status_created_at_idx" ON "proxy_debug_traces" ("final_status", "created_at"); -CREATE INDEX "proxy_debug_traces_model_created_at_idx" ON "proxy_debug_traces" ("requested_model", "created_at"); -CREATE INDEX "proxy_debug_traces_session_created_at_idx" ON "proxy_debug_traces" ("session_id", "created_at"); +CREATE TABLE IF NOT EXISTS "site_api_endpoints" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY NOT NULL PRIMARY KEY, "site_id" INTEGER NOT NULL, "url" TEXT NOT NULL, "enabled" BOOLEAN DEFAULT true, "sort_order" INTEGER DEFAULT 0, "cooldown_until" TEXT, "last_selected_at" TEXT, "last_failed_at" TEXT, "last_failure_reason" TEXT, "created_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), "updated_at" TEXT DEFAULT to_char(timezone('UTC', CURRENT_TIMESTAMP), 'YYYY-MM-DD HH24:MI:SS'), FOREIGN KEY ("site_id") REFERENCES "sites"("id") ON DELETE CASCADE); +CREATE UNIQUE INDEX "site_api_endpoints_site_url_unique" ON "site_api_endpoints" ("site_id", "url"); +CREATE INDEX "site_api_endpoints_site_cooldown_idx" ON "site_api_endpoints" ("site_id", "cooldown_until"); +CREATE INDEX "site_api_endpoints_site_enabled_sort_idx" ON "site_api_endpoints" ("site_id", "enabled", "sort_order"); diff --git a/src/server/db/generated/schemaContract.json b/src/server/db/generated/schemaContract.json index cf7df5f..9159156 100644 --- a/src/server/db/generated/schemaContract.json +++ b/src/server/db/generated/schemaContract.json @@ -1308,6 +1308,76 @@ } } }, + "site_api_endpoints": { + "columns": { + "id": { + "logicalType": "integer", + "notNull": true, + "defaultValue": null, + "primaryKey": true + }, + "site_id": { + "logicalType": "integer", + "notNull": true, + "defaultValue": null, + "primaryKey": false + }, + "url": { + "logicalType": "text", + "notNull": true, + "defaultValue": null, + "primaryKey": false + }, + "enabled": { + "logicalType": "boolean", + "notNull": false, + "defaultValue": "true", + "primaryKey": false + }, + "sort_order": { + "logicalType": "integer", + "notNull": false, + "defaultValue": "0", + "primaryKey": false + }, + "cooldown_until": { + "logicalType": "datetime", + "notNull": false, + "defaultValue": null, + "primaryKey": false + }, + "last_selected_at": { + "logicalType": "datetime", + "notNull": false, + "defaultValue": null, + "primaryKey": false + }, + "last_failed_at": { + "logicalType": "datetime", + "notNull": false, + "defaultValue": null, + "primaryKey": false + }, + "last_failure_reason": { + "logicalType": "text", + "notNull": false, + "defaultValue": null, + "primaryKey": false + }, + "created_at": { + "logicalType": "datetime", + "notNull": false, + "defaultValue": "datetime('now')", + "primaryKey": false + }, + "updated_at": { + "logicalType": "datetime", + "notNull": false, + "defaultValue": "datetime('now')", + "primaryKey": false + } + } + }, "site_disabled_models": { "columns": { "id": { @@ -1967,6 +2037,34 @@ ], "unique": true }, + { + "name": "site_api_endpoints_site_cooldown_idx", + "table": "site_api_endpoints", + "columns": [ + "site_id", + "cooldown_until" + ], + "unique": false + }, + { + "name": "site_api_endpoints_site_enabled_sort_idx", + "table": "site_api_endpoints", + "columns": [ + "site_id", + "enabled", + "sort_order" + ], + "unique": false + }, + { + "name": "site_api_endpoints_site_url_unique", + "table": "site_api_endpoints", + "columns": [ + "site_id", + "url" + ], + "unique": true + }, { "name": "site_disabled_models_site_id_idx", "table": "site_disabled_models", @@ -2106,6 +2204,14 @@ "source_key" ] }, + { + "name": "site_api_endpoints_site_url_unique", + "table": "site_api_endpoints", + "columns": [ + "site_id", + "url" + ] + }, { "name": "site_disabled_models_site_model_unique", "table": "site_disabled_models", @@ -2253,6 +2359,17 @@ ], "onDelete": "CASCADE" }, + { + "table": "site_api_endpoints", + "columns": [ + "site_id" + ], + "referencedTable": "sites", + "referencedColumns": [ + "id" + ], + "onDelete": "CASCADE" + }, { "table": "site_disabled_models", "columns": [ diff --git a/src/server/db/schema.ts b/src/server/db/schema.ts index b750e5b..ced7883 100644 --- a/src/server/db/schema.ts +++ b/src/server/db/schema.ts @@ -22,6 +22,24 @@ export const sites = sqliteTable('sites', { platformUrlUnique: uniqueIndex('sites_platform_url_unique').on(table.platform, table.url), })); +export const siteApiEndpoints = sqliteTable('site_api_endpoints', { + id: integer('id').primaryKey({ autoIncrement: true }), + siteId: integer('site_id').notNull().references(() => sites.id, { onDelete: 'cascade' }), + url: text('url').notNull(), + enabled: integer('enabled', { mode: 'boolean' }).default(true), + sortOrder: integer('sort_order').default(0), + cooldownUntil: text('cooldown_until'), + lastSelectedAt: text('last_selected_at'), + lastFailedAt: text('last_failed_at'), + lastFailureReason: text('last_failure_reason'), + createdAt: text('created_at').default(sql`(datetime('now'))`), + updatedAt: text('updated_at').default(sql`(datetime('now'))`), +}, (table) => ({ + siteUrlUnique: uniqueIndex('site_api_endpoints_site_url_unique').on(table.siteId, table.url), + siteEnabledSortIdx: index('site_api_endpoints_site_enabled_sort_idx').on(table.siteId, table.enabled, table.sortOrder), + siteCooldownIdx: index('site_api_endpoints_site_cooldown_idx').on(table.siteId, table.cooldownUntil), +})); + export const siteDisabledModels = sqliteTable('site_disabled_models', { id: integer('id').primaryKey({ autoIncrement: true }), siteId: integer('site_id').notNull().references(() => sites.id, { onDelete: 'cascade' }), diff --git a/src/server/db/siteSchemaCompatibility.test.ts b/src/server/db/siteSchemaCompatibility.test.ts index c4f8efb..c5b1df3 100644 --- a/src/server/db/siteSchemaCompatibility.test.ts +++ b/src/server/db/siteSchemaCompatibility.test.ts @@ -33,6 +33,25 @@ function createInspector( describe('ensureSiteSchemaCompatibility', () => { it.each([ + { + dialect: 'sqlite' as const, + expectedSql: [ + 'ALTER TABLE sites ADD COLUMN proxy_url text;', + 'ALTER TABLE sites ADD COLUMN use_system_proxy integer DEFAULT 0;', + 'UPDATE sites SET use_system_proxy = 0 WHERE use_system_proxy IS NULL;', + 'ALTER TABLE sites ADD COLUMN custom_headers text;', + 'ALTER TABLE sites ADD COLUMN external_checkin_url text;', + 'ALTER TABLE sites ADD COLUMN global_weight real DEFAULT 1;', + 'UPDATE sites SET global_weight = 1 WHERE global_weight IS NULL OR global_weight <= 0;', + 'CREATE TABLE IF NOT EXISTS site_api_endpoints (id integer PRIMARY KEY AUTOINCREMENT NOT NULL, site_id integer NOT NULL REFERENCES sites(id) ON DELETE cascade, url text NOT NULL, enabled integer DEFAULT 1, sort_order integer DEFAULT 0, cooldown_until text, last_selected_at text, last_failed_at text, last_failure_reason text, created_at text DEFAULT (datetime(\'now\')), updated_at text DEFAULT (datetime(\'now\')));', + 'CREATE UNIQUE INDEX IF NOT EXISTS site_api_endpoints_site_url_unique ON site_api_endpoints (site_id, url);', + 'CREATE INDEX IF NOT EXISTS site_api_endpoints_site_enabled_sort_idx ON site_api_endpoints (site_id, enabled, sort_order);', + 'CREATE INDEX IF NOT EXISTS site_api_endpoints_site_cooldown_idx ON site_api_endpoints (site_id, cooldown_until);', + 'CREATE TABLE IF NOT EXISTS site_disabled_models (id integer PRIMARY KEY AUTOINCREMENT NOT NULL, site_id integer NOT NULL REFERENCES sites(id) ON DELETE cascade, model_name text NOT NULL, created_at text DEFAULT (datetime(\'now\')));', + 'CREATE UNIQUE INDEX IF NOT EXISTS site_disabled_models_site_model_unique ON site_disabled_models (site_id, model_name);', + 'CREATE INDEX IF NOT EXISTS site_disabled_models_site_id_idx ON site_disabled_models (site_id);', + ], + }, { dialect: 'postgres' as const, expectedSql: [ @@ -43,6 +62,10 @@ describe('ensureSiteSchemaCompatibility', () => { 'ALTER TABLE "sites" ADD COLUMN "external_checkin_url" TEXT', 'ALTER TABLE "sites" ADD COLUMN "global_weight" DOUBLE PRECISION DEFAULT 1', 'UPDATE "sites" SET "global_weight" = 1 WHERE "global_weight" IS NULL OR "global_weight" <= 0', + 'CREATE TABLE IF NOT EXISTS "site_api_endpoints" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "site_id" INTEGER NOT NULL REFERENCES "sites"("id") ON DELETE CASCADE, "url" TEXT NOT NULL, "enabled" BOOLEAN DEFAULT TRUE, "sort_order" INTEGER DEFAULT 0, "cooldown_until" TEXT, "last_selected_at" TEXT, "last_failed_at" TEXT, "last_failure_reason" TEXT, "created_at" TEXT DEFAULT to_char(timezone(\'UTC\', CURRENT_TIMESTAMP), \'YYYY-MM-DD HH24:MI:SS\'), "updated_at" TEXT DEFAULT to_char(timezone(\'UTC\', CURRENT_TIMESTAMP), \'YYYY-MM-DD HH24:MI:SS\'))', + 'CREATE UNIQUE INDEX IF NOT EXISTS "site_api_endpoints_site_url_unique" ON "site_api_endpoints" ("site_id", "url")', + 'CREATE INDEX IF NOT EXISTS "site_api_endpoints_site_enabled_sort_idx" ON "site_api_endpoints" ("site_id", "enabled", "sort_order")', + 'CREATE INDEX IF NOT EXISTS "site_api_endpoints_site_cooldown_idx" ON "site_api_endpoints" ("site_id", "cooldown_until")', 'CREATE TABLE IF NOT EXISTS "site_disabled_models" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "site_id" INTEGER NOT NULL REFERENCES "sites"("id") ON DELETE CASCADE, "model_name" TEXT NOT NULL, "created_at" TEXT)', 'CREATE UNIQUE INDEX IF NOT EXISTS "site_disabled_models_site_model_unique" ON "site_disabled_models" ("site_id", "model_name")', 'CREATE INDEX IF NOT EXISTS "site_disabled_models_site_id_idx" ON "site_disabled_models" ("site_id")', @@ -58,6 +81,10 @@ describe('ensureSiteSchemaCompatibility', () => { 'ALTER TABLE `sites` ADD COLUMN `external_checkin_url` TEXT NULL', 'ALTER TABLE `sites` ADD COLUMN `global_weight` DOUBLE DEFAULT 1', 'UPDATE `sites` SET `global_weight` = 1 WHERE `global_weight` IS NULL OR `global_weight` <= 0', + 'CREATE TABLE IF NOT EXISTS `site_api_endpoints` (`id` INT AUTO_INCREMENT PRIMARY KEY, `site_id` INT NOT NULL, `url` TEXT NOT NULL, `enabled` BOOLEAN DEFAULT TRUE, `sort_order` INT DEFAULT 0, `cooldown_until` VARCHAR(191) NULL, `last_selected_at` VARCHAR(191) NULL, `last_failed_at` VARCHAR(191) NULL, `last_failure_reason` TEXT NULL, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), \'%Y-%m-%d %H:%i:%s\')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), \'%Y-%m-%d %H:%i:%s\')), CONSTRAINT `site_api_endpoints_site_fk` FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE)', + 'CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`, `url`(191))', + 'CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`, `enabled`, `sort_order`)', + 'CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`, `cooldown_until`(191))', 'CREATE TABLE IF NOT EXISTS `site_disabled_models` (`id` INT AUTO_INCREMENT PRIMARY KEY, `site_id` INT NOT NULL, `model_name` VARCHAR(191) NOT NULL, `created_at` TEXT NULL, CONSTRAINT `site_disabled_models_site_fk` FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE)', 'CREATE UNIQUE INDEX `site_disabled_models_site_model_unique` ON `site_disabled_models` (`site_id`, `model_name`(191))', 'CREATE INDEX `site_disabled_models_site_id_idx` ON `site_disabled_models` (`site_id`)', diff --git a/src/server/db/siteSchemaCompatibility.ts b/src/server/db/siteSchemaCompatibility.ts index 4e08f57..2a79887 100644 --- a/src/server/db/siteSchemaCompatibility.ts +++ b/src/server/db/siteSchemaCompatibility.ts @@ -73,6 +73,31 @@ export const SITE_COLUMN_COMPATIBILITY_SPECS: SiteColumnCompatibilitySpec[] = [ ]; export const SITE_TABLE_COMPATIBILITY_SPECS: SiteTableCompatibilitySpec[] = [ + { + table: 'site_api_endpoints', + createSql: { + sqlite: 'CREATE TABLE IF NOT EXISTS site_api_endpoints (id integer PRIMARY KEY AUTOINCREMENT NOT NULL, site_id integer NOT NULL REFERENCES sites(id) ON DELETE cascade, url text NOT NULL, enabled integer DEFAULT 1, sort_order integer DEFAULT 0, cooldown_until text, last_selected_at text, last_failed_at text, last_failure_reason text, created_at text DEFAULT (datetime(\'now\')), updated_at text DEFAULT (datetime(\'now\')));', + mysql: 'CREATE TABLE IF NOT EXISTS `site_api_endpoints` (`id` INT AUTO_INCREMENT PRIMARY KEY, `site_id` INT NOT NULL, `url` TEXT NOT NULL, `enabled` BOOLEAN DEFAULT TRUE, `sort_order` INT DEFAULT 0, `cooldown_until` VARCHAR(191) NULL, `last_selected_at` VARCHAR(191) NULL, `last_failed_at` VARCHAR(191) NULL, `last_failure_reason` TEXT NULL, `created_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), \'%Y-%m-%d %H:%i:%s\')), `updated_at` VARCHAR(191) DEFAULT (DATE_FORMAT(NOW(), \'%Y-%m-%d %H:%i:%s\')), CONSTRAINT `site_api_endpoints_site_fk` FOREIGN KEY (`site_id`) REFERENCES `sites`(`id`) ON DELETE CASCADE)', + postgres: 'CREATE TABLE IF NOT EXISTS "site_api_endpoints" ("id" INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "site_id" INTEGER NOT NULL REFERENCES "sites"("id") ON DELETE CASCADE, "url" TEXT NOT NULL, "enabled" BOOLEAN DEFAULT TRUE, "sort_order" INTEGER DEFAULT 0, "cooldown_until" TEXT, "last_selected_at" TEXT, "last_failed_at" TEXT, "last_failure_reason" TEXT, "created_at" TEXT DEFAULT to_char(timezone(\'UTC\', CURRENT_TIMESTAMP), \'YYYY-MM-DD HH24:MI:SS\'), "updated_at" TEXT DEFAULT to_char(timezone(\'UTC\', CURRENT_TIMESTAMP), \'YYYY-MM-DD HH24:MI:SS\'))', + }, + postCreateSql: { + sqlite: [ + 'CREATE UNIQUE INDEX IF NOT EXISTS site_api_endpoints_site_url_unique ON site_api_endpoints (site_id, url);', + 'CREATE INDEX IF NOT EXISTS site_api_endpoints_site_enabled_sort_idx ON site_api_endpoints (site_id, enabled, sort_order);', + 'CREATE INDEX IF NOT EXISTS site_api_endpoints_site_cooldown_idx ON site_api_endpoints (site_id, cooldown_until);', + ], + mysql: [ + 'CREATE UNIQUE INDEX `site_api_endpoints_site_url_unique` ON `site_api_endpoints` (`site_id`, `url`(191))', + 'CREATE INDEX `site_api_endpoints_site_enabled_sort_idx` ON `site_api_endpoints` (`site_id`, `enabled`, `sort_order`)', + 'CREATE INDEX `site_api_endpoints_site_cooldown_idx` ON `site_api_endpoints` (`site_id`, `cooldown_until`(191))', + ], + postgres: [ + 'CREATE UNIQUE INDEX IF NOT EXISTS "site_api_endpoints_site_url_unique" ON "site_api_endpoints" ("site_id", "url")', + 'CREATE INDEX IF NOT EXISTS "site_api_endpoints_site_enabled_sort_idx" ON "site_api_endpoints" ("site_id", "enabled", "sort_order")', + 'CREATE INDEX IF NOT EXISTS "site_api_endpoints_site_cooldown_idx" ON "site_api_endpoints" ("site_id", "cooldown_until")', + ], + }, + }, { table: 'site_disabled_models', createSql: { diff --git a/src/server/proxy-core/surfaces/chatSurface.ts b/src/server/proxy-core/surfaces/chatSurface.ts index c24ee11..5b289f3 100644 --- a/src/server/proxy-core/surfaces/chatSurface.ts +++ b/src/server/proxy-core/surfaces/chatSurface.ts @@ -60,6 +60,7 @@ import { selectSurfaceChannelForAttempt, trySurfaceOauthRefreshRecovery, } from './sharedSurface.js'; +import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js'; import { buildSurfaceProxyDebugResponseHeaders, captureSurfaceProxyDebugSuccessResponseBody, @@ -316,67 +317,145 @@ export async function handleChatSurfaceRequest( downstreamHeaders: request.headers as Record, }) ); - const buildEndpointRequest = ( - endpoint: 'chat' | 'messages' | 'responses', - options: { forceNormalizeClaudeBody?: boolean } = {}, - ) => { - const upstreamStream = isStream || (isCodexSite && endpoint === 'responses'); - const endpointRequest = buildUpstreamEndpointRequest({ - endpoint, - modelName, - stream: upstreamStream, - tokenValue: selected.tokenValue, - oauthProvider: oauth?.provider, - oauthProjectId: oauth?.projectId, - sitePlatform: selected.site.platform, - siteUrl: selected.site.url, - openaiBody: resolvedOpenAiBody, - downstreamFormat, - claudeOriginalBody, - forceNormalizeClaudeBody: options.forceNormalizeClaudeBody, - downstreamHeaders: request.headers as Record, - providerHeaders: buildProviderHeaders(), - codexSessionCacheKey, - }); - return { - endpoint, - path: endpointRequest.path, - headers: endpointRequest.headers, - body: endpointRequest.body as Record, - runtime: endpointRequest.runtime, - }; - }; - const dispatchRequest = createSurfaceDispatchRequest({ - site: selected.site, - accountExtraConfig: selected.account.extraConfig, - }); - const endpointStrategy = downstreamTransformer.compatibility.createEndpointStrategy({ - downstreamFormat, - endpointCandidates, - modelName, - requestedModelHint: requestedModel, - sitePlatform: selected.site.platform, - isStream: isStream || isCodexSite, - buildRequest: ({ endpoint, forceNormalizeClaudeBody }) => buildEndpointRequest( - endpoint, - { forceNormalizeClaudeBody }, - ), - dispatchRequest, - }); - const tryRecover = async (ctx: Parameters>[0]) => { - if ((ctx.response.status === 401 || ctx.response.status === 403) && oauth) { - const recovered = await trySurfaceOauthRefreshRecovery({ - ctx, - selected, - siteUrl: selected.site.url, - buildRequest: (endpoint) => buildEndpointRequest(endpoint), - dispatchRequest, + const executeEndpointResultForSiteApiBaseUrl = async (siteApiBaseUrl: string) => { + const buildEndpointRequest = ( + endpoint: 'chat' | 'messages' | 'responses', + options: { forceNormalizeClaudeBody?: boolean } = {}, + ) => { + const upstreamStream = isStream || (isCodexSite && endpoint === 'responses'); + const endpointRequest = buildUpstreamEndpointRequest({ + endpoint, + modelName, + stream: upstreamStream, + tokenValue: selected.tokenValue, + oauthProvider: oauth?.provider, + oauthProjectId: oauth?.projectId, + sitePlatform: selected.site.platform, + siteUrl: siteApiBaseUrl, + openaiBody: resolvedOpenAiBody, + downstreamFormat, + claudeOriginalBody, + forceNormalizeClaudeBody: options.forceNormalizeClaudeBody, + downstreamHeaders: request.headers as Record, + providerHeaders: buildProviderHeaders(), + codexSessionCacheKey, }); - if (recovered?.upstream?.ok) { - return recovered; + return { + endpoint, + path: endpointRequest.path, + headers: endpointRequest.headers, + body: endpointRequest.body as Record, + runtime: endpointRequest.runtime, + }; + }; + const dispatchRequest = createSurfaceDispatchRequest({ + site: selected.site, + siteUrl: siteApiBaseUrl, + accountExtraConfig: selected.account.extraConfig, + }); + const endpointStrategy = downstreamTransformer.compatibility.createEndpointStrategy({ + downstreamFormat, + endpointCandidates, + modelName, + requestedModelHint: requestedModel, + sitePlatform: selected.site.platform, + isStream: isStream || isCodexSite, + buildRequest: ({ endpoint, forceNormalizeClaudeBody }) => buildEndpointRequest( + endpoint, + { forceNormalizeClaudeBody }, + ), + dispatchRequest, + }); + const tryRecover = async (ctx: Parameters>[0]) => { + if ((ctx.response.status === 401 || ctx.response.status === 403) && oauth) { + const recovered = await trySurfaceOauthRefreshRecovery({ + ctx, + selected, + siteUrl: siteApiBaseUrl, + buildRequest: (endpoint) => buildEndpointRequest(endpoint), + dispatchRequest, + }); + if (recovered?.upstream?.ok) { + return recovered; + } } - } - return endpointStrategy.tryRecover(ctx); + return endpointStrategy.tryRecover(ctx); + }; + const debugAttemptBase = reserveSurfaceProxyDebugAttemptBase(debugTrace, endpointCandidates.length); + return executeEndpointFlow({ + siteUrl: siteApiBaseUrl, + disableCrossProtocolFallback: config.disableCrossProtocolFallback, + endpointCandidates, + buildRequest: (endpoint) => buildEndpointRequest(endpoint), + dispatchRequest, + tryRecover, + onAttemptFailure: async (ctx) => { + const memoryWrite = recordUpstreamEndpointFailure({ + ...endpointRuntimeContext, + endpoint: ctx.request.endpoint, + status: ctx.response.status, + errorText: ctx.rawErrText, + }); + await safeInsertSurfaceProxyDebugAttempt(debugTrace, { + attemptIndex: debugAttemptBase + ctx.endpointIndex, + endpoint: ctx.request.endpoint, + requestPath: ctx.request.path, + targetUrl: ctx.targetUrl, + runtimeExecutor: ctx.request.runtime?.executor || 'default', + requestHeaders: ctx.request.headers, + requestBody: ctx.request.body, + responseStatus: ctx.response.status, + responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response), + responseBody: parseSurfaceProxyDebugTextPayload(ctx.rawErrText), + rawErrorText: ctx.rawErrText, + recoverApplied: ctx.recoverApplied === true, + downgradeDecision: false, + downgradeReason: null, + memoryWrite, + }); + }, + onAttemptSuccess: async (ctx) => { + const memoryWrite = recordUpstreamEndpointSuccess({ + ...endpointRuntimeContext, + endpoint: ctx.request.endpoint, + }); + const responseBody = await captureSurfaceProxyDebugSuccessResponseBody(debugTrace, ctx); + await safeInsertSurfaceProxyDebugAttempt(debugTrace, { + attemptIndex: debugAttemptBase + ctx.endpointIndex, + endpoint: ctx.request.endpoint, + requestPath: ctx.request.path, + targetUrl: ctx.targetUrl, + runtimeExecutor: ctx.request.runtime?.executor || 'default', + requestHeaders: ctx.request.headers, + requestBody: ctx.request.body, + responseStatus: ctx.response.status, + responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response), + responseBody, + rawErrorText: null, + recoverApplied: ctx.recoverApplied === true, + downgradeDecision: false, + downgradeReason: null, + memoryWrite, + }); + }, + shouldDowngrade: endpointStrategy.shouldDowngrade, + onDowngrade: async (ctx) => { + await safeUpdateSurfaceProxyDebugAttempt(debugTrace, debugAttemptBase + ctx.endpointIndex, { + downgradeDecision: true, + downgradeReason: ctx.errText, + rawErrorText: ctx.rawErrText, + }); + return failureToolkit.log({ + selected, + modelRequested: requestedModel, + status: 'failed', + httpStatus: ctx.response.status, + latencyMs: Date.now() - startTime, + errorMessage: ctx.errText, + retryCount, + }); + }, + }); }; let startTime = Date.now(); const leaseResult = await acquireSurfaceChannelLease({ @@ -418,113 +497,18 @@ export async function handleChatSurfaceRequest( const channelLease = leaseResult.lease; try { - const debugAttemptBase = reserveSurfaceProxyDebugAttemptBase(debugTrace, endpointCandidates.length); - const endpointResult = await executeEndpointFlow({ - siteUrl: selected.site.url, - disableCrossProtocolFallback: config.disableCrossProtocolFallback, - endpointCandidates, - buildRequest: (endpoint) => buildEndpointRequest(endpoint), - dispatchRequest, - tryRecover, - onAttemptFailure: async (ctx) => { - const memoryWrite = recordUpstreamEndpointFailure({ - ...endpointRuntimeContext, - endpoint: ctx.request.endpoint, - status: ctx.response.status, - errorText: ctx.rawErrText, - }); - await safeInsertSurfaceProxyDebugAttempt(debugTrace, { - attemptIndex: debugAttemptBase + ctx.endpointIndex, - endpoint: ctx.request.endpoint, - requestPath: ctx.request.path, - targetUrl: ctx.targetUrl, - runtimeExecutor: ctx.request.runtime?.executor || 'default', - requestHeaders: ctx.request.headers, - requestBody: ctx.request.body, - responseStatus: ctx.response.status, - responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response), - responseBody: parseSurfaceProxyDebugTextPayload(ctx.rawErrText), - rawErrorText: ctx.rawErrText, - recoverApplied: ctx.recoverApplied === true, - downgradeDecision: false, - downgradeReason: null, - memoryWrite, - }); - }, - onAttemptSuccess: async (ctx) => { - const memoryWrite = recordUpstreamEndpointSuccess({ - ...endpointRuntimeContext, - endpoint: ctx.request.endpoint, - }); - const responseBody = await captureSurfaceProxyDebugSuccessResponseBody(debugTrace, ctx); - await safeInsertSurfaceProxyDebugAttempt(debugTrace, { - attemptIndex: debugAttemptBase + ctx.endpointIndex, - endpoint: ctx.request.endpoint, - requestPath: ctx.request.path, - targetUrl: ctx.targetUrl, - runtimeExecutor: ctx.request.runtime?.executor || 'default', - requestHeaders: ctx.request.headers, - requestBody: ctx.request.body, - responseStatus: ctx.response.status, - responseHeaders: buildSurfaceProxyDebugResponseHeaders(ctx.response), - responseBody, - rawErrorText: null, - recoverApplied: ctx.recoverApplied === true, - downgradeDecision: false, - downgradeReason: null, - memoryWrite, - }); - }, - shouldDowngrade: endpointStrategy.shouldDowngrade, - onDowngrade: async (ctx) => { - await safeUpdateSurfaceProxyDebugAttempt(debugTrace, debugAttemptBase + ctx.endpointIndex, { - downgradeDecision: true, - downgradeReason: ctx.errText, - rawErrorText: ctx.rawErrText, - }); - return failureToolkit.log({ - selected, - modelRequested: requestedModel, - status: 'failed', - httpStatus: ctx.response.status, - latencyMs: Date.now() - startTime, - errorMessage: ctx.errText, - retryCount, - }); - }, - }); - - if (!endpointResult.ok) { - clearSurfaceStickyChannel({ - stickySessionKey, - selected, - }); - const failureOutcome = await failureToolkit.handleUpstreamFailure({ - selected, - requestedModel, - modelName, - status: endpointResult.status || 502, - errText: endpointResult.errText || 'unknown error', - rawErrText: endpointResult.rawErrText, - latencyMs: Date.now() - startTime, - retryCount, - }); - const terminalFailureOutcome = failureOutcome.action === 'retry' - ? (canRetryChannelSelection(retryCount, forcedChannelId) - ? null - : finalizeRetryAsUpstreamFailure(endpointResult.status || 502, endpointResult.errText || 'unknown error')) - : failureOutcome; - if (!terminalFailureOutcome) { - retryCount += 1; - continue; + const endpointResult = await runWithSiteApiEndpointPool(selected.site, async (target) => { + const result = await executeEndpointResultForSiteApiBaseUrl(target.baseUrl); + if (!result.ok) { + const upstreamFailure = new SiteApiEndpointRequestError(result.errText || 'unknown error', { + status: result.status || 502, + rawErrText: result.rawErrText || result.errText || 'unknown error', + }) as SiteApiEndpointRequestError & { siteApiEndpointUpstreamFailure?: boolean }; + upstreamFailure.siteApiEndpointUpstreamFailure = true; + throw upstreamFailure; } - await finalizeDebugFailure( - terminalFailureOutcome.status, - terminalFailureOutcome.payload, - null, - ); - return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload); - } + return result; + }); const upstream = endpointResult.upstream; const successfulUpstreamPath = endpointResult.upstreamPath; @@ -944,14 +928,48 @@ export async function handleChatSurfaceRequest( }); return reply.send(downstreamResponse); - } catch (err: any) { - clearSurfaceStickyChannel({ - stickySessionKey, - selected, - }); - const failureOutcome = await failureToolkit.handleExecutionError({ + } catch (err: any) { + clearSurfaceStickyChannel({ + stickySessionKey, + selected, + }); + const endpointFailureStatus = typeof err?.status === 'number' ? err.status : null; + const isSiteApiEndpointFailure = ( + err instanceof SiteApiEndpointRequestError + || err?.name === 'SiteApiEndpointRequestError' + || err?.siteApiEndpointUpstreamFailure === true + || (endpointFailureStatus !== null && endpointFailureStatus >= 500) + ); + if (isSiteApiEndpointFailure) { + const failureOutcome = await failureToolkit.handleUpstreamFailure({ selected, requestedModel, + modelName, + status: endpointFailureStatus || 502, + errText: err.message || 'unknown error', + rawErrText: err.rawErrText || err.message || 'unknown error', + latencyMs: Date.now() - startTime, + retryCount, + }); + const terminalFailureOutcome = failureOutcome.action === 'retry' + ? (canRetryChannelSelection(retryCount, forcedChannelId) + ? null + : finalizeRetryAsUpstreamFailure(endpointFailureStatus || 502, err.message || 'unknown error')) + : failureOutcome; + if (!terminalFailureOutcome) { + retryCount += 1; + continue; + } + await finalizeDebugFailure( + terminalFailureOutcome.status, + terminalFailureOutcome.payload, + null, + ); + return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload); + } + const failureOutcome = await failureToolkit.handleExecutionError({ + selected, + requestedModel, modelName, errorMessage: err?.message || 'network failure', latencyMs: Date.now() - startTime, @@ -1229,92 +1247,89 @@ export async function handleClaudeCountTokensSurfaceRequest( }; try { - let upstreamRequest = buildRequest(); - const dispatchRequest = createSurfaceDispatchRequest({ - site: selected.site, - accountExtraConfig: selected.account.extraConfig, - }); - let upstream = await dispatchRequest(upstreamRequest); - let recoverApplied = false; + const countTokensResult = await runWithSiteApiEndpointPool(selected.site, async (target) => { + let upstreamRequest = buildRequest(); + const dispatchRequest = createSurfaceDispatchRequest({ + site: selected.site, + siteUrl: target.baseUrl, + accountExtraConfig: selected.account.extraConfig, + }); + let upstream = await dispatchRequest(upstreamRequest); + let recoverApplied = false; - if ((upstream.status === 401 || upstream.status === 403) && oauth) { - const recoverContext = { - request: upstreamRequest, - response: upstream, - rawErrText: '', + if ((upstream.status === 401 || upstream.status === 403) && oauth) { + const recoverContext = { + request: upstreamRequest, + response: upstream, + rawErrText: '', + }; + const recovered = await trySurfaceOauthRefreshRecovery({ + ctx: recoverContext, + selected, + siteUrl: target.baseUrl, + buildRequest: () => buildRequest(), + dispatchRequest, + captureFailureBody: false, + }); + if (recovered?.upstream?.ok) { + upstreamRequest = buildRequest(); + upstream = recovered.upstream; + recoverApplied = true; + } else { + upstreamRequest = recoverContext.request; + upstream = recoverContext.response; + } + } + + const latency = Date.now() - startTime; + const contentType = upstream.headers.get('content-type') || 'application/json'; + const text = await readRuntimeResponseText(upstream); + let payload: unknown = text; + try { + payload = JSON.parse(text); + } catch { + payload = text; + } + await safeInsertSurfaceProxyDebugAttempt(debugTrace, { + attemptIndex: retryCount, + endpoint: upstreamRequest.endpoint, + requestPath: upstreamRequest.path, + targetUrl: `${target.baseUrl}${upstreamRequest.path}`, + runtimeExecutor: upstreamRequest.runtime?.executor || 'default', + requestHeaders: upstreamRequest.headers, + requestBody: upstreamRequest.body, + responseStatus: upstream.status, + responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream), + responseBody: payload, + rawErrorText: upstream.ok ? null : text, + recoverApplied, + downgradeDecision: false, + downgradeReason: null, + memoryWrite: null, + }); + if (!upstream.ok) { + const errText = typeof payload === 'string' ? payload : JSON.stringify(payload); + throw new SiteApiEndpointRequestError(errText || 'unknown error', { + status: upstream.status, + rawErrText: typeof payload === 'string' ? payload : text, + }); + } + return { + upstream, + upstreamRequest, + contentType, + payload, + latency, }; - const recovered = await trySurfaceOauthRefreshRecovery({ - ctx: recoverContext, - selected, - siteUrl: selected.site.url, - buildRequest: () => buildRequest(), - dispatchRequest, - captureFailureBody: false, - }); - if (recovered?.upstream?.ok) { - upstreamRequest = buildRequest(); - upstream = recovered.upstream; - recoverApplied = true; - } else { - upstreamRequest = recoverContext.request; - upstream = recoverContext.response; - } - } - - const latency = Date.now() - startTime; - const contentType = upstream.headers.get('content-type') || 'application/json'; - const text = await readRuntimeResponseText(upstream); - let payload: unknown = text; - try { - payload = JSON.parse(text); - } catch { - payload = text; - } - await safeInsertSurfaceProxyDebugAttempt(debugTrace, { - attemptIndex: retryCount, - endpoint: upstreamRequest.endpoint, - requestPath: upstreamRequest.path, - targetUrl: `${selected.site.url}${upstreamRequest.path}`, - runtimeExecutor: upstreamRequest.runtime?.executor || 'default', - requestHeaders: upstreamRequest.headers, - requestBody: upstreamRequest.body, - responseStatus: upstream.status, - responseHeaders: buildSurfaceProxyDebugResponseHeaders(upstream), - responseBody: payload, - rawErrorText: upstream.ok ? null : text, - recoverApplied, - downgradeDecision: false, - downgradeReason: null, - memoryWrite: null, }); - if (!upstream.ok) { - clearSurfaceStickyChannel({ - stickySessionKey, - selected, - }); - const failureOutcome = await failureToolkit.handleUpstreamFailure({ - selected, - requestedModel, - modelName, - status: upstream.status, - errText: typeof payload === 'string' ? payload : JSON.stringify(payload), - rawErrText: typeof payload === 'string' ? payload : text, - latencyMs: latency, - retryCount, - }); - const terminalFailureOutcome = failureOutcome.action === 'retry' - ? (canRetryChannelSelection(retryCount, forcedChannelId) - ? null - : finalizeRetryAsUpstreamFailure(upstream.status, typeof payload === 'string' ? payload : JSON.stringify(payload))) - : failureOutcome; - if (!terminalFailureOutcome) { - retryCount += 1; - continue; - } - await finalizeDebugFailure(terminalFailureOutcome.status, terminalFailureOutcome.payload, upstreamRequest.path); - return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload); - } + const { + upstream, + upstreamRequest, + contentType, + payload, + latency, + } = countTokensResult; tokenRouter.recordSuccess(selected.channel.id, latency, 0, modelName); recordDownstreamCostUsage(request, 0); @@ -1344,6 +1359,36 @@ export async function handleClaudeCountTokensSurfaceRequest( stickySessionKey, selected, }); + const endpointFailureStatus = typeof error?.status === 'number' ? error.status : null; + const isSiteApiEndpointFailure = ( + error instanceof SiteApiEndpointRequestError + || error?.name === 'SiteApiEndpointRequestError' + || error?.siteApiEndpointUpstreamFailure === true + || (endpointFailureStatus !== null && endpointFailureStatus >= 500) + ); + if (isSiteApiEndpointFailure) { + const failureOutcome = await failureToolkit.handleUpstreamFailure({ + selected, + requestedModel, + modelName, + status: endpointFailureStatus || 502, + errText: error.message || 'unknown error', + rawErrText: error.rawErrText || error.message || 'unknown error', + latencyMs: Date.now() - startTime, + retryCount, + }); + const terminalFailureOutcome = failureOutcome.action === 'retry' + ? (canRetryChannelSelection(retryCount, forcedChannelId) + ? null + : finalizeRetryAsUpstreamFailure(endpointFailureStatus || 502, error.message || 'unknown error')) + : failureOutcome; + if (!terminalFailureOutcome) { + retryCount += 1; + continue; + } + await finalizeDebugFailure(terminalFailureOutcome.status, terminalFailureOutcome.payload, null); + return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload); + } const failureOutcome = await failureToolkit.handleExecutionError({ selected, requestedModel, diff --git a/src/server/proxy-core/surfaces/openAiResponsesSurface.ts b/src/server/proxy-core/surfaces/openAiResponsesSurface.ts index 3b041b9..31e88fe 100644 --- a/src/server/proxy-core/surfaces/openAiResponsesSurface.ts +++ b/src/server/proxy-core/surfaces/openAiResponsesSurface.ts @@ -70,6 +70,7 @@ import { safeUpdateSurfaceProxyDebugSelection, startSurfaceProxyDebugTrace, } from '../../services/proxyDebugTraceRuntime.js'; +import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js'; import { buildForcedChannelUnavailableMessage, canRetryChannelSelection, @@ -423,122 +424,84 @@ export async function handleOpenAiResponsesSurfaceRequest( downstreamHeaders: request.headers as Record, }) ); - const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => { - const upstreamStream = isStream || (isCodexSite && endpoint === 'responses'); - const endpointRequest = buildUpstreamEndpointRequest({ - endpoint, - modelName, - stream: upstreamStream, - tokenValue: selected.tokenValue, - oauthProvider: oauth?.provider, - oauthProjectId: oauth?.projectId, - sitePlatform: selected.site.platform, - siteUrl: selected.site.url, - openaiBody: openAiBody, - downstreamFormat: 'responses', - responsesOriginalBody: normalizedResponsesBody, - downstreamHeaders: request.headers as Record, - providerHeaders: buildProviderHeaders(), - }); - const upstreamPath = ( - isCompactRequest && endpoint === 'responses' - ? `${endpointRequest.path}/compact` - : endpointRequest.path - ); - return { - endpoint, - path: upstreamPath, - headers: endpointRequest.headers, - body: endpointRequest.body as Record, - runtime: endpointRequest.runtime, - }; - }; - const baseDispatchRequest = createSurfaceDispatchRequest({ - site: selected.site, - accountExtraConfig: selected.account.extraConfig, - }); - const dispatchRequest = ( - endpointRequest: BuiltEndpointRequest, - targetUrl?: string, - ) => { - if (!isCodexSite || endpointRequest.path !== '/responses') { - return baseDispatchRequest(endpointRequest, targetUrl); - } - const sessionId = getCodexSessionHeaderValue(endpointRequest.headers); - return runCodexHttpSessionTask( - sessionId, - () => baseDispatchRequest(endpointRequest, targetUrl), - ); - }; - const endpointStrategy = openAiResponsesTransformer.compatibility.createEndpointStrategy({ - isStream: isStream || isCodexSite, - requiresNativeResponsesFileUrl, - dispatchRequest, - }); - const tryRecover = async (ctx: Parameters>[0]) => { - if (oauth && shouldRefreshOauthResponsesRequest({ - oauthProvider: oauth.provider, - status: ctx.response.status, - response: ctx.response, - rawErrText: ctx.rawErrText || '', - })) { - const recovered = await trySurfaceOauthRefreshRecovery({ - ctx, - selected, - siteUrl: selected.site.url, - buildRequest: (endpoint) => buildEndpointRequest(endpoint), - dispatchRequest, + const executeEndpointResultForSiteApiBaseUrl = async (siteApiBaseUrl: string) => { + const buildEndpointRequest = (endpoint: 'chat' | 'messages' | 'responses') => { + const upstreamStream = isStream || (isCodexSite && endpoint === 'responses'); + const endpointRequest = buildUpstreamEndpointRequest({ + endpoint, + modelName, + stream: upstreamStream, + tokenValue: selected.tokenValue, + oauthProvider: oauth?.provider, + oauthProjectId: oauth?.projectId, + sitePlatform: selected.site.platform, + siteUrl: siteApiBaseUrl, + openaiBody: openAiBody, + downstreamFormat: 'responses', + responsesOriginalBody: normalizedResponsesBody, + downstreamHeaders: request.headers as Record, + providerHeaders: buildProviderHeaders(), }); - if (recovered?.upstream?.ok) { - return recovered; + const upstreamPath = ( + isCompactRequest && endpoint === 'responses' + ? `${endpointRequest.path}/compact` + : endpointRequest.path + ); + return { + endpoint, + path: upstreamPath, + headers: endpointRequest.headers, + body: endpointRequest.body as Record, + runtime: endpointRequest.runtime, + }; + }; + const baseDispatchRequest = createSurfaceDispatchRequest({ + site: selected.site, + siteUrl: siteApiBaseUrl, + accountExtraConfig: selected.account.extraConfig, + }); + const dispatchRequest = ( + endpointRequest: BuiltEndpointRequest, + targetUrl?: string, + ) => { + if (!isCodexSite || !endpointRequest.path.startsWith('/responses')) { + return baseDispatchRequest(endpointRequest, targetUrl); } - } - return endpointStrategy.tryRecover(ctx); - }; + const sessionId = getCodexSessionHeaderValue(endpointRequest.headers); + return runCodexHttpSessionTask( + sessionId, + () => baseDispatchRequest(endpointRequest, targetUrl), + ); + }; + const endpointStrategy = openAiResponsesTransformer.compatibility.createEndpointStrategy({ + isStream: isStream || isCodexSite, + requiresNativeResponsesFileUrl, + dispatchRequest, + }); + const tryRecover = async (ctx: Parameters>[0]) => { + if (oauth && shouldRefreshOauthResponsesRequest({ + oauthProvider: oauth.provider, + status: ctx.response.status, + response: ctx.response, + rawErrText: ctx.rawErrText || '', + })) { + const recovered = await trySurfaceOauthRefreshRecovery({ + ctx, + selected, + siteUrl: siteApiBaseUrl, + buildRequest: (endpoint) => buildEndpointRequest(endpoint), + dispatchRequest, + }); + if (recovered?.upstream?.ok) { + return recovered; + } + } + return endpointStrategy.tryRecover(ctx); + }; - const startTime = Date.now(); - const leaseResult = await acquireSurfaceChannelLease({ - stickySessionKey, - selected, - }); - if (leaseResult.status === 'timeout') { - clearSurfaceStickyChannel({ - stickySessionKey, - selected, - }); - const busyMessage = buildSurfaceChannelBusyMessage(leaseResult.waitMs); - await failureToolkit.log({ - selected, - modelRequested: requestedModel, - status: 'failed', - httpStatus: 503, - latencyMs: leaseResult.waitMs, - errorMessage: busyMessage, - retryCount, - }); - if (retryCount < maxRetries && canRetryChannelSelection(retryCount, forcedChannelId)) { - retryCount += 1; - continue; - } - await finalizeDebugFailure(503, { - error: { - message: busyMessage, - type: 'server_error', - }, - }); - return reply.code(503).send({ - error: { - message: busyMessage, - type: 'server_error', - }, - }); - } - const channelLease = leaseResult.lease; - - try { const debugAttemptBase = reserveSurfaceProxyDebugAttemptBase(debugTrace, endpointCandidates.length); - const endpointResult = await executeEndpointFlow({ - siteUrl: selected.site.url, + return executeEndpointFlow({ + siteUrl: siteApiBaseUrl, disableCrossProtocolFallback: config.disableCrossProtocolFallback, endpointCandidates, buildRequest: (endpoint) => buildEndpointRequest(endpoint), @@ -611,77 +574,99 @@ export async function handleOpenAiResponsesSurfaceRequest( }); }, }); - - if (!endpointResult.ok) { - clearSurfaceStickyChannel({ - stickySessionKey, - selected, - }); - const failureOutcome = await failureToolkit.handleUpstreamFailure({ - selected, - requestedModel, - modelName, - status: endpointResult.status || 502, - errText: endpointResult.errText || 'unknown error', - rawErrText: endpointResult.rawErrText, - latencyMs: Date.now() - startTime, - retryCount, - }); - const terminalFailureOutcome = failureOutcome.action === 'retry' - ? (canRetryChannelSelection(retryCount, forcedChannelId) - ? null - : finalizeRetryAsUpstreamFailure(endpointResult.status || 502, endpointResult.errText || 'unknown error')) - : failureOutcome; - if (!terminalFailureOutcome) { - retryCount += 1; - continue; - } - await finalizeDebugFailure( - terminalFailureOutcome.status, - terminalFailureOutcome.payload, - null, - ); - return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload); - } - - const upstream = endpointResult.upstream; - const successfulUpstreamPath = endpointResult.upstreamPath; - const finalizeStreamSuccess = async ( - parsedUsage: UsageSummary, - latency: number, - streamDebugBody: unknown, - upstreamUsagePresent: boolean, - ) => { - try { - await recordSurfaceSuccess({ - selected, - requestedModel, - modelName, - parsedUsage, - upstreamUsagePresent, - requestStartedAtMs: startTime, - latencyMs: latency, - retryCount, - upstreamPath: successfulUpstreamPath, - logSuccess: failureToolkit.log, - recordDownstreamCost: (estimatedCost) => { - recordDownstreamCostUsage(request, estimatedCost); - }, - bestEffortMetrics: { - errorLabel: '[responses] post-stream bookkeeping failed:', - }, - }); - } catch (error) { - console.error('[responses] post-stream success logging failed:', error); - } - await finalizeDebugSuccess( - 200, - successfulUpstreamPath, - buildSurfaceProxyDebugResponseHeaders(upstream), - streamDebugBody, - ); }; + const startTime = Date.now(); + const leaseResult = await acquireSurfaceChannelLease({ + stickySessionKey, + selected, + }); + if (leaseResult.status === 'timeout') { + clearSurfaceStickyChannel({ + stickySessionKey, + selected, + }); + const busyMessage = buildSurfaceChannelBusyMessage(leaseResult.waitMs); + await failureToolkit.log({ + selected, + modelRequested: requestedModel, + status: 'failed', + httpStatus: 503, + latencyMs: leaseResult.waitMs, + errorMessage: busyMessage, + retryCount, + }); + if (retryCount < maxRetries && canRetryChannelSelection(retryCount, forcedChannelId)) { + retryCount += 1; + continue; + } + await finalizeDebugFailure(503, { + error: { + message: busyMessage, + type: 'server_error', + }, + }); + return reply.code(503).send({ + error: { + message: busyMessage, + type: 'server_error', + }, + }); + } + const channelLease = leaseResult.lease; + + try { + const endpointResult = await runWithSiteApiEndpointPool(selected.site, async (target) => { + const result = await executeEndpointResultForSiteApiBaseUrl(target.baseUrl); + if (!result.ok) { + const upstreamFailure = new SiteApiEndpointRequestError(result.errText || 'unknown error', { + status: result.status || 502, + rawErrText: result.rawErrText || result.errText || 'unknown error', + }) as SiteApiEndpointRequestError & { siteApiEndpointUpstreamFailure?: boolean }; + upstreamFailure.siteApiEndpointUpstreamFailure = true; + throw upstreamFailure; + } + return result; + }); + + const upstream = endpointResult.upstream; + const successfulUpstreamPath = endpointResult.upstreamPath; + const finalizeStreamSuccess = async ( + parsedUsage: UsageSummary, + latency: number, + streamDebugBody: unknown, + upstreamUsagePresent: boolean, + ) => { + try { + await recordSurfaceSuccess({ + selected, + requestedModel, + modelName, + parsedUsage, + upstreamUsagePresent, + requestStartedAtMs: startTime, + latencyMs: latency, + retryCount, + upstreamPath: successfulUpstreamPath, + logSuccess: failureToolkit.log, + recordDownstreamCost: (estimatedCost) => { + recordDownstreamCostUsage(request, estimatedCost); + }, + bestEffortMetrics: { + errorLabel: '[responses] post-stream bookkeeping failed:', + }, + }); + } catch (error) { + console.error('[responses] post-stream success logging failed:', error); + } + await finalizeDebugSuccess( + 200, + successfulUpstreamPath, + buildSurfaceProxyDebugResponseHeaders(upstream), + streamDebugBody, + ); + }; + if (isStream) { const upstreamContentType = (upstream.headers.get('content-type') || '').toLowerCase(); const startSseResponse = () => { @@ -1126,22 +1111,56 @@ export async function handleOpenAiResponsesSurfaceRequest( stickySessionKey, selected, }); + const endpointFailureStatus = typeof err?.status === 'number' ? err.status : null; + const isSiteApiEndpointFailure = ( + err instanceof SiteApiEndpointRequestError + || err?.name === 'SiteApiEndpointRequestError' + || err?.siteApiEndpointUpstreamFailure === true + || (endpointFailureStatus !== null && endpointFailureStatus >= 500) + ); + if (isSiteApiEndpointFailure) { + const failureOutcome = await failureToolkit.handleUpstreamFailure({ + selected, + requestedModel, + modelName, + status: endpointFailureStatus || 502, + errText: err?.message || 'unknown error', + rawErrText: err?.rawErrText || err?.message || 'unknown error', + latencyMs: Date.now() - startTime, + retryCount, + }); + const terminalFailureOutcome = failureOutcome.action === 'retry' + ? (canRetryChannelSelection(retryCount, forcedChannelId) + ? null + : finalizeRetryAsUpstreamFailure(endpointFailureStatus || 502, err?.message || 'unknown error')) + : failureOutcome; + if (!terminalFailureOutcome) { + retryCount += 1; + continue; + } + await finalizeDebugFailure( + terminalFailureOutcome.status, + terminalFailureOutcome.payload, + null, + ); + return reply.code(terminalFailureOutcome.status).send(terminalFailureOutcome.payload); + } const failureOutcome = await failureToolkit.handleExecutionError({ selected, requestedModel, - modelName, - errorMessage: err?.message || 'network failure', - latencyMs: Date.now() - startTime, - retryCount, - }); - const terminalFailureOutcome = failureOutcome.action === 'retry' - ? (canRetryChannelSelection(retryCount, forcedChannelId) - ? null - : finalizeRetryAsExecutionFailure(err?.message || 'network failure')) - : failureOutcome; - if (!terminalFailureOutcome) { - retryCount += 1; - continue; + modelName, + errorMessage: err?.message || 'network failure', + latencyMs: Date.now() - startTime, + retryCount, + }); + const terminalFailureOutcome = failureOutcome.action === 'retry' + ? (canRetryChannelSelection(retryCount, forcedChannelId) + ? null + : finalizeRetryAsExecutionFailure(err?.message || 'network failure')) + : failureOutcome; + if (!terminalFailureOutcome) { + retryCount += 1; + continue; } await finalizeDebugFailure( terminalFailureOutcome.status, diff --git a/src/server/proxy-core/surfaces/sharedSurface.ts b/src/server/proxy-core/surfaces/sharedSurface.ts index 73570e8..fb49cbc 100644 --- a/src/server/proxy-core/surfaces/sharedSurface.ts +++ b/src/server/proxy-core/surfaces/sharedSurface.ts @@ -249,14 +249,15 @@ export async function writeSurfaceProxyLog(input: { export function createSurfaceDispatchRequest(input: { site: SiteProxyConfigLike & { url: string }; accountExtraConfig?: string | null; + siteUrl?: string; }) { const channelProxyUrl = resolveChannelProxyUrl(input.site, input.accountExtraConfig); return ( request: BuiltEndpointRequest, targetUrl?: string, - ) => ( + ) => ( dispatchRuntimeRequest({ - siteUrl: input.site.url, + siteUrl: input.siteUrl ?? input.site.url, targetUrl, request, buildInit: (_requestUrl, requestForFetch) => withSiteRecordProxyRequestInit(input.site, { diff --git a/src/server/routes/api/accounts.apiSite.test.ts b/src/server/routes/api/accounts.apiSite.test.ts new file mode 100644 index 0000000..65b8791 --- /dev/null +++ b/src/server/routes/api/accounts.apiSite.test.ts @@ -0,0 +1,271 @@ +import Fastify, { type FastifyInstance } from 'fastify'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +const verifyTokenMock = vi.fn(); +const getModelsMock = vi.fn(); + +vi.mock('../../services/platforms/index.js', () => ({ + getAdapter: () => ({ + verifyToken: (...args: unknown[]) => verifyTokenMock(...args), + getModels: (...args: unknown[]) => getModelsMock(...args), + }), +})); + +type DbModule = typeof import('../../db/index.js'); + +describe('accounts api endpoint host selection', { timeout: 15_000 }, () => { + let app: FastifyInstance; + let db: DbModule['db']; + let schema: DbModule['schema']; + let dataDir = ''; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-accounts-api-site-')); + process.env.DATA_DIR = dataDir; + + await import('../../db/migrate.js'); + const dbModule = await import('../../db/index.js'); + const routesModule = await import('./accounts.js'); + db = dbModule.db; + schema = dbModule.schema; + + app = Fastify(); + await app.register(routesModule.accountsRoutes); + }); + + beforeEach(async () => { + verifyTokenMock.mockReset(); + getModelsMock.mockReset(); + + await db.delete(schema.proxyLogs).run(); + await db.delete(schema.checkinLogs).run(); + await db.delete(schema.routeChannels).run(); + await db.delete(schema.tokenRoutes).run(); + await db.delete(schema.tokenModelAvailability).run(); + await db.delete(schema.modelAvailability).run(); + await db.delete(schema.accountTokens).run(); + await db.delete(schema.accounts).run(); + await db.delete(schema.siteApiEndpoints).run(); + await db.delete(schema.sites).run(); + }); + + afterAll(async () => { + await app.close(); + delete process.env.DATA_DIR; + }); + + it('uses the configured ai endpoint for API key verification', async () => { + getModelsMock.mockResolvedValueOnce(['gpt-4o-mini']); + + const site = await db.insert(schema.sites).values({ + name: 'Nihao Panel', + url: 'https://nih.cc', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api.nih.cc', + enabled: true, + sortOrder: 0, + }).run(); + + const response = await app.inject({ + method: 'POST', + url: '/api/accounts/verify-token', + payload: { + siteId: site.id, + accessToken: 'sk-nihao', + credentialMode: 'apikey', + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + success: true, + tokenType: 'apikey', + modelCount: 1, + }); + expect(getModelsMock).toHaveBeenCalledWith('https://api.nih.cc', 'sk-nihao', undefined); + expect(verifyTokenMock).not.toHaveBeenCalled(); + }); + + it('rotates API key verification across configured ai endpoints after a retryable failure', async () => { + getModelsMock + .mockRejectedValueOnce(new Error('HTTP 502: temporary upstream failure')) + .mockResolvedValueOnce(['gpt-4o-mini']); + + const site = await db.insert(schema.sites).values({ + name: 'Nihao Pool', + url: 'https://nih.cc', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-a.nih.cc', + enabled: true, + sortOrder: 0, + }, + { + siteId: site.id, + url: 'https://api-b.nih.cc', + enabled: true, + sortOrder: 1, + }, + ]).run(); + + const response = await app.inject({ + method: 'POST', + url: '/api/accounts/verify-token', + payload: { + siteId: site.id, + accessToken: 'sk-rotate', + credentialMode: 'apikey', + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + success: true, + tokenType: 'apikey', + modelCount: 1, + }); + expect(getModelsMock).toHaveBeenNthCalledWith(1, 'https://api-a.nih.cc', 'sk-rotate', undefined); + expect(getModelsMock).toHaveBeenNthCalledWith(2, 'https://api-b.nih.cc', 'sk-rotate', undefined); + + const endpoints = await db.select().from(schema.siteApiEndpoints).all(); + const firstEndpoint = endpoints.find((item) => item.url === 'https://api-a.nih.cc'); + const secondEndpoint = endpoints.find((item) => item.url === 'https://api-b.nih.cc'); + expect(firstEndpoint?.cooldownUntil).toBeTruthy(); + expect(firstEndpoint?.lastFailureReason).toContain('HTTP 502'); + expect(secondEndpoint?.lastSelectedAt).toBeTruthy(); + }); + + it('keeps session verification on the panel host even when api endpoints exist', async () => { + verifyTokenMock.mockResolvedValueOnce({ + tokenType: 'session', + userInfo: { username: 'nihao-user' }, + balance: 12.5, + apiToken: 'sk-derived', + }); + + const site = await db.insert(schema.sites).values({ + name: 'Nihao Panel', + url: 'https://nih.cc', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api.nih.cc', + enabled: true, + sortOrder: 0, + }).run(); + + const response = await app.inject({ + method: 'POST', + url: '/api/accounts/verify-token', + payload: { + siteId: site.id, + accessToken: 'session-token', + credentialMode: 'session', + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + success: true, + tokenType: 'session', + apiToken: 'sk-derived', + }); + expect(verifyTokenMock).toHaveBeenCalledWith('https://nih.cc', 'session-token', undefined); + expect(getModelsMock).not.toHaveBeenCalled(); + }); + + it('uses the configured ai endpoint when adding an API key connection', async () => { + getModelsMock.mockResolvedValueOnce(['gpt-4o-mini']); + + const site = await db.insert(schema.sites).values({ + name: 'Nihao Panel', + url: 'https://nih.cc', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api.nih.cc', + enabled: true, + sortOrder: 0, + }).run(); + + const response = await app.inject({ + method: 'POST', + url: '/api/accounts', + payload: { + siteId: site.id, + accessToken: 'sk-nihao-create', + credentialMode: 'apikey', + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + tokenType: 'apikey', + }); + expect(getModelsMock).toHaveBeenCalledWith('https://api.nih.cc', 'sk-nihao-create', undefined); + }); + + it('rotates API key account creation across configured ai endpoints after a retryable failure', async () => { + getModelsMock + .mockRejectedValueOnce(new Error('HTTP 502: temporary upstream failure')) + .mockResolvedValueOnce(['gpt-4o-mini']); + + const site = await db.insert(schema.sites).values({ + name: 'Nihao Create Pool', + url: 'https://nih.cc', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-create-a.nih.cc', + enabled: true, + sortOrder: 0, + }, + { + siteId: site.id, + url: 'https://api-create-b.nih.cc', + enabled: true, + sortOrder: 1, + }, + ]).run(); + + const response = await app.inject({ + method: 'POST', + url: '/api/accounts', + payload: { + siteId: site.id, + accessToken: 'sk-nihao-create-rotate', + credentialMode: 'apikey', + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + tokenType: 'apikey', + }); + expect(getModelsMock).toHaveBeenNthCalledWith(1, 'https://api-create-a.nih.cc', 'sk-nihao-create-rotate', undefined); + expect(getModelsMock).toHaveBeenNthCalledWith(2, 'https://api-create-b.nih.cc', 'sk-nihao-create-rotate', undefined); + }); +}); diff --git a/src/server/routes/api/accounts.ts b/src/server/routes/api/accounts.ts index 8a97c92..46f881c 100644 --- a/src/server/routes/api/accounts.ts +++ b/src/server/routes/api/accounts.ts @@ -42,6 +42,7 @@ import { parseAccountUpdatePayload, parseAccountVerifyTokenPayload, } from '../../contracts/accountsRoutePayloads.js'; +import { requireSiteApiBaseUrl, runWithSiteApiEndpointPool } from '../../services/siteApiEndpointService.js'; type AccountWithSiteRow = { accounts: typeof schema.accounts.$inferSelect; @@ -314,6 +315,31 @@ function isVerificationTimeoutError(error: unknown): boolean { return lowered.includes('timeout') || lowered.includes('timed out') || lowered.includes('abort'); } +function buildAccountVerifyTimeoutMessage(): string { + return `Token verification timed out (${Math.max(1, Math.round(ACCOUNT_VERIFY_TIMEOUT_MS / 1000))}s)`; +} + +async function getModelsWithSiteApiEndpointPool( + site: typeof schema.sites.$inferSelect, + adapter: NonNullable>, + accessToken: string, + platformUserId?: number, +): Promise { + const timeoutMessage = buildAccountVerifyTimeoutMessage(); + const deadline = Date.now() + ACCOUNT_VERIFY_TIMEOUT_MS; + return runWithSiteApiEndpointPool(site, (target) => { + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) { + throw new Error(timeoutMessage); + } + return withTimeout( + () => adapter.getModels(target.baseUrl, accessToken, platformUserId), + remainingMs, + timeoutMessage, + ); + }); +} + function resolveUserIdFailureReason(message: string, hasProvidedUserId: boolean): VerifyFailureReason { const lowered = String(message || '').trim().toLowerCase(); if (!lowered) return null; @@ -671,7 +697,9 @@ export async function accountsRoutes(app: FastifyInstance) { : undefined; const hasProvidedUserId = parsedPlatformUserId !== undefined; const skipRawShieldDetection = normalizedPlatform === 'new-api' || normalizedPlatform === 'anyrouter'; - const diagnoseVerificationFailure = async (): Promise => { + const diagnoseVerificationFailure = async ( + options: { useApiEndpointPool?: boolean } = {}, + ): Promise => { const parseFailureReason = (bodyText: string, contentType: string): VerifyFailureReason => { const text = bodyText || ''; const ct = (contentType || '').toLowerCase(); @@ -720,21 +748,39 @@ export async function accountsRoutes(app: FastifyInstance) { }); } - for (const headers of headerVariants) { - try { - const testRes = await fetch( - `${site.url}/api/user/self`, - withSiteRecordProxyRequestInit(site, { - headers, - signal: AbortSignal.timeout(ACCOUNT_VERIFY_DIAG_TIMEOUT_MS), - }), - ); - const bodyText = await testRes.text(); - const contentType = testRes.headers.get('content-type') || ''; - const reason = parseFailureReason(bodyText, contentType); - if (reason) return reason; - } catch { } + const tryBaseUrl = async (baseUrl: string): Promise => { + let sawNetworkError = false; + let sawResponse = false; + for (const headers of headerVariants) { + try { + const testRes = await fetch( + `${baseUrl.replace(/\/+$/, '')}/api/user/self`, + withSiteRecordProxyRequestInit(site, { + headers, + signal: AbortSignal.timeout(ACCOUNT_VERIFY_DIAG_TIMEOUT_MS), + }), + ); + sawResponse = true; + const bodyText = await testRes.text(); + const contentType = testRes.headers.get('content-type') || ''; + const reason = parseFailureReason(bodyText, contentType); + if (reason) return reason; + } catch { + sawNetworkError = true; + } + } + if (sawNetworkError && !sawResponse) { + throw new Error(`diagnostic request timed out for ${baseUrl}`); + } + return null; + }; + + if (options.useApiEndpointPool) { + const diagnosticBaseUrl = await requireSiteApiBaseUrl(site); + return await tryBaseUrl(diagnosticBaseUrl); } + + return await tryBaseUrl(site.url); } catch { } return null; @@ -768,7 +814,9 @@ export async function accountsRoutes(app: FastifyInstance) { }; if (!hasProvidedUserId && (normalizedPlatform === 'new-api' || normalizedPlatform === 'anyrouter')) { - const preflightReason = await diagnoseVerificationFailure(); + const preflightReason = await diagnoseVerificationFailure({ + useApiEndpointPool: credentialMode === 'apikey', + }); if (preflightReason === 'needs-user-id') { return buildVerificationFailureResponse(preflightReason); } @@ -776,10 +824,11 @@ export async function accountsRoutes(app: FastifyInstance) { if (credentialMode === 'apikey') { try { - const models = await withTimeout( - () => adapter.getModels(site.url, accessToken, parsedPlatformUserId), - ACCOUNT_VERIFY_TIMEOUT_MS, - `Token verification timed out (${Math.max(1, Math.round(ACCOUNT_VERIFY_TIMEOUT_MS / 1000))}s)`, + const models = await getModelsWithSiteApiEndpointPool( + site, + adapter, + accessToken, + parsedPlatformUserId, ); const availableModels = Array.isArray(models) ? models.filter((item) => typeof item === 'string' && item.trim().length > 0) : []; if (availableModels.length === 0) { @@ -796,7 +845,9 @@ export async function accountsRoutes(app: FastifyInstance) { }; } catch (err: any) { if (isVerificationTimeoutError(err)) { - const failure = buildVerificationFailureResponse(await diagnoseVerificationFailure()); + const failure = buildVerificationFailureResponse(await diagnoseVerificationFailure({ + useApiEndpointPool: true, + })); if (failure) return failure; } return { @@ -1114,7 +1165,12 @@ export async function accountsRoutes(app: FastifyInstance) { if (!apiToken) apiToken = rawAccessToken; } else { try { - const models = await adapter.getModels(site.url, rawAccessToken, body.platformUserId); + const models = await getModelsWithSiteApiEndpointPool( + site, + adapter, + rawAccessToken, + body.platformUserId, + ); verifiedModels = Array.isArray(models) ? models.filter((item) => typeof item === 'string' && item.trim().length > 0) : []; diff --git a/src/server/routes/api/sites.api-endpoints.test.ts b/src/server/routes/api/sites.api-endpoints.test.ts new file mode 100644 index 0000000..b24dd58 --- /dev/null +++ b/src/server/routes/api/sites.api-endpoints.test.ts @@ -0,0 +1,226 @@ +import Fastify, { type FastifyInstance } from 'fastify'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { asc, eq } from 'drizzle-orm'; + +type DbModule = typeof import('../../db/index.js'); + +describe('sites api endpoints', () => { + let app: FastifyInstance; + let db: DbModule['db']; + let schema: DbModule['schema']; + let dataDir = ''; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-sites-api-endpoints-')); + process.env.DATA_DIR = dataDir; + + await import('../../db/migrate.js'); + const dbModule = await import('../../db/index.js'); + const routesModule = await import('./sites.js'); + db = dbModule.db; + schema = dbModule.schema; + + app = Fastify(); + await app.register(routesModule.sitesRoutes); + }); + + beforeEach(async () => { + await db.delete(schema.accounts).run(); + await db.delete(schema.siteApiEndpoints).run(); + await db.delete(schema.sites).run(); + }); + + afterAll(async () => { + await app.close(); + delete process.env.DATA_DIR; + }); + + it('accepts apiEndpoints on create and returns them in stable sort order with server-owned metadata', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/sites', + payload: { + name: 'split-host-site', + url: 'https://panel.example.com', + platform: 'new-api', + apiEndpoints: [ + { + url: 'https://api-b.example.com', + enabled: false, + sortOrder: 1, + cooldownUntil: '2099-01-01T00:00:00.000Z', + lastFailureReason: 'should-be-ignored', + }, + { + url: 'https://api-a.example.com/', + enabled: true, + sortOrder: 0, + lastSelectedAt: '2099-01-01T00:00:00.000Z', + }, + ], + }, + }); + + expect(response.statusCode).toBe(200); + const payload = response.json() as { + id: number; + apiEndpoints?: Array>; + }; + expect(payload.apiEndpoints).toEqual([ + expect.objectContaining({ + url: 'https://api-a.example.com', + enabled: true, + sortOrder: 0, + cooldownUntil: null, + lastSelectedAt: null, + lastFailedAt: null, + lastFailureReason: null, + }), + expect.objectContaining({ + url: 'https://api-b.example.com', + enabled: false, + sortOrder: 1, + cooldownUntil: null, + lastSelectedAt: null, + lastFailedAt: null, + lastFailureReason: null, + }), + ]); + + const stored = await db.select().from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.siteId, payload.id)) + .orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id)) + .all(); + expect(stored.map((row) => row.url)).toEqual([ + 'https://api-a.example.com', + 'https://api-b.example.com', + ]); + }); + + it('replaces apiEndpoints on update', async () => { + const created = await app.inject({ + method: 'POST', + url: '/api/sites', + payload: { + name: 'replace-site', + url: 'https://panel.example.com', + platform: 'new-api', + apiEndpoints: [ + { url: 'https://api-old.example.com', enabled: true, sortOrder: 0 }, + ], + }, + }); + expect(created.statusCode).toBe(200); + const site = created.json() as { id: number }; + + const updated = await app.inject({ + method: 'PUT', + url: `/api/sites/${site.id}`, + payload: { + apiEndpoints: [ + { url: 'https://api-b.example.com', enabled: true, sortOrder: 1 }, + { url: 'https://api-a.example.com', enabled: false, sortOrder: 0 }, + ], + }, + }); + + expect(updated.statusCode).toBe(200); + const payload = updated.json() as { apiEndpoints?: Array> }; + expect(payload.apiEndpoints).toEqual([ + expect.objectContaining({ + url: 'https://api-a.example.com', + enabled: false, + sortOrder: 0, + }), + expect.objectContaining({ + url: 'https://api-b.example.com', + enabled: true, + sortOrder: 1, + }), + ]); + + const stored = await db.select().from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.siteId, site.id)) + .orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id)) + .all(); + expect(stored.map((row) => row.url)).toEqual([ + 'https://api-a.example.com', + 'https://api-b.example.com', + ]); + }); + + it('rejects duplicate api endpoint urls under the same site after normalization', async () => { + const response = await app.inject({ + method: 'POST', + url: '/api/sites', + payload: { + name: 'duplicate-endpoints-site', + url: 'https://panel.example.com', + platform: 'new-api', + apiEndpoints: [ + { url: 'https://api.example.com/', enabled: true, sortOrder: 0 }, + { url: 'https://api.example.com', enabled: false, sortOrder: 1 }, + ], + }, + }); + + expect(response.statusCode).toBe(400); + expect((response.json() as { error?: string }).error).toContain('Duplicate apiEndpoints url'); + }); + + it('includes apiEndpoints in GET /api/sites using stable endpoint order', async () => { + const site = await db.insert(schema.sites).values({ + name: 'listed-site', + url: 'https://panel.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-b.example.com', + enabled: true, + sortOrder: 1, + cooldownUntil: '2026-03-31T12:05:00.000Z', + lastFailedAt: '2026-03-31T12:00:00.000Z', + lastFailureReason: 'HTTP 502', + }, + { + siteId: site.id, + url: 'https://api-a.example.com', + enabled: false, + sortOrder: 0, + }, + ]).run(); + + const response = await app.inject({ + method: 'GET', + url: '/api/sites', + }); + + expect(response.statusCode).toBe(200); + const payload = response.json() as Array<{ + id: number; + apiEndpoints?: Array>; + }>; + const listed = payload.find((row) => row.id === site.id); + expect(listed?.apiEndpoints).toEqual([ + expect.objectContaining({ + url: 'https://api-a.example.com', + enabled: false, + sortOrder: 0, + }), + expect.objectContaining({ + url: 'https://api-b.example.com', + enabled: true, + sortOrder: 1, + cooldownUntil: '2026-03-31T12:05:00.000Z', + lastFailureReason: 'HTTP 502', + }), + ]); + }); +}); diff --git a/src/server/routes/api/sites.ts b/src/server/routes/api/sites.ts index 142fb67..e7236d6 100644 --- a/src/server/routes/api/sites.ts +++ b/src/server/routes/api/sites.ts @@ -1,7 +1,7 @@ import { FastifyInstance, FastifyReply } from 'fastify'; import { db, schema } from '../../db/index.js'; -import { insertAndGetById } from '../../db/insertHelpers.js'; -import { and, eq } from 'drizzle-orm'; +import { getInsertedRowId } from '../../db/insertHelpers.js'; +import { and, asc, eq, inArray } from 'drizzle-orm'; import { detectSite } from '../../services/siteDetector.js'; import { invalidateSiteProxyCache, normalizeSiteUrl, parseSiteProxyUrlInput } from '../../services/siteProxy.js'; import { formatUtcSqlDateTime } from '../../services/localTimeService.js'; @@ -16,6 +16,7 @@ import { parseSiteUpdatePayload, } from '../../contracts/siteRoutePayloads.js'; import { getSiteInitializationPreset } from '../../../shared/siteInitializationPresets.js'; +import { normalizeSiteApiEndpointBaseUrl } from '../../services/siteApiEndpointService.js'; function normalizeSiteStatus(input: unknown): 'active' | 'disabled' | null { if (input === undefined || input === null) return null; @@ -103,6 +104,165 @@ function normalizeSitePlatform(value: string | undefined): string | null { return normalized || null; } +type SiteApiEndpointInputRow = { + url: string; + enabled: boolean; + sortOrder: number; +}; + +function normalizeSiteApiEndpointBoolean(input: unknown): boolean | null { + return normalizePinnedFlag(input); +} + +function normalizeSiteApiEndpointsInput(input: unknown): { + valid: boolean; + present: boolean; + apiEndpoints: SiteApiEndpointInputRow[]; + error?: string; +} { + if (input === undefined) { + return { valid: true, present: false, apiEndpoints: [] }; + } + if (input === null) { + return { valid: true, present: true, apiEndpoints: [] }; + } + if (!Array.isArray(input)) { + return { + valid: false, + present: true, + apiEndpoints: [], + error: 'Invalid apiEndpoints. Expected an array.', + }; + } + + const seenUrls = new Set(); + const apiEndpoints: SiteApiEndpointInputRow[] = []; + + for (let index = 0; index < input.length; index += 1) { + const row = input[index]; + if (!row || typeof row !== 'object') { + return { + valid: false, + present: true, + apiEndpoints: [], + error: 'Invalid apiEndpoints item. Expected an object.', + }; + } + + const rawUrl = typeof (row as { url?: unknown }).url === 'string' + ? (row as { url: string }).url + : ''; + const normalizedUrl = normalizeSiteApiEndpointBaseUrl(rawUrl); + if (!normalizedUrl) { + return { + valid: false, + present: true, + apiEndpoints: [], + error: 'Invalid apiEndpoints url. Expected a valid http(s) URL.', + }; + } + + let parsedUrl: URL; + try { + parsedUrl = new URL(normalizedUrl); + } catch { + return { + valid: false, + present: true, + apiEndpoints: [], + error: 'Invalid apiEndpoints url. Expected a valid http(s) URL.', + }; + } + if (parsedUrl.protocol !== 'http:' && parsedUrl.protocol !== 'https:') { + return { + valid: false, + present: true, + apiEndpoints: [], + error: 'Invalid apiEndpoints url. Expected a valid http(s) URL.', + }; + } + + if (seenUrls.has(normalizedUrl)) { + return { + valid: false, + present: true, + apiEndpoints: [], + error: `Duplicate apiEndpoints url: ${normalizedUrl}`, + }; + } + seenUrls.add(normalizedUrl); + + const normalizedEnabled = normalizeSiteApiEndpointBoolean((row as { enabled?: unknown }).enabled); + if ((row as { enabled?: unknown }).enabled !== undefined && normalizedEnabled === null) { + return { + valid: false, + present: true, + apiEndpoints: [], + error: 'Invalid apiEndpoints enabled value. Expected boolean.', + }; + } + + const normalizedSortOrder = normalizeSortOrder((row as { sortOrder?: unknown }).sortOrder); + if ((row as { sortOrder?: unknown }).sortOrder !== undefined && normalizedSortOrder === null) { + return { + valid: false, + present: true, + apiEndpoints: [], + error: 'Invalid apiEndpoints sortOrder value. Expected non-negative integer.', + }; + } + + apiEndpoints.push({ + url: normalizedUrl, + enabled: normalizedEnabled ?? true, + sortOrder: normalizedSortOrder ?? index, + }); + } + + return { valid: true, present: true, apiEndpoints }; +} + +async function loadSiteApiEndpointsBySiteIds(siteIds: number[]) { + if (siteIds.length === 0) { + return new Map>(); + } + + const rows = await db.select().from(schema.siteApiEndpoints) + .where(inArray(schema.siteApiEndpoints.siteId, siteIds)) + .orderBy( + asc(schema.siteApiEndpoints.siteId), + asc(schema.siteApiEndpoints.sortOrder), + asc(schema.siteApiEndpoints.id), + ) + .all(); + + const bySiteId = new Map>(); + for (const row of rows) { + const current = bySiteId.get(row.siteId) || []; + current.push({ + ...row, + url: normalizeSiteApiEndpointBaseUrl(row.url), + }); + bySiteId.set(row.siteId, current); + } + return bySiteId; +} + +async function attachSiteApiEndpoints(siteRows: T[]) { + const bySiteId = await loadSiteApiEndpointsBySiteIds(siteRows.map((row) => row.id)); + return siteRows.map((row) => ({ + ...row, + apiEndpoints: bySiteId.get(row.id) || [], + })); +} + +async function loadSiteWithApiEndpoints(siteId: number) { + const site = await db.select().from(schema.sites).where(eq(schema.sites.id, siteId)).get(); + if (!site) return null; + const [hydrated] = await attachSiteApiEndpoints([site]); + return hydrated || null; +} + function getErrorChain(error: unknown): ErrorLike[] { const chain: ErrorLike[] = []; const seen = new Set(); @@ -272,6 +432,7 @@ export async function sitesRoutes(app: FastifyInstance) { // List all sites app.get('/api/sites', async () => { const siteRows = await db.select().from(schema.sites).all(); + const siteRowsWithApiEndpoints = await attachSiteApiEndpoints(siteRows); const accountRows = await db.select({ siteId: schema.accounts.siteId, balance: schema.accounts.balance, @@ -285,7 +446,7 @@ export async function sitesRoutes(app: FastifyInstance) { subscriptionBySiteId[row.siteId] = aggregateSiteSubscription(subscriptionBySiteId[row.siteId], row.extraConfig); } - return siteRows.map((site) => ({ + return siteRowsWithApiEndpoints.map((site) => ({ ...site, totalBalance: Math.round((totalBalanceBySiteId[site.id] || 0) * 1_000_000) / 1_000_000, subscriptionSummary: subscriptionBySiteId[site.id] || null, @@ -298,7 +459,7 @@ export async function sitesRoutes(app: FastifyInstance) { if (!parsedBody.success) { return reply.code(400).send({ error: parsedBody.error }); } - + const createBody = parsedBody.data as typeof parsedBody.data & { apiEndpoints?: unknown }; const { name, url, @@ -312,7 +473,8 @@ export async function sitesRoutes(app: FastifyInstance) { isPinned, sortOrder, globalWeight, - } = parsedBody.data; + apiEndpoints, + } = createBody; const normalizedStatus = normalizeSiteStatus(status); if (status !== undefined && !normalizedStatus) { return reply.code(400).send({ error: 'Invalid site status. Expected active or disabled.' }); @@ -351,6 +513,10 @@ export async function sitesRoutes(app: FastifyInstance) { if (initializationPresetId != null && initializationPresetId !== '' && !explicitInitializationPreset) { return reply.code(400).send({ error: 'Invalid initializationPresetId.' }); } + const normalizedApiEndpoints = normalizeSiteApiEndpointsInput(apiEndpoints); + if (!normalizedApiEndpoints.valid) { + return reply.code(400).send({ error: normalizedApiEndpoints.error || 'Invalid apiEndpoints.' }); + } const existingSites = await db.select().from(schema.sites).all(); const maxSortOrder = existingSites.reduce((max, site) => Math.max(max, site.sortOrder || 0), -1); @@ -380,10 +546,8 @@ export async function sitesRoutes(app: FastifyInstance) { let inserted; try { - inserted = await insertAndGetById({ - table: schema.sites, - idColumn: schema.sites.id, - values: { + inserted = await db.transaction(async (tx) => { + const siteInsert = await tx.insert(schema.sites).values({ name, url: canonicalUrl, platform: detectedPlatform, @@ -395,9 +559,19 @@ export async function sitesRoutes(app: FastifyInstance) { isPinned: normalizedPinned ?? false, sortOrder: normalizedSortOrder ?? (maxSortOrder + 1), globalWeight: normalizedGlobalWeight ?? 1, - }, - insertErrorMessage: 'Create site failed', - loadErrorMessage: 'Create site failed', + }).run(); + const siteId = getInsertedRowId(siteInsert); + if (siteId && normalizedApiEndpoints.present && normalizedApiEndpoints.apiEndpoints.length > 0) { + await tx.insert(schema.siteApiEndpoints).values( + normalizedApiEndpoints.apiEndpoints.map((row) => ({ + siteId, + url: row.url, + enabled: row.enabled, + sortOrder: row.sortOrder, + })), + ).run(); + } + return siteInsert; }); } catch (error) { if (isSitesPlatformUrlConflict(error)) { @@ -405,9 +579,17 @@ export async function sitesRoutes(app: FastifyInstance) { } throw error; } + const siteId = getInsertedRowId(inserted); + if (!siteId) { + return reply.code(500).send({ error: 'Create site failed' }); + } + const result = await loadSiteWithApiEndpoints(siteId); + if (!result) { + return reply.code(500).send({ error: 'Create site failed' }); + } invalidateSiteCaches(); return { - ...inserted, + ...result, ...(responseInitializationPresetId ? { initializationPresetId: responseInitializationPresetId } : {}), }; }); @@ -430,7 +612,7 @@ export async function sitesRoutes(app: FastifyInstance) { } const updates: any = {}; - const body = parsedBody.data; + const body = parsedBody.data as typeof parsedBody.data & { apiEndpoints?: unknown }; const normalizedStatus = normalizeSiteStatus(body.status); if (body.status !== undefined && !normalizedStatus) { return reply.code(400).send({ error: 'Invalid site status. Expected active or disabled.' }); @@ -463,6 +645,10 @@ export async function sitesRoutes(app: FastifyInstance) { if (!normalizedCustomHeaders.valid) { return reply.code(400).send({ error: normalizedCustomHeaders.error || 'Invalid customHeaders.' }); } + const normalizedApiEndpoints = normalizeSiteApiEndpointsInput(body.apiEndpoints); + if (!normalizedApiEndpoints.valid) { + return reply.code(400).send({ error: normalizedApiEndpoints.error || 'Invalid apiEndpoints.' }); + } const canonicalPlatform = normalizeSitePlatform(body.platform); const nextUrl = body.url !== undefined ? normalizeCanonicalSiteUrl(body.url) : existingSite.url; @@ -498,7 +684,24 @@ export async function sitesRoutes(app: FastifyInstance) { if (body.globalWeight !== undefined) updates.globalWeight = normalizedGlobalWeight; updates.updatedAt = new Date().toISOString(); try { - await db.update(schema.sites).set(updates).where(eq(schema.sites.id, id)).run(); + await db.transaction(async (tx) => { + await tx.update(schema.sites).set(updates).where(eq(schema.sites.id, id)).run(); + if (normalizedApiEndpoints.present) { + await tx.delete(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.siteId, id)) + .run(); + if (normalizedApiEndpoints.apiEndpoints.length > 0) { + await tx.insert(schema.siteApiEndpoints).values( + normalizedApiEndpoints.apiEndpoints.map((row) => ({ + siteId: id, + url: row.url, + enabled: row.enabled, + sortOrder: row.sortOrder, + })), + ).run(); + } + } + }); } catch (error) { if (isSitesPlatformUrlConflict(error)) { return sendSiteBindingConflict(reply, nextPlatform, nextUrl); @@ -512,7 +715,7 @@ export async function sitesRoutes(app: FastifyInstance) { invalidateSiteCaches(); - return await db.select().from(schema.sites).where(eq(schema.sites.id, id)).get(); + return await loadSiteWithApiEndpoints(id); }); // Delete a site diff --git a/src/server/routes/proxy/chat.codex-oauth.test.ts b/src/server/routes/proxy/chat.codex-oauth.test.ts index 5fe3262..2d6b106 100644 --- a/src/server/routes/proxy/chat.codex-oauth.test.ts +++ b/src/server/routes/proxy/chat.codex-oauth.test.ts @@ -55,6 +55,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../services/proxyUsageFallbackService.js', () => ({ @@ -68,12 +69,33 @@ vi.mock('../../services/oauth/refreshSingleflight.js', () => ({ vi.mock('../../db/index.js', () => ({ db: { insert: (arg: any) => dbInsertMock(arg), + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => [], + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), }, hasProxyLogBillingDetailsColumn: async () => false, hasProxyLogClientColumns: async () => false, hasProxyLogDownstreamApiKeyIdColumn: async () => false, schema: { proxyLogs: {}, + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, }, })); @@ -144,7 +166,9 @@ describe('chat proxy codex oauth compatibility', () => { }); afterAll(async () => { - await app.close(); + if (app) { + await app.close(); + } }); it('strips codex-unsupported responses fields and aggregates the SSE result for /v1/messages callers', async () => { diff --git a/src/server/routes/proxy/chat.count-tokens.test.ts b/src/server/routes/proxy/chat.count-tokens.test.ts index f72901a..c2d2c05 100644 --- a/src/server/routes/proxy/chat.count-tokens.test.ts +++ b/src/server/routes/proxy/chat.count-tokens.test.ts @@ -59,6 +59,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../services/proxyUsageFallbackService.js', () => ({ @@ -85,12 +86,33 @@ vi.mock('../../services/oauth/quota.js', () => ({ vi.mock('../../db/index.js', () => ({ db: { insert: (arg: any) => dbInsertMock(arg), + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => [], + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), }, hasProxyLogBillingDetailsColumn: async () => false, hasProxyLogClientColumns: async () => false, hasProxyLogDownstreamApiKeyIdColumn: async () => false, schema: { proxyLogs: {}, + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, }, })); @@ -147,7 +169,9 @@ describe('claude count_tokens proxy route', () => { }); afterAll(async () => { - await app.close(); + if (app) { + await app.close(); + } }); it('forwards /v1/messages/count_tokens to the claude count_tokens upstream path', async () => { diff --git a/src/server/routes/proxy/chat.siteApiEndpoint.test.ts b/src/server/routes/proxy/chat.siteApiEndpoint.test.ts new file mode 100644 index 0000000..7434780 --- /dev/null +++ b/src/server/routes/proxy/chat.siteApiEndpoint.test.ts @@ -0,0 +1,239 @@ +import Fastify, { type FastifyInstance } from 'fastify'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { asc, eq } from 'drizzle-orm'; +import { config } from '../../config.js'; +import { resetUpstreamEndpointRuntimeState } from '../../services/upstreamEndpointRuntimeMemory.js'; + +const fetchMock = vi.fn(); +const selectChannelMock = vi.fn(); +const selectNextChannelMock = vi.fn(); +const recordSuccessMock = vi.fn(); +const recordFailureMock = vi.fn(); +const refreshModelsAndRebuildRoutesMock = vi.fn(); +const reportProxyAllFailedMock = vi.fn(); +const reportTokenExpiredMock = vi.fn(); +const estimateProxyCostMock = vi.fn(async (_arg?: any) => 0); +const buildProxyBillingDetailsMock = vi.fn(async (_arg?: any) => null); +const fetchModelPricingCatalogMock = vi.fn(async (_arg?: any): Promise => null); +const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(async ({ usage }: any) => ({ + ...usage, + estimatedCostFromQuota: 0, + recoveredFromSelfLog: false, +})); +const insertProxyLogMock = vi.fn(); + +vi.mock('undici', () => ({ + fetch: (...args: unknown[]) => fetchMock(...args), +})); + +vi.mock('../../services/tokenRouter.js', () => ({ + tokenRouter: { + selectChannel: (...args: unknown[]) => selectChannelMock(...args), + selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args), + recordSuccess: (...args: unknown[]) => recordSuccessMock(...args), + recordFailure: (...args: unknown[]) => recordFailureMock(...args), + }, +})); + +vi.mock('../../services/modelService.js', () => ({ + refreshModelsAndRebuildRoutes: (...args: unknown[]) => refreshModelsAndRebuildRoutesMock(...args), +})); + +vi.mock('../../services/alertService.js', () => ({ + reportProxyAllFailed: (...args: unknown[]) => reportProxyAllFailedMock(...args), + reportTokenExpired: (...args: unknown[]) => reportTokenExpiredMock(...args), +})); + +vi.mock('../../services/alertRules.js', () => ({ + isTokenExpiredError: () => false, +})); + +vi.mock('../../services/modelPricingService.js', () => ({ + estimateProxyCost: (arg: any) => estimateProxyCostMock(arg), + buildProxyBillingDetails: (arg: any) => buildProxyBillingDetailsMock(arg), + fetchModelPricingCatalog: (arg: any) => fetchModelPricingCatalogMock(arg), +})); + +vi.mock('../../services/proxyRetryPolicy.js', () => ({ + shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], +})); + +vi.mock('../../services/proxyUsageFallbackService.js', () => ({ + resolveProxyUsageWithSelfLogFallback: (arg: any) => resolveProxyUsageWithSelfLogFallbackMock(arg), +})); + +vi.mock('../../services/proxyLogStore.js', () => ({ + insertProxyLog: (...args: unknown[]) => insertProxyLogMock(...args), +})); + +type DbModule = typeof import('../../db/index.js'); + +describe('chat proxy site api endpoint rotation', () => { + let app: FastifyInstance; + let db: DbModule['db']; + let schema: DbModule['schema']; + let dataDir = ''; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-chat-site-api-endpoint-')); + process.env.DATA_DIR = dataDir; + + await import('../../db/migrate.js'); + const dbModule = await import('../../db/index.js'); + const routesModule = await import('./chat.js'); + db = dbModule.db; + schema = dbModule.schema; + + app = Fastify(); + await app.register(routesModule.chatProxyRoute); + }); + + beforeEach(async () => { + fetchMock.mockReset(); + selectChannelMock.mockReset(); + selectNextChannelMock.mockReset(); + recordSuccessMock.mockReset(); + recordFailureMock.mockReset(); + refreshModelsAndRebuildRoutesMock.mockReset(); + reportProxyAllFailedMock.mockReset(); + reportTokenExpiredMock.mockReset(); + estimateProxyCostMock.mockClear(); + buildProxyBillingDetailsMock.mockClear(); + fetchModelPricingCatalogMock.mockReset(); + resolveProxyUsageWithSelfLogFallbackMock.mockClear(); + insertProxyLogMock.mockReset(); + resetUpstreamEndpointRuntimeState(); + + await db.delete(schema.proxyLogs).run(); + await db.delete(schema.routeChannels).run(); + await db.delete(schema.tokenRoutes).run(); + await db.delete(schema.tokenModelAvailability).run(); + await db.delete(schema.modelAvailability).run(); + await db.delete(schema.accountTokens).run(); + await db.delete(schema.accounts).run(); + await db.delete(schema.siteApiEndpoints).run(); + await db.delete(schema.sites).run(); + + fetchModelPricingCatalogMock.mockResolvedValue(null); + (config as any).codexHeaderDefaults = { + userAgent: '', + betaFeatures: '', + }; + (config as any).payloadRules = { + default: [], + defaultRaw: [], + override: [], + overrideRaw: [], + filter: [], + }; + (config as any).disableCrossProtocolFallback = false; + config.proxyEmptyContentFailEnabled = false; + config.proxyErrorKeywords = []; + }); + + afterAll(async () => { + if (app) { + await app.close(); + } + delete process.env.DATA_DIR; + }); + + it('rotates to the next configured ai endpoint for retryable /v1/chat/completions failures', async () => { + const site = await db.insert(schema.sites).values({ + name: 'nihao-panel', + url: 'https://nih.cc', + platform: 'openai', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'nihao-user', + accessToken: '', + apiToken: 'sk-nihao', + status: 'active', + checkinEnabled: false, + extraConfig: JSON.stringify({ credentialMode: 'apikey' }), + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-a.nih.cc', + enabled: true, + sortOrder: 0, + }, + { + siteId: site.id, + url: 'https://api-b.nih.cc', + enabled: true, + sortOrder: 1, + }, + ]).run(); + + selectChannelMock.mockReturnValue({ + channel: { id: 11, routeId: 22 }, + site, + account, + tokenName: 'default', + tokenValue: 'sk-nihao', + actualModel: 'gpt-4o-mini', + }); + selectNextChannelMock.mockReturnValue(null); + + fetchMock + .mockResolvedValueOnce(new Response('bad gateway', { status: 502 })) + .mockResolvedValueOnce(new Response('bad gateway via responses', { status: 502 })) + .mockResolvedValueOnce(new Response('bad gateway via messages', { status: 502 })) + .mockResolvedValueOnce(new Response(JSON.stringify({ + id: 'chatcmpl-ok', + object: 'chat.completion', + created: 1_706_000_000, + model: 'gpt-4o-mini', + choices: [{ + index: 0, + message: { role: 'assistant', content: 'ok via api-b' }, + finish_reason: 'stop', + }], + usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const response = await app.inject({ + method: 'POST', + url: '/v1/chat/completions', + payload: { + model: 'gpt-4o-mini', + messages: [{ role: 'user', content: 'hi' }], + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()?.choices?.[0]?.message?.content).toBe('ok via api-b'); + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(String(fetchMock.mock.calls[0]?.[0] || '')).toBe('https://api-a.nih.cc/v1/chat/completions'); + expect(String(fetchMock.mock.calls[1]?.[0] || '')).toBe('https://api-a.nih.cc/v1/responses'); + expect(String(fetchMock.mock.calls[2]?.[0] || '')).toBe('https://api-a.nih.cc/v1/messages'); + expect(String(fetchMock.mock.calls[3]?.[0] || '')).toBe('https://api-b.nih.cc/v1/chat/completions'); + expect(selectNextChannelMock).not.toHaveBeenCalled(); + expect(recordFailureMock).not.toHaveBeenCalled(); + expect(recordSuccessMock).toHaveBeenCalledTimes(1); + + const storedEndpoints = await db.select().from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.siteId, site.id)) + .orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id)) + .all(); + expect(storedEndpoints[0]).toMatchObject({ + url: 'https://api-a.nih.cc', + lastFailureReason: 'HTTP 502: [upstream:/v1/messages] Upstream returned HTTP 502: bad gateway via messages', + }); + expect(storedEndpoints[0]?.cooldownUntil).toBeTruthy(); + expect(storedEndpoints[1]?.lastSelectedAt).toBeTruthy(); + }); +}); diff --git a/src/server/routes/proxy/chat.stream.test.ts b/src/server/routes/proxy/chat.stream.test.ts index 70f0595..9b2497e 100644 --- a/src/server/routes/proxy/chat.stream.test.ts +++ b/src/server/routes/proxy/chat.stream.test.ts @@ -60,6 +60,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../services/proxyUsageFallbackService.js', () => ({ @@ -69,12 +70,33 @@ vi.mock('../../services/proxyUsageFallbackService.js', () => ({ vi.mock('../../db/index.js', () => ({ db: { insert: (arg: any) => dbInsertMock(arg), + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => [], + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), }, hasProxyLogBillingDetailsColumn: async () => false, hasProxyLogClientColumns: async () => false, hasProxyLogDownstreamApiKeyIdColumn: async () => false, schema: { proxyLogs: {}, + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, }, })); @@ -133,7 +155,9 @@ describe('chat proxy stream behavior', () => { }); afterAll(async () => { - await app.close(); + if (app) { + await app.close(); + } }); it('converts non-SSE upstream streaming responses into SSE events', async () => { diff --git a/src/server/routes/proxy/completions.siteApiEndpoint.test.ts b/src/server/routes/proxy/completions.siteApiEndpoint.test.ts new file mode 100644 index 0000000..f919a21 --- /dev/null +++ b/src/server/routes/proxy/completions.siteApiEndpoint.test.ts @@ -0,0 +1,213 @@ +import Fastify, { type FastifyInstance } from 'fastify'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { asc, eq } from 'drizzle-orm'; + +const fetchMock = vi.fn(); +const selectChannelMock = vi.fn(); +const selectNextChannelMock = vi.fn(); +const recordSuccessMock = vi.fn(); +const recordFailureMock = vi.fn(); +const refreshModelsAndRebuildRoutesMock = vi.fn(); +const reportProxyAllFailedMock = vi.fn(); +const reportTokenExpiredMock = vi.fn(); +const insertProxyLogMock = vi.fn(); +const resolveProxyUsageWithSelfLogFallbackMock = vi.fn(); +const resolveProxyLogBillingMock = vi.fn(); + +vi.mock('undici', () => ({ + fetch: (...args: unknown[]) => fetchMock(...args), +})); + +vi.mock('../../services/tokenRouter.js', () => ({ + tokenRouter: { + selectChannel: (...args: unknown[]) => selectChannelMock(...args), + selectNextChannel: (...args: unknown[]) => selectNextChannelMock(...args), + recordSuccess: (...args: unknown[]) => recordSuccessMock(...args), + recordFailure: (...args: unknown[]) => recordFailureMock(...args), + }, + invalidateTokenRouterCache: vi.fn(), +})); + +vi.mock('../../services/routeRefreshWorkflow.js', () => ({ + refreshModelsAndRebuildRoutes: (...args: unknown[]) => refreshModelsAndRebuildRoutesMock(...args), +})); + +vi.mock('../../services/alertService.js', () => ({ + reportProxyAllFailed: (...args: unknown[]) => reportProxyAllFailedMock(...args), + reportTokenExpired: (...args: unknown[]) => reportTokenExpiredMock(...args), +})); + +vi.mock('../../services/proxyLogStore.js', () => ({ + insertProxyLog: (...args: unknown[]) => insertProxyLogMock(...args), +})); + +vi.mock('../../services/proxyUsageFallbackService.js', () => ({ + resolveProxyUsageWithSelfLogFallback: (...args: unknown[]) => resolveProxyUsageWithSelfLogFallbackMock(...args), +})); + +vi.mock('./proxyBilling.js', () => ({ + resolveProxyLogBilling: (...args: unknown[]) => resolveProxyLogBillingMock(...args), +})); + +type DbModule = typeof import('../../db/index.js'); + +describe('/v1/completions site api endpoint rotation', () => { + let app: FastifyInstance; + let db: DbModule['db']; + let schema: DbModule['schema']; + let dataDir = ''; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-completions-site-api-endpoint-')); + process.env.DATA_DIR = dataDir; + + await import('../../db/migrate.js'); + const dbModule = await import('../../db/index.js'); + const routesModule = await import('./completions.js'); + db = dbModule.db; + schema = dbModule.schema; + + app = Fastify(); + await app.register(routesModule.completionsProxyRoute); + }); + + beforeEach(async () => { + fetchMock.mockReset(); + selectChannelMock.mockReset(); + selectNextChannelMock.mockReset(); + recordSuccessMock.mockReset(); + recordFailureMock.mockReset(); + refreshModelsAndRebuildRoutesMock.mockReset(); + reportProxyAllFailedMock.mockReset(); + reportTokenExpiredMock.mockReset(); + insertProxyLogMock.mockReset(); + resolveProxyUsageWithSelfLogFallbackMock.mockReset(); + resolveProxyLogBillingMock.mockReset(); + + resolveProxyUsageWithSelfLogFallbackMock.mockResolvedValue({ + promptTokens: 1, + completionTokens: 2, + totalTokens: 3, + }); + resolveProxyLogBillingMock.mockResolvedValue({ + estimatedCost: 0, + billingDetails: null, + }); + + await db.delete(schema.proxyLogs).run(); + await db.delete(schema.routeChannels).run(); + await db.delete(schema.tokenRoutes).run(); + await db.delete(schema.tokenModelAvailability).run(); + await db.delete(schema.modelAvailability).run(); + await db.delete(schema.accountTokens).run(); + await db.delete(schema.accounts).run(); + await db.delete(schema.siteApiEndpoints).run(); + await db.delete(schema.sites).run(); + }); + + afterAll(async () => { + await app.close(); + delete process.env.DATA_DIR; + }); + + it('cools down a retryable failed endpoint and retries the next endpoint within the same site', async () => { + const site = await db.insert(schema.sites).values({ + name: 'nihao-panel', + url: 'https://nih.cc', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'nihao-user', + accessToken: '', + apiToken: 'sk-nihao', + status: 'active', + checkinEnabled: false, + extraConfig: JSON.stringify({ credentialMode: 'apikey' }), + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-a.nih.cc', + enabled: true, + sortOrder: 0, + }, + { + siteId: site.id, + url: 'https://api-b.nih.cc', + enabled: true, + sortOrder: 1, + }, + ]).run(); + + selectChannelMock.mockResolvedValue({ + channel: { id: 11, routeId: 22 }, + site, + account, + tokenName: 'default', + tokenValue: 'sk-nihao', + actualModel: 'gpt-4o-mini', + }); + selectNextChannelMock.mockResolvedValue(null); + + fetchMock + .mockResolvedValueOnce(new Response('bad gateway', { status: 502 })) + .mockResolvedValueOnce(new Response(JSON.stringify({ + id: 'cmpl-ok', + object: 'text_completion', + choices: [{ text: 'ok' }], + usage: { + prompt_tokens: 1, + completion_tokens: 2, + total_tokens: 3, + }, + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const response = await app.inject({ + method: 'POST', + url: '/v1/completions', + headers: { + authorization: 'Bearer sk-downstream', + }, + payload: { + model: 'gpt-4o-mini', + prompt: 'hello', + }, + }); + + expect(response.statusCode).toBe(200); + expect(response.json()).toMatchObject({ + id: 'cmpl-ok', + object: 'text_completion', + }); + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(String(fetchMock.mock.calls[0]?.[0] || '')).toBe('https://api-a.nih.cc/v1/completions'); + expect(String(fetchMock.mock.calls[1]?.[0] || '')).toBe('https://api-b.nih.cc/v1/completions'); + expect(selectNextChannelMock).not.toHaveBeenCalled(); + expect(recordFailureMock).not.toHaveBeenCalled(); + expect(recordSuccessMock).toHaveBeenCalledTimes(1); + + const storedEndpoints = await db.select().from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.siteId, site.id)) + .orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id)) + .all(); + expect(storedEndpoints[0]).toMatchObject({ + url: 'https://api-a.nih.cc', + lastFailureReason: 'HTTP 502: bad gateway', + }); + expect(storedEndpoints[0]?.cooldownUntil).toBeTruthy(); + expect(storedEndpoints[1]).toMatchObject({ + url: 'https://api-b.nih.cc', + }); + expect(storedEndpoints[1]?.lastSelectedAt).toBeTruthy(); + }); +}); diff --git a/src/server/routes/proxy/completions.ts b/src/server/routes/proxy/completions.ts index 8e2e703..a4bd54f 100644 --- a/src/server/routes/proxy/completions.ts +++ b/src/server/routes/proxy/completions.ts @@ -10,14 +10,15 @@ import { ensureModelAllowedForDownstreamKey, getDownstreamRoutingPolicy, recordD import { withSiteRecordProxyRequestInit } from '../../services/siteProxy.js'; import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js'; import { composeProxyLogMessage } from './logPathMeta.js'; -import { formatUtcSqlDateTime } from '../../services/localTimeService.js'; -import { detectProxyFailure } from './proxyFailureJudge.js'; -import { resolveProxyLogBilling } from './proxyBilling.js'; +import { formatUtcSqlDateTime } from '../../services/localTimeService.js'; +import { detectProxyFailure } from './proxyFailureJudge.js'; +import { resolveProxyLogBilling } from './proxyBilling.js'; import { getProxyAuthContext } from '../../middleware/auth.js'; import { buildUpstreamUrl } from './upstreamUrl.js'; import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js'; import { insertProxyLog } from '../../services/proxyLogStore.js'; import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js'; +import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js'; import { buildForcedChannelUnavailableMessage, canRetryChannelSelection, @@ -72,68 +73,30 @@ export async function completionsProxyRoute(app: FastifyInstance) { excludeChannelIds.push(selected.channel.id); - const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/completions'); const upstreamModel = selected.actualModel || requestedModel; const forwardBody = { ...body, model: upstreamModel }; const startTime = Date.now(); - - try { - const upstream = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${selected.tokenValue}`, - }, - body: JSON.stringify(forwardBody), - }, getProxyUrlFromExtraConfig(selected.account.extraConfig))); - - if (!upstream.ok) { - const errText = await upstream.text().catch(() => 'unknown error'); - await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: upstream.status, - errorText: errText, - modelName: upstreamModel, - })); - logProxy( - selected, - requestedModel, - 'failed', - upstream.status, - Date.now() - startTime, - errText, - retryCount, - downstreamApiKeyId, - 0, - 0, - 0, - 0, - null, - clientContext, - downstreamPath, - ); - - if (isTokenExpiredError({ status: upstream.status, message: errText })) { - await reportTokenExpired({ - accountId: selected.account.id, - username: selected.account.username, - siteName: selected.site.name, - detail: `HTTP ${upstream.status}`, - }); - } - - if (shouldRetryProxyRequest(upstream.status, errText) && canRetryChannelSelection(retryCount, forcedChannelId)) { - retryCount++; - continue; + try { + const upstream = await runWithSiteApiEndpointPool(selected.site, async (target) => { + const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/completions'); + const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${selected.tokenValue}`, + }, + body: JSON.stringify(forwardBody), + }, getProxyUrlFromExtraConfig(selected.account.extraConfig))); + if (!response.ok) { + const errText = await response.text().catch(() => 'unknown error'); + throw new SiteApiEndpointRequestError(errText || 'unknown error', { + status: response.status, + }); } - - await reportProxyAllFailed({ - model: requestedModel, - reason: `upstream returned HTTP ${upstream.status}`, - }); - return reply.code(upstream.status).send({ error: { message: errText, type: 'upstream_error' } }); - } - - if (isStream) { + return response; + }); + + if (isStream) { reply.raw.writeHead(200, { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache', @@ -328,18 +291,20 @@ export async function completionsProxyRoute(app: FastifyInstance) { ); return reply.send(data); } catch (err: any) { + const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0; + const errorText = err?.message || 'network failure'; await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: 0, - errorText: err.message, + status, + errorText, modelName: upstreamModel, })); logProxy( selected, requestedModel, 'failed', - 0, + status, Date.now() - startTime, - err.message, + errorText, retryCount, downstreamApiKeyId, 0, @@ -350,18 +315,26 @@ export async function completionsProxyRoute(app: FastifyInstance) { clientContext, downstreamPath, ); - if (canRetryChannelSelection(retryCount, forcedChannelId)) { + if (status > 0 && isTokenExpiredError({ status, message: errorText })) { + await reportTokenExpired({ + accountId: selected.account.id, + username: selected.account.username, + siteName: selected.site.name, + detail: `HTTP ${status}`, + }); + } + if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) { retryCount++; continue; } - await reportProxyAllFailed({ - model: requestedModel, - reason: err.message || 'network failure', - }); - return reply.code(502).send({ - error: { message: `Upstream error: ${err.message}`, type: 'upstream_error' }, - }); - } + await reportProxyAllFailed({ + model: requestedModel, + reason: errorText || 'network failure', + }); + return reply.code(status || 502).send({ + error: { message: status > 0 ? errorText : `Upstream error: ${errorText}`, type: 'upstream_error' }, + }); + } } }); } diff --git a/src/server/routes/proxy/downstreamClientContext.routes.test.ts b/src/server/routes/proxy/downstreamClientContext.routes.test.ts index 147b94d..d484541 100644 --- a/src/server/routes/proxy/downstreamClientContext.routes.test.ts +++ b/src/server/routes/proxy/downstreamClientContext.routes.test.ts @@ -58,6 +58,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../services/proxyUsageFallbackService.js', () => ({ @@ -67,9 +68,30 @@ vi.mock('../../services/proxyUsageFallbackService.js', () => ({ vi.mock('../../db/index.js', () => ({ db: { insert: (arg: any) => dbInsertMock(arg), + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => [], + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), }, schema: { proxyLogs: {}, + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, }, hasProxyLogBillingDetailsColumn: async () => false, hasProxyLogClientColumns: async () => false, @@ -116,7 +138,9 @@ describe('downstream client context route logging', () => { }); afterAll(async () => { - await app.close(); + if (app) { + await app.close(); + } }); it('includes Codex client and session metadata in /v1/responses failure logs', async () => { diff --git a/src/server/routes/proxy/embeddings.ts b/src/server/routes/proxy/embeddings.ts index e488f27..57551cd 100644 --- a/src/server/routes/proxy/embeddings.ts +++ b/src/server/routes/proxy/embeddings.ts @@ -12,11 +12,12 @@ import { getProxyUrlFromExtraConfig } from '../../services/accountExtraConfig.js import { composeProxyLogMessage } from './logPathMeta.js'; import { formatUtcSqlDateTime } from '../../services/localTimeService.js'; import { resolveProxyLogBilling } from './proxyBilling.js'; -import { getProxyAuthContext } from '../../middleware/auth.js'; +import { getProxyAuthContext } from '../../middleware/auth.js'; import { buildUpstreamUrl } from './upstreamUrl.js'; import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js'; import { insertProxyLog } from '../../services/proxyLogStore.js'; import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js'; +import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js'; import { buildForcedChannelUnavailableMessage, canRetryChannelSelection, @@ -66,68 +67,45 @@ export async function embeddingsProxyRoute(app: FastifyInstance) { return reply.code(503).send({ error: { message: noChannelMessage, type: 'server_error' } }); } excludeChannelIds.push(selected.channel.id); - - const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/embeddings'); const upstreamModel = selected.actualModel || requestedModel; const forwardBody = { ...body, model: upstreamModel }; const startTime = Date.now(); - - try { - const upstream = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${selected.tokenValue}`, - }, - body: JSON.stringify(forwardBody), - }, getProxyUrlFromExtraConfig(selected.account.extraConfig))); - - const text = await upstream.text(); - if (!upstream.ok) { - await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: upstream.status, - errorText: text, - modelName: upstreamModel, - })); - logProxy( - selected, - requestedModel, - 'failed', - upstream.status, - Date.now() - startTime, - text, - retryCount, - downstreamApiKeyId, - 0, - 0, - 0, - 0, - null, - clientContext, - downstreamPath, - ); - - if (isTokenExpiredError({ status: upstream.status, message: text })) { - await reportTokenExpired({ - accountId: selected.account.id, - username: selected.account.username, - siteName: selected.site.name, - detail: `HTTP ${upstream.status}`, - }); - } - - if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) { - retryCount++; - continue; + try { + const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => { + const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/embeddings'); + const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${selected.tokenValue}`, + }, + body: JSON.stringify(forwardBody), + }, getProxyUrlFromExtraConfig(selected.account.extraConfig))); + const status = response.status; + let responseText = ''; + try { + responseText = await response.text(); + } catch (error) { + if (!response.ok) { + throw new SiteApiEndpointRequestError('unknown error', { + status, + cause: error, + }); + } + throw error; } - - await reportProxyAllFailed({ - model: requestedModel, - reason: `upstream returned HTTP ${upstream.status}`, - }); - return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } }); - } - + if (!response.ok) { + throw new SiteApiEndpointRequestError(responseText || 'unknown error', { + status, + rawErrText: responseText || null, + }); + } + return { + upstream: response, + text: responseText, + }; + }); + let data: any = {}; try { data = JSON.parse(text); } catch { data = {}; } const latency = Date.now() - startTime; @@ -165,18 +143,20 @@ export async function embeddingsProxyRoute(app: FastifyInstance) { ); return reply.code(upstream.status).send(data); } catch (err: any) { + const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0; + const errorText = err?.message || 'network failure'; await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: 0, - errorText: err.message, + status, + errorText, modelName: upstreamModel, })); logProxy( selected, requestedModel, 'failed', - 0, + status, Date.now() - startTime, - err.message, + errorText, retryCount, downstreamApiKeyId, 0, @@ -187,16 +167,29 @@ export async function embeddingsProxyRoute(app: FastifyInstance) { clientContext, downstreamPath, ); - if (canRetryChannelSelection(retryCount, forcedChannelId)) { + if (status > 0 && isTokenExpiredError({ status, message: errorText })) { + await reportTokenExpired({ + accountId: selected.account.id, + username: selected.account.username, + siteName: selected.site.name, + detail: `HTTP ${status}`, + }); + } + if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) { retryCount++; continue; } - await reportProxyAllFailed({ - model: requestedModel, - reason: err.message || 'network failure', - }); - return reply.code(502).send({ error: { message: err.message, type: 'upstream_error' } }); - } + await reportProxyAllFailed({ + model: requestedModel, + reason: errorText || 'network failure', + }); + return reply.code(status || 502).send({ + error: { + message: status > 0 ? errorText : `Upstream error: ${errorText}`, + type: 'upstream_error', + }, + }); + } } }); } diff --git a/src/server/routes/proxy/images.edits.test.ts b/src/server/routes/proxy/images.edits.test.ts index db5507f..c130a46 100644 --- a/src/server/routes/proxy/images.edits.test.ts +++ b/src/server/routes/proxy/images.edits.test.ts @@ -52,17 +52,39 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../db/index.js', () => ({ db: { insert: (arg: any) => dbInsertMock(arg), + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => [], + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), }, hasProxyLogBillingDetailsColumn: async () => false, hasProxyLogClientColumns: async () => false, hasProxyLogDownstreamApiKeyIdColumn: async () => false, schema: { proxyLogs: {}, + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, }, })); @@ -115,7 +137,9 @@ describe('/v1/images/edits route', () => { }); afterAll(async () => { - await app.close(); + if (app) { + await app.close(); + } }); it('accepts multipart image edit requests and forwards them to /v1/images/edits', async () => { diff --git a/src/server/routes/proxy/images.ts b/src/server/routes/proxy/images.ts index 8a5b6c1..40a9c48 100644 --- a/src/server/routes/proxy/images.ts +++ b/src/server/routes/proxy/images.ts @@ -16,6 +16,7 @@ import { buildUpstreamUrl } from './upstreamUrl.js'; import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js'; import { insertProxyLog } from '../../services/proxyLogStore.js'; import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js'; +import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js'; import { buildForcedChannelUnavailableMessage, canRetryChannelSelection, @@ -66,60 +67,32 @@ export async function imagesProxyRoute(app: FastifyInstance) { } excludeChannelIds.push(selected.channel.id); - - const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/images/generations'); const upstreamModel = selected.actualModel || requestedModel; const forwardBody = { ...body, model: upstreamModel }; const startTime = Date.now(); try { - const upstream = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${selected.tokenValue}`, - }, - body: JSON.stringify(forwardBody), - }, getProxyUrlFromExtraConfig(selected.account.extraConfig))); - - const text = await upstream.text(); - if (!upstream.ok) { - await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: upstream.status, - errorText: text, - modelName: upstreamModel, - })); - logProxy( - selected, - requestedModel, - 'failed', - upstream.status, - Date.now() - startTime, - text, - retryCount, - downstreamApiKeyId, - 0, - downstreamPath, - clientContext, - ); - if (isTokenExpiredError({ status: upstream.status, message: text })) { - await reportTokenExpired({ - accountId: selected.account.id, - username: selected.account.username, - siteName: selected.site.name, - detail: `HTTP ${upstream.status}`, + const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => { + const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/images/generations'); + const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${selected.tokenValue}`, + }, + body: JSON.stringify(forwardBody), + }, getProxyUrlFromExtraConfig(selected.account.extraConfig))); + const responseText = await response.text(); + if (!response.ok) { + throw new SiteApiEndpointRequestError(responseText || 'unknown error', { + status: response.status, }); } - if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) { - retryCount++; - continue; - } - await reportProxyAllFailed({ - model: requestedModel, - reason: `upstream returned HTTP ${upstream.status}`, - }); - return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } }); - } + return { + upstream: response, + text: responseText, + }; + }); const data = parseUpstreamImageResponse(text); if (!data.ok) { @@ -175,34 +148,47 @@ export async function imagesProxyRoute(app: FastifyInstance) { logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId, estimatedCost, downstreamPath, clientContext); return reply.code(upstream.status).send(data.value); } catch (err: any) { + const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0; + const errorText = err?.message || 'network failure'; await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: 0, - errorText: err.message, + status, + errorText, modelName: upstreamModel, })); logProxy( selected, requestedModel, 'failed', - 0, + status, Date.now() - startTime, - err.message, + errorText, retryCount, downstreamApiKeyId, 0, downstreamPath, clientContext, ); - if (canRetryChannelSelection(retryCount, forcedChannelId)) { + if (status > 0 && isTokenExpiredError({ status, message: errorText })) { + await reportTokenExpired({ + accountId: selected.account.id, + username: selected.account.username, + siteName: selected.site.name, + detail: `HTTP ${status}`, + }); + } + if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) { retryCount++; continue; } await reportProxyAllFailed({ model: requestedModel, - reason: err.message || 'network failure', + reason: errorText || 'network failure', }); - return reply.code(502).send({ - error: { message: `Upstream error: ${err.message}`, type: 'upstream_error' }, + return reply.code(status || 502).send({ + error: { + message: status > 0 ? errorText : `Upstream error: ${errorText}`, + type: 'upstream_error', + }, }); } } @@ -254,72 +240,45 @@ export async function imagesProxyRoute(app: FastifyInstance) { } excludeChannelIds.push(selected.channel.id); - const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/images/edits'); const upstreamModel = selected.actualModel || requestedModel; const startTime = Date.now(); try { - const requestInit = multipartForm - ? withSiteRecordProxyRequestInit(selected.site, { - method: 'POST', - headers: { - Authorization: `Bearer ${selected.tokenValue}`, - }, - body: cloneFormDataWithOverrides(multipartForm, { - model: upstreamModel, - }) as any, - }, getProxyUrlFromExtraConfig(selected.account.extraConfig)) - : withSiteRecordProxyRequestInit(selected.site, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${selected.tokenValue}`, - }, - body: JSON.stringify({ - ...(jsonBody || {}), - model: upstreamModel, - }), - }, getProxyUrlFromExtraConfig(selected.account.extraConfig)); - - const upstream = await fetch(targetUrl, requestInit); - const text = await upstream.text(); - if (!upstream.ok) { - await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: upstream.status, - errorText: text, - modelName: upstreamModel, - })); - logProxy( - selected, - requestedModel, - 'failed', - upstream.status, - Date.now() - startTime, - text, - retryCount, - downstreamApiKeyId, - 0, - downstreamPath, - clientContext, - ); - if (isTokenExpiredError({ status: upstream.status, message: text })) { - await reportTokenExpired({ - accountId: selected.account.id, - username: selected.account.username, - siteName: selected.site.name, - detail: `HTTP ${upstream.status}`, + const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => { + const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/images/edits'); + const requestInit = multipartForm + ? withSiteRecordProxyRequestInit(selected.site, { + method: 'POST', + headers: { + Authorization: `Bearer ${selected.tokenValue}`, + }, + body: cloneFormDataWithOverrides(multipartForm, { + model: upstreamModel, + }) as any, + }, getProxyUrlFromExtraConfig(selected.account.extraConfig)) + : withSiteRecordProxyRequestInit(selected.site, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${selected.tokenValue}`, + }, + body: JSON.stringify({ + ...(jsonBody || {}), + model: upstreamModel, + }), + }, getProxyUrlFromExtraConfig(selected.account.extraConfig)); + const response = await fetch(targetUrl, requestInit); + const responseText = await response.text(); + if (!response.ok) { + throw new SiteApiEndpointRequestError(responseText || 'unknown error', { + status: response.status, }); } - if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) { - retryCount++; - continue; - } - await reportProxyAllFailed({ - model: requestedModel, - reason: `upstream returned HTTP ${upstream.status}`, - }); - return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } }); - } + return { + upstream: response, + text: responseText, + }; + }); const data = parseUpstreamImageResponse(text); if (!data.ok) { @@ -375,34 +334,47 @@ export async function imagesProxyRoute(app: FastifyInstance) { logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId, estimatedCost, downstreamPath, clientContext); return reply.code(upstream.status).send(data.value); } catch (err: any) { + const status = err instanceof SiteApiEndpointRequestError ? (err.status || 0) : 0; + const errorText = err?.message || 'network failure'; await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: 0, - errorText: err.message, + status, + errorText, modelName: upstreamModel, })); logProxy( selected, requestedModel, 'failed', - 0, + status, Date.now() - startTime, - err.message, + errorText, retryCount, downstreamApiKeyId, 0, downstreamPath, clientContext, ); - if (canRetryChannelSelection(retryCount, forcedChannelId)) { + if (status > 0 && isTokenExpiredError({ status, message: errorText })) { + await reportTokenExpired({ + accountId: selected.account.id, + username: selected.account.username, + siteName: selected.site.name, + detail: `HTTP ${status}`, + }); + } + if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) { retryCount++; continue; } await reportProxyAllFailed({ model: requestedModel, - reason: err.message || 'network failure', + reason: errorText || 'network failure', }); - return reply.code(502).send({ - error: { message: `Upstream error: ${err.message}`, type: 'upstream_error' }, + return reply.code(status || 502).send({ + error: { + message: status > 0 ? errorText : `Upstream error: ${errorText}`, + type: 'upstream_error', + }, }); } } diff --git a/src/server/routes/proxy/responses.codex-oauth.test.ts b/src/server/routes/proxy/responses.codex-oauth.test.ts index 47f7499..445e1ef 100644 --- a/src/server/routes/proxy/responses.codex-oauth.test.ts +++ b/src/server/routes/proxy/responses.codex-oauth.test.ts @@ -69,6 +69,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../services/proxyUsageFallbackService.js', () => ({ @@ -86,12 +87,33 @@ vi.mock('../../services/oauth/quota.js', () => ({ vi.mock('../../db/index.js', () => ({ db: { insert: (arg: any) => dbInsertMock(arg), + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => [], + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), }, hasProxyLogBillingDetailsColumn: async () => false, hasProxyLogClientColumns: async () => false, hasProxyLogDownstreamApiKeyIdColumn: async () => false, schema: { proxyLogs: {}, + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, }, })); @@ -205,7 +227,9 @@ describe('responses proxy codex oauth refresh', () => { config.proxyStickySessionEnabled = originalProxyStickySessionEnabled; config.proxySessionChannelConcurrencyLimit = originalProxySessionChannelConcurrencyLimit; config.proxySessionChannelQueueWaitMs = originalProxySessionChannelQueueWaitMs; - await app.close(); + if (app) { + await app.close(); + } }); it('refreshes codex oauth token and retries the same responses request on 401', async () => { diff --git a/src/server/routes/proxy/responses.compact-upstream.test.ts b/src/server/routes/proxy/responses.compact-upstream.test.ts index ce93ae2..7d6166f 100644 --- a/src/server/routes/proxy/responses.compact-upstream.test.ts +++ b/src/server/routes/proxy/responses.compact-upstream.test.ts @@ -57,6 +57,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../services/proxyUsageFallbackService.js', () => ({ @@ -66,12 +67,33 @@ vi.mock('../../services/proxyUsageFallbackService.js', () => ({ vi.mock('../../db/index.js', () => ({ db: { insert: (arg: any) => dbInsertMock(arg), + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => [], + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), }, hasProxyLogBillingDetailsColumn: async () => false, hasProxyLogClientColumns: async () => false, hasProxyLogDownstreamApiKeyIdColumn: async () => false, schema: { proxyLogs: {}, + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, }, })); @@ -112,7 +134,9 @@ describe('responses proxy compact upstream routing', () => { }); afterAll(async () => { - await app.close(); + if (app) { + await app.close(); + } }); it('forwards compact requests to the upstream /v1/responses/compact path first', async () => { diff --git a/src/server/routes/proxy/responses.websocket.test.ts b/src/server/routes/proxy/responses.websocket.test.ts index eae898d..5418494 100644 --- a/src/server/routes/proxy/responses.websocket.test.ts +++ b/src/server/routes/proxy/responses.websocket.test.ts @@ -78,6 +78,7 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../services/proxyUsageFallbackService.js', () => ({ @@ -91,12 +92,33 @@ vi.mock('../../services/oauth/quota.js', () => ({ vi.mock('../../db/index.js', () => ({ db: { insert: (arg: any) => dbInsertMock(arg), + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => [], + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), }, hasProxyLogBillingDetailsColumn: async () => false, hasProxyLogClientColumns: async () => false, hasProxyLogDownstreamApiKeyIdColumn: async () => false, schema: { proxyLogs: {}, + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, }, })); @@ -366,9 +388,15 @@ describe('responses websocket transport', () => { afterAll(async () => { (config as any).codexUpstreamWebsocketEnabled = originalCodexUpstreamWebsocketEnabled; - await new Promise((resolve) => rejectedUpgradeServer.close(() => resolve())); - await new Promise((resolve) => upstreamServer.close(() => resolve())); - await app.close(); + if (rejectedUpgradeServer) { + await new Promise((resolve) => rejectedUpgradeServer.close(() => resolve())); + } + if (upstreamServer) { + await new Promise((resolve) => upstreamServer.close(() => resolve())); + } + if (app) { + await app.close(); + } }); it('accepts response.create over GET /v1/responses websocket and forwards streamed responses events', async () => { diff --git a/src/server/routes/proxy/search.test.ts b/src/server/routes/proxy/search.test.ts index a618257..870e1df 100644 --- a/src/server/routes/proxy/search.test.ts +++ b/src/server/routes/proxy/search.test.ts @@ -48,17 +48,39 @@ vi.mock('../../services/modelPricingService.js', () => ({ vi.mock('../../services/proxyRetryPolicy.js', () => ({ shouldRetryProxyRequest: () => false, + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../db/index.js', () => ({ db: { insert: (arg: any) => dbInsertMock(arg), + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => [], + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), }, hasProxyLogBillingDetailsColumn: async () => false, hasProxyLogClientColumns: async () => false, hasProxyLogDownstreamApiKeyIdColumn: async () => false, schema: { proxyLogs: {}, + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, }, })); @@ -95,7 +117,9 @@ describe('/v1/search route', () => { }); afterAll(async () => { - await app.close(); + if (app) { + await app.close(); + } }); it('defaults model to __search and forwards to the upstream /v1/search endpoint', async () => { diff --git a/src/server/routes/proxy/search.ts b/src/server/routes/proxy/search.ts index e5dbaa6..038f6ea 100644 --- a/src/server/routes/proxy/search.ts +++ b/src/server/routes/proxy/search.ts @@ -14,6 +14,7 @@ import { buildUpstreamUrl } from './upstreamUrl.js'; import { detectDownstreamClientContext, type DownstreamClientContext } from './downstreamClientContext.js'; import { insertProxyLog } from '../../services/proxyLogStore.js'; import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js'; +import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js'; import { buildForcedChannelUnavailableMessage, canRetryChannelSelection, @@ -94,7 +95,6 @@ export async function searchProxyRoute(app: FastifyInstance) { } excludeChannelIds.push(selected.channel.id); - const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/search'); const upstreamModel = selected.actualModel || requestedModel; const forwardBody = { ...body, @@ -104,52 +104,27 @@ export async function searchProxyRoute(app: FastifyInstance) { const startTime = Date.now(); try { - const upstream = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${selected.tokenValue}`, - }, - body: JSON.stringify(forwardBody), - }, getProxyUrlFromExtraConfig(selected.account.extraConfig))); - - const text = await upstream.text(); - if (!upstream.ok) { - await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: upstream.status, - errorText: text, - modelName: upstreamModel, - })); - logProxy( - selected, - requestedModel, - 'failed', - upstream.status, - Date.now() - startTime, - text, - retryCount, - downstreamApiKeyId, - clientContext, - downstreamPath, - ); - if (isTokenExpiredError({ status: upstream.status, message: text })) { - await reportTokenExpired({ - accountId: selected.account.id, - username: selected.account.username, - siteName: selected.site.name, - detail: `HTTP ${upstream.status}`, + const { upstream, text } = await runWithSiteApiEndpointPool(selected.site, async (target) => { + const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/search'); + const response = await fetch(targetUrl, withSiteRecordProxyRequestInit(selected.site, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${selected.tokenValue}`, + }, + body: JSON.stringify(forwardBody), + }, getProxyUrlFromExtraConfig(selected.account.extraConfig))); + const responseText = await response.text(); + if (!response.ok) { + throw new SiteApiEndpointRequestError(responseText || 'unknown error', { + status: response.status, }); } - if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) { - retryCount += 1; - continue; - } - await reportProxyAllFailed({ - model: requestedModel, - reason: `upstream returned HTTP ${upstream.status}`, - }); - return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } }); - } + return { + upstream: response, + text: responseText, + }; + }); let data: any = {}; try { data = JSON.parse(text); } catch { data = { data: [] }; } @@ -162,33 +137,46 @@ export async function searchProxyRoute(app: FastifyInstance) { logProxy(selected, requestedModel, 'success', upstream.status, latency, null, retryCount, downstreamApiKeyId, clientContext, downstreamPath); return reply.code(upstream.status).send(data); } catch (error: any) { + const status = error instanceof SiteApiEndpointRequestError ? (error.status || 0) : 0; + const errorText = error?.message || 'network error'; await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: 0, - errorText: error?.message || 'network error', + status, + errorText, modelName: upstreamModel, })); logProxy( selected, requestedModel, 'failed', - 0, + status, Date.now() - startTime, - error?.message || 'network error', + errorText, retryCount, downstreamApiKeyId, clientContext, downstreamPath, ); - if (canRetryChannelSelection(retryCount, forcedChannelId)) { + if (status > 0 && isTokenExpiredError({ status, message: errorText })) { + await reportTokenExpired({ + accountId: selected.account.id, + username: selected.account.username, + siteName: selected.site.name, + detail: `HTTP ${status}`, + }); + } + if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) { retryCount += 1; continue; } await reportProxyAllFailed({ model: requestedModel, - reason: error?.message || 'network failure', + reason: errorText || 'network failure', }); - return reply.code(502).send({ - error: { message: error?.message || 'network failure', type: 'upstream_error' }, + return reply.code(status || 502).send({ + error: { + message: status > 0 ? errorText : `Upstream error: ${errorText}`, + type: 'upstream_error', + }, }); } } diff --git a/src/server/routes/proxy/videos.test.ts b/src/server/routes/proxy/videos.test.ts index 8394da7..4722e7e 100644 --- a/src/server/routes/proxy/videos.test.ts +++ b/src/server/routes/proxy/videos.test.ts @@ -10,10 +10,13 @@ const refreshModelsAndRebuildRoutesMock = vi.fn(); const reportProxyAllFailedMock = vi.fn(); const reportTokenExpiredMock = vi.fn(); const estimateProxyCostMock = vi.fn(async () => 0); +const shouldRetryProxyRequestMock = vi.fn(); const saveProxyVideoTaskMock = vi.fn(); const getProxyVideoTaskByPublicIdMock = vi.fn(); const deleteProxyVideoTaskByPublicIdMock = vi.fn(); const refreshProxyVideoTaskSnapshotMock = vi.fn(); +const resolveProxyVideoTaskSiteMock = vi.fn(); +let siteApiEndpointRows: Array> = []; vi.mock('undici', async () => { const actual = await vi.importActual('undici'); @@ -50,7 +53,8 @@ vi.mock('../../services/modelPricingService.js', () => ({ })); vi.mock('../../services/proxyRetryPolicy.js', () => ({ - shouldRetryProxyRequest: () => false, + shouldRetryProxyRequest: (...args: unknown[]) => shouldRetryProxyRequestMock(...args), + RETRYABLE_TIMEOUT_PATTERNS: [/(request timed out|connection timed out|read timeout|\btimed out\b)/i], })); vi.mock('../../services/proxyVideoTaskStore.js', () => ({ @@ -58,6 +62,35 @@ vi.mock('../../services/proxyVideoTaskStore.js', () => ({ getProxyVideoTaskByPublicId: (...args: unknown[]) => getProxyVideoTaskByPublicIdMock(...args), deleteProxyVideoTaskByPublicId: (...args: unknown[]) => deleteProxyVideoTaskByPublicIdMock(...args), refreshProxyVideoTaskSnapshot: (...args: unknown[]) => refreshProxyVideoTaskSnapshotMock(...args), + resolveProxyVideoTaskSite: (...args: unknown[]) => resolveProxyVideoTaskSiteMock(...args), +})); + +vi.mock('../../db/index.js', () => ({ + db: { + select: () => ({ + from: () => ({ + where: () => ({ + orderBy: () => ({ + all: async () => siteApiEndpointRows, + }), + }), + }), + }), + update: () => ({ + set: () => ({ + where: () => ({ + run: async () => undefined, + }), + }), + }), + }, + schema: { + siteApiEndpoints: { + id: {}, + siteId: {}, + sortOrder: {}, + }, + }, })); describe('/v1/videos routes', () => { @@ -93,10 +126,14 @@ describe('/v1/videos routes', () => { reportProxyAllFailedMock.mockReset(); reportTokenExpiredMock.mockReset(); estimateProxyCostMock.mockClear(); + shouldRetryProxyRequestMock.mockReset(); saveProxyVideoTaskMock.mockReset(); getProxyVideoTaskByPublicIdMock.mockReset(); deleteProxyVideoTaskByPublicIdMock.mockReset(); refreshProxyVideoTaskSnapshotMock.mockReset(); + resolveProxyVideoTaskSiteMock.mockReset(); + siteApiEndpointRows = []; + shouldRetryProxyRequestMock.mockReturnValue(false); selectChannelMock.mockReturnValue({ channel: { id: 11, routeId: 22 }, @@ -110,7 +147,9 @@ describe('/v1/videos routes', () => { }); afterAll(async () => { - await app.close(); + if (app) { + await app.close(); + } }); it('creates an upstream video task and stores a local public id mapping', async () => { @@ -189,7 +228,55 @@ describe('/v1/videos routes', () => { }); }); + it('stores the selected api endpoint base url for split-host video tasks', async () => { + siteApiEndpointRows = [ + { + id: 91, + siteId: 44, + url: 'https://api-videos.example.com', + enabled: true, + sortOrder: 0, + }, + ]; + selectChannelMock.mockReturnValue({ + channel: { id: 11, routeId: 22 }, + site: { id: 44, name: 'demo-site', url: 'https://panel.example.com', platform: 'openai' }, + account: { id: 33, username: 'demo-user' }, + tokenName: 'default', + tokenValue: 'sk-demo', + actualModel: 'sora-2', + }); + saveProxyVideoTaskMock.mockResolvedValue({ + publicId: 'vid_local_split', + upstreamVideoId: 'vid_upstream_split', + }); + fetchMock.mockResolvedValue(new Response(JSON.stringify({ + id: 'vid_upstream_split', + object: 'video', + status: 'queued', + model: 'sora-2', + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const response = await app.inject({ + method: 'POST', + url: '/v1/videos', + payload: { + model: 'sora-2', + prompt: 'split-host create', + }, + }); + + expect(response.statusCode).toBe(200); + expect(saveProxyVideoTaskMock).toHaveBeenCalledWith(expect.objectContaining({ + siteUrl: 'https://api-videos.example.com', + })); + }); + it('resolves local video ids back to the upstream task on GET', async () => { + resolveProxyVideoTaskSiteMock.mockResolvedValue(null); getProxyVideoTaskByPublicIdMock.mockResolvedValue({ publicId: 'vid_local_123', upstreamVideoId: 'vid_upstream_123', @@ -228,6 +315,116 @@ describe('/v1/videos routes', () => { }); }); + it('re-resolves account-backed video tasks through the site api endpoint pool on GET', async () => { + siteApiEndpointRows = [ + { + id: 91, + siteId: 44, + url: 'https://api-videos.example.com', + enabled: true, + sortOrder: 0, + }, + ]; + resolveProxyVideoTaskSiteMock.mockResolvedValue({ + id: 44, + name: 'demo-site', + url: 'https://panel.example.com', + platform: 'openai', + }); + getProxyVideoTaskByPublicIdMock.mockResolvedValue({ + publicId: 'vid_local_456', + upstreamVideoId: 'vid_upstream_456', + siteUrl: 'https://panel.example.com', + tokenValue: 'sk-demo', + accountId: 33, + }); + fetchMock.mockResolvedValue(new Response(JSON.stringify({ + id: 'vid_upstream_456', + object: 'video', + status: 'queued', + model: 'sora-2', + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const response = await app.inject({ + method: 'GET', + url: '/v1/videos/vid_local_456', + }); + + expect(response.statusCode).toBe(200); + const [targetUrl] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(targetUrl).toBe('https://api-videos.example.com/v1/videos/vid_upstream_456'); + }); + + it('uses the persisted api base url when the backing site can no longer be resolved', async () => { + resolveProxyVideoTaskSiteMock.mockResolvedValue(null); + getProxyVideoTaskByPublicIdMock.mockResolvedValue({ + publicId: 'vid_local_fallback', + upstreamVideoId: 'vid_upstream_fallback', + siteUrl: 'https://api-fallback.example.com', + tokenValue: 'sk-demo', + accountId: 33, + }); + fetchMock.mockResolvedValue(new Response(JSON.stringify({ + id: 'vid_upstream_fallback', + object: 'video', + status: 'running', + }), { + status: 200, + headers: { 'content-type': 'application/json' }, + })); + + const response = await app.inject({ + method: 'GET', + url: '/v1/videos/vid_local_fallback', + }); + + expect(response.statusCode).toBe(200); + const [targetUrl] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(targetUrl).toBe('https://api-fallback.example.com/v1/videos/vid_upstream_fallback'); + }); + + it('preserves retryable upstream failures after mapped GET retries exhaust', async () => { + shouldRetryProxyRequestMock.mockReturnValue(true); + siteApiEndpointRows = [ + { + id: 91, + siteId: 44, + url: 'https://api-videos.example.com', + enabled: true, + sortOrder: 0, + }, + ]; + resolveProxyVideoTaskSiteMock.mockResolvedValue({ + id: 44, + name: 'demo-site', + url: 'https://panel.example.com', + platform: 'openai', + }); + getProxyVideoTaskByPublicIdMock.mockResolvedValue({ + publicId: 'vid_local_retry_get', + upstreamVideoId: 'vid_upstream_retry_get', + siteUrl: 'https://api-videos.example.com', + tokenValue: 'sk-demo', + accountId: 33, + }); + fetchMock.mockResolvedValue(new Response('temporary unavailable', { + status: 502, + headers: { 'content-type': 'text/plain' }, + })); + + const response = await app.inject({ + method: 'GET', + url: '/v1/videos/vid_local_retry_get', + }); + + expect(response.statusCode).toBe(502); + expect(response.headers['content-type']).toContain('text/plain'); + expect(response.body).toBe('temporary unavailable'); + }); + it('deletes the upstream task and local mapping on DELETE', async () => { getProxyVideoTaskByPublicIdMock.mockResolvedValue({ publicId: 'vid_local_123', @@ -245,4 +442,43 @@ describe('/v1/videos routes', () => { expect(response.statusCode).toBe(204); expect(deleteProxyVideoTaskByPublicIdMock).toHaveBeenCalledWith('vid_local_123'); }); + + it('preserves retryable upstream failures after mapped DELETE retries exhaust', async () => { + shouldRetryProxyRequestMock.mockReturnValue(true); + siteApiEndpointRows = [ + { + id: 91, + siteId: 44, + url: 'https://api-videos.example.com', + enabled: true, + sortOrder: 0, + }, + ]; + resolveProxyVideoTaskSiteMock.mockResolvedValue({ + id: 44, + name: 'demo-site', + url: 'https://panel.example.com', + platform: 'openai', + }); + getProxyVideoTaskByPublicIdMock.mockResolvedValue({ + publicId: 'vid_local_retry_delete', + upstreamVideoId: 'vid_upstream_retry_delete', + siteUrl: 'https://api-videos.example.com', + tokenValue: 'sk-demo', + accountId: 33, + }); + fetchMock.mockResolvedValue(new Response('temporary unavailable', { + status: 502, + headers: { 'content-type': 'text/plain' }, + })); + + const response = await app.inject({ + method: 'DELETE', + url: '/v1/videos/vid_local_retry_delete', + }); + + expect(response.statusCode).toBe(502); + expect(response.headers['content-type']).toContain('text/plain'); + expect(response.body).toBe('temporary unavailable'); + }); }); diff --git a/src/server/routes/proxy/videos.ts b/src/server/routes/proxy/videos.ts index 46457bb..ddb9f40 100644 --- a/src/server/routes/proxy/videos.ts +++ b/src/server/routes/proxy/videos.ts @@ -14,6 +14,7 @@ import { deleteProxyVideoTaskByPublicId, getProxyVideoTaskByPublicId, refreshProxyVideoTaskSnapshot, + resolveProxyVideoTaskSite, saveProxyVideoTask, } from '../../services/proxyVideoTaskStore.js'; import { getProxyMaxChannelRetries } from '../../services/proxyChannelRetry.js'; @@ -23,6 +24,7 @@ import { getTesterForcedChannelId, selectProxyChannelForAttempt, } from '../../proxy-core/channelSelection.js'; +import { runWithSiteApiEndpointPool, SiteApiEndpointRequestError } from '../../services/siteApiEndpointService.js'; function rewriteVideoResponsePublicId(payload: unknown, publicId: string): unknown { if (!payload || typeof payload !== 'object') return payload; @@ -80,60 +82,48 @@ export async function videosProxyRoute(app: FastifyInstance) { } excludeChannelIds.push(selected.channel.id); - const targetUrl = buildUpstreamUrl(selected.site.url, '/v1/videos'); const upstreamModel = selected.actualModel || requestedModel; const startTime = Date.now(); try { - const accountProxy = getProxyUrlFromExtraConfig(selected.account.extraConfig); - const requestInit = multipartForm - ? withSiteRecordProxyRequestInit(selected.site, { - method: 'POST', - headers: { - Authorization: `Bearer ${selected.tokenValue}`, - }, - body: cloneFormDataWithOverrides(multipartForm, { - model: upstreamModel, - }) as any, - }, accountProxy) - : withSiteRecordProxyRequestInit(selected.site, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${selected.tokenValue}`, - }, - body: JSON.stringify({ - ...(jsonBody || {}), - model: upstreamModel, - }), - }, accountProxy); - - const upstream = await fetch(targetUrl, requestInit); - const text = await upstream.text(); - if (!upstream.ok) { - await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: upstream.status, - errorText: text, - modelName: upstreamModel, - })); - if (isTokenExpiredError({ status: upstream.status, message: text })) { - await reportTokenExpired({ - accountId: selected.account.id, - username: selected.account.username, - siteName: selected.site.name, - detail: `HTTP ${upstream.status}`, + const { upstream, text, baseUrl } = await runWithSiteApiEndpointPool(selected.site, async (target) => { + const targetUrl = buildUpstreamUrl(target.baseUrl, '/v1/videos'); + const accountProxy = getProxyUrlFromExtraConfig(selected.account.extraConfig); + const requestInit = multipartForm + ? withSiteRecordProxyRequestInit(selected.site, { + method: 'POST', + headers: { + Authorization: `Bearer ${selected.tokenValue}`, + }, + body: cloneFormDataWithOverrides(multipartForm, { + model: upstreamModel, + }) as any, + }, accountProxy) + : withSiteRecordProxyRequestInit(selected.site, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${selected.tokenValue}`, + }, + body: JSON.stringify({ + ...(jsonBody || {}), + model: upstreamModel, + }), + }, accountProxy); + const response = await fetch(targetUrl, requestInit); + const responseText = await response.text(); + if (!response.ok) { + throw new SiteApiEndpointRequestError(responseText || 'unknown error', { + status: response.status, + rawErrText: responseText || null, }); } - if (shouldRetryProxyRequest(upstream.status, text) && canRetryChannelSelection(retryCount, forcedChannelId)) { - retryCount += 1; - continue; - } - await reportProxyAllFailed({ - model: requestedModel, - reason: `upstream returned HTTP ${upstream.status}`, - }); - return reply.code(upstream.status).send({ error: { message: text, type: 'upstream_error' } }); - } + return { + baseUrl: target.baseUrl, + upstream: response, + text: responseText, + }; + }); let data: any = {}; try { data = JSON.parse(text); } catch { data = {}; } @@ -146,7 +136,7 @@ export async function videosProxyRoute(app: FastifyInstance) { const mapping = await saveProxyVideoTask({ upstreamVideoId, - siteUrl: selected.site.url, + siteUrl: baseUrl, tokenValue: selected.tokenValue, requestedModel, actualModel: upstreamModel, @@ -174,21 +164,34 @@ export async function videosProxyRoute(app: FastifyInstance) { recordDownstreamCostUsage(request, estimatedCost); return reply.code(upstream.status).send(rewriteVideoResponsePublicId(data, mapping.publicId)); } catch (error: any) { + const status = error instanceof SiteApiEndpointRequestError ? (error.status || 0) : 0; + const errorText = error?.message || 'network failure'; await recordTokenRouterEventBestEffort('record channel failure', () => tokenRouter.recordFailure(selected.channel.id, { - status: 0, - errorText: error?.message || 'network failure', + status, + errorText, modelName: upstreamModel, })); - if (canRetryChannelSelection(retryCount, forcedChannelId)) { + if (status > 0 && isTokenExpiredError({ status, message: errorText })) { + await reportTokenExpired({ + accountId: selected.account.id, + username: selected.account.username, + siteName: selected.site.name, + detail: `HTTP ${status}`, + }); + } + if ((status > 0 ? shouldRetryProxyRequest(status, errorText) : true) && canRetryChannelSelection(retryCount, forcedChannelId)) { retryCount += 1; continue; } await reportProxyAllFailed({ model: requestedModel, - reason: error?.message || 'network failure', + reason: errorText || 'network failure', }); - return reply.code(502).send({ - error: { message: error?.message || 'network failure', type: 'upstream_error' }, + return reply.code(status || 502).send({ + error: { + message: status > 0 ? errorText : `Upstream error: ${errorText}`, + type: 'upstream_error', + }, }); } } @@ -202,13 +205,15 @@ export async function videosProxyRoute(app: FastifyInstance) { }); } - const targetUrl = buildUpstreamUrl(mapping.siteUrl, `/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`); - const upstream = await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, { - method: 'GET', - headers: { - Authorization: `Bearer ${mapping.tokenValue}`, - }, - })); + let upstream: Awaited>; + try { + ({ upstream } = await requestMappedVideoTaskUpstream(mapping, 'GET')); + } catch (error) { + if (isSiteApiEndpointFailure(error)) { + return sendVideoTaskEndpointFailure(reply, error); + } + throw error; + } const text = await upstream.text(); try { const data = JSON.parse(text); @@ -233,13 +238,15 @@ export async function videosProxyRoute(app: FastifyInstance) { }); } - const targetUrl = buildUpstreamUrl(mapping.siteUrl, `/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`); - const upstream = await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, { - method: 'DELETE', - headers: { - Authorization: `Bearer ${mapping.tokenValue}`, - }, - })); + let upstream: Awaited>; + try { + ({ upstream } = await requestMappedVideoTaskUpstream(mapping, 'DELETE')); + } catch (error) { + if (isSiteApiEndpointFailure(error)) { + return sendVideoTaskEndpointFailure(reply, error); + } + throw error; + } if (upstream.ok) { await deleteProxyVideoTaskByPublicId(mapping.publicId); return reply.code(upstream.status).send(); @@ -252,6 +259,63 @@ export async function videosProxyRoute(app: FastifyInstance) { }); } +async function requestMappedVideoTaskUpstream( + mapping: NonNullable>>, + method: 'GET' | 'DELETE', +): Promise<{ upstream: Awaited> }> { + const buildRequest = async (baseUrl: string) => { + const targetUrl = buildUpstreamUrl(baseUrl, `/v1/videos/${encodeURIComponent(mapping.upstreamVideoId)}`); + const upstream = await fetch(targetUrl, await withSiteProxyRequestInit(targetUrl, { + method, + headers: { + Authorization: `Bearer ${mapping.tokenValue}`, + }, + })); + if (!upstream.ok) { + const errorText = await upstream.clone().text().catch(() => ''); + if (shouldRetryProxyRequest(upstream.status, errorText || `HTTP ${upstream.status}`)) { + throw new SiteApiEndpointRequestError(errorText || `HTTP ${upstream.status}`, { + status: upstream.status, + rawErrText: errorText || null, + }); + } + } + return { upstream }; + }; + + const site = await resolveProxyVideoTaskSite(mapping.accountId); + if (site) { + return runWithSiteApiEndpointPool(site, (target) => buildRequest(target.baseUrl)); + } + + return buildRequest(mapping.siteUrl); +} + +function isSiteApiEndpointFailure(error: unknown): error is SiteApiEndpointRequestError { + return error instanceof SiteApiEndpointRequestError + || (typeof error === 'object' && error !== null && (error as { name?: unknown }).name === 'SiteApiEndpointRequestError'); +} + +function sendVideoTaskEndpointFailure( + reply: FastifyReply, + error: { status?: number | null; rawErrText?: string | null; message?: string | null }, +) { + const status = typeof error.status === 'number' && error.status > 0 ? error.status : 502; + const rawText = (typeof error.rawErrText === 'string' && error.rawErrText.trim()) + ? error.rawErrText + : (typeof error.message === 'string' ? error.message.trim() : ''); + if (!rawText) { + return reply.code(status).send({ + error: { message: 'Upstream request failed', type: 'upstream_error' }, + }); + } + try { + return reply.code(status).send(JSON.parse(rawText)); + } catch { + return reply.code(status).type('text/plain').send(rawText); + } +} + async function recordTokenRouterEventBestEffort( label: string, operation: () => Promise, diff --git a/src/server/services/backupService.test.ts b/src/server/services/backupService.test.ts index 71e068c..1a1ad92 100644 --- a/src/server/services/backupService.test.ts +++ b/src/server/services/backupService.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vites import { mkdtempSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { eq } from 'drizzle-orm'; +import { asc, eq } from 'drizzle-orm'; type DbModule = typeof import('../db/index.js'); type BackupServiceModule = typeof import('./backupService.js'); @@ -155,6 +155,19 @@ describe('backupService', () => { createdAt: now, }).run(); + await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api-roundtrip.example.com', + enabled: true, + sortOrder: 0, + cooldownUntil: null, + lastSelectedAt: now, + lastFailedAt: null, + lastFailureReason: null, + createdAt: now, + updatedAt: now, + }).run(); + await db.insert(schema.modelAvailability).values([ { accountId: account.id, @@ -199,6 +212,15 @@ describe('backupService', () => { expect(exported.accounts.siteDisabledModels).toEqual([ { siteId: site.id, modelName: 'gpt-hidden' }, ]); + expect(exported.accounts.siteApiEndpoints).toEqual([ + expect.objectContaining({ + siteId: site.id, + url: 'https://api-roundtrip.example.com', + enabled: true, + sortOrder: 0, + cooldownUntil: null, + }), + ]); expect(exported.accounts.manualModels).toEqual([ { accountId: account.id, modelName: 'gpt-manual' }, ]); @@ -579,6 +601,33 @@ describe('backupService', () => { createdAt: exportedAt, }).run(); + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-a.example.com', + enabled: true, + sortOrder: 0, + cooldownUntil: null, + lastSelectedAt: exportedAt, + lastFailedAt: null, + lastFailureReason: null, + createdAt: exportedAt, + updatedAt: exportedAt, + }, + { + siteId: site.id, + url: 'https://api-b.example.com', + enabled: false, + sortOrder: 1, + cooldownUntil: exportedAt, + lastSelectedAt: null, + lastFailedAt: exportedAt, + lastFailureReason: 'HTTP 429', + createdAt: exportedAt, + updatedAt: exportedAt, + }, + ]).run(); + await db.insert(schema.modelAvailability).values([ { accountId: account.id, @@ -678,6 +727,22 @@ describe('backupService', () => { updatedAt: localRuntimeAt, }).where(eq(schema.sites.id, site.id)).run(); + await db.delete(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.siteId, site.id)) + .run(); + await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api-local-only.example.com', + enabled: true, + sortOrder: 9, + cooldownUntil: null, + lastSelectedAt: localRuntimeAt, + lastFailedAt: null, + lastFailureReason: null, + createdAt: localRuntimeAt, + updatedAt: localRuntimeAt, + }).run(); + await db.update(schema.tokenRoutes).set({ displayName: 'local-route-name', updatedAt: localRuntimeAt, @@ -821,6 +886,11 @@ describe('backupService', () => { const restoredAccount = await db.select().from(schema.accounts).where(eq(schema.accounts.id, account.id)).get(); const restoredRoute = await db.select().from(schema.tokenRoutes).where(eq(schema.tokenRoutes.id, route.id)).get(); const restoredChannel = await db.select().from(schema.routeChannels).where(eq(schema.routeChannels.id, insertedChannel!.id)).get(); + const restoredSiteApiEndpoints = await db.select() + .from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.siteId, site.id)) + .orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id)) + .all(); const restoredDisabledModels = await db.select().from(schema.siteDisabledModels).all(); const restoredAvailability = await db.select().from(schema.modelAvailability).all(); const restoredTokenAvailability = await db.select().from(schema.tokenModelAvailability).all(); @@ -848,6 +918,25 @@ describe('backupService', () => { expect(restoredChannel?.cooldownLevel).toBe(2); expect(restoredChannel?.cooldownUntil).toBe(localRuntimeAt); + expect(restoredSiteApiEndpoints).toEqual([ + expect.objectContaining({ + siteId: site.id, + url: 'https://api-a.example.com', + enabled: true, + sortOrder: 0, + cooldownUntil: null, + lastFailureReason: null, + }), + expect.objectContaining({ + siteId: site.id, + url: 'https://api-b.example.com', + enabled: false, + sortOrder: 1, + cooldownUntil: exportedAt, + lastFailureReason: 'HTTP 429', + }), + ]); + expect(restoredDisabledModels).toEqual([ expect.objectContaining({ siteId: site.id, modelName: 'gpt-backup-disabled' }), ]); diff --git a/src/server/services/backupService.ts b/src/server/services/backupService.ts index 2b2939a..4598b6d 100644 --- a/src/server/services/backupService.ts +++ b/src/server/services/backupService.ts @@ -38,6 +38,7 @@ export interface BackupWebdavState { } type SiteRow = typeof schema.sites.$inferSelect; +type SiteApiEndpointRow = typeof schema.siteApiEndpoints.$inferSelect; type AccountRow = typeof schema.accounts.$inferSelect; type AccountTokenRow = typeof schema.accountTokens.$inferSelect; type TokenRouteRow = typeof schema.tokenRoutes.$inferSelect; @@ -100,6 +101,7 @@ type BackupDownstreamApiKeyRow = Pick { const [ sites, + siteApiEndpoints, accounts, accountTokens, tokenRoutes, @@ -1292,6 +1295,13 @@ async function exportAccountsSection(): Promise { downstreamApiKeys, ] = await Promise.all([ db.select().from(schema.sites).orderBy(asc(schema.sites.id)).all(), + db.select().from(schema.siteApiEndpoints) + .orderBy( + asc(schema.siteApiEndpoints.siteId), + asc(schema.siteApiEndpoints.sortOrder), + asc(schema.siteApiEndpoints.id), + ) + .all(), db.select().from(schema.accounts).orderBy(asc(schema.accounts.id)).all(), db.select().from(schema.accountTokens).orderBy(asc(schema.accountTokens.id)).all(), db.select().from(schema.tokenRoutes).orderBy(asc(schema.tokenRoutes.id)).all(), @@ -1309,6 +1319,7 @@ async function exportAccountsSection(): Promise { return { sites, + siteApiEndpoints, accounts: accounts.map(({ balanceUsed: _balanceUsed, lastCheckinAt: _lastCheckinAt, lastBalanceRefresh: _lastBalanceRefresh, ...row }) => row), accountTokens, tokenRoutes, @@ -1389,6 +1400,9 @@ function coerceAccountsSection(input: unknown): AccountsBackupSection | null { if (!isRecord(input)) return null; const sites = Array.isArray(input.sites) ? input.sites as SiteRow[] : null; + const siteApiEndpoints = Array.isArray(input.siteApiEndpoints) + ? input.siteApiEndpoints as SiteApiEndpointRow[] + : undefined; const accounts = Array.isArray(input.accounts) ? input.accounts as BackupAccountRow[] : null; const accountTokens = Array.isArray(input.accountTokens) ? input.accountTokens as AccountTokenRow[] : null; const tokenRoutes = Array.isArray(input.tokenRoutes) ? input.tokenRoutes as TokenRouteRow[] : null; @@ -1410,6 +1424,7 @@ function coerceAccountsSection(input: unknown): AccountsBackupSection | null { return { sites, + siteApiEndpoints, accounts, accountTokens, tokenRoutes, @@ -1534,6 +1549,22 @@ async function importAccountsSection(section: AccountsBackupSection): Promise { expect(siteStatement?.values[customHeadersIndex]).toBe('{"x-site-scope":"internal"}'); }); + it('includes site api endpoints when building migration statements', () => { + const statements = __databaseMigrationServiceTestUtils.buildStatements({ + version: 'test', + timestamp: Date.now(), + accounts: { + sites: [{ + id: 1, + name: 'demo', + url: 'https://example.com', + platform: 'new-api', + status: 'active', + }], + siteApiEndpoints: [{ + id: 9, + siteId: 1, + url: 'https://api.example.com', + enabled: true, + sortOrder: 2, + cooldownUntil: '2026-03-31T12:05:00.000Z', + lastSelectedAt: '2026-03-31T12:00:00.000Z', + lastFailedAt: '2026-03-31T11:59:00.000Z', + lastFailureReason: 'HTTP 502', + createdAt: '2026-03-30T00:00:00.000Z', + updatedAt: '2026-03-31T12:05:00.000Z', + }], + siteAnnouncements: [], + siteDisabledModels: [], + accounts: [], + accountTokens: [], + checkinLogs: [], + modelAvailability: [], + tokenModelAvailability: [], + tokenRoutes: [], + routeChannels: [], + routeGroupSources: [], + proxyLogs: [], + proxyVideoTasks: [], + proxyFiles: [], + downstreamApiKeys: [], + events: [], + }, + preferences: { + settings: [], + }, + } as any); + + const endpointStatement = statements.find((statement) => statement.table === 'site_api_endpoints'); + expect(endpointStatement?.columns).toEqual([ + 'id', + 'site_id', + 'url', + 'enabled', + 'sort_order', + 'cooldown_until', + 'last_selected_at', + 'last_failed_at', + 'last_failure_reason', + 'created_at', + 'updated_at', + ]); + expect(endpointStatement?.values).toEqual([ + 9, + 1, + 'https://api.example.com', + true, + 2, + '2026-03-31T12:05:00.000Z', + '2026-03-31T12:00:00.000Z', + '2026-03-31T11:59:00.000Z', + 'HTTP 502', + '2026-03-30T00:00:00.000Z', + '2026-03-31T12:05:00.000Z', + ]); + }); + it('serializes parsed JSON-column values when building migration statements', () => { const statements = __databaseMigrationServiceTestUtils.buildStatements({ version: 'test', diff --git a/src/server/services/databaseMigrationService.ts b/src/server/services/databaseMigrationService.ts index f3e5f2e..0df7bc7 100644 --- a/src/server/services/databaseMigrationService.ts +++ b/src/server/services/databaseMigrationService.ts @@ -29,6 +29,7 @@ type BackupSnapshot = { timestamp: number; accounts: { sites: Array>; + siteApiEndpoints: Array>; siteAnnouncements: Array>; siteDisabledModels: Array>; accounts: Array>; @@ -58,6 +59,7 @@ export interface DatabaseMigrationSummary { timestamp: number; rows: { sites: number; + siteApiEndpoints: number; siteAnnouncements: number; siteDisabledModels: number; accounts: number; @@ -251,6 +253,7 @@ async function toBackupSnapshot(): Promise { timestamp: Date.now(), accounts: { sites: await db.select().from(schema.sites).all() as Array>, + siteApiEndpoints: await db.select().from(schema.siteApiEndpoints).all() as Array>, siteAnnouncements: await db.select().from(schema.siteAnnouncements).all() as Array>, siteDisabledModels: await db.select().from(schema.siteDisabledModels).all() as Array>, accounts: await db.select().from(schema.accounts).all() as Array>, @@ -299,6 +302,7 @@ async function clearTargetData(client: SqlClient): Promise { 'accounts', 'site_announcements', 'site_disabled_models', + 'site_api_endpoints', 'token_routes', 'sites', 'downstream_api_keys', @@ -340,6 +344,38 @@ function buildStatements( }); } + for (const row of snapshot.accounts.siteApiEndpoints || []) { + statements.push({ + table: 'site_api_endpoints', + columns: [ + 'id', + 'site_id', + 'url', + 'enabled', + 'sort_order', + 'cooldown_until', + 'last_selected_at', + 'last_failed_at', + 'last_failure_reason', + 'created_at', + 'updated_at', + ], + values: [ + asNumber(row.id, 0), + asNumber(row.siteId, 0), + asNullableString(row.url), + asBoolean(row.enabled, true), + asNumber(row.sortOrder, 0), + asNullableString(row.cooldownUntil), + asNullableString(row.lastSelectedAt), + asNullableString(row.lastFailedAt), + asNullableString(row.lastFailureReason), + asNullableString(row.createdAt), + asNullableString(row.updatedAt), + ], + }); + } + for (const row of snapshot.accounts.siteDisabledModels) { statements.push({ table: 'site_disabled_models', @@ -708,6 +744,7 @@ async function syncPostgresSequences(client: SqlClient): Promise { if (client.dialect !== 'postgres') return; const tables = [ 'sites', + 'site_api_endpoints', 'site_announcements', 'site_disabled_models', 'accounts', @@ -776,6 +813,7 @@ export async function migrateCurrentDatabase(input: DatabaseMigrationInput): Pro timestamp: snapshot.timestamp, rows: { sites: snapshot.accounts.sites.length, + siteApiEndpoints: snapshot.accounts.siteApiEndpoints.length, siteAnnouncements: snapshot.accounts.siteAnnouncements.length, siteDisabledModels: snapshot.accounts.siteDisabledModels.length, accounts: snapshot.accounts.accounts.length, diff --git a/src/server/services/modelService.discovery.test.ts b/src/server/services/modelService.discovery.test.ts index ced9a69..a1045e9 100644 --- a/src/server/services/modelService.discovery.test.ts +++ b/src/server/services/modelService.discovery.test.ts @@ -75,6 +75,7 @@ describe('refreshModelsForAccount credential discovery', () => { await db.delete(schema.modelAvailability).run(); await db.delete(schema.accountTokens).run(); await db.delete(schema.accounts).run(); + await db.delete(schema.siteApiEndpoints).run(); await db.delete(schema.settings).run(); await db.delete(schema.sites).run(); const { config } = await import('../config.js'); @@ -134,6 +135,49 @@ describe('refreshModelsForAccount credential discovery', () => { expect(tokenRows).toHaveLength(0); }); + it('uses the configured ai endpoint for direct model discovery credentials', async () => { + getApiTokenMock.mockResolvedValue(null); + getModelsMock.mockImplementation(async (baseUrl: string, token: string) => ( + baseUrl === 'https://api.nih.cc' && token === 'session-token' + ? ['gpt-4.1'] + : [] + )); + + const site = await db.insert(schema.sites).values({ + name: 'nihao-panel', + url: 'https://nih.cc', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api.nih.cc', + enabled: true, + sortOrder: 0, + }).run(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'nihao-user', + accessToken: 'session-token', + apiToken: null, + status: 'active', + extraConfig: JSON.stringify({ credentialMode: 'session' }), + }).returning().get(); + + const result = await refreshModelsForAccount(account.id); + + expect(result).toMatchObject({ + accountId: account.id, + refreshed: true, + status: 'success', + modelCount: 1, + modelsPreview: ['gpt-4.1'], + }); + expect(getModelsMock).toHaveBeenCalledWith('https://api.nih.cc', 'session-token', undefined); + }); + it('deduplicates discovered model names before writing availability rows', async () => { getApiTokenMock.mockResolvedValue(null); getModelsMock.mockResolvedValue(['? ', '?', 'GPT-4.1', 'gpt-4.1']); @@ -694,6 +738,145 @@ describe('refreshModelsForAccount credential discovery', () => { ]); }); + it('uses the configured ai endpoint for codex cloud model discovery', async () => { + getApiTokenMock.mockResolvedValue(null); + getModelsMock.mockRejectedValue(new Error('codex plan discovery should not call adapter.getModels')); + undiciFetchMock.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + models: [ + { id: 'gpt-5.3-codex' }, + ], + }), + text: async () => JSON.stringify({ ok: true }), + }); + + const site = await db.insert(schema.sites).values({ + name: 'codex-panel-site', + url: 'https://chatgpt.com/panel-codex', + platform: 'codex', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://chatgpt.com/backend-api/codex', + enabled: true, + sortOrder: 0, + }).run(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'codex-user@example.com', + accessToken: 'oauth-access-token', + apiToken: null, + status: 'active', + extraConfig: JSON.stringify({ + credentialMode: 'session', + oauth: { + provider: 'codex', + accountId: 'chatgpt-account-456', + email: 'codex-user@example.com', + planType: 'plus', + }, + }), + }).returning().get(); + + const result = await refreshModelsForAccount(account.id); + + expect(result).toMatchObject({ + accountId: account.id, + refreshed: true, + status: 'success', + modelCount: 1, + modelsPreview: ['gpt-5.3-codex'], + }); + expect(String(undiciFetchMock.mock.calls[0]?.[0] || '')).toBe('https://chatgpt.com/backend-api/codex/models?client_version=1.0.0'); + }); + + it('rotates codex cloud discovery across configured ai endpoints after a retryable failure', async () => { + getApiTokenMock.mockResolvedValue(null); + getModelsMock.mockRejectedValue(new Error('codex plan discovery should not call adapter.getModels')); + undiciFetchMock + .mockResolvedValueOnce({ + ok: false, + status: 502, + json: async () => ({ error: 'bad gateway' }), + text: async () => 'bad gateway', + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + models: [ + { id: 'gpt-5.3-codex' }, + ], + }), + text: async () => JSON.stringify({ ok: true }), + }); + + const site = await db.insert(schema.sites).values({ + name: 'codex-pool-site', + url: 'https://chatgpt.com/panel-codex', + platform: 'codex', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://chatgpt.com/backend-api/codex-a', + enabled: true, + sortOrder: 0, + }, + { + siteId: site.id, + url: 'https://chatgpt.com/backend-api/codex-b', + enabled: true, + sortOrder: 1, + }, + ]).run(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'codex-pool-user@example.com', + accessToken: 'oauth-access-token', + apiToken: null, + status: 'active', + extraConfig: JSON.stringify({ + credentialMode: 'session', + oauth: { + provider: 'codex', + accountId: 'chatgpt-account-789', + email: 'codex-pool-user@example.com', + planType: 'plus', + }, + }), + }).returning().get(); + + const result = await refreshModelsForAccount(account.id); + + expect(result).toMatchObject({ + accountId: account.id, + refreshed: true, + status: 'success', + modelCount: 1, + modelsPreview: ['gpt-5.3-codex'], + }); + expect(String(undiciFetchMock.mock.calls[0]?.[0] || '')).toBe('https://chatgpt.com/backend-api/codex-a/models?client_version=1.0.0'); + expect(String(undiciFetchMock.mock.calls[1]?.[0] || '')).toBe('https://chatgpt.com/backend-api/codex-b/models?client_version=1.0.0'); + + const endpoints = await db.select().from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.siteId, site.id)) + .all(); + const firstEndpoint = endpoints.find((item) => item.url === 'https://chatgpt.com/backend-api/codex-a'); + const secondEndpoint = endpoints.find((item) => item.url === 'https://chatgpt.com/backend-api/codex-b'); + expect(firstEndpoint?.cooldownUntil).toBeTruthy(); + expect(firstEndpoint?.lastFailureReason).toContain('HTTP 502'); + expect(secondEndpoint?.lastSelectedAt).toBeTruthy(); + }); + it('discovers Claude OAuth models from the upstream /v1/models response', async () => { getApiTokenMock.mockResolvedValue(null); getModelsMock.mockRejectedValue(new Error('claude oauth discovery should not call adapter.getModels')); @@ -1276,6 +1459,86 @@ describe('refreshModelsForAccount credential discovery', () => { }); }); + it('rotates antigravity discovery across configured ai endpoints before using built-in fallback hosts', async () => { + getApiTokenMock.mockResolvedValue(null); + getModelsMock.mockRejectedValue(new Error('antigravity oauth discovery should not call adapter.getModels')); + undiciFetchMock + .mockResolvedValueOnce({ + ok: false, + status: 503, + json: async () => ({ error: 'endpoint-a unavailable' }), + text: async () => 'endpoint-a unavailable', + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + json: async () => ({ + models: { + 'gemini-3-pro-preview': { displayName: 'Gemini 3 Pro Preview' }, + }, + }), + text: async () => JSON.stringify({ ok: true }), + }); + + const site = await db.insert(schema.sites).values({ + name: 'antigravity-endpoint-pool-site', + url: 'https://cloudcode-panel.example.com', + platform: 'antigravity', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-antigravity-a.example.com', + enabled: true, + sortOrder: 0, + }, + { + siteId: site.id, + url: 'https://api-antigravity-b.example.com', + enabled: true, + sortOrder: 1, + }, + ]).run(); + + const account = await db.insert(schema.accounts).values({ + siteId: site.id, + username: 'antigravity-endpoint-pool@example.com', + accessToken: 'antigravity-access-token', + apiToken: null, + status: 'active', + extraConfig: JSON.stringify({ + credentialMode: 'session', + oauth: { + provider: 'antigravity', + email: 'antigravity-endpoint-pool@example.com', + projectId: 'project-demo', + }, + }), + }).returning().get(); + + const result = await refreshModelsForAccount(account.id); + + expect(result).toMatchObject({ + accountId: account.id, + refreshed: true, + status: 'success', + modelCount: 1, + modelsPreview: ['gemini-3-pro-preview'], + }); + expect(undiciFetchMock).toHaveBeenCalledTimes(2); + expect(String(undiciFetchMock.mock.calls[0]?.[0] || '')).toBe('https://api-antigravity-a.example.com/v1internal:fetchAvailableModels'); + expect(String(undiciFetchMock.mock.calls[1]?.[0] || '')).toBe('https://api-antigravity-b.example.com/v1internal:fetchAvailableModels'); + + const endpoints = await db.select().from(schema.siteApiEndpoints).all(); + const firstEndpoint = endpoints.find((item) => item.url === 'https://api-antigravity-a.example.com'); + const secondEndpoint = endpoints.find((item) => item.url === 'https://api-antigravity-b.example.com'); + expect(firstEndpoint?.cooldownUntil).toBeTruthy(); + expect(firstEndpoint?.lastFailureReason).toContain('HTTP 503'); + expect(secondEndpoint?.lastSelectedAt).toBeTruthy(); + }); + it('preserves manual models after successful model refresh', async () => { getApiTokenMock.mockResolvedValue(null); getModelsMock.mockResolvedValue(['gpt-4.1', 'claude-opus-4-6']); diff --git a/src/server/services/modelService.ts b/src/server/services/modelService.ts index 5fdf28d..844f0fa 100644 --- a/src/server/services/modelService.ts +++ b/src/server/services/modelService.ts @@ -26,6 +26,7 @@ import { withAccountProxyOverride } from './siteProxy.js'; import { isCodexPlatform } from './oauth/codexAccount.js'; import { buildStoredOauthStateFromAccount, getOauthInfoFromAccount } from './oauth/oauthAccount.js'; import { refreshOauthAccessTokenSingleflight } from './oauth/refreshSingleflight.js'; +import { requireSiteApiBaseUrl } from './siteApiEndpointService.js'; import { discoverAntigravityModelsFromCloud, discoverClaudeModelsFromCloud, @@ -775,6 +776,30 @@ export async function refreshModelsForAccount( } } + let aiBaseUrl: string; + try { + aiBaseUrl = await requireSiteApiBaseUrl(site); + } catch (err) { + const rawMessage = (err as { message?: string })?.message || '模型获取失败'; + const errorCode = classifyModelDiscoveryError(rawMessage); + const errorMessage = rawMessage; + await setAccountRuntimeHealth(account.id, { + state: 'unhealthy', + reason: errorMessage, + source: 'model-discovery', + checkedAt: new Date().toISOString(), + }); + await restorePreviousAvailability(); + return buildFailedRefreshResult({ + accountId, + errorCode, + errorMessage, + tokenScanned: 0, + discoveredByCredential: false, + discoveredApiToken: !!discoveredApiToken, + }); + } + const accountModels = new Map(); // lowercase key → original name (first-wins) const modelLatency = new Map(); let scannedTokenCount = 0; @@ -813,7 +838,7 @@ export async function refreshModelsForAccount( models = normalizeModels( await withTimeout( () => withAccountProxyOverride(accountProxyUrl, - () => adapter.getModels(site.url, credential, platformUserId)), + () => adapter.getModels(aiBaseUrl, credential, platformUserId)), MODEL_DISCOVERY_TIMEOUT_MS, `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`, ), @@ -841,7 +866,7 @@ export async function refreshModelsForAccount( models = normalizeModels( await withTimeout( () => withAccountProxyOverride(accountProxyUrl, - () => adapter.getModels(site.url, token.token, platformUserId)), + () => adapter.getModels(aiBaseUrl, token.token, platformUserId)), MODEL_DISCOVERY_TIMEOUT_MS, `model discovery timeout (${Math.round(MODEL_DISCOVERY_TIMEOUT_MS / 1000)}s)`, ), diff --git a/src/server/services/platformDiscoveryRegistry.ts b/src/server/services/platformDiscoveryRegistry.ts index 7ac158b..d573c0b 100644 --- a/src/server/services/platformDiscoveryRegistry.ts +++ b/src/server/services/platformDiscoveryRegistry.ts @@ -1,6 +1,7 @@ import { fetch } from 'undici'; import { schema } from '../db/index.js'; import { withSiteRecordProxyRequestInit } from './siteProxy.js'; +import { runWithSiteApiEndpointPool } from './siteApiEndpointService.js'; import { getOauthInfoFromAccount } from './oauth/oauthAccount.js'; import { CLAUDE_DEFAULT_ANTHROPIC_VERSION } from './oauth/claudeProvider.js'; import { @@ -136,15 +137,18 @@ export async function discoverCodexModelsFromCloud(input: { headers['Chatgpt-Account-Id'] = oauth.accountId; } - const response = await fetch( - buildCodexModelsEndpoint(input.site.url), - withSiteRecordProxyRequestInit(input.site, { method: 'GET', headers }), - ); - if (!response.ok) { - const text = await response.text().catch(() => ''); - throw new Error(`HTTP ${response.status}: ${text || 'codex model discovery failed'}`); - } - return normalizeDiscoveredModels(extractCodexModelIds(await response.json())); + const payload = await runWithSiteApiEndpointPool(input.site, async (target) => { + const response = await fetch( + buildCodexModelsEndpoint(target.baseUrl), + withSiteRecordProxyRequestInit(input.site, { method: 'GET', headers }), + ); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`HTTP ${response.status}: ${text || 'codex model discovery failed'}`); + } + return response.json(); + }); + return normalizeDiscoveredModels(extractCodexModelIds(payload)); } export async function discoverClaudeModelsFromCloud(input: { @@ -155,22 +159,25 @@ export async function discoverClaudeModelsFromCloud(input: { if (!accessToken) { throw new Error('claude oauth access token missing'); } - const response = await fetch( - `${input.site.url.replace(/\/+$/, '')}/v1/models`, - withSiteRecordProxyRequestInit(input.site, { - method: 'GET', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'anthropic-version': CLAUDE_DEFAULT_ANTHROPIC_VERSION, - }, - }), - ); - if (!response.ok) { - const text = await response.text().catch(() => ''); - throw new Error(`HTTP ${response.status}: ${text || 'claude oauth model discovery failed'}`); - } - return normalizeDiscoveredModels(extractClaudeModelIds(await response.json())); + const payload = await runWithSiteApiEndpointPool(input.site, async (target) => { + const response = await fetch( + `${target.baseUrl.replace(/\/+$/, '')}/v1/models`, + withSiteRecordProxyRequestInit(input.site, { + method: 'GET', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + 'anthropic-version': CLAUDE_DEFAULT_ANTHROPIC_VERSION, + }, + }), + ); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`HTTP ${response.status}: ${text || 'claude oauth model discovery failed'}`); + } + return response.json(); + }); + return normalizeDiscoveredModels(extractClaudeModelIds(payload)); } export async function validateGeminiCliOauthConnection(input: { @@ -220,38 +227,45 @@ export async function discoverAntigravityModelsFromCloud(input: { const oauth = getOauthInfoFromAccount(input.account); const projectId = (oauth?.projectId || '').trim(); const requestBody = projectId ? { project: projectId } : {}; - let lastError = ''; + return runWithSiteApiEndpointPool(input.site, async (target) => { + let lastError = ''; + const selectedBaseUrl = normalizeBaseUrl(target.baseUrl || ANTIGRAVITY_UPSTREAM_BASE_URL) || ANTIGRAVITY_UPSTREAM_BASE_URL; + const discoveryBaseUrls = target.endpointId + ? [selectedBaseUrl] + : buildAntigravityDiscoveryBaseUrls(selectedBaseUrl); - for (const baseUrl of buildAntigravityDiscoveryBaseUrls(input.site.url || ANTIGRAVITY_UPSTREAM_BASE_URL)) { - try { - const response = await fetch( - `${baseUrl}/v1internal:fetchAvailableModels`, - withSiteRecordProxyRequestInit(input.site, { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - 'User-Agent': ANTIGRAVITY_MODELS_USER_AGENT, - }, - body: JSON.stringify(requestBody), - }), - ); - if (!response.ok) { - lastError = await response.text().catch(() => '') || `HTTP ${response.status}`; - continue; - } + for (const discoveryBaseUrl of discoveryBaseUrls) { + try { + const response = await fetch( + `${discoveryBaseUrl}/v1internal:fetchAvailableModels`, + withSiteRecordProxyRequestInit(input.site, { + method: 'POST', + headers: { + Authorization: `Bearer ${accessToken}`, + Accept: 'application/json', + 'Content-Type': 'application/json', + 'User-Agent': ANTIGRAVITY_MODELS_USER_AGENT, + }, + body: JSON.stringify(requestBody), + }), + ); + if (!response.ok) { + const text = await response.text().catch(() => ''); + lastError = `HTTP ${response.status}: ${text || '未获取到可用模型'}`; + continue; + } - const payload = await response.json(); - const models = normalizeDiscoveredModels(extractAntigravityModelIds(payload)); - if (models.length > 0) { - return models; + const payload = await response.json(); + const models = normalizeDiscoveredModels(extractAntigravityModelIds(payload)); + if (models.length > 0) { + return models; + } + lastError = '未获取到可用模型'; + } catch (error) { + lastError = error instanceof Error ? `${discoveryBaseUrl}: ${error.message}` : String(error); } - lastError = '未获取到可用模型'; - } catch (error) { - lastError = error instanceof Error ? `${baseUrl}: ${error.message}` : String(error); } - } - throw new Error(lastError || '未获取到可用模型'); + throw new Error(lastError || '未获取到可用模型'); + }); } diff --git a/src/server/services/proxyVideoTaskStore.ts b/src/server/services/proxyVideoTaskStore.ts index f1739c6..6bd0dca 100644 --- a/src/server/services/proxyVideoTaskStore.ts +++ b/src/server/services/proxyVideoTaskStore.ts @@ -89,6 +89,18 @@ export async function getProxyVideoTaskByPublicId(publicId: string): Promise { + if (!Number.isFinite(accountId) || !accountId || accountId <= 0) { + return null; + } + const row = await db.select() + .from(schema.accounts) + .innerJoin(schema.sites, eq(schema.accounts.siteId, schema.sites.id)) + .where(eq(schema.accounts.id, accountId)) + .get(); + return row?.sites ?? null; +} + export async function deleteProxyVideoTaskByPublicId(publicId: string): Promise { await db.delete(schema.proxyVideoTasks) .where(eq(schema.proxyVideoTasks.publicId, publicId)) diff --git a/src/server/services/siteApiEndpointService.test.ts b/src/server/services/siteApiEndpointService.test.ts new file mode 100644 index 0000000..d2c421f --- /dev/null +++ b/src/server/services/siteApiEndpointService.test.ts @@ -0,0 +1,340 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { asc, eq } from 'drizzle-orm'; + +type DbModule = typeof import('../db/index.js'); +type SiteApiEndpointServiceModule = typeof import('./siteApiEndpointService.js'); + +describe('siteApiEndpointService', () => { + let db: DbModule['db']; + let schema: DbModule['schema']; + let selectSiteApiEndpointTarget: SiteApiEndpointServiceModule['selectSiteApiEndpointTarget']; + let recordSiteApiEndpointFailure: SiteApiEndpointServiceModule['recordSiteApiEndpointFailure']; + let recordSiteApiEndpointSuccess: SiteApiEndpointServiceModule['recordSiteApiEndpointSuccess']; + let dataDir = ''; + + beforeAll(async () => { + dataDir = mkdtempSync(join(tmpdir(), 'metapi-site-api-endpoint-service-')); + process.env.DATA_DIR = dataDir; + + await import('../db/migrate.js'); + const dbModule = await import('../db/index.js'); + const serviceModule = await import('./siteApiEndpointService.js'); + + db = dbModule.db; + schema = dbModule.schema; + selectSiteApiEndpointTarget = serviceModule.selectSiteApiEndpointTarget; + recordSiteApiEndpointFailure = serviceModule.recordSiteApiEndpointFailure; + recordSiteApiEndpointSuccess = serviceModule.recordSiteApiEndpointSuccess; + }); + + beforeEach(async () => { + await db.delete(schema.siteApiEndpoints).run(); + await db.delete(schema.sites).run(); + }); + + afterAll(() => { + delete process.env.DATA_DIR; + }); + + it('returns a synthetic site-url fallback when the site has no configured api endpoints', async () => { + const site = await db.insert(schema.sites).values({ + name: 'panel-only-site', + url: 'https://panel.example.com/', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z'); + + expect(selected).toMatchObject({ + kind: 'site-fallback', + siteId: site.id, + endpointId: null, + baseUrl: 'https://panel.example.com', + configuredEndpointCount: 0, + }); + }); + + it('selects the least recently selected enabled endpoint when sort order is tied', async () => { + const site = await db.insert(schema.sites).values({ + name: 'pool-site', + url: 'https://panel.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-b.example.com', + enabled: true, + sortOrder: 1, + lastSelectedAt: '2026-03-31T11:59:00.000Z', + }, + { + siteId: site.id, + url: 'https://api-a.example.com/', + enabled: true, + sortOrder: 0, + lastSelectedAt: '2026-03-31T11:00:00.000Z', + }, + ]).run(); + + const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z'); + + expect(selected).toMatchObject({ + kind: 'endpoint', + siteId: site.id, + baseUrl: 'https://api-a.example.com', + configuredEndpointCount: 2, + endpoint: expect.objectContaining({ + url: 'https://api-a.example.com/', + sortOrder: 0, + }), + }); + }); + + it('prefers lower sortOrder before lastSelectedAt when selecting an enabled endpoint', async () => { + const site = await db.insert(schema.sites).values({ + name: 'ordered-site', + url: 'https://panel.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-secondary.example.com', + enabled: true, + sortOrder: 1, + lastSelectedAt: '2026-03-31T11:00:00.000Z', + }, + { + siteId: site.id, + url: 'https://api-primary.example.com', + enabled: true, + sortOrder: 0, + lastSelectedAt: '2026-03-31T11:59:00.000Z', + }, + ]).run(); + + const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z'); + + expect(selected).toMatchObject({ + kind: 'endpoint', + siteId: site.id, + baseUrl: 'https://api-primary.example.com', + configuredEndpointCount: 2, + endpoint: expect.objectContaining({ + url: 'https://api-primary.example.com', + sortOrder: 0, + }), + }); + }); + + it('skips disabled endpoints and endpoints that are still cooling down', async () => { + const site = await db.insert(schema.sites).values({ + name: 'filtered-site', + url: 'https://panel.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-disabled.example.com', + enabled: false, + sortOrder: 0, + }, + { + siteId: site.id, + url: 'https://api-cooling.example.com', + enabled: true, + sortOrder: 1, + cooldownUntil: '2026-03-31T12:05:00.000Z', + }, + { + siteId: site.id, + url: 'https://api-ready.example.com', + enabled: true, + sortOrder: 2, + cooldownUntil: '2026-03-31T11:55:00.000Z', + }, + ]).run(); + + const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z'); + + expect(selected).toMatchObject({ + kind: 'endpoint', + baseUrl: 'https://api-ready.example.com', + configuredEndpointCount: 3, + }); + }); + + it('returns null when the site has configured api endpoints but none are currently eligible', async () => { + const site = await db.insert(schema.sites).values({ + name: 'exhausted-site', + url: 'https://panel.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + await db.insert(schema.siteApiEndpoints).values([ + { + siteId: site.id, + url: 'https://api-disabled.example.com', + enabled: false, + sortOrder: 0, + }, + { + siteId: site.id, + url: 'https://api-cooling.example.com', + enabled: true, + sortOrder: 1, + cooldownUntil: '2026-03-31T12:05:00.000Z', + }, + ]).run(); + + const selected = await selectSiteApiEndpointTarget(site, '2026-03-31T12:00:00.000Z'); + + expect(selected).toBeNull(); + }); + + it('records retryable failures with a 5-minute cooldown', async () => { + const site = await db.insert(schema.sites).values({ + name: 'retryable-site', + url: 'https://panel.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const endpoint = await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api-retryable.example.com', + enabled: true, + sortOrder: 0, + }).returning().get(); + + const result = await recordSiteApiEndpointFailure(endpoint.id, { + status: 502, + message: 'Bad gateway', + }, '2026-03-31T12:00:00.000Z'); + + expect(result).toMatchObject({ + retryable: true, + rotateToNextEndpoint: true, + cooldownUntil: '2026-03-31T12:05:00.000Z', + failureReason: 'HTTP 502: Bad gateway', + }); + + const stored = await db.select().from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.id, endpoint.id)) + .get(); + expect(stored).toMatchObject({ + cooldownUntil: '2026-03-31T12:05:00.000Z', + lastFailedAt: '2026-03-31T12:00:00.000Z', + lastFailureReason: 'HTTP 502: Bad gateway', + }); + }); + + it('parses retryable HTTP status codes from failure messages when no explicit status is provided', async () => { + const site = await db.insert(schema.sites).values({ + name: 'message-status-site', + url: 'https://panel.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const endpoint = await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api-message-status.example.com', + enabled: true, + sortOrder: 0, + }).returning().get(); + + const result = await recordSiteApiEndpointFailure(endpoint.id, { + message: 'HTTP 502: upstream temporarily unavailable', + }, '2026-03-31T12:00:00.000Z'); + + expect(result).toMatchObject({ + retryable: true, + rotateToNextEndpoint: true, + cooldownUntil: '2026-03-31T12:05:00.000Z', + failureReason: 'HTTP 502: upstream temporarily unavailable', + }); + }); + + it('records auth and validation failures without triggering cooldown rotation', async () => { + const site = await db.insert(schema.sites).values({ + name: 'non-retryable-site', + url: 'https://panel.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const endpoint = await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api-auth.example.com', + enabled: true, + sortOrder: 0, + cooldownUntil: '2026-03-31T11:00:00.000Z', + }).returning().get(); + + const result = await recordSiteApiEndpointFailure(endpoint.id, { + status: 401, + message: 'Invalid token', + }, '2026-03-31T12:00:00.000Z'); + + expect(result).toMatchObject({ + retryable: false, + rotateToNextEndpoint: false, + cooldownUntil: null, + failureReason: 'HTTP 401: Invalid token', + }); + + const stored = await db.select().from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.id, endpoint.id)) + .get(); + expect(stored).toMatchObject({ + cooldownUntil: null, + lastFailedAt: '2026-03-31T12:00:00.000Z', + lastFailureReason: 'HTTP 401: Invalid token', + }); + }); + + it('clears cooldown metadata and updates lastSelectedAt after a recorded success', async () => { + const site = await db.insert(schema.sites).values({ + name: 'success-site', + url: 'https://panel.example.com', + platform: 'new-api', + status: 'active', + }).returning().get(); + + const endpoint = await db.insert(schema.siteApiEndpoints).values({ + siteId: site.id, + url: 'https://api-success.example.com', + enabled: true, + sortOrder: 0, + cooldownUntil: '2026-03-31T12:05:00.000Z', + lastFailedAt: '2026-03-31T12:00:00.000Z', + lastFailureReason: 'HTTP 502: Bad gateway', + }).returning().get(); + + await recordSiteApiEndpointSuccess(endpoint.id, '2026-03-31T12:01:00.000Z'); + + const stored = await db.select().from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.id, endpoint.id)) + .orderBy(asc(schema.siteApiEndpoints.id)) + .get(); + expect(stored).toMatchObject({ + cooldownUntil: null, + lastSelectedAt: '2026-03-31T12:01:00.000Z', + lastFailureReason: null, + }); + }); +}); diff --git a/src/server/services/siteApiEndpointService.ts b/src/server/services/siteApiEndpointService.ts new file mode 100644 index 0000000..9e577b2 --- /dev/null +++ b/src/server/services/siteApiEndpointService.ts @@ -0,0 +1,287 @@ +import { asc, eq } from 'drizzle-orm'; +import { db, schema } from '../db/index.js'; +import { RETRYABLE_TIMEOUT_PATTERNS } from './proxyRetryPolicy.js'; + +const RETRYABLE_STATUS_CODES = new Set([408, 429, 500, 502, 503, 504]); +const NON_RETRYABLE_STATUS_CODES = new Set([400, 401, 403, 404, 422]); +const NETWORK_FAILURE_PATTERNS = [ + /network error/i, + /fetch failed/i, + /socket hang up/i, + /econnreset/i, + /econnrefused/i, + /enotfound/i, + /ehostunreach/i, + /ecanceled/i, + ...RETRYABLE_TIMEOUT_PATTERNS, +]; + +export const SITE_API_ENDPOINT_COOLDOWN_MS = 5 * 60 * 1000; + +type SiteRow = typeof schema.sites.$inferSelect; +type SiteApiEndpointRow = typeof schema.siteApiEndpoints.$inferSelect; + +export interface SiteApiEndpointTarget { + kind: 'site-fallback' | 'endpoint'; + siteId: number; + endpointId: number | null; + baseUrl: string; + configuredEndpointCount: number; + endpoint: SiteApiEndpointRow | null; +} + +export interface SiteApiEndpointFailureInput { + status?: number | null; + message?: string | null; + error?: unknown; +} + +export interface SiteApiEndpointFailureDisposition { + retryable: boolean; + rotateToNextEndpoint: boolean; + failureReason: string; +} + +export interface RecordedSiteApiEndpointFailure extends SiteApiEndpointFailureDisposition { + cooldownUntil: string | null; +} + +export class SiteApiEndpointRequestError extends Error { + readonly status: number | null; + readonly rawErrText: string | null; + + constructor(message: string, options?: { status?: number | null; rawErrText?: string | null; cause?: unknown }) { + super(message, options?.cause !== undefined ? { cause: options.cause } : undefined); + this.name = 'SiteApiEndpointRequestError'; + this.status = typeof options?.status === 'number' ? options.status : null; + this.rawErrText = typeof options?.rawErrText === 'string' && options.rawErrText.trim() + ? options.rawErrText + : null; + } +} + +export function normalizeSiteApiEndpointBaseUrl(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) return ''; + try { + const parsed = new URL(trimmed); + parsed.search = ''; + parsed.hash = ''; + return parsed.toString().replace(/\/+$/, ''); + } catch { + return trimmed.replace(/\/+$/, ''); + } +} + +function toIsoTimestamp(now?: string | Date): string { + if (typeof now === 'string' && now.trim()) return now; + if (now instanceof Date) return now.toISOString(); + return new Date().toISOString(); +} + +function compareNullableTimeAsc(left?: string | null, right?: string | null): number { + if (!left && !right) return 0; + if (!left) return -1; + if (!right) return 1; + return left.localeCompare(right); +} + +function isEndpointCoolingDown(endpoint: SiteApiEndpointRow, nowIso: string): boolean { + return !!endpoint.cooldownUntil && endpoint.cooldownUntil > nowIso; +} + +function extractFailureMessage(input: SiteApiEndpointFailureInput): string { + const direct = typeof input.message === 'string' ? input.message.trim() : ''; + if (direct) return direct; + const errorMessage = input.error instanceof Error ? input.error.message.trim() : ''; + return errorMessage; +} + +function formatFailureReason(status: number | null, message: string): string { + if (status && message) { + if (message.match(new RegExp(`^HTTP\\s+${status}\\b`, 'i'))) { + return message; + } + return `HTTP ${status}: ${message}`; + } + if (status) return `HTTP ${status}`; + return message || 'endpoint failure'; +} + +function parseStatusFromFailureMessage(message: string): number | null { + const matched = message.match(/\bHTTP\s+(\d{3})\b/i); + if (!matched) return null; + const status = Number.parseInt(matched[1] || '', 10); + return Number.isFinite(status) ? status : null; +} + +export function classifySiteApiEndpointFailure( + input: SiteApiEndpointFailureInput, +): SiteApiEndpointFailureDisposition { + const message = extractFailureMessage(input); + const status = typeof input.status === 'number' + ? input.status + : parseStatusFromFailureMessage(message); + const failureReason = formatFailureReason(status, message); + + if (status !== null) { + if (RETRYABLE_STATUS_CODES.has(status)) { + return { retryable: true, rotateToNextEndpoint: true, failureReason }; + } + if (NON_RETRYABLE_STATUS_CODES.has(status)) { + return { retryable: false, rotateToNextEndpoint: false, failureReason }; + } + } + + if (NETWORK_FAILURE_PATTERNS.some((pattern) => pattern.test(message))) { + return { retryable: true, rotateToNextEndpoint: true, failureReason }; + } + + return { retryable: false, rotateToNextEndpoint: false, failureReason }; +} + +export async function selectSiteApiEndpointTarget( + site: SiteRow, + now?: string | Date, +): Promise { + const nowIso = toIsoTimestamp(now); + const endpoints = await db.select().from(schema.siteApiEndpoints) + .where(eq(schema.siteApiEndpoints.siteId, site.id)) + .orderBy(asc(schema.siteApiEndpoints.sortOrder), asc(schema.siteApiEndpoints.id)) + .all(); + + if (endpoints.length === 0) { + return { + kind: 'site-fallback', + siteId: site.id, + endpointId: null, + baseUrl: normalizeSiteApiEndpointBaseUrl(site.url), + configuredEndpointCount: 0, + endpoint: null, + }; + } + + const eligible = endpoints + .filter((endpoint) => (endpoint.enabled ?? true) && !isEndpointCoolingDown(endpoint, nowIso)) + .sort((left, right) => { + const sortOrder = (left.sortOrder ?? 0) - (right.sortOrder ?? 0); + if (sortOrder !== 0) return sortOrder; + const selectionOrder = compareNullableTimeAsc(left.lastSelectedAt, right.lastSelectedAt); + if (selectionOrder !== 0) return selectionOrder; + return (left.id ?? 0) - (right.id ?? 0); + }); + + const selected = eligible[0]; + if (!selected) return null; + + return { + kind: 'endpoint', + siteId: site.id, + endpointId: selected.id, + baseUrl: normalizeSiteApiEndpointBaseUrl(selected.url), + configuredEndpointCount: endpoints.length, + endpoint: selected, + }; +} + +export async function resolveSiteApiBaseUrl( + site: SiteRow, + now?: string | Date, +): Promise { + const target = await selectSiteApiEndpointTarget(site, now); + return target?.baseUrl || null; +} + +export async function requireSiteApiBaseUrl( + site: SiteRow, + now?: string | Date, +): Promise { + const baseUrl = await resolveSiteApiBaseUrl(site, now); + if (baseUrl) return baseUrl; + throw new Error('当前站点的 AI 请求地址均不可用'); +} + +export async function recordSiteApiEndpointFailure( + endpointId: number, + input: SiteApiEndpointFailureInput, + now?: string | Date, +): Promise { + const nowIso = toIsoTimestamp(now); + const disposition = classifySiteApiEndpointFailure(input); + const cooldownUntil = disposition.retryable + ? new Date(Date.parse(nowIso) + SITE_API_ENDPOINT_COOLDOWN_MS).toISOString() + : null; + + await db.update(schema.siteApiEndpoints).set({ + cooldownUntil, + lastFailedAt: nowIso, + lastFailureReason: disposition.failureReason, + updatedAt: nowIso, + }).where(eq(schema.siteApiEndpoints.id, endpointId)).run(); + + return { + ...disposition, + cooldownUntil, + }; +} + +export async function recordSiteApiEndpointSuccess( + endpointId: number, + now?: string | Date, +): Promise { + const nowIso = toIsoTimestamp(now); + await db.update(schema.siteApiEndpoints).set({ + cooldownUntil: null, + lastSelectedAt: nowIso, + lastFailureReason: null, + updatedAt: nowIso, + }).where(eq(schema.siteApiEndpoints.id, endpointId)).run(); +} + +export async function runWithSiteApiEndpointPool( + site: SiteRow, + operation: (target: SiteApiEndpointTarget) => Promise, +): Promise { + const attemptedEndpointIds = new Set(); + let lastError: unknown; + + while (true) { + const target = await selectSiteApiEndpointTarget(site); + if (!target) { + if (lastError) throw lastError; + throw new Error('当前站点的 AI 请求地址均不可用'); + } + if (target.endpointId && attemptedEndpointIds.has(target.endpointId)) { + if (lastError) throw lastError; + throw new Error('当前站点的 AI 请求地址均不可用'); + } + + try { + const result = await operation(target); + if (target.endpointId) { + try { + await recordSiteApiEndpointSuccess(target.endpointId); + } catch (error) { + console.warn('[siteApiEndpointService] failed to record endpoint success', error); + } + } + return result; + } catch (error) { + lastError = error; + if (!target.endpointId) { + throw error; + } + + const recordedFailure = await recordSiteApiEndpointFailure(target.endpointId, { + status: error instanceof SiteApiEndpointRequestError ? error.status : undefined, + message: error instanceof Error ? error.message : String(error ?? ''), + error, + }); + if (!recordedFailure.rotateToNextEndpoint) { + throw error; + } + + attemptedEndpointIds.add(target.endpointId); + } + } +} diff --git a/src/web/pages/Sites.tsx b/src/web/pages/Sites.tsx index 7d4c6df..43588d4 100644 --- a/src/web/pages/Sites.tsx +++ b/src/web/pages/Sites.tsx @@ -20,11 +20,14 @@ import { shouldIgnoreRowSelectionClick } from './helpers/rowSelection.js'; import { resolveInitialConnectionSegment } from './helpers/defaultConnectionSegment.js'; import { buildSiteSaveAction, + emptySiteApiEndpoint, emptySiteCustomHeader, emptySiteForm, + serializeSiteApiEndpoints, serializeSiteCustomHeaders, siteFormFromSite, type SiteEditorState, + type SiteApiEndpointField, type SiteForm, } from './helpers/sitesEditor.js'; import { @@ -59,12 +62,33 @@ type SiteRow = { totalBalance?: number; subscriptionSummary?: SiteSubscriptionSummary | null; createdAt?: string; + apiEndpoints?: Array<{ + id?: number; + url: string; + enabled?: boolean; + sortOrder?: number; + cooldownUntil?: string | null; + lastFailureReason?: string | null; + }>; }; function hasConfiguredCustomHeaders(customHeaders?: string | null): boolean { return typeof customHeaders === 'string' && customHeaders.trim().length > 0; } +function getConfiguredSiteApiEndpoints(site?: Pick | null) { + return Array.isArray(site?.apiEndpoints) + ? site.apiEndpoints.filter((item) => typeof item?.url === 'string' && item.url.trim()) + : []; +} + +function buildSiteApiEndpointSummary(site?: Pick | null): string { + const endpoints = getConfiguredSiteApiEndpoints(site); + if (endpoints.length <= 0) return '跟随主站点 URL'; + const enabledCount = endpoints.filter((item) => item.enabled !== false).length; + return `${enabledCount}/${endpoints.length} 条启用`; +} + function formatUsd(value?: number | null): string { return `$${(value || 0).toFixed(2)}`; } @@ -203,7 +227,27 @@ export default function Sites() { const [sortMode, setSortMode] = useState('custom'); const [highlightSiteId, setHighlightSiteId] = useState(null); const [editor, setEditor] = useState(null); - const [form, setForm] = useState(emptySiteForm()); + const apiEndpointDraftIdRef = useRef(0); + const createApiEndpointDraftId = () => { + apiEndpointDraftIdRef.current += 1; + return `site-api-endpoint-draft-${apiEndpointDraftIdRef.current}`; + }; + const createEmptyApiEndpointRow = (): SiteApiEndpointField => ({ + ...emptySiteApiEndpoint(), + draftId: createApiEndpointDraftId(), + }); + const hydrateSiteForm = (value: SiteForm): SiteForm => { + const sourceRows = value.apiEndpoints.length > 0 ? value.apiEndpoints : [createEmptyApiEndpointRow()]; + return { + ...value, + apiEndpoints: sourceRows.map((endpoint) => ({ + ...endpoint, + draftId: endpoint.draftId || createApiEndpointDraftId(), + })), + }; + }; + const createEmptySiteForm = (): SiteForm => hydrateSiteForm(emptySiteForm()); + const [form, setForm] = useState(() => createEmptySiteForm()); const [detecting, setDetecting] = useState(false); const [saving, setSaving] = useState(false); const [deleting, setDeleting] = useState(null); @@ -364,7 +408,7 @@ export default function Sites() { const closeEditor = () => { setEditor(null); - setForm(emptySiteForm()); + setForm(createEmptySiteForm()); setSelectedInitializationPresetId(null); }; @@ -381,14 +425,14 @@ export default function Sites() { return; } setEditor({ mode: 'add' }); - setForm(emptySiteForm()); + setForm(createEmptySiteForm()); setSelectedInitializationPresetId(null); scrollToEditorTop(); }; const openEdit = (site: SiteRow) => { setEditor({ mode: 'edit', editingSiteId: site.id }); - setForm(siteFormFromSite(site)); + setForm(hydrateSiteForm(siteFormFromSite(site))); setSelectedInitializationPresetId(detectSiteInitializationPreset(site.url, site.platform)?.id || null); scrollToEditorTop(); // Load disabled models and discovered models independently so a best-effort @@ -471,6 +515,11 @@ export default function Sites() { toast.error(serializedCustomHeaders.error || '自定义请求头格式不正确'); return; } + const serializedApiEndpoints = serializeSiteApiEndpoints(form.apiEndpoints); + if (!serializedApiEndpoints.valid) { + toast.error(serializedApiEndpoints.error || 'AI 请求地址格式不正确'); + return; + } const payload = { name: form.name.trim(), @@ -480,6 +529,7 @@ export default function Sites() { initializationPresetId: selectedInitializationPresetId, proxyUrl: form.proxyUrl.trim(), useSystemProxy: !!form.useSystemProxy, + apiEndpoints: serializedApiEndpoints.apiEndpoints, customHeaders: serializedCustomHeaders.customHeaders, globalWeight: Number(parsedGlobalWeight.toFixed(3)), }; @@ -551,6 +601,48 @@ export default function Sites() { }); }; + const updateApiEndpointRow = (index: number, patch: Partial) => { + setForm((prev) => ({ + ...prev, + apiEndpoints: prev.apiEndpoints.map((item, itemIndex) => ( + itemIndex === index + ? { ...item, ...patch } + : item + )), + })); + }; + + const addApiEndpointRow = () => { + setForm((prev) => ({ + ...prev, + apiEndpoints: [...prev.apiEndpoints, createEmptyApiEndpointRow()], + })); + }; + + const removeApiEndpointRow = (index: number) => { + setForm((prev) => { + const nextEndpoints = prev.apiEndpoints.filter((_, itemIndex) => itemIndex !== index); + return { + ...prev, + apiEndpoints: nextEndpoints.length > 0 ? nextEndpoints : [createEmptyApiEndpointRow()], + }; + }); + }; + + const moveApiEndpointRow = (index: number, direction: 'up' | 'down') => { + setForm((prev) => { + const targetIndex = direction === 'up' ? index - 1 : index + 1; + if (targetIndex < 0 || targetIndex >= prev.apiEndpoints.length) return prev; + const nextEndpoints = [...prev.apiEndpoints]; + const [current] = nextEndpoints.splice(index, 1); + nextEndpoints.splice(targetIndex, 0, current); + return { + ...prev, + apiEndpoints: nextEndpoints, + }; + }); + }; + const handleSiteCreatedChoice = (choice: 'session' | 'apikey' | 'later') => { if (!createdSiteForChoice) return; @@ -935,7 +1027,8 @@ export default function Sites() { />
setForm((prev) => ({ ...prev, url: e.target.value }))} onBlur={() => { @@ -1008,6 +1101,100 @@ export default function Sites() {
)} +
+ 主站点 URL 用于登录、签到、面板接口和系统访问令牌管理；如果 AI 请求地址和主站点不同，请在下面的 AI 请求地址池里填写。 +
+
+
+
+ AI 请求地址池 +
+ +
+
+ 这里只用于 `/v1/*`、模型发现和 API Key 验证。不填时默认跟随主站点 URL；多条地址会按列表顺序参与轮询，禁用的地址不会参与调度。 +
+ {form.apiEndpoints.map((endpoint, index) => ( +
+
+ updateApiEndpointRow(index, { url: e.target.value })} + style={{ ...formInputStyle, flex: 1, fontFamily: 'var(--font-mono)' }} + /> + +
+
+
+ 顺序 #{index + 1} + {endpoint.cooldownUntil ? 冷却至 {formatDateTimeLocal(endpoint.cooldownUntil)} : null} + {endpoint.lastFailureReason ? 最近失败: {endpoint.lastFailureReason} : null} +
+
+ + + +
+
+
+ ))} +
- 填写后优先使用站点代理；留空则使用系统代理或直连(取决于设置开关状态)。 + 这里只是 HTTP/SOCKS 代理地址，不是上游 AI API 地址。填写后优先使用站点代理；留空则使用系统代理或直连(取决于设置开关状态)。