feat: review python code & optimize code structure

This commit is contained in:
Chenyme committed 2025-11-20 17:15:02 +08:00
1 parent 0c6c36f999
commit db73406dae
19 files changed
+1509 -2167

No files matched your search

+203 -540
View File
File diff suppressed because it is too large. Load diff
+10 -33
View File
@@ -1,9 +1,4 @@
# -*- coding: utf-8 -*-
"""
聊天API路由模块
提供OpenAI兼容的聊天API接口,支持与Grok模型的交互。
"""
"""聊天API路由 - OpenAI兼容的聊天接口"""
from fastapi import APIRouter, Depends, HTTPException
from typing import Optional
@@ -15,38 +10,20 @@ from app.core.logger import logger
from app.services.grok.client import GrokClient
from app.models.openai_schema import OpenAIChatRequest
# 聊天路由
router = APIRouter(prefix="/chat", tags=["聊天"])
@router.post("/completions", response_model=None)
async def chat_completions(
request: OpenAIChatRequest,
_: Optional[str] = Depends(auth_manager.verify)
):
"""
创建聊天补全
兼容OpenAI聊天API的端点,支持流式和非流式响应。
Args:
request: OpenAI格式的聊天请求
_: 认证依赖(自动验证)
Returns:
OpenAIChatCompletionResponse: 非流式响应
StreamingResponse: 流式响应
Raises:
HTTPException: 当请求处理失败时
"""
async def chat_completions(request: OpenAIChatRequest, _: Optional[str] = Depends(auth_manager.verify)):
"""创建聊天补全(支持流式和非流式)"""
try:
logger.info(f"[Chat] 收到聊天请求")
logger.info("[Chat] 收到聊天请求")
# 调用Grok客户端处理请求
# 调用Grok客户端
result = await GrokClient.openai_to_grok(request.model_dump())
# 如果是流式响应,GrokClient已经返回了Iterator,直接包装为StreamingResponse
# 流式响应
if request.stream:
return StreamingResponse(
content=result,
@@ -58,11 +35,11 @@ async def chat_completions(
}
)
# 非流式响应直接返回
# 非流式响应
return result
except GrokApiException as e:
logger.error(f"[Chat] Grok API错误: {str(e)} - 详情: {e.details}")
logger.error(f"[Chat] Grok API错误: {e} - 详情: {e.details}")
raise HTTPException(
status_code=500,
detail={
@@ -74,7 +51,7 @@ async def chat_completions(
}
)
except Exception as e:
logger.error(f"[Chat] 聊天请求处理失败: {str(e)}")
logger.error(f"[Chat] 处理失败: {e}")
raise HTTPException(
status_code=500,
detail={
+8 -13
View File
@@ -1,4 +1,4 @@
"""图片服务API路由"""
"""图片服务API - 提供缓存的图片和视频文件"""
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse
@@ -13,31 +13,26 @@ router = APIRouter()
@router.get("/images/{img_path:path}")
async def get_image(img_path: str):
"""获取缓存的图片或视频
Args:
img_path: 文件路径,格式如 users-xxx-generated-xxx-image.jpg 或 users-xxx-generated-xxx-video.mp4
Returns:
文件响应
img_path: 文件路径(格式:users-xxx-generated-xxx-image.jpg)
"""
try:
# 将路径转换回原始格式(短横线转斜杠)
# 转换路径(短横线→斜杠)
original_path = "/" + img_path.replace('-', '/')
# 判断是图片还是视频
# 判断类型
is_video = any(original_path.lower().endswith(ext) for ext in ['.mp4', '.webm', '.mov', '.avi'])
if is_video:
# 检查视频缓存
cache_path = video_cache_service.get_cached(original_path)
media_type = "video/mp4"
else:
# 检查图片缓存
cache_path = image_cache_service.get_cached(original_path)
media_type = "image/jpeg"
if cache_path and cache_path.exists():
logger.debug(f"[MediaAPI] 返回缓存文件: {cache_path}")
logger.debug(f"[MediaAPI] 返回缓存: {cache_path}")
return FileResponse(
path=str(cache_path),
media_type=media_type,
@@ -48,11 +43,11 @@ async def get_image(img_path: str):
)
# 文件不存在
logger.warning(f"[MediaAPI] 文件未找到: {original_path}")
logger.warning(f"[MediaAPI] 未找到: {original_path}")
raise HTTPException(status_code=404, detail="File not found")
except HTTPException:
raise
except Exception as e:
logger.error(f"[MediaAPI] 获取文件失败: {e}")
logger.error(f"[MediaAPI] 获取失败: {e}")
raise HTTPException(status_code=500, detail=str(e))
+18 -57
View File
@@ -1,8 +1,4 @@
"""
模型接口模块
提供 OpenAI 兼容的 /v1/models 端点,返回系统支持的所有模型列表。
"""
"""模型接口 - OpenAI兼容的模型列表端点"""
import time
from typing import Dict, Any, List, Optional
@@ -12,43 +8,27 @@ from app.models.grok_models import Models
from app.core.auth import auth_manager
from app.core.logger import logger
# 配置日志
# 创建路由器
router = APIRouter(tags=["模型"])
@router.get("/models")
async def list_models(_: Optional[str] = Depends(auth_manager.verify)) -> Dict[str, Any]:
"""
获取可用模型列表
返回 OpenAI 兼容的模型列表格式,包含系统支持的所有 Grok 模型的详细信息。
Args:
_: 认证依赖(自动验证)
Returns:
Dict[str, Any]: 包含模型列表的响应数据
"""
"""获取可用模型列表"""
try:
logger.debug("[Models] 请求获取模型列表")
logger.debug("[Models] 请求模型列表")
# 获取当前时间戳
current_timestamp = int(time.time())
# 构建模型数据列表
timestamp = int(time.time())
model_data: List[Dict[str, Any]] = []
for model in Models:
model_id = model.value
config = Models.get_model_info(model_id)
# 基础信息
model_info = {
"id": model_id,
"object": "model",
"created": current_timestamp,
"created": timestamp,
"owned_by": "x-ai",
"display_name": config.get("display_name", model_id),
"description": config.get("description", ""),
@@ -61,22 +41,16 @@ async def list_models(_: Optional[str] = Depends(auth_manager.verify)) -> Dict[s
model_data.append(model_info)
# 构建响应
response = {
"object": "list",
"data": model_data
}
logger.debug(f"[Models] 成功返回 {len(model_data)} 个模型")
return response
logger.debug(f"[Models] 返回 {len(model_data)} 个模型")
return {"object": "list", "data": model_data}
except Exception as e:
logger.error(f"[Models] 获取模型列表时发生错误: {str(e)}")
logger.error(f"[Models] 获取列表失败: {e}")
raise HTTPException(
status_code=500,
detail={
"error": {
"message": f"Failed to retrieve models: {str(e)}",
"message": f"Failed to retrieve models: {e}",
"type": "internal_error",
"code": "model_list_error"
}
@@ -86,22 +60,13 @@ async def list_models(_: Optional[str] = Depends(auth_manager.verify)) -> Dict[s
@router.get("/models/{model_id}")
async def get_model(model_id: str, _: Optional[str] = Depends(auth_manager.verify)) -> Dict[str, Any]:
"""
获取特定模型信息
Args:
model_id (str): 模型ID
_: 认证依赖(自动验证)
Returns:
Dict[str, Any]: 模型详细信息
"""
"""获取特定模型信息"""
try:
logger.debug(f"[Models] 请求获取模型信息: {model_id}")
logger.debug(f"[Models] 请求模型: {model_id}")
# 验证模型是否存在
# 验证模型
if not Models.is_valid_model(model_id):
logger.warning(f"[Models] 请求的模型不存在: {model_id}")
logger.warning(f"[Models] 模型不存在: {model_id}")
raise HTTPException(
status_code=404,
detail={
@@ -113,17 +78,13 @@ async def get_model(model_id: str, _: Optional[str] = Depends(auth_manager.verif
}
)
# 获取当前时间戳
current_timestamp = int(time.time())
# 获取模型配置
timestamp = int(time.time())
config = Models.get_model_info(model_id)
# 构建模型信息
model_info = {
"id": model_id,
"object": "model",
"created": current_timestamp,
"created": timestamp,
"owned_by": "x-ai",
"display_name": config.get("display_name", model_id),
"description": config.get("description", ""),
@@ -134,18 +95,18 @@ async def get_model(model_id: str, _: Optional[str] = Depends(auth_manager.verif
"default_top_p": config.get("default_top_p", 0.95)
}
logger.debug(f"[Models] 成功返回模型信息: {model_id}")
logger.debug(f"[Models] 返回模型: {model_id}")
return model_info
except HTTPException:
raise
except Exception as e:
logger.error(f"[Models] 获取模型信息时发生错误: {str(e)}")
logger.error(f"[Models] 获取模型失败: {e}")
raise HTTPException(
status_code=500,
detail={
"error": {
"message": f"Failed to retrieve model: {str(e)}",
"message": f"Failed to retrieve model: {e}",
"type": "internal_error",
"code": "model_retrieve_error"
}
+23 -18
View File
@@ -1,53 +1,58 @@
"""认证模块"""
"""认证模块 - API令牌验证"""
from typing import Optional
from fastapi import Depends, HTTPException
from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from app.core.config import setting
from app.core.logger import logger
# Bearer安全方案
security = HTTPBearer(auto_error=False)
def _build_error(message: str, code: str = "invalid_token") -> dict:
"""构建认证错误"""
return {
"error": {
"message": message,
"type": "authentication_error",
"code": code
}
}
class AuthManager:
"""认证管理器"""
"""认证管理器 - 验证API令牌"""
@staticmethod
def verify(credentials: Optional[HTTPAuthorizationCredentials] = Depends(security)) -> Optional[str]:
"""验证认证令牌"""
"""验证令牌"""
api_key = setting.grok_config.get("api_key")
# 未设置时跳过
if not api_key:
logger.debug("[Auth] 未设置API_KEY,跳过验证。")
logger.debug("[Auth] 未设置API_KEY,跳过验证")
return credentials.credentials if credentials else None
# 检查令牌
if not credentials:
raise HTTPException(
status_code=401,
detail={
"error": {
"message": "缺少认证令牌",
"type": "authentication_error",
"code": "missing_token"
}
}
detail=_build_error("缺少认证令牌", "missing_token")
)
# 验证令牌
if credentials.credentials != api_key:
raise HTTPException(
status_code=401,
detail={
"error": {
"message": f"令牌无效,长度: {len(credentials.credentials)}",
"type": "authentication_error",
"code": "invalid_token"
}
}
detail=_build_error(f"令牌无效,长度: {len(credentials.credentials)}", "invalid_token")
)
logger.debug("[Auth] 令牌认证成功")
return credentials.credentials
# 全局实例
auth_manager = AuthManager()
+124 -125
View File
@@ -1,164 +1,163 @@
"""配置管理器"""
"""配置管理器 - 管理应用配置的读写"""
import toml
from pathlib import Path
from typing import Dict, Any
from typing import Dict, Any, Optional, Literal
# 默认配置
DEFAULT_GROK = {
"api_key": "",
"proxy_url": "",
"cache_proxy_url": "",
"cf_clearance": "",
"x_statsig_id": "",
"dynamic_statsig": True,
"filtered_tags": "xaiartifact,xai:tool_usage_card,grok:render",
"stream_chunk_timeout": 120,
"stream_total_timeout": 600,
"stream_first_response_timeout": 30,
"temporary": True,
"show_thinking": True
}
DEFAULT_GLOBAL = {
"base_url": "http://localhost:8000",
"log_level": "INFO",
"image_mode": "url",
"admin_password": "admin",
"admin_username": "admin",
"image_cache_max_size_mb": 512,
"video_cache_max_size_mb": 1024
}
class ConfigManager:
"""配置管理器"""
def __init__(self) -> None:
"""初始化"""
# 配置文件路径
"""初始化配置"""
self.config_path: Path = Path(__file__).parents[2] / "data" / "setting.toml"
# 确保配置文件存在(首次启动时创建默认配置)
self._ensure_config_exists()
# 加载配置
self._storage: Optional[Any] = None
self._ensure_exists()
self.global_config: Dict[str, Any] = self.load("global")
self.grok_config: Dict[str, Any] = self.load("grok")
self._storage = None
def _ensure_config_exists(self) -> None:
"""确保配置文件存在,如果不存在则创建默认配置"""
def _ensure_exists(self) -> None:
"""确保配置存在"""
if not self.config_path.exists():
# 创建 data 目录
self.config_path.parent.mkdir(parents=True, exist_ok=True)
# 创建默认配置文件
self._create_default_config()
self._create_default()
def _create_default_config(self) -> None:
"""创建默认配置文件"""
default_config = {
"grok": {
"api_key": "",
"proxy_url": "",
"cache_proxy_url": "",
"cf_clearance": "",
"x_statsig_id": "",
"dynamic_statsig": True,
"filtered_tags": "xaiartifact,xai:tool_usage_card,grok:render",
"stream_chunk_timeout": 120,
"stream_total_timeout": 600,
"stream_first_response_timeout": 30,
"temporary": True,
"show_thinking": False
},
"global": {
"base_url": "http://localhost:8000",
"log_level": "INFO",
"image_mode": "url",
"admin_password": "admin",
"admin_username": "admin",
"image_cache_max_size_mb": 512,
"video_cache_max_size_mb": 1024
}
}
def _create_default(self) -> None:
"""创建默认配置"""
default = {"grok": DEFAULT_GROK.copy(), "global": DEFAULT_GLOBAL.copy()}
with open(self.config_path, "w", encoding="utf-8") as f:
toml.dump(default_config, f)
toml.dump(default, f)
def _normalize_proxy(self, proxy: str) -> str:
"""标准化代理URL(socks5:// → socks5h://)"""
if proxy and proxy.startswith("socks5://"):
return proxy.replace("socks5://", "socks5h://", 1)
return proxy
def _normalize_cf(self, cf: str) -> str:
"""标准化CF Clearance(自动添加前缀)"""
if cf and not cf.startswith("cf_clearance="):
return f"cf_clearance={cf}"
return cf
def set_storage(self, storage) -> None:
def set_storage(self, storage: Any) -> None:
"""设置存储实例"""
self._storage = storage
def load(self, section: str) -> Dict[str, Any]:
"""配置加载器"""
def load(self, section: Literal["global", "grok"]) -> Dict[str, Any]:
"""加载配置节"""
try:
with open(self.config_path, "r", encoding="utf-8") as f:
config = toml.load(f)[section]
# 自动将 SOCKS5 转换为 SOCKS5H
if section == "grok" and "proxy_url" in config:
proxy_url = config["proxy_url"]
if proxy_url and proxy_url.startswith("socks5://"):
config["proxy_url"] = proxy_url.replace("socks5://", "socks5h://", 1)
# 标准化Grok配置
if section == "grok":
if "proxy_url" in config:
config["proxy_url"] = self._normalize_proxy(config["proxy_url"])
if "cf_clearance" in config:
config["cf_clearance"] = self._normalize_cf(config["cf_clearance"])
# 自动为 CF Clearance 添加前缀
if section == "grok" and "cf_clearance" in config:
cf_clearance = config["cf_clearance"]
if cf_clearance and not cf_clearance.startswith("cf_clearance="):
config["cf_clearance"] = f"cf_clearance={cf_clearance}"
return config
return config
except Exception as e:
raise Exception(f"[Setting] 配置加载失败: {e}")
raise Exception(f"[Setting] 配置加载失败: {e}") from e
async def reload(self) -> None:
"""重新加载配置(用于从存储同步后)"""
"""重新加载配置"""
self.global_config = self.load("global")
self.grok_config = self.load("grok")
async def save(self, global_config: Dict[str, Any] = None, grok_config: Dict[str, Any] = None) -> None:
"""保存配置到存储"""
if not self._storage:
# 如果没有设置存储,使用传统文件保存方式
import aiofiles
async with aiofiles.open(self.config_path, "r", encoding="utf-8") as f:
content = await f.read()
config = toml.loads(content)
if global_config:
config["global"].update(global_config)
if grok_config:
# 处理 cf_clearance,移除前缀后保存
processed_grok_config = grok_config.copy()
if "cf_clearance" in processed_grok_config:
cf_clearance = processed_grok_config["cf_clearance"]
if cf_clearance and cf_clearance.startswith("cf_clearance="):
processed_grok_config["cf_clearance"] = cf_clearance.replace("cf_clearance=", "", 1)
config["grok"].update(processed_grok_config)
async with aiofiles.open(self.config_path, "w", encoding="utf-8") as f:
await f.write(toml.dumps(config))
else:
# 使用存储抽象层
config_data = await self._storage.load_config()
if global_config:
config_data["global"].update(global_config)
if grok_config:
# 处理 cf_clearance,移除前缀后保存
processed_grok_config = grok_config.copy()
if "cf_clearance" in processed_grok_config:
cf_clearance = processed_grok_config["cf_clearance"]
if cf_clearance and cf_clearance.startswith("cf_clearance="):
processed_grok_config["cf_clearance"] = cf_clearance.replace("cf_clearance=", "", 1)
config_data["grok"].update(processed_grok_config)
await self._storage.save_config(config_data)
async def _save_file(self, updates: Dict[str, Dict[str, Any]]) -> None:
"""保存到文件"""
import aiofiles
async with aiofiles.open(self.config_path, "r", encoding="utf-8") as f:
config = toml.loads(await f.read())
for section, data in updates.items():
if section in config:
config[section].update(data)
async with aiofiles.open(self.config_path, "w", encoding="utf-8") as f:
await f.write(toml.dumps(config))
async def _save_storage(self, updates: Dict[str, Dict[str, Any]]) -> None:
"""保存到存储"""
config = await self._storage.load_config()
for section, data in updates.items():
if section in config:
config[section].update(data)
await self._storage.save_config(config)
def _prepare_grok(self, grok: Dict[str, Any]) -> Dict[str, Any]:
"""准备Grok配置(移除前缀)"""
processed = grok.copy()
if "cf_clearance" in processed:
cf = processed["cf_clearance"]
if cf and cf.startswith("cf_clearance="):
processed["cf_clearance"] = cf.replace("cf_clearance=", "", 1)
return processed
async def save(self, global_config: Optional[Dict[str, Any]] = None, grok_config: Optional[Dict[str, Any]] = None) -> None:
"""保存配置"""
updates = {}
if global_config:
updates["global"] = global_config
if grok_config:
updates["grok"] = self._prepare_grok(grok_config)
# 选择存储方式
if self._storage:
await self._save_storage(updates)
else:
await self._save_file(updates)
# 重新加载配置
await self.reload()
def get_service_proxy(self) -> str:
"""获取服务代理URL(用于 client 和 upload)"""
return self.grok_config.get("proxy_url", "")
def get_cache_proxy(self) -> str:
"""获取缓存代理URL(用于 cache)
def get_proxy(self, proxy_type: Literal["service", "cache"] = "service") -> str:
"""获取代理URL
逻辑:
- 如果只设置了 proxy_url,缓存和服务都使用 proxy_url
- 如果同时设置了 proxy_url 和 cache_proxy_url,缓存使用 cache_proxy_url
Args:
proxy_type: 代理类型
- service: 服务代理(client/upload)
- cache: 缓存代理(cache)
"""
cache_proxy = self.grok_config.get("cache_proxy_url", "")
service_proxy = self.grok_config.get("proxy_url", "")
if proxy_type == "cache":
cache_proxy = self.grok_config.get("cache_proxy_url", "")
if cache_proxy:
return cache_proxy
# 如果设置了 cache_proxy_url,优先使用
if cache_proxy:
return cache_proxy
# 否则使用 proxy_url(服务代理)
return service_proxy
return self.grok_config.get("proxy_url", "")
# 全局设置
# 全局实例
setting = ConfigManager()
if __name__ == "__main__":
print(setting.global_config)
print(setting.grok_config)
+54 -56
View File
@@ -1,4 +1,4 @@
"""异常处理器"""
"""异常处理器 - OpenAI兼容的错误响应"""
from fastapi import Request, status
from fastapi.responses import JSONResponse
@@ -6,22 +6,60 @@ from fastapi.exceptions import RequestValidationError
from starlette.exceptions import HTTPException as StarletteHTTPException
class GrokApiException(Exception):
"""Grok API 业务异常"""
# HTTP错误映射
HTTP_ERROR_MAP = {
400: ("invalid_request_error", "请求格式错误或缺少必填参数"),
401: ("invalid_request_error", "令牌认证失败"),
403: ("permission_error", "没有权限访问此资源"),
404: ("invalid_request_error", "请求的资源不存在"),
429: ("rate_limit_error", "请求频率超出限制,请稍后再试"),
500: ("api_error", "内部服务器错误"),
503: ("api_error", "服务暂时不可用"),
}
def __init__(self, message: str, error_code: str = None, details: dict = None):
# Grok错误码映射
GROK_STATUS_MAP = {
"NO_AUTH_TOKEN": status.HTTP_401_UNAUTHORIZED,
"INVALID_TOKEN": status.HTTP_401_UNAUTHORIZED,
"HTTP_ERROR": status.HTTP_502_BAD_GATEWAY,
"NETWORK_ERROR": status.HTTP_503_SERVICE_UNAVAILABLE,
"JSON_ERROR": status.HTTP_502_BAD_GATEWAY,
"API_ERROR": status.HTTP_502_BAD_GATEWAY,
"STREAM_ERROR": status.HTTP_502_BAD_GATEWAY,
"NO_RESPONSE": status.HTTP_502_BAD_GATEWAY,
"TOKEN_SAVE_ERROR": status.HTTP_500_INTERNAL_SERVER_ERROR,
"NO_AVAILABLE_TOKEN": status.HTTP_503_SERVICE_UNAVAILABLE,
}
GROK_TYPE_MAP = {
"NO_AUTH_TOKEN": "authentication_error",
"INVALID_TOKEN": "authentication_error",
"HTTP_ERROR": "api_error",
"NETWORK_ERROR": "api_error",
"JSON_ERROR": "api_error",
"API_ERROR": "api_error",
"STREAM_ERROR": "api_error",
"NO_RESPONSE": "api_error",
"TOKEN_SAVE_ERROR": "api_error",
"NO_AVAILABLE_TOKEN": "api_error",
}
class GrokApiException(Exception):
"""Grok API业务异常"""
def __init__(self, message: str, error_code: str = None, details: dict = None, context: dict = None):
self.message = message
self.error_code = error_code
self.details = details or {}
self.context = context or {}
super().__init__(self.message)
def build_error_response(message: str, error_type: str, code: str = None, param: str = None) -> dict:
"""构建OpenAI兼容的错误响应"""
error = {
"message": message,
"type": error_type,
}
error = {"message": message, "type": error_type}
if code:
error["code"] = code
if param:
@@ -32,18 +70,8 @@ def build_error_response(message: str, error_type: str, code: str = None, param:
async def http_exception_handler(_: Request, exc: StarletteHTTPException) -> JSONResponse:
"""处理HTTP异常"""
error_map = {
400: ("invalid_request_error", "请求格式错误或缺少必填参数。"),
401: ("invalid_request_error", "令牌认证失败。"),
403: ("permission_error", "没有权限访问此资源。"),
404: ("invalid_request_error", "请求的资源不存在。"),
429: ("rate_limit_error", "请求频率超出限制,请稍后再试。"),
500: ("api_error", "内部服务器错误。"),
503: ("api_error", "服务暂时不可用。"),
}
error_type, default_message = error_map.get(exc.status_code, ("api_error", str(exc.detail)))
message = str(exc.detail) if exc.detail else default_message
error_type, default_msg = HTTP_ERROR_MAP.get(exc.status_code, ("api_error", str(exc.detail)))
message = str(exc.detail) if exc.detail else default_msg
return JSONResponse(
status_code=exc.status_code,
@@ -55,7 +83,7 @@ async def validation_exception_handler(_: Request, exc: RequestValidationError)
"""处理验证错误"""
errors = exc.errors()
param = errors[0]["loc"][-1] if errors and errors[0].get("loc") else None
message = errors[0]["msg"] if errors and errors[0].get("msg") else "请求参数错误。"
message = errors[0]["msg"] if errors and errors[0].get("msg") else "请求参数错误"
return JSONResponse(
status_code=status.HTTP_400_BAD_REQUEST,
@@ -64,36 +92,9 @@ async def validation_exception_handler(_: Request, exc: RequestValidationError)
async def grok_api_exception_handler(_: Request, exc: GrokApiException) -> JSONResponse:
"""处理Grok API业务异常"""
# 根据错误码映射HTTP状态码
status_code_map = {
"NO_AUTH_TOKEN": status.HTTP_401_UNAUTHORIZED,
"INVALID_TOKEN": status.HTTP_401_UNAUTHORIZED,
"HTTP_ERROR": status.HTTP_502_BAD_GATEWAY,
"NETWORK_ERROR": status.HTTP_503_SERVICE_UNAVAILABLE,
"JSON_ERROR": status.HTTP_502_BAD_GATEWAY,
"API_ERROR": status.HTTP_502_BAD_GATEWAY,
"STREAM_ERROR": status.HTTP_502_BAD_GATEWAY,
"NO_RESPONSE": status.HTTP_502_BAD_GATEWAY,
"TOKEN_SAVE_ERROR": status.HTTP_500_INTERNAL_SERVER_ERROR,
"NO_AVAILABLE_TOKEN": status.HTTP_503_SERVICE_UNAVAILABLE,
}
http_status = status_code_map.get(exc.error_code, status.HTTP_500_INTERNAL_SERVER_ERROR)
error_type_map = {
"NO_AUTH_TOKEN": "authentication_error",
"INVALID_TOKEN": "authentication_error",
"HTTP_ERROR": "api_error",
"NETWORK_ERROR": "api_error",
"JSON_ERROR": "api_error",
"API_ERROR": "api_error",
"STREAM_ERROR": "api_error",
"NO_RESPONSE": "api_error",
"TOKEN_SAVE_ERROR": "api_error",
"NO_AVAILABLE_TOKEN": "api_error",
}
error_type = error_type_map.get(exc.error_code, "api_error")
"""处理Grok API异常"""
http_status = GROK_STATUS_MAP.get(exc.error_code, status.HTTP_500_INTERNAL_SERVER_ERROR)
error_type = GROK_TYPE_MAP.get(exc.error_code, "api_error")
return JSONResponse(
status_code=http_status,
@@ -105,15 +106,12 @@ async def global_exception_handler(_: Request, exc: Exception) -> JSONResponse:
"""处理未捕获异常"""
return JSONResponse(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
content=build_error_response(
"服务器遇到意外错误,请重试。",
"api_error"
)
content=build_error_response("服务器遇到意外错误,请重试", "api_error")
)
def register_exception_handlers(app) -> None:
"""注册OpenAI兼容的异常处理器"""
"""注册异常处理器"""
app.add_exception_handler(StarletteHTTPException, http_exception_handler)
app.add_exception_handler(RequestValidationError, validation_exception_handler)
app.add_exception_handler(GrokApiException, grok_api_exception_handler)
+53 -37
View File
@@ -1,27 +1,32 @@
"""全局日志模块"""
"""全局日志模块 - 单例模式的日志管理器"""
import sys
import logging
from pathlib import Path
from logging.handlers import RotatingFileHandler
from app.core.config import setting
# 过滤模式
FILTER_PATTERNS = [
"chunk: b'", # SSE原始字节
"Got event:", # SSE事件
"Closing", # SSE关闭
]
class MCPLogFilter(logging.Filter):
"""MCP 日志过滤器 - 过滤掉包含大量数据的 DEBUG 日志"""
"""MCP日志过滤器 - 过滤大量数据的DEBUG日志"""
def filter(self, record):
# 过滤掉包含原始字节数据的 SSE 日志
if record.name == "sse_starlette.sse" and "chunk: b'" in record.getMessage():
return False
# 过滤掉 SSE 的一些冗余日志
def filter(self, record: logging.LogRecord) -> bool:
"""过滤日志"""
# 过滤SSE的DEBUG日志
if record.name == "sse_starlette.sse" and record.levelno == logging.DEBUG:
msg = record.getMessage()
if any(x in msg for x in ["Got event:", "Closing", "chunk:"]):
return False
return not any(p in msg for p in FILTER_PATTERNS)
# 过滤掉 MCP streamable_http 的一些 DEBUG 日志
# 过滤MCP streamable_http的DEBUG日志
if "mcp.server.streamable_http" in record.name and record.levelno == logging.DEBUG:
return False
@@ -29,65 +34,76 @@ class MCPLogFilter(logging.Filter):
class LoggerManager:
"""日志管理器"""
"""日志管理器(单例)"""
_instance = None
_initialized = False
def __new__(cls):
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __init__(self):
"""初始化日志"""
"""初始化日志系统"""
if LoggerManager._initialized:
return
# 日志配置
# 配置
log_dir = Path(__file__).parents[2] / "logs"
log_dir.mkdir(exist_ok=True)
log_level = setting.global_config.get("log_level", "INFO").upper()
log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
log_file = log_dir / "app.log"
# 配置根日志器
# 根日志器
self.logger = logging.getLogger()
self.logger.setLevel(log_level)
# 避免重复添加处理器
# 避免重复添加
if self.logger.handlers:
return
# 创建格式器
# 格式器和过滤器
formatter = logging.Formatter(log_format)
# 创建日志过滤器
mcp_filter = MCPLogFilter()
# 控制台处理器
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(log_level)
console_handler.setFormatter(formatter)
console_handler.addFilter(mcp_filter)
console = logging.StreamHandler(sys.stdout)
console.setLevel(log_level)
console.setFormatter(formatter)
console.addFilter(mcp_filter)
# 文件处理器
# 文件处理器(10MB,5个备份)
file_handler = RotatingFileHandler(
log_file, maxBytes=10 * 1024 * 1024, backupCount=5, encoding="utf-8"
log_file, maxBytes=10*1024*1024, backupCount=5, encoding="utf-8"
)
file_handler.setLevel(log_level)
file_handler.setFormatter(formatter)
file_handler.addFilter(mcp_filter)
# 添加处理器到根日志器
self.logger.addHandler(console_handler)
# 添加处理器
self.logger.addHandler(console)
self.logger.addHandler(file_handler)
# 配置第三方库日志级别,避免过多调试信息
logging.getLogger("asyncio").setLevel(logging.WARNING)
logging.getLogger("uvicorn").setLevel(logging.INFO)
logging.getLogger("fastapi").setLevel(logging.INFO)
logging.getLogger("aiomysql").setLevel(logging.WARNING)
# FastMCP 相关日志 - 关闭
logging.getLogger("mcp").setLevel(logging.CRITICAL)
logging.getLogger("fastmcp").setLevel(logging.CRITICAL)
# 配置第三方库
self._configure_third_party()
LoggerManager._initialized = True
def _configure_third_party(self):
"""配置第三方库日志级别"""
config = {
"asyncio": logging.WARNING,
"uvicorn": logging.INFO,
"fastapi": logging.INFO,
"aiomysql": logging.WARNING,
"mcp": logging.CRITICAL,
"fastmcp": logging.CRITICAL,
}
for name, level in config.items():
logging.getLogger(name).setLevel(level)
def debug(self, msg: str) -> None:
"""调试日志"""
@@ -110,5 +126,5 @@ class LoggerManager:
self.logger.critical(msg)
# 全局日志器实例
# 全局实例
logger = LoggerManager()
+98 -123
View File
@@ -1,7 +1,7 @@
"""存储抽象层 - 支持文件、MySQL和Redis存储"""
import os
import json
import orjson
import toml
import asyncio
import warnings
@@ -47,7 +47,7 @@ class BaseStorage(ABC):
class FileStorage(BaseStorage):
"""文件存储实现"""
"""文件存储"""
def __init__(self, data_dir: Path):
self.data_dir = data_dir
@@ -60,89 +60,87 @@ class FileStorage(BaseStorage):
"""初始化文件存储"""
self.data_dir.mkdir(parents=True, exist_ok=True)
# 初始化token文件
if not self.token_file.exists():
await self._write_file(self.token_file, json.dumps({"sso": {}, "ssoSuper": {}}, indent=2, ensure_ascii=False))
logger.info("[Storage] 创建新的token文件")
await self._write(self.token_file, orjson.dumps({"sso": {}, "ssoSuper": {}}, option=orjson.OPT_INDENT_2).decode())
logger.info("[Storage] 创建token文件")
# 初始化配置文件
if not self.config_file.exists():
default_config = {
default = {
"global": {"api_keys": [], "admin_username": "admin", "admin_password": "admin"},
"grok": {"proxy_url": "", "cf_clearance": "", "x_statsig_id": ""}
}
await self._write_file(self.config_file, toml.dumps(default_config))
logger.info("[Storage] 创建新的配置文件")
await self._write(self.config_file, toml.dumps(default))
logger.info("[Storage] 创建配置文件")
async def _read_file(self, file_path: Path) -> str:
"""读取文件内容"""
async with aiofiles.open(file_path, "r", encoding="utf-8") as f:
async def _read(self, path: Path) -> str:
"""读取文件"""
async with aiofiles.open(path, "r", encoding="utf-8") as f:
return await f.read()
async def _write_file(self, file_path: Path, content: str) -> None:
"""写入文件内容"""
async with aiofiles.open(file_path, "w", encoding="utf-8") as f:
async def _write(self, path: Path, content: str) -> None:
"""写入文件"""
async with aiofiles.open(path, "w", encoding="utf-8") as f:
await f.write(content)
async def _load_json(self, file_path: Path, default: Dict[str, Any], lock: asyncio.Lock) -> Dict[str, Any]:
"""加载JSON文件"""
async def _load_json(self, path: Path, default: Dict, lock: asyncio.Lock) -> Dict[str, Any]:
"""加载JSON"""
try:
async with lock:
if not file_path.exists():
if not path.exists():
return default
return json.loads(await self._read_file(file_path))
return orjson.loads(await self._read(path))
except Exception as e:
logger.error(f"[Storage] 加载{file_path.name}失败: {e}")
logger.error(f"[Storage] 加载{path.name}失败: {e}")
return default
async def _save_json(self, file_path: Path, data: Dict[str, Any], lock: asyncio.Lock) -> None:
"""保存JSON文件"""
async def _save_json(self, path: Path, data: Dict, lock: asyncio.Lock) -> None:
"""保存JSON"""
try:
async with lock:
await self._write_file(file_path, json.dumps(data, indent=2, ensure_ascii=False))
await self._write(path, orjson.dumps(data, option=orjson.OPT_INDENT_2).decode())
except Exception as e:
logger.error(f"[Storage] 保存{file_path.name}失败: {e}")
logger.error(f"[Storage] 保存{path.name}失败: {e}")
raise
async def _load_toml(self, file_path: Path, default: Dict[str, Any], lock: asyncio.Lock) -> Dict[str, Any]:
"""加载TOML文件"""
async def _load_toml(self, path: Path, default: Dict, lock: asyncio.Lock) -> Dict[str, Any]:
"""加载TOML"""
try:
async with lock:
if not file_path.exists():
if not path.exists():
return default
return toml.loads(await self._read_file(file_path))
return toml.loads(await self._read(path))
except Exception as e:
logger.error(f"[Storage] 加载{file_path.name}失败: {e}")
logger.error(f"[Storage] 加载{path.name}失败: {e}")
return default
async def _save_toml(self, file_path: Path, data: Dict[str, Any], lock: asyncio.Lock) -> None:
"""保存TOML文件"""
async def _save_toml(self, path: Path, data: Dict, lock: asyncio.Lock) -> None:
"""保存TOML"""
try:
async with lock:
await self._write_file(file_path, toml.dumps(data))
await self._write(path, toml.dumps(data))
except Exception as e:
logger.error(f"[Storage] 保存{file_path.name}失败: {e}")
logger.error(f"[Storage] 保存{path.name}失败: {e}")
raise
async def load_tokens(self) -> Dict[str, Any]:
"""加载token数据"""
"""加载token"""
return await self._load_json(self.token_file, {"sso": {}, "ssoSuper": {}}, self._token_lock)
async def save_tokens(self, data: Dict[str, Any]) -> None:
"""保存token数据"""
"""保存token"""
await self._save_json(self.token_file, data, self._token_lock)
async def load_config(self) -> Dict[str, Any]:
"""加载配置数据"""
"""加载配置"""
return await self._load_toml(self.config_file, {"global": {}, "grok": {}}, self._config_lock)
async def save_config(self, data: Dict[str, Any]) -> None:
"""保存配置数据"""
"""保存配置"""
await self._save_toml(self.config_file, data, self._config_lock)
class MysqlStorage(BaseStorage):
"""MySQL存储实现"""
"""MySQL存储"""
def __init__(self, database_url: str, data_dir: Path):
self.database_url = database_url
@@ -155,22 +153,15 @@ class MysqlStorage(BaseStorage):
try:
import aiomysql
parsed = self._parse_url(self.database_url)
logger.info(f"[Storage] 解析数据库连接: {parsed['user']}@{parsed['host']}:{parsed['port']}/{parsed['db']}")
logger.info(f"[Storage] MySQL: {parsed['user']}@{parsed['host']}:{parsed['port']}/{parsed['db']}")
# 创建数据库
await self._create_db(parsed)
# 创建连接池
self._pool = await aiomysql.create_pool(
host=parsed['host'], port=parsed['port'], user=parsed['user'],
password=parsed['password'], db=parsed['db'], charset="utf8mb4",
autocommit=True, maxsize=10
)
# 创建表
await self._create_tables()
# 初始化文件存储和同步数据
await self._file.init_db()
await self._sync_data()
@@ -181,26 +172,26 @@ class MysqlStorage(BaseStorage):
raise
def _parse_url(self, url: str) -> Dict[str, Any]:
"""解析数据库URL"""
parsed = urlparse(url)
"""解析URL"""
p = urlparse(url)
return {
'user': unquote(parsed.username) if parsed.username else "",
'password': unquote(parsed.password) if parsed.password else "",
'host': parsed.hostname,
'port': parsed.port or 3306,
'db': parsed.path[1:] if parsed.path else "grok2api"
'user': unquote(p.username) if p.username else "",
'password': unquote(p.password) if p.password else "",
'host': p.hostname,
'port': p.port or 3306,
'db': p.path[1:] if p.path else "grok2api"
}
async def _create_db(self, parsed: Dict[str, Any]) -> None:
async def _create_db(self, parsed: Dict) -> None:
"""创建数据库"""
import aiomysql
temp_pool = await aiomysql.create_pool(
pool = await aiomysql.create_pool(
host=parsed['host'], port=parsed['port'], user=parsed['user'],
password=parsed['password'], charset="utf8mb4", autocommit=True, maxsize=1
)
try:
async with temp_pool.acquire() as conn:
async with pool.acquire() as conn:
async with conn.cursor() as cursor:
with warnings.catch_warnings():
warnings.filterwarnings('ignore', message='.*database exists')
@@ -208,10 +199,10 @@ class MysqlStorage(BaseStorage):
f"CREATE DATABASE IF NOT EXISTS `{parsed['db']}` "
f"CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
)
logger.info(f"[Storage] MySQL数据库 '{parsed['db']}' 已就绪")
logger.info(f"[Storage] 数据库 '{parsed['db']}' 就绪")
finally:
temp_pool.close()
await temp_pool.wait_closed()
pool.close()
await pool.wait_closed()
async def _create_tables(self) -> None:
"""创建表"""
@@ -240,7 +231,7 @@ class MysqlStorage(BaseStorage):
warnings.filterwarnings('ignore', message='.*already exists')
for sql in tables.values():
await cursor.execute(sql)
logger.info("[Storage] MySQL表创建/验证成功")
logger.info("[Storage] MySQL表就绪")
async def _sync_data(self) -> None:
"""同步数据"""
@@ -252,39 +243,33 @@ class MysqlStorage(BaseStorage):
await self._file.save_tokens(data)
else:
await self._file.save_config(data)
logger.info(f"[Storage] {table.split('_')[1]}数据已从数据库同步到文件")
logger.info(f"[Storage] {table.split('_')[1]}数据已从DB同步")
else:
if table == "grok_tokens":
file_data = await self._file.load_tokens()
if file_data.get(key) or file_data.get("ssoSuper"):
await self._save_db(table, file_data)
logger.info("[Storage] Token数据已从文件初始化到数据库")
else:
file_data = await self._file.load_config()
if file_data.get(key) or file_data.get("grok"):
await self._save_db(table, file_data)
logger.info("[Storage] 配置数据已从文件初始化到数据库")
file_data = await (self._file.load_tokens() if table == "grok_tokens" else self._file.load_config())
if file_data.get(key) or (table == "grok_tokens" and file_data.get("ssoSuper")):
await self._save_db(table, file_data)
logger.info(f"[Storage] {table.split('_')[1]}数据已初始化到DB")
except Exception as e:
logger.warning(f"[Storage] 数据同步失败: {e}")
logger.warning(f"[Storage] 同步失败: {e}")
async def _load_db(self, table: str) -> Optional[Dict[str, Any]]:
"""从数据库加载数据"""
async def _load_db(self, table: str) -> Optional[Dict]:
"""从DB加载"""
try:
async with self._pool.acquire() as conn:
async with conn.cursor() as cursor:
await cursor.execute(f"SELECT data FROM {table} ORDER BY id DESC LIMIT 1")
result = await cursor.fetchone()
return json.loads(result[0]) if result else None
return orjson.loads(result[0]) if result else None
except Exception as e:
logger.error(f"[Storage] 从数据库加载{table}失败: {e}")
logger.error(f"[Storage] 加载{table}失败: {e}")
return None
async def _save_db(self, table: str, data: Dict[str, Any]) -> None:
"""保存数据到数据库"""
async def _save_db(self, table: str, data: Dict) -> None:
"""保存到DB"""
try:
async with self._pool.acquire() as conn:
async with conn.cursor() as cursor:
json_data = json.dumps(data, ensure_ascii=False)
json_data = orjson.dumps(data).decode()
await cursor.execute(f"SELECT id FROM {table} ORDER BY id DESC LIMIT 1")
result = await cursor.fetchone()
@@ -293,24 +278,24 @@ class MysqlStorage(BaseStorage):
else:
await cursor.execute(f"INSERT INTO {table} (data) VALUES (%s)", (json_data,))
except Exception as e:
logger.error(f"[Storage] 保存数据到{table}失败: {e}")
logger.error(f"[Storage] 保存{table}失败: {e}")
raise
async def load_tokens(self) -> Dict[str, Any]:
"""加载token数据"""
"""加载token"""
return await self._file.load_tokens()
async def save_tokens(self, data: Dict[str, Any]) -> None:
"""保存token数据"""
"""保存token"""
await self._file.save_tokens(data)
await self._save_db("grok_tokens", data)
async def load_config(self) -> Dict[str, Any]:
"""加载配置数据"""
"""加载配置"""
return await self._file.load_config()
async def save_config(self, data: Dict[str, Any]) -> None:
"""保存配置数据"""
"""保存配置"""
await self._file.save_config(data)
await self._save_db("grok_settings", data)
@@ -319,11 +304,11 @@ class MysqlStorage(BaseStorage):
if self._pool:
self._pool.close()
await self._pool.wait_closed()
logger.info("[Storage] MySQL连接池已关闭")
logger.info("[Storage] MySQL已关闭")
class RedisStorage(BaseStorage):
"""Redis存储实现"""
"""Redis存储"""
def __init__(self, redis_url: str, data_dir: Path):
self.redis_url = redis_url
@@ -336,20 +321,17 @@ class RedisStorage(BaseStorage):
try:
import redis.asyncio as redis
parsed = self._parse_url(self.redis_url)
logger.info(f"[Storage] 解析Redis URL: host={parsed['host']}, port={parsed['port']}, db={parsed.get('db', 0)}, username={parsed.get('username')}, password={'***' if parsed.get('password') else None}")
logger.info(f"[Storage] Redis: {parsed['host']}:{parsed['port']}/{parsed['db']}")
# 创建Redis连接
self._redis = redis.Redis(
host=parsed['host'], port=parsed['port'], password=parsed.get('password'),
username=parsed.get('username'), db=parsed.get('db', 0),
encoding="utf-8", decode_responses=True
)
# 测试连接
await self._redis.ping()
logger.info(f"[Storage] Redis连接成功: {parsed['host']}:{parsed['port']}/{parsed['db']}")
logger.info(f"[Storage] Redis连接成功")
# 初始化文件存储和同步数据
await self._file.init_db()
await self._sync_data()
@@ -363,17 +345,16 @@ class RedisStorage(BaseStorage):
"""解析Redis URL"""
if url.startswith('redis://'):
url = url[8:]
parsed = urlparse(f'//{url}')
p = urlparse(f'//{url}')
result = {
'host': parsed.hostname or 'localhost',
'port': parsed.port or 6379,
'db': int(parsed.path.lstrip('/')) if parsed.path and parsed.path != '/' else 0,
'username': unquote(parsed.username) if parsed.username else None,
'password': unquote(parsed.password) if parsed.password else None
'host': p.hostname or 'localhost',
'port': p.port or 6379,
'db': int(p.path.lstrip('/')) if p.path and p.path != '/' else 0,
'username': unquote(p.username) if p.username else None,
'password': unquote(p.password) if p.password else None
}
# Redis 6+ 默认用户名
if result['password'] and not result['username']:
result['username'] = 'default'
@@ -388,44 +369,43 @@ class RedisStorage(BaseStorage):
]:
data = await self._redis.get(key)
if data:
parsed = json.loads(data)
parsed = orjson.loads(data)
if key == "grok:tokens":
await self._file.save_tokens(parsed)
else:
await self._file.save_config(parsed)
logger.info(f"[Storage] {key.split(':')[1]}数据已从Redis同步到文件")
logger.info(f"[Storage] {key.split(':')[1]}数据已从Redis同步")
else:
file_data = await file_func()
if file_data.get(key_name) or (key == "grok:tokens" and file_data.get("ssoSuper")):
json_data = json.dumps(file_data, ensure_ascii=False)
await self._redis.set(key, json_data)
logger.info(f"[Storage] {key.split(':')[1]}数据已从文件初始化到Redis")
await self._redis.set(key, orjson.dumps(file_data).decode())
logger.info(f"[Storage] {key.split(':')[1]}数据已初始化到Redis")
except Exception as e:
logger.warning(f"[Storage] 数据同步失败: {e}")
logger.warning(f"[Storage] 同步失败: {e}")
async def _save_redis(self, key: str, data: Dict[str, Any]) -> None:
async def _save_redis(self, key: str, data: Dict) -> None:
"""保存到Redis"""
try:
await self._redis.set(key, json.dumps(data, ensure_ascii=False))
await self._redis.set(key, orjson.dumps(data).decode())
except Exception as e:
logger.error(f"[Storage] 保存到Redis失败: {e}")
logger.error(f"[Storage] 保存Redis失败: {e}")
raise
async def load_tokens(self) -> Dict[str, Any]:
"""加载token数据"""
"""加载token"""
return await self._file.load_tokens()
async def save_tokens(self, data: Dict[str, Any]) -> None:
"""保存token数据"""
"""保存token"""
await self._file.save_tokens(data)
await self._save_redis("grok:tokens", data)
async def load_config(self) -> Dict[str, Any]:
"""加载配置数据"""
"""加载配置"""
return await self._file.load_config()
async def save_config(self, data: Dict[str, Any]) -> None:
"""保存配置数据"""
"""保存配置"""
await self._file.save_config(data)
await self._save_redis("grok:settings", data)
@@ -433,11 +413,11 @@ class RedisStorage(BaseStorage):
"""关闭连接"""
if self._redis:
await self._redis.close()
logger.info("[Storage] Redis连接已关闭")
logger.info("[Storage] Redis已关闭")
class StorageManager:
"""存储管理器"""
"""存储管理器(单例)"""
_instance: Optional['StorageManager'] = None
_storage: Optional[BaseStorage] = None
@@ -457,22 +437,17 @@ class StorageManager:
url = os.getenv("DATABASE_URL", "")
data_dir = Path(__file__).parents[2] / "data"
storage_classes = {
"mysql": MysqlStorage,
"redis": RedisStorage,
"file": FileStorage
}
classes = {"mysql": MysqlStorage, "redis": RedisStorage, "file": FileStorage}
if mode in ("mysql", "redis") and not url:
raise ValueError(f"{mode.upper()}模式需要DATABASE_URL环境变量")
raise ValueError(f"{mode.upper()}模式需要DATABASE_URL")
storage_class = storage_classes.get(mode, FileStorage)
storage_class = classes.get(mode, FileStorage)
self._storage = storage_class(url, data_dir) if mode != "file" else storage_class(data_dir)
await self._storage.init_db()
self._initialized = True
logger.info(f"[Storage] 使用{mode}存储模式")
logger.info("[Storage] 存储管理器初始化完成")
logger.info(f"[Storage] 使用{mode}模式")
def get_storage(self) -> BaseStorage:
"""获取存储实例"""
@@ -486,5 +461,5 @@ class StorageManager:
await self._storage.close()
# 全局存储管理器实例
# 全局实例
storage_manager = StorageManager()
+133 -121
View File
@@ -1,122 +1,127 @@
from enum import Enum
"""Grok 模型配置和枚举定义"""
# 模型配置字典
_MODEL_CONFIG = {
"grok-3-fast": {
"grok_model": ("grok-3", "MODEL_MODE_FAST"),
"rate_limit_model": "grok-3",
"cost": {"type": "low_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok 3 Fast",
"description": "Fast and efficient Grok 3 model",
"raw_model_path": "xai/grok-3",
"default_temperature": 1.0,
"default_max_output_tokens": 8192,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4-fast": {
"grok_model": ("grok-4-mini-thinking-tahoe", "MODEL_MODE_GROK_4_MINI_THINKING"),
"rate_limit_model": "grok-4-mini-thinking-tahoe",
"cost": {"type": "low_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok 4 Fast",
"description": "Fast version of Grok 4 with mini thinking capabilities",
"raw_model_path": "xai/grok-4-mini-thinking-tahoe",
"default_temperature": 1.0,
"default_max_output_tokens": 8192,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4-fast-expert": {
"grok_model": ("grok-4-mini-thinking-tahoe", "MODEL_MODE_EXPERT"),
"rate_limit_model": "grok-4-mini-thinking-tahoe",
"cost": {"type": "high_cost", "multiplier": 4, "description": "计4次调用"},
"requires_super": False,
"display_name": "Grok 4 Fast Expert",
"description": "Expert mode of Grok 4 Fast with enhanced reasoning",
"raw_model_path": "xai/grok-4-mini-thinking-tahoe",
"default_temperature": 1.0,
"default_max_output_tokens": 32768,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4-expert": {
"grok_model": ("grok-4", "MODEL_MODE_EXPERT"),
"rate_limit_model": "grok-4",
"cost": {"type": "high_cost", "multiplier": 4, "description": "计4次调用"},
"requires_super": False,
"display_name": "Grok 4 Expert",
"description": "Full Grok 4 model with expert mode capabilities",
"raw_model_path": "xai/grok-4",
"default_temperature": 1.0,
"default_max_output_tokens": 32768,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4-heavy": {
"grok_model": ("grok-4-heavy", "MODEL_MODE_HEAVY"),
"rate_limit_model": "grok-4-heavy",
"cost": {"type": "independent", "multiplier": 1, "description": "独立计费,只有Super用户可用"},
"requires_super": True,
"display_name": "Grok 4 Heavy",
"description": "Most powerful Grok 4 model with heavy computational capabilities. Requires Super Token for access.",
"raw_model_path": "xai/grok-4-heavy",
"default_temperature": 1.0,
"default_max_output_tokens": 65536,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4.1": {
"grok_model": ("grok-4-1-non-thinking-w-tool", "MODEL_MODE_GROK_4_1"),
"rate_limit_model": "grok-4-1-non-thinking-w-tool",
"cost": {"type": "low_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok 4.1",
"description": "Latest Grok 4.1 model with tool capabilities",
"raw_model_path": "xai/grok-4-1-non-thinking-w-tool",
"default_temperature": 1.0,
"default_max_output_tokens": 8192,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4.1-thinking": {
"grok_model": ("grok-4-1-thinking-1108b", "MODEL_MODE_AUTO"),
"rate_limit_model": "grok-4-1-thinking-1108b",
"cost": {"type": "high_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok 4.1 Thinking",
"description": "Grok 4.1 model with advanced thinking and tool capabilities",
"raw_model_path": "xai/grok-4-1-thinking-1108b",
"default_temperature": 1.0,
"default_max_output_tokens": 32768,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-imagine-0.9": {
"grok_model": ("grok-3", "MODEL_MODE_FAST"),
"rate_limit_model": "grok-3",
"cost": {"type": "low_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok Imagine 0.9",
"description": "Video generation model powered by Grok",
"raw_model_path": "xai/grok-imagine-0.9",
"default_temperature": 1.0,
"default_max_output_tokens": 8192,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95,
"is_video_model": True
}
from enum import Enum
from typing import Dict, Any, Tuple
# 模型配置
_MODEL_CONFIG: Dict[str, Dict[str, Any]] = {
"grok-3-fast": {
"grok_model": ("grok-3", "MODEL_MODE_FAST"),
"rate_limit_model": "grok-3",
"cost": {"type": "low_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok 3 Fast",
"description": "Fast and efficient Grok 3 model",
"raw_model_path": "xai/grok-3",
"default_temperature": 1.0,
"default_max_output_tokens": 8192,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4-fast": {
"grok_model": ("grok-4-mini-thinking-tahoe", "MODEL_MODE_GROK_4_MINI_THINKING"),
"rate_limit_model": "grok-4-mini-thinking-tahoe",
"cost": {"type": "low_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok 4 Fast",
"description": "Fast version of Grok 4 with mini thinking capabilities",
"raw_model_path": "xai/grok-4-mini-thinking-tahoe",
"default_temperature": 1.0,
"default_max_output_tokens": 8192,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4-fast-expert": {
"grok_model": ("grok-4-mini-thinking-tahoe", "MODEL_MODE_EXPERT"),
"rate_limit_model": "grok-4-mini-thinking-tahoe",
"cost": {"type": "high_cost", "multiplier": 4, "description": "计4次调用"},
"requires_super": False,
"display_name": "Grok 4 Fast Expert",
"description": "Expert mode of Grok 4 Fast with enhanced reasoning",
"raw_model_path": "xai/grok-4-mini-thinking-tahoe",
"default_temperature": 1.0,
"default_max_output_tokens": 32768,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4-expert": {
"grok_model": ("grok-4", "MODEL_MODE_EXPERT"),
"rate_limit_model": "grok-4",
"cost": {"type": "high_cost", "multiplier": 4, "description": "计4次调用"},
"requires_super": False,
"display_name": "Grok 4 Expert",
"description": "Full Grok 4 model with expert mode capabilities",
"raw_model_path": "xai/grok-4",
"default_temperature": 1.0,
"default_max_output_tokens": 32768,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4-heavy": {
"grok_model": ("grok-4-heavy", "MODEL_MODE_HEAVY"),
"rate_limit_model": "grok-4-heavy",
"cost": {"type": "independent", "multiplier": 1, "description": "独立计费,只有Super用户可用"},
"requires_super": True,
"display_name": "Grok 4 Heavy",
"description": "Most powerful Grok 4 model with heavy computational capabilities. Requires Super Token for access.",
"raw_model_path": "xai/grok-4-heavy",
"default_temperature": 1.0,
"default_max_output_tokens": 65536,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4.1": {
"grok_model": ("grok-4-1-non-thinking-w-tool", "MODEL_MODE_GROK_4_1"),
"rate_limit_model": "grok-4-1-non-thinking-w-tool",
"cost": {"type": "low_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok 4.1",
"description": "Latest Grok 4.1 model with tool capabilities",
"raw_model_path": "xai/grok-4-1-non-thinking-w-tool",
"default_temperature": 1.0,
"default_max_output_tokens": 8192,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-4.1-thinking": {
"grok_model": ("grok-4-1-thinking-1108b", "MODEL_MODE_AUTO"),
"rate_limit_model": "grok-4-1-thinking-1108b",
"cost": {"type": "high_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok 4.1 Thinking",
"description": "Grok 4.1 model with advanced thinking and tool capabilities",
"raw_model_path": "xai/grok-4-1-thinking-1108b",
"default_temperature": 1.0,
"default_max_output_tokens": 32768,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95
},
"grok-imagine-0.9": {
"grok_model": ("grok-3", "MODEL_MODE_FAST"),
"rate_limit_model": "grok-3",
"cost": {"type": "low_cost", "multiplier": 1, "description": "计1次调用"},
"requires_super": False,
"display_name": "Grok Imagine 0.9",
"description": "Video generation model powered by Grok",
"raw_model_path": "xai/grok-imagine-0.9",
"default_temperature": 1.0,
"default_max_output_tokens": 8192,
"supported_max_output_tokens": 131072,
"default_top_p": 0.95,
"is_video_model": True
}
}
class TokenType(Enum):
"""Token类型枚举"""
NORMAL = "ssoNormal" # 普通用户Token
SUPER = "ssoSuper" # 超级用户Token
"""Token类型"""
NORMAL = "ssoNormal"
SUPER = "ssoSuper"
class Models(Enum):
"""支持的模型枚举"""
"""支持的模型"""
GROK_3_FAST = "grok-3-fast"
GROK_4_1 = "grok-4.1"
GROK_4_1_THINKING = "grok-4.1-thinking"
@@ -127,8 +132,8 @@ class Models(Enum):
GROK_IMAGINE_0_9 = "grok-imagine-0.9"
@classmethod
def get_model_info(cls, model: str) -> dict:
"""获取模型的完整配置信息"""
def get_model_info(cls, model: str) -> Dict[str, Any]:
"""获取模型配置"""
return _MODEL_CONFIG.get(model, {})
@classmethod
@@ -137,15 +142,22 @@ class Models(Enum):
return model in _MODEL_CONFIG
@classmethod
def to_grok(cls, model: str) -> tuple[str, str]:
"""转换为Grok内部模型名和模式类型"""
def to_grok(cls, model: str) -> Tuple[str, str]:
"""转换为Grok内部模型名和模式
Returns:
(模型名, 模式类型) 元组
"""
config = _MODEL_CONFIG.get(model)
if config:
return config["grok_model"]
return model, "MODEL_MODE_FAST"
return config["grok_model"] if config else (model, "MODEL_MODE_FAST")
@classmethod
def to_rate_limit(cls, model: str) -> str:
"""转换为速率限制接口模型名"""
"""转换为速率限制模型名"""
config = _MODEL_CONFIG.get(model)
return config["rate_limit_model"] if config else model
return config["rate_limit_model"] if config else model
@classmethod
def get_all_model_names(cls) -> list[str]:
"""获取所有模型名称"""
return list(_MODEL_CONFIG.keys())
+26 -38
View File
@@ -1,16 +1,18 @@
"""OpenAI 请求-响应模型"""
"""OpenAI 请求-响应模型定义"""
from fastapi import HTTPException
from typing import Optional, List, Union, Dict, Any
from pydantic import BaseModel, Field, field_validator
from app.models.grok_models import Models
class OpenAIChatRequest(BaseModel):
"""OpenAI 聊天请求模型"""
"""OpenAI聊天请求"""
model: str = Field(..., description="模型名称", min_length=1)
messages: List[Dict[str, Any]] = Field(..., description="消息列表", min_length=1)
stream: bool = Field(False, description="启用流式响应")
stream: bool = Field(False, description="流式响应")
temperature: Optional[float] = Field(0.7, ge=0, le=2, description="采样温度")
max_tokens: Optional[int] = Field(None, ge=1, le=100000, description="最大Token数")
top_p: Optional[float] = Field(1.0, ge=0, le=1, description="采样参数")
@@ -20,31 +22,19 @@ class OpenAIChatRequest(BaseModel):
def validate_messages(cls, v):
"""验证消息格式"""
if not v:
raise HTTPException(
status_code=400,
detail="消息列表不能为空"
)
raise HTTPException(status_code=400, detail="消息列表不能为空")
for msg in v:
if not isinstance(msg, dict):
raise HTTPException(
status_code=400,
detail="每个消息必须是一个字典"
)
raise HTTPException(status_code=400, detail="每个消息必须是字典")
if 'role' not in msg:
raise HTTPException(
status_code=400,
detail="消息缺少必填字段 'role'"
)
raise HTTPException(status_code=400, detail="消息缺少 'role' 字段")
if 'content' not in msg:
raise HTTPException(
status_code=400,
detail="消息缺少必填字段 'content'"
)
raise HTTPException(status_code=400, detail="消息缺少 'content' 字段")
if msg['role'] not in ['system', 'user', 'assistant']:
raise HTTPException(
status_code=400,
detail=f"无效的角色 '{msg['role']}', 必须是 'system', 'user' 或 'assistant'"
detail=f"无效角色 '{msg['role']}', 必须是 system/user/assistant"
)
return v
@@ -53,29 +43,27 @@ class OpenAIChatRequest(BaseModel):
@field_validator('model')
def validate_model(cls, v):
"""验证模型名称"""
allowed_models = [
'grok-3-fast', 'grok-4-fast', 'grok-4-fast-expert',
'grok-4-expert', 'grok-4-heavy', 'grok-imagine-0.9'
]
if v not in allowed_models:
if not Models.is_valid_model(v):
supported = Models.get_all_model_names()
raise HTTPException(
status_code=400,
detail=f"不支持的模型 '{v}', 支持的模型: {', '.join(allowed_models)}"
detail=f"不支持的模型 '{v}', 支持: {', '.join(supported)}"
)
return v
class OpenAIChatCompletionMessage(BaseModel):
"""聊天完成消息"""
role: str = Field(..., description="角色")
content: str = Field(..., description="消息内容")
content: str = Field(..., description="内容")
reference_id: Optional[str] = Field(default=None, description="参考ID")
annotations: Optional[List[str]] = Field(default=None, description="注释")
class OpenAIChatCompletionChoice(BaseModel):
"""聊天完成选项"""
index: int = Field(..., description="选项索引")
message: OpenAIChatCompletionMessage = Field(..., description="响应消息")
index: int = Field(..., description="索引")
message: OpenAIChatCompletionMessage = Field(..., description="消息")
logprobs: Optional[float] = Field(default=None, description="对数概率")
finish_reason: str = Field(default="stop", description="完成原因")
@@ -85,20 +73,20 @@ class OpenAIChatCompletionResponse(BaseModel):
id: str = Field(..., description="响应ID")
object: str = Field("chat.completion", description="对象类型")
created: int = Field(..., description="创建时间戳")
model: str = Field(..., description="使用的模型")
choices: List[OpenAIChatCompletionChoice] = Field(..., description="响应选项")
model: str = Field(..., description="模型")
choices: List[OpenAIChatCompletionChoice] = Field(..., description="选项")
usage: Optional[Dict[str, Any]] = Field(None, description="令牌使用")
class OpenAIChatCompletionChunkMessage(BaseModel):
"""流式响应消息片段"""
"""流式消息片段"""
role: str = Field(..., description="角色")
content: str = Field(..., description="消息内容")
content: str = Field(..., description="内容")
class OpenAIChatCompletionChunkChoice(BaseModel):
"""流式响应选项"""
index: int = Field(..., description="选项索引")
"""流式选项"""
index: int = Field(..., description="索引")
delta: Optional[Union[Dict[str, Any], OpenAIChatCompletionChunkMessage]] = Field(
None, description="Delta数据"
)
@@ -106,10 +94,10 @@ class OpenAIChatCompletionChunkChoice(BaseModel):
class OpenAIChatCompletionChunkResponse(BaseModel):
"""流式聊天完成响应"""
"""流式聊天响应"""
id: str = Field(..., description="响应ID")
object: str = Field(default="chat.completion.chunk", description="对象类型")
created: int = Field(..., description="创建时间戳")
model: str = Field(..., description="使用的模型")
model: str = Field(..., description="模型")
system_fingerprint: Optional[str] = Field(default=None, description="系统指纹")
choices: List[OpenAIChatCompletionChunkChoice] = Field(..., description="响应选项")
choices: List[OpenAIChatCompletionChunkChoice] = Field(..., description="选项")
+120 -102
View File
@@ -1,9 +1,9 @@
"""缓存服务模块"""
"""缓存服务模块 - 提供图片和视频的下载、缓存和清理功能"""
import asyncio
import base64
from pathlib import Path
from typing import Optional
from typing import Optional, Tuple
from curl_cffi.requests import AsyncSession
from app.core.config import setting
@@ -11,170 +11,188 @@ from app.core.logger import logger
from app.services.grok.statsig import get_dynamic_headers
# 常量定义
MIME_TYPES = {
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png',
'.gif': 'image/gif', '.webp': 'image/webp', '.bmp': 'image/bmp',
}
DEFAULT_MIME = 'image/jpeg'
ASSETS_URL = "https://assets.grok.com"
class CacheService:
"""缓存服务基类"""
def __init__(self, cache_type: str):
"""初始化缓存服务"""
def __init__(self, cache_type: str, timeout: float = 30.0):
self.cache_type = cache_type
self.cache_dir = Path(f"data/temp/{cache_type}")
self.cache_dir.mkdir(parents=True, exist_ok=True)
self.timeout = timeout
self._cleanup_lock = asyncio.Lock()
def _cache_path(self, file_path: str) -> Path:
"""获取缓存文件的完整路径"""
filename = file_path.lstrip('/').replace('/', '-')
return self.cache_dir / filename
def _get_path(self, file_path: str) -> Path:
"""转换文件路径为缓存路径"""
return self.cache_dir / file_path.lstrip('/').replace('/', '-')
async def download_file(self, file_path: str, auth_token: str, timeout: float = 30.0) -> Optional[Path]:
def _log(self, level: str, msg: str):
"""统一日志输出"""
getattr(logger, level)(f"[{self.cache_type.upper()}Cache] {msg}")
def _build_headers(self, file_path: str, auth_token: str) -> dict:
"""构建请求头"""
cf = setting.grok_config.get("cf_clearance", "")
return {
**get_dynamic_headers(pathname=file_path),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "same-site",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"Referer": "https://grok.com/",
"Cookie": f"{auth_token};{cf}" if cf else auth_token
}
async def download(self, file_path: str, auth_token: str, timeout: Optional[float] = None) -> Optional[Path]:
"""下载并缓存文件"""
cache_path = self._cache_path(file_path)
cache_path = self._get_path(file_path)
if cache_path.exists():
logger.debug(f"[{self.cache_type.upper()}Cache] 文件缓存成功")
self._log("debug", "文件已缓存")
return cache_path
try:
cf_clearance = setting.grok_config.get("cf_clearance", "")
headers = {
**get_dynamic_headers(pathname=file_path),
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "same-site",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"Referer": "https://grok.com/",
"Cookie": f"{auth_token};{cf_clearance}" if cf_clearance else auth_token
}
# 使用缓存代理
proxy_url = setting.get_cache_proxy()
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else {}
proxy = setting.get_proxy("cache")
proxies = {"http": proxy, "https": proxy} if proxy else {}
if proxy_url:
logger.debug(f"[{self.cache_type.upper()}Cache] 使用缓存代理: {proxy_url.split('@')[-1] if '@' in proxy_url else proxy_url}")
if proxy:
self._log("debug", f"使用代理: {proxy.split('@')[-1] if '@' in proxy else proxy}")
async with AsyncSession() as session:
logger.debug(f"[{self.cache_type.upper()}Cache] 缓存生成视频文件: https://assets.grok.com{file_path}")
url = f"{ASSETS_URL}{file_path}"
self._log("debug", f"下载: {url}")
response = await session.get(
f"https://assets.grok.com{file_path}",
headers=headers,
url,
headers=self._build_headers(file_path, auth_token),
proxies=proxies,
timeout=timeout,
timeout=timeout or self.timeout,
allow_redirects=True,
impersonate="chrome133a"
)
response.raise_for_status()
cache_path.write_bytes(response.content)
logger.debug(f"[{self.cache_type.upper()}Cache] 文件缓存成功")
asyncio.create_task(self.cleanup_cache())
self._log("debug", "缓存成功")
# 异步清理(带错误处理)
asyncio.create_task(self._safe_cleanup())
return cache_path
except Exception as e:
logger.error(f"[{self.cache_type.upper()}Cache] 下载文件失败: {e}")
self._log("error", f"下载失败: {e}")
return None
def get_cached(self, file_path: str) -> Optional[Path]:
"""获取缓存的文件路径"""
cache_path = self._cache_path(file_path)
return cache_path if cache_path.exists() else None
"""获取已缓存的文件"""
path = self._get_path(file_path)
return path if path.exists() else None
async def cleanup_cache(self):
"""清理缓存目录,确保不超过配置的大小限制"""
async def _safe_cleanup(self):
"""安全清理(捕获异常)"""
try:
max_size_mb = setting.global_config.get(f"{self.cache_type}_cache_max_size_mb", 500)
max_size_bytes = max_size_mb * 1024 * 1024
files = [(fp, (stat := fp.stat()).st_size, stat.st_mtime)
for fp in self.cache_dir.glob("*") if fp.is_file()]
total_size = sum(size for _, size, _ in files)
if total_size <= max_size_bytes:
logger.debug(f"[{self.cache_type.upper()}Cache] 缓存大小统计: {total_size / 1024 / 1024:.2f}MB,未达到限制")
return
logger.info(f"[{self.cache_type.upper()}Cache] 清理缓存,当前大小 {total_size / 1024 / 1024:.2f}MB 超过限制 {max_size_mb}MB")
files.sort(key=lambda x: x[2])
for file_path, size, _ in files:
if total_size <= max_size_bytes:
break
file_path.unlink()
total_size -= size
logger.debug(f"[{self.cache_type.upper()}Cache] 清理缓存文件: {file_path}")
logger.info(f"[{self.cache_type.upper()}Cache] 清理缓存完成,当前大小 {total_size / 1024 / 1024:.2f}MB")
await self.cleanup()
except Exception as e:
logger.error(f"[{self.cache_type.upper()}Cache] 清理缓存失败: {e}")
self._log("error", f"后台清理失败: {e}")
async def cleanup(self):
"""清理超限缓存"""
if self._cleanup_lock.locked():
return
async with self._cleanup_lock:
try:
max_mb = setting.global_config.get(f"{self.cache_type}_cache_max_size_mb", 500)
max_bytes = max_mb * 1024 * 1024
# 获取文件信息 (path, size, mtime)
files = [(f, (s := f.stat()).st_size, s.st_mtime)
for f in self.cache_dir.glob("*") if f.is_file()]
total = sum(size for _, size, _ in files)
if total <= max_bytes:
return
self._log("info", f"清理缓存 {total/1024/1024:.1f}MB -> {max_mb}MB")
# 删除最旧的文件
for path, size, _ in sorted(files, key=lambda x: x[2]):
if total <= max_bytes:
break
path.unlink()
total -= size
self._log("info", f"清理完成: {total/1024/1024:.1f}MB")
except Exception as e:
self._log("error", f"清理失败: {e}")
class ImageCacheService(CacheService):
class ImageCache(CacheService):
"""图片缓存服务"""
def __init__(self):
super().__init__("image")
super().__init__("image", timeout=30.0)
async def download_image(self, image_path: str, auth_token: str) -> Optional[Path]:
"""下载并缓存图片"""
return await self.download_file(image_path, auth_token, timeout=30.0)
def get_cached(self, image_path: str) -> Optional[Path]:
"""获取缓存的图片路径"""
return super().get_cached(image_path)
async def download_image(self, path: str, token: str) -> Optional[Path]:
"""下载图片"""
return await self.download(path, token)
@staticmethod
def to_base64(image_path: Path) -> Optional[str]:
"""将图片转换为 base64 编码"""
"""图片转base64"""
try:
if not image_path.exists():
logger.error(f"[ImageCache] 图片文件不存在: {image_path}")
logger.error(f"[ImageCache] 文件不存在: {image_path}")
return None
with open(image_path, "rb") as f:
base64_data = base64.b64encode(f.read()).decode('utf-8')
mime_type = {'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png',
'.gif': 'image/gif', '.webp': 'image/webp'}.get(image_path.suffix.lower(), 'image/jpeg')
return f"data:{mime_type};base64,{base64_data}"
data = base64.b64encode(image_path.read_bytes()).decode()
mime = MIME_TYPES.get(image_path.suffix.lower(), DEFAULT_MIME)
return f"data:{mime};base64,{data}"
except Exception as e:
logger.error(f"[ImageCache] 图片转 base64 失败: {e}")
logger.error(f"[ImageCache] 转换失败: {e}")
return None
async def download_base64(self, image_path: str, auth_token: str) -> Optional[str]:
"""下载图片并转换为 base64 编码(转换后立即删除缓存文件)"""
async def download_base64(self, path: str, token: str) -> Optional[str]:
"""下载并转为base64(自动删除临时文件)"""
try:
cache_path = await self.download_file(image_path, auth_token, timeout=30.0)
cache_path = await self.download(path, token)
if not cache_path:
return None
base64_str = self.to_base64(cache_path)
result = self.to_base64(cache_path)
# 清理临时文件
try:
cache_path.unlink()
logger.debug(f"[ImageCache] 已删除临时文件: {cache_path}")
except Exception as e:
logger.warning(f"[ImageCache] 删除临时文件失败: {e}")
return base64_str
return result
except Exception as e:
logger.error(f"[ImageCache] 下载并转换 base64 失败: {e}")
logger.error(f"[ImageCache] 下载base64失败: {e}")
return None
class VideoCacheService(CacheService):
class VideoCache(CacheService):
"""视频缓存服务"""
def __init__(self):
super().__init__("video")
super().__init__("video", timeout=60.0)
async def download_video(self, video_path: str, auth_token: str) -> Optional[Path]:
"""下载并缓存视频"""
return await self.download_file(video_path, auth_token, timeout=60.0)
def get_cached(self, video_path: str) -> Optional[Path]:
"""获取缓存的视频路径"""
return super().get_cached(video_path)
async def download_video(self, path: str, token: str) -> Optional[Path]:
"""下载视频"""
return await self.download(path, token)
# 全局实例
image_cache_service = ImageCacheService()
video_cache_service = VideoCacheService()
image_cache_service = ImageCache()
video_cache_service = VideoCache()
+163 -200
View File
@@ -1,9 +1,8 @@
"""Grok API 客户端模块"""
"""Grok API 客户端 - 处理OpenAI到Grok的请求转换和响应处理"""
import asyncio
import json
from typing import Dict, List, Tuple, Any
import orjson
from typing import Dict, List, Tuple, Any, Optional
from curl_cffi import requests as curl_requests
from app.core.config import setting
@@ -16,136 +15,148 @@ from app.services.grok.upload import ImageUploadManager
from app.services.grok.create import PostCreateManager
from app.core.exception import GrokApiException
# 常量定义
GROK_API_ENDPOINT = "https://grok.com/rest/app-chat/conversations/new"
REQUEST_TIMEOUT = 120
IMPERSONATE_BROWSER = "chrome133a"
MAX_RETRY = 3 # 最大重试次数
# 常量
API_ENDPOINT = "https://grok.com/rest/app-chat/conversations/new"
TIMEOUT = 120
BROWSER = "chrome133a"
MAX_RETRY = 3
MAX_UPLOADS = 5
class GrokClient:
"""Grok API 客户端"""
_upload_sem = asyncio.Semaphore(MAX_UPLOADS)
@staticmethod
async def openai_to_grok(openai_request: dict):
"""转换OpenAI请求为Grok请求并处理响应"""
model = openai_request["model"]
messages = openai_request["messages"]
stream = openai_request.get("stream", False)
# 提取消息内容和图片URL
content, image_urls = GrokClient._extract_content(messages)
model_name, model_mode = Models.to_grok(model)
is_video_model = Models.get_model_info(model).get("is_video_model", False)
async def openai_to_grok(request: dict):
"""转换OpenAI请求为Grok请求"""
model = request["model"]
content, images = GrokClient._extract_content(request["messages"])
stream = request.get("stream", False)
# 视频模型特殊处理
if is_video_model:
if len(image_urls) > 1:
logger.warning(f"[Client] 视频模型只允许一张图片,当前有{len(image_urls)}张,只使用第一张")
image_urls = image_urls[:1]
# 重试逻辑
return await GrokClient._try(model, content, image_urls, model_name, model_mode, is_video_model, stream)
# 获取模型信息
info = Models.get_model_info(model)
grok_model, mode = Models.to_grok(model)
is_video = info.get("is_video_model", False)
# 视频模型限制
if is_video and len(images) > 1:
logger.warning(f"[Client] 视频模型仅支持1张图片,已截取前1张")
images = images[:1]
return await GrokClient._retry(model, content, images, grok_model, mode, is_video, stream)
@staticmethod
async def _try(model: str, content: str, image_urls: List[str], model_name: str, model_mode: str, is_video: bool, stream: bool):
"""带重试的请求执行"""
async def _retry(model: str, content: str, images: List[str], grok_model: str, mode: str, is_video: bool, stream: bool):
"""重试请求"""
last_err = None
for i in range(MAX_RETRY):
try:
# 获取token
auth_token = token_manager.get_token(model)
token = token_manager.get_token(model)
img_ids, img_uris = await GrokClient._upload(images, token)
# 上传图片
imgs, uris = await GrokClient._upload_imgs(image_urls, auth_token)
# 视频模型 - 创建会话
# 视频模型创建会话
post_id = None
if is_video and imgs and uris:
try:
create_result = await PostCreateManager.create(imgs[0], uris[0], auth_token)
if create_result and create_result.get("success"):
post_id = create_result.get("post_id")
else:
logger.warning(f"[Client] 创建会话失败,继续使用原有流程")
except Exception as e:
logger.warning(f"[Client] 创建会话异常: {e},继续使用原有流程")
if is_video and img_ids and img_uris:
post_id = await GrokClient._create_post(img_ids[0], img_uris[0], token)
# 构建并发送请求
payload = GrokClient._build_payload(content, model_name, model_mode, imgs, uris, is_video, post_id)
return await GrokClient._send_request(payload, auth_token, model, stream, post_id)
payload = GrokClient._build_payload(content, grok_model, mode, img_ids, img_uris, is_video, post_id)
return await GrokClient._request(payload, token, model, stream, post_id)
except GrokApiException as e:
last_err = e
# 401/429 可重试,其他错误直接抛出
# 仅401/429可重试
if e.error_code not in ["HTTP_ERROR", "NO_AVAILABLE_TOKEN"]:
raise
# 检查是否为可重试的状态码
status = e.context.get("status") if e.context else None
if status not in [401, 429]:
raise
if i < MAX_RETRY - 1:
logger.warning(f"[Client] 请求失败(状态码:{status}), 重试 {i+1}/{MAX_RETRY}")
await asyncio.sleep(0.5) # 短暂延迟
else:
logger.error(f"[Client] 重试{MAX_RETRY}次后仍失败")
logger.warning(f"[Client] 失败(状态:{status}), 重试 {i+1}/{MAX_RETRY}")
await asyncio.sleep(0.5)
raise last_err if last_err else GrokApiException("请求失败", "REQUEST_ERROR")
raise last_err or GrokApiException("请求失败", "REQUEST_ERROR")
@staticmethod
def _extract_content(messages: List[Dict]) -> Tuple[str, List[str]]:
"""提取消息内容和图片URL"""
content_parts = []
image_urls = []
"""提取文本和图片"""
texts, images = [], []
for msg in messages:
msg_content = msg.get("content", "")
# 处理复杂消息格式
if isinstance(msg_content, list):
for item in msg_content:
item_type = item.get("type")
if item_type == "text":
content_parts.append(item.get("text", ""))
elif item_type == "image_url":
url = item.get("image_url", {}).get("url", "")
if url:
image_urls.append(url)
# 处理纯文本消息
content = msg.get("content", "")
if isinstance(content, list):
for item in content:
if item.get("type") == "text":
texts.append(item.get("text", ""))
elif item.get("type") == "image_url":
if url := item.get("image_url", {}).get("url"):
images.append(url)
else:
content_parts.append(msg_content)
return "".join(content_parts), image_urls
texts.append(content)
return "".join(texts), images
@staticmethod
async def _upload_imgs(image_urls: List[str], auth_token: str) -> Tuple[List[str], List[str]]:
"""上传图片并返回附件ID列表"""
image_attachments = []
image_uris = []
# 并发上传所有图片
tasks = [ImageUploadManager.upload(url, auth_token) for url in image_urls]
results = await asyncio.gather(*tasks, return_exceptions=True)
for url, (file_id, file_uri) in zip(image_urls, results):
if isinstance(file_id, Exception):
logger.warning(f"[Client] 图片上传失败: {url}, 错误: {file_id}")
elif file_id:
image_attachments.append(file_id)
image_uris.append(file_uri)
return image_attachments, image_uris
async def _upload(urls: List[str], token: str) -> Tuple[List[str], List[str]]:
"""并发上传图片"""
if not urls:
return [], []
async def upload_limited(url):
async with GrokClient._upload_sem:
return await ImageUploadManager.upload(url, token)
results = await asyncio.gather(*[upload_limited(u) for u in urls], return_exceptions=True)
ids, uris = [], []
for url, result in zip(urls, results):
if isinstance(result, Exception):
logger.warning(f"[Client] 上传失败: {url} - {result}")
elif isinstance(result, tuple) and len(result) == 2:
fid, furi = result
if fid:
ids.append(fid)
uris.append(furi)
return ids, uris
@staticmethod
def _build_payload(content: str, model_name: str, model_mode: str, image_attachments: List[str], image_uris: List[str], is_video_model: bool = False, post_id: str = None) -> Dict[str, Any]:
"""构建Grok API请求载荷"""
payload = {
async def _create_post(file_id: str, file_uri: str, token: str) -> Optional[str]:
"""创建视频会话"""
try:
result = await PostCreateManager.create(file_id, file_uri, token)
if result and result.get("success"):
return result.get("post_id")
except Exception as e:
logger.warning(f"[Client] 创建会话失败: {e}")
return None
@staticmethod
def _build_payload(content: str, model: str, mode: str, img_ids: List[str], img_uris: List[str], is_video: bool = False, post_id: str = None) -> Dict:
"""构建请求载荷"""
# 视频模型特殊处理
if is_video and img_uris:
img_msg = f"https://grok.com/imagine/{post_id}" if post_id else f"https://assets.grok.com/post/{img_uris[0]}"
return {
"temporary": True,
"modelName": "grok-3",
"message": f"{img_msg} {content} --mode=custom",
"fileAttachments": img_ids,
"toolOverrides": {"videoGen": True}
}
# 标准载荷
return {
"temporary": setting.grok_config.get("temporary", True),
"modelName": model_name,
"modelName": model,
"message": content,
"fileAttachments": image_attachments,
"fileAttachments": img_ids,
"imageAttachments": [],
"disableSearch": False,
"enableImageGeneration": True,
@@ -160,144 +171,96 @@ class GrokClient:
"isReasoning": False,
"webpageUrls": [],
"disableTextFollowUps": True,
"responseMetadata": {"requestModelDetails": {"modelId": model_name}},
"responseMetadata": {"requestModelDetails": {"modelId": model}},
"disableMemory": False,
"forceSideBySide": False,
"modelMode": model_mode,
"modelMode": mode,
"isAsyncChat": False
}
# 视频模型配置
if is_video_model and image_uris:
image_url = image_uris[0]
# 构建 URL 消息
if post_id:
image_message = f"https://grok.com/imagine/{post_id} {content} --mode=custom"
else:
image_message = f"https://assets.grok.com/post/{image_url} {content} --mode=custom"
payload = {
"temporary": True,
"modelName": "grok-3",
"message": image_message,
"fileAttachments": image_attachments,
"toolOverrides": {"videoGen": True}
}
return payload
@staticmethod
async def _send_request(payload: dict, auth_token: str, model: str, stream: bool, post_id: str = None):
"""发送HTTP请求到Grok API"""
# 验证认证令牌
if not auth_token:
async def _request(payload: dict, token: str, model: str, stream: bool, post_id: str = None):
"""发送请求"""
if not token:
raise GrokApiException("认证令牌缺失", "NO_AUTH_TOKEN")
try:
# 构建请求头
headers = GrokClient._build_headers(auth_token)
# 构建请求
headers = GrokClient._build_headers(token)
if model == "grok-imagine-0.9":
# 传入会话ID
file_attachments = payload.get("fileAttachments", [])
referer_id = post_id if post_id else (file_attachments[0] if file_attachments else "")
if referer_id:
headers["Referer"] = f"https://grok.com/imagine/{referer_id}"
ref_id = post_id or payload.get("fileAttachments", [""])[0]
if ref_id:
headers["Referer"] = f"https://grok.com/imagine/{ref_id}"
# 使用服务代理
proxy_url = setting.get_service_proxy()
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
proxy = setting.get_proxy("service")
proxies = {"http": proxy, "https": proxy} if proxy else None
# 构建请求参数
request_kwargs = {
"headers": headers,
"data": json.dumps(payload),
"impersonate": IMPERSONATE_BROWSER,
"timeout": REQUEST_TIMEOUT,
"stream": True,
"proxies": proxies
}
# 在线程池中执行同步HTTP请求,避免阻塞事件循环
# 执行请求
response = await asyncio.to_thread(
curl_requests.post,
GROK_API_ENDPOINT,
**request_kwargs
API_ENDPOINT,
headers=headers,
data=orjson.dumps(payload),
impersonate=BROWSER,
timeout=TIMEOUT,
stream=True,
proxies=proxies
)
# 处理非成功响应
if response.status_code != 200:
GrokClient._handle_error(response, auth_token)
# 请求成功,重置失败计数
asyncio.create_task(token_manager.reset_failure(auth_token))
# 处理并返回响应
return await GrokClient._process_response(response, auth_token, model, stream)
GrokClient._handle_error(response, token)
# 成功 - 重置失败计数
asyncio.create_task(token_manager.reset_failure(token))
# 处理响应
result = (GrokResponseProcessor.process_stream(response, token) if stream
else await GrokResponseProcessor.process_normal(response, token, model))
asyncio.create_task(GrokClient._update_limits(token, model))
return result
except curl_requests.RequestsError as e:
logger.error(f"[Client] 网络请求错误: {e}")
logger.error(f"[Client] 网络错误: {e}")
raise GrokApiException(f"网络错误: {e}", "NETWORK_ERROR") from e
except json.JSONDecodeError as e:
logger.error(f"[Client] JSON解析错误: {e}")
raise GrokApiException(f"JSON解析错误: {e}", "JSON_ERROR") from e
except Exception as e:
logger.error(f"[Client] 未知请求错误: {type(e).__name__}: {e}")
raise GrokApiException(f"请求处理错误: {e}", "REQUEST_ERROR") from e
logger.error(f"[Client] 请求错误: {e}")
raise GrokApiException(f"请求错误: {e}", "REQUEST_ERROR") from e
@staticmethod
def _build_headers(auth_token: str) -> Dict[str, str]:
def _build_headers(token: str) -> Dict[str, str]:
"""构建请求头"""
headers = get_dynamic_headers("/rest/app-chat/conversations/new")
# 构建Cookie
cf_clearance = setting.grok_config.get("cf_clearance", "")
headers["Cookie"] = f"{auth_token};{cf_clearance}" if cf_clearance else auth_token
cf = setting.grok_config.get("cf_clearance", "")
headers["Cookie"] = f"{token};{cf}" if cf else token
return headers
@staticmethod
def _handle_error(response, auth_token: str):
"""处理错误响应"""
# 处理 403 错误
def _handle_error(response, token: str):
"""处理错误"""
if response.status_code == 403:
error_message = "服务器IP被Block,请尝试 1. 更换服务器IP 2. 使用代理IP 3. 服务器登陆Grok.com,过盾后F12找到CF值填入后台设置"
error_data = {"cf_blocked": True, "status": 403}
logger.warning(f"[Client] {error_message}")
msg = "您的IP被拦截,请尝试以下方法之一: 1.更换IP 2.使用代理 3.配置CF值"
data = {"cf_blocked": True, "status": 403}
logger.warning(f"[Client] {msg}")
else:
# 其他错误尝试解析 JSON
try:
error_data = response.json()
error_message = str(error_data)
except Exception as e:
error_data = response.text
error_message = error_data[:200] if error_data else str(e)
# 记录Token失败
asyncio.create_task(token_manager.record_failure(auth_token, response.status_code, error_message))
data = response.json()
msg = str(data)
except:
data = response.text
msg = data[:200] if data else "未知错误"
asyncio.create_task(token_manager.record_failure(token, response.status_code, msg))
raise GrokApiException(
f"请求失败: {response.status_code} - {error_message}",
f"请求失败: {response.status_code} - {msg}",
"HTTP_ERROR",
{"status": response.status_code, "data": error_data}
{"status": response.status_code, "data": data}
)
@staticmethod
async def _process_response(response, auth_token: str, model: str, stream: bool):
"""处理API响应"""
if stream:
result = GrokResponseProcessor.process_stream(response, auth_token)
asyncio.create_task(GrokClient._update_rate_limits(auth_token, model))
else:
result = await GrokResponseProcessor.process_normal(response, auth_token, model)
asyncio.create_task(GrokClient._update_rate_limits(auth_token, model))
return result
@staticmethod
async def _update_rate_limits(auth_token: str, model: str):
"""异步更新速率限制信息"""
async def _update_limits(token: str, model: str):
"""更新速率限制"""
try:
await token_manager.check_limits(auth_token, model)
await token_manager.check_limits(token, model)
except Exception as e:
logger.error(f"[Client] 更新速率限制失败: {e}")
logger.error(f"[Client] 更新限制失败: {e}")
+49 -59
View File
@@ -1,6 +1,6 @@
"""Post创建管理器"""
"""Post创建管理器 - 用于视频生成前的会话创建"""
import json
import orjson
from typing import Dict, Any, Optional
from curl_cffi.requests import AsyncSession
@@ -9,75 +9,65 @@ from app.core.exception import GrokApiException
from app.core.config import setting
from app.core.logger import logger
# 常量定义
CREATE_ENDPOINT = "https://grok.com/rest/media/post/create"
REQUEST_TIMEOUT = 30
IMPERSONATE_BROWSER = "chrome133a"
# 常量
ENDPOINT = "https://grok.com/rest/media/post/create"
TIMEOUT = 30
BROWSER = "chrome133a"
class PostCreateManager:
"""
Grok 会话创建管理器
提供图片会话创建功能,用于视频生成前的准备工作
"""
"""会话创建管理器"""
@staticmethod
async def create(file_id: str, file_uri: str, auth_token: str) -> Optional[Dict[str, Any]]:
"""
创建会话记录
"""创建会话记录
Args:
file_id: 上传后的文件ID
file_uri: 上传后的文件URI
file_id: 文件ID
file_uri: 文件URI
auth_token: 认证令牌
Returns:
创建的会话信息,包含会话ID等
会话信息字典,包含post_id等
"""
try:
# 验证参数
if not file_id or not file_uri:
raise GrokApiException("会话ID或URI缺失", "INVALID_PARAMS")
if not auth_token:
raise GrokApiException("认证令牌缺失", "NO_AUTH_TOKEN")
# 参数验证
if not file_id or not file_uri:
raise GrokApiException("文件ID或URI缺失", "INVALID_PARAMS")
if not auth_token:
raise GrokApiException("认证令牌缺失", "NO_AUTH_TOKEN")
# 构建创建数据
media_url = f"https://assets.grok.com/{file_uri}"
create_data = {
"media_url": media_url,
try:
# 构建请求
data = {
"media_url": f"https://assets.grok.com/{file_uri}",
"media_type": "MEDIA_POST_TYPE_IMAGE"
}
# 获取认证令牌和cookie
cf_clearance = setting.grok_config.get("cf_clearance", "")
cookie = f"{auth_token};{cf_clearance}" if cf_clearance else auth_token
# 获取代理配置
proxy_url = setting.grok_config.get("proxy_url", "")
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
cf = setting.grok_config.get("cf_clearance", "")
headers = {
**get_dynamic_headers("/rest/media/post/create"),
"Cookie": f"{auth_token};{cf}" if cf else auth_token
}
proxy = setting.grok_config.get("proxy_url", "")
proxies = {"http": proxy, "https": proxy} if proxy else None
# 发送异步请求
# 发送请求
async with AsyncSession() as session:
response = await session.post(
CREATE_ENDPOINT,
headers={
**get_dynamic_headers("/rest/media/post/create"),
"Cookie": cookie,
},
json=create_data,
impersonate=IMPERSONATE_BROWSER,
timeout=REQUEST_TIMEOUT,
proxies=proxies,
ENDPOINT,
headers=headers,
json=data,
impersonate=BROWSER,
timeout=TIMEOUT,
proxies=proxies
)
# 检查响应
if response.status_code == 200:
result = response.json()
post_id = result.get("post", {}).get("id", "")
logger.debug(f"[PostCreate] 创建会话成功,会话ID: {post_id}")
logger.debug(f"[PostCreate] 成功,会话ID: {post_id}")
return {
"post_id": post_id,
"file_id": file_id,
@@ -85,19 +75,19 @@ class PostCreateManager:
"success": True,
"data": result
}
else:
error_msg = f"状态码: {response.status_code}"
try:
error_data = response.json()
error_msg = f"{error_msg}, 详情: {error_data}"
except:
error_msg = f"{error_msg}, 详情: {response.text[:200]}"
logger.error(f"[PostCreate] 创建会话失败: {error_msg}")
raise GrokApiException(f"创建会话失败: {error_msg}", "CREATE_ERROR")
# 错误处理
try:
error = response.json()
msg = f"状态码: {response.status_code}, 详情: {error}"
except:
msg = f"状态码: {response.status_code}, 详情: {response.text[:200]}"
logger.error(f"[PostCreate] 失败: {msg}")
raise GrokApiException(f"创建失败: {msg}", "CREATE_ERROR")
except GrokApiException:
raise
except Exception as e:
logger.error(f"[PostCreate] 创建会话异常: {e}")
raise GrokApiException(f"创建会话异常: {e}", "CREATE_ERROR") from e
logger.error(f"[PostCreate] 异常: {e}")
raise GrokApiException(f"创建异常: {e}", "CREATE_ERROR") from e
+156 -247
View File
@@ -1,10 +1,10 @@
"""Grok API 响应处理器模块"""
"""Grok API 响应处理器 - 处理流式和非流式响应"""
import json
import orjson
import uuid
import time
import asyncio
from typing import AsyncGenerator
from typing import AsyncGenerator, Tuple
from app.core.config import setting
from app.core.exception import GrokApiException
@@ -21,61 +21,43 @@ from app.services.grok.cache import image_cache_service, video_cache_service
class StreamTimeoutManager:
"""流式响应超时管理器"""
"""流式响应超时管理"""
def __init__(self, chunk_timeout: int = 120, first_response_timeout: int = 30, total_timeout: int = 600):
"""初始化超时管理器
Args:
chunk_timeout: 数据块间隔超时(秒)
first_response_timeout: 首次响应超时(秒)
total_timeout: 总超时限制(秒,0表示不限制)
"""
def __init__(self, chunk_timeout: int = 120, first_timeout: int = 30, total_timeout: int = 600):
self.chunk_timeout = chunk_timeout
self.first_response_timeout = first_response_timeout
self.first_timeout = first_timeout
self.total_timeout = total_timeout
self.start_time = asyncio.get_event_loop().time()
self.last_chunk_time = self.start_time
self.first_chunk_received = False
self.first_received = False
def check_timeout(self) -> tuple[bool, str]:
"""检查是否超时
def check_timeout(self) -> Tuple[bool, str]:
"""检查超时"""
now = asyncio.get_event_loop().time()
Returns:
(is_timeout, timeout_message): 是否超时及超时信息
"""
current_time = asyncio.get_event_loop().time()
if not self.first_received and now - self.start_time > self.first_timeout:
return True, f"首次响应超时({self.first_timeout}秒)"
# 检查首次响应超时
if not self.first_chunk_received:
if current_time - self.start_time > self.first_response_timeout:
return True, f"首次响应超时 ({self.first_response_timeout}秒未收到首个数据块)"
if self.total_timeout > 0 and now - self.start_time > self.total_timeout:
return True, f"总超时({self.total_timeout}秒)"
# 检查总超时
if self.total_timeout > 0:
if current_time - self.start_time > self.total_timeout:
return True, f"流式响应总超时 ({self.total_timeout}秒)"
# 检查数据块间隔超时
if self.first_chunk_received:
if current_time - self.last_chunk_time > self.chunk_timeout:
return True, f"数据块间隔超时 ({self.chunk_timeout}秒无新数据)"
if self.first_received and now - self.last_chunk_time > self.chunk_timeout:
return True, f"数据块超时({self.chunk_timeout}秒)"
return False, ""
def mark_chunk_received(self):
"""标记收到数据块"""
def mark_received(self):
"""标记收到数据"""
self.last_chunk_time = asyncio.get_event_loop().time()
self.first_chunk_received = True
self.first_received = True
def get_total_duration(self) -> float:
"""获取总耗时(秒)"""
def duration(self) -> float:
"""获取总耗时"""
return asyncio.get_event_loop().time() - self.start_time
class GrokResponseProcessor:
"""Grok API 响应处理器"""
"""Grok响应处理器"""
@staticmethod
async def process_normal(response, auth_token: str, model: str = None) -> OpenAIChatCompletionResponse:
@@ -86,7 +68,7 @@ class GrokResponseProcessor:
if not chunk:
continue
data = json.loads(chunk.decode("utf-8"))
data = orjson.loads(chunk)
# 错误检查
if error := data.get("error"):
@@ -96,133 +78,57 @@ class GrokResponseProcessor:
{"code": error.get("code")}
)
# 提取响应数据
grok_resp = data.get("result", {}).get("response", {})
# 提取视频数据
# 视频响应
if video_resp := grok_resp.get("streamingVideoGenerationResponse"):
if video_url := video_resp.get("videoUrl"):
logger.debug(f"[Processor] 检测到视频生成: {video_url}")
full_video_url = f"https://assets.grok.com/{video_url}"
# 下载并缓存视频
try:
cache_path = await video_cache_service.download_video(f"/{video_url}", auth_token)
if cache_path:
video_path = video_url.replace('/', '-')
base_url = setting.global_config.get("base_url", "")
local_video_url = f"{base_url}/images/{video_path}" if base_url else f"/images/{video_path}"
content = f'<video src="{local_video_url}" controls="controls" width="500" height="300"></video>\n'
else:
content = f'<video src="{full_video_url}" controls="controls" width="500" height="300"></video>\n'
except Exception as e:
logger.warning(f"[Processor] 缓存视频失败: {e}")
content = f'<video src="{full_video_url}" controls="controls" width="500" height="300"></video>\n'
# 返回视频响应
result = OpenAIChatCompletionResponse(
id=f"chatcmpl-{uuid.uuid4()}",
object="chat.completion",
created=int(time.time()),
model=model or "grok-imagine-0.9",
choices=[OpenAIChatCompletionChoice(
index=0,
message=OpenAIChatCompletionMessage(
role="assistant",
content=content
),
finish_reason="stop"
)],
usage=None
)
content = await GrokResponseProcessor._build_video_content(video_url, auth_token)
result = GrokResponseProcessor._build_response(content, model or "grok-imagine-0.9")
response_closed = True
response.close()
return result
# 提取模型响应
# 模型响应
model_response = grok_resp.get("modelResponse")
if not model_response:
continue
# 检查 modelResponse 中的错误
if error_msg := model_response.get("error"):
raise GrokApiException(
f"模型响应错误: {error_msg}",
"MODEL_ERROR"
)
raise GrokApiException(f"模型错误: {error_msg}", "MODEL_ERROR")
# 构建响应内容
model_name = model_response.get("model")
# 构建内容
content = model_response.get("message", "")
model_name = model_response.get("model")
# 提取图片数据
# 处理图片
if images := model_response.get("generatedImageUrls"):
# 获取图片返回模式
image_mode = setting.global_config.get("image_mode", "url")
content = await GrokResponseProcessor._append_images(content, images, auth_token)
for img in images:
try:
if image_mode == "base64":
# base64 模式:下载并转换为 base64
base64_str = await image_cache_service.download_base64(f"/{img}", auth_token)
if base64_str:
content += f"\n![Generated Image]({base64_str})"
else:
content += f"\n![Generated Image](https://assets.grok.com/{img})"
else:
# url 模式:缓存并返回链接
cache_path = await image_cache_service.download_image(f"/{img}", auth_token)
if cache_path:
img_path = img.replace('/', '-')
base_url = setting.global_config.get("base_url", "")
img_url = f"{base_url}/images/{img_path}" if base_url else f"/images/{img_path}"
content += f"\n![Generated Image]({img_url})"
else:
content += f"\n![Generated Image](https://assets.grok.com/{img})"
except Exception as e:
logger.warning(f"[Processor] 处理图片失败: {e}")
content += f"\n![Generated Image](https://assets.grok.com/{img})"
# 返回 OpenAI 响应格式
result = OpenAIChatCompletionResponse(
id=f"chatcmpl-{uuid.uuid4()}",
object="chat.completion",
created=int(time.time()),
model=model_name,
choices=[OpenAIChatCompletionChoice(
index=0,
message=OpenAIChatCompletionMessage(
role="assistant",
content=content
),
finish_reason="stop"
)],
usage=None
)
result = GrokResponseProcessor._build_response(content, model_name)
response_closed = True
response.close()
return result
raise GrokApiException("无响应数据", "NO_RESPONSE")
except json.JSONDecodeError as e:
except orjson.JSONDecodeError as e:
logger.error(f"[Processor] JSON解析失败: {e}")
raise GrokApiException(f"JSON解析失败: {e}", "JSON_ERROR") from e
except Exception as e:
logger.error(f"[Processor] 处理响应时发生未知错误: {type(e).__name__}: {e}")
logger.error(f"[Processor] 处理错误: {type(e).__name__}: {e}")
raise GrokApiException(f"响应处理错误: {e}", "PROCESS_ERROR") from e
finally:
# 确保响应对象被关闭,避免双重释放
if not response_closed and hasattr(response, 'close'):
try:
response.close()
except Exception as e:
logger.warning(f"[Processor] 关闭响应对象时出错: {e}")
logger.warning(f"[Processor] 关闭响应失败: {e}")
@staticmethod
async def process_stream(response, auth_token: str) -> AsyncGenerator[str, None]:
"""处理流式响应"""
# 流式生成状态
# 状态变量
is_image = False
is_thinking = False
thinking_finished = False
@@ -234,15 +140,15 @@ class GrokResponseProcessor:
response_closed = False
show_thinking = setting.grok_config.get("show_thinking", True)
# 初始化超时管理器
timeout_manager = StreamTimeoutManager(
# 超时管理
timeout_mgr = StreamTimeoutManager(
chunk_timeout=setting.grok_config.get("stream_chunk_timeout", 120),
first_response_timeout=setting.grok_config.get("stream_first_response_timeout", 30),
first_timeout=setting.grok_config.get("stream_first_response_timeout", 30),
total_timeout=setting.grok_config.get("stream_total_timeout", 600)
)
def make_chunk(chunk_content: str, finish: str = None):
"""生成OpenAI格式的响应块"""
def make_chunk(content: str, finish: str = None):
"""生成响应块"""
chunk_data = OpenAIChatCompletionChunkResponse(
id=f"chatcmpl-{uuid.uuid4()}",
created=int(time.time()),
@@ -251,275 +157,278 @@ class GrokResponseProcessor:
index=chunk_index,
delta=OpenAIChatCompletionChunkMessage(
role="assistant",
content=chunk_content
) if chunk_content else {},
content=content
) if content else {},
finish_reason=finish
)]
).model_dump()
# SSE 格式返回
return f"data: {json.dumps(chunk_data)}\n\n"
)
return f"data: {chunk_data.model_dump_json()}\n\n"
try:
for chunk in response.iter_lines():
# 超时检查
is_timeout, timeout_msg = timeout_manager.check_timeout()
is_timeout, timeout_msg = timeout_mgr.check_timeout()
if is_timeout:
logger.warning(f"[Processor] {timeout_msg}")
yield make_chunk("", "stop")
yield "data: [DONE]\n\n"
return
logger.debug(f"[Processor] 接收到数据块, 长度: {len(chunk)} bytes")
logger.debug(f"[Processor] 收到数据块: {len(chunk)} bytes")
if not chunk:
continue
try:
data = json.loads(chunk.decode("utf-8"))
data = orjson.loads(chunk)
# 错误检查
if error := data.get("error"):
error_msg = error.get('message', '未知错误')
logger.error(f"[Processor] Grok API返回错误: {error_msg}")
logger.error(f"[Processor] API错误: {error_msg}")
yield make_chunk(f"Error: {error_msg}", "stop")
yield "data: [DONE]\n\n"
return
# 提取响应数据
grok_resp = data.get("result", {}).get("response", {})
logger.debug(f"[Processor] 解析响应数据, 长度: {len(grok_resp)} bytes")
logger.debug(f"[Processor] 解析响应: {len(grok_resp)} bytes")
if not grok_resp:
continue
# 更新超时计时器
timeout_manager.mark_chunk_received()
timeout_mgr.mark_received()
# 更新模型名称
# 更新模型
if user_resp := grok_resp.get("userResponse"):
if m := user_resp.get("model"):
model = m
# 提取视频数据
# 视频处理
if video_resp := grok_resp.get("streamingVideoGenerationResponse"):
progress = video_resp.get("progress", 0)
v_url = video_resp.get("videoUrl")
# 处理进度更新
# 进度更新
if progress > last_video_progress:
last_video_progress = progress
# 思考状态
if show_thinking:
# 添加 <think> 标签
if not video_progress_started:
content = f"<think>视频已生成{progress}%\n"
content = f"<think>视频已生成{progress}%\\n"
video_progress_started = True
elif progress < 100:
content = f"视频已生成{progress}%\n"
content = f"视频已生成{progress}%\\n"
else:
# 关闭 <think> 标签
content = f"视频已生成{progress}%</think>\n"
content = f"视频已生成{progress}%</think>\\n"
yield make_chunk(content)
chunk_index += 1
# 处理视频URL
# 视频URL
if v_url:
logger.debug(f"[Processor] 视频生成完成")
full_video_url = f"https://assets.grok.com/{v_url}"
try:
cache_path = await video_cache_service.download_video(f"/{v_url}", auth_token)
if cache_path:
video_path = v_url.replace('/', '-')
base_url = setting.global_config.get("base_url", "")
local_video_url = f"{base_url}/images/{video_path}" if base_url else f"/images/{video_path}"
video_content = f'<video src="{local_video_url}" controls="controls"></video>\n'
else:
video_content = f'<video src="{full_video_url}" controls="controls"></video>\n'
except Exception as e:
logger.warning(f"[Processor] 缓存视频失败: {e}")
video_content = f'<video src="{full_video_url}" controls="controls"></video>\n'
logger.debug("[Processor] 视频生成完成")
video_content = await GrokResponseProcessor._build_video_content(v_url, auth_token)
yield make_chunk(video_content)
chunk_index += 1
continue
# 检查生成模式
# 图片模式
if grok_resp.get("imageAttachmentInfo"):
is_image = True
# 获取token
token = grok_resp.get("token", "")
# 提取图片数据
# 图片处理
if is_image:
if model_resp := grok_resp.get("modelResponse"):
# 获取图片返回模式
image_mode = setting.global_config.get("image_mode", "url")
# 初始化内容变量
content = ""
# 生成图片链接并缓存
for img in model_resp.get("generatedImageUrls", []):
try:
if image_mode == "base64":
# base64 模式:下载并转换为 base64
# Base64模式 - 分块发送
base64_str = await image_cache_service.download_base64(f"/{img}", auth_token)
if base64_str:
# 分块发送 base64 数据,每 8KB 一个 chunk
markdown_prefix = "![Generated Image](data:"
markdown_suffix = ")\n"
# 提取 data URL 的 mime 和 base64 部分
# 分块发送大数据
if not base64_str.startswith("data:"):
parts = base64_str.split(",", 1)
if len(parts) == 2:
mime_part = parts[0] + ","
b64_data = parts[1]
# 发送前缀
yield make_chunk(markdown_prefix + mime_part)
yield make_chunk(f"![Generated Image](data:{parts[0]},")
chunk_index += 1
# 分块发送 base64 数据
chunk_size = 8192
for i in range(0, len(b64_data), chunk_size):
chunk_data = b64_data[i:i + chunk_size]
yield make_chunk(chunk_data)
# 8KB分块
for i in range(0, len(parts[1]), 8192):
yield make_chunk(parts[1][i:i+8192])
chunk_index += 1
# 发送后缀
yield make_chunk(markdown_suffix)
yield make_chunk(")\\n")
chunk_index += 1
else:
yield make_chunk(f"![Generated Image]({base64_str})\n")
yield make_chunk(f"![Generated Image]({base64_str})\\n")
chunk_index += 1
else:
yield make_chunk(f"![Generated Image]({base64_str})\n")
yield make_chunk(f"![Generated Image]({base64_str})\\n")
chunk_index += 1
else:
yield make_chunk(f"![Generated Image](https://assets.grok.com/{img})\n")
yield make_chunk(f"![Generated Image](https://assets.grok.com/{img})\\n")
chunk_index += 1
else:
# url 模式:缓存并返回链接
# URL模式
await image_cache_service.download_image(f"/{img}", auth_token)
# 本地图片路径
img_path = img.replace('/', '-')
base_url = setting.global_config.get("base_url", "")
img_url = f"{base_url}/images/{img_path}" if base_url else f"/images/{img_path}"
content += f"![Generated Image]({img_url})\n"
content += f"![Generated Image]({img_url})\\n"
except Exception as e:
logger.warning(f"[Processor] 处理图片失败: {e}")
content += f"![Generated Image](https://assets.grok.com/{img})\n"
content += f"![Generated Image](https://assets.grok.com/{img})\\n"
# 发送内容
yield make_chunk(content.strip(), "stop")
return
elif token:
yield make_chunk(token)
chunk_index += 1
# 提取对话数据
# 对话处理
else:
# 过滤 list 格式的 token
if isinstance(token, list):
continue
# 过滤特定标签
if any(tag in token for tag in filtered_tags if token):
continue
# 获取当前状态
current_is_thinking = grok_resp.get("isThinking", False)
message_tag = grok_resp.get("messageTag")
# 跳过后续的 <think> 标签
if thinking_finished and current_is_thinking:
continue
# 检查 toolUsageCardId
# 搜索结果处理
if grok_resp.get("toolUsageCardId"):
if web_search := grok_resp.get("webSearchResults"):
if current_is_thinking:
if show_thinking:
# 封装搜索结果
for result in web_search.get("results", []):
title = result.get("title", "")
url = result.get("url", "")
preview = result.get("preview", "")
preview_clean = preview.replace("\n", "") if isinstance(preview, str) else ""
token += f'\n- [{title}]({url} "{preview_clean}")'
token += "\n"
preview_clean = preview.replace("\\n", "") if isinstance(preview, str) else ""
token += f'\\n- [{title}]({url} "{preview_clean}")'
token += "\\n"
else:
# show_thinking=false 时跳过 thinking 状态下的搜索结果
continue
else:
# 有 webSearchResults 但 isThinking 为 false
continue
else:
# 没有 webSearchResults
continue
if token:
content = token
# header 在 token 后换行
if message_tag == "header":
content = f"\n\n{token}\n\n"
# is_thinking 状态切换
# Thinking状态切换
should_skip = False
if not is_thinking and current_is_thinking:
# 进入 thinking 状态
if show_thinking:
content = f"<think>\n{content}"
content = f"<think>\\n{content}"
else:
should_skip = True
elif is_thinking and not current_is_thinking:
# 退出 thinking 状态
if show_thinking:
content = f"\n</think>\n{content}"
content = f"\\n</think>\\n{content}"
thinking_finished = True
elif current_is_thinking:
# 处于 thinking 状态中
if not show_thinking:
should_skip = True
# 只在不需要跳过时才发送
if not should_skip:
yield make_chunk(content)
chunk_index += 1
is_thinking = current_is_thinking
except (json.JSONDecodeError, UnicodeDecodeError) as e:
logger.warning(f"[Processor] 解析chunk失败: {e}")
except (orjson.JSONDecodeError, UnicodeDecodeError) as e:
logger.warning(f"[Processor] 解析失败: {e}")
continue
except Exception as e:
logger.warning(f"[Processor] 处理chunk出错: {e}")
logger.warning(f"[Processor] 处理出错: {e}")
continue
# 发送结束块
yield make_chunk("", "stop")
# 发送流结束标记
yield "data: [DONE]\n\n"
# 记录流式响应统计
logger.info(f"[Processor] 流式响应完成,总耗时: {timeout_manager.get_total_duration():.2f}秒")
logger.info(f"[Processor] 流式完成,耗时: {timeout_mgr.duration():.2f}秒")
except Exception as e:
logger.error(f"[Processor] 流式处理发生严重错误: {e}")
logger.error(f"[Processor] 严重错误: {e}")
yield make_chunk(f"处理错误: {e}", "error")
# 发送流结束标记
yield "data: [DONE]\n\n"
finally:
# 确保响应对象被关闭
if not response_closed and hasattr(response, 'close'):
try:
response.close()
logger.debug("[Processor] 流式响应对象已关闭")
logger.debug("[Processor] 响应已关闭")
except Exception as e:
logger.warning(f"[Processor] 关闭流式响应对象时出错: {e}")
logger.warning(f"[Processor] 关闭失败: {e}")
@staticmethod
async def _build_video_content(video_url: str, auth_token: str) -> str:
"""构建视频内容"""
logger.debug(f"[Processor] 检测到视频: {video_url}")
full_url = f"https://assets.grok.com/{video_url}"
try:
cache_path = await video_cache_service.download_video(f"/{video_url}", auth_token)
if cache_path:
video_path = video_url.replace('/', '-')
base_url = setting.global_config.get("base_url", "")
local_url = f"{base_url}/images/{video_path}" if base_url else f"/images/{video_path}"
return f'<video src="{local_url}" controls="controls" width="500" height="300"></video>\\n'
except Exception as e:
logger.warning(f"[Processor] 缓存视频失败: {e}")
return f'<video src="{full_url}" controls="controls" width="500" height="300"></video>\\n'
@staticmethod
async def _append_images(content: str, images: list, auth_token: str) -> str:
"""追加图片到内容"""
image_mode = setting.global_config.get("image_mode", "url")
for img in images:
try:
if image_mode == "base64":
base64_str = await image_cache_service.download_base64(f"/{img}", auth_token)
if base64_str:
content += f"\\n![Generated Image]({base64_str})"
else:
content += f"\\n![Generated Image](https://assets.grok.com/{img})"
else:
cache_path = await image_cache_service.download_image(f"/{img}", auth_token)
if cache_path:
img_path = img.replace('/', '-')
base_url = setting.global_config.get("base_url", "")
img_url = f"{base_url}/images/{img_path}" if base_url else f"/images/{img_path}"
content += f"\\n![Generated Image]({img_url})"
else:
content += f"\\n![Generated Image](https://assets.grok.com/{img})"
except Exception as e:
logger.warning(f"[Processor] 处理图片失败: {e}")
content += f"\\n![Generated Image](https://assets.grok.com/{img})"
return content
@staticmethod
def _build_response(content: str, model: str) -> OpenAIChatCompletionResponse:
"""构建响应对象"""
return OpenAIChatCompletionResponse(
id=f"chatcmpl-{uuid.uuid4()}",
object="chat.completion",
created=int(time.time()),
model=model,
choices=[OpenAIChatCompletionChoice(
index=0,
message=OpenAIChatCompletionMessage(
role="assistant",
content=content
),
finish_reason="stop"
)],
usage=None
)
+44 -68
View File
@@ -1,4 +1,4 @@
"""Grok 请求头管理模块"""
"""Grok 请求头管理 - 生成动态请求头和Statsig ID"""
import base64
import random
@@ -10,95 +10,71 @@ from app.core.logger import logger
from app.core.config import setting
def _generate_random_string(length: int, use_letters: bool = True) -> str:
"""生成随机字符串
Args:
length: 字符串长度
use_letters: 是否使用字母(True)或数字+字母(False)
Returns:
随机字符串
"""
if use_letters:
# 生成随机字母(小写)
return ''.join(random.choices(string.ascii_lowercase, k=length))
else:
# 生成随机数字和字母组合(小写)
return ''.join(random.choices(string.ascii_lowercase + string.digits, k=length))
# 基础请求头
BASE_HEADERS = {
"Accept": "*/*",
"Accept-Language": "zh-CN,zh;q=0.9",
"Accept-Encoding": "gzip, deflate, br, zstd",
"Connection": "keep-alive",
"Origin": "https://grok.com",
"Priority": "u=1, i",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
"Sec-Ch-Ua": '"Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"',
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": '"macOS"',
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
"Baggage": "sentry-environment=production,sentry-public_key=b311e0f2690c81f25e2c4cf6d4f7ce1c",
}
def _random_string(length: int, letters_only: bool = True) -> str:
"""生成随机字符串"""
chars = string.ascii_lowercase if letters_only else string.ascii_lowercase + string.digits
return ''.join(random.choices(chars, k=length))
def _generate_statsig_id() -> str:
"""动态生成 x-statsig-id
"""生成x-statsig-id
随机选择两种格式之一:
随机选择两种格式:
1. e:TypeError: Cannot read properties of null (reading 'children['xxxxx']')
其中 xxxxx 是5位随机字符串
2. e:TypeError: Cannot read properties of undefined (reading 'xxxxxxxxxx')
其中 xxxxxxxxxx 是10位随机字母
Returns:
base64 编码后的字符串
"""
# 随机选择一种格式
format_type = random.choice([1, 2])
if format_type == 1:
# 格式1: children['xxxxx']
random_str = _generate_random_string(5, use_letters=False)
error_msg = f"e:TypeError: Cannot read properties of null (reading 'children['{random_str}']')"
if random.choice([True, False]):
rand = _random_string(5, letters_only=False)
msg = f"e:TypeError: Cannot read properties of null (reading 'children['{rand}']')"
else:
# 格式2: 'xxxxxxxxxx'
random_str = _generate_random_string(10, use_letters=True)
error_msg = f"e:TypeError: Cannot read properties of undefined (reading '{random_str}')"
rand = _random_string(10)
msg = f"e:TypeError: Cannot read properties of undefined (reading '{rand}')"
# base64 编码
encoded = base64.b64encode(error_msg.encode('utf-8')).decode('utf-8')
return encoded
return base64.b64encode(msg.encode()).decode()
def get_dynamic_headers(pathname: str = "/rest/app-chat/conversations/new") -> Dict[str, str]:
"""获取请求头
Args:
pathname: 请求路径
Returns:
请求头字典
完整的请求头字典
"""
# 检查是否启用动态生成
dynamic_statsig = setting.grok_config.get("dynamic_statsig", False)
if dynamic_statsig:
# 动态生成 x-statsig-id
# 获取或生成statsig-id
if setting.grok_config.get("dynamic_statsig", False):
statsig_id = _generate_statsig_id()
logger.debug(f"[Statsig] 动态生成值 {statsig_id}")
logger.debug(f"[Statsig] 动态生成: {statsig_id}")
else:
# 使用配置文件中的固定值
statsig_id = setting.grok_config.get("x_statsig_id")
logger.debug(f"[Statsig] 使用固定值 {statsig_id}")
if not statsig_id:
raise ValueError("配置文件中未设置 x_statsig_id")
logger.debug(f"[Statsig] 使用固定值: {statsig_id}")
# 构建基础请求头
headers = {
"Accept": "*/*",
"Accept-Language": "zh-CN,zh;q=0.9",
"Accept-Encoding": "gzip, deflate, br, zstd",
"Content-Type": "application/json" if "upload-file" not in pathname else "text/plain;charset=UTF-8",
"Connection": "keep-alive",
"Origin": "https://grok.com",
"Priority": "u=1, i",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/133.0.0.0 Safari/537.36",
"Sec-Ch-Ua": '"Not(A:Brand";v="99", "Google Chrome";v="133", "Chromium";v="133"',
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": '"macOS"',
"Sec-Fetch-Dest": "empty",
"Sec-Fetch-Mode": "cors",
"Sec-Fetch-Site": "same-origin",
"Baggage": "sentry-environment=production,sentry-public_key=b311e0f2690c81f25e2c4cf6d4f7ce1c",
"x-statsig-id": statsig_id,
"x-xai-request-id": str(uuid.uuid4())
}
# 构建请求头
headers = BASE_HEADERS.copy()
headers["x-statsig-id"] = statsig_id
headers["x-xai-request-id"] = str(uuid.uuid4())
headers["Content-Type"] = "text/plain;charset=UTF-8" if "upload-file" in pathname else "application/json"
return headers
+149 -249
View File
@@ -1,6 +1,6 @@
"""Grok Token 管理器模块"""
"""Grok Token 管理器 - 单例模式的Token负载均衡和状态管理"""
import json
import orjson
import time
import asyncio
import aiofiles
@@ -14,37 +14,28 @@ from app.core.logger import logger
from app.core.config import setting
from app.services.grok.statsig import get_dynamic_headers
# 常量定义
RATE_LIMIT_ENDPOINT = "https://grok.com/rest/rate-limits"
REQUEST_TIMEOUT = 30
IMPERSONATE_BROWSER = "chrome133a"
MAX_FAILURE_COUNT = 3
TOKEN_INVALID_CODE = 401 # SSO Token失效
STATSIG_INVALID_CODE = 403 # x-statsig-id失效
# 常量
RATE_LIMIT_API = "https://grok.com/rest/rate-limits"
TIMEOUT = 30
BROWSER = "chrome133a"
MAX_FAILURES = 3
TOKEN_INVALID = 401
STATSIG_INVALID = 403
class GrokTokenManager:
"""
Grok Token管理器
单例模式的Token管理器,负责:
- Token文件的读写操作
- Token负载均衡
- Token状态管理
- 支持普通Token和Super Token
"""
"""Token管理器(单例)"""
_instance: Optional['GrokTokenManager'] = None
_lock = asyncio.Lock()
def __new__(cls) -> 'GrokTokenManager':
"""单例模式实现"""
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def __init__(self):
"""初始化Token管理器"""
if hasattr(self, '_initialized'):
return
@@ -52,79 +43,65 @@ class GrokTokenManager:
self._file_lock = asyncio.Lock()
self.token_file.parent.mkdir(parents=True, exist_ok=True)
self._storage = None
# 同步加载初始数据
self._load_data()
self._initialized = True
logger.debug(f"[Token] 管理器初始化完成,文件: {self.token_file}")
logger.debug(f"[Token] 初始化完成: {self.token_file}")
def set_storage(self, storage) -> None:
"""设置存储实例"""
self._storage = storage
def _load_data(self) -> None:
"""同步加载Token数据(仅用于初始化)"""
default_data = {
TokenType.NORMAL.value: {},
TokenType.SUPER.value: {}
}
"""加载Token数据"""
default = {TokenType.NORMAL.value: {}, TokenType.SUPER.value: {}}
try:
if self.token_file.exists():
with open(self.token_file, "r", encoding="utf-8") as f:
self.token_data = json.load(f)
self.token_data = orjson.loads(f.read())
else:
self.token_data = default_data
logger.debug("[Token] 创建新的Token数据文件")
except (json.JSONDecodeError, IOError) as e:
logger.error(f"[Token] 加载Token数据失败: {str(e)}")
self.token_data = default_data
self.token_data = default
logger.debug("[Token] 创建新数据文件")
except Exception as e:
logger.error(f"[Token] 加载失败: {e}")
self.token_data = default
async def _save_data(self) -> None:
"""异步保存Token数据到存储"""
"""保存Token数据"""
try:
if not self._storage:
# 如果没有设置存储,使用传统文件保存方式(向后兼容)
async with self._file_lock:
async with aiofiles.open(self.token_file, "w", encoding="utf-8") as f:
await f.write(json.dumps(self.token_data, indent=2, ensure_ascii=False))
await f.write(orjson.dumps(self.token_data, option=orjson.OPT_INDENT_2).decode())
else:
# 使用存储抽象层
await self._storage.save_tokens(self.token_data)
except IOError as e:
logger.error(f"[Token] 保存Token数据失败: {str(e)}")
raise GrokApiException(
f"Token数据保存失败: {str(e)}",
"TOKEN_SAVE_ERROR",
{"file_path": str(self.token_file)}
)
logger.error(f"[Token] 保存失败: {e}")
raise GrokApiException(f"保存失败: {e}", "TOKEN_SAVE_ERROR", {"file": str(self.token_file)})
@staticmethod
def _extract_sso(auth_token: str) -> Optional[str]:
"""从认证令牌中提取SSO值"""
"""提取SSO值"""
if "sso=" in auth_token:
return auth_token.split("sso=")[1].split(";")[0]
logger.warning("[Token] 无法从认证令牌中提取SSO值")
logger.warning("[Token] 无法提取SSO值")
return None
def _find_token(self, sso_value: str) -> Tuple[Optional[str], Optional[Dict[str, Any]]]:
"""查找Token数据,返回(token_type, token_data)"""
def _find_token(self, sso: str) -> Tuple[Optional[str], Optional[Dict]]:
"""查找Token"""
for token_type in [TokenType.NORMAL.value, TokenType.SUPER.value]:
if sso_value in self.token_data[token_type]:
return token_type, self.token_data[token_type][sso_value]
if sso in self.token_data[token_type]:
return token_type, self.token_data[token_type][sso]
return None, None
async def add_token(self, tokens: list[str], token_type: TokenType) -> None:
"""添加Token到管理器"""
"""添加Token"""
if not tokens:
logger.debug("[Token] 尝试添加空的Token列表")
return
added_count = 0
count = 0
for token in tokens:
if not token or not token.strip():
logger.debug("[Token] 跳过空的Token")
continue
self.token_data[token_type.value][token] = {
@@ -138,310 +115,233 @@ class GrokTokenManager:
"tags": [],
"note": ""
}
added_count += 1
count += 1
await self._save_data()
logger.info(f"[Token] 成功添加 {added_count} 个 {token_type.value} Token")
logger.info(f"[Token] 添加 {count} 个 {token_type.value} Token")
async def delete_token(self, tokens: list[str], token_type: TokenType) -> None:
"""删除指定的Token"""
"""删除Token"""
if not tokens:
logger.debug("[Token] 尝试删除空的Token列表")
return
deleted_count = 0
count = 0
for token in tokens:
if token in self.token_data[token_type.value]:
del self.token_data[token_type.value][token]
deleted_count += 1
else:
logger.debug(f"[Token] Token不存在: {token[:10]}...")
count += 1
await self._save_data()
logger.info(f"[Token] 成功删除 {deleted_count} 个 {token_type.value} Token")
logger.info(f"[Token] 删除 {count} 个 {token_type.value} Token")
async def update_token_tags(self, token: str, token_type: TokenType, tags: list[str]) -> None:
"""更新Token的标签"""
"""更新Token标签"""
if token not in self.token_data[token_type.value]:
logger.warning(f"[Token] Token不存在: {token[:10]}...")
raise GrokApiException(
"Token不存在",
"TOKEN_NOT_FOUND",
{"token": token[:10]}
)
# 确保tags是列表且不包含空字符串
cleaned_tags = [tag.strip() for tag in tags if tag and tag.strip()]
self.token_data[token_type.value][token]["tags"] = cleaned_tags
raise GrokApiException("Token不存在", "TOKEN_NOT_FOUND", {"token": token[:10]})
cleaned = [t.strip() for t in tags if t and t.strip()]
self.token_data[token_type.value][token]["tags"] = cleaned
await self._save_data()
logger.info(f"[Token] 成功更新Token {token[:10]}... 的标签: {cleaned_tags}")
logger.info(f"[Token] 更新标签: {token[:10]}... -> {cleaned}")
async def update_token_note(self, token: str, token_type: TokenType, note: str) -> None:
"""更新Token的备注"""
"""更新Token备注"""
if token not in self.token_data[token_type.value]:
logger.warning(f"[Token] Token不存在: {token[:10]}...")
raise GrokApiException(
"Token不存在",
"TOKEN_NOT_FOUND",
{"token": token[:10]}
)
raise GrokApiException("Token不存在", "TOKEN_NOT_FOUND", {"token": token[:10]})
self.token_data[token_type.value][token]["note"] = note.strip()
await self._save_data()
logger.info(f"[Token] 成功更新Token {token[:10]}... 的备注")
logger.info(f"[Token] 更新备注: {token[:10]}...")
def get_tokens(self) -> Dict[str, Any]:
"""获取所有Token数据"""
"""获取所有Token"""
return self.token_data.copy()
def get_token(self, model: str) -> str:
"""获取指定模型的Token"""
jwt_token = self.select_token(model)
return f"sso-rw={jwt_token};sso={jwt_token}"
"""获取Token"""
jwt = self.select_token(model)
return f"sso-rw={jwt};sso={jwt}"
def select_token(self, model: str) -> str:
"""根据模型类型和剩余次数选择最优Token"""
def select_best_token(tokens_dict: Dict[str, Any]) -> Tuple[Optional[str], Optional[int]]:
"""从 token 字典中选择最佳 token"""
unused_tokens = [] # remaining = -1 的 token
used_tokens = [] # remaining > 0 的 token
"""选择最优Token"""
def select_best(tokens: Dict[str, Any], field: str) -> Tuple[Optional[str], Optional[int]]:
"""选择最佳Token"""
unused, used = [], []
for token_key, token_data in tokens_dict.items():
# 跳过已失效的Token
if token_data.get("status") == "expired":
for key, data in tokens.items():
if data.get("status") == "expired":
continue
remaining = int(token_data.get(remaining_field, -1))
# 跳过已限流的 token
remaining = int(data.get(field, -1))
if remaining == 0:
continue
# 分类存储
if remaining == -1:
unused_tokens.append(token_key)
unused.append(key)
elif remaining > 0:
used_tokens.append((token_key, remaining))
# 优先返回尚未使用的 token
if unused_tokens:
return unused_tokens[0], -1
# 否则返回次数最多的 token
if used_tokens:
used_tokens.sort(key=lambda x: x[1], reverse=True)
return used_tokens[0][0], used_tokens[0][1]
used.append((key, remaining))
if unused:
return unused[0], -1
if used:
used.sort(key=lambda x: x[1], reverse=True)
return used[0][0], used[0][1]
return None, None
max_token_key = None
max_remaining = None
# 深拷贝
token_data_snapshot = {
# 快照
snapshot = {
TokenType.NORMAL.value: self.token_data[TokenType.NORMAL.value].copy(),
TokenType.SUPER.value: self.token_data[TokenType.SUPER.value].copy()
}
# 选择策略
if model == "grok-4-heavy":
# grok-4-heavy 只能使用Super Token + heavy remaining queries
remaining_field = "heavyremainingQueries"
max_token_key, max_remaining = select_best_token(token_data_snapshot[TokenType.SUPER.value])
field = "heavyremainingQueries"
token_key, remaining = select_best(snapshot[TokenType.SUPER.value], field)
else:
# 其他模型使用 remaining Queries
remaining_field = "remainingQueries"
field = "remainingQueries"
token_key, remaining = select_best(snapshot[TokenType.NORMAL.value], field)
if token_key is None:
token_key, remaining = select_best(snapshot[TokenType.SUPER.value], field)
# 优先使用普通Token
max_token_key, max_remaining = select_best_token(token_data_snapshot[TokenType.NORMAL.value])
# 如果普通Token没有可用的,尝试使用Super Token
if max_token_key is None:
max_token_key, max_remaining = select_best_token(token_data_snapshot[TokenType.SUPER.value])
if max_token_key is None:
if token_key is None:
raise GrokApiException(
f"没有可用Token用于模型 {model}",
f"没有可用Token: {model}",
"NO_AVAILABLE_TOKEN",
{
"model": model,
"normal_count": len(token_data_snapshot[TokenType.NORMAL.value]),
"super_count": len(token_data_snapshot[TokenType.SUPER.value])
"normal": len(snapshot[TokenType.NORMAL.value]),
"super": len(snapshot[TokenType.SUPER.value])
}
)
status_text = "未使用" if max_remaining == -1 else f"剩余{max_remaining}次"
logger.debug(f"[Token] 正在为模型 {model} 分配Token ({status_text})")
return max_token_key
status = "未使用" if remaining == -1 else f"剩余{remaining}次"
logger.debug(f"[Token] 分配Token: {model} ({status})")
return token_key
async def check_limits(self, auth_token: str, model: str) -> Optional[Dict[str, Any]]:
"""检查并更新模型速率限制"""
"""检查速率限制"""
try:
rate_limit_model_name = Models.to_rate_limit(model)
# 准备请求
payload = {"requestKind": "DEFAULT", "modelName": rate_limit_model_name}
cf_clearance = setting.grok_config.get("cf_clearance", "")
cookie = f"{auth_token};{cf_clearance}" if cf_clearance else auth_token
headers = get_dynamic_headers("/rest/rate-limits")
headers["Cookie"] = cookie
# 获取代理配置
proxy_url = setting.grok_config.get("proxy_url", "")
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
rate_model = Models.to_rate_limit(model)
payload = {"requestKind": "DEFAULT", "modelName": rate_model}
cf = setting.grok_config.get("cf_clearance", "")
headers = get_dynamic_headers("/rest/rate-limits")
headers["Cookie"] = f"{auth_token};{cf}" if cf else auth_token
proxy = setting.grok_config.get("proxy_url", "")
proxies = {"http": proxy, "https": proxy} if proxy else None
# 发送异步请求
async with AsyncSession() as session:
response = await session.post(
RATE_LIMIT_ENDPOINT,
RATE_LIMIT_API,
headers=headers,
json=payload,
impersonate=IMPERSONATE_BROWSER,
timeout=REQUEST_TIMEOUT,
impersonate=BROWSER,
timeout=TIMEOUT,
proxies=proxies
)
if response.status_code == 200:
rate_limit_data = response.json()
# 保存速率限制信息
sso_value = self._extract_sso(auth_token)
if sso_value:
data = response.json()
sso = self._extract_sso(auth_token)
if sso:
if model == "grok-4-heavy":
await self.update_limits(sso_value, normal=None, heavy=rate_limit_data.get("remainingQueries", -1))
logger.info(f"[Token] 更新 Token 限制: sso={sso_value[:10]}..., heavy={rate_limit_data.get('remainingQueries', -1)}")
await self.update_limits(sso, normal=None, heavy=data.get("remainingQueries", -1))
logger.info(f"[Token] 更新限制: {sso[:10]}..., heavy={data.get('remainingQueries', -1)}")
else:
await self.update_limits(sso_value, normal=rate_limit_data.get("remainingTokens", -1), heavy=None)
logger.info(f"[Token] 更新 Token 限制: sso={sso_value[:10]}..., basic={rate_limit_data.get('remainingTokens', -1)}")
return rate_limit_data
await self.update_limits(sso, normal=data.get("remainingTokens", -1), heavy=None)
logger.info(f"[Token] 更新限制: {sso[:10]}..., basic={data.get('remainingTokens', -1)}")
return data
else:
logger.warning(f"[Token] 获取速率限制失败,状态码: {response.status_code}")
# 根据HTTP状态码处理不同的错误
sso_value = self._extract_sso(auth_token)
if sso_value:
logger.warning(f"[Token] 获取限制失败: {response.status_code}")
sso = self._extract_sso(auth_token)
if sso:
if response.status_code == 401:
# Token失效
await self.record_failure(auth_token, 401, "Token authentication failed")
await self.record_failure(auth_token, 401, "Token失效")
elif response.status_code == 403:
# 服务器被block,不改变token状态,但记录失败信息
await self.record_failure(auth_token, 403, "Server blocked (Cloudflare)")
await self.record_failure(auth_token, 403, "服务器被Block")
else:
# 其他错误,设置为限流状态
await self.record_failure(auth_token, response.status_code, f"Rate limit or other error: {response.status_code}")
await self.record_failure(auth_token, response.status_code, f"错误: {response.status_code}")
return None
except Exception as e:
logger.error(f"[Token] 检查速率限制时发生错误: {str(e)}")
logger.error(f"[Token] 检查限制错误: {e}")
return None
async def update_limits(self, sso_value: str, normal: Optional[int] = None, heavy: Optional[int] = None) -> None:
"""更新Token限制信息"""
async def update_limits(self, sso: str, normal: Optional[int] = None, heavy: Optional[int] = None) -> None:
"""更新限制"""
try:
for token_type in [TokenType.NORMAL.value, TokenType.SUPER.value]:
if sso_value in self.token_data[token_type]:
if sso in self.token_data[token_type]:
if normal is not None:
self.token_data[token_type][sso_value]["remainingQueries"] = normal
self.token_data[token_type][sso]["remainingQueries"] = normal
if heavy is not None:
self.token_data[token_type][sso_value]["heavyremainingQueries"] = heavy
self.token_data[token_type][sso]["heavyremainingQueries"] = heavy
await self._save_data()
logger.info(f"[Token] 已更新Token {sso_value[:10]}... 的限制信息")
logger.info(f"[Token] 更新限制: {sso[:10]}...")
return
logger.warning(f"[Token] 未找到SSO值为 {sso_value[:10]}... 的Token")
logger.warning(f"[Token] 未找到: {sso[:10]}...")
except Exception as e:
logger.error(f"[Token] 更新Token限制时发生错误: {str(e)}")
logger.error(f"[Token] 更新限制错误: {e}")
async def record_failure(self, auth_token: str, status_code: int, error_message: str) -> None:
"""记录Token失败信息
错误码说明:
- 401: SSO Token失效,会标记Token为expired
- 403: 服务器IP被Block,不影响Token状态
Args:
auth_token: 完整的认证Token (格式: sso-rw=xxx;sso=xxx)
status_code: HTTP状态码
error_message: 错误信息
"""
async def record_failure(self, auth_token: str, status: int, msg: str) -> None:
"""记录失败"""
try:
# 403错误是服务器IP被Block,不是Token问题
if status_code == STATSIG_INVALID_CODE:
logger.warning(
f"[Token] 服务器IP被Block (403),请 1. 更换服务器IP 2. 使用代理IP "
f"3. 服务器登陆Grok.com,过盾后F12找到CF值填入后台设置"
)
if status == STATSIG_INVALID:
logger.warning("[Token] IP被Block,请: 1.更换IP 2.使用代理 3.配置CF值")
return
sso_value = self._extract_sso(auth_token)
if not sso_value:
sso = self._extract_sso(auth_token)
if not sso:
return
_, token_data = self._find_token(sso_value)
if not token_data:
logger.warning(f"[Token] 未找到SSO值为 {sso_value[:10]}... 的Token")
_, data = self._find_token(sso)
if not data:
logger.warning(f"[Token] 未找到: {sso[:10]}...")
return
# 更新失败计数
token_data["failedCount"] = token_data.get("failedCount", 0) + 1
token_data["lastFailureTime"] = int(time.time() * 1000)
token_data["lastFailureReason"] = f"{status_code}: {error_message}"
data["failedCount"] = data.get("failedCount", 0) + 1
data["lastFailureTime"] = int(time.time() * 1000)
data["lastFailureReason"] = f"{status}: {msg}"
logger.warning(
f"[Token] Token {sso_value[:10]}... 失败 (状态码: {status_code}), "
f"失败次数: {token_data['failedCount']}/{MAX_FAILURE_COUNT}, "
f"原因: {error_message}"
f"[Token] 失败: {sso[:10]}... (状态:{status}), "
f"次数: {data['failedCount']}/{MAX_FAILURES}, 原因: {msg}"
)
# 只有401错误(SSO Token失效)且失败次数达到上限时,标记为失效
if status_code == TOKEN_INVALID_CODE and token_data["failedCount"] >= MAX_FAILURE_COUNT:
token_data["status"] = "expired"
logger.error(
f"[Token] SSO Token {sso_value[:10]}... 已被标记为失效 "
f"(连续401错误{token_data['failedCount']}次)"
)
if status == TOKEN_INVALID and data["failedCount"] >= MAX_FAILURES:
data["status"] = "expired"
logger.error(f"[Token] 标记失效: {sso[:10]}... (连续401错误{data['failedCount']}次)")
await self._save_data()
except Exception as e:
logger.error(f"[Token] 记录Token失败信息时发生错误: {str(e)}")
logger.error(f"[Token] 记录失败错误: {e}")
async def reset_failure(self, auth_token: str) -> None:
"""重置Token失败计数
当Token成功完成请求时调用此方法,用于清除失败记录。
Args:
auth_token: 完整的认证Token (格式: sso-rw=xxx;sso=xxx)
"""
"""重置失败计数"""
try:
sso_value = self._extract_sso(auth_token)
if not sso_value:
sso = self._extract_sso(auth_token)
if not sso:
return
_, token_data = self._find_token(sso_value)
if not token_data:
logger.warning(f"[Token] 未找到SSO值为 {sso_value[:10]}... 的Token")
_, data = self._find_token(sso)
if not data:
return
# 只有在有失败记录时才重置并保存
if token_data.get("failedCount", 0) > 0:
token_data["failedCount"] = 0
token_data["lastFailureTime"] = None
token_data["lastFailureReason"] = None
if data.get("failedCount", 0) > 0:
data["failedCount"] = 0
data["lastFailureTime"] = None
data["lastFailureReason"] = None
await self._save_data()
logger.info(f"[Token] Token {sso_value[:10]}... 失败计数已重置")
logger.info(f"[Token] 重置失败计数: {sso[:10]}...")
except Exception as e:
logger.error(f"[Token] 重置Token失败计数时发生错误: {str(e)}")
logger.error(f"[Token] 重置失败错误: {e}")
# 全局Token管理器实例
# 全局实例
token_manager = GrokTokenManager()
+78 -81
View File
@@ -1,10 +1,9 @@
"""图片上传管理器"""
"""图片上传管理器 - 支持Base64和URL图片上传"""
import base64
import re
from typing import Tuple, Optional
from urllib.parse import urlparse
from curl_cffi.requests import AsyncSession
from app.services.grok.statsig import get_dynamic_headers
@@ -12,136 +11,134 @@ from app.core.exception import GrokApiException
from app.core.config import setting
from app.core.logger import logger
# 常量定义
UPLOAD_ENDPOINT = "https://grok.com/rest/app-chat/upload-file"
REQUEST_TIMEOUT = 30
IMPERSONATE_BROWSER = "chrome133a"
DEFAULT_MIME_TYPE = "image/jpeg"
DEFAULT_EXTENSION = "jpg"
# 常量
UPLOAD_API = "https://grok.com/rest/app-chat/upload-file"
TIMEOUT = 30
BROWSER = "chrome133a"
# MIME类型
MIME_TYPES = {
'.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png',
'.gif': 'image/gif', '.webp': 'image/webp', '.bmp': 'image/bmp',
}
DEFAULT_MIME = "image/jpeg"
DEFAULT_EXT = "jpg"
class ImageUploadManager:
"""
Grok图片上传管理器
提供图片上传功能,支持:
- Base64格式图片上传
- URL图片下载并上传
- 多种图片格式支持
"""
"""图片上传管理器"""
@staticmethod
async def upload(image_input: str, auth_token: str) -> str:
"""上传图片到Grok,支持Base64或URL"""
async def upload(image_input: str, auth_token: str) -> Tuple[str, str]:
"""上传图片(支持Base64或URL)
Returns:
(file_id, file_uri) 元组
"""
try:
# 判断类型并处理
if ImageUploadManager._is_url(image_input):
# 下载 URL 图片
image_buffer, mime_type = await ImageUploadManager._download(image_input)
# 获取图片信息
file_name, _ = ImageUploadManager._get_info("", mime_type)
buffer, mime = await ImageUploadManager._download(image_input)
filename, _ = ImageUploadManager._get_info("", mime)
else:
# 处理 base64 数据
image_buffer = image_input.split(",")[1] if "data:image" in image_input else image_input
buffer = image_input.split(",")[1] if "data:image" in image_input else image_input
filename, mime = ImageUploadManager._get_info(image_input)
# 获取图片信息
file_name, mime_type = ImageUploadManager._get_info(image_input)
# 构建上传数据
upload_data = {
"fileName": file_name,
"fileMimeType": mime_type,
"content": image_buffer,
# 构建数据
data = {
"fileName": filename,
"fileMimeType": mime,
"content": buffer,
}
# 获取认证令牌
if not auth_token:
raise GrokApiException("认证令牌缺失或为空", "NO_AUTH_TOKEN")
raise GrokApiException("认证令牌缺失", "NO_AUTH_TOKEN")
cf_clearance = setting.grok_config.get("cf_clearance", "")
cookie = f"{auth_token};{cf_clearance}" if cf_clearance else auth_token
# 请求配置
cf = setting.grok_config.get("cf_clearance", "")
headers = {
**get_dynamic_headers("/rest/app-chat/upload-file"),
"Cookie": f"{auth_token};{cf}" if cf else auth_token,
}
proxy_url = setting.grok_config.get("proxy_url", "")
proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None
proxy = setting.grok_config.get("proxy_url", "")
proxies = {"http": proxy, "https": proxy} if proxy else None
# 发送异步请求
# 上传
async with AsyncSession() as session:
response = await session.post(
UPLOAD_ENDPOINT,
headers={
**get_dynamic_headers("/rest/app-chat/upload-file"),
"Cookie": cookie,
},
json=upload_data,
impersonate=IMPERSONATE_BROWSER,
timeout=REQUEST_TIMEOUT,
UPLOAD_API,
headers=headers,
json=data,
impersonate=BROWSER,
timeout=TIMEOUT,
proxies=proxies,
)
# 检查响应
if response.status_code == 200:
result = response.json()
file_id = result.get("fileMetadataId", "")
file_uri = result.get("fileUri", "")
logger.debug(f"[Upload] 图片上传成功,文件ID: {file_id}")
logger.debug(f"[Upload] 成功,ID: {file_id}")
return file_id, file_uri
return "", ""
except Exception as e:
logger.warning(f"[Upload] 上传图片失败: {e}")
return ""
logger.warning(f"[Upload] 失败: {e}")
return "", ""
@staticmethod
def _is_url(image_input: str) -> bool:
"""检查输入是否为有效的URL"""
def _is_url(input_str: str) -> bool:
"""检查是否为URL"""
try:
result = urlparse(image_input)
result = urlparse(input_str)
return all([result.scheme, result.netloc]) and result.scheme in ['http', 'https']
except Exception as e:
logger.warning(f"[Upload] URL解析失败: {e}")
except:
return False
@staticmethod
async def _download(url: str) -> Tuple[str, str]:
"""下载图片并转换为Base64"""
"""下载图片并转Base64
Returns:
(base64_string, mime_type) 元组
"""
try:
async with AsyncSession() as session:
response = await session.get(url, timeout=5)
response.raise_for_status()
# 获取内容类型
content_type = response.headers.get('content-type', DEFAULT_MIME_TYPE)
content_type = response.headers.get('content-type', DEFAULT_MIME)
if not content_type.startswith('image/'):
content_type = DEFAULT_MIME_TYPE
content_type = DEFAULT_MIME
# 转换为 Base64
image_base64 = base64.b64encode(response.content).decode('utf-8')
return image_base64, content_type
b64 = base64.b64encode(response.content).decode()
return b64, content_type
except Exception as e:
logger.warning(f"[Upload] 下载图片失败: {e}")
logger.warning(f"[Upload] 下载失败: {e}")
return "", ""
@staticmethod
def _get_info(image_data: str, mime_type: Optional[str] = None) -> Tuple[str, str]:
"""获取图片文件名和MIME类型"""
# mime_type 有值,直接使用
"""获取文件名和MIME类型
Returns:
(file_name, mime_type) 元组
"""
# 已提供MIME类型
if mime_type:
extension = mime_type.split("/")[1] if "/" in mime_type else DEFAULT_EXTENSION
file_name = f"image.{extension}"
return file_name, mime_type
ext = mime_type.split("/")[1] if "/" in mime_type else DEFAULT_EXT
return f"image.{ext}", mime_type
# mime_type 没有值,使用默认值
mime_type = DEFAULT_MIME_TYPE
extension = DEFAULT_EXTENSION
# 从Base64提取
mime = DEFAULT_MIME
ext = DEFAULT_EXT
# 从 Base64 数据中提取 MIME 类型
if "data:image" in image_data:
match = re.search(r"data:([a-zA-Z0-9]+/[a-zA-Z0-9-.+]+);base64,", image_data)
if match:
mime_type = match.group(1)
extension = mime_type.split("/")[1]
if match := re.search(r"data:([a-zA-Z0-9]+/[a-zA-Z0-9-.+]+);base64,", image_data):
mime = match.group(1)
ext = mime.split("/")[1]
file_name = f"image.{extension}"
return file_name, mime_type
return f"image.{ext}", mime