diff --git a/api/admin/v1/monitor.go b/api/admin/v1/monitor.go index 9d9688e..f6e44e1 100644 --- a/api/admin/v1/monitor.go +++ b/api/admin/v1/monitor.go @@ -37,6 +37,10 @@ type MonitorModelPerformanceReq struct { g.Meta `path:"/monitor/model-performance" method:"get" mime:"json" tags:"管理后台-监控" summary:"模型性能指标"` StartDate string `json:"start_date" in:"query"` // YYYY-MM-DD，缺省=近30天起点 EndDate string `json:"end_date" in:"query"` // YYYY-MM-DD，缺省=今天 + // 可选渠道筛选：0=全部渠道（选定渠道后当天数据改从 bil_usage_logs 明细实时聚合；0 哨兵被占用，无法筛「无渠道」） + ChannelId int64 `json:"channel_id" in:"query" d:"0"` + // 可选模型筛选：空=全部模型（与 bil_usage_daily.model_name 同口径） + ModelName string `json:"model_name" in:"query"` } type MonitorModelPerformanceRes struct { diff --git a/docs/reference/goframe-conventions.md b/docs/reference/goframe-conventions.md index bd95dcb..d914a5f 100644 --- a/docs/reference/goframe-conventions.md +++ b/docs/reference/goframe-conventions.md @@ -719,3 +719,15 @@ err := g.DB().Transaction(dbCtx, func(txCtx context.Context, tx gdb.TX) error { - **前端编辑表单慎用 `x || 默认值` 兜底**：对数值字段，0 会被 falsy 规则吞掉，既掩盖真实数据，又可能在保存时把合法的 0 改写为默认值。 排查信号：某字段在库里莫名变成 0/空串，而创建时明明写过合法值——查该表所有 UPDATE 路径中是否存在非指针字段的无条件赋值。 + +### 2026-08-16：httptest.NewServer 直接包裹未 Start 的 g.Server，每个请求都在 Session.Close() panic + +**问题**：`go test -v` 下凡是用 `httptest.NewServer(g.Server(guid.S()))` 方式测 HTTP 处理逻辑的用例，每个请求都向 stderr 刷一段 `http: panic serving ... nil pointer dereference`（`gsession.(*Session).Close`）。测试仍然 PASS——响应在 panic 前已完整写出、panic 发生在 deferred `handleAfterRequestDone` 中且被 net/http recover——导致问题长期被掩盖（不带 `-v` 时 `go test` 丢弃通过用例的输出）。涉及 `internal/middleware`、`internal/response`、`internal/handler/setup` 三处。 + +**原因**：gf v2.10.2 只在 `Server.Start()` 中初始化 `sessionManager`（ghttp_server.go），而 `ServeHTTP` 对每个请求无条件执行 `request.Session.Close()`，后者第一行就解引用 `s.manager.storage`。未经 `Start()` 的 server 其 `sessionManager` 为 nil → `Session.manager` 为 nil → 必 panic。无导出 API 可以单独初始化 manager，绕不开。 + +**修复**：新增共享测试辅助 `internal/testutil.StartGFServer(t, s)`——把 server 绑定到 `127.0.0.1:0`（内核随机分配端口避免冲突，绑定回环地址避免 Windows 防火墙弹窗）走真实 `s.Start()`，返回 `http://127.0.0.1:{port}` 供用例直接请求，`t.Cleanup` 注册 `s.Shutdown()`。`Start()` 返回时 listener 已创建完毕（`startServer` 内 `wg.Wait()` 等待 `CreateListener` 完成后才返回），`GetListenedPort()` 立即可用，无需轮询。三处测试全部改用该辅助。 + +**正确做法**：测试中需要真实走 ghttp 请求链路（中间件、handler、hook）时，一律用 `testutil.StartGFServer(t, s)` 启动真实 server；**禁止**把 `g.Server` 直接塞进 `httptest.NewServer`——那只会执行 `ServeHTTP` 路径而不执行 `Start()` 的初始化（session manager、graceful listener 等），属于未定义用法。`httptest.NewServer` 仅用于包装自建的普通 `http.Handler`。 + +排查信号：`go test -v` 输出中出现成段的 `http: panic serving` + `gsession.(*Session).Close` 栈。 diff --git a/docs/协议文档/大模型协议Usage与缓存Token语义调研.md b/docs/协议文档/大模型协议Usage与缓存Token语义调研.md new file mode 100644 index 0000000..631cc35 --- /dev/null +++ b/docs/协议文档/大模型协议Usage与缓存Token语义调研.md @@ -0,0 +1,331 @@ +# 大模型协议 Usage 与缓存 Token 语义调研（官方联网核验版） + +## 文档信息 + +- 核验日期：2026-08-16 +- 核验方式：联网读取 OpenAI、Anthropic、Google 官方 API Reference 与 Prompt Caching 文档正文。 +- 覆盖范围：OpenAI Chat Completions、OpenAI Responses、Claude Messages API、Gemini GenerateContent API。 +- 不包含：Azure OpenAI、Amazon Bedrock、Vertex AI 等托管平台对字段的二次封装，也不把第三方兼容网关的扩展字段视为官方字段。 + +## 核心结论 + +| 协议 | 顶层输入字段 | 缓存读取字段 | 缓存写入字段 | 顶层输入是否包含缓存 | +| --- | --- | --- | --- | --- | +| OpenAI Chat Completions | `prompt_tokens` | `prompt_tokens_details.cached_tokens` | `prompt_tokens_details.cache_write_tokens` | **包含缓存读取和缓存写入** | +| OpenAI Responses | `input_tokens` | `input_tokens_details.cached_tokens` | `input_tokens_details.cache_write_tokens` | **包含缓存读取和缓存写入** | +| Claude Messages | `input_tokens` | `cache_read_input_tokens` | `cache_creation_input_tokens` | **不包含，三个字段是独立桶** | +| Gemini GenerateContent | `promptTokenCount` | `cachedContentTokenCount` | GenerateContent usage 无对应字段 | **包含缓存读取** | + +不能把四种协议的“输入 token”直接按同一口径使用： + +```text +OpenAI / Responses: +普通输入 = 顶层输入 - cached_tokens - cache_write_tokens + +Claude: +总输入 = input_tokens + cache_read_input_tokens + cache_creation_input_tokens + +Gemini: +普通输入 = promptTokenCount - cachedContentTokenCount +``` + +其中 `cache_write_tokens` 是 OpenAI 官方文档中的新字段。官方 Prompt Caching 文档说明：GPT-5.6 及后续模型系列会报告缓存写入 token；早期模型没有该字段或不产生对应的额外缓存写入计费。 + +## OpenAI Chat Completions + +### 官方 Usage 结构 + +顶层字段： + +| 字段 | 官方语义 | +| --- | --- | +| `prompt_tokens` | prompt 使用的 token 数 | +| `completion_tokens` | 生成 completion 使用的 token 数 | +| `total_tokens` | `prompt_tokens + completion_tokens` | +| `prompt_tokens_details` | 输入 token 明细 | +| `completion_tokens_details` | 输出 token 明细 | + +截至核验日期，`prompt_tokens_details` 的官方字段为： + +| 字段 | 官方语义 | +| --- | --- | +| `audio_tokens` | prompt 中的音频输入 token | +| `cache_write_tokens` | 写入缓存的 prompt token 原始数量 | +| `cached_tokens` | prompt 中命中缓存的 token | +| `image_tokens` | prompt 中的图像输入 token | +| `text_tokens` | prompt 中的文本输入 token | + +`completion_tokens_details` 的官方字段为： + +| 字段 | 官方语义 | +| --- | --- | +| `accepted_prediction_tokens` | Predicted Outputs 中被实际采用的预测 token | +| `audio_tokens` | 模型生成的音频 token | +| `reasoning_tokens` | 模型生成的推理 token | +| `rejected_prediction_tokens` | Predicted Outputs 中未被采用的预测 token；官方明确说明仍计入 completion token、计费及上下文限制 | +| `text_tokens` | 模型生成的文本 token | + +### 缓存口径 + +OpenAI Prompt Caching 官方示例表明，顶层输入包含缓存读取、缓存写入和普通输入。例如： + +```json +{ + "usage": { + "input_tokens": 2600, + "input_tokens_details": { + "cached_tokens": 2000, + "cache_write_tokens": 400 + } + } +} +``` + +该示例的官方解释是：2000 个 token 从缓存读取，400 个 token 写入缓存，剩余 200 个既未读取也未写入。因此 Chat Completions 对应公式为： + +```text +普通输入 token = + prompt_tokens + - prompt_tokens_details.cached_tokens + - prompt_tokens_details.cache_write_tokens +``` + +`text_tokens`、`image_tokens`、`audio_tokens` 是按模态的拆分；`cached_tokens`、`cache_write_tokens` 是按缓存状态的拆分。两组维度可能重叠，不能把所有 details 字段直接求和。 + +## OpenAI Responses + +### 官方 Usage 结构 + +```json +{ + "usage": { + "input_tokens": 2600, + "output_tokens": 120, + "total_tokens": 2720, + "input_tokens_details": { + "cached_tokens": 2000, + "cache_write_tokens": 400 + }, + "output_tokens_details": { + "reasoning_tokens": 50 + } + } +} +``` + +截至核验日期，Responses 官方 API Reference 中的字段为： + +| 位置 | 字段 | 官方语义 | +| --- | --- | --- | +| `usage` | `input_tokens` | 输入 token 数 | +| `usage` | `output_tokens` | 输出 token 数 | +| `usage` | `total_tokens` | 使用的总 token 数 | +| `input_tokens_details` | `cached_tokens` | 从缓存取回的 token 数 | +| `input_tokens_details` | `cache_write_tokens` | 写入缓存的输入 token 数 | +| `output_tokens_details` | `reasoning_tokens` | 推理 token 数 | + +缓存公式为： + +```text +普通输入 token = + input_tokens + - input_tokens_details.cached_tokens + - input_tokens_details.cache_write_tokens +``` + +Responses 当前官方 schema 没有在 `input_tokens_details` 中声明 `text_tokens`、`image_tokens`、`audio_tokens`，也没有在 `output_tokens_details` 中声明预测或模态明细。兼容服务可能返回扩展字段，但不能将它们标注成 OpenAI Responses 官方字段。 + +## Claude Messages API + +### 官方 Usage 结构 + +Claude 官方 API Reference 当前声明的 token 字段包括： + +| 字段 | 官方语义 | +| --- | --- | +| `input_tokens` | 未从缓存读取、也未用于创建缓存的输入 token；Prompt Caching 文档进一步说明，它通常对应最后一个缓存断点之后的 token | +| `cache_creation_input_tokens` | 用于创建缓存条目的输入 token | +| `cache_read_input_tokens` | 从缓存读取的输入 token | +| `output_tokens` | 输出 token 的权威总数，包含 thinking token | +| `cache_creation.ephemeral_5m_input_tokens` | 创建 5 分钟缓存条目的 token | +| `cache_creation.ephemeral_1h_input_tokens` | 创建 1 小时缓存条目的 token | +| `output_tokens_details.thinking_tokens` | 内部推理 token 明细，是 `output_tokens` 的子集 | + +Usage 中还包含以下非 token 元数据： + +- `inference_geo` +- `server_tool_use.web_fetch_requests` +- `server_tool_use.web_search_requests` +- `service_tier`，可能为 `standard`、`priority`、`batch` 或 `null` + +### 缓存口径 + +Anthropic 官方 API Reference 和 Prompt Caching 文档均明确给出： + +```text +总输入 token = + input_tokens + + cache_creation_input_tokens + + cache_read_input_tokens +``` + +因此，Claude 的 `input_tokens` **不包含**缓存读取和缓存创建 token。它与 OpenAI、Gemini 的顶层输入字段语义不同。 + +另外： + +- Claude 没有原生 `total_tokens` 字段。 +- `cache_creation.ephemeral_5m_input_tokens + ephemeral_1h_input_tokens` 等于 `cache_creation_input_tokens`，它们是 TTL 细分，不能重复相加。 +- `output_tokens_details.thinking_tokens <= output_tokens`；`output_tokens` 仍是计费使用的权威输出总量。 +- Amazon Bedrock 等托管平台可能使用不同字段名和缓存规则，应单独核验对应平台文档。 + +## Gemini GenerateContent API + +### 官方 UsageMetadata 结构 + +截至核验日期，Gemini 官方 `UsageMetadata` 字段为： + +| 字段 | 官方语义 | +| --- | --- | +| `promptTokenCount` | prompt 的总有效 token 数；官方明确说明设置 `cachedContent` 时仍包含缓存内容 token | +| `cachedContentTokenCount` | prompt 中缓存内容的 token 数 | +| `candidatesTokenCount` | 所有生成候选响应的 token 总数 | +| `toolUsePromptTokenCount` | tool-use prompt 中的 token 数 | +| `thoughtsTokenCount` | thinking 模型的思考 token 数 | +| `totalTokenCount` | 生成请求的总 token 数，官方定义为 prompt + thoughts + response candidates | +| `promptTokensDetails[]` | 请求输入按模态的 token 明细 | +| `cacheTokensDetails[]` | 缓存输入按模态的 token 明细 | +| `candidatesTokensDetails[]` | 响应候选按模态的 token 明细 | +| `toolUsePromptTokensDetails[]` | tool-use 请求输入按模态的 token 明细 | +| `serviceTier` | 请求使用的服务层级，非 token 字段 | + +### 缓存口径 + +Google 官方 API Reference 的原意非常明确：`promptTokenCount` 在设置 `cachedContent` 后仍表示总有效 prompt 大小，包含缓存内容。因此： + +```text +普通输入 token = promptTokenCount - cachedContentTokenCount +``` + +Gemini 的 `thoughtsTokenCount` 不是 `candidatesTokenCount` 的子集。官方给出的总量定义是： + +```text +totalTokenCount = promptTokenCount + thoughtsTokenCount + candidatesTokenCount +``` + +所以有 thinking 时，不能使用 `promptTokenCount + candidatesTokenCount` 代替 `totalTokenCount`。程序应优先采用上游返回的 `totalTokenCount`。 + +## 推荐的内部统一模型 + +不建议只保存通用的 `input_tokens` 和 `output_tokens`。为了准确计费和协议转换，建议至少保存： + +```text +input_total +input_ordinary +cache_read +cache_write +output_total +visible_output +reasoning_or_thinking +provider_total +``` + +推荐映射： + +| 协议 | `input_total` | `input_ordinary` | `cache_read` | `cache_write` | +| --- | --- | --- | --- | --- | +| OpenAI Chat | `prompt_tokens` | `prompt_tokens - cached_tokens - cache_write_tokens` | `cached_tokens` | `cache_write_tokens` | +| OpenAI Responses | `input_tokens` | `input_tokens - cached_tokens - cache_write_tokens` | `cached_tokens` | `cache_write_tokens` | +| Claude | 三个输入桶之和 | `input_tokens` | `cache_read_input_tokens` | `cache_creation_input_tokens` | +| Gemini | `promptTokenCount` | `promptTokenCount - cachedContentTokenCount` | `cachedContentTokenCount` | 无对应生成响应字段 | + +输出侧需要额外注意： + +- OpenAI 的 `completion_tokens` / `output_tokens` 已包含 `reasoning_tokens`。 +- Claude 的 `output_tokens` 已包含 `thinking_tokens`。 +- Gemini 的 `candidatesTokenCount` 不包含 `thoughtsTokenCount`；其完整生成侧数量应依据 `totalTokenCount - promptTokenCount` 或直接分别保留 candidates 与 thoughts。 +- 所有减法都应做非负保护，以防第三方兼容服务返回不一致数据。 + +## 本仓库与最新官方协议的差异 + +### 已正确处理的部分 + +- OpenAI Responses 已设置 `CacheIncludedInPrompt=true`，符合 OpenAI 顶层输入包含缓存的语义：[`relay/channel/openai/responses.go`](relay/channel/openai/responses.go)。 +- Gemini 已设置 `CacheIncludedInPrompt=true`，符合 `promptTokenCount` 包含 `cachedContentTokenCount` 的语义：[`relay/channel/gemini/response.go`](relay/channel/gemini/response.go)。 +- Claude 将缓存读取和缓存创建保留为独立桶，符合 Claude 语义：[`relay/channel/claude/response.go`](relay/channel/claude/response.go)。 + +### 需要更新或明确标注的部分 + +1. OpenAI Chat 通用明细缺少官方 `cache_write_tokens`。 + + [`relaykit/dto/usage.go`](relaykit/dto/usage.go) 目前有项目自定义的 `cached_creation_tokens`，但它不是 OpenAI 官方 JSON 字段 `cache_write_tokens`。两者不应使用同一个 JSON 名称或隐式混用。 + +2. OpenAI Responses 输入明细缺少 `cache_write_tokens`。 + + [`relay/dto/openai_responses.go`](relay/dto/openai_responses.go) 的 `InputTokenDetails` 当前只有 `cached_tokens` 及模态扩展字段，应增加官方 `cache_write_tokens`。 + +3. Responses DTO 中存在非当前官方 schema 的扩展字段。 + + `input_tokens_details.text_tokens/audio_tokens/image_tokens` 以及输出侧的文本、音频、预测 token 字段不在本次联网核验到的 OpenAI Responses 官方 schema 中。如果为了兼容第三方上游而保留，应明确标记为兼容扩展，不能当作 OpenAI 官方字段。 + +4. Claude DTO 缺少最新 Usage 字段。 + + [`relaykit/dto/claude.go`](relaykit/dto/claude.go) 当前缺少： + + - `output_tokens_details.thinking_tokens` + - `inference_geo` + - `server_tool_use.web_fetch_requests` + +5. Gemini DTO 缺少 `usageMetadata.serviceTier`。 + + [`relaykit/dto/gemini.go`](relaykit/dto/gemini.go) 已覆盖主要 token 字段和模态明细，但未覆盖当前官方 `serviceTier`。 + +6. Claude 转 OpenAI 的 usage 口径不符合 OpenAI 子集语义。 + + [`relaykit/relayconvert/internal/oai_chat/claude_to_openai_response.go`](relaykit/relayconvert/internal/oai_chat/claude_to_openai_response.go) 当前将 Claude `input_tokens` 直接映射为 OpenAI `prompt_tokens`，同时把 `cache_read_input_tokens` 放入 `cached_tokens`。这样 `cached_tokens` 并未包含在 `prompt_tokens` 中，与 OpenAI 官方语义不一致。 + + 严格的协议映射应为： + + ```text + prompt_tokens = + input_tokens + + cache_read_input_tokens + + cache_creation_input_tokens + + prompt_tokens_details.cached_tokens = cache_read_input_tokens + prompt_tokens_details.cache_write_tokens = cache_creation_input_tokens + total_tokens = prompt_tokens + completion_tokens + ``` + +7. Claude 内部 `TotalTokens` 不是跨协议总量。 + + 当前 Claude 路径使用 `input_tokens + output_tokens` 生成内部 `TotalTokens`，缓存 token 另存。这可以作为内部计费表示，但不能把该值解释为 Claude 请求实际处理的全部 token，也不能直接映射为 OpenAI `total_tokens`。 + +## 官方来源 + +以下页面均在 2026-08-16 实际联网获取并读取正文： + +### OpenAI + +- [Chat Completions API Reference](https://developers.openai.com/api/reference/resources/chat/subresources/completions/methods/create) +- [Responses API Reference](https://developers.openai.com/api/reference/resources/responses/methods/create) +- [Prompt Caching Guide](https://developers.openai.com/api/docs/guides/prompt-caching) + +### Anthropic + +- [Messages API Reference](https://platform.claude.com/docs/en/api/messages) +- [Prompt Caching](https://platform.claude.com/docs/en/build-with-claude/prompt-caching) + +### Google Gemini + +- [GenerateContent API Reference - UsageMetadata](https://ai.google.dev/api/generate-content#UsageMetadata) + +## 结论 + +对于缓存输入，四种协议应按两类处理： + +```text +包含型：OpenAI Chat、OpenAI Responses、Gemini +独立桶型：Claude +``` + +最新版 OpenAI 又进一步把包含型输入拆成“缓存读取、缓存写入、普通输入”三个计费类别。因此，项目后续不应只维护 `CacheIncludedInPrompt` 一个布尔值，还应显式支持 `cache_write_tokens`，并在协议转换时分别处理缓存读、缓存写和普通输入。 diff --git a/docs/大模型协议Usage字段调研.md b/docs/大模型协议Usage字段调研.md new file mode 100644 index 0000000..1314c3f --- /dev/null +++ b/docs/大模型协议Usage字段调研.md @@ -0,0 +1,219 @@ +# 大模型协议 Usage 字段调研 + +> 调研对象：OpenAI Chat Completions、OpenAI Responses、Claude Messages、Gemini generateContent 四个协议响应中的 token 用量（usage）结构。 +> 核心关注点：**存在缓存时，输入 token 是否包含缓存 token**。 +> 调研日期：2026-08-17（基于各官方文档当时的版本） + +## 核心结论 + +四家协议里三家是「包含」口径，**Claude 是唯一的「排除」口径**： + +| 协议 | 输入字段 | 缓存 token 是否含在内 | 未缓存输入怎么得到 | +|---|---|---|---| +| OpenAI Chat Completions | `prompt_tokens` | ✅ 包含，`cached_tokens` 是它的子集 | `prompt_tokens - cached_tokens` | +| OpenAI Responses | `input_tokens` | ✅ 包含，`cached_tokens` 是它的子集 | `input_tokens - cached_tokens - cache_write_tokens` | +| **Claude Messages** | `input_tokens` | ❌ **排除**，缓存是独立并列字段 | `input_tokens` 本身就是未缓存部分；总输入 = 三个字段**相加** | +| Gemini | `promptTokenCount` | ✅ 包含，`cachedContentTokenCount` 是它的子集 | `promptTokenCount - cachedContentTokenCount` | + +## 1. OpenAI Chat Completions（`/v1/chat/completions`） + +官方文档示例（chat/object）： + +```json +"usage": { + "prompt_tokens": 1117, + "completion_tokens": 46, + "total_tokens": 1163, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } +} +``` + +字段全集（据官方 SDK `completion_usage.py` 的 schema）： + +| 字段 | 说明 | +|---|---| +| `prompt_tokens` | 总输入 token，**包含**缓存命中部分 | +| `completion_tokens` | 总输出 token，**包含** reasoning tokens | +| `total_tokens` | = prompt_tokens + completion_tokens | +| `prompt_tokens_details.cached_tokens` | 缓存命中读到的输入 token（prompt_tokens 的子集） | +| `prompt_tokens_details.cache_write_tokens` | 本次写入缓存的 token（较新字段，prompt_tokens 的子集） | +| `prompt_tokens_details.audio_tokens / text_tokens / image_tokens` | 按模态细分（prompt_tokens 的子集） | +| `completion_tokens_details.reasoning_tokens` | 推理 token（completion_tokens 的子集） | +| `completion_tokens_details.audio_tokens / text_tokens` | 按模态细分（completion_tokens 的子集） | +| `completion_tokens_details.accepted_prediction_tokens / rejected_prediction_tokens` | Predicted Outputs 功能相关 | + +包含关系的官方例证（prompt caching guide）：`prompt_tokens: 2006, cached_tokens: 1920` —— 1920 是 2006 的一部分，剩 86 个未命中。 + +缓存计费：命中部分按折扣价计费（guide 现行口径 0.1×，实际各模型定价表不同，如 gpt-4o 为 0.5×）。可缓存的最小前缀为 1024 token，按 128 token 增量命中。 + +## 2. OpenAI Responses API（`/v1/responses`） + +usage 结构明显比 Chat Completions 精简（据官方 SDK `response_usage.py`）： + +```json +"usage": { + "input_tokens": 2600, + "input_tokens_details": { + "cached_tokens": 2000, + "cache_write_tokens": 400 + }, + "output_tokens": 300, + "output_tokens_details": { + "reasoning_tokens": 120 + }, + "total_tokens": 2900 +} +``` + +| 字段 | 说明 | +|---|---| +| `input_tokens` | 总输入，**包含**缓存命中和缓存写入（示例：2600 = 2000 cached + 400 write + 200 新输入） | +| `input_tokens_details.cached_tokens` | 缓存命中（子集） | +| `input_tokens_details.cache_write_tokens` | 本次写入缓存（子集）——Responses 特有字段，Chat Completions 侧较新才补上 | +| `output_tokens` | 总输出，**包含** reasoning_tokens（子集） | +| `output_tokens_details.reasoning_tokens` | 推理 token | +| `total_tokens` | = input_tokens + output_tokens | + +注意： +- **没有**模态细分、没有预测 token 字段。 +- 复用 `previous_response_id` 时每次都为整个已见对话历史计费，`input_tokens` 会随轮次增长，`cached_tokens` 会很大。 + +## 3. Claude Messages API（`/v1/messages`） + +官方 API reference 的 usage 字段全集： + +```json +"usage": { + "input_tokens": 2048, + "cache_read_input_tokens": 1800, + "cache_creation_input_tokens": 248, + "output_tokens": 503, + "cache_creation": { + "ephemeral_5m_input_tokens": 148, + "ephemeral_1h_input_tokens": 100 + }, + "output_tokens_details": { "thinking_tokens": 0 }, + "server_tool_use": { "web_search_requests": 0, "web_fetch_requests": 2 }, + "service_tier": "standard", + "inference_geo": "global" +} +``` + +| 字段 | 说明 | +|---|---| +| `input_tokens` | **仅计最后一个 cache 断点之后的输入**（未走缓存的常规输入） | +| `cache_read_input_tokens` | 从缓存读到的 | +| `cache_creation_input_tokens` | 本次写入缓存的 | +| `cache_creation.ephemeral_5m_input_tokens / ephemeral_1h_input_tokens` | 按 TTL 细分的缓存写入，两者之和 = cache_creation_input_tokens | +| `output_tokens` | 总输出（含 thinking），是计费权威值 | +| `output_tokens_details.thinking_tokens` | 思考 token（output_tokens 的子集，只读观测用） | +| `server_tool_use.web_search_requests / web_fetch_requests` | 服务端工具调用次数（按次计费） | +| `service_tier` | standard / priority / batch | +| `inference_geo` | 推理所在地理区域 | + +官方原文（关键）： + +> "Total input tokens in a request is the summation of `input_tokens`, `cache_creation_input_tokens`, and `cache_read_input_tokens`." + +> "The `input_tokens` field represents only the tokens that come after the last cache breakpoint in your request - not all the input tokens you sent." + +计费倍率（官方 prompt caching 文档）： + +| Token 类型 | 倍率（相对基础输入价） | +|---|---| +| 常规输入 | 1.0× | +| 5 分钟缓存写入（ephemeral_5m） | 1.25× | +| 1 小时缓存写入（ephemeral_1h） | 2× | +| 缓存读取 | 0.1× | + +老响应没有 `cache_creation` 细分时，`cache_creation_input_tokens` 默认按 5m（1.25×）计。 + +## 4. Gemini（`generateContent` 的 `usageMetadata`） + +注意：Gemini 的用量不在 `usage` 字段里，而是响应顶层的 `usageMetadata`，且为 camelCase： + +```json +"usageMetadata": { + "promptTokenCount": 1050, + "cachedContentTokenCount": 1000, + "candidatesTokenCount": 200, + "thoughtsTokenCount": 150, + "toolUsePromptTokenCount": 0, + "totalTokenCount": 1400, + "promptTokensDetails": [{ "modality": "TEXT", "tokenCount": 1050 }], + "cacheTokensDetails": [{ "modality": "TEXT", "tokenCount": 1000 }], + "candidatesTokensDetails": [{ "modality": "TEXT", "tokenCount": 200 }] +} +``` + +| 字段 | 说明 | +|---|---| +| `promptTokenCount` | 总输入，**包含**缓存部分 | +| `cachedContentTokenCount` | 其中命中缓存的（显式 context caching；2.5 系列隐式缓存命中也体现在此字段） | +| `candidatesTokenCount` | 输出 token（不含思考） | +| `thoughtsTokenCount` | 思考 token，**输出侧**，按输出价计费 | +| `toolUsePromptTokenCount` | 工具使用 prompt 的 token | +| `totalTokenCount` | = promptTokenCount + thoughtsTokenCount + candidatesTokenCount | +| `promptTokensDetails[] / cacheTokensDetails[] / candidatesTokensDetails[] / toolUsePromptTokensDetails[]` | 按模态（TEXT/IMAGE/AUDIO/VIDEO）细分的列表 | +| `serviceTier` | 服务层级 | + +官方原文（关键）： + +> "When `cachedContent` is set, this is still the total effective prompt size meaning this includes the number of tokens in the cached content." + +> "Total token count for the generation request (prompt + thoughts + response candidates)." + +## 计费换算公式 + +``` +OpenAI 系（两协议同构）: + 输入成本 = (input_tokens − cached_tokens) × P_in + cached_tokens × P_cached + 输出成本 = output_tokens × P_out # reasoning 已含在 output 里，不另计 + +Claude: + 输入成本 = input_tokens × P_in + + cache_creation_5m × P_in × 1.25 + cache_creation_1h × P_in × 2 + + cache_read × P_in × 0.1 + 输出成本 = output_tokens × P_out # thinking 已含在 output 里 + （总输入量展示用 = 三项相加，切勿把 input_tokens 当总输入） + +Gemini: + 输入成本 = (promptTokenCount − cachedContentTokenCount) × P_in + cachedContentTokenCount × P_cached + 输出成本 = (candidatesTokenCount + thoughtsTokenCount) × P_out +``` + +## 协议互转的坑 + +最容易踩的坑是 **Claude ↔ 其他三家**： + +- 把 Claude 的 `input_tokens` 直接映射成 OpenAI 的 `prompt_tokens`，会导致缓存场景下输入量少算（Claude 侧）或多算（反向）。 +- 正确做法：Claude → 其他，做**加法**（`input + cache_read + cache_creation` → 对方口径的「总输入」）；其他 → Claude，做**减法**（总输入 − 缓存命中 → `input_tokens`，缓存命中 → `cache_read_input_tokens`）。 +- 注意 OpenAI 的 `cached_tokens` 是「命中读取」语义，没有 Claude 的「写入 1.25×/2×」概念；反向转换时 Claude 的写入 token 一般并入常规输入或缓存读取，需按计费策略明确选一边。 + +## 流式响应中 usage 的位置 + +| 协议 | 流式 usage 出现位置 | +|---|---| +| OpenAI chat | 需设 `stream_options: {"include_usage": true}`，最后一个 chunk 带 usage | +| Responses | `response.completed` 事件的 `response.usage` | +| Claude | `message_start` 带输入侧 usage，`message_delta` 累计 `output_tokens`（已知问题：部分流式场景 `message_delta` 只带 `output_tokens`，输入侧字段缺失，需保留 `message_start` 的值合并） | +| Gemini | `streamGenerateContent` 最后一个 chunk 的 `usageMetadata` | + +## 参考来源 + +- [Gemini API — Generating content（usageMetadata 完整规格）](https://ai.google.dev/api/generate-content) +- [OpenAI — Chat Completions API Reference（chat object）](https://platform.openai.com/docs/api-reference/chat/object) +- [OpenAI — Prompt caching guide](https://developers.openai.com/api/docs/guides/prompt-caching) +- [openai-python — completion_usage.py（PromptTokensDetails/CompletionTokensDetails 权威 schema）](https://github.com/openai/openai-python/blob/main/src/openai/types/completion_usage.py) +- [openai-python — response_usage.py（Responses usage 权威 schema）](https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_usage.py) +- [Anthropic — Messages API Reference（usage object）](https://platform.claude.com/docs/en/api/messages) +- [Anthropic — Prompt caching（计费倍率与 token 拆解）](https://platform.claude.com/docs/en/docs/build-with-claude/prompt-caching) diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index e976bf7..eb2a3b1 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -149,6 +149,7 @@ var ( // Global middleware s.Use(middleware.Recovery) s.Use(middleware.RequestId) + s.Use(middleware.ServiceName) // Setup mode guard: block all requests until initialization is complete s.Use(func(r *ghttp.Request) { @@ -431,6 +432,10 @@ func registerRelayRoutes(server *ghttp.Server) { group.POST("/completions", relay.HandleCompletions) group.POST("/responses", relay.HandleResponses) group.POST("/responses/compact", relay.HandleResponses) + // Responses 生命周期端点（Redis 路由还原渠道后透传上游） + group.GET("/responses/{id}", relay.HandleResponsesRetrieve) + group.POST("/responses/{id}/cancel", relay.HandleResponsesCancel) + group.DELETE("/responses/{id}", relay.HandleResponsesDelete) group.POST("/messages", relay.HandleMessages) group.POST("/audio/speech", relay.HandleAudioSpeech) group.POST("/audio/transcriptions", relay.HandleAudioTranscription) diff --git a/internal/dao/internal/bil_usage_daily.go b/internal/dao/internal/bil_usage_daily.go index 3b2ceb0..41210b8 100644 --- a/internal/dao/internal/bil_usage_daily.go +++ b/internal/dao/internal/bil_usage_daily.go @@ -21,42 +21,48 @@ type BilUsageDailyDao struct { // BilUsageDailyColumns defines and stores column names for the table bil_usage_daily. type BilUsageDailyColumns struct { - Id string // - StatDate string // 统计日期（按 created_at 的日期分桶） - TenantId string // 租户ID - ProjectId string // 项目ID（0=无项目/个人Key） - ModelName string // 模型名称 - ChannelId string // 渠道ID（0=无渠道） - Status string // 请求状态：success/error/timeout/cancelled - RequestCount string // 请求数（COUNT(*)） - InputTokens string // 输入Token合计 - OutputTokens string // 输出Token合计 - TotalCost string // 客户侧成本合计（USD，源自 bil_usage_logs.total_cost） - AccountCost string // 上游账户成本合计（USD，源自 bil_usage_logs.account_cost，用于利润分析） - SumLatencyMs string // 总延迟合计（ms，源自 bil_usage_logs.latency_ms；视图按 SUM/COUNT 求均值） - SumFirstTokenMs string // 首Token延迟合计（ms，源自 bil_usage_logs.first_token_ms；视图按 SUM/COUNT 求均值） - CreatedAt string // - UpdatedAt string // + Id string // + StatDate string // 统计日期（按 created_at 的日期分桶） + TenantId string // 租户ID + ProjectId string // 项目ID（0=无项目/个人Key） + ModelName string // 模型名称 + ChannelId string // 渠道ID（0=无渠道） + Status string // 请求状态：success/error/timeout/cancelled + RequestCount string // 请求数（COUNT(*)） + InputTokens string // 输入Token合计 + OutputTokens string // 输出Token合计 + TotalCost string // 客户侧成本合计（USD，源自 bil_usage_logs.total_cost） + AccountCost string // 上游账户成本合计（USD，源自 bil_usage_logs.account_cost，用于利润分析） + SumLatencyMs string // 总延迟合计（ms，源自 bil_usage_logs.latency_ms；视图按 SUM/COUNT 求均值） + SumFirstTokenMs string // 首Token延迟合计（ms，源自 bil_usage_logs.first_token_ms；视图按 SUM/COUNT 求均值） + CreatedAt string // + UpdatedAt string // + CacheCreationTokens string // 缓存创建 token 数（Claude cache_creation，SUM 聚合） + CacheReadTokens string // 缓存命中读取 token 数（Claude cache_read / OpenAI cached_tokens，SUM 聚合） + CacheHitRequestCount string // 命中缓存的请求数（明细 cache_read_tokens>0 计 1，含失败状态行） } // bilUsageDailyColumns holds the columns for the table bil_usage_daily. var bilUsageDailyColumns = BilUsageDailyColumns{ - Id: "id", - StatDate: "stat_date", - TenantId: "tenant_id", - ProjectId: "project_id", - ModelName: "model_name", - ChannelId: "channel_id", - Status: "status", - RequestCount: "request_count", - InputTokens: "input_tokens", - OutputTokens: "output_tokens", - TotalCost: "total_cost", - AccountCost: "account_cost", - SumLatencyMs: "sum_latency_ms", - SumFirstTokenMs: "sum_first_token_ms", - CreatedAt: "created_at", - UpdatedAt: "updated_at", + Id: "id", + StatDate: "stat_date", + TenantId: "tenant_id", + ProjectId: "project_id", + ModelName: "model_name", + ChannelId: "channel_id", + Status: "status", + RequestCount: "request_count", + InputTokens: "input_tokens", + OutputTokens: "output_tokens", + TotalCost: "total_cost", + AccountCost: "account_cost", + SumLatencyMs: "sum_latency_ms", + SumFirstTokenMs: "sum_first_token_ms", + CreatedAt: "created_at", + UpdatedAt: "updated_at", + CacheCreationTokens: "cache_creation_tokens", + CacheReadTokens: "cache_read_tokens", + CacheHitRequestCount: "cache_hit_request_count", } // NewBilUsageDailyDao creates and returns a new DAO object for table data access. diff --git a/internal/dao/internal/mdl_tenant_models.go b/internal/dao/internal/mdl_tenant_models.go index 4f603b1..885b674 100644 --- a/internal/dao/internal/mdl_tenant_models.go +++ b/internal/dao/internal/mdl_tenant_models.go @@ -27,7 +27,7 @@ type MdlTenantModelsColumns struct { Enabled string // 是否启用（禁用后该租户无法调用此模型） CustomInputPrice string // 租户自定义输入价格（NULL 表示使用默认定价） CustomOutputPrice string // 租户自定义输出价格（NULL 表示使用默认定价） - Multiplier string // 租户价格倍率（VIP 折扣等，最终价格 = 基础价格 × 模型倍率 × 租户倍率） + Multiplier string // 租户价格倍率（VIP 折扣）。作为 tenant_multiplier 参与最终价格计算：最终价格 = 基础价格 × 租户倍率。倍率来源优先级：discount_ratio > multiplier > 租户等级 price_multiplier CreatedAt string // 创建时间 UpdatedAt string // 更新时间 BillingMode string // 覆盖模型计费方式（NULL表示跟随模型默认） diff --git a/internal/handler/relay/response_capture.go b/internal/handler/relay/response_capture.go index 09e3585..8bb6fb0 100644 --- a/internal/handler/relay/response_capture.go +++ b/internal/handler/relay/response_capture.go @@ -1,10 +1,12 @@ package relay import ( + "bytes" "net/http" "strings" "sync" "sync/atomic" + "unicode/utf8" ) const maxCaptureSize = 1 << 18 // 256KB total capture budget @@ -58,8 +60,15 @@ func (w *ResponseCaptureWriter) Write(b []byte) (int, error) { if len(b) <= remaining { w.buf = append(w.buf, b...) } else { - w.buf = append(w.buf, b[:remaining]...) - w.appendToTail(b[remaining:]) + // 切点回退到多字节字符的起始字节（UTF-8 最长 4 字节，最多回退 3 字节）， + // 整字符归 tail，避免 head 末尾留下半个字符产生无效 UTF-8。 + // 跨 Write 到达的割裂字符无法在此对齐，由 Body() 出口清洗兜底。 + cut := remaining + for cut > 0 && remaining-cut < 3 && !utf8.RuneStart(b[cut]) { + cut-- + } + w.buf = append(w.buf, b[:cut]...) + w.appendToTail(b[cut:]) } } else { w.appendToTail(b) @@ -96,22 +105,31 @@ func (w *ResponseCaptureWriter) Flush() { // Body 返回已捕获的响应体内容。 // 短响应（<= headLimit）直接返回；长响应保留 head + truncation marker + tail。 +// 两个无效 UTF-8 来源都在出口兜底：分块到达的流可能在任意字节边界切开多字节字符 +// （跨 Write 的割裂无法在写入侧对齐），上游也可能直接输出非法字节；不清洗会导致 +// 审计日志 INSERT 被 UTF8 编码的 PostgreSQL 以 invalid byte sequence 拒绝、整条丢失。 func (w *ResponseCaptureWriter) Body() string { w.mu.Lock() defer w.mu.Unlock() if w.tailBuf == nil { - return string(w.buf) + return strings.ToValidUTF8(string(w.buf), string(utf8.RuneError)) } var sb strings.Builder sb.Write(w.buf) sb.WriteString("\n...[truncated]...\n") if w.tailWrapped { - sb.Write(w.tailBuf[w.tailPos:]) + // 环形缓冲区起点落在残行中间：残行前半已被覆盖、不可恢复， + // 跳到下一行行首，让 tail 从完整 SSE 行开始；找不到换行则原样输出。 + tail := w.tailBuf[w.tailPos:] + if idx := bytes.IndexByte(tail, '\n'); idx >= 0 { + tail = tail[idx+1:] + } + sb.Write(tail) sb.Write(w.tailBuf[:w.tailPos]) } else { sb.Write(w.tailBuf[:w.tailPos]) } - return sb.String() + return strings.ToValidUTF8(sb.String(), string(utf8.RuneError)) } // StatusCode 返回捕获的 HTTP 状态码 diff --git a/internal/handler/relay/response_capture_test.go b/internal/handler/relay/response_capture_test.go index 34f822c..04a93af 100644 --- a/internal/handler/relay/response_capture_test.go +++ b/internal/handler/relay/response_capture_test.go @@ -3,7 +3,9 @@ package relay import ( "net/http" "net/http/httptest" + "strings" "testing" + "unicode/utf8" ) func TestResponseCaptureWriterCommitted(t *testing.T) { @@ -46,3 +48,71 @@ func TestResponseCaptureWriterKeepsFirstStatus(t *testing.T) { t.Fatalf("status = %d, want first status %d", got, http.StatusAccepted) } } + +// 捕获体最终要写入 UTF8 编码的 PostgreSQL，任何无效 UTF-8 字节都会让审计 INSERT +// 整条被拒。以下用例覆盖合法短响应、上游脏字节、head/tail 分界、环形回绕等边界， +// 保证 Body() 恒返回合法 UTF-8。 +func TestResponseCaptureBodyRoundTrip(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte("data: 你好\n\n")) + if got := capture.Body(); got != "data: 你好\n\n" { + t.Fatalf("short valid body should round-trip, got %q", got) + } +} + +// 上游直接输出非法字节（供应商 bug、非 UTF-8 错误页），出口清洗兜底 +func TestResponseCaptureBodySanitizesInvalidUpstreamBytes(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte("ok \xe4\xb8")) // 半个“中” + got := capture.Body() + if !utf8.ValidString(got) { + t.Fatalf("body should be valid UTF-8, got %q", got) + } + if !strings.ContainsRune(got, utf8.RuneError) { + t.Fatalf("invalid sequence should be replaced with U+FFFD, got %q", got) + } +} + +// head/tail 分界点落在多字节字符中间时，整字符应归 tail，不留半个字符 +func TestResponseCaptureHeadTailSplitKeepsCharIntact(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte(strings.Repeat("a", headLimit-2))) + _, _ = capture.Write([]byte("中xyz")) + got := capture.Body() + if !utf8.ValidString(got) { + t.Fatal("body should be valid UTF-8 after boundary split") + } + if !strings.HasSuffix(got, "中xyz") { + t.Fatalf("char straddling the boundary should stay intact in tail, got %q", got[len(got)-20:]) + } +} + +// 环形缓冲区回绕后起点落在字符中间（且无换行可对齐），仅靠出口清洗兜底 +func TestResponseCaptureRingWrapBodyValid(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte(strings.Repeat("a", headLimit))) + // 21846 个“中”共 65538 字节，回绕后 tailPos=2，环起点落在“中”的最后一个字节上 + _, _ = capture.Write([]byte(strings.Repeat("中", 21846))) + if got := capture.Body(); !utf8.ValidString(got) { + t.Fatal("body should be valid UTF-8 after ring wrap") + } +} + +// 环形回绕起点落在残行中间时应跳到下一行行首，tail 从完整 SSE 行开始 +func TestResponseCaptureRingWrapAlignsToLineStart(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte(strings.Repeat("a", headLimit))) + // “d:中中\n”每行 9 字节，14564 行共 131076 字节，回绕后 tailPos=4（落在行首“中”中间） + _, _ = capture.Write([]byte(strings.Repeat("d:中中\n", 14564))) + body := capture.Body() + if !utf8.ValidString(body) { + t.Fatal("body should be valid UTF-8 after ring wrap") + } + prefix := strings.Repeat("a", headLimit) + "\n...[truncated]...\n" + if !strings.HasPrefix(body, prefix) { + t.Fatal("body should start with head + truncation marker") + } + if rest := body[len(prefix):]; !strings.HasPrefix(rest, "d:") { + t.Fatalf("tail should start at a complete line, got %q", rest) + } +} diff --git a/internal/handler/relay/responses.go b/internal/handler/relay/responses.go new file mode 100644 index 0000000..8e45b25 --- /dev/null +++ b/internal/handler/relay/responses.go @@ -0,0 +1,53 @@ +package relay + +import ( + "net/http" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/ghttp" + + relay_constant "github.com/qianfree/team-api/relay/constant" + "github.com/qianfree/team-api/relay/handler" +) + +// HandleResponsesRetrieve 处理 GET /v1/responses/{id}（retrieve，background 模式轮询用） +func HandleResponsesRetrieve(r *ghttp.Request) { + handleResponsesLifecycle(r, http.MethodGet) +} + +// HandleResponsesCancel 处理 POST /v1/responses/{id}/cancel（取消进行中的响应） +func HandleResponsesCancel(r *ghttp.Request) { + handleResponsesLifecycle(r, http.MethodPost) +} + +// HandleResponsesDelete 处理 DELETE /v1/responses/{id}（删除已存储的响应） +func HandleResponsesDelete(r *ghttp.Request) { + handleResponsesLifecycle(r, http.MethodDelete) +} + +// handleResponsesLifecycle Responses 生命周期端点入口： +// 无请求体（ApiKeyAuth 纯 header 鉴权、ContentFilter 空 body 放行）， +// 不走 validateRelayRequest（无 model），scope/IP 校验对齐 HandleModels 先例。 +func handleResponsesLifecycle(r *ghttp.Request, method string) { + responseID := r.Get("id").String() + if responseID == "" { + r.Response.WriteStatus(http.StatusBadRequest, g.Map{ + "error": g.Map{"type": "invalid_request_error", "message": "response id is required"}, + }) + return + } + + rc := buildRelayContext(r) + if !billingProvider.CheckScope(rc.Scope, "responses") { + handler.WriteRelayError(r.Response.Writer, relay_constant.NewAuthError("API key scope denied")) + return + } + if !billingProvider.CheckIPWhitelist(rc.KeyIpWhitelist, rc.ClientIP) { + handler.WriteRelayError(r.Response.Writer, relay_constant.NewAuthError("IP address is not allowed")) + return + } + + if err := handler.HandleResponsesLifecycle(r.Context(), method, responseID, rc, dataProvider); err != nil { + handler.WriteRelayError(r.Response.Writer, err) + } +} diff --git a/internal/handler/setup/setup_test.go b/internal/handler/setup/setup_test.go index 3bbd618..72b0aa1 100644 --- a/internal/handler/setup/setup_test.go +++ b/internal/handler/setup/setup_test.go @@ -3,7 +3,6 @@ package setup_test import ( "encoding/json" "net/http" - "net/http/httptest" "strings" "testing" @@ -12,15 +11,14 @@ import ( "github.com/qianfree/team-api/internal/consts" setuphandler "github.com/qianfree/team-api/internal/handler/setup" + "github.com/qianfree/team-api/internal/testutil" ) func TestHandleSetupInitializeValidationErrors(t *testing.T) { s := g.Server(guid.S()) - s.SetDumpRouterMap(false) s.BindHandler("/initialize", setuphandler.HandleSetupInitialize) - server := httptest.NewServer(s) - defer server.Close() + baseURL := testutil.StartGFServer(t, s) tests := []struct { name string @@ -46,8 +44,8 @@ func TestHandleSetupInitializeValidationErrors(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - res, err := server.Client().Post( - server.URL+"/initialize", + res, err := http.Post( + baseURL+"/initialize", "application/json", strings.NewReader(tt.body), ) diff --git a/internal/logic/common/model_perf_metrics.go b/internal/logic/common/model_perf_metrics.go index 4d453eb..0568bab 100644 --- a/internal/logic/common/model_perf_metrics.go +++ b/internal/logic/common/model_perf_metrics.go @@ -23,6 +23,8 @@ import ( // SUM(latency_ms) 对 NULL 跳过、但 SUM/COUNT 用全部请求数作分母的语义一致） // - 平均首 Token = Σttft / Σttft_n（首 Token 单独计数，避免非流式/无首Token请求计入分母拉低均值） // - 成本以整数 micro-USD 累加（对齐 Redis 钱包 micro 规范，杜绝 HINCRBYFLOAT 漂移） +// - 缓存指标与日聚合对齐：cache 命中请求数（chit）按「该次请求 cache_read>0」计 1， +// 仅写入缓存（creation>0、read=0）不计命中 // // 可靠性为 best-effort：Redis 不可用时计数丢弃、查询返回空，不影响请求主链路，页面退化为历史日表。 const ( @@ -41,6 +43,9 @@ type ModelPerfRecord struct { InputTokens int OutputTokens int CostUSD float64 + // 缓存 token（命中判定由本模块从 CacheReadTokens>0 推导，调用方无需单独传标志） + CacheCreationTokens int + CacheReadTokens int } // ModelPerfCounter 当日某模型的累计计数（Redis 小时桶聚合结果）。 @@ -54,6 +59,10 @@ type ModelPerfCounter struct { Tin int64 Tout int64 CostMicro int64 + // 缓存累计：creation/read token 与 命中缓存的请求数（与 bil_usage_daily 同口径） + CacheCreation int64 + CacheRead int64 + CacheHitReq int64 } func modelPerfModelsKey(date string) string { @@ -84,7 +93,8 @@ func microFromUSD(usd float64) int64 { // modelPerfIncrLua 一次 RTT 把样本原子累加到「当日模型集合 + 小时桶」并续期。 // KEYS[1] = perf_models:{date}；KEYS[2] = perf_model:{hex}:{hour} // ARGV[1] = 模型名；ARGV[2] = TTL 秒；ARGV[3] = ok(1/0)；ARGV[4] = lat ms； -// ARGV[5] = ttft ms；ARGV[6] = input tokens；ARGV[7] = output tokens；ARGV[8] = cost micro-USD。 +// ARGV[5] = ttft ms；ARGV[6] = input tokens；ARGV[7] = output tokens；ARGV[8] = cost micro-USD； +// ARGV[9] = cache creation tokens；ARGV[10] = cache read tokens（>0 时 chit 同步 +1）。 const modelPerfIncrLua = ` redis.call("SADD", KEYS[1], ARGV[1]) redis.call("EXPIRE", KEYS[1], ARGV[2]) @@ -98,14 +108,20 @@ end if tonumber(ARGV[6]) > 0 then redis.call("HINCRBY", KEYS[2], "tin", tonumber(ARGV[6])) end if tonumber(ARGV[7]) > 0 then redis.call("HINCRBY", KEYS[2], "tout", tonumber(ARGV[7])) end if tonumber(ARGV[8]) > 0 then redis.call("HINCRBY", KEYS[2], "cost", tonumber(ARGV[8])) end +if tonumber(ARGV[9]) > 0 then redis.call("HINCRBY", KEYS[2], "ccr", tonumber(ARGV[9])) end +if tonumber(ARGV[10]) > 0 then + redis.call("HINCRBY", KEYS[2], "crd", tonumber(ARGV[10])) + redis.call("HINCRBY", KEYS[2], "chit", 1) +end redis.call("EXPIRE", KEYS[2], ARGV[2]) return 1 ` // modelPerfReadLua 累加某模型当日所有小时桶计数。KEYS = 当日各小时桶 key。 -// 返回扁平数组 [req, ok, lat, ttft, ttft_n, tin, tout, cost]。 +// 返回扁平数组 [req, ok, lat, ttft, ttft_n, tin, tout, cost, ccr, crd, chit]。 +// 旧版本桶缺失 ccr/crd/chit 字段时 tonumber(nil) or 0 兜底为 0，平滑上线。 const modelPerfReadLua = ` -local req,ok,lat,ttft,ttftn,tin,tout,cost = 0,0,0,0,0,0,0,0 +local req,ok,lat,ttft,ttftn,tin,tout,cost,ccr,crd,chit = 0,0,0,0,0,0,0,0,0,0,0 for i = 1, #KEYS do local h = redis.call("HGETALL", KEYS[i]) for j = 1, #h, 2 do @@ -118,10 +134,13 @@ for i = 1, #KEYS do elseif f == "ttft_n" then ttftn = ttftn + v elseif f == "tin" then tin = tin + v elseif f == "tout" then tout = tout + v - elseif f == "cost" then cost = cost + v end + elseif f == "cost" then cost = cost + v + elseif f == "ccr" then ccr = ccr + v + elseif f == "crd" then crd = crd + v + elseif f == "chit" then chit = chit + v end end end -return {req, ok, lat, ttft, ttftn, tin, tout, cost} +return {req, ok, lat, ttft, ttftn, tin, tout, cost, ccr, crd, chit} ` // RecordModelPerfMetrics 将一次请求的模型性能采样累加到 Redis 当日小时桶。 @@ -153,6 +172,7 @@ func RecordModelPerfMetrics(ctx context.Context, r ModelPerfRecord) { boolToInt(r.Success), int64(math.Round(r.LatencyMs)), int64(r.FirstTokenMs), int64(r.InputTokens), int64(r.OutputTokens), microFromUSD(r.CostUSD), + int64(r.CacheCreationTokens), int64(r.CacheReadTokens), } if _, err := g.Redis().Do(bgCtx, "EVAL", args...); err != nil { // 指标是 best-effort，Redis 抖动只记 debug，不刷告警 @@ -203,14 +223,17 @@ func GetModelPerfMetricsToday(ctx context.Context, date string) (map[string]Mode } vals := res.Vars() out[model] = ModelPerfCounter{ - Req: vals[0].Int64(), - Ok: vals[1].Int64(), - Lat: vals[2].Int64(), - Ttft: vals[3].Int64(), - TtftN: vals[4].Int64(), - Tin: vals[5].Int64(), - Tout: vals[6].Int64(), - CostMicro: vals[7].Int64(), + Req: vals[0].Int64(), + Ok: vals[1].Int64(), + Lat: vals[2].Int64(), + Ttft: vals[3].Int64(), + TtftN: vals[4].Int64(), + Tin: vals[5].Int64(), + Tout: vals[6].Int64(), + CostMicro: vals[7].Int64(), + CacheCreation: vals[8].Int64(), + CacheRead: vals[9].Int64(), + CacheHitReq: vals[10].Int64(), } } return out, nil diff --git a/internal/logic/common/usage_log_writer.go b/internal/logic/common/usage_log_writer.go index e8ff3a9..1850fc5 100644 --- a/internal/logic/common/usage_log_writer.go +++ b/internal/logic/common/usage_log_writer.go @@ -239,8 +239,27 @@ func (w *UsageLogWriter) flushBatch(records []any) { ctx := gctx.New() _, err := w.dbInstance().Model(w.table).Ctx(ctx).Data(batch).Batch(len(batch)).Insert() if err != nil { - w.failed.Add(int64(len(batch))) - g.Log().Errorf(ctx, "usage_log_writer: batch insert %d records failed: %v", len(batch), err) + // 批量写入失败（如某条记录含非法 UTF-8 字节被 PG 整体拒绝）时降级为逐条写入， + // 避免一条脏数据拖垮整批记录（usage_logs 为计费数据，audit_logs 不允许丢失）； + // 单条仍失败才丢弃该条，并汇总打日志。 + g.Log().Errorf(ctx, "usage_log_writer: batch insert %d records failed: %v, fallback to per-record insert", len(batch), err) + saved := 0 + var firstErr error + for _, rec := range batch { + if _, e := w.dbInstance().Model(w.table).Ctx(ctx).Data(rec).Insert(); e != nil { + if firstErr == nil { + firstErr = e + } + continue + } + saved++ + } + w.completed.Add(int64(saved)) + dropped := len(batch) - saved + w.failed.Add(int64(dropped)) + if firstErr != nil { + g.Log().Errorf(ctx, "usage_log_writer: %d/%d records dropped in per-record fallback, first error: %v", dropped, len(batch), firstErr) + } return } w.completed.Add(int64(len(batch))) diff --git a/internal/logic/monitor/api_metrics.go b/internal/logic/monitor/api_metrics.go index 2015a43..f6deedf 100644 --- a/internal/logic/monitor/api_metrics.go +++ b/internal/logic/monitor/api_metrics.go @@ -121,6 +121,10 @@ type modelPerfRow struct { TotalCost float64 `json:"total_cost"` SumLatencyMs int64 `json:"sum_latency_ms"` SumFirstTokenMs int64 `json:"sum_first_token_ms"` + // 缓存聚合列（000016 迁移新增，口径与 bil_usage_logs 明细一致） + CacheCreationTokens int64 `json:"cache_creation_tokens"` + CacheReadTokens int64 `json:"cache_read_tokens"` + CacheHitRequestCount int64 `json:"cache_hit_request_count"` } // gradeSuccessRate 按成功率返回分级。阈值：≥99 excellent、≥95 good、≥90 warning、<90 critical。 @@ -146,15 +150,23 @@ type perfModelAgg struct { sumLatencyMs int64 sumFirstTokenMs int64 totalCost float64 + // 缓存累计（历史日表 + 当日热桶同口径累加） + cacheCreationTokens int64 + cacheReadTokens int64 + cacheHitRequests int64 } -// GetModelPerformance 从 bil_usage_daily 按模型聚合历史性能指标，并合并当日 Redis 实时热桶 -// （见 common.RecordModelPerfMetrics，桶粒度小时），使页面当天即可看到最新数据，不再滞后一天。 +// GetModelPerformance 从 bil_usage_daily 按模型聚合历史性能指标，并合并当日实时数据， +// 支持可选的渠道（channelID，0=全部）与模型（modelName，空=全部）过滤：选定渠道后可查看 +// 该渠道下各模型的性能。历史段经 bil_usage_daily（唯一键含 channel_id）过滤； +// 当天段在渠道过滤下改走 bil_usage_logs 明细实时聚合（热桶无渠道维度），两段口径一致。 // 指标口径与 bil_usage_daily 一致：延迟按 SUM 求和、视图以 SUM/COUNT 求均值（分位数不可跨桶加，p95/p99 留后续）。 -func GetModelPerformance(ctx context.Context, startDate, endDate string) (any, error) { +func GetModelPerformance(ctx context.Context, startDate, endDate string, channelID int64, modelName string) (any, error) { today := time.Now().Format("2006-01-02") - // 历史段：stat_date < today（当天由 Redis 热桶实时提供，避免与每日 01:00 的日聚合重复/滞后） + // 历史段：stat_date < today（当天由实时数据提供，避免与每日 01:00 的日聚合重复/滞后）。 + // 渠道/模型过滤用「参数=哨兵值 OR 等值匹配」写法保持 SQL 常量字符串，未筛选时等价于无条件； + // $4/$5 显式转型规避 PostgreSQL 对 OR 两侧参数的类型推断歧义。 const query = ` SELECT model_name, @@ -164,32 +176,139 @@ func GetModelPerformance(ctx context.Context, startDate, endDate string) (any, e COALESCE(SUM(output_tokens), 0) AS output_tokens, COALESCE(SUM(total_cost), 0) AS total_cost, COALESCE(SUM(sum_latency_ms), 0) AS sum_latency_ms, - COALESCE(SUM(sum_first_token_ms), 0) AS sum_first_token_ms + COALESCE(SUM(sum_first_token_ms), 0) AS sum_first_token_ms, + COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens, + COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens, + COALESCE(SUM(cache_hit_request_count), 0) AS cache_hit_request_count FROM bil_usage_daily WHERE stat_date >= $1 AND stat_date <= $2 AND stat_date < $3 AND model_name <> 'unknown' + AND ($4::bigint = 0 OR channel_id = $4) + AND ($5::text = '' OR model_name = $5) GROUP BY model_name ` var rows []modelPerfRow - if err := g.DB().Ctx(ctx).Raw(query, startDate, endDate, today).Scan(&rows); err != nil { + if err := g.DB().Ctx(ctx).Raw(query, startDate, endDate, today, channelID, modelName).Scan(&rows); err != nil { return nil, err } - // 当日 Redis 热桶（best-effort：仅当查询窗口包含今天时参与；Redis 异常则忽略当日数据，退化为历史日表） + // 当天段（best-effort：仅当查询窗口包含今天时参与；数据源失败则忽略当日数据，退化为历史日表）： + // - 选定渠道：Redis 热桶无渠道维度，改从 bil_usage_logs 当天明细实时聚合（输出同构，合并逻辑复用） + // - 未选渠道：保持热桶路径与现状一致；仅选模型时热桶键即模型名，按键过滤 todayCounts := map[string]common.ModelPerfCounter{} if endDate == today { - tc, err := common.GetModelPerfMetricsToday(ctx, today) - if err != nil { - g.Log().Warningf(ctx, "model performance today redis read failed, fallback to daily only: %v", err) + if channelID > 0 { + tc, err := getModelPerfTodayFromLogs(ctx, today, channelID, modelName) + if err != nil { + g.Log().Warningf(ctx, "model performance today logs aggregate failed, fallback to daily only: %v", err) + } else { + todayCounts = tc + } } else { - todayCounts = tc + tc, err := common.GetModelPerfMetricsToday(ctx, today) + if err != nil { + g.Log().Warningf(ctx, "model performance today redis read failed, fallback to daily only: %v", err) + } else { + todayCounts = filterTodayCounts(tc, modelName) + } } } return map[string]any{"list": buildModelPerformanceList(rows, todayCounts)}, nil } +// filterTodayCounts 热桶结果按模型过滤（热桶键即模型名；modelName 空 = 原样返回）。 +// 未命中返回 nil，交给合并层按空处理。 +func filterTodayCounts(tc map[string]common.ModelPerfCounter, modelName string) map[string]common.ModelPerfCounter { + if modelName == "" { + return tc + } + if c, ok := tc[modelName]; ok { + return map[string]common.ModelPerfCounter{modelName: c} + } + return nil +} + +// modelPerfTodayRow 当天明细聚合行（bil_usage_logs 按模型分组，仅渠道筛选路径使用）。 +// 列名对齐 common.ModelPerfCounter 语义；口径与 usage_aggregation 写入 bil_usage_daily 一致： +// ok=成功请求、ttft=Σ首Token（无首Token请求计 0）、ttft_n=有首Token样本数、chit=命中缓存的请求数。 +type modelPerfTodayRow struct { + ModelName string `json:"model_name"` + Req int64 `json:"req"` + Ok int64 `json:"ok"` + Lat int64 `json:"lat"` + Ttft int64 `json:"ttft"` + TtftN int64 `json:"ttft_n"` + Tin int64 `json:"tin"` + Tout int64 `json:"tout"` + TotalCost float64 `json:"total_cost"` + CacheCreation int64 `json:"cache_creation"` + CacheRead int64 `json:"cache_read"` + Chit int64 `json:"chit"` +} + +// getModelPerfTodayFromLogs 渠道筛选下的当天数据源：从 bil_usage_logs 明细按模型实时聚合。 +// Redis 热桶（perf_model:{model}:{hour}）无渠道维度，选定渠道后当天必须回明细表； +// 时间区间沿用日聚合的半开区间口径 [today 00:00, tomorrow 00:00)，命中 created_at 的 BRIN 索引 +// （tomorrow 从 today 参数推导而非另取 now，避免跨午夜瞬间的边界漂移）。 +// 输出与 common.GetModelPerfMetricsToday 同构，合并逻辑复用不分叉。 +func getModelPerfTodayFromLogs(ctx context.Context, today string, channelID int64, modelName string) (map[string]common.ModelPerfCounter, error) { + dayStart, err := time.ParseInLocation("2006-01-02", today, time.Local) + if err != nil { + return nil, err + } + start := today + " 00:00:00" + end := dayStart.AddDate(0, 0, 1).Format("2006-01-02") + " 00:00:00" + const query = ` + SELECT + model_name AS model_name, + COUNT(*) AS req, + COUNT(*) FILTER (WHERE status = 'success') AS ok, + COALESCE(SUM(latency_ms), 0) AS lat, + COALESCE(SUM(first_token_ms), 0) AS ttft, + COUNT(*) FILTER (WHERE first_token_ms > 0) AS ttft_n, + COALESCE(SUM(input_tokens), 0) AS tin, + COALESCE(SUM(output_tokens), 0) AS tout, + COALESCE(SUM(total_cost), 0) AS total_cost, + COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation, + COALESCE(SUM(cache_read_tokens), 0) AS cache_read, + COUNT(*) FILTER (WHERE cache_read_tokens > 0) AS chit + FROM bil_usage_logs + WHERE created_at >= $1 AND created_at < $2 AND channel_id = $3 + AND ($4::text = '' OR model_name = $4) + GROUP BY model_name + ` + var rows []modelPerfTodayRow + if err := g.DB().Ctx(ctx).Raw(query, start, end, channelID, modelName).Scan(&rows); err != nil { + return nil, err + } + return rowsToTodayCounters(rows), nil +} + +// rowsToTodayCounters 当天明细聚合行 → 热桶同构计数。 +// total_cost 以 float 聚合后转整数 micro-USD（common.microFromUSD 未导出，此处同式内联）， +// 合并侧统一经 billing.FromMicro 还原 USD，保证两条当天数据路径口径一致。 +func rowsToTodayCounters(rows []modelPerfTodayRow) map[string]common.ModelPerfCounter { + out := make(map[string]common.ModelPerfCounter, len(rows)) + for _, r := range rows { + out[r.ModelName] = common.ModelPerfCounter{ + Req: r.Req, + Ok: r.Ok, + Lat: r.Lat, + Ttft: r.Ttft, + TtftN: r.TtftN, + Tin: r.Tin, + Tout: r.Tout, + CostMicro: int64(math.Round(r.TotalCost * 1e6)), + CacheCreation: r.CacheCreation, + CacheRead: r.CacheRead, + CacheHitReq: r.Chit, + } + } + return out +} + // buildModelPerformanceList 将历史日表聚合行与当日 Redis 热桶合并为统一的模型性能列表， -// 统一重算派生指标（成功率/均延迟/均首Token/TPS/分级）并按 request_count 降序输出。 +// 统一重算派生指标（成功率/均延迟/均首Token/TPS/分级/缓存命中率）并按模型名称升序输出。 func buildModelPerformanceList(rows []modelPerfRow, todayCounts map[string]common.ModelPerfCounter) []map[string]any { agg := make(map[string]*perfModelAgg) ensure := func(model string) *perfModelAgg { @@ -212,6 +331,9 @@ func buildModelPerformanceList(rows []modelPerfRow, todayCounts map[string]commo a.sumLatencyMs += r.SumLatencyMs a.sumFirstTokenMs += r.SumFirstTokenMs a.totalCost += r.TotalCost + a.cacheCreationTokens += r.CacheCreationTokens + a.cacheReadTokens += r.CacheReadTokens + a.cacheHitRequests += r.CacheHitRequestCount } for model, c := range todayCounts { if c.Req <= 0 || model == "" || model == "unknown" { @@ -225,6 +347,9 @@ func buildModelPerformanceList(rows []modelPerfRow, todayCounts map[string]commo a.sumLatencyMs += c.Lat a.sumFirstTokenMs += c.Ttft a.totalCost += billing.FromMicro(c.CostMicro).InexactFloat64() + a.cacheCreationTokens += c.CacheCreation + a.cacheReadTokens += c.CacheRead + a.cacheHitRequests += c.CacheHitReq } list := make([]map[string]any, 0, len(agg)) @@ -238,19 +363,34 @@ func buildModelPerformanceList(rows []modelPerfRow, todayCounts map[string]commo if a.sumLatencyMs > 0 { tps = float64(a.outputTokens) / (float64(a.sumLatencyMs) / 1000) } + // 缓存命中率（Token 级）= 命中读取 / 总输入。input_tokens 已统一为「含缓存总输入」口径 + //（Claude 渠道入库时补加缓存读写，OpenAI/Gemini 原生即含），分母直接取 input_tokens， + // 跨渠道口径一致。请求级命中率分母为 request_count，不受该口径影响。 + var cacheHitRate, cacheHitReqRate float64 + if a.inputTokens > 0 { + cacheHitRate = float64(a.cacheReadTokens) * 100 / float64(a.inputTokens) + } + if a.requestCount > 0 { + cacheHitReqRate = float64(a.cacheHitRequests) * 100 / float64(a.requestCount) + } list = append(list, map[string]any{ - "model_name": model, - "request_count": a.requestCount, - "success_count": a.successCount, - "success_rate": math.Round(successRate*100) / 100, - "grade": gradeSuccessRate(successRate), - "input_tokens": a.inputTokens, - "output_tokens": a.outputTokens, - "total_tokens": a.inputTokens + a.outputTokens, - "total_cost": a.totalCost, - "avg_latency_ms": math.Round(avgLatency*100) / 100, - "avg_first_token_ms": math.Round(avgTTFT*100) / 100, - "tps": math.Round(tps*100) / 100, + "model_name": model, + "request_count": a.requestCount, + "success_count": a.successCount, + "success_rate": math.Round(successRate*100) / 100, + "grade": gradeSuccessRate(successRate), + "input_tokens": a.inputTokens, + "output_tokens": a.outputTokens, + "total_tokens": a.inputTokens + a.outputTokens, + "total_cost": a.totalCost, + "avg_latency_ms": math.Round(avgLatency*100) / 100, + "avg_first_token_ms": math.Round(avgTTFT*100) / 100, + "tps": math.Round(tps*100) / 100, + "cache_creation_tokens": a.cacheCreationTokens, + "cache_read_tokens": a.cacheReadTokens, + "cache_hit_rate": math.Round(cacheHitRate*100) / 100, + "cache_hit_request_count": a.cacheHitRequests, + "cache_hit_request_rate": math.Round(cacheHitReqRate*100) / 100, }) } // 默认按模型名称升序排列，便于运营按名称定位具体模型；用户可在前端表格点任意列切换排序。 diff --git a/internal/logic/monitor/api_metrics_test.go b/internal/logic/monitor/api_metrics_test.go index e84079f..590bc42 100644 --- a/internal/logic/monitor/api_metrics_test.go +++ b/internal/logic/monitor/api_metrics_test.go @@ -36,9 +36,13 @@ func assertPerfField(t *testing.T, list []map[string]any, model, field string, w } func TestBuildModelPerformanceList_HistoryOnly(t *testing.T) { + // input_tokens 为「含缓存总输入」统一口径（base + cache_read + cache_creation）： + // gpt-4o 行 1000 base + 800 read = 1800；claude 行 100 base + 200 creation + 700 read = 1000 rows := []modelPerfRow{ - {ModelName: "gpt-4o", RequestCount: 100, SuccessCount: 90, InputTokens: 1000, OutputTokens: 2000, TotalCost: 1.5, SumLatencyMs: 50000, SumFirstTokenMs: 8000}, - {ModelName: "claude-3-5-sonnet", RequestCount: 50, SuccessCount: 1, InputTokens: 100, OutputTokens: 50, TotalCost: 0.1, SumLatencyMs: 30000, SumFirstTokenMs: 0}, + {ModelName: "gpt-4o", RequestCount: 100, SuccessCount: 90, InputTokens: 1800, OutputTokens: 2000, TotalCost: 1.5, SumLatencyMs: 50000, SumFirstTokenMs: 8000, + CacheReadTokens: 800, CacheHitRequestCount: 80}, + {ModelName: "claude-3-5-sonnet", RequestCount: 50, SuccessCount: 1, InputTokens: 1000, OutputTokens: 50, TotalCost: 0.1, SumLatencyMs: 30000, SumFirstTokenMs: 0, + CacheCreationTokens: 200, CacheReadTokens: 700, CacheHitRequestCount: 40}, } list := buildModelPerformanceList(rows, nil) @@ -51,23 +55,37 @@ func TestBuildModelPerformanceList_HistoryOnly(t *testing.T) { assertPerfField(t, list, "gpt-4o", "avg_latency_ms", 500.0, 0) assertPerfField(t, list, "gpt-4o", "avg_first_token_ms", 80.0, 0) // 8000/100 assertPerfField(t, list, "gpt-4o", "tps", 40.0, 0) // 2000 / (50000/1000) - assertPerfField(t, list, "gpt-4o", "total_tokens", 0, 3000) + assertPerfField(t, list, "gpt-4o", "total_tokens", 0, 3800) // 1800+2000 assertPerfField(t, list, "gpt-4o", "total_cost", 1.5, 0) + // 缓存：无写入，命中率 = 800/1800 = 44.44%；请求级 = 80/100 = 80% + assertPerfField(t, list, "gpt-4o", "cache_read_tokens", 0, 800) + assertPerfField(t, list, "gpt-4o", "cache_creation_tokens", 0, 0) + assertPerfField(t, list, "gpt-4o", "cache_hit_rate", 44.44, 0) + assertPerfField(t, list, "gpt-4o", "cache_hit_request_count", 0, 80) + assertPerfField(t, list, "gpt-4o", "cache_hit_request_rate", 80.0, 0) + // claude 成功率 2%（1/50）→ critical assertPerfField(t, list, "claude-3-5-sonnet", "success_rate", 2.0, 0) assertGrade(t, list, "claude-3-5-sonnet", "critical") assertPerfField(t, list, "claude-3-5-sonnet", "avg_latency_ms", 600.0, 0) assertPerfField(t, list, "claude-3-5-sonnet", "tps", 1.67, 0) // 50/(30000/1000)=1.6666 + + // 缓存：命中率 = 700/1000 = 70%；请求级 = 40/50 = 80% + assertPerfField(t, list, "claude-3-5-sonnet", "cache_hit_rate", 70.0, 0) + assertPerfField(t, list, "claude-3-5-sonnet", "cache_hit_request_rate", 80.0, 0) } func TestBuildModelPerformanceList_MergeToday(t *testing.T) { + // input_tokens 为「含缓存总输入」统一口径： + // 历史行 1000 base + 800 read + 100 creation = 1900；当日 Tin = 50 base + 180 read + 20 creation = 250 rows := []modelPerfRow{ - {ModelName: "gpt-4o", RequestCount: 100, SuccessCount: 90, InputTokens: 1000, OutputTokens: 2000, TotalCost: 1.5, SumLatencyMs: 50000, SumFirstTokenMs: 8000}, + {ModelName: "gpt-4o", RequestCount: 100, SuccessCount: 90, InputTokens: 1900, OutputTokens: 2000, TotalCost: 1.5, SumLatencyMs: 50000, SumFirstTokenMs: 8000, + CacheCreationTokens: 100, CacheReadTokens: 800, CacheHitRequestCount: 70}, } today := map[string]common.ModelPerfCounter{ - "gpt-4o": {Req: 10, Ok: 9, Lat: 6000, Ttft: 500, TtftN: 10, Tin: 50, Tout: 150, CostMicro: 500000}, // 500000 micro = 0.5 USD - "new-model-day": {Req: 1, Ok: 0, Lat: 8000, Ttft: 0, TtftN: 0, Tin: 100, Tout: 0, CostMicro: 12345}, + "gpt-4o": {Req: 10, Ok: 9, Lat: 6000, Ttft: 500, TtftN: 10, Tin: 250, Tout: 150, CostMicro: 500000, CacheCreation: 20, CacheRead: 180, CacheHitReq: 8}, // 500000 micro = 0.5 USD + "new-model-day": {Req: 1, Ok: 0, Lat: 8000, Ttft: 0, TtftN: 0, Tin: 160, Tout: 0, CostMicro: 12345, CacheRead: 60, CacheHitReq: 1}, // 100 base + 60 read } list := buildModelPerformanceList(rows, today) @@ -76,9 +94,17 @@ func TestBuildModelPerformanceList_MergeToday(t *testing.T) { assertPerfField(t, list, "gpt-4o", "success_count", 0, 99) assertPerfField(t, list, "gpt-4o", "success_rate", 90.0, 0) assertPerfField(t, list, "gpt-4o", "avg_latency_ms", 509.09, 0) // 56000/110 = 509.0909 - assertPerfField(t, list, "gpt-4o", "total_tokens", 0, 3200) // (1000+50)+(2000+150) + assertPerfField(t, list, "gpt-4o", "total_tokens", 0, 4300) // (1900+250)+(2000+150) assertPerfField(t, list, "gpt-4o", "total_cost", 2.0, 0) + // 缓存合并：read 800+180=980、creation 100+20=120、命中请求 70+8=78； + // 命中率 = 980/(1900+250) = 980/2150 = 45.58%；请求级 = 78/110 = 70.91% + assertPerfField(t, list, "gpt-4o", "cache_read_tokens", 0, 980) + assertPerfField(t, list, "gpt-4o", "cache_creation_tokens", 0, 120) + assertPerfField(t, list, "gpt-4o", "cache_hit_rate", 45.58, 0) + assertPerfField(t, list, "gpt-4o", "cache_hit_request_count", 0, 78) + assertPerfField(t, list, "gpt-4o", "cache_hit_request_rate", 70.91, 0) + // 仅当日模型 assertPerfField(t, list, "new-model-day", "request_count", 0, 1) assertPerfField(t, list, "new-model-day", "total_cost", 0.012345, 0) @@ -86,10 +112,83 @@ func TestBuildModelPerformanceList_MergeToday(t *testing.T) { assertPerfField(t, list, "new-model-day", "avg_latency_ms", 8000.0, 0) assertPerfField(t, list, "new-model-day", "tps", 0.0, 0) // 无输出 token + // 仅当日缓存：命中率 = 60/160 = 37.5%；请求级 = 1/1 = 100% + assertPerfField(t, list, "new-model-day", "cache_hit_rate", 37.5, 0) + assertPerfField(t, list, "new-model-day", "cache_hit_request_rate", 100.0, 0) + // 按模型名称升序：gpt-4o（'g'）在 new-model-day（'n'）前 _ = assertOrder(t, list, "gpt-4o", "new-model-day") } +// 缓存边界：仅写入未命中（creation>0、read=0）不算命中请求；无缓存活动时分母为 0 不 panic、比率为 0。 +func TestBuildModelPerformanceList_CacheEdgeCases(t *testing.T) { + rows := []modelPerfRow{ + // 首次写入缓存，尚未命中：500 base + 500 creation（含缓存总输入 1000）/ read 0 / 命中请求 0 + {ModelName: "write-only", RequestCount: 10, SuccessCount: 10, InputTokens: 1000, OutputTokens: 100, + CacheCreationTokens: 500}, + // 完全无缓存活动（embedding 类模型） + {ModelName: "text-embedding-3", RequestCount: 20, SuccessCount: 20, InputTokens: 4000, OutputTokens: 0}, + } + list := buildModelPerformanceList(rows, nil) + + assertPerfField(t, list, "write-only", "cache_creation_tokens", 0, 500) + assertPerfField(t, list, "write-only", "cache_read_tokens", 0, 0) + assertPerfField(t, list, "write-only", "cache_hit_rate", 0.0, 0) // 0/1000 + assertPerfField(t, list, "write-only", "cache_hit_request_count", 0, 0) // 仅写入不计命中 + assertPerfField(t, list, "write-only", "cache_hit_request_rate", 0.0, 0) // 0/10 + + assertPerfField(t, list, "text-embedding-3", "cache_hit_rate", 0.0, 0) + assertPerfField(t, list, "text-embedding-3", "cache_hit_request_rate", 0.0, 0) +} + +// 当天明细聚合行 → 热桶同构计数：字段映射、cost 转 micro（math.Round）、条件计数透传，并与合并层贯通。 +func TestRowsToTodayCounters(t *testing.T) { + rows := []modelPerfTodayRow{ + {ModelName: "gpt-4o", Req: 30, Ok: 28, Lat: 15000, Ttft: 4200, TtftN: 25, Tin: 3000, Tout: 6000, + TotalCost: 0.123456, CacheCreation: 100, CacheRead: 900, Chit: 20}, + {ModelName: "tiny", Req: 2, Ok: 2, TotalCost: 0.0000004}, + } + m := rowsToTodayCounters(rows) + c := m["gpt-4o"] + if c.Req != 30 || c.Ok != 28 || c.Lat != 15000 || c.Ttft != 4200 || c.TtftN != 25 || + c.Tin != 3000 || c.Tout != 6000 || c.CostMicro != 123456 || + c.CacheCreation != 100 || c.CacheRead != 900 || c.CacheHitReq != 20 { + t.Errorf("unexpected counter: %+v", c) + } + // 极小金额四舍五入到 0 micro（对齐 common.microFromUSD 的 math.Round 语义） + if m["tiny"].CostMicro != 0 { + t.Errorf("CostMicro = %d, want 0", m["tiny"].CostMicro) + } + // 与合并层贯通：明细行经转换后与热桶路径产出等价（USD 还原一致） + list := buildModelPerformanceList(nil, m) + assertPerfField(t, list, "gpt-4o", "request_count", 0, 30) + assertPerfField(t, list, "gpt-4o", "total_cost", 0.123456, 0) + assertPerfField(t, list, "gpt-4o", "cache_hit_request_count", 0, 20) +} + +func TestRowsToTodayCounters_Empty(t *testing.T) { + if m := rowsToTodayCounters(nil); len(m) != 0 { + t.Errorf("expected empty map, got %v", m) + } +} + +// 热桶路径的模型过滤：命中只留一键；未命中返回空；空模型名原样返回。 +func TestFilterTodayCounts(t *testing.T) { + tc := map[string]common.ModelPerfCounter{ + "gpt-4o": {Req: 3}, + "claude-3-5-sonnet": {Req: 5}, + } + if got := filterTodayCounts(tc, ""); len(got) != 2 { + t.Errorf("empty model should keep all, got %v", got) + } + if got := filterTodayCounts(tc, "gpt-4o"); len(got) != 1 || got["gpt-4o"].Req != 3 { + t.Errorf("unexpected filtered map: %v", got) + } + if got := filterTodayCounts(tc, "no-such-model"); len(got) != 0 { + t.Errorf("miss should be empty, got %v", got) + } +} + func TestBuildModelPerformanceList_FilterUnknownAndEmpty(t *testing.T) { rows := []modelPerfRow{ {ModelName: "unknown", RequestCount: 999, SuccessCount: 0}, diff --git a/internal/logic/monitor/monitor.go b/internal/logic/monitor/monitor.go index 788faf9..a2bf509 100644 --- a/internal/logic/monitor/monitor.go +++ b/internal/logic/monitor/monitor.go @@ -71,7 +71,8 @@ func (s *sMonitor) ModelPerformance(ctx context.Context, req *v1.MonitorModelPer return nil, err } startDate, endDate := normalizeMonitorDateRange(req.StartDate, req.EndDate) - data, err := GetModelPerformance(ctx, startDate, endDate) + // 渠道/模型为可选过滤（channel_id=0、model_name 空 = 不过滤），透传给聚合层拼 SQL 条件 + data, err := GetModelPerformance(ctx, startDate, endDate, req.ChannelId, req.ModelName) if err != nil { return nil, err } diff --git a/internal/logic/relay/provider.go b/internal/logic/relay/provider.go index 180495e..f9fc2c7 100644 --- a/internal/logic/relay/provider.go +++ b/internal/logic/relay/provider.go @@ -343,9 +343,22 @@ func (p *DataProviderImpl) RecordUsage(ctx context.Context, record *common.Usage InputTokens: record.PromptTokens, OutputTokens: record.CompletionTokens, CostUSD: record.TotalCost, + // 缓存 token 透传给热桶（命中请求判定由热桶模块从 read>0 推导） + CacheCreationTokens: record.CacheCreationTokens, + CacheReadTokens: record.CacheReadTokens, }) } +// sanitizeUTF8 清洗字符串字段：将非法 UTF-8 字节替换为 U+FFFD 替换符。 +// 上游错误文本/请求头/回显模型名等可能夹带 0xa0 等单字节脏数据，PostgreSQL 在协议层 +// 会以 invalid byte sequence 拒绝整批插入，必须在落库前清洗保证字符串合法。 +func sanitizeUTF8(s string) string { + if utf8.ValidString(s) { + return s + } + return strings.ToValidUTF8(s, "�") +} + // buildUsageLogDO 将 UsageRecord 转换为 DO 对象 func buildUsageLogDO(record *common.UsageRecord) do.BilUsageLogs { requestType := record.RequestType @@ -363,23 +376,23 @@ func buildUsageLogDO(record *common.UsageRecord) do.BilUsageLogs { ApiKeyId: record.ApiKeyID, ProjectId: record.ProjectID, ChannelId: record.ChannelID, - ModelName: record.ModelName, - RequestId: record.RequestID, + ModelName: sanitizeUTF8(record.ModelName), + RequestId: sanitizeUTF8(record.RequestID), RelayMode: record.RelayMode, InputTokens: record.PromptTokens, OutputTokens: record.CompletionTokens, TotalCost: record.TotalCost, LatencyMs: int(record.LatencyMs), - Status: record.Status, - ErrorMessage: record.ErrorMessage, - ClientIp: record.ClientIP, + Status: sanitizeUTF8(record.Status), + ErrorMessage: sanitizeUTF8(record.ErrorMessage), + ClientIp: sanitizeUTF8(record.ClientIP), InputCost: record.InputCost, OutputCost: record.OutputCost, CacheCreationCost: record.CacheCreationCost, CacheReadCost: record.CacheReadCost, ActualCost: record.ActualCost, - Currency: record.Currency, + Currency: sanitizeUTF8(record.Currency), PreDeductAmount: record.PreDeductAmount, RefundAmount: record.RefundAmount, SupplementAmount: record.SupplementAmount, @@ -392,32 +405,32 @@ func buildUsageLogDO(record *common.UsageRecord) do.BilUsageLogs { ImageOutputTokens: record.ImageOutputTokens, ReasoningTokens: record.ReasoningTokens, - RequestedModel: record.RequestedModel, - UpstreamModel: record.UpstreamModel, + RequestedModel: sanitizeUTF8(record.RequestedModel), + UpstreamModel: sanitizeUTF8(record.UpstreamModel), RequestType: requestType, - UserAgent: record.UserAgent, + UserAgent: sanitizeUTF8(record.UserAgent), FirstTokenMs: record.FirstTokenMs, - ServiceTier: record.ServiceTier, - ReasoningEffort: record.ReasoningEffort, - InboundEndpoint: record.InboundEndpoint, + ServiceTier: sanitizeUTF8(record.ServiceTier), + ReasoningEffort: sanitizeUTF8(record.ReasoningEffort), + InboundEndpoint: sanitizeUTF8(record.InboundEndpoint), - ChannelName: record.ChannelName, + ChannelName: sanitizeUTF8(record.ChannelName), ChannelType: record.ChannelType, - BillingMode: record.BillingMode, - BillingSource: record.BillingSource, + BillingMode: sanitizeUTF8(record.BillingMode), + BillingSource: sanitizeUTF8(record.BillingSource), RateMultiplier: record.RateMultiplier, RetryIndex: record.RetryIndex, - StreamEndReason: record.StreamEndReason, + StreamEndReason: sanitizeUTF8(record.StreamEndReason), ImageCount: record.ImageCount, - ImageSize: record.ImageSize, + ImageSize: sanitizeUTF8(record.ImageSize), - BillingSnapshot: jsonNullIfEmpty(record.BillingSnapshot), - BillingSummary: record.BillingSummary, - TaskId: record.TaskID, + BillingSnapshot: jsonNullIfEmpty(sanitizeUTF8(record.BillingSnapshot)), + BillingSummary: sanitizeUTF8(record.BillingSummary), + TaskId: sanitizeUTF8(record.TaskID), } } @@ -627,6 +640,9 @@ func (p *DataProviderImpl) UpdateTaskAudit(ctx context.Context, record *common.A } func truncateBody(s string, maxLen int) string { + // 请求/响应体可能携带无效 UTF-8 字节（捕获侧按字节切割、上游/客户端原始输出）， + // 落库前统一清洗为合法 UTF-8，避免 INSERT 被 UTF8 编码的 PostgreSQL 拒绝、整条记录丢失。 + s = strings.ToValidUTF8(s, string(utf8.RuneError)) // maxLen <= 0 表示不截断（配置了独立审计库时完整记录） if maxLen <= 0 || len(s) <= maxLen { return s diff --git a/internal/logic/relay/provider_test.go b/internal/logic/relay/provider_test.go index 45c3f77..db2a274 100644 --- a/internal/logic/relay/provider_test.go +++ b/internal/logic/relay/provider_test.go @@ -2,6 +2,7 @@ package relay import ( "fmt" + "reflect" "strings" "testing" "time" @@ -224,3 +225,58 @@ func TestBuildUsageLogDO_FieldMapping(t *testing.T) { t.Errorf("non-empty BillingSnapshot should pass through, got %v", got) } } + +func TestBuildUsageLogDO_SanitizesInvalidUTF8(t *testing.T) { + // 构造带原始 0xa0 字节（非法 UTF-8，PG 协议层会整体拒绝插入）的 UsageRecord， + // 验证 buildUsageLogDO 已对所有字符串字段清洗为合法 UTF-8。 + bad := "pre\xa0post" + rec := &common.UsageRecord{ + TenantID: 7, + ModelName: bad, + RequestID: bad, + Status: bad, + ErrorMessage: bad, + ClientIP: bad, + Currency: bad, + RequestedModel: bad, + UpstreamModel: bad, + UserAgent: bad, + ServiceTier: bad, + ReasoningEffort: bad, + InboundEndpoint: bad, + ChannelName: bad, + BillingMode: bad, + BillingSource: bad, + StreamEndReason: bad, + ImageSize: bad, + BillingSnapshot: `{"err":"` + bad + `"}`, + BillingSummary: bad, + TaskID: bad, + } + + d := buildUsageLogDO(rec) + + // 逐一校验所有字符串字段 + v := reflect.ValueOf(d) + for i := 0; i < v.NumField(); i++ { + field := v.Field(i) + if field.Kind() != reflect.Interface || field.IsNil() { + continue + } + s, ok := field.Interface().(string) + if !ok { + continue + } + if !utf8.ValidString(s) { + t.Errorf("field %s still contains invalid UTF-8: %q", v.Type().Field(i).Name, s) + } + if strings.Contains(s, "\xa0") { + t.Errorf("field %s still contains raw 0xa0 byte: %q", v.Type().Field(i).Name, s) + } + } + + // 校验清洗结果：0xa0 被替换为 U+FFFD + if got, _ := d.ErrorMessage.(string); got != "pre�post" { + t.Errorf("ErrorMessage = %q, want %q", got, "pre�post") + } +} diff --git a/internal/logic/task/usage_aggregation.go b/internal/logic/task/usage_aggregation.go index 8c4e108..b792951 100644 --- a/internal/logic/task/usage_aggregation.go +++ b/internal/logic/task/usage_aggregation.go @@ -13,7 +13,9 @@ const aggregateUsageSQL = ` INSERT INTO bil_usage_daily (stat_date, tenant_id, project_id, model_name, channel_id, status, request_count, input_tokens, output_tokens, total_cost, account_cost, - sum_latency_ms, sum_first_token_ms, updated_at) + sum_latency_ms, sum_first_token_ms, + cache_creation_tokens, cache_read_tokens, cache_hit_request_count, + updated_at) SELECT DATE(created_at) AS stat_date, tenant_id AS tenant_id, @@ -28,6 +30,9 @@ SELECT COALESCE(SUM(account_cost), 0) AS account_cost, COALESCE(SUM(latency_ms), 0) AS sum_latency_ms, COALESCE(SUM(first_token_ms), 0) AS sum_first_token_ms, + COALESCE(SUM(cache_creation_tokens), 0) AS cache_creation_tokens, + COALESCE(SUM(cache_read_tokens), 0) AS cache_read_tokens, + COALESCE(SUM(CASE WHEN cache_read_tokens > 0 THEN 1 ELSE 0 END), 0) AS cache_hit_request_count, now() FROM bil_usage_logs WHERE created_at >= $1 AND created_at < $2 @@ -39,14 +44,17 @@ GROUP BY COALESCE(channel_id, 0), COALESCE(NULLIF(status, ''), 'unknown') ON CONFLICT (stat_date, tenant_id, project_id, model_name, channel_id, status) DO UPDATE SET - request_count = EXCLUDED.request_count, - input_tokens = EXCLUDED.input_tokens, - output_tokens = EXCLUDED.output_tokens, - total_cost = EXCLUDED.total_cost, - account_cost = EXCLUDED.account_cost, - sum_latency_ms = EXCLUDED.sum_latency_ms, + request_count = EXCLUDED.request_count, + input_tokens = EXCLUDED.input_tokens, + output_tokens = EXCLUDED.output_tokens, + total_cost = EXCLUDED.total_cost, + account_cost = EXCLUDED.account_cost, + sum_latency_ms = EXCLUDED.sum_latency_ms, sum_first_token_ms = EXCLUDED.sum_first_token_ms, - updated_at = now() + cache_creation_tokens = EXCLUDED.cache_creation_tokens, + cache_read_tokens = EXCLUDED.cache_read_tokens, + cache_hit_request_count = EXCLUDED.cache_hit_request_count, + updated_at = now() ` // AggregateUsageRange 将 bil_usage_logs 在 [startDate, endDate)（半开区间）的用量 diff --git a/internal/logic/tenant/personal_dashboard.go b/internal/logic/tenant/personal_dashboard.go index 026bab8..dc94faf 100644 --- a/internal/logic/tenant/personal_dashboard.go +++ b/internal/logic/tenant/personal_dashboard.go @@ -150,7 +150,8 @@ func (s *sTenant) PersonalDashboard(ctx context.Context, req *v1.PersonalDashboa CacheReadTokens: lat.CacheReadTkns, TotalInputTokens: lat.TotalInputTkns, } - totalForRatio := float64(lat.TotalInputTkns + lat.CacheCreationTkns + lat.CacheReadTkns) + // input_tokens 已统一为「含缓存总输入」口径（Claude 渠道入库时补加缓存），分母直接取总和 + totalForRatio := float64(lat.TotalInputTkns) if totalForRatio > 0 { cache.HitRatio = math.Round(float64(lat.CacheReadTkns)/totalForRatio*10000) / 10000 } diff --git a/internal/middleware/request_id.go b/internal/middleware/request_id.go index c3741fd..05c6add 100644 --- a/internal/middleware/request_id.go +++ b/internal/middleware/request_id.go @@ -26,3 +26,13 @@ func RequestId(r *ghttp.Request) { r.Middleware.Next() } + +// serviceName 随响应头 X-Service-Name 返回的稳定服务标识。 +const serviceName = "team-api" + +// ServiceName 在每个响应上写入服务标识头 X-Service-Name， +// 与 RequestId 同属全局注册的响应头中间件。 +func ServiceName(r *ghttp.Request) { + r.Response.Header().Set("X-Service-Name", serviceName) + r.Middleware.Next() +} diff --git a/internal/middleware/request_id_test.go b/internal/middleware/request_id_test.go new file mode 100644 index 0000000..e991518 --- /dev/null +++ b/internal/middleware/request_id_test.go @@ -0,0 +1,58 @@ +package middleware + +import ( + "net/http" + "testing" + + "github.com/gogf/gf/v2/frame/g" + "github.com/gogf/gf/v2/net/ghttp" + "github.com/gogf/gf/v2/util/guid" + + "github.com/qianfree/team-api/internal/testutil" +) + +// TestGlobalResponseHeaders 验证两个全局响应头中间件： +// RequestId 生成/透传 X-Request-Id，ServiceName 在所有响应上写入 X-Service-Name。 +func TestGlobalResponseHeaders(t *testing.T) { + server := g.Server(guid.S()) + server.Use(RequestId) + server.Use(ServiceName) + server.BindHandler("/ping", func(r *ghttp.Request) { + r.Response.WriteHeader(http.StatusNoContent) + }) + + baseURL := testutil.StartGFServer(t, server) + + t.Run("服务端生成请求 ID 并写入服务标识头", func(t *testing.T) { + resp, err := http.Get(baseURL + "/ping") + if err != nil { + t.Fatalf("GET /ping: %v", err) + } + defer resp.Body.Close() + + if got := resp.Header.Get("X-Service-Name"); got != serviceName { + t.Errorf("X-Service-Name = %q, want %q", got, serviceName) + } + if got := resp.Header.Get("X-Request-Id"); !clientRequestIdRegexp.MatchString(got) { + t.Errorf("X-Request-Id = %q, want 服务端生成的合法 ID", got) + } + }) + + t.Run("合法的客户端请求 ID 被透传", func(t *testing.T) { + const clientID = "client-req-123" + req, err := http.NewRequest(http.MethodGet, baseURL+"/ping", nil) + if err != nil { + t.Fatalf("构造请求失败: %v", err) + } + req.Header.Set("X-Request-Id", clientID) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("GET /ping: %v", err) + } + defer resp.Body.Close() + + if got := resp.Header.Get("X-Request-Id"); got != clientID { + t.Errorf("X-Request-Id = %q, want %q", got, clientID) + } + }) +} diff --git a/internal/model/do/bil_usage_daily.go b/internal/model/do/bil_usage_daily.go index 4a5efc8..09720bd 100644 --- a/internal/model/do/bil_usage_daily.go +++ b/internal/model/do/bil_usage_daily.go @@ -11,21 +11,24 @@ import ( // BilUsageDaily is the golang structure of table bil_usage_daily for DAO operations like Where/Data. type BilUsageDaily struct { - g.Meta `orm:"table:bil_usage_daily, do:true"` - Id any // - StatDate *gtime.Time // 统计日期（按 created_at 的日期分桶） - TenantId any // 租户ID - ProjectId any // 项目ID（0=无项目/个人Key） - ModelName any // 模型名称 - ChannelId any // 渠道ID（0=无渠道） - Status any // 请求状态：success/error/timeout/cancelled - RequestCount any // 请求数（COUNT(*)） - InputTokens any // 输入Token合计 - OutputTokens any // 输出Token合计 - TotalCost any // 客户侧成本合计（USD，源自 bil_usage_logs.total_cost） - AccountCost any // 上游账户成本合计（USD，源自 bil_usage_logs.account_cost，用于利润分析） - SumLatencyMs any // 总延迟合计（ms，源自 bil_usage_logs.latency_ms；视图按 SUM/COUNT 求均值） - SumFirstTokenMs any // 首Token延迟合计（ms，源自 bil_usage_logs.first_token_ms；视图按 SUM/COUNT 求均值） - CreatedAt *gtime.Time // - UpdatedAt *gtime.Time // + g.Meta `orm:"table:bil_usage_daily, do:true"` + Id any // + StatDate *gtime.Time // 统计日期（按 created_at 的日期分桶） + TenantId any // 租户ID + ProjectId any // 项目ID（0=无项目/个人Key） + ModelName any // 模型名称 + ChannelId any // 渠道ID（0=无渠道） + Status any // 请求状态：success/error/timeout/cancelled + RequestCount any // 请求数（COUNT(*)） + InputTokens any // 输入Token合计 + OutputTokens any // 输出Token合计 + TotalCost any // 客户侧成本合计（USD，源自 bil_usage_logs.total_cost） + AccountCost any // 上游账户成本合计（USD，源自 bil_usage_logs.account_cost，用于利润分析） + SumLatencyMs any // 总延迟合计（ms，源自 bil_usage_logs.latency_ms；视图按 SUM/COUNT 求均值） + SumFirstTokenMs any // 首Token延迟合计（ms，源自 bil_usage_logs.first_token_ms；视图按 SUM/COUNT 求均值） + CreatedAt *gtime.Time // + UpdatedAt *gtime.Time // + CacheCreationTokens any // 缓存创建 token 数（Claude cache_creation，SUM 聚合） + CacheReadTokens any // 缓存命中读取 token 数（Claude cache_read / OpenAI cached_tokens，SUM 聚合） + CacheHitRequestCount any // 命中缓存的请求数（明细 cache_read_tokens>0 计 1，含失败状态行） } diff --git a/internal/model/do/mdl_tenant_models.go b/internal/model/do/mdl_tenant_models.go index c849def..e510c84 100644 --- a/internal/model/do/mdl_tenant_models.go +++ b/internal/model/do/mdl_tenant_models.go @@ -19,7 +19,7 @@ type MdlTenantModels struct { Enabled any // 是否启用（禁用后该租户无法调用此模型） CustomInputPrice *decimal.Decimal // 租户自定义输入价格（NULL 表示使用默认定价） CustomOutputPrice *decimal.Decimal // 租户自定义输出价格（NULL 表示使用默认定价） - Multiplier any // 租户价格倍率（VIP 折扣等，最终价格 = 基础价格 × 模型倍率 × 租户倍率） + Multiplier any // 租户价格倍率（VIP 折扣）。作为 tenant_multiplier 参与最终价格计算：最终价格 = 基础价格 × 租户倍率。倍率来源优先级：discount_ratio > multiplier > 租户等级 price_multiplier CreatedAt *gtime.Time // 创建时间 UpdatedAt *gtime.Time // 更新时间 BillingMode any // 覆盖模型计费方式（NULL表示跟随模型默认） diff --git a/internal/model/entity/bil_usage_daily.go b/internal/model/entity/bil_usage_daily.go index 96a3937..77d3330 100644 --- a/internal/model/entity/bil_usage_daily.go +++ b/internal/model/entity/bil_usage_daily.go @@ -11,20 +11,23 @@ import ( // BilUsageDaily is the golang structure for table bil_usage_daily. type BilUsageDaily struct { - Id int64 `json:"id" orm:"id" description:""` // - StatDate *gtime.Time `json:"stat_date" orm:"stat_date" description:"统计日期（按 created_at 的日期分桶）"` // 统计日期（按 created_at 的日期分桶） - TenantId int64 `json:"tenant_id" orm:"tenant_id" description:"租户ID"` // 租户ID - ProjectId int64 `json:"project_id" orm:"project_id" description:"项目ID（0=无项目/个人Key）"` // 项目ID（0=无项目/个人Key） - ModelName string `json:"model_name" orm:"model_name" description:"模型名称"` // 模型名称 - ChannelId int64 `json:"channel_id" orm:"channel_id" description:"渠道ID（0=无渠道）"` // 渠道ID（0=无渠道） - Status string `json:"status" orm:"status" description:"请求状态：success/error/timeout/cancelled"` // 请求状态：success/error/timeout/cancelled - RequestCount int `json:"request_count" orm:"request_count" description:"请求数（COUNT(*)）"` // 请求数（COUNT(*)） - InputTokens int64 `json:"input_tokens" orm:"input_tokens" description:"输入Token合计"` // 输入Token合计 - OutputTokens int64 `json:"output_tokens" orm:"output_tokens" description:"输出Token合计"` // 输出Token合计 - TotalCost decimal.Decimal `json:"total_cost" orm:"total_cost" description:"客户侧成本合计（USD，源自 bil_usage_logs.total_cost）"` // 客户侧成本合计（USD，源自 bil_usage_logs.total_cost） - AccountCost decimal.Decimal `json:"account_cost" orm:"account_cost" description:"上游账户成本合计（USD，源自 bil_usage_logs.account_cost，用于利润分析）"` // 上游账户成本合计（USD，源自 bil_usage_logs.account_cost，用于利润分析） - SumLatencyMs int64 `json:"sum_latency_ms" orm:"sum_latency_ms" description:"总延迟合计（ms，源自 bil_usage_logs.latency_ms；视图按 SUM/COUNT 求均值）"` // 总延迟合计（ms，源自 bil_usage_logs.latency_ms；视图按 SUM/COUNT 求均值） - SumFirstTokenMs int64 `json:"sum_first_token_ms" orm:"sum_first_token_ms" description:"首Token延迟合计（ms，源自 bil_usage_logs.first_token_ms；视图按 SUM/COUNT 求均值）"` // 首Token延迟合计（ms，源自 bil_usage_logs.first_token_ms；视图按 SUM/COUNT 求均值） - CreatedAt *gtime.Time `json:"created_at" orm:"created_at" description:""` // - UpdatedAt *gtime.Time `json:"updated_at" orm:"updated_at" description:""` // + Id int64 `json:"id" orm:"id" description:""` // + StatDate *gtime.Time `json:"stat_date" orm:"stat_date" description:"统计日期（按 created_at 的日期分桶）"` // 统计日期（按 created_at 的日期分桶） + TenantId int64 `json:"tenant_id" orm:"tenant_id" description:"租户ID"` // 租户ID + ProjectId int64 `json:"project_id" orm:"project_id" description:"项目ID（0=无项目/个人Key）"` // 项目ID（0=无项目/个人Key） + ModelName string `json:"model_name" orm:"model_name" description:"模型名称"` // 模型名称 + ChannelId int64 `json:"channel_id" orm:"channel_id" description:"渠道ID（0=无渠道）"` // 渠道ID（0=无渠道） + Status string `json:"status" orm:"status" description:"请求状态：success/error/timeout/cancelled"` // 请求状态：success/error/timeout/cancelled + RequestCount int `json:"request_count" orm:"request_count" description:"请求数（COUNT(*)）"` // 请求数（COUNT(*)） + InputTokens int64 `json:"input_tokens" orm:"input_tokens" description:"输入Token合计"` // 输入Token合计 + OutputTokens int64 `json:"output_tokens" orm:"output_tokens" description:"输出Token合计"` // 输出Token合计 + TotalCost decimal.Decimal `json:"total_cost" orm:"total_cost" description:"客户侧成本合计（USD，源自 bil_usage_logs.total_cost）"` // 客户侧成本合计（USD，源自 bil_usage_logs.total_cost） + AccountCost decimal.Decimal `json:"account_cost" orm:"account_cost" description:"上游账户成本合计（USD，源自 bil_usage_logs.account_cost，用于利润分析）"` // 上游账户成本合计（USD，源自 bil_usage_logs.account_cost，用于利润分析） + SumLatencyMs int64 `json:"sum_latency_ms" orm:"sum_latency_ms" description:"总延迟合计（ms，源自 bil_usage_logs.latency_ms；视图按 SUM/COUNT 求均值）"` // 总延迟合计（ms，源自 bil_usage_logs.latency_ms；视图按 SUM/COUNT 求均值） + SumFirstTokenMs int64 `json:"sum_first_token_ms" orm:"sum_first_token_ms" description:"首Token延迟合计（ms，源自 bil_usage_logs.first_token_ms；视图按 SUM/COUNT 求均值）"` // 首Token延迟合计（ms，源自 bil_usage_logs.first_token_ms；视图按 SUM/COUNT 求均值） + CreatedAt *gtime.Time `json:"created_at" orm:"created_at" description:""` // + UpdatedAt *gtime.Time `json:"updated_at" orm:"updated_at" description:""` // + CacheCreationTokens int64 `json:"cache_creation_tokens" orm:"cache_creation_tokens" description:"缓存创建 token 数（Claude cache_creation，SUM 聚合）"` // 缓存创建 token 数（Claude cache_creation，SUM 聚合） + CacheReadTokens int64 `json:"cache_read_tokens" orm:"cache_read_tokens" description:"缓存命中读取 token 数（Claude cache_read / OpenAI cached_tokens，SUM 聚合）"` // 缓存命中读取 token 数（Claude cache_read / OpenAI cached_tokens，SUM 聚合） + CacheHitRequestCount int64 `json:"cache_hit_request_count" orm:"cache_hit_request_count" description:"命中缓存的请求数（明细 cache_read_tokens>0 计 1，含失败状态行）"` // 命中缓存的请求数（明细 cache_read_tokens>0 计 1，含失败状态行） } diff --git a/internal/model/entity/mdl_tenant_models.go b/internal/model/entity/mdl_tenant_models.go index ca7b682..4ef62d5 100644 --- a/internal/model/entity/mdl_tenant_models.go +++ b/internal/model/entity/mdl_tenant_models.go @@ -17,7 +17,7 @@ type MdlTenantModels struct { Enabled bool `json:"enabled" orm:"enabled" description:"是否启用（禁用后该租户无法调用此模型）"` // 是否启用（禁用后该租户无法调用此模型） CustomInputPrice *decimal.Decimal `json:"custom_input_price" orm:"custom_input_price" description:"租户自定义输入价格（NULL 表示使用默认定价）"` // 租户自定义输入价格（NULL 表示使用默认定价） CustomOutputPrice *decimal.Decimal `json:"custom_output_price" orm:"custom_output_price" description:"租户自定义输出价格（NULL 表示使用默认定价）"` // 租户自定义输出价格（NULL 表示使用默认定价） - Multiplier decimal.Decimal `json:"multiplier" orm:"multiplier" description:"租户价格倍率（VIP 折扣等，最终价格 = 基础价格 × 模型倍率 × 租户倍率）"` // 租户价格倍率（VIP 折扣等，最终价格 = 基础价格 × 模型倍率 × 租户倍率） + Multiplier decimal.Decimal `json:"multiplier" orm:"multiplier" description:"租户价格倍率（VIP 折扣）。作为 tenant_multiplier 参与最终价格计算：最终价格 = 基础价格 × 租户倍率。倍率来源优先级：discount_ratio > multiplier > 租户等级 price_multiplier"` // 租户价格倍率（VIP 折扣）。作为 tenant_multiplier 参与最终价格计算：最终价格 = 基础价格 × 租户倍率。倍率来源优先级：discount_ratio > multiplier > 租户等级 price_multiplier CreatedAt *gtime.Time `json:"created_at" orm:"created_at" description:"创建时间"` // 创建时间 UpdatedAt *gtime.Time `json:"updated_at" orm:"updated_at" description:"更新时间"` // 更新时间 BillingMode string `json:"billing_mode" orm:"billing_mode" description:"覆盖模型计费方式（NULL表示跟随模型默认）"` // 覆盖模型计费方式（NULL表示跟随模型默认） diff --git a/internal/plugin/interfaces.go b/internal/plugin/interfaces.go index ca89dc0..7a58c2e 100644 --- a/internal/plugin/interfaces.go +++ b/internal/plugin/interfaces.go @@ -112,4 +112,11 @@ type TokenUsage struct { PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` TotalTokens int `json:"total_tokens"` + + // 缓存/推理明细（上游未返回时为 0）。PromptTokens 始终为「含缓存总输入」， + // cache 各项是其子集明细；CacheIncludedInPrompt 标记原始口径（false=Claude 排除口径） + CacheReadTokens int `json:"cache_read_tokens,omitempty"` // 缓存命中读取 + CacheCreationTokens int `json:"cache_creation_tokens,omitempty"` // 写入缓存（Claude cache_creation / OpenAI cache_write） + ReasoningTokens int `json:"reasoning_tokens,omitempty"` // 推理 token（CompletionTokens 子集） + CacheIncludedInPrompt bool `json:"-"` // 原始响应的 PromptTokens 是否已含缓存（OpenAI/Gemini=true，Claude=false） } diff --git a/internal/response/response_test.go b/internal/response/response_test.go index 31468de..4c8567c 100644 --- a/internal/response/response_test.go +++ b/internal/response/response_test.go @@ -3,7 +3,6 @@ package response_test import ( "encoding/json" "net/http" - "net/http/httptest" "testing" "github.com/gogf/gf/v2/frame/g" @@ -11,11 +10,11 @@ import ( "github.com/gogf/gf/v2/util/guid" apiresponse "github.com/qianfree/team-api/internal/response" + "github.com/qianfree/team-api/internal/testutil" ) func TestErrorResponseStatusAndCode(t *testing.T) { s := g.Server(guid.S()) - s.SetDumpRouterMap(false) s.BindHandler("/standard", func(r *ghttp.Request) { apiresponse.ErrorMsg(r, http.StatusBadRequest, "bad request") }) @@ -26,8 +25,7 @@ func TestErrorResponseStatusAndCode(t *testing.T) { apiresponse.ErrorWithCode(r, 10079, 10079, "invalid username") }) - server := httptest.NewServer(s) - defer server.Close() + baseURL := testutil.StartGFServer(t, s) tests := []struct { name string @@ -57,7 +55,7 @@ func TestErrorResponseStatusAndCode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - res, err := server.Client().Get(server.URL + tt.path) + res, err := http.Get(baseURL + tt.path) if err != nil { t.Fatalf("GET %s: %v", tt.path, err) } diff --git a/internal/testutil/gfserver.go b/internal/testutil/gfserver.go new file mode 100644 index 0000000..74482be --- /dev/null +++ b/internal/testutil/gfserver.go @@ -0,0 +1,38 @@ +// Package testutil 提供跨包共享的测试辅助工具。 +package testutil + +import ( + "fmt" + "testing" + + "github.com/gogf/gf/v2/net/ghttp" +) + +// StartGFServer 在 127.0.0.1 的随机端口上真实启动 ghttp.Server， +// 返回可访问的基础 URL，并通过 t.Cleanup 在测试结束后关闭服务。 +// +// 必须走真实 Start()，禁止把 g.Server 直接塞进 httptest.NewServer： +// gf 只在 Start() 中初始化 sessionManager（ghttp_server.go），未经 Start 的 +// server 处理请求时会在 handleAfterRequestDone 的 Session.Close() 中解引用 +// nil manager 而 panic——响应虽已写出、panic 被 net/http recover，测试侥幸 +// 通过，但每个请求都会向 stderr 刷一段 panic 栈。 +// +// 绑定 127.0.0.1 避免触发 Windows 防火墙弹窗；端口 0 由内核分配避免冲突。 +func StartGFServer(t *testing.T, s *ghttp.Server) string { + t.Helper() + s.SetAddr("127.0.0.1:0") + s.SetDumpRouterMap(false) + if err := s.Start(); err != nil { + t.Fatalf("启动测试 server 失败: %v", err) + } + port := s.GetListenedPort() + if port <= 0 { + t.Fatalf("测试 server 未监听端口: %d", port) + } + t.Cleanup(func() { + if err := s.Shutdown(); err != nil { + t.Logf("关闭测试 server 失败: %v", err) + } + }) + return fmt.Sprintf("http://127.0.0.1:%d", port) +} diff --git a/migrations/000016_add_cache_columns_to_bil_usage_daily.sql b/migrations/000016_add_cache_columns_to_bil_usage_daily.sql new file mode 100644 index 0000000..3b3c60b --- /dev/null +++ b/migrations/000016_add_cache_columns_to_bil_usage_daily.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- 模型性能监控：bil_usage_daily 增加缓存聚合列（数据源自 bil_usage_logs 已有的缓存明细字段）。 +-- cache_creation_tokens 缓存创建 token 数（Claude cache_creation_input_tokens） +-- cache_read_tokens 缓存命中读取 token 数（Claude cache_read / OpenAI cached_tokens） +-- cache_hit_request_count 命中缓存的请求数（明细行 cache_read_tokens > 0 计 1，与 request_count 同分母） +-- 上线后需用 aggregate-usage 命令回填历史区间（ON CONFLICT 幂等）。 + +ALTER TABLE bil_usage_daily ADD COLUMN cache_creation_tokens BIGINT DEFAULT 0 NOT NULL; +COMMENT ON COLUMN bil_usage_daily.cache_creation_tokens IS '缓存创建 token 数（Claude cache_creation，SUM 聚合）'; + +ALTER TABLE bil_usage_daily ADD COLUMN cache_read_tokens BIGINT DEFAULT 0 NOT NULL; +COMMENT ON COLUMN bil_usage_daily.cache_read_tokens IS '缓存命中读取 token 数（Claude cache_read / OpenAI cached_tokens，SUM 聚合）'; + +ALTER TABLE bil_usage_daily ADD COLUMN cache_hit_request_count BIGINT DEFAULT 0 NOT NULL; +COMMENT ON COLUMN bil_usage_daily.cache_hit_request_count IS '命中缓存的请求数（明细 cache_read_tokens>0 计 1，含失败状态行）'; + +-- +goose Down +ALTER TABLE bil_usage_daily DROP COLUMN IF EXISTS cache_creation_tokens; +ALTER TABLE bil_usage_daily DROP COLUMN IF EXISTS cache_read_tokens; +ALTER TABLE bil_usage_daily DROP COLUMN IF EXISTS cache_hit_request_count; diff --git a/migrations/000017_unify_input_tokens_cache_semantics.sql b/migrations/000017_unify_input_tokens_cache_semantics.sql new file mode 100644 index 0000000..5fcf7cd --- /dev/null +++ b/migrations/000017_unify_input_tokens_cache_semantics.sql @@ -0,0 +1,30 @@ +-- +goose Up +-- input_tokens 口径统一：bil_usage_logs.input_tokens 从此统一为「含缓存的总输入」。 +-- 历史问题：OpenAI/Gemini 渠道（promptTokenCount/prompt_tokens 原生含缓存）与 Claude 渠道 +-- （input_tokens 原生不含缓存）写入同一列但语义不同，跨渠道 SUM 聚合与缓存命中率分母失真。 +-- 写侧已改为统一按 TotalInputTokens() 入库（Claude 口径补加 cache_read + cache_creation）， +-- 本迁移回填历史 Claude 渠道行（channel_type=2）使存量数据同口径。 +-- 上线后需用 aggregate-usage 命令回填 bil_usage_daily 历史区间（ON CONFLICT 幂等）： +-- team-api aggregate-usage --from <最早用量日期> --to <今天> + +UPDATE bil_usage_logs +SET input_tokens = input_tokens + COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0), + total_tokens = total_tokens + COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0) +WHERE channel_type = 2 + AND (COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0)) > 0; + +-- 列语义注释同步更新（cache_creation_tokens 兼收 OpenAI Responses 的 cache_write_tokens） +COMMENT ON COLUMN bil_usage_logs.input_tokens IS '输入 token 数（含缓存总输入：base + cache_read + cache_creation，跨渠道统一口径）'; +COMMENT ON COLUMN bil_usage_logs.cache_creation_tokens IS '写入缓存的 token 数（Claude cache_creation_input_tokens / OpenAI Responses cache_write_tokens）'; +COMMENT ON COLUMN bil_usage_daily.input_tokens IS '输入Token合计（含缓存总输入，源自 bil_usage_logs.input_tokens）'; +COMMENT ON COLUMN bil_usage_daily.cache_creation_tokens IS '写入缓存的 token 数（Claude cache_creation / OpenAI cache_write，SUM 聚合）'; + +-- +goose Down +-- 逆转回填：Claude 渠道行恢复「不含缓存」口径（OpenAI/Gemini 行本就含缓存，不受影响）。 +-- 注意：Down 后需再次执行 aggregate-usage 回滚 bil_usage_daily。 + +UPDATE bil_usage_logs +SET input_tokens = input_tokens - COALESCE(cache_read_tokens, 0) - COALESCE(cache_creation_tokens, 0), + total_tokens = total_tokens - COALESCE(cache_read_tokens, 0) - COALESCE(cache_creation_tokens, 0) +WHERE channel_type = 2 + AND (COALESCE(cache_read_tokens, 0) + COALESCE(cache_creation_tokens, 0)) > 0; diff --git a/relay/channel/claude/response.go b/relay/channel/claude/response.go index dc38240..649d2cd 100644 --- a/relay/channel/claude/response.go +++ b/relay/channel/claude/response.go @@ -283,10 +283,13 @@ func (a *Adaptor) handleStreamToOpenAI(ctx context.Context, resp *http.Response, if reason == "" { reason = "stop" } + // Claude 的 input_tokens 不含缓存（三项并列），OpenAI 的 prompt_tokens 含缓存 + //（cached 是其子集），客户端可见 usage 做加法；计费返回值另行按 Claude 口径构建 + promptTotal := usage.InputTokens + usage.CacheReadInputTokens + usage.CacheCreationInputTokens usageObj := &dto.UsageWithDetails{ - PromptTokens: usage.InputTokens, + PromptTokens: promptTotal, CompletionTokens: usage.OutputTokens, - TotalTokens: usage.InputTokens + usage.OutputTokens, + TotalTokens: promptTotal + usage.OutputTokens, PromptTokensDetails: common.CommonTokenDetailsToDto(claudeUsageToTokenDetails(&usage)), } if usageObj.CompletionTokens == 0 { @@ -656,10 +659,21 @@ func claudeToOpenAIResponse(claudeResp *dto.ClaudeResponse, info *common.RelayIn resp.Choices[0].Message = message if claudeResp.Usage != nil { + // Claude 的 input_tokens 不含缓存（三项并列），OpenAI 的 prompt_tokens 含缓存 + //（cached 是其子集），转换做加法并透出缓存明细，客户端按 OpenAI 语义解析才正确 + promptTotal := claudeResp.Usage.InputTokens + + claudeResp.Usage.CacheReadInputTokens + + claudeResp.Usage.CacheCreationInputTokens resp.Usage = dto.UsageWithDetails{ - PromptTokens: claudeResp.Usage.InputTokens, + PromptTokens: promptTotal, CompletionTokens: claudeResp.Usage.OutputTokens, - TotalTokens: claudeResp.Usage.InputTokens + claudeResp.Usage.OutputTokens, + TotalTokens: promptTotal + claudeResp.Usage.OutputTokens, + } + if claudeResp.Usage.CacheReadInputTokens > 0 || claudeResp.Usage.CacheCreationInputTokens > 0 { + resp.Usage.PromptTokensDetails = &dto.TokenDetails{ + CachedTokens: claudeResp.Usage.CacheReadInputTokens, + CachedCreationTokens: claudeResp.Usage.CacheCreationInputTokens, + } } } diff --git a/relay/channel/codex/adaptor.go b/relay/channel/codex/adaptor.go index 37372fd..7688684 100644 --- a/relay/channel/codex/adaptor.go +++ b/relay/channel/codex/adaptor.go @@ -95,10 +95,13 @@ func (a *Adaptor) ConvertRequest(ctx context.Context, info *common.RelayInfo, re // 官方兼容性字段处理（对齐 Codex CLI 行为）： // - store 必须为 false（chatgpt.com 官方后端硬性要求） - // - 移除 max_output_tokens / temperature（上游不支持这两个字段，透传会被拒） + // - 移除 max_output_tokens / temperature（上游不支持，透传会被拒） + // - 移除 frequency_penalty / presence_penalty（非官方 Responses 参数，防御性剥离） rawMap["store"] = json.RawMessage("false") delete(rawMap, "max_output_tokens") delete(rawMap, "temperature") + delete(rawMap, "frequency_penalty") + delete(rawMap, "presence_penalty") // instructions 字段必须存在，缺省时默认空串 if _, ok := rawMap["instructions"]; !ok { rawMap["instructions"] = json.RawMessage(`""`) diff --git a/relay/channel/gemini/response.go b/relay/channel/gemini/response.go index 23a1a01..28ecad0 100644 --- a/relay/channel/gemini/response.go +++ b/relay/channel/gemini/response.go @@ -518,10 +518,18 @@ func (a *Adaptor) handleStreamToOpenAI(ctx context.Context, resp *http.Response, } if totalUsage.PromptTokenCount > 0 || totalUsage.CandidatesTokenCount > 0 { + // OpenAI 口径：prompt 含 cached（子集），completion 含 thoughts（子集）， + // 与计费用量 geminiUsageToCommon 保持同一换算 endChunk.Usage = &dto.UsageWithDetails{ PromptTokens: totalUsage.PromptTokenCount, - CompletionTokens: totalUsage.CandidatesTokenCount, + CompletionTokens: totalUsage.CandidatesTokenCount + totalUsage.ThoughtsTokenCount, TotalTokens: totalUsage.TotalTokenCount, + PromptTokensDetails: &dto.TokenDetails{ + CachedTokens: totalUsage.CachedContentTokenCount, + }, + CompletionTokenDetails: &dto.TokenDetails{ + ReasoningTokens: totalUsage.ThoughtsTokenCount, + }, } } @@ -623,10 +631,18 @@ func geminiToOpenAIResponse(geminiResp *dto.GeminiChatResponse, info *common.Rel resp.Choices = append(resp.Choices, choice) if geminiResp.UsageMetadata != nil { + // OpenAI 口径：prompt 含 cached（子集），completion 含 thoughts（子集）， + // 与计费用量 geminiUsageToCommon 保持同一换算 resp.Usage = dto.UsageWithDetails{ PromptTokens: geminiResp.UsageMetadata.PromptTokenCount, - CompletionTokens: geminiResp.UsageMetadata.CandidatesTokenCount, + CompletionTokens: geminiResp.UsageMetadata.CandidatesTokenCount + geminiResp.UsageMetadata.ThoughtsTokenCount, TotalTokens: geminiResp.UsageMetadata.TotalTokenCount, + PromptTokensDetails: &dto.TokenDetails{ + CachedTokens: geminiResp.UsageMetadata.CachedContentTokenCount, + }, + CompletionTokenDetails: &dto.TokenDetails{ + ReasoningTokens: geminiResp.UsageMetadata.ThoughtsTokenCount, + }, } } @@ -645,9 +661,15 @@ func geminiUsageToCommon(um *dto.GeminiUsageMetadata) *common.Usage { return &common.Usage{} } usage := &common.Usage{ - PromptTokens: um.PromptTokenCount, - CompletionTokens: um.CandidatesTokenCount, + PromptTokens: um.PromptTokenCount, + // Gemini 的 candidatesTokenCount 不含思考 token，thoughtsTokenCount 是输出侧 + // 独立字段（按输出价计费）。OpenAI 口径的 completion 含 reasoning（子集语义）， + // 计费用量必须 candidates+thoughts 合计，否则思考 token 漏计费 + CompletionTokens: um.CandidatesTokenCount + um.ThoughtsTokenCount, TotalTokens: um.TotalTokenCount, + // Gemini 的 promptTokenCount 已含 cachedContentTokenCount（cached 为其子集）， + // 置 true 让计费先扣减缓存部分，避免「input 全价 + cache 价」双重计费 + CacheIncludedInPrompt: true, PromptTokensDetails: &common.TokenDetails{ CachedTokens: um.CachedContentTokenCount, }, diff --git a/relay/channel/gemini/response_test.go b/relay/channel/gemini/response_test.go index 04e4679..1935f81 100644 --- a/relay/channel/gemini/response_test.go +++ b/relay/channel/gemini/response_test.go @@ -3,6 +3,8 @@ package gemini import ( "strings" "testing" + + "github.com/qianfree/team-api/relay/dto" ) // TestBuildGeminiUpstreamError_ResourceExhausted 验证真实 Gemini 429（区域配额耗尽） @@ -39,3 +41,33 @@ func TestBuildGeminiUpstreamError_FallbackStatusCode(t *testing.T) { t.Errorf("StatusCode = %d, want 503 (fallback)", err.StatusCode) } } + +// TestGeminiUsageToCommon_CacheSemantics 验证 Gemini usage 转换的缓存与思考语义： +// 1. Gemini 的 promptTokenCount 已含 cachedContentTokenCount（cached 为其子集）， +// 必须置 CacheIncludedInPrompt=true 让计费扣减缓存部分，否则缓存 token 会被 +// 「input 全价 + cache 价」双重计费； +// 2. Gemini 的 candidatesTokenCount 不含思考 token，thoughtsTokenCount 是输出侧 +// 独立字段（按输出价计费），completion 必须为 candidates+thoughts 合计，否则思考漏计费。 +func TestGeminiUsageToCommon_CacheSemantics(t *testing.T) { + usage := geminiUsageToCommon(&dto.GeminiUsageMetadata{ + PromptTokenCount: 414, + CandidatesTokenCount: 219, + TotalTokenCount: 633, + CachedContentTokenCount: 231, + ThoughtsTokenCount: 100, + }) + + if !usage.CacheIncludedInPrompt { + t.Error("CacheIncludedInPrompt = false, want true (cachedContentTokenCount 是 promptTokenCount 的子集)") + } + // 计费输出须含思考 token：completion = candidates(219) + thoughts(100) + if usage.PromptTokens != 414 || usage.CompletionTokens != 319 || usage.TotalTokens != 633 { + t.Errorf("token counts = %+v, want prompt=414 completion=319 total=633", usage) + } + if usage.PromptTokensDetails == nil || usage.PromptTokensDetails.CachedTokens != 231 { + t.Errorf("cached tokens = %+v, want 231", usage.PromptTokensDetails) + } + if usage.CompletionTokenDetails == nil || usage.CompletionTokenDetails.ReasoningTokens != 100 { + t.Errorf("reasoning tokens = %+v, want 100", usage.CompletionTokenDetails) + } +} diff --git a/relay/channel/ollama/adaptor.go b/relay/channel/ollama/adaptor.go index f3406bf..ab6bfc0 100644 --- a/relay/channel/ollama/adaptor.go +++ b/relay/channel/ollama/adaptor.go @@ -184,7 +184,20 @@ func (a *Adaptor) handleChatNonStreamResponse(ctx context.Context, resp *http.Re } // 转换为 OpenAI ChatCompletion 格式 - finishReason := "stop" + message := dto.Message{ + Role: ollamaResp.Message.Role, + Content: ollamaResp.Message.Content, + } + if reasoning := ollamaThinkingToOpenAI(ollamaResp.Message.Thinking); reasoning != nil { + message.ReasoningContent = reasoning + } + if toolCalls := legacyOllamaToolCallsToOpenAI(ollamaResp.Message.ToolCalls); len(toolCalls) > 0 { + message.ToolCalls = toolCalls + } + finishReason := ollamaDoneReasonToFinishReason(ollamaResp.DoneReason) + if len(message.ToolCalls) > 0 { + finishReason = "tool_calls" + } openaiResp := dto.ChatCompletionResponse{ ID: fmt.Sprintf("chatcmpl-%s", info.RequestID), Object: "chat.completion", @@ -192,11 +205,8 @@ func (a *Adaptor) handleChatNonStreamResponse(ctx context.Context, resp *http.Re Model: info.OriginModelName, Choices: []dto.Choice{ { - Index: 0, - Message: dto.Message{ - Role: ollamaResp.Message.Role, - Content: ollamaResp.Message.Content, - }, + Index: 0, + Message: message, FinishReason: finishReason, }, }, @@ -244,6 +254,8 @@ func (a *Adaptor) handleChatStreamResponse(ctx context.Context, resp *http.Respo var usage common.Usage var transferredTextLen int // 已转发的文本长度，供流中断输出估算 + sawToolCalls := false + nextToolCallIndex := 0 for scanner.Scan() { select { case <-ctx.Done(): @@ -273,7 +285,10 @@ func (a *Adaptor) handleChatStreamResponse(ctx context.Context, resp *http.Respo usage.TotalTokens = ollamaResp.PromptEvalCount + ollamaResp.EvalCount // 发送带 finish_reason 的结束 chunk - finishReason := "stop" + finishReason := ollamaDoneReasonToFinishReason(ollamaResp.DoneReason) + if sawToolCalls { + finishReason = "tool_calls" + } endChunk := dto.ChatCompletionStreamResponse{ ID: fmt.Sprintf("chatcmpl-%s", info.RequestID), Object: "chat.completion.chunk", @@ -297,6 +312,17 @@ func (a *Adaptor) handleChatStreamResponse(ctx context.Context, resp *http.Respo // 构建 OpenAI 流式 chunk transferredTextLen += len(ollamaResp.Message.Content) + delta := dto.Message{ + Role: ollamaResp.Message.Role, + Content: ollamaResp.Message.Content, + } + if thinking := ollamaThinkingToOpenAI(ollamaResp.Message.Thinking); thinking != nil { + delta.ReasoningContent = thinking + } + if toolCalls := legacyOllamaStreamToolCalls(ollamaResp.Message.ToolCalls, &nextToolCallIndex); len(toolCalls) > 0 { + delta.ToolCalls = toolCalls + sawToolCalls = true + } chunk := dto.ChatCompletionStreamResponse{ ID: fmt.Sprintf("chatcmpl-%s", info.RequestID), Object: "chat.completion.chunk", @@ -304,10 +330,7 @@ func (a *Adaptor) handleChatStreamResponse(ctx context.Context, resp *http.Respo Choices: []dto.StreamChoice{ { Index: 0, - Delta: dto.Message{ - Role: ollamaResp.Message.Role, - Content: ollamaResp.Message.Content, - }, + Delta: delta, }, }, } @@ -522,3 +545,87 @@ func writeStreamChunk(w http.ResponseWriter, chunk any) { data, _ := json.Marshal(chunk) _ = helper.WriteSSEData(w, string(data)) } + +// ollamaThinkingToOpenAI 将 Ollama thinking 字段（JSON 字符串）转为 OpenAI reasoning_content。 +// 字段缺失、null、空串或非字符串时返回 nil，避免产出空的 reasoning_content。 +func ollamaThinkingToOpenAI(thinking json.RawMessage) *string { + if len(thinking) == 0 { + return nil + } + var s string + if err := json.Unmarshal(thinking, &s); err != nil { + return nil + } + // JSON null 反序列化不报错且保持零值，空思考内容同样不产出字段 + if s == "" { + return nil + } + return &s +} + +// ollamaDoneReasonToFinishReason 将 Ollama done_reason 映射为 OpenAI finish_reason。 +// stop/length 均为 OpenAI 合法取值，其余（如 load 等运行态原因）归一为 stop。 +func ollamaDoneReasonToFinishReason(doneReason string) string { + if doneReason == "length" { + return "length" + } + return "stop" +} + +// legacyOllamaToolCallsToOpenAI 将 Ollama tool_calls 转换为 OpenAI tool_calls（非流式回退路径）。 +func legacyOllamaToolCallsToOpenAI(toolCalls []OllamaToolCall) []dto.ToolCall { + if len(toolCalls) == 0 { + return nil + } + out := make([]dto.ToolCall, 0, len(toolCalls)) + for i, tc := range toolCalls { + id := tc.ID + if id == "" { + id = fmt.Sprintf("call_%d", i) + } + argsBytes := []byte("{}") + if tc.Function.Arguments != nil { + if b, err := json.Marshal(tc.Function.Arguments); err == nil { + argsBytes = b + } + } + out = append(out, dto.ToolCall{ + ID: id, + Type: "function", + Function: dto.FunctionCall{ + Name: tc.Function.Name, + Arguments: string(argsBytes), + }, + }) + } + return out +} + +// legacyOllamaStreamToolCalls 将 Ollama tool_calls 转换为 OpenAI 流式增量 delta（回退路径）， +// 为每个调用分配稳定递增的 index。 +func legacyOllamaStreamToolCalls(toolCalls []OllamaToolCall, nextIndex *int) []dto.ToolCall { + out := make([]dto.ToolCall, 0, len(toolCalls)) + for _, tc := range toolCalls { + id := tc.ID + if id == "" { + id = fmt.Sprintf("call_%d", *nextIndex) + } + argsBytes := []byte("{}") + if tc.Function.Arguments != nil { + if b, err := json.Marshal(tc.Function.Arguments); err == nil { + argsBytes = b + } + } + out = append(out, dto.ToolCall{ + Index: *nextIndex, + ID: id, + Type: "function", + Function: dto.FunctionCall{ + Name: tc.Function.Name, + Arguments: string(argsBytes), + }, + }) + *nextIndex++ + } + return out +} diff --git a/relay/channel/ollama/converter.go b/relay/channel/ollama/converter.go index c30ad2a..2c4f288 100644 --- a/relay/channel/ollama/converter.go +++ b/relay/channel/ollama/converter.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/qianfree/team-api/relay/common" + "github.com/qianfree/team-api/relay/dto" ) // convertChatRequest 将 OpenAI Chat Completions 请求转换为 Ollama Chat 格式 @@ -22,8 +23,12 @@ func convertChatRequest(requestBody []byte, info *common.RelayInfo) ([]byte, err // 解析消息 var openaiMessages []struct { - Role string `json:"role"` - Content any `json:"content"` + Role string `json:"role"` + Content any `json:"content"` + ToolCallID string `json:"tool_call_id"` + Name string `json:"name"` + ReasoningContent *string `json:"reasoning_content"` + ToolCalls []dto.ToolCall `json:"tool_calls"` } if raw, ok := rawReq["messages"]; ok { if err := json.Unmarshal(raw, &openaiMessages); err != nil { @@ -32,11 +37,12 @@ func convertChatRequest(requestBody []byte, info *common.RelayInfo) ([]byte, err } ollamaMessages := make([]OllamaMessage, 0, len(openaiMessages)) + toolNamesByCallID := make(map[string]string) for _, msg := range openaiMessages { ollamaMsg := OllamaMessage{ Role: msg.Role, } - // content 可以是 string 或 []ContentPart + // content 可以是 string 或 []ContentPart（多模态） switch c := msg.Content.(type) { case string: ollamaMsg.Content = c @@ -47,8 +53,7 @@ func convertChatRequest(requestBody []byte, info *common.RelayInfo) ([]byte, err if !ok { continue } - partType, _ := partMap["type"].(string) - if partType == "text" { + if partType, _ := partMap["type"].(string); partType == "text" { if text, ok := partMap["text"].(string); ok { ollamaMsg.Content += text } @@ -61,6 +66,45 @@ func convertChatRequest(requestBody []byte, info *common.RelayInfo) ([]byte, err ollamaMsg.Content = string(b) } } + // 保留跨轮次的工具调用与推理上下文（与 relaykit 转换器行为对齐） + if msg.Role == "tool" { + ollamaMsg.ToolCallID = msg.ToolCallID + ollamaMsg.ToolName = msg.Name + if ollamaMsg.ToolName == "" { + // 消息未带 name 时按 tool_call_id 从前面 assistant 的 tool_calls 反查工具名； + // Ollama 的 tool 消息靠 tool_name 关联结果（tool_call_id 仅原样透传）。 + ollamaMsg.ToolName = toolNamesByCallID[msg.ToolCallID] + } + } + if msg.Role == "assistant" { + if msg.ReasoningContent != nil { + if b, err := json.Marshal(*msg.ReasoningContent); err == nil { + ollamaMsg.Thinking = b + } + } + if len(msg.ToolCalls) > 0 { + calls := make([]OllamaToolCall, 0, len(msg.ToolCalls)) + for _, tc := range msg.ToolCalls { + oc := OllamaToolCall{ID: tc.ID} + oc.Function.Name = tc.Function.Name + var args any + if tc.Function.Arguments != "" { + if err := json.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil { + return nil, fmt.Errorf("parse tool call arguments: %w", err) + } + } + if args == nil { + args = map[string]any{} + } + oc.Function.Arguments = args + calls = append(calls, oc) + if tc.ID != "" { + toolNamesByCallID[tc.ID] = tc.Function.Name + } + } + ollamaMsg.ToolCalls = calls + } + } ollamaMessages = append(ollamaMessages, ollamaMsg) } @@ -75,6 +119,9 @@ func convertChatRequest(requestBody []byte, info *common.RelayInfo) ([]byte, err Messages: ollamaMessages, Stream: stream, Options: options, + Tools: legacyOllamaTools(rawReq), + Format: legacyOllamaFormat(rawReq), + Think: legacyOllamaThink(rawReq), } return json.Marshal(ollamaReq) @@ -218,3 +265,101 @@ func buildOptions(rawReq map[string]json.RawMessage) map[string]any { } return options } + +// legacyOllamaTools 将原始 OpenAI tools 映射为 Ollama tools（回退路径）。 +func legacyOllamaTools(rawReq map[string]json.RawMessage) []OllamaTool { + raw, ok := rawReq["tools"] + if !ok { + return nil + } + var tools []struct { + Type string `json:"type"` + Function struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Parameters any `json:"parameters,omitempty"` + } `json:"function"` + } + if err := json.Unmarshal(raw, &tools); err != nil { + return nil + } + out := make([]OllamaTool, 0, len(tools)) + for _, t := range tools { + if t.Type != "function" { + continue + } + out = append(out, OllamaTool{ + Type: "function", + Function: OllamaToolFunction{ + Name: t.Function.Name, + Description: t.Function.Description, + Parameters: t.Function.Parameters, + }, + }) + } + if len(out) == 0 { + return nil + } + return out +} + +// legacyOllamaFormat 将原始 response_format 映射为 Ollama format（回退路径）。 +func legacyOllamaFormat(rawReq map[string]json.RawMessage) any { + raw, ok := rawReq["response_format"] + if !ok { + return nil + } + var rf struct { + Type string `json:"type"` + JSONSchema any `json:"json_schema,omitempty"` + } + if err := json.Unmarshal(raw, &rf); err != nil { + return nil + } + switch rf.Type { + case "json", "json_object": + return "json" + case "json_schema": + return legacyExtractJSONSchema(rf.JSONSchema) + default: + return nil + } +} + +// legacyOllamaThink 将原始 reasoning_effort 映射为 Ollama think（回退路径）。 +// "none" 关闭思考；low/medium/high/max 作为思考档位透传；未知取值跳过。 +func legacyOllamaThink(rawReq map[string]json.RawMessage) json.RawMessage { + raw, ok := rawReq["reasoning_effort"] + if !ok { + return nil + } + var effort string + if err := json.Unmarshal(raw, &effort); err != nil || effort == "" { + return nil + } + var value any + switch effort { + case "none": + value = false + case "low", "medium", "high", "max": + value = effort + default: + return nil + } + b, err := json.Marshal(value) + if err != nil { + return nil + } + return b +} + +// legacyExtractJSONSchema 从 OpenAI json_schema 包装结构（{name, description, schema, strict}） +// 中提取裸 schema；Ollama 的 format 字段期望 schema 对象本身。 +func legacyExtractJSONSchema(v any) any { + if m, ok := v.(map[string]any); ok { + if schema, ok := m["schema"]; ok { + return schema + } + } + return v +} diff --git a/relay/channel/ollama/dto.go b/relay/channel/ollama/dto.go index 52bf081..7b36baa 100644 --- a/relay/channel/ollama/dto.go +++ b/relay/channel/ollama/dto.go @@ -6,6 +6,9 @@ import relaykitdto "github.com/qianfree/team-api/relaykit/dto" type OllamaChatRequest = relaykitdto.OllamaChatRequest type OllamaMessage = relaykitdto.OllamaMessage type OllamaChatResponse = relaykitdto.OllamaChatResponse +type OllamaTool = relaykitdto.OllamaTool +type OllamaToolFunction = relaykitdto.OllamaToolFunction +type OllamaToolCall = relaykitdto.OllamaToolCall // OllamaGenerateRequest Ollama Generate（文本补全）请求 type OllamaGenerateRequest struct { diff --git a/relay/channel/openai/adaptor_responses_test.go b/relay/channel/openai/adaptor_responses_test.go index 1077818..fd797a2 100644 --- a/relay/channel/openai/adaptor_responses_test.go +++ b/relay/channel/openai/adaptor_responses_test.go @@ -255,6 +255,191 @@ func TestAdaptor_DoResponse_ResponsesUpstreamStream(t *testing.T) { if usage.PromptTokens != 10 || usage.CompletionTokens != 20 || usage.TotalTokens != 30 { t.Errorf("usage = %+v, want prompt=10 completion=20 total=30", usage) } + if usage.PromptTokensDetails == nil || usage.PromptTokensDetails.CachedTokens != 5 { + t.Errorf("cached usage = %+v, want cached=5", usage) + } +} + +// fakeResponseRouteStore 路由存储 fake，用于断言 Record 调用 +type fakeResponseRouteStore struct { + calls []fakeRouteCall +} + +type fakeRouteCall struct { + tenantID int64 + responseID string + route common.ResponseRoute +} + +func (f *fakeResponseRouteStore) Record(_ context.Context, tenantID int64, responseID string, route common.ResponseRoute) { + f.calls = append(f.calls, fakeRouteCall{tenantID: tenantID, responseID: responseID, route: route}) +} + +func (f *fakeResponseRouteStore) Lookup(context.Context, int64, string) (common.ResponseRoute, bool) { + return common.ResponseRoute{}, false +} + +func (f *fakeResponseRouteStore) Delete(context.Context, int64, string) {} + +// TestChatCompletionToResponsesResponse_StoreFalseAndEcho 合成响应的保真度： +// store 恒为 false（合成响应不落上游存储，不可 retrieve）； +// temperature/top_p/max_output_tokens/instructions 从请求快照 echo，快照缺失回退默认值。 +func TestChatCompletionToResponsesResponse_StoreFalseAndEcho(t *testing.T) { + buildChatResp := func() *dto.ChatCompletionResponse { + return &dto.ChatCompletionResponse{ + ID: "abc123", + Created: 1700000000, + Model: "gpt-4o", + Choices: []dto.Choice{{ + Index: 0, + Message: dto.Message{Role: "assistant", Content: "hello"}, + FinishReason: "stop", + }}, + Usage: dto.UsageWithDetails{ + PromptTokensDetails: &dto.TokenDetails{}, + CompletionTokenDetails: &dto.TokenDetails{}, + }, + } + } + + t.Run("echo from request snapshot", func(t *testing.T) { + temp, topP := 0.7, 0.9 + maxOut := uint(1024) + info := &common.RelayInfo{ + OriginModelName: "gpt-4o", + ResponsesRequest: &dto.OpenAIResponsesRequest{ + Temperature: &temp, + TopP: &topP, + MaxOutputTokens: &maxOut, + Instructions: json.RawMessage(`"be brief"`), + }, + } + resp := chatCompletionToResponsesResponse(buildChatResp(), info) + if resp.Store { + t.Error("store should be false（合成响应不可 retrieve）") + } + if resp.Temperature == nil || *resp.Temperature != 0.7 { + t.Errorf("temperature = %v, want 0.7", resp.Temperature) + } + if resp.TopP == nil || *resp.TopP != 0.9 { + t.Errorf("top_p = %v, want 0.9", resp.TopP) + } + if resp.MaxOutputTokens == nil || *resp.MaxOutputTokens != 1024 { + t.Errorf("max_output_tokens = %v, want 1024", resp.MaxOutputTokens) + } + if instr, ok := resp.Instructions.(json.RawMessage); !ok || string(instr) != `"be brief"` { + t.Errorf("instructions = %v, want echo", resp.Instructions) + } + }) + + t.Run("nil snapshot falls back to defaults", func(t *testing.T) { + info := &common.RelayInfo{OriginModelName: "gpt-4o"} + resp := chatCompletionToResponsesResponse(buildChatResp(), info) + if resp.Store { + t.Error("store should be false") + } + if resp.Temperature == nil || *resp.Temperature != 1.0 { + t.Errorf("temperature = %v, want default 1.0", resp.Temperature) + } + if resp.TopP == nil || *resp.TopP != 1.0 { + t.Errorf("top_p = %v, want default 1.0", resp.TopP) + } + }) +} + +// TestBuildResponsesObjectMap_StoreFalseEcho 流式合成 response 对象同样 store:false + echo。 +func TestBuildResponsesObjectMap_StoreFalseEcho(t *testing.T) { + temp := 0.5 + info := &common.RelayInfo{ + ResponsesRequest: &dto.OpenAIResponsesRequest{Temperature: &temp}, + } + m := buildResponsesObjectMap("resp_1", 1700000000, "completed", "gpt-4o", []any{}, nil, nil, info) + if m["store"] != false { + t.Errorf("store = %v, want false", m["store"]) + } + if m["temperature"] != 0.5 { + t.Errorf("temperature = %v, want 0.5", m["temperature"]) + } +} + +// TestAdaptor_DoResponse_ResponsesUpstreamNonStream_RecordsRoute 直连非流式响应： +// 解析出 response id 后记录 response_id → 渠道路由（tenant 隔离、lookupModel 口径）。 +func TestAdaptor_DoResponse_ResponsesUpstreamNonStream_RecordsRoute(t *testing.T) { + fake := &fakeResponseRouteStore{} + old := common.DefaultResponseRouteStore + common.DefaultResponseRouteStore = fake + t.Cleanup(func() { common.DefaultResponseRouteStore = old }) + + respBody := `{"id":"resp_route1","object":"response","status":"completed","output":[]}` + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": {"application/json"}}, + Body: io.NopCloser(strings.NewReader(respBody)), + } + info := responsesUpstreamInfo(constant.RelayModeResponses, false) + info.TenantID = 42 + info.BaseModelName = "gpt-4o" + info.ChannelMeta.ChannelID = 7 + + rec := httptest.NewRecorder() + a := &Adaptor{} + if _, err := a.DoResponse(context.Background(), resp, info, rec); err != nil { + t.Fatalf("DoResponse error: %v", err) + } + if len(fake.calls) != 1 { + t.Fatalf("Record calls = %d, want 1", len(fake.calls)) + } + call := fake.calls[0] + if call.tenantID != 42 || call.responseID != "resp_route1" { + t.Errorf("call = %+v", call) + } + if call.route.ChannelID != 7 || call.route.ModelName != "gpt-4o" { + t.Errorf("route = %+v", call.route) + } +} + +// TestAdaptor_DoResponse_ResponsesUpstreamStream_RecordsRoute 直连流式响应： +// response.created 事件即记录路由（cancel 尽早可用），completed 再刷新。 +func TestAdaptor_DoResponse_ResponsesUpstreamStream_RecordsRoute(t *testing.T) { + fake := &fakeResponseRouteStore{} + old := common.DefaultResponseRouteStore + common.DefaultResponseRouteStore = fake + t.Cleanup(func() { common.DefaultResponseRouteStore = old }) + + ss := strings.Join([]string{ + "event: response.created", + `data: {"type":"response.created","response":{"id":"resp_s1","object":"response","status":"in_progress"}}`, + "", + "event: response.completed", + `data: {"type":"response.completed","response":{"id":"resp_s1","status":"completed","usage":{"input_tokens":1,"output_tokens":2,"total_tokens":3}}}`, + "", + "data: [DONE]", + "", + }, "\n") + resp := &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": {"text/event-stream"}}, + Body: io.NopCloser(strings.NewReader(ss)), + } + info := responsesUpstreamInfo(constant.RelayModeResponses, true) + info.TenantID = 42 + info.BaseModelName = "gpt-4o" + info.ChannelMeta.ChannelID = 7 + + rec := httptest.NewRecorder() + a := &Adaptor{} + if _, err := a.DoResponse(context.Background(), resp, info, rec); err != nil { + t.Fatalf("DoResponse error: %v", err) + } + if len(fake.calls) < 2 { + t.Fatalf("Record calls = %d, want >=2（created + completed）", len(fake.calls)) + } + if fake.calls[0].responseID != "resp_s1" || fake.calls[0].tenantID != 42 { + t.Errorf("first call = %+v", fake.calls[0]) + } + if fake.calls[len(fake.calls)-1].route.ChannelID != 7 { + t.Errorf("last call route = %+v", fake.calls[len(fake.calls)-1].route) + } } // TestAdaptor_DoResponse_ResponsesUpstreamNonStream_Error 非 200 上游响应： diff --git a/relay/channel/openai/claude_response.go b/relay/channel/openai/claude_response.go index 99283af..5a6a5c7 100644 --- a/relay/channel/openai/claude_response.go +++ b/relay/channel/openai/claude_response.go @@ -102,6 +102,7 @@ func handleClaudeInboundStream(ctx context.Context, resp *http.Response, info *c contentIndex int inputTokens int outputTokens int + cachedTokens int // prompt_tokens_details.cached_tokens（OpenAI 口径：已含于 prompt_tokens） currentBlockType string // 跟踪当前 block 类型: "text" 或 "tool_use" transferredTextLen int // 已转发的文本/思考内容长度，供流中断输出估算 ) @@ -146,15 +147,23 @@ func handleClaudeInboundStream(ctx context.Context, resp *http.Response, info *c currentBlockType = "" } - // 发送 message_delta（stop_reason）+ message_stop + // 发送 message_delta（stop_reason）+ message_stop。 + // Claude 语义：input_tokens 不含 cache_read（OpenAI 的 prompt_tokens 含 cached，扣减后映射）； + // OpenAI 的 usage 在流尾才返回、message_start 时不可得，最终用量在此补报 reason := common.OpenAIFinishReasonToClaude(finishReason) + deltaInput := inputTokens - cachedTokens + if deltaInput < 0 { + deltaInput = 0 + } delta := dto.ClaudeResponse{ Type: "message_delta", Delta: &dto.ClaudeDelta{ StopReason: strPtr(reason), }, Usage: &dto.ClaudeUsage{ - OutputTokens: outputTokens, + InputTokens: deltaInput, + CacheReadInputTokens: cachedTokens, + OutputTokens: outputTokens, }, } writeClaudeSSE(writer, "message_delta", &delta) @@ -175,6 +184,14 @@ func handleClaudeInboundStream(ctx context.Context, resp *http.Response, info *c if streamResp.Usage != nil { inputTokens = streamResp.Usage.PromptTokens outputTokens = streamResp.Usage.CompletionTokens + if streamResp.Usage.PromptTokensDetails != nil { + cachedTokens = streamResp.Usage.PromptTokensDetails.CachedTokens + } + // 缓存/思考明细透传给计费；OpenAI 的 prompt_tokens 含 cached（子集语义）， + // 置 CacheIncludedInPrompt 让计费按明细扣减缓存部分，避免缓存按 input 全价计费 + usage.PromptTokensDetails = common.DtoTokenDetailsToCommon(streamResp.Usage.PromptTokensDetails) + usage.CompletionTokenDetails = common.DtoTokenDetailsToCommon(streamResp.Usage.CompletionTokenDetails) + usage.CacheIncludedInPrompt = true } // 发送 message_start（首块数据时） @@ -327,6 +344,11 @@ func handleClaudeInboundStream(ctx context.Context, resp *http.Response, info *c if err := scanner.Err(); err != nil && err != io.EOF && ctx.Err() == nil { info.StreamStatus.SetEndReason(common.StreamEndReasonError, err) + // 读流出错：带上已收到的 usage，避免整段按零值计费漏计 + usage.PromptTokens = inputTokens + usage.CompletionTokens = outputTokens + usage.TotalTokens = inputTokens + outputTokens + usage.CacheIncludedInPrompt = true return &usage, fmt.Errorf("stream scanner error: %w", err) } @@ -423,6 +445,18 @@ func openAIToClaudeResponse(openaiResp *dto.ChatCompletionResponse, info *common stopReason = common.OpenAIFinishReasonToClaude(openaiResp.Choices[0].FinishReason) } + // OpenAI 的 prompt_tokens 已含 cached（子集语义），Claude 的 input_tokens 不含 cache_read， + // 需扣减后映射，否则 Claude 客户端按本协议语义计账时缓存部分被重复计入。 + // PromptTokensDetails 可能为 nil（多数 OpenAI 兼容上游不返回该字段），必须判空。 + cachedTokens := 0 + if openaiResp.Usage.PromptTokensDetails != nil { + cachedTokens = openaiResp.Usage.PromptTokensDetails.CachedTokens + } + inputTokens := openaiResp.Usage.PromptTokens - cachedTokens + if inputTokens < 0 { + inputTokens = 0 + } + return dto.ClaudeResponse{ ID: fmt.Sprintf("msg_%s", info.RequestID), Type: "message", @@ -432,9 +466,9 @@ func openAIToClaudeResponse(openaiResp *dto.ChatCompletionResponse, info *common StopSequence: nil, Model: modelName, Usage: &dto.ClaudeUsage{ - InputTokens: openaiResp.Usage.PromptTokens, + InputTokens: inputTokens, OutputTokens: openaiResp.Usage.CompletionTokens, - CacheReadInputTokens: openaiResp.Usage.PromptTokensDetails.CachedTokens, + CacheReadInputTokens: cachedTokens, }, } } diff --git a/relay/channel/openai/claude_response_test.go b/relay/channel/openai/claude_response_test.go new file mode 100644 index 0000000..1a558dd --- /dev/null +++ b/relay/channel/openai/claude_response_test.go @@ -0,0 +1,126 @@ +package openai + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/qianfree/team-api/relay/common" + "github.com/qianfree/team-api/relay/dto" +) + +// newClaudeInboundTestInfo 构造 Claude 入站转换测试用的 RelayInfo。 +func newClaudeInboundTestInfo() *common.RelayInfo { + return &common.RelayInfo{ + RequestID: "req-claude-inbound", + OriginModelName: "gpt-4o", + StreamStatus: common.NewStreamStatus(), + ChannelMeta: &common.ChannelMeta{}, + } +} + +// TestOpenAIToClaudeResponse_UsageNilDetails 上游不返回 prompt_tokens_details（多数 OpenAI 兼容上游） +// 时不得空指针 panic，且无缓存可扣减。 +func TestOpenAIToClaudeResponse_UsageNilDetails(t *testing.T) { + info := newClaudeInboundTestInfo() + openaiResp := &dto.ChatCompletionResponse{ + ID: "chatcmpl-1", + Model: "gpt-4o", + Choices: []dto.Choice{{Index: 0, Message: dto.Message{Role: "assistant", Content: "hi"}, FinishReason: "stop"}}, + Usage: dto.UsageWithDetails{PromptTokens: 100, CompletionTokens: 20, TotalTokens: 120}, + } + + resp := openAIToClaudeResponse(openaiResp, info) // PromptTokensDetails 为 nil，此处原会 panic + + if resp.Usage.InputTokens != 100 { + t.Errorf("InputTokens = %d, want 100 (no cache to deduct)", resp.Usage.InputTokens) + } + if resp.Usage.CacheReadInputTokens != 0 { + t.Errorf("CacheReadInputTokens = %d, want 0", resp.Usage.CacheReadInputTokens) + } + if resp.Usage.OutputTokens != 20 { + t.Errorf("OutputTokens = %d, want 20", resp.Usage.OutputTokens) + } +} + +// TestOpenAIToClaudeResponse_UsageWithCache 上游返回缓存明细时，usage 需按 Claude 语义映射： +// OpenAI 的 prompt_tokens 含 cached（子集），Claude 的 input_tokens 不含 cache_read， +// 直接透传会导致客户端按本协议语义计账时缓存部分重复计入。 +func TestOpenAIToClaudeResponse_UsageWithCache(t *testing.T) { + info := newClaudeInboundTestInfo() + openaiResp := &dto.ChatCompletionResponse{ + ID: "chatcmpl-2", + Model: "gpt-4o", + Choices: []dto.Choice{{Index: 0, Message: dto.Message{Role: "assistant", Content: "hi"}, FinishReason: "stop"}}, + Usage: dto.UsageWithDetails{ + PromptTokens: 100, + CompletionTokens: 20, + TotalTokens: 120, + PromptTokensDetails: &dto.TokenDetails{CachedTokens: 30}, + }, + } + + resp := openAIToClaudeResponse(openaiResp, info) + + if resp.Usage.InputTokens != 70 { + t.Errorf("InputTokens = %d, want 70 (prompt 100 - cached 30)", resp.Usage.InputTokens) + } + if resp.Usage.CacheReadInputTokens != 30 { + t.Errorf("CacheReadInputTokens = %d, want 30", resp.Usage.CacheReadInputTokens) + } +} + +// TestHandleClaudeInboundStream_CacheUsage 流式路径缓存用量： +// 1. 计费 usage 携带缓存明细并置 CacheIncludedInPrompt（计费扣减缓存部分，避免按 input 全价计费）； +// 2. 最终 message_delta 按 Claude 语义补报扣减后的 input_tokens 与 cache_read_input_tokens +// （OpenAI 的 usage 在流尾才返回，message_start 无法提供真实值）。 +func TestHandleClaudeInboundStream_CacheUsage(t *testing.T) { + sse := `data: {"id":"chatcmpl-1","object":"chat.completion.chunk","model":"gpt-4o","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"}}]} + +data: {"id":"chatcmpl-1","object":"chat.completion.chunk","model":"gpt-4o","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]} + +data: {"id":"chatcmpl-1","object":"chat.completion.chunk","model":"gpt-4o","choices":[],"usage":{"prompt_tokens":100,"completion_tokens":20,"total_tokens":120,"prompt_tokens_details":{"cached_tokens":30},"completion_tokens_details":{"reasoning_tokens":5}}} + +data: [DONE] + +` + upstream := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(sse)), + Header: http.Header{}, + } + + info := newClaudeInboundTestInfo() + rec := httptest.NewRecorder() + + usage, err := handleClaudeInboundStream(context.Background(), upstream, info, rec) + if err != nil { + t.Fatalf("handleClaudeInboundStream error: %v", err) + } + + // 计费侧：OpenAI 原始口径（prompt 含 cached）+ 明细 + 扣减标记 + if usage.PromptTokens != 100 || usage.CompletionTokens != 20 || usage.TotalTokens != 120 { + t.Errorf("billing usage = %+v, want prompt=100 completion=20 total=120", usage) + } + if usage.PromptTokensDetails == nil || usage.PromptTokensDetails.CachedTokens != 30 { + t.Errorf("cache details = %+v, want cached=30", usage.PromptTokensDetails) + } + if !usage.CacheIncludedInPrompt { + t.Error("CacheIncludedInPrompt = false, want true (OpenAI prompt 含 cached)") + } + + // 客户端侧：message_delta 补报 Claude 语义 usage + body := rec.Body.String() + if !strings.Contains(body, `"input_tokens":70`) { + t.Errorf("message_delta missing deducted input_tokens=70, got: %s", body) + } + if !strings.Contains(body, `"cache_read_input_tokens":30`) { + t.Errorf("message_delta missing cache_read_input_tokens=30, got: %s", body) + } + if !strings.Contains(body, `"output_tokens":20`) { + t.Errorf("message_delta missing output_tokens=20, got: %s", body) + } +} diff --git a/relay/channel/openai/converter.go b/relay/channel/openai/converter.go index 306ce7a..36a1d78 100644 --- a/relay/channel/openai/converter.go +++ b/relay/channel/openai/converter.go @@ -175,6 +175,18 @@ func convertClaudeUserMessage(msg dto.ClaudeMessage) []dto.Message { if b, err := json.Marshal(cMap); err == nil { toolContent = string(b) } + } else if cArr, ok := m["content"].([]any); ok { + // 内容块数组（Claude 规范形式，如 [{type:"text",text:"..."}]）： + // 拼接文本块，否则工具结果会被替换为空字符串导致内容丢失 + var parts []string + for _, item := range cArr { + if b, ok := item.(map[string]any); ok && b["type"] == "text" { + if text, ok := b["text"].(string); ok { + parts = append(parts, text) + } + } + } + toolContent = c2oJoinParts(parts) } if len(contentParts) > 0 { if len(contentParts) == 1 && contentParts[0].Type == "text" { @@ -594,6 +606,18 @@ func ConvertResponsesToOpenAI(requestBody []byte, info *common.RelayInfo) (io.Re return nil, fmt.Errorf("parse responses request: %w", err) } + // 有状态请求快速失败：previous_response_id 的会话历史存储在上游 Responses 服务侧， + // chat-only 渠道无法还原（本网关不存储响应体），降级转换会静默丢失全部上下文。 + // 返回哨兵错误，由 relay_handler 按渠道级致命上报调度 FSM 换渠道。 + if req.PreviousResponseID != "" { + return nil, fmt.Errorf("stateful responses (previous_response_id) not supported by chat-only channels: %w", constant.ErrStatefulResponsesUnsupported) + } + + // stash 请求快照，供上游 chat 响应合成回 Responses 格式时 echo 请求参数 + if info != nil { + info.ResponsesRequest = &req + } + chatReq := make(map[string]any) if info.ChannelMeta.IsModelMapped { chatReq["model"] = info.ChannelMeta.UpstreamModelName @@ -655,6 +679,23 @@ func ConvertResponsesToOpenAI(requestBody []byte, info *common.RelayInfo) (io.Re if req.PromptCacheKey != "" { chatReq["prompt_cache_key"] = req.PromptCacheKey } + if len(req.Text) > 0 { + if rf := r2cParseTextFormat(req.Text); rf != nil { + chatReq["response_format"] = rf + } + } + if len(req.FrequencyPenalty) > 0 { + var v float64 + if err := json.Unmarshal(req.FrequencyPenalty, &v); err == nil { + chatReq["frequency_penalty"] = v + } + } + if len(req.PresencePenalty) > 0 { + var v float64 + if err := json.Unmarshal(req.PresencePenalty, &v); err == nil { + chatReq["presence_penalty"] = v + } + } if len(req.Metadata) > 0 { chatReq["metadata"] = json.RawMessage(req.Metadata) } @@ -681,6 +722,18 @@ type r2cContentPart struct { ImageURL string `json:"image_url,omitempty"` URL string `json:"url,omitempty"` Detail string `json:"detail,omitempty"` + // input_audio：Responses 为 {"type":"input_audio","input_audio":{"data","format"}}， + // chat 同形，原样透传 + InputAudio *r2cInputAudio `json:"input_audio,omitempty"` + // input_file：Responses 为扁平 {"type":"input_file","file_data","filename"}， + // 转换为 chat 的 {"type":"file","file":{"file_data","filename"}} + FileData string `json:"file_data,omitempty"` + Filename string `json:"filename,omitempty"` +} + +type r2cInputAudio struct { + Data string `json:"data,omitempty"` + Format string `json:"format,omitempty"` } func r2cConvertInputToMessages(input json.RawMessage) ([]map[string]any, error) { @@ -740,6 +793,24 @@ func r2cConvertMessage(item r2cInputItem) map[string]any { switch part.Type { case "input_text": chatParts = append(chatParts, map[string]any{"type": "text", "text": part.Text}) + case "input_audio": + // 音频输入：chat 的 input_audio 与 Responses 同形（input_audio:{data,format}） + if part.InputAudio != nil && part.InputAudio.Data != "" { + audio := map[string]any{"data": part.InputAudio.Data} + if part.InputAudio.Format != "" { + audio["format"] = part.InputAudio.Format + } + chatParts = append(chatParts, map[string]any{"type": "input_audio", "input_audio": audio}) + } + case "input_file": + // 文件输入：Responses 扁平 {file_data,filename} → chat 的 file:{file_data,filename} + if part.FileData != "" { + file := map[string]any{"file_data": part.FileData} + if part.Filename != "" { + file["filename"] = part.Filename + } + chatParts = append(chatParts, map[string]any{"type": "file", "file": file}) + } case "input_image": imageURL := part.ImageURL if imageURL == "" { @@ -805,6 +876,44 @@ func r2cConvertToolChoice(tcRaw json.RawMessage) any { return tc } +// r2cParseTextFormat 解析 Responses text.format（扁平 {type,name,schema,strict}） +// 为 chat 的 response_format（json_schema 时嵌套为 json_schema:{name,schema,strict}）。 +// text 或未知类型返回 nil（chat 无对应字段，不映射）。 +func r2cParseTextFormat(raw json.RawMessage) map[string]any { + var textCfg struct { + Format struct { + Type string `json:"type"` + Name string `json:"name"` + Schema json.RawMessage `json:"schema"` + Strict *bool `json:"strict"` + } `json:"format"` + } + if err := json.Unmarshal(raw, &textCfg); err != nil { + return nil + } + switch textCfg.Format.Type { + case "json_object": + return map[string]any{"type": "json_object"} + case "json_schema": + jsonSchema := make(map[string]any, 3) + if textCfg.Format.Name != "" { + jsonSchema["name"] = textCfg.Format.Name + } + if len(textCfg.Format.Schema) > 0 { + var schema any + if err := json.Unmarshal(textCfg.Format.Schema, &schema); err == nil { + jsonSchema["schema"] = schema + } + } + if textCfg.Format.Strict != nil { + jsonSchema["strict"] = *textCfg.Format.Strict + } + return map[string]any{"type": "json_schema", "json_schema": jsonSchema} + default: + return nil + } +} + // ===== OpenAI → Responses 请求转换（Responses API Bridge） ===== // ConvertOpenAIToResponses 将 Chat Completions 请求体转换为 Responses API 请求体 @@ -835,12 +944,23 @@ func ConvertOpenAIToResponses(body []byte, info *common.RelayInfo) ([]byte, erro if req.TopP != nil { respReq["top_p"] = *req.TopP } + // prompt_cache_key 是官方 Responses API 参数，原样透传； + // presence/frequency penalty 不属于官方 Responses API，透传会被严格上游 + // （如 api.openai.com）以未知参数拒绝，chat 入站时不注入、静默丢弃。 + if req.PromptCacheKey != "" { + respReq["prompt_cache_key"] = req.PromptCacheKey + } if maxTokens := c2rGetMaxTokens(req); maxTokens > 0 { respReq["max_output_tokens"] = maxTokens } if req.ResponseFormat != nil { - respReq["text"] = map[string]any{"format": req.ResponseFormat} + if tf := c2rBuildTextFormat(req.ResponseFormat); tf != nil { + respReq["text"] = tf + } } + // 桥接的响应不可被 chat 客户端经 previous_response_id 引用（chat 协议无此概念）， + // 显式 store:false 避免上游无谓存储；渠道配置 DisableStore 时 SanitizeFields 会删掉该字段 + respReq["store"] = false if req.ReasoningEffort != "" { respReq["reasoning"] = map[string]any{"effort": req.ReasoningEffort, "summary": "detailed"} } @@ -1059,6 +1179,32 @@ func c2rGetMaxTokens(req dto.GeneralOpenAIRequest) int { return max } +// c2rBuildTextFormat 将 chat 的 response_format 转换为 Responses 的 text 配置。 +// chat 的 json_schema 为嵌套 {type,json_schema:{name,schema,strict}}， +// Responses 的 format 为扁平 {type,name,schema,strict}——需解包提升，不能原样塞入。 +// 其余类型（json_object）两侧同形；无法识别时返回 nil 不映射。 +func c2rBuildTextFormat(rf *dto.ResponseFormat) map[string]any { + if rf == nil { + return nil + } + switch rf.Type { + case "json_object": + return map[string]any{"format": map[string]any{"type": "json_object"}} + case "json_schema": + format := map[string]any{"type": "json_schema"} + if js, ok := rf.JSONSchema.(map[string]any); ok { + for _, k := range []string{"name", "schema", "strict"} { + if v, ok := js[k]; ok { + format[k] = v + } + } + } + return map[string]any{"format": format} + default: + return nil + } +} + // ===== Responses → Chat 响应转换 ===== // ResponsesResponseToChatCompletions 将 Responses API 非流式响应转换为 Chat Completions 响应 @@ -1320,9 +1466,7 @@ func HandleResponsesStreamToChat(ctx context.Context, resp *http.Response, info createAt = int64(streamResp.Response.CreatedAt) } if streamResp.Response.Usage != nil { - totalUsage.PromptTokens = streamResp.Response.Usage.InputTokens - totalUsage.CompletionTokens = streamResp.Response.Usage.OutputTokens - totalUsage.TotalTokens = streamResp.Response.Usage.TotalTokens + totalUsage = *responsesUsageToCommon(streamResp.Response.Usage) if totalUsage.TotalTokens == 0 { totalUsage.TotalTokens = totalUsage.PromptTokens + totalUsage.CompletionTokens } diff --git a/relay/channel/openai/converter_test.go b/relay/channel/openai/converter_test.go index 5849ed7..cab7c47 100644 --- a/relay/channel/openai/converter_test.go +++ b/relay/channel/openai/converter_test.go @@ -2,8 +2,12 @@ package openai import ( "encoding/json" + "errors" + "io" "testing" + "github.com/qianfree/team-api/relay/common" + "github.com/qianfree/team-api/relay/constant" "github.com/qianfree/team-api/relay/dto" ) @@ -199,3 +203,300 @@ func TestConvertClaudeUserMessage_StringContent(t *testing.T) { t.Errorf("got %+v", msgs[0]) } } + +// TestConvertClaudeUserMessage_ToolResultArrayContent tool_result.content 为内容块数组 +// （Claude 规范形式 [{type:"text",text:"..."}]）时，文本块须拼接保留， +// 否则工具结果会被替换为空字符串导致内容丢失。 +func TestConvertClaudeUserMessage_ToolResultArrayContent(t *testing.T) { + msgs := convertClaudeUserMessage(dto.ClaudeMessage{ + Role: "user", + Content: []any{ + map[string]any{ + "type": "tool_result", + "tool_use_id": "toolu_123", + "content": []any{ + map[string]any{"type": "text", "text": "result-a"}, + map[string]any{"type": "text", "text": "result-b"}, + }, + }, + }, + }) + if len(msgs) != 1 { + t.Fatalf("got %d messages, want 1", len(msgs)) + } + if msgs[0].Role != "tool" { + t.Errorf("role = %q, want tool", msgs[0].Role) + } + if msgs[0].ToolCallID != "toolu_123" { + t.Errorf("ToolCallID = %q, want toolu_123", msgs[0].ToolCallID) + } + if msgs[0].Content != "result-a\nresult-b" { + t.Errorf("content = %q, want %q", msgs[0].Content, "result-a\nresult-b") + } +} + +// TestConvertResponsesToOpenAI_PenaltyPassthrough Responses 入站转 chat 出站时 +// 透传 presence/frequency penalty 与 prompt_cache_key（vLLM 等 OpenAI 兼容上游接受）。 +func TestConvertResponsesToOpenAI_PenaltyPassthrough(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + body := []byte(`{"model":"gpt-4o","input":"hi","frequency_penalty":0.5,"presence_penalty":0.2}`) + out, err := ConvertResponsesToOpenAI(body, info) + if err != nil { + t.Fatalf("ConvertResponsesToOpenAI: %v", err) + } + raw, _ := io.ReadAll(out) + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("bad json: %v\n%s", err, raw) + } + if m["frequency_penalty"] != 0.5 { + t.Errorf("frequency_penalty = %v, want 0.5", m["frequency_penalty"]) + } + if m["presence_penalty"] != 0.2 { + t.Errorf("presence_penalty = %v, want 0.2", m["presence_penalty"]) + } + if _, ok := m["messages"]; !ok { + t.Error("messages should be present") + } +} + +// TestConvertOpenAIToResponses_PenaltyDropped chat 入站转 Responses 出站时 +// 丢弃官方不支持的 presence/frequency penalty（透传会被严格上游拒绝）， +// 保留官方参数 prompt_cache_key。 +func TestConvertOpenAIToResponses_PenaltyDropped(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + body := []byte(`{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"frequency_penalty":0.5,"presence_penalty":0.2,"prompt_cache_key":"abc"}`) + out, err := ConvertOpenAIToResponses(body, info) + if err != nil { + t.Fatalf("ConvertOpenAIToResponses: %v", err) + } + var m map[string]any + if err := json.Unmarshal(out, &m); err != nil { + t.Fatalf("bad json: %v\n%s", err, out) + } + if _, ok := m["frequency_penalty"]; ok { + t.Error("frequency_penalty should be dropped (not official Responses API)") + } + if _, ok := m["presence_penalty"]; ok { + t.Error("presence_penalty should be dropped (not official Responses API)") + } + if m["prompt_cache_key"] != "abc" { + t.Errorf("prompt_cache_key = %v, want abc", m["prompt_cache_key"]) + } + if _, ok := m["input"]; !ok { + t.Error("input should be present") + } +} + +// TestConvertResponsesToOpenAI_PreviousResponseIDRejected 有状态请求（previous_response_id） +// 落在 chat-only 渠道时必须快速失败：降级转换会静默丢失全部会话上下文。 +// 错误须携带哨兵以供 relay_handler 识别并驱动调度 FSM 换渠道。 +func TestConvertResponsesToOpenAI_PreviousResponseIDRejected(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + body := []byte(`{"model":"gpt-4o","input":"hi","previous_response_id":"resp_prev"}`) + _, err := ConvertResponsesToOpenAI(body, info) + if err == nil { + t.Fatal("previous_response_id should be rejected on chat-only conversion") + } + if !errors.Is(err, constant.ErrStatefulResponsesUnsupported) { + t.Errorf("error should wrap ErrStatefulResponsesUnsupported, got: %v", err) + } +} + +// TestConvertResponsesToOpenAI_TextFormat Responses text.format（扁平结构）转 chat +// response_format（json_schema 嵌套结构）；text 类型不映射。 +func TestConvertResponsesToOpenAI_TextFormat(t *testing.T) { + cases := []struct { + name string + text string + want map[string]any + }{ + { + name: "json_object", + text: `{"format":{"type":"json_object"}}`, + want: map[string]any{"type": "json_object"}, + }, + { + name: "json_schema", + text: `{"format":{"type":"json_schema","name":"out","schema":{"type":"object"},"strict":true}}`, + want: map[string]any{ + "type": "json_schema", + "json_schema": map[string]any{ + "name": "out", + "schema": map[string]any{"type": "object"}, + "strict": true, + }, + }, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + body := []byte(`{"model":"gpt-4o","input":"hi","text":` + c.text + `}`) + out, err := ConvertResponsesToOpenAI(body, info) + if err != nil { + t.Fatalf("ConvertResponsesToOpenAI: %v", err) + } + raw, _ := io.ReadAll(out) + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("bad json: %v\n%s", err, raw) + } + got, ok := m["response_format"].(map[string]any) + if !ok { + t.Fatalf("response_format missing or wrong type: %v", m["response_format"]) + } + wantJSON, _ := json.Marshal(c.want) + gotJSON, _ := json.Marshal(got) + if string(gotJSON) != string(wantJSON) { + t.Errorf("response_format = %s, want %s", gotJSON, wantJSON) + } + }) + } + + t.Run("text format not mapped", func(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + body := []byte(`{"model":"gpt-4o","input":"hi","text":{"format":{"type":"text"}}}`) + out, err := ConvertResponsesToOpenAI(body, info) + if err != nil { + t.Fatalf("ConvertResponsesToOpenAI: %v", err) + } + raw, _ := io.ReadAll(out) + var m map[string]any + _ = json.Unmarshal(raw, &m) + if _, ok := m["response_format"]; ok { + t.Error("text format should not map to response_format") + } + }) +} + +// TestConvertResponsesToOpenAI_StashesResponsesRequest 转换时 stash 请求快照， +// 供上游 chat 响应合成回 Responses 格式时 echo 请求参数。 +func TestConvertResponsesToOpenAI_StashesResponsesRequest(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + temp := 0.7 + // 先填充再转换，断言转换覆盖为最新请求体 + info.ResponsesRequest = &dto.OpenAIResponsesRequest{Model: "stale"} + body := []byte(`{"model":"gpt-4o","input":"hi","temperature":0.7}`) + if _, err := ConvertResponsesToOpenAI(body, info); err != nil { + t.Fatalf("ConvertResponsesToOpenAI: %v", err) + } + if info.ResponsesRequest == nil { + t.Fatal("ResponsesRequest should be stashed") + } + if info.ResponsesRequest.Model != "gpt-4o" { + t.Errorf("stashed model = %q, want gpt-4o", info.ResponsesRequest.Model) + } + if info.ResponsesRequest.Temperature == nil || *info.ResponsesRequest.Temperature != temp { + t.Errorf("stashed temperature = %v, want 0.7", info.ResponsesRequest.Temperature) + } +} + +// TestConvertResponsesToOpenAI_InputAudioAndFile 多模态输入透传： +// input_audio 与 Responses 同形透传；input_file 扁平结构转 chat 的 file 嵌套结构。 +func TestConvertResponsesToOpenAI_InputAudioAndFile(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + body := []byte(`{"model":"gpt-4o","input":[{"type":"message","role":"user","content":[` + + `{"type":"input_text","text":"listen"},` + + `{"type":"input_audio","input_audio":{"data":"QUJD","format":"wav"}},` + + `{"type":"input_file","file_data":"data:text/plain;base64,aGk=","filename":"a.txt"}` + + `]}]}`) + out, err := ConvertResponsesToOpenAI(body, info) + if err != nil { + t.Fatalf("ConvertResponsesToOpenAI: %v", err) + } + raw, _ := io.ReadAll(out) + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatalf("bad json: %v\n%s", err, raw) + } + msgs, _ := m["messages"].([]any) + if len(msgs) != 1 { + t.Fatalf("messages = %v", msgs) + } + parts, _ := msgs[0].(map[string]any)["content"].([]any) + if len(parts) != 3 { + t.Fatalf("content parts = %v, want 3（audio/file 不再被丢弃）", parts) + } + audio, _ := parts[1].(map[string]any) + if audio["type"] != "input_audio" { + t.Errorf("audio part = %v", audio) + } + ia, _ := audio["input_audio"].(map[string]any) + if ia["data"] != "QUJD" || ia["format"] != "wav" { + t.Errorf("input_audio = %v", ia) + } + file, _ := parts[2].(map[string]any) + if file["type"] != "file" { + t.Errorf("file part = %v", file) + } + f, _ := file["file"].(map[string]any) + if f["file_data"] != "data:text/plain;base64,aGk=" || f["filename"] != "a.txt" { + t.Errorf("file = %v", f) + } +} + +// TestConvertOpenAIToResponses_TextFormatUnpack chat response_format 转 Responses +// text.format：json_schema 需解包为扁平结构（不能原样塞入嵌套形状）。 +func TestConvertOpenAIToResponses_TextFormatUnpack(t *testing.T) { + t.Run("json_schema", func(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + body := []byte(`{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],` + + `"response_format":{"type":"json_schema","json_schema":{"name":"out","schema":{"type":"object"},"strict":true}}}`) + out, err := ConvertOpenAIToResponses(body, info) + if err != nil { + t.Fatalf("ConvertOpenAIToResponses: %v", err) + } + var m map[string]any + if err := json.Unmarshal(out, &m); err != nil { + t.Fatalf("bad json: %v\n%s", err, out) + } + text, _ := m["text"].(map[string]any) + format, _ := text["format"].(map[string]any) + if format["type"] != "json_schema" { + t.Fatalf("format = %v", format) + } + if format["name"] != "out" { + t.Errorf("format.name = %v, want out", format["name"]) + } + if _, nested := format["json_schema"]; nested { + t.Error("format should be flat（json_schema 嵌套必须解包）") + } + if s, _ := format["schema"].(map[string]any); s == nil { + t.Errorf("format.schema = %v", format["schema"]) + } + }) + t.Run("json_object", func(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + body := []byte(`{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}],"response_format":{"type":"json_object"}}`) + out, err := ConvertOpenAIToResponses(body, info) + if err != nil { + t.Fatalf("ConvertOpenAIToResponses: %v", err) + } + var m map[string]any + _ = json.Unmarshal(out, &m) + text, _ := m["text"].(map[string]any) + format, _ := text["format"].(map[string]any) + if format["type"] != "json_object" { + t.Errorf("format = %v", format) + } + }) +} + +// TestConvertOpenAIToResponses_StoreFalse 桥接方向显式 store:false： +// chat 客户端无法经 previous_response_id 引用响应，无需上游存储。 +func TestConvertOpenAIToResponses_StoreFalse(t *testing.T) { + info := &common.RelayInfo{ChannelMeta: &common.ChannelMeta{}} + body := []byte(`{"model":"gpt-4o","messages":[{"role":"user","content":"hi"}]}`) + out, err := ConvertOpenAIToResponses(body, info) + if err != nil { + t.Fatalf("ConvertOpenAIToResponses: %v", err) + } + var m map[string]any + if err := json.Unmarshal(out, &m); err != nil { + t.Fatalf("bad json: %v\n%s", err, out) + } + if v, ok := m["store"]; !ok || v != false { + t.Errorf("store = %v(%T), want explicit false", m["store"], m["store"]) + } +} diff --git a/relay/channel/openai/responses.go b/relay/channel/openai/responses.go index 748e988..625ed09 100644 --- a/relay/channel/openai/responses.go +++ b/relay/channel/openai/responses.go @@ -72,9 +72,42 @@ func (a *Adaptor) handleResponsesInboundNonStream(ctx context.Context, resp *htt }, nil } +// responsesRequestEcho 合成 Responses 响应时需回显（echo）的请求参数 +type responsesRequestEcho struct { + temperature *float64 + topP *float64 + maxOutputTokens *int + instructions any +} + +// extractResponsesRequestEcho 从 info.ResponsesRequest 提取合成响应应 echo 的请求参数。 +// 快照缺失（直连路径/异常）时回退 OpenAI 默认值（temperature=1.0 / top_p=1.0，其余 nil）。 +func extractResponsesRequestEcho(info *common.RelayInfo) responsesRequestEcho { + echo := responsesRequestEcho{temperature: float64Ptr(1.0), topP: float64Ptr(1.0)} + if info == nil || info.ResponsesRequest == nil { + return echo + } + rr := info.ResponsesRequest + if rr.Temperature != nil { + echo.temperature = rr.Temperature + } + if rr.TopP != nil { + echo.topP = rr.TopP + } + if rr.MaxOutputTokens != nil { + m := int(*rr.MaxOutputTokens) + echo.maxOutputTokens = &m + } + if len(rr.Instructions) > 0 { + echo.instructions = json.RawMessage(rr.Instructions) + } + return echo +} + // chatCompletionToResponsesResponse 将 Chat Completions 响应转换为 Responses API 响应 func chatCompletionToResponsesResponse(chatResp *dto.ChatCompletionResponse, info *common.RelayInfo) *dto.OpenAIResponsesResponse { modelName := info.OriginModelName + echo := extractResponsesRequestEcho(info) // 构建 output output := make([]dto.ResponsesOutput, 0) @@ -129,29 +162,31 @@ func chatCompletionToResponsesResponse(chatResp *dto.ChatCompletionResponse, inf Status: json.RawMessage(`"completed"`), Error: nil, IncompleteDetails: nil, - Instructions: nil, - MaxOutputTokens: nil, + Instructions: echo.instructions, + MaxOutputTokens: echo.maxOutputTokens, Model: modelName, Output: output, ParallelToolCalls: true, PreviousResponseID: nil, Reasoning: &dto.ResponsesReasoning{Effort: nil, Summary: nil}, - Store: true, - Temperature: float64Ptr(1.0), - Text: &dto.ResponsesText{Format: dto.ResponsesTextFormat{Type: "text"}}, - ToolChoice: "auto", - Tools: make([]any, 0), - TopP: float64Ptr(1.0), - Truncation: "disabled", - User: nil, - Metadata: make(map[string]any), + // store:false 是真实语义：合成响应不落上游存储，客户端不可经 GET /v1/responses/{id} retrieve + Store: false, + Temperature: echo.temperature, + Text: &dto.ResponsesText{Format: dto.ResponsesTextFormat{Type: "text"}}, + ToolChoice: "auto", + Tools: make([]any, 0), + TopP: echo.topP, + Truncation: "disabled", + User: nil, + Metadata: make(map[string]any), Usage: &dto.ResponsesUsage{ InputTokens: chatResp.Usage.PromptTokens, OutputTokens: chatResp.Usage.CompletionTokens, TotalTokens: chatResp.Usage.TotalTokens, InputTokensDetails: &dto.InputTokenDetails{ - CachedTokens: chatResp.Usage.PromptTokensDetails.CachedTokens, - AudioTokens: chatResp.Usage.PromptTokensDetails.AudioTokens, + CachedTokens: chatResp.Usage.PromptTokensDetails.CachedTokens, + CacheWriteTokens: chatResp.Usage.PromptTokensDetails.CacheWriteTokens, + AudioTokens: chatResp.Usage.PromptTokensDetails.AudioTokens, }, OutputTokenDetails: &dto.OutputTokenDetails{ ReasoningTokens: chatResp.Usage.CompletionTokenDetails.ReasoningTokens, @@ -279,7 +314,7 @@ func (a *Adaptor) handleResponsesInboundStream(ctx context.Context, resp *http.R // response.created emitResponsesSSE(writer, "response.created", map[string]any{ "type": "response.created", - "response": buildResponsesObjectMap(respID, createdAt, "in_progress", modelName, []any{}, nil, nil), + "response": buildResponsesObjectMap(respID, createdAt, "in_progress", modelName, []any{}, nil, nil, info), }) // response.output_item.added @@ -567,7 +602,7 @@ func (a *Adaptor) handleResponsesInboundStream(ctx context.Context, resp *http.R completedAt := int(time.Now().Unix()) emitResponsesSSE(writer, "response.completed", map[string]any{ "type": "response.completed", - "response": buildResponsesObjectMap(respID, createdAt, "completed", modelName, finalOutput, buildResponsesUsageMap(&usage), &completedAt), + "response": buildResponsesObjectMap(respID, createdAt, "completed", modelName, finalOutput, buildResponsesUsageMap(&usage), &completedAt, info), }) if info.StreamStatus.GetEndReason() == "" { @@ -609,8 +644,11 @@ func emitResponsesSSE(w http.ResponseWriter, eventType string, data any) { } } -// buildResponsesObjectMap 构建 Responses API response 对象的完整字段 map -func buildResponsesObjectMap(respID string, createdAt int, status string, model string, output any, usageObj map[string]any, completedAt *int) map[string]any { +// buildResponsesObjectMap 构建 Responses API response 对象的完整字段 map。 +// 请求参数从 info.ResponsesRequest echo（快照缺失时回退默认值）； +// store 恒为 false——合成响应不落上游存储，客户端不可经生命周期端点 retrieve。 +func buildResponsesObjectMap(respID string, createdAt int, status string, model string, output any, usageObj map[string]any, completedAt *int, info *common.RelayInfo) map[string]any { + echo := extractResponsesRequestEcho(info) m := map[string]any{ "id": respID, "object": "response", @@ -618,19 +656,19 @@ func buildResponsesObjectMap(respID string, createdAt int, status string, model "status": status, "error": nil, "incomplete_details": nil, - "instructions": nil, - "max_output_tokens": nil, + "instructions": echo.instructions, + "max_output_tokens": echo.maxOutputTokens, "model": model, "output": output, "parallel_tool_calls": true, "previous_response_id": nil, "reasoning": map[string]any{"effort": nil, "summary": nil}, - "store": true, - "temperature": 1.0, + "store": false, + "temperature": *echo.temperature, "text": map[string]any{"format": map[string]any{"type": "text"}}, "tool_choice": "auto", "tools": []any{}, - "top_p": 1.0, + "top_p": *echo.topP, "truncation": "disabled", "user": nil, "metadata": map[string]any{}, @@ -650,8 +688,9 @@ func buildResponsesUsageMap(usage *common.Usage) map[string]any { outputDetails := map[string]any{"reasoning_tokens": 0} if usage.PromptTokensDetails != nil { inputDetails = map[string]any{ - "cached_tokens": usage.PromptTokensDetails.CachedTokens, - "audio_tokens": usage.PromptTokensDetails.AudioTokens, + "cached_tokens": usage.PromptTokensDetails.CachedTokens, + "cache_write_tokens": usage.PromptTokensDetails.CacheWriteTokens, + "audio_tokens": usage.PromptTokensDetails.AudioTokens, } } if usage.CompletionTokenDetails != nil { @@ -690,10 +729,11 @@ func responsesUsageToCommon(u *dto.ResponsesUsage) *common.Usage { usage.TotalTokens = u.TotalTokens if d := u.InputTokensDetails; d != nil { usage.PromptTokensDetails = &common.TokenDetails{ - CachedTokens: d.CachedTokens, - TextTokens: d.TextTokens, - AudioTokens: d.AudioTokens, - ImageTokens: d.ImageTokens, + CachedTokens: d.CachedTokens, + CacheWriteTokens: d.CacheWriteTokens, + TextTokens: d.TextTokens, + AudioTokens: d.AudioTokens, + ImageTokens: d.ImageTokens, } } if d := u.OutputTokenDetails; d != nil { @@ -708,6 +748,19 @@ func responsesUsageToCommon(u *dto.ResponsesUsage) *common.Usage { return usage } +// recordResponseRoute 记录 response_id → 渠道路由（Redis），供 GET/DELETE/cancel +// 生命周期端点还原原始请求落到的渠道。ModelName 存 lookupModel 口径（BaseModelName）， +// 与 MaterializeSelection 入参一致。 +func recordResponseRoute(ctx context.Context, info *common.RelayInfo, responseID string) { + if responseID == "" || info == nil || info.ChannelMeta == nil { + return + } + common.DefaultResponseRouteStore.Record(ctx, info.TenantID, responseID, common.ResponseRoute{ + ChannelID: info.ChannelMeta.ChannelID, + ModelName: info.BaseModelName, + }) +} + // handleResponsesUpstreamNonStream 上游为 Responses 协议时的非流式响应： // 解析 usage 后原样透传上游响应体（模型映射时回写客户端请求的模型名）。 func (a *Adaptor) handleResponsesUpstreamNonStream(ctx context.Context, resp *http.Response, info *common.RelayInfo, writer http.ResponseWriter) (*common.Usage, error) { @@ -740,6 +793,7 @@ func (a *Adaptor) handleResponsesUpstreamNonStream(ctx context.Context, resp *ht // 解析 usage（计费用），响应体仍透传上游原始内容 var responsesResp dto.OpenAIResponsesResponse if err := json.Unmarshal(body, &responsesResp); err == nil { + recordResponseRoute(ctx, info, responsesResp.ID) return responsesUsageToCommon(responsesResp.Usage), nil } return &common.Usage{}, nil @@ -817,10 +871,14 @@ func (a *Adaptor) handleResponsesUpstreamStream(ctx context.Context, resp *http. var streamResp dto.ResponsesStreamResponse if err := json.Unmarshal([]byte(data), &streamResp); err == nil { switch streamResp.Type { - case "response.completed", "response.done": - if r := streamResp.Response; r != nil && r.Usage != nil { - if u := responsesUsageToCommon(r.Usage); u != nil { - usage = *u + case "response.created", "response.completed", "response.done": + if r := streamResp.Response; r != nil { + // 路由记录：created 即记录使 cancel 尽早可用，completed/done 刷新 TTL（SET 幂等） + recordResponseRoute(ctx, info, r.ID) + if streamResp.Type != "response.created" && r.Usage != nil { + if u := responsesUsageToCommon(r.Usage); u != nil { + usage = *u + } } } case "response.output_text.delta": diff --git a/relay/common/provider.go b/relay/common/provider.go index ec7f65b..ebe243c 100644 --- a/relay/common/provider.go +++ b/relay/common/provider.go @@ -158,7 +158,7 @@ type UsageRecord struct { ErrorMessage string // Cache token 细分 - CacheCreationTokens int // 缓存创建 token（合并） + CacheCreationTokens int // 写入缓存的 token（Claude cache_creation / OpenAI Responses cache_write） CacheCreation5mTokens int // Claude 5分钟缓存创建 CacheCreation1hTokens int // Claude 1小时缓存创建 CacheReadTokens int // 缓存读取 token diff --git a/relay/common/relay_info.go b/relay/common/relay_info.go index acfb6c3..78c5c61 100644 --- a/relay/common/relay_info.go +++ b/relay/common/relay_info.go @@ -6,6 +6,7 @@ import ( "time" "github.com/qianfree/team-api/relay/constant" + "github.com/qianfree/team-api/relay/dto" "github.com/qianfree/team-api/relaykit/relayconvert/convmeta" "github.com/qianfree/team-api/relaykit/types" ) @@ -129,6 +130,11 @@ type RelayInfo struct { // UseResponsesAPI 桥接标志：客户端发送 Chat Completions，但请求应通过 Responses API 发送到上游 UseResponsesAPI bool + // ResponsesRequest responses 入站请求快照（ConvertResponsesToOpenAI 解析后 stash）， + // 供 chat 上游响应合成回 Responses 格式时 echo temperature/top_p/max_output_tokens/instructions。 + // 直连（原生 Responses 上游）路径不填充，合成时保持默认值。 + ResponsesRequest *dto.OpenAIResponsesRequest + // Thinking 后缀路由（从模型名解析，供适配器消费） ThinkingEnabled bool // 是否有 -thinking 后缀 ThinkingDisabled bool // 是否有 -nothinking 后缀 diff --git a/relay/common/response_route.go b/relay/common/response_route.go new file mode 100644 index 0000000..2460d53 --- /dev/null +++ b/relay/common/response_route.go @@ -0,0 +1,107 @@ +package common + +import ( + "context" + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/gogf/gf/v2/database/gredis" + "github.com/gogf/gf/v2/frame/g" +) + +// ResponseRoute response_id → 渠道路由记录，供生命周期端点 +// （GET /v1/responses/{id}、POST .../cancel、DELETE）还原原始请求落到的渠道。 +// ModelName 存 lookupModel 口径（去 thinking 后缀），与 MaterializeSelection 入参一致。 +type ResponseRoute struct { + ChannelID int64 `json:"channel_id"` + ModelName string `json:"model_name"` +} + +// ResponseRouteStore 路由存储接口（测试可注入 fake） +type ResponseRouteStore interface { + // Record 记录路由（幂等 SET，重复调用刷新 TTL）。热路径调用，失败只记日志不中断。 + Record(ctx context.Context, tenantID int64, responseID string, route ResponseRoute) + // Lookup 查询路由；miss 或存储不可用返回 false（调用方按 404 处理） + Lookup(ctx context.Context, tenantID int64, responseID string) (ResponseRoute, bool) + // Delete 删除路由（DELETE 生命周期端点成功后调用） + Delete(ctx context.Context, tenantID int64, responseID string) +} + +// DefaultResponseRouteStore 包级单例，adaptor 响应处理与 lifecycle handler 共用 +var DefaultResponseRouteStore ResponseRouteStore = &redisResponseRouteStore{} + +// responseRouteTTL 路由保留期，与 OpenAI Responses 存储保留期（30 天）对齐； +// 超期后上游本身也不再可 retrieve，路由条目自然过期即可。 +const responseRouteTTL = 30 * 24 * time.Hour + +// responseRouteKey 路由按租户隔离，防止跨租户探测 response_id +func responseRouteKey(tenantID int64, responseID string) string { + return fmt.Sprintf("relay:resp:route:%d:%s", tenantID, responseID) +} + +// redisResponseRouteStore Redis 实现。Redis 未配置/不可用时全程降级 no-op： +// Record 静默丢弃、Lookup 恒 miss、Delete 静默——生命周期端点退化为"不可用"而非报错。 +type redisResponseRouteStore struct{} + +var ( + redisProbeOnce sync.Once + redisClient *gredis.Redis +) + +// redisClientOrNil 返回 Redis 客户端。注意 gf v2 的 g.Redis() 在未配置 redis 时 +// 会 panic（gins 懒加载抛错）而非返回 nil，故以 recover 探测并缓存结果（仅探测一次， +// 配置在进程内不动态增删 redis 节点，探测失败后全程按不可用处理）。 +func redisClientOrNil() *gredis.Redis { + redisProbeOnce.Do(func() { + defer func() { + // g.Redis() panic → 保持 redisClient = nil + _ = recover() + }() + redisClient = g.Redis() + }) + return redisClient +} + +func (s *redisResponseRouteStore) Record(ctx context.Context, tenantID int64, responseID string, route ResponseRoute) { + r := redisClientOrNil() + if r == nil || responseID == "" { + return + } + val, err := json.Marshal(route) + if err != nil { + g.Log().Debugf(ctx, "[ResponseRoute] marshal route failed: %v", err) + return + } + if _, err := r.Do(ctx, "SET", responseRouteKey(tenantID, responseID), string(val), "EX", int(responseRouteTTL.Seconds())); err != nil { + // 路由记录失败不影响响应转发，仅损失后续 retrieve/cancel 能力 + g.Log().Debugf(ctx, "[ResponseRoute] record failed: responseID=%s err=%v", responseID, err) + } +} + +func (s *redisResponseRouteStore) Lookup(ctx context.Context, tenantID int64, responseID string) (ResponseRoute, bool) { + r := redisClientOrNil() + if r == nil || responseID == "" { + return ResponseRoute{}, false + } + res, err := r.Do(ctx, "GET", responseRouteKey(tenantID, responseID)) + if err != nil || res.IsNil() { + return ResponseRoute{}, false + } + var route ResponseRoute + if err := json.Unmarshal([]byte(res.String()), &route); err != nil { + return ResponseRoute{}, false + } + return route, true +} + +func (s *redisResponseRouteStore) Delete(ctx context.Context, tenantID int64, responseID string) { + r := redisClientOrNil() + if r == nil || responseID == "" { + return + } + if _, err := r.Do(ctx, "DEL", responseRouteKey(tenantID, responseID)); err != nil { + g.Log().Debugf(ctx, "[ResponseRoute] delete failed: responseID=%s err=%v", responseID, err) + } +} diff --git a/relay/common/response_route_test.go b/relay/common/response_route_test.go new file mode 100644 index 0000000..03fe694 --- /dev/null +++ b/relay/common/response_route_test.go @@ -0,0 +1,31 @@ +package common + +import ( + "context" + "testing" +) + +// TestResponseRouteKey_Format 路由 key 按租户隔离、包含 response_id。 +func TestResponseRouteKey_Format(t *testing.T) { + got := responseRouteKey(42, "resp_abc") + want := "relay:resp:route:42:resp_abc" + if got != want { + t.Errorf("responseRouteKey = %q, want %q", got, want) + } +} + +// TestResponseRouteStore_NilRedisNoop Redis 未配置（单测环境）时全程降级 no-op： +// Record/Delete 不 panic，Lookup 恒 miss。 +func TestResponseRouteStore_NilRedisNoop(t *testing.T) { + store := DefaultResponseRouteStore + ctx := context.Background() + + // 不应 panic + store.Record(ctx, 1, "resp_x", ResponseRoute{ChannelID: 1, ModelName: "gpt-4o"}) + store.Delete(ctx, 1, "resp_x") + + // Lookup 在无 Redis（或 Redis 不可达）时返回 miss，不报错 + if _, ok := store.Lookup(ctx, 1, "resp_missing"); ok { + t.Error("Lookup should miss when redis is unavailable") + } +} diff --git a/relay/common/usage.go b/relay/common/usage.go index 15b20f9..ee320cc 100644 --- a/relay/common/usage.go +++ b/relay/common/usage.go @@ -4,10 +4,14 @@ import "github.com/qianfree/team-api/relay/dto" // TokenDetails Token 使用量细分 type TokenDetails struct { - CachedTokens int `json:"cached_tokens,omitempty"` - CachedCreationTokens int `json:"cached_creation_tokens,omitempty"` // Claude cache_creation_input_tokens - CachedCreation5mTokens int `json:"cached_creation_5m_tokens,omitempty"` // Claude 5分钟缓存创建 - CachedCreation1hTokens int `json:"cached_creation_1h_tokens,omitempty"` // Claude 1小时缓存创建 + CachedTokens int `json:"cached_tokens,omitempty"` + CachedCreationTokens int `json:"cached_creation_tokens,omitempty"` // Claude cache_creation_input_tokens + CachedCreation5mTokens int `json:"cached_creation_5m_tokens,omitempty"` // Claude 5分钟缓存创建 + CachedCreation1hTokens int `json:"cached_creation_1h_tokens,omitempty"` // Claude 1小时缓存创建 + // CacheWriteTokens 本次写入缓存的 token（OpenAI Responses cache_write_tokens，观测字段）。 + // 注意：OpenAI 缓存写入按普通输入价计费（含于 PromptTokens），不得并入 CachedCreationTokens + // 计费桶（Claude 写入才有独立 CacheCreationPrice），计费引擎不读取本字段 + CacheWriteTokens int `json:"cache_write_tokens,omitempty"` AudioTokens int `json:"audio_tokens,omitempty"` TextTokens int `json:"text_tokens,omitempty"` ImageTokens int `json:"image_tokens,omitempty"` @@ -30,6 +34,20 @@ type Usage struct { CacheIncludedInPrompt bool `json:"-"` } +// TotalInputTokens 返回「含缓存的总输入 token」：口径未包含缓存（Claude）时补加缓存读/写， +// 口径已包含（OpenAI/Gemini）时原样返回。bil_usage_logs.input_tokens 统一按此口径入库， +// 保证跨渠道 SUM 聚合语义一致（cache_read/cache_creation 列为其子集明细）。 +func (u *Usage) TotalInputTokens() int { + if u == nil { + return 0 + } + total := u.PromptTokens + if !u.CacheIncludedInPrompt && u.PromptTokensDetails != nil { + total += u.PromptTokensDetails.CachedTokens + u.PromptTokensDetails.CachedCreationTokens + } + return total +} + // DtoTokenDetailsToCommon 将 dto.TokenDetails 转换为 common.TokenDetails func DtoTokenDetailsToCommon(d *dto.TokenDetails) *TokenDetails { if d == nil { @@ -40,6 +58,7 @@ func DtoTokenDetailsToCommon(d *dto.TokenDetails) *TokenDetails { CachedCreationTokens: d.CachedCreationTokens, CachedCreation5mTokens: d.CachedCreation5mTokens, CachedCreation1hTokens: d.CachedCreation1hTokens, + CacheWriteTokens: d.CacheWriteTokens, AudioTokens: d.AudioTokens, TextTokens: d.TextTokens, ImageTokens: d.ImageTokens, @@ -59,6 +78,7 @@ func CommonTokenDetailsToDto(c *TokenDetails) *dto.TokenDetails { CachedCreationTokens: c.CachedCreationTokens, CachedCreation5mTokens: c.CachedCreation5mTokens, CachedCreation1hTokens: c.CachedCreation1hTokens, + CacheWriteTokens: c.CacheWriteTokens, AudioTokens: c.AudioTokens, TextTokens: c.TextTokens, ImageTokens: c.ImageTokens, diff --git a/relay/common/usage_test.go b/relay/common/usage_test.go new file mode 100644 index 0000000..5560f14 --- /dev/null +++ b/relay/common/usage_test.go @@ -0,0 +1,44 @@ +package common + +import "testing" + +// TestTotalInputTokens 验证「含缓存总输入」归一化口径： +// OpenAI/Gemini 口径（CacheIncludedInPrompt=true，prompt 已含缓存）原样返回； +// Claude 口径（false，input_tokens 与缓存三项并列）补加 cache_read + cache_creation。 +func TestTotalInputTokens(t *testing.T) { + // OpenAI 口径：prompt=2006 已含 cached=1920，总输入即 2006，不得再叠加 + openai := &Usage{ + PromptTokens: 2006, + CompletionTokens: 46, + CacheIncludedInPrompt: true, + PromptTokensDetails: &TokenDetails{CachedTokens: 1920}, + } + if got := openai.TotalInputTokens(); got != 2006 { + t.Errorf("TotalInputTokens() = %d, want 2006 (included 口径不叠加)", got) + } + + // Claude 口径：input=204 + cache_read=1800 + cache_creation=248 = 2252 + claude := &Usage{ + PromptTokens: 204, + CompletionTokens: 503, + PromptTokensDetails: &TokenDetails{ + CachedTokens: 1800, + CachedCreationTokens: 248, + }, + } + if got := claude.TotalInputTokens(); got != 2252 { + t.Errorf("TotalInputTokens() = %d, want 2252 (excluded 口径补加缓存)", got) + } + + // Claude 口径但无缓存明细：原样返回 + plain := &Usage{PromptTokens: 100} + if got := plain.TotalInputTokens(); got != 100 { + t.Errorf("TotalInputTokens() = %d, want 100", got) + } + + // nil 安全 + var nilUsage *Usage + if got := nilUsage.TotalInputTokens(); got != 0 { + t.Errorf("TotalInputTokens() on nil = %d, want 0", got) + } +} diff --git a/relay/constant/errors.go b/relay/constant/errors.go index 58b1095..652f5df 100644 --- a/relay/constant/errors.go +++ b/relay/constant/errors.go @@ -17,6 +17,11 @@ import ( // Relay 层哨兵错误 var ErrAllChannelsFailed = gerror.New("all channels failed") +// ErrStatefulResponsesUnsupported responses 有状态请求（previous_response_id）落在 +// 不支持 Responses 协议的 chat-only 渠道上，降级转换会静默丢失会话上下文，必须快速失败。 +// 由 ConvertResponsesToOpenAI 返回，relay_handler 捕获后按渠道级致命上报调度 FSM 换渠道。 +var ErrStatefulResponsesUnsupported = gerror.New("previous_response_id requires a responses-native channel") + // RelayError 包装 relay 层错误，携带上游返回的状态码和信息 type RelayError struct { StatusCode int diff --git a/relay/dto/openai_responses.go b/relay/dto/openai_responses.go index c1951c2..8beedcc 100644 --- a/relay/dto/openai_responses.go +++ b/relay/dto/openai_responses.go @@ -37,6 +37,8 @@ type OpenAIResponsesRequest struct { PromptCacheKey string `json:"prompt_cache_key,omitempty"` PromptCacheRetention string `json:"prompt_cache_retention,omitempty"` SafetyIdentifier string `json:"safety_identifier,omitempty"` + FrequencyPenalty json.RawMessage `json:"frequency_penalty,omitempty"` + PresencePenalty json.RawMessage `json:"presence_penalty,omitempty"` } // Reasoning 推理参数（用于 o1/o3 等推理模型） @@ -116,10 +118,11 @@ type ResponsesTextFormat struct { // InputTokenDetails 输入 token 细分 type InputTokenDetails struct { - CachedTokens int `json:"cached_tokens"` - TextTokens int `json:"text_tokens,omitempty"` - AudioTokens int `json:"audio_tokens,omitempty"` - ImageTokens int `json:"image_tokens,omitempty"` + CachedTokens int `json:"cached_tokens"` + CacheWriteTokens int `json:"cache_write_tokens,omitempty"` // 本次写入缓存的 token（input_tokens 子集，按普通输入价计费） + TextTokens int `json:"text_tokens,omitempty"` + AudioTokens int `json:"audio_tokens,omitempty"` + ImageTokens int `json:"image_tokens,omitempty"` } // OutputTokenDetails 输出 token 细分 diff --git a/relay/handler/dispatch_glue.go b/relay/handler/dispatch_glue.go index 8dafef6..d92fd1a 100644 --- a/relay/handler/dispatch_glue.go +++ b/relay/handler/dispatch_glue.go @@ -66,15 +66,27 @@ func deliveryStateOfRequestErr(err error) dispatch.DeliveryState { return dispatch.DeliveryMaybeSent } -// reportMaterializeFailure 选择物化失败（Key 解密失败/目录缺失）时按渠道级致命上报， -// 让 FSM 决定换渠道或终止。不发起过上游请求，送达状态为 NotSent。 -func reportMaterializeFailure(ctx context.Context, sess *dispatch.RouteSession, mErr error) dispatch.RetryDecision { - failErr := types.NewError(mErr, types.ErrorCodeChannelNoAvailableKey) +// reportChannelFatal 渠道级致命错误上报（请求未送达上游），让 FSM 决定换渠道或终止。 +// code 必须为 channel: 前缀错误码（classify 归 ErrClassChannelFatal → failover）。 +func reportChannelFatal(ctx context.Context, sess *dispatch.RouteSession, err error, code types.ErrorCode) dispatch.RetryDecision { + failErr := types.NewError(err, code) decision, _ := sess.Report(ctx, 0, failErr, dispatch.DeliveryNotSent, 0, 0) trackRetryDecision(0, failErr, dispatch.DeliveryNotSent, decision) return decision } +// reportMaterializeFailure 选择物化失败（Key 解密失败/目录缺失）时按渠道级致命上报， +// 让 FSM 决定换渠道或终止。不发起过上游请求，送达状态为 NotSent。 +func reportMaterializeFailure(ctx context.Context, sess *dispatch.RouteSession, mErr error) dispatch.RetryDecision { + return reportChannelFatal(ctx, sess, mErr, types.ErrorCodeChannelNoAvailableKey) +} + +// reportProtocolMismatch 渠道协议能力不匹配（如 responses 有状态请求落在 chat-only +// 渠道）时上报：换渠道可能得到支持 Responses 协议的渠道，不等于渠道本身故障。 +func reportProtocolMismatch(ctx context.Context, sess *dispatch.RouteSession, err error) dispatch.RetryDecision { + return reportChannelFatal(ctx, sess, err, types.ErrorCodeChannelProtocolMismatch) +} + // sleepBackoff 执行退避等待，客户端断开时立即返回。 func sleepBackoff(ctx context.Context, d time.Duration) { if d <= 0 { diff --git a/relay/handler/relay_handler.go b/relay/handler/relay_handler.go index fe284e9..2a8ce19 100644 --- a/relay/handler/relay_handler.go +++ b/relay/handler/relay_handler.go @@ -310,7 +310,13 @@ func settleSuccessfulRequest( firstTokenMs = int(info.FirstResponseTime.Sub(info.StartTime).Milliseconds()) } - // 构建用量记录 + // 构建用量记录。input_tokens 统一按「含缓存总输入」口径入库（Claude 口径未含缓存时补加）， + // 保证跨渠道 SUM 聚合语义一致；total_tokens 同步按 totalInput+output 重算 + totalInput := usage.TotalInputTokens() + totalTokens := totalInput + usage.CompletionTokens + if totalTokens == 0 { + totalTokens = usage.TotalTokens + } usageRecord := &common.UsageRecord{ TenantID: rc.TenantID, UserID: rc.UserID, @@ -319,9 +325,9 @@ func settleSuccessfulRequest( ChannelID: selection.ChannelID, ModelName: v.modelName, RelayMode: int(v.relayMode), - PromptTokens: usage.PromptTokens, + PromptTokens: totalInput, CompletionTokens: usage.CompletionTokens, - TotalTokens: usage.TotalTokens, + TotalTokens: totalTokens, CachedTokens: tokenDetailField(usage.PromptTokensDetails, func(d *common.TokenDetails) int { return d.CachedTokens }), AudioTokens: tokenDetailField(usage.PromptTokensDetails, func(d *common.TokenDetails) int { return d.AudioTokens }) + tokenDetailField(usage.CompletionTokenDetails, func(d *common.TokenDetails) int { return d.AudioTokens }), @@ -333,8 +339,11 @@ func settleSuccessfulRequest( RequestID: rc.RequestID, Status: "success", - // Cache token 明细 - CacheCreationTokens: usage.CacheCreationTokens, + // Cache token 明细。cache_creation_tokens 列统一记录「写入缓存的 token」： + // Claude 为 cache_creation_input_tokens，OpenAI Responses 为 cache_write_tokens（两者物理语义相同， + // 仅计价不同——后者按普通输入价计费且已含于 input_tokens，不参与计费扣减，仅作观测） + CacheCreationTokens: usage.CacheCreationTokens + + tokenDetailField(usage.PromptTokensDetails, func(d *common.TokenDetails) int { return d.CacheWriteTokens }), CacheCreation5mTokens: tokenDetailField(usage.PromptTokensDetails, func(d *common.TokenDetails) int { return d.CachedCreation5mTokens }), CacheCreation1hTokens: tokenDetailField(usage.PromptTokensDetails, func(d *common.TokenDetails) int { return d.CachedCreation1hTokens }), CacheReadTokens: tokenDetailField(usage.PromptTokensDetails, func(d *common.TokenDetails) int { return d.CachedTokens }), @@ -586,6 +595,23 @@ func RelayHandler(ctx context.Context, body []byte, path string, headers http.He // 转换请求（直连模式跳过协议转换和参数改写） convertedBody, err := convertRequestBody(ctx, info, body, adaptor) if err != nil { + // responses 有状态请求（previous_response_id）落在 chat-only 渠道：协议能力不匹配 + // 而非请求错误，换渠道可能命中 Responses 原生渠道——按渠道级致命上报驱动 failover， + // 而非直接返回（直接返回会让有能力渠道存在时也失败） + if errors.Is(err, constant.ErrStatefulResponsesUnsupported) { + channelErrors = append(channelErrors, fmt.Sprintf("attempt=%d channel=%d(%s) model=%s convert_error=[%v]", + attempt, selection.ChannelID, selection.ChannelName, v.modelName, err)) + g.Log().Warningf(ctx, "[RelayHandler] Stateful responses not supported by chat-only channel, failing over: channel=%d(%s) model=%s attempt=%d", + selection.ChannelID, selection.ChannelName, v.modelName, attempt) + if reportProtocolMismatch(ctx, sess, err) == dispatch.DecisionAbort { + if billing != nil && preDeductAmount > 0 { + _ = billing.SettleFailed(ctx, rc.TenantID, rc.RequestID, preDeductAmount) + } + return nil, v.billingResult, constant.NewRequestError( + "previous_response_id is only supported on responses-native channels; no eligible channel remains for this model. Resend the full conversation input without previous_response_id.", nil) + } + continue + } if billing != nil && preDeductAmount > 0 { _ = billing.SettleFailed(ctx, rc.TenantID, rc.RequestID, preDeductAmount) } diff --git a/relay/handler/responses_lifecycle.go b/relay/handler/responses_lifecycle.go new file mode 100644 index 0000000..835237d --- /dev/null +++ b/relay/handler/responses_lifecycle.go @@ -0,0 +1,141 @@ +package handler + +import ( + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/gogf/gf/v2/frame/g" + + "github.com/qianfree/team-api/relay/channel" + "github.com/qianfree/team-api/relay/common" + "github.com/qianfree/team-api/relay/constant" + "github.com/qianfree/team-api/relay/override" +) + +// Responses 生命周期动作（POST {id}/cancel 以 method 区分，其余按 HTTP method 透传） +const ( + responsesLifecycleCancel = "/cancel" +) + +// HandleResponsesLifecycle 处理 Responses 生命周期端点： +// GET /v1/responses/{id}（retrieve）、POST /v1/responses/{id}/cancel、DELETE /v1/responses/{id}。 +// 经 Redis 路由表（response_id → 渠道，Record 于直连转发时写入）还原原始渠道后透传上游。 +// 无请求体、无计费（对齐异步任务查询端点先例）；错误以 *constant.RelayError 返回， +// 由调用方经 WriteRelayError 写为 OpenAI 格式。 +func HandleResponsesLifecycle(ctx context.Context, method string, responseID string, rc *RelayContext, provider common.DataProvider) error { + if responseID == "" { + return constant.NewRequestError("response id is required", nil) + } + + // 1. 查路由：miss（未记录/已过期/Redis 不可用）或渠道已不可物化均按 404 处理 + route, ok := common.DefaultResponseRouteStore.Lookup(ctx, rc.TenantID, responseID) + if !ok { + return &constant.RelayError{StatusCode: http.StatusNotFound, Message: "Response not found", Type: "invalid_request_error"} + } + + selection, err := provider.MaterializeSelection(ctx, route.ChannelID, 0, route.ModelName) + if err != nil { + g.Log().Debugf(ctx, "[ResponsesLifecycle] materialize channel failed: responseID=%s channel=%d err=%v", responseID, route.ChannelID, err) + return &constant.RelayError{StatusCode: http.StatusNotFound, Message: "Response not found", Type: "invalid_request_error"} + } + + adaptor := channel.GetAdaptor(selection.ChannelType) + if adaptor == nil { + return constant.NewUpstreamError(http.StatusInternalServerError, fmt.Sprintf("unsupported channel type: %d", selection.ChannelType), nil) + } + + info := buildLifecycleRelayInfo(rc, selection) + info.RequestURLPath = "/responses/" + responseID + // 守卫：路由记录只产生于 UpstreamSpeaksResponses 渠道，此处防御 chat-only 渠道 + //（GetRequestURL 会返回 /v1/chat/completions 导致拼错 URL） + if !info.ChannelMeta.UpstreamSpeaksResponses() { + return &constant.RelayError{StatusCode: http.StatusNotFound, Message: "Response not found", Type: "invalid_request_error"} + } + adaptor.Init(info) + + upstreamURL, err := buildLifecycleUpstreamURL(adaptor, info, responseID, method == http.MethodPost) + if err != nil { + return constant.NewUpstreamError(http.StatusInternalServerError, err.Error(), nil) + } + + httpReq, err := http.NewRequestWithContext(ctx, method, upstreamURL, nil) + if err != nil { + return constant.NewUpstreamError(http.StatusInternalServerError, "create lifecycle request failed", err) + } + if err := adaptor.SetupRequestHeader(httpReq.Header, info); err != nil { + return constant.NewUpstreamError(http.StatusInternalServerError, "setup request header failed", err) + } + // 与标准转发链路一致：应用渠道级 Header Override + if hdrOverrides, hdrErr := override.ApplyHeaderOverride(info); hdrErr == nil && len(hdrOverrides) > 0 { + override.MergeHeaderOverrides(httpReq.Header, hdrOverrides) + } + + client := common.NewPooledClient(info.ChannelMeta.Settings.GetTimeoutSeconds(info.RelayMode), info.ChannelMeta.Settings.UseProxy) + resp, err := client.Do(httpReq) + if err != nil { + return constant.NewUpstreamError(http.StatusBadGateway, "upstream lifecycle request failed", err) + } + defer resp.Body.Close() + + // 原样透传上游响应（状态码 + Content-Type + body） + if ct := resp.Header.Get("Content-Type"); ct != "" { + rc.Writer.Header().Set("Content-Type", ct) + } + rc.Writer.WriteHeader(resp.StatusCode) + _, _ = io.Copy(rc.Writer, resp.Body) + + // DELETE 成功后清理路由记录（上游已不可 retrieve） + if method == http.MethodDelete && resp.StatusCode >= 200 && resp.StatusCode < 300 { + common.DefaultResponseRouteStore.Delete(ctx, rc.TenantID, responseID) + } + return nil +} + +// buildLifecycleRelayInfo 构建生命周期转发的最小 RelayInfo（不经调度 FSM， +// 字段拷贝自路由记录物化的 ChannelSelection，照 buildRelayInfo 的 ChannelMeta 部分； +// RequestURLPath 由调用方按目标 responseID 覆写） +func buildLifecycleRelayInfo(rc *RelayContext, selection *common.ChannelSelection) *common.RelayInfo { + return &common.RelayInfo{ + Context: context.Background(), + TenantID: rc.TenantID, + UserID: rc.UserID, + ApiKeyID: rc.ApiKeyID, + ProjectID: rc.ProjectID, + RequestID: rc.RequestID, + RelayMode: int(constant.RelayModeResponses), + OriginModelName: selection.UpstreamModelName, + StartTime: time.Now(), + ChannelMeta: &common.ChannelMeta{ + ChannelID: selection.ChannelID, + ChannelType: selection.ChannelType, + ChannelName: selection.ChannelName, + BaseURL: selection.BaseURL, + ApiKey: selection.ApiKey, + UpstreamModelName: selection.UpstreamModelName, + IsModelMapped: selection.IsModelMapped, + Settings: selection.Settings, + + SupportsResponses: selection.SupportsResponses, + ChatViaResponses: selection.ChatViaResponses, + }, + } +} + +// buildLifecycleUpstreamURL 构建生命周期端点的上游 URL： +// 复用 adaptor.GetRequestURL（RelayModeResponses + UpstreamSpeaksResponses 时返回 +// POST 基址，如 openai 的 /v1/responses、codex 的 /backend-api/codex/responses）， +// 追加 /{responseID}（cancel 再追加 /cancel），对各 Responses 上游天然形状正确。 +func buildLifecycleUpstreamURL(adaptor common.Adaptor, info *common.RelayInfo, responseID string, isCancel bool) (string, error) { + base, err := adaptor.GetRequestURL(info) + if err != nil { + return "", err + } + url := base + "/" + responseID + if isCancel { + url += responsesLifecycleCancel + } + return url, nil +} diff --git a/relay/handler/responses_lifecycle_test.go b/relay/handler/responses_lifecycle_test.go new file mode 100644 index 0000000..5a71754 --- /dev/null +++ b/relay/handler/responses_lifecycle_test.go @@ -0,0 +1,38 @@ +package handler + +import ( + "testing" + + "github.com/qianfree/team-api/relay/channel/openai" + "github.com/qianfree/team-api/relay/common" + "github.com/qianfree/team-api/relay/constant" +) + +// TestBuildLifecycleUpstreamURL 生命周期端点上游 URL：复用 adaptor 的 Responses +// POST 基址追加 /{id}（cancel 再追加 /cancel），对 openai 形状的渠道天然正确。 +func TestBuildLifecycleUpstreamURL(t *testing.T) { + info := &common.RelayInfo{ + RelayMode: int(constant.RelayModeResponses), + ChannelMeta: &common.ChannelMeta{ + BaseURL: "https://up.example.com", + SupportsResponses: true, + }, + } + a := &openai.Adaptor{} + + got, err := buildLifecycleUpstreamURL(a, info, "resp_1", false) + if err != nil { + t.Fatalf("buildLifecycleUpstreamURL: %v", err) + } + if want := "https://up.example.com/v1/responses/resp_1"; got != want { + t.Errorf("retrieve URL = %q, want %q", got, want) + } + + got, err = buildLifecycleUpstreamURL(a, info, "resp_1", true) + if err != nil { + t.Fatalf("buildLifecycleUpstreamURL(cancel): %v", err) + } + if want := "https://up.example.com/v1/responses/resp_1/cancel"; got != want { + t.Errorf("cancel URL = %q, want %q", got, want) + } +} diff --git a/relay/relaykit_bridge/response.go b/relay/relaykit_bridge/response.go index dde6ae6..879a554 100644 --- a/relay/relaykit_bridge/response.go +++ b/relay/relaykit_bridge/response.go @@ -102,6 +102,8 @@ func convertResponseViaRelaykit(ctx context.Context, info *common.RelayInfo, ups // UsageFromConvertedChatResponse 从 relaykit 转换后的 OpenAI ChatCompletionResponse 响应体中提取用量。 // 非流式响应桥接成功后用于构建计费用量（转换器已把用量写入响应体的 Usage 字段）。 +// 转换后的 usage 统一为 OpenAI 口径（prompt_tokens 含缓存，cached 为其子集），故置 +// CacheIncludedInPrompt 让计费按明细扣减缓存部分，避免双重计费。 // 解析失败返回 (nil, false)，调用方可回退到从原始上游体提取或返回空用量。 func UsageFromConvertedChatResponse(body []byte) (*common.Usage, bool) { var resp dto.ChatCompletionResponse @@ -109,12 +111,23 @@ func UsageFromConvertedChatResponse(body []byte) (*common.Usage, bool) { return nil, false } return &common.Usage{ - PromptTokens: resp.Usage.PromptTokens, - CompletionTokens: resp.Usage.CompletionTokens, - TotalTokens: resp.Usage.TotalTokens, + PromptTokens: resp.Usage.PromptTokens, + CompletionTokens: resp.Usage.CompletionTokens, + TotalTokens: resp.Usage.TotalTokens, + CacheCreationTokens: convertedCacheCreationTokens(resp.Usage.PromptTokensDetails), + CacheIncludedInPrompt: true, + PromptTokensDetails: common.DtoTokenDetailsToCommon(resp.Usage.PromptTokensDetails), + CompletionTokenDetails: common.DtoTokenDetailsToCommon(resp.Usage.CompletionTokenDetails), }, true } +func convertedCacheCreationTokens(details *dto.TokenDetails) int { + if details == nil { + return 0 + } + return details.CachedCreationTokens +} + // relaykitResponseConverterID 根据 (上游原生格式, 客户端格式) 返回响应转换器 ID。 // 返回空串表示没有匹配的 relaykit 响应转换器（调用方回退旧路径）。 func relaykitResponseConverterID(upstream, clientFormat constant.RelayFormat) string { diff --git a/relay/relaykit_bridge/response_test.go b/relay/relaykit_bridge/response_test.go index 3e7a528..b63f688 100644 --- a/relay/relaykit_bridge/response_test.go +++ b/relay/relaykit_bridge/response_test.go @@ -23,8 +23,8 @@ func TestRelaykitResponseConverterID(t *testing.T) { {constant.RelayFormatCoze, constant.RelayFormatOpenAI, relayconvert.ConverterOpenAIChatToCoze}, {constant.RelayFormatDify, constant.RelayFormatOpenAI, relayconvert.ConverterOpenAIChatToDify}, {constant.RelayFormatOllama, constant.RelayFormatOpenAI, relayconvert.ConverterOpenAIChatToOllama}, - {constant.RelayFormatOpenAI, constant.RelayFormatOpenAI, ""}, // 同格式 - {constant.RelayFormatClaude, constant.RelayFormatGemini, ""}, // 未知方向 + {constant.RelayFormatOpenAI, constant.RelayFormatOpenAI, ""}, // 同格式 + {constant.RelayFormatClaude, constant.RelayFormatGemini, ""}, // 未知方向 } for _, c := range cases { if got := relaykitResponseConverterID(c.upstream, c.client); got != c.want { @@ -37,13 +37,20 @@ func TestRelaykitResponseConverterID(t *testing.T) { func TestUsageFromConvertedChatResponse(t *testing.T) { body, _ := json.Marshal(dto.ChatCompletionResponse{ - Usage: dto.UsageWithDetails{PromptTokens: 5, CompletionTokens: 3, TotalTokens: 8}, + Usage: dto.UsageWithDetails{ + PromptTokens: 5, CompletionTokens: 3, TotalTokens: 8, + PromptTokensDetails: &dto.TokenDetails{CachedTokens: 2, CachedCreationTokens: 1}, + CompletionTokenDetails: &dto.TokenDetails{ReasoningTokens: 4}, + }, }) usage, ok := UsageFromConvertedChatResponse(body) if !ok { t.Fatal("expected ok=true") } - if usage == nil || usage.PromptTokens != 5 || usage.CompletionTokens != 3 { + if usage == nil || usage.PromptTokens != 5 || usage.CompletionTokens != 3 || + usage.CacheCreationTokens != 1 || usage.PromptTokensDetails == nil || + usage.PromptTokensDetails.CachedTokens != 2 || usage.CompletionTokenDetails == nil || + usage.CompletionTokenDetails.ReasoningTokens != 4 { t.Errorf("unexpected usage: %+v", usage) } diff --git a/relay/relaykit_bridge/stream.go b/relay/relaykit_bridge/stream.go index 2a267ca..c5da8ff 100644 --- a/relay/relaykit_bridge/stream.go +++ b/relay/relaykit_bridge/stream.go @@ -62,7 +62,10 @@ func convertStreamViaRelaykit(ctx context.Context, info *common.RelayInfo, upstr safeWriter := helper.NewSafeWriter(writer) defer helper.PingTicker(safeWriter, 15*time.Second)() - capturedUsage := &common.Usage{} + // relaykit 转换器输出的 usage 统一为 OpenAI 口径：prompt_tokens 已含缓存 + //（Gemini 的 cached ⊆ promptTokenCount；Claude 转换时已做 input+cache_read+cache_creation 加法）。 + // 置 CacheIncludedInPrompt 让计费按明细扣减缓存部分，避免「input 全价 + cache 价」双重计费 + capturedUsage := &common.Usage{CacheIncludedInPrompt: true} var ( gotFinish bool // 转换器是否已产出带 finish_reason 的结束 chunk firstChunk bool @@ -84,6 +87,9 @@ func convertStreamViaRelaykit(ctx context.Context, info *common.RelayInfo, upstr capturedUsage.PromptTokens = streamChunk.Usage.PromptTokens capturedUsage.CompletionTokens = streamChunk.Usage.CompletionTokens capturedUsage.TotalTokens = streamChunk.Usage.TotalTokens + capturedUsage.CacheCreationTokens = convertedCacheCreationTokens(streamChunk.Usage.PromptTokensDetails) + capturedUsage.PromptTokensDetails = common.DtoTokenDetailsToCommon(streamChunk.Usage.PromptTokensDetails) + capturedUsage.CompletionTokenDetails = common.DtoTokenDetailsToCommon(streamChunk.Usage.CompletionTokenDetails) } if len(streamChunk.Choices) > 0 && streamChunk.Choices[0].FinishReason != nil { gotFinish = true diff --git a/relay/relaykit_bridge/stream_test.go b/relay/relaykit_bridge/stream_test.go index a4d17ef..4367b39 100644 --- a/relay/relaykit_bridge/stream_test.go +++ b/relay/relaykit_bridge/stream_test.go @@ -30,7 +30,7 @@ func newStreamTestRelayInfo(channelType constant.ProviderType, clientFormat cons // 并正确提取 usage、写入 [DONE] 收尾、设置正常结束原因。 func TestConvertStreamViaRelaykit_ClaudeToOpenAI(t *testing.T) { // 复用 relaykit 内 claude_to_openai_stream_test.go 的 BasicStream 报文 - claudeStream := `data: {"type":"message_start","message":{"id":"msg_123","type":"message","role":"assistant","model":"claude-3-opus-20240229","usage":{"input_tokens":10,"output_tokens":0}}} + claudeStream := `data: {"type":"message_start","message":{"id":"msg_123","type":"message","role":"assistant","model":"claude-3-opus-20240229","usage":{"input_tokens":10,"output_tokens":0,"cache_read_input_tokens":4,"cache_creation_input_tokens":3}}} data: {"type":"content_block_start","index":0,"content_block":{"type":"text"}} @@ -56,12 +56,20 @@ data: {"type":"message_stop"} if usage == nil { t.Fatal("expected non-nil usage") } - if usage.PromptTokens != 10 { - t.Errorf("PromptTokens = %d, want 10", usage.PromptTokens) + // 转换器已按 OpenAI 口径做加法：prompt = input(10) + cache_read(4) + cache_creation(3) + if usage.PromptTokens != 17 { + t.Errorf("PromptTokens = %d, want 17", usage.PromptTokens) } if usage.CompletionTokens != 7 { t.Errorf("CompletionTokens = %d, want 7", usage.CompletionTokens) } + if usage.PromptTokensDetails == nil || usage.PromptTokensDetails.CachedTokens != 4 || usage.CacheCreationTokens != 3 { + t.Errorf("cache usage = %+v, want read=4 creation=3", usage) + } + // 转换后的 prompt 已含缓存（OpenAI 子集语义），计费前须扣减缓存部分避免双重计费 + if !usage.CacheIncludedInPrompt { + t.Error("CacheIncludedInPrompt = false, want true for Claude upstream (converted usage is OpenAI semantics)") + } body := rec.Body.String() if !strings.Contains(body, "chat.completion.chunk") { @@ -84,6 +92,50 @@ data: {"type":"message_stop"} } } +// TestConvertStreamViaRelaykit_GeminiToOpenAI 验证 Gemini SSE 流经 relaykit 转换为 OpenAI SSE， +// 缓存/思考 token 明细正确透出，且因 Gemini 的 promptTokenCount 已含 cached（子集语义） +// 计费标记 CacheIncludedInPrompt=true（计费时扣减缓存部分，避免双重计费）。 +func TestConvertStreamViaRelaykit_GeminiToOpenAI(t *testing.T) { + // 报文格式同 relaykit golden 用例 03_basic_streaming，末 chunk 带缓存/思考用量 + geminiStream := `data: {"candidates":[{"index":0,"content":{"role":"model","parts":[{"text":"Hello!"}]}}]} + +data: {"candidates":[{"index":0,"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":20,"candidatesTokenCount":5,"totalTokenCount":25,"cachedContentTokenCount":8,"thoughtsTokenCount":3}} + +` + + info := newStreamTestRelayInfo(constant.ProviderGemini, constant.RelayFormatOpenAI) + rec := httptest.NewRecorder() + + usage, ok := convertStreamViaRelaykit(context.Background(), info, strings.NewReader(geminiStream), rec) + if !ok { + t.Fatal("expected ok=true (handled), got false") + } + if usage == nil { + t.Fatal("expected non-nil usage") + } + // 转换器已按 OpenAI 口径合并思考 token：completion = candidates(5) + thoughts(3) + if usage.PromptTokens != 20 || usage.CompletionTokens != 8 || usage.TotalTokens != 25 { + t.Errorf("token counts = %+v, want prompt=20 completion=8 total=25", usage) + } + if usage.PromptTokensDetails == nil || usage.PromptTokensDetails.CachedTokens != 8 { + t.Errorf("cached tokens = %+v, want 8", usage.PromptTokensDetails) + } + if usage.CompletionTokenDetails == nil || usage.CompletionTokenDetails.ReasoningTokens != 3 { + t.Errorf("reasoning tokens = %+v, want 3", usage.CompletionTokenDetails) + } + if !usage.CacheIncludedInPrompt { + t.Error("CacheIncludedInPrompt = false, want true for Gemini upstream (cached ⊆ prompt)") + } + + body := rec.Body.String() + if !strings.Contains(body, "Hello!") { + t.Errorf("output missing streamed text, got: %s", body) + } + if !strings.HasSuffix(strings.TrimSpace(body), "data: [DONE]") { + t.Errorf("output should end with [DONE], got tail: %q", tail(body, 40)) + } +} + // TestConvertStreamViaRelaykit_SameFormatFallback 同格式（OpenAI→OpenAI）无需转换，应回退（ok=false）。 func TestConvertStreamViaRelaykit_SameFormatFallback(t *testing.T) { info := newStreamTestRelayInfo(constant.ProviderOpenAI, constant.RelayFormatOpenAI) diff --git a/relaykit/dispatch/classify_test.go b/relaykit/dispatch/classify_test.go index 02ef042..3916d7f 100644 --- a/relaykit/dispatch/classify_test.go +++ b/relaykit/dispatch/classify_test.go @@ -49,6 +49,7 @@ func TestClassify_NewAPIError错误码(t *testing.T) { }{ {"invalid_key 归凭证", types.ErrorCodeChannelInvalidKey, ErrClassCredential}, {"no_available_key 归渠道级", types.ErrorCodeChannelNoAvailableKey, ErrClassChannelFatal}, + {"protocol_mismatch 归渠道级（驱动换渠道）", types.ErrorCodeChannelProtocolMismatch, ErrClassChannelFatal}, {"model_not_found 归渠道级", types.ErrorCodeModelNotFound, ErrClassChannelFatal}, {"response_time_exceeded 归超时", types.ErrorCodeChannelResponseTimeExceeded, ErrClassTimeout}, {"aws_client_error 归渠道级", types.ErrorCodeChannelAwsClientError, ErrClassChannelFatal}, diff --git a/relaykit/dto/ollama.go b/relaykit/dto/ollama.go index d709428..a655409 100644 --- a/relaykit/dto/ollama.go +++ b/relaykit/dto/ollama.go @@ -6,31 +6,63 @@ // 不依赖任何宿主类型（RelayInfo / GoFrame），仅含协议字段。 package dto +import "encoding/json" + // OllamaChatRequest Ollama Chat 请求（/api/chat）。 type OllamaChatRequest struct { Model string `json:"model"` Messages []OllamaMessage `json:"messages"` Stream bool `json:"stream"` Options map[string]any `json:"options,omitempty"` + Tools []OllamaTool `json:"tools,omitempty"` + Format any `json:"format,omitempty"` + Think json.RawMessage `json:"think,omitempty"` } // OllamaMessage Ollama 消息格式。Content 为纯文本（多模态图片经 base64 放入 Images）。 type OllamaMessage struct { - Role string `json:"role"` - Content string `json:"content"` - Images []string `json:"images,omitempty"` + Role string `json:"role"` + Content string `json:"content"` + Images []string `json:"images,omitempty"` + ToolCalls []OllamaToolCall `json:"tool_calls,omitempty"` + ToolName string `json:"tool_name,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + Thinking json.RawMessage `json:"thinking,omitempty"` } // OllamaChatResponse Ollama Chat 响应（流式与非流式共用）。 // // 流式（NDJSON）每行一个本结构；非流式为单个本结构。 -// 最后一条（Done==true）携带 PromptEvalCount / EvalCount 用量。 +// 最后一条（Done==true）携带 PromptEvalCount / EvalCount 用量与 DoneReason 结束原因。 type OllamaChatResponse struct { Model string `json:"model"` CreatedAt string `json:"created_at"` Message OllamaMessage `json:"message"` Done bool `json:"done"` + DoneReason string `json:"done_reason,omitempty"` TotalDuration int64 `json:"total_duration,omitempty"` PromptEvalCount int `json:"prompt_eval_count,omitempty"` EvalCount int `json:"eval_count,omitempty"` } + +// OllamaTool Ollama 工具定义（OpenAI function 格式一致）。 +type OllamaTool struct { + Type string `json:"type"` + Function OllamaToolFunction `json:"function"` +} + +// OllamaToolFunction 工具函数定义。 +type OllamaToolFunction struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Parameters any `json:"parameters,omitempty"` +} + +// OllamaToolCall Ollama 工具调用。Arguments 为 JSON 对象（非字符串，区别于 OpenAI 的字符串编码）。 +type OllamaToolCall struct { + ID string `json:"id,omitempty"` + Function struct { + Name string `json:"name"` + Arguments any `json:"arguments"` + } `json:"function"` +} diff --git a/relaykit/dto/usage.go b/relaykit/dto/usage.go index 4f2ed0b..36be348 100644 --- a/relaykit/dto/usage.go +++ b/relaykit/dto/usage.go @@ -20,16 +20,17 @@ type UsageWithDetails struct { // TokenDetails Token 类型细分 type TokenDetails struct { - CachedTokens int `json:"cached_tokens,omitempty"` - CachedCreationTokens int `json:"cached_creation_tokens,omitempty"` // Claude cache_creation_input_tokens - CachedCreation5mTokens int `json:"cached_creation_5m_tokens,omitempty"` // Claude 5分钟缓存创建 - CachedCreation1hTokens int `json:"cached_creation_1h_tokens,omitempty"` // Claude 1小时缓存创建 - AudioTokens int `json:"audio_tokens,omitempty"` - TextTokens int `json:"text_tokens,omitempty"` - ImageTokens int `json:"image_tokens,omitempty"` - ReasoningTokens int `json:"reasoning_tokens,omitempty"` - AcceptedPredictionTokens int `json:"accepted_prediction_tokens,omitempty"` - RejectedPredictionTokens int `json:"rejected_prediction_tokens,omitempty"` + CachedTokens int `json:"cached_tokens,omitempty"` + CachedCreationTokens int `json:"cached_creation_tokens,omitempty"` // Claude cache_creation_input_tokens + CachedCreation5mTokens int `json:"cached_creation_5m_tokens,omitempty"` // Claude 5分钟缓存创建 + CachedCreation1hTokens int `json:"cached_creation_1h_tokens,omitempty"` // Claude 1小时缓存创建 + CacheWriteTokens int `json:"cache_write_tokens,omitempty"` // OpenAI Responses cache_write_tokens（观测字段，prompt_tokens 子集） + AudioTokens int `json:"audio_tokens,omitempty"` + TextTokens int `json:"text_tokens,omitempty"` + ImageTokens int `json:"image_tokens,omitempty"` + ReasoningTokens int `json:"reasoning_tokens,omitempty"` + AcceptedPredictionTokens int `json:"accepted_prediction_tokens,omitempty"` + RejectedPredictionTokens int `json:"rejected_prediction_tokens,omitempty"` } // CompletionsRequest 文本补全请求 diff --git a/relaykit/relayconvert/internal/oai_chat/claude_to_openai_response.go b/relaykit/relayconvert/internal/oai_chat/claude_to_openai_response.go index 28236f8..64149ac 100644 --- a/relaykit/relayconvert/internal/oai_chat/claude_to_openai_response.go +++ b/relaykit/relayconvert/internal/oai_chat/claude_to_openai_response.go @@ -63,12 +63,23 @@ func (c *ClaudeToOpenAIResponseConverter) ConvertResponse( message := convertClaudeContentToMessage(claudeResp.Content) openaiResp.Choices[0].Message = message - // 转换 usage + // 转换 usage。Claude 的 input_tokens 不含缓存（三项并列），OpenAI 的 prompt_tokens + // 含缓存（cached 是其子集），转换必须做加法：prompt = input + cache_read + cache_creation， + // 否则缓存场景下客户端按 OpenAI 语义解析会少算输入量（甚至出现 cached > prompt 的矛盾） if claudeResp.Usage != nil { + promptTotal := claudeResp.Usage.InputTokens + + claudeResp.Usage.CacheReadInputTokens + + claudeResp.Usage.CacheCreationInputTokens openaiResp.Usage = dto.UsageWithDetails{ - PromptTokens: claudeResp.Usage.InputTokens, + PromptTokens: promptTotal, CompletionTokens: claudeResp.Usage.OutputTokens, - TotalTokens: claudeResp.Usage.InputTokens + claudeResp.Usage.OutputTokens, + TotalTokens: promptTotal + claudeResp.Usage.OutputTokens, + } + if claudeResp.Usage.CacheReadInputTokens > 0 || claudeResp.Usage.CacheCreationInputTokens > 0 { + openaiResp.Usage.PromptTokensDetails = &dto.TokenDetails{ + CachedTokens: claudeResp.Usage.CacheReadInputTokens, + CachedCreationTokens: claudeResp.Usage.CacheCreationInputTokens, + } } } diff --git a/relaykit/relayconvert/internal/oai_chat/claude_to_openai_response_test.go b/relaykit/relayconvert/internal/oai_chat/claude_to_openai_response_test.go index 9c1ece0..1ea0113 100644 --- a/relaykit/relayconvert/internal/oai_chat/claude_to_openai_response_test.go +++ b/relaykit/relayconvert/internal/oai_chat/claude_to_openai_response_test.go @@ -112,6 +112,56 @@ func TestClaudeToOpenAIResponseConverter_BasicConversion(t *testing.T) { } } +// TestClaudeToOpenAIResponseConverter_CacheUsageAddition 验证缓存 token 的加法口径： +// Claude 的 input_tokens 不含缓存（三项并列），OpenAI 的 prompt_tokens 含缓存（cached 为其子集）， +// 转换必须做 input+cache_read+cache_creation 加法，否则缓存场景下客户端少算输入量、 +// 出现 cached > prompt 的矛盾数据。 +func TestClaudeToOpenAIResponseConverter_CacheUsageAddition(t *testing.T) { + converter := &ClaudeToOpenAIResponseConverter{} + ctx := context.Background() + + textContent := "Hi" + claudeResp := &dto.ClaudeResponse{ + ID: "msg_cache", + Type: "message", + Role: "assistant", + Model: "claude-sonnet-4", + StopReason: "end_turn", + Content: []dto.ClaudeContentBlock{ + {Type: "text", Text: &textContent}, + }, + Usage: &dto.ClaudeUsage{ + InputTokens: 204, + OutputTokens: 50, + CacheReadInputTokens: 1800, + CacheCreationInputTokens: 248, + }, + } + + result, err := converter.ConvertResponse(ctx, nil, claudeResp) + if err != nil { + t.Fatalf("ConvertResponse failed: %v", err) + } + openaiResp, ok := result.(*dto.ChatCompletionResponse) + if !ok { + t.Fatalf("Expected *dto.ChatCompletionResponse, got %T", result) + } + + // prompt = 204 + 1800 + 248 = 2252 + if openaiResp.Usage.PromptTokens != 2252 { + t.Errorf("PromptTokens = %d, want 2252 (input+cache_read+cache_creation)", openaiResp.Usage.PromptTokens) + } + if openaiResp.Usage.CompletionTokens != 50 { + t.Errorf("CompletionTokens = %d, want 50", openaiResp.Usage.CompletionTokens) + } + if openaiResp.Usage.TotalTokens != 2302 { + t.Errorf("TotalTokens = %d, want 2302", openaiResp.Usage.TotalTokens) + } + if d := openaiResp.Usage.PromptTokensDetails; d == nil || d.CachedTokens != 1800 || d.CachedCreationTokens != 248 { + t.Errorf("PromptTokensDetails = %+v, want cached=1800 creation=248", openaiResp.Usage.PromptTokensDetails) + } +} + func TestClaudeToOpenAIResponseConverter_ThinkingContent(t *testing.T) { converter := &ClaudeToOpenAIResponseConverter{} ctx := context.Background() diff --git a/relaykit/relayconvert/internal/oai_chat/claude_to_openai_stream.go b/relaykit/relayconvert/internal/oai_chat/claude_to_openai_stream.go index 60d25f1..8bb21e4 100644 --- a/relaykit/relayconvert/internal/oai_chat/claude_to_openai_stream.go +++ b/relaykit/relayconvert/internal/oai_chat/claude_to_openai_stream.go @@ -222,15 +222,24 @@ func (c *ClaudeToOpenAIStreamConverter) ConvertStreamResponse( } case "message_stop": - // 末尾 chunk，包含 finish_reason 和 usage + // 末尾 chunk，包含 finish_reason 和 usage。 + // Claude 的 input_tokens 不含缓存（三项并列），OpenAI 的 prompt_tokens 含缓存 + //（cached 是其子集），转换必须做加法，否则缓存场景下客户端少算输入量 reason := finishReason if reason == "" { reason = "stop" } + promptTotal := usage.InputTokens + usage.CacheReadInputTokens + usage.CacheCreationInputTokens usageObj := &dto.UsageWithDetails{ - PromptTokens: usage.InputTokens, + PromptTokens: promptTotal, CompletionTokens: usage.OutputTokens, - TotalTokens: usage.InputTokens + usage.OutputTokens, + TotalTokens: promptTotal + usage.OutputTokens, + } + if usage.CacheReadInputTokens > 0 || usage.CacheCreationInputTokens > 0 { + usageObj.PromptTokensDetails = &dto.TokenDetails{ + CachedTokens: usage.CacheReadInputTokens, + CachedCreationTokens: usage.CacheCreationInputTokens, + } } if usageObj.CompletionTokens == 0 { // 未提供时根据文本长度估算 diff --git a/relaykit/relayconvert/internal/oai_chat/claude_to_openai_stream_test.go b/relaykit/relayconvert/internal/oai_chat/claude_to_openai_stream_test.go index 6e545a7..597ffc4 100644 --- a/relaykit/relayconvert/internal/oai_chat/claude_to_openai_stream_test.go +++ b/relaykit/relayconvert/internal/oai_chat/claude_to_openai_stream_test.go @@ -468,3 +468,55 @@ func createClaudeSSEStream(events []map[string]any) *bytes.Buffer { } return &buf } + +// TestClaudeToOpenAIStreamConverter_CacheUsageAddition 验证流式转换的缓存加法口径： +// 末 chunk 的 prompt_tokens 须为 input+cache_read+cache_creation 合计（OpenAI 子集语义）。 +func TestClaudeToOpenAIStreamConverter_CacheUsageAddition(t *testing.T) { + converter := &ClaudeToOpenAIStreamConverter{} + ctx := context.Background() + + claudeStream := `data: {"type":"message_start","message":{"id":"msg_cache","type":"message","role":"assistant","model":"claude-sonnet-4","usage":{"input_tokens":204,"output_tokens":0,"cache_read_input_tokens":1800,"cache_creation_input_tokens":248}}} + +data: {"type":"content_block_start","index":0,"content_block":{"type":"text"}} + +data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi"}} + +data: {"type":"content_block_stop","index":0} + +data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":50}} + +data: {"type":"message_stop"} + +` + + reader := strings.NewReader(claudeStream) + var chunks []*dto.ChatCompletionStreamResponse + chunkWriter := func(chunk any) error { + if c, ok := chunk.(*dto.ChatCompletionStreamResponse); ok { + chunks = append(chunks, c) + } + return nil + } + + if err := converter.ConvertStreamResponse(ctx, nil, reader, chunkWriter); err != nil { + t.Fatalf("ConvertStreamResponse failed: %v", err) + } + + lastChunk := chunks[len(chunks)-1] + if lastChunk.Usage == nil { + t.Fatal("Last chunk should have usage") + } + // prompt = 204 + 1800 + 248 = 2252 + if lastChunk.Usage.PromptTokens != 2252 { + t.Errorf("PromptTokens = %d, want 2252 (input+cache_read+cache_creation)", lastChunk.Usage.PromptTokens) + } + if lastChunk.Usage.CompletionTokens != 50 { + t.Errorf("CompletionTokens = %d, want 50", lastChunk.Usage.CompletionTokens) + } + if lastChunk.Usage.TotalTokens != 2302 { + t.Errorf("TotalTokens = %d, want 2302", lastChunk.Usage.TotalTokens) + } + if d := lastChunk.Usage.PromptTokensDetails; d == nil || d.CachedTokens != 1800 || d.CachedCreationTokens != 248 { + t.Errorf("PromptTokensDetails = %+v, want cached=1800 creation=248", lastChunk.Usage.PromptTokensDetails) + } +} diff --git a/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_response.go b/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_response.go index a11ae20..372c0c6 100644 --- a/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_response.go +++ b/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_response.go @@ -155,13 +155,26 @@ func (c *GeminiToOpenAIResponseConverter) ConvertResponse( openaiResp.Choices = append(openaiResp.Choices, choice) - // 转换 usage + // 转换 usage。 + // OpenAI 口径：prompt 含 cached（子集），completion 含 thoughts（子集）—— + // Gemini 的 candidatesTokenCount 不含思考 token，须与 thoughtsTokenCount 合计， + // 否则计费捕获与客户端按 OpenAI 语义解析都会漏掉思考部分 if geminiResp.UsageMetadata != nil { openaiResp.Usage = dto.UsageWithDetails{ PromptTokens: geminiResp.UsageMetadata.PromptTokenCount, - CompletionTokens: geminiResp.UsageMetadata.CandidatesTokenCount, + CompletionTokens: geminiResp.UsageMetadata.CandidatesTokenCount + geminiResp.UsageMetadata.ThoughtsTokenCount, TotalTokens: geminiResp.UsageMetadata.TotalTokenCount, } + if geminiResp.UsageMetadata.CachedContentTokenCount > 0 { + openaiResp.Usage.PromptTokensDetails = &dto.TokenDetails{ + CachedTokens: geminiResp.UsageMetadata.CachedContentTokenCount, + } + } + if geminiResp.UsageMetadata.ThoughtsTokenCount > 0 { + openaiResp.Usage.CompletionTokenDetails = &dto.TokenDetails{ + ReasoningTokens: geminiResp.UsageMetadata.ThoughtsTokenCount, + } + } } return openaiResp, nil diff --git a/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_response_test.go b/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_response_test.go index 3238db2..4bb17b8 100644 --- a/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_response_test.go +++ b/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_response_test.go @@ -872,6 +872,57 @@ func TestGeminiToOpenAIStreamConverter_MultipleCandidates(t *testing.T) { } } +// TestGeminiToOpenAIResponseConverter_ThoughtsUsage 验证思考 token 计入输出： +// Gemini 的 candidatesTokenCount 不含思考，OpenAI 口径的 completion 含 reasoning（子集）， +// 转换必须输出 candidates+thoughts 合计，否则计费捕获与客户端解析都会漏掉思考部分。 +func TestGeminiToOpenAIResponseConverter_ThoughtsUsage(t *testing.T) { + converter := &GeminiToOpenAIResponseConverter{} + ctx := context.Background() + + geminiResp := &dto.GeminiChatResponse{ + Candidates: []dto.GeminiCandidate{{ + Content: &dto.GeminiContent{ + Role: "model", + Parts: []dto.GeminiPart{{Text: "Hello!"}}, + }, + FinishReason: "STOP", + }}, + UsageMetadata: &dto.GeminiUsageMetadata{ + PromptTokenCount: 1050, + CandidatesTokenCount: 200, + ThoughtsTokenCount: 150, + TotalTokenCount: 1400, + CachedContentTokenCount: 1000, + }, + } + + result, err := converter.ConvertResponse(ctx, nil, geminiResp) + if err != nil { + t.Fatalf("ConvertResponse failed: %v", err) + } + openaiResp, ok := result.(*dto.ChatCompletionResponse) + if !ok { + t.Fatalf("Expected *dto.ChatCompletionResponse, got %T", result) + } + + if openaiResp.Usage.PromptTokens != 1050 { + t.Errorf("PromptTokens = %d, want 1050", openaiResp.Usage.PromptTokens) + } + // completion = candidates(200) + thoughts(150)，thoughts 作为 reasoning 子集透出 + if openaiResp.Usage.CompletionTokens != 350 { + t.Errorf("CompletionTokens = %d, want 350 (candidates+thoughts)", openaiResp.Usage.CompletionTokens) + } + if openaiResp.Usage.TotalTokens != 1400 { + t.Errorf("TotalTokens = %d, want 1400", openaiResp.Usage.TotalTokens) + } + if d := openaiResp.Usage.PromptTokensDetails; d == nil || d.CachedTokens != 1000 { + t.Errorf("PromptTokensDetails = %+v, want cached=1000", openaiResp.Usage.PromptTokensDetails) + } + if d := openaiResp.Usage.CompletionTokenDetails; d == nil || d.ReasoningTokens != 150 { + t.Errorf("CompletionTokenDetails = %+v, want reasoning=150", openaiResp.Usage.CompletionTokenDetails) + } +} + func TestGeminiToOpenAIStreamConverter_FileDataStreaming(t *testing.T) { converter := &GeminiToOpenAIStreamConverter{} ctx := context.Background() diff --git a/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_stream.go b/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_stream.go index 38f90ef..7571bf3 100644 --- a/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_stream.go +++ b/relaykit/relayconvert/internal/oai_gemini/gemini_to_openai_stream.go @@ -240,11 +240,20 @@ func (c *GeminiToOpenAIStreamConverter) ConvertStreamResponse( } if totalUsage.PromptTokenCount > 0 || totalUsage.CandidatesTokenCount > 0 { + // OpenAI 口径：prompt 含 cached（子集），completion 含 thoughts（子集）—— + // Gemini 的 candidatesTokenCount 不含思考 token，须与 thoughtsTokenCount 合计， + // 否则计费捕获与客户端按 OpenAI 语义解析都会漏掉思考部分 finalChunk.Usage = &dto.UsageWithDetails{ PromptTokens: totalUsage.PromptTokenCount, - CompletionTokens: totalUsage.CandidatesTokenCount, + CompletionTokens: totalUsage.CandidatesTokenCount + totalUsage.ThoughtsTokenCount, TotalTokens: totalUsage.TotalTokenCount, } + if totalUsage.CachedContentTokenCount > 0 { + finalChunk.Usage.PromptTokensDetails = &dto.TokenDetails{CachedTokens: totalUsage.CachedContentTokenCount} + } + if totalUsage.ThoughtsTokenCount > 0 { + finalChunk.Usage.CompletionTokenDetails = &dto.TokenDetails{ReasoningTokens: totalUsage.ThoughtsTokenCount} + } } if err := chunkWriter(finalChunk); err != nil { diff --git a/relaykit/relayconvert/internal/ollama_chat/ollama_chat_test.go b/relaykit/relayconvert/internal/ollama_chat/ollama_chat_test.go index fe0a45b..f8cf54e 100644 --- a/relaykit/relayconvert/internal/ollama_chat/ollama_chat_test.go +++ b/relaykit/relayconvert/internal/ollama_chat/ollama_chat_test.go @@ -3,6 +3,7 @@ package ollama_chat import ( "bytes" "context" + "encoding/json" "testing" "github.com/qianfree/team-api/relaykit/dto" @@ -27,8 +28,8 @@ func TestOpenAIToOllamaRequestConverter_BasicWithOptions(t *testing.T) { c := &OpenAIToOllamaRequestConverter{} temp, topP, maxTok, freq, pres := 0.5, 0.8, 100, 0.1, 0.2 req := &dto.GeneralOpenAIRequest{ - Model: "openai-model", - Messages: []dto.Message{{Role: "user", Content: "hi"}}, + Model: "openai-model", + Messages: []dto.Message{{Role: "user", Content: "hi"}}, Temperature: &temp, TopP: &topP, MaxTokens: &maxTok, FrequencyPenalty: &freq, PresencePenalty: &pres, Stop: "STOP", } @@ -66,7 +67,7 @@ func TestOpenAIToOllamaRequestConverter_MaxCompletionTokens(t *testing.T) { c := &OpenAIToOllamaRequestConverter{} mct := 256 req := &dto.GeneralOpenAIRequest{ - Messages: []dto.Message{{Role: "user", Content: "hi"}}, + Messages: []dto.Message{{Role: "user", Content: "hi"}}, MaxCompletionTokens: &mct, } res, _ := c.ConvertRequest(context.Background(), nil, req) @@ -191,3 +192,246 @@ func TestOllamaToOpenAIStreamConverter_NoDoneFallback(t *testing.T) { t.Error("expected terminal finish chunk when no done line") } } + +// ---------- 请求转换：tools / format / think ---------- + +func TestOpenAIToOllamaRequestConverter_ToolsFormatThink(t *testing.T) { + c := &OpenAIToOllamaRequestConverter{} + req := &dto.GeneralOpenAIRequest{ + Messages: []dto.Message{{Role: "user", Content: "hi"}}, + Tools: []dto.Tool{{ + Type: "function", + Function: dto.FunctionDef{ + Name: "get_weather", + Description: "get weather", + Parameters: map[string]any{"type": "object", "properties": map[string]any{"city": map[string]any{"type": "string"}}}, + }, + }}, + ResponseFormat: &dto.ResponseFormat{Type: "json_schema", JSONSchema: map[string]any{"name": "weather", "schema": map[string]any{"type": "object", "properties": map[string]any{"city": map[string]any{"type": "string"}}}}}, + ReasoningEffort: "high", + } + res, err := c.ConvertRequest(context.Background(), nil, req) + if err != nil { + t.Fatalf("ConvertRequest: %v", err) + } + r := res.(*dto.OllamaChatRequest) + if len(r.Tools) != 1 || r.Tools[0].Type != "function" || r.Tools[0].Function.Name != "get_weather" { + t.Errorf("Tools = %+v", r.Tools) + } + // json_schema 应解包为裸 schema，不带外层包装字段 + format, ok := r.Format.(map[string]any) + if !ok || format["type"] != "object" { + t.Errorf("Format = %#v, want bare schema {type:object}", r.Format) + } + if _, hasName := format["name"]; hasName { + t.Errorf("Format should not keep json_schema wrapper name, got %#v", r.Format) + } + if string(r.Think) != `"high"` { + t.Errorf("Think = %s, want \"high\"", r.Think) + } +} + +func TestOpenAIToOllamaRequestConverter_JSONFormatAndThinkNone(t *testing.T) { + c := &OpenAIToOllamaRequestConverter{} + req := &dto.GeneralOpenAIRequest{ + Messages: []dto.Message{{Role: "user", Content: "hi"}}, + ResponseFormat: &dto.ResponseFormat{Type: "json_object"}, + ReasoningEffort: "none", + } + res, err := c.ConvertRequest(context.Background(), nil, req) + if err != nil { + t.Fatalf("ConvertRequest: %v", err) + } + r := res.(*dto.OllamaChatRequest) + if r.Format != "json" { + t.Errorf("Format = %#v, want json", r.Format) + } + if string(r.Think) != "false" { + t.Errorf("Think = %s, want false", r.Think) + } +} + +func TestOpenAIToOllamaRequestConverter_ToolCallContext(t *testing.T) { + c := &OpenAIToOllamaRequestConverter{} + rc := "thinking text" + req := &dto.GeneralOpenAIRequest{ + Messages: []dto.Message{ + { + Role: "assistant", + Content: "", + ReasoningContent: &rc, + ToolCalls: []dto.ToolCall{{ + ID: "call_abc", + Type: "function", + Function: dto.FunctionCall{ + Name: "get_weather", + Arguments: `{"city":"Paris"}`, + }, + }}, + }, + { + Role: "tool", + Content: "sunny", + ToolCallID: "call_abc", + }, + }, + } + res, err := c.ConvertRequest(context.Background(), nil, req) + if err != nil { + t.Fatalf("ConvertRequest: %v", err) + } + r := res.(*dto.OllamaChatRequest) + asst := r.Messages[0] + if string(asst.Thinking) != `"thinking text"` { + t.Errorf("assistant Thinking = %s, want \"thinking text\"", asst.Thinking) + } + if len(asst.ToolCalls) != 1 || asst.ToolCalls[0].ID != "call_abc" { + t.Fatalf("assistant ToolCalls = %+v", asst.ToolCalls) + } + args, ok := asst.ToolCalls[0].Function.Arguments.(map[string]any) + if !ok || args["city"] != "Paris" { + t.Errorf("tool call arguments = %#v", asst.ToolCalls[0].Function.Arguments) + } + toolMsg := r.Messages[1] + if toolMsg.ToolCallID != "call_abc" { + t.Errorf("tool ToolCallID = %q, want call_abc", toolMsg.ToolCallID) + } + if toolMsg.ToolName != "get_weather" { + t.Errorf("tool ToolName = %q, want get_weather (fallback from tool_calls)", toolMsg.ToolName) + } +} + +// ---------- 非流式响应转换：tool_calls / thinking ---------- + +func TestOllamaToOpenAIResponseConverter_ToolCallsAndThinking(t *testing.T) { + c := &OllamaToOpenAIResponseConverter{} + resp := &dto.OllamaChatResponse{ + Message: dto.OllamaMessage{ + Role: "assistant", + Content: "", + Thinking: json.RawMessage(`"step by step"`), + ToolCalls: []dto.OllamaToolCall{{ + ID: "call_upstream", + }}, + }, + PromptEvalCount: 5, + EvalCount: 7, + } + resp.Message.ToolCalls[0].Function.Name = "get_weather" + resp.Message.ToolCalls[0].Function.Arguments = map[string]any{"city": "Paris"} + + res, err := c.ConvertResponse(context.Background(), nil, resp) + if err != nil { + t.Fatalf("ConvertResponse: %v", err) + } + o := res.(*dto.ChatCompletionResponse) + msg := o.Choices[0].Message + if o.Choices[0].FinishReason != "tool_calls" { + t.Errorf("finish reason = %q, want tool_calls", o.Choices[0].FinishReason) + } + if msg.ReasoningContent == nil || *msg.ReasoningContent != "step by step" { + t.Errorf("reasoning content = %v", msg.ReasoningContent) + } + if len(msg.ToolCalls) != 1 { + t.Fatalf("tool calls = %+v", msg.ToolCalls) + } + tc := msg.ToolCalls[0] + if tc.ID != "call_upstream" || tc.Type != "function" { + t.Errorf("tool call = %+v", tc) + } + if tc.Function.Name != "get_weather" || tc.Function.Arguments != `{"city":"Paris"}` { + t.Errorf("tool call function = %+v", tc.Function) + } +} + +func TestOllamaToOpenAIResponseConverter_ThinkingOnly(t *testing.T) { + c := &OllamaToOpenAIResponseConverter{} + resp := &dto.OllamaChatResponse{ + Message: dto.OllamaMessage{Role: "assistant", Content: "answer", Thinking: json.RawMessage(`"draft"`)}, + } + res, err := c.ConvertResponse(context.Background(), nil, resp) + if err != nil { + t.Fatalf("ConvertResponse: %v", err) + } + o := res.(*dto.ChatCompletionResponse) + if o.Choices[0].FinishReason != "stop" { + t.Errorf("finish reason = %q, want stop", o.Choices[0].FinishReason) + } + if o.Choices[0].Message.ReasoningContent == nil || *o.Choices[0].Message.ReasoningContent != "draft" { + t.Errorf("reasoning content = %v", o.Choices[0].Message.ReasoningContent) + } + if len(o.Choices[0].Message.ToolCalls) != 0 { + t.Errorf("tool calls should be empty, got %+v", o.Choices[0].Message.ToolCalls) + } +} + +func TestOllamaToOpenAIResponseConverter_ThinkingNullOrEmpty(t *testing.T) { + c := &OllamaToOpenAIResponseConverter{} + for _, raw := range []string{"null", `""`} { + resp := &dto.OllamaChatResponse{ + Message: dto.OllamaMessage{Role: "assistant", Content: "answer", Thinking: json.RawMessage(raw)}, + } + res, err := c.ConvertResponse(context.Background(), nil, resp) + if err != nil { + t.Fatalf("ConvertResponse(thinking=%s): %v", raw, err) + } + o := res.(*dto.ChatCompletionResponse) + if o.Choices[0].Message.ReasoningContent != nil { + t.Errorf("thinking=%s: reasoning content = %q, want nil", raw, *o.Choices[0].Message.ReasoningContent) + } + } +} + +func TestOllamaToOpenAIResponseConverter_DoneReasonLength(t *testing.T) { + c := &OllamaToOpenAIResponseConverter{} + resp := &dto.OllamaChatResponse{ + Message: dto.OllamaMessage{Role: "assistant", Content: "partial"}, + Done: true, + DoneReason: "length", + } + res, err := c.ConvertResponse(context.Background(), nil, resp) + if err != nil { + t.Fatalf("ConvertResponse: %v", err) + } + o := res.(*dto.ChatCompletionResponse) + if o.Choices[0].FinishReason != "length" { + t.Errorf("finish reason = %q, want length", o.Choices[0].FinishReason) + } +} + +// ---------- 流式响应转换：tool_calls ---------- + +func TestOllamaToOpenAIStreamConverter_ToolCalls(t *testing.T) { + c := &OllamaToOpenAIStreamConverter{} + ndjson := `{"model":"llama3","message":{"role":"assistant","content":"","tool_calls":[{"id":"call_1","function":{"name":"get_weather","arguments":{"city":"Paris"}}}]},"done":false}` + "\n" + + `{"model":"llama3","message":{"role":"assistant","content":""},"done":true,"prompt_eval_count":10,"eval_count":5}` + "\n" + var chunks []*dto.ChatCompletionStreamResponse + err := c.ConvertStreamResponse(context.Background(), nil, bytes.NewReader([]byte(ndjson)), func(chunk any) error { + if sc, ok := chunk.(*dto.ChatCompletionStreamResponse); ok { + chunks = append(chunks, sc) + } + return nil + }) + if err != nil { + t.Fatalf("ConvertStreamResponse: %v", err) + } + // role 首包 + tool_calls 增量 + finish(usage) 共三段 + if len(chunks) != 3 { + t.Fatalf("chunks = %d, want 3", len(chunks)) + } + toolDelta := chunks[1].Choices[0].Delta + if len(toolDelta.ToolCalls) != 1 { + t.Fatalf("tool delta = %+v", toolDelta.ToolCalls) + } + tc := toolDelta.ToolCalls[0] + if tc.Index != 0 || tc.ID != "call_1" || tc.Type != "function" { + t.Errorf("tool call = %+v", tc) + } + if tc.Function.Name != "get_weather" || tc.Function.Arguments != `{"city":"Paris"}` { + t.Errorf("tool call function = %+v", tc.Function) + } + last := chunks[2] + if last.Choices[0].FinishReason == nil || *last.Choices[0].FinishReason != "tool_calls" { + t.Errorf("finish reason = %v, want tool_calls", last.Choices[0].FinishReason) + } +} diff --git a/relaykit/relayconvert/internal/ollama_chat/ollama_to_openai_response.go b/relaykit/relayconvert/internal/ollama_chat/ollama_to_openai_response.go index 2921041..f16decb 100644 --- a/relaykit/relayconvert/internal/ollama_chat/ollama_to_openai_response.go +++ b/relaykit/relayconvert/internal/ollama_chat/ollama_to_openai_response.go @@ -2,6 +2,7 @@ package ollama_chat import ( "context" + "encoding/json" "fmt" "github.com/qianfree/team-api/relaykit/dto" @@ -46,18 +47,31 @@ func (c *OllamaToOpenAIResponseConverter) ConvertResponse( modelName = info.GetOriginModelName() } + message := dto.Message{ + Role: ollamaResp.Message.Role, + Content: ollamaResp.Message.Content, + } + if reasoning := ollamaThinkingToOpenAI(ollamaResp.Message.Thinking); reasoning != nil { + message.ReasoningContent = reasoning + } + if toolCalls := ollamaToolCallsToOpenAI(ollamaResp.Message.ToolCalls); len(toolCalls) > 0 { + message.ToolCalls = toolCalls + } + + finishReason := ollamaDoneReasonToFinishReason(ollamaResp.DoneReason) + if len(message.ToolCalls) > 0 { + finishReason = "tool_calls" + } + openaiResp := &dto.ChatCompletionResponse{ ID: fmt.Sprintf("chatcmpl-%d", getCurrentTimestamp()), Object: "chat.completion", Created: 0, Model: modelName, Choices: []dto.Choice{{ - Index: 0, - Message: dto.Message{ - Role: ollamaResp.Message.Role, - Content: ollamaResp.Message.Content, - }, - FinishReason: "stop", + Index: 0, + Message: message, + FinishReason: finishReason, }}, Usage: dto.UsageWithDetails{ PromptTokens: ollamaResp.PromptEvalCount, @@ -68,3 +82,56 @@ func (c *OllamaToOpenAIResponseConverter) ConvertResponse( return openaiResp, nil } + +// ollamaToolCallsToOpenAI 将 Ollama tool_calls（对象形式 arguments）转换为 +// OpenAI tool_calls（字符串编码 arguments）。 +func ollamaToolCallsToOpenAI(toolCalls []dto.OllamaToolCall) []dto.ToolCall { + out := make([]dto.ToolCall, 0, len(toolCalls)) + for i, tc := range toolCalls { + id := tc.ID + if id == "" { + id = fmt.Sprintf("call_%d", i) + } + argsBytes := []byte("{}") + if tc.Function.Arguments != nil { + if b, err := json.Marshal(tc.Function.Arguments); err == nil { + argsBytes = b + } + } + out = append(out, dto.ToolCall{ + ID: id, + Type: "function", + Function: dto.FunctionCall{ + Name: tc.Function.Name, + Arguments: string(argsBytes), + }, + }) + } + return out +} + +// ollamaThinkingToOpenAI 将 Ollama thinking 字段（JSON 字符串）转为 OpenAI reasoning_content。 +// 字段缺失、null、空串或非字符串时返回 nil，避免产出空的 reasoning_content。 +func ollamaThinkingToOpenAI(thinking json.RawMessage) *string { + if len(thinking) == 0 { + return nil + } + var s string + if err := json.Unmarshal(thinking, &s); err != nil { + return nil + } + // JSON null 反序列化不报错且保持零值，空思考内容同样不产出字段 + if s == "" { + return nil + } + return &s +} + +// ollamaDoneReasonToFinishReason 将 Ollama done_reason 映射为 OpenAI finish_reason。 +// stop/length 均为 OpenAI 合法取值，其余（如 load 等运行态原因）归一为 stop。 +func ollamaDoneReasonToFinishReason(doneReason string) string { + if doneReason == "length" { + return "length" + } + return "stop" +} diff --git a/relaykit/relayconvert/internal/ollama_chat/ollama_to_openai_stream.go b/relaykit/relayconvert/internal/ollama_chat/ollama_to_openai_stream.go index 862bf07..8f4a4af 100644 --- a/relaykit/relayconvert/internal/ollama_chat/ollama_to_openai_stream.go +++ b/relaykit/relayconvert/internal/ollama_chat/ollama_to_openai_stream.go @@ -78,6 +78,8 @@ func (c *OllamaToOpenAIStreamConverter) ConvertStreamResponse( roleChunkSent := false finishEmitted := false + sawToolCalls := false + nextToolCallIndex := 0 for scanner.Scan() { select { @@ -102,7 +104,10 @@ func (c *OllamaToOpenAIStreamConverter) ConvertStreamResponse( CompletionTokens: ollamaResp.EvalCount, TotalTokens: ollamaResp.PromptEvalCount + ollamaResp.EvalCount, } - reason := "stop" + reason := ollamaDoneReasonToFinishReason(ollamaResp.DoneReason) + if sawToolCalls { + reason = "tool_calls" + } if err := chunkWriter(newChunk(dto.Message{}, &reason, usage)); err != nil { return err } @@ -114,6 +119,13 @@ func (c *OllamaToOpenAIStreamConverter) ConvertStreamResponse( delta := dto.Message{ Content: ollamaResp.Message.Content, } + if thinking := ollamaThinkingToOpenAI(ollamaResp.Message.Thinking); thinking != nil { + delta.ReasoningContent = thinking + } + if toolCalls := ollamaStreamToolCalls(ollamaResp.Message.ToolCalls, &nextToolCallIndex); len(toolCalls) > 0 { + delta.ToolCalls = toolCalls + sawToolCalls = true + } if !roleChunkSent { delta.Role = "assistant" emptyContent := "" @@ -137,6 +149,9 @@ func (c *OllamaToOpenAIStreamConverter) ConvertStreamResponse( } } reason := "stop" + if sawToolCalls { + reason = "tool_calls" + } if err := chunkWriter(newChunk(dto.Message{}, &reason, nil)); err != nil { return err } @@ -147,3 +162,32 @@ func (c *OllamaToOpenAIStreamConverter) ConvertStreamResponse( } return nil } + +// ollamaStreamToolCalls 将 Ollama tool_calls 转换为 OpenAI 流式增量 delta， +// 为每个调用分配稳定递增的 index。 +func ollamaStreamToolCalls(toolCalls []dto.OllamaToolCall, nextIndex *int) []dto.ToolCall { + out := make([]dto.ToolCall, 0, len(toolCalls)) + for _, tc := range toolCalls { + id := tc.ID + if id == "" { + id = fmt.Sprintf("call_%d", *nextIndex) + } + argsBytes := []byte("{}") + if tc.Function.Arguments != nil { + if b, err := json.Marshal(tc.Function.Arguments); err == nil { + argsBytes = b + } + } + out = append(out, dto.ToolCall{ + Index: *nextIndex, + ID: id, + Type: "function", + Function: dto.FunctionCall{ + Name: tc.Function.Name, + Arguments: string(argsBytes), + }, + }) + *nextIndex++ + } + return out +} diff --git a/relaykit/relayconvert/internal/ollama_chat/openai_to_ollama_request.go b/relaykit/relayconvert/internal/ollama_chat/openai_to_ollama_request.go index 8bc176f..9220548 100644 --- a/relaykit/relayconvert/internal/ollama_chat/openai_to_ollama_request.go +++ b/relaykit/relayconvert/internal/ollama_chat/openai_to_ollama_request.go @@ -6,6 +6,7 @@ package ollama_chat import ( "context" + "encoding/json" "fmt" "strings" @@ -53,13 +54,48 @@ func (c *OpenAIToOllamaRequestConverter) ConvertRequest( } } - // 消息转换：Ollama Content 为纯文本（多模态文本部分拼接） + // 消息转换：Ollama Content 为纯文本（多模态文本部分拼接）。 + // 保留跨轮次的工具调用与推理上下文： + // assistant 消息保留 tool_calls（含 id 与对象形式的 arguments）， + // tool 消息保留 tool_call_id/tool_name，reasoning_content 转为 thinking 回传。 ollamaMessages := make([]dto.OllamaMessage, 0, len(openaiReq.Messages)) + toolNamesByCallID := make(map[string]string) for _, msg := range openaiReq.Messages { - ollamaMessages = append(ollamaMessages, dto.OllamaMessage{ + om := dto.OllamaMessage{ Role: msg.Role, Content: extractOllamaContent(msg.Content), - }) + } + if msg.Role == "tool" { + om.ToolCallID = msg.ToolCallID + om.ToolName = msg.Name + if om.ToolName == "" { + // 消息未带 name 时按 tool_call_id 从前面 assistant 的 tool_calls 反查工具名； + // Ollama 的 tool 消息靠 tool_name 关联结果（tool_call_id 仅原样透传）。 + om.ToolName = toolNamesByCallID[msg.ToolCallID] + } + } + if msg.Role == "assistant" { + if msg.ReasoningContent != nil { + thinking, err := json.Marshal(*msg.ReasoningContent) + if err != nil { + return nil, fmt.Errorf("marshal ollama thinking: %w", err) + } + om.Thinking = thinking + } + if len(msg.ToolCalls) > 0 { + calls, err := toOllamaToolCalls(msg.ToolCalls) + if err != nil { + return nil, err + } + for _, tc := range msg.ToolCalls { + if tc.ID != "" { + toolNamesByCallID[tc.ID] = tc.Function.Name + } + } + om.ToolCalls = calls + } + } + ollamaMessages = append(ollamaMessages, om) } stream := false @@ -67,11 +103,24 @@ func (c *OpenAIToOllamaRequestConverter) ConvertRequest( stream = info.GetIsStream() } + think, err := toOllamaThink(openaiReq.ReasoningEffort) + if err != nil { + return nil, err + } + + format, err := toOllamaResponseFormat(openaiReq.ResponseFormat) + if err != nil { + return nil, err + } + ollamaReq := &dto.OllamaChatRequest{ Model: model, Messages: ollamaMessages, Stream: stream, Options: buildOllamaOptions(openaiReq), + Tools: toOllamaTools(openaiReq.Tools), + Format: format, + Think: think, } return ollamaReq, nil @@ -139,3 +188,99 @@ func buildOllamaOptions(req *dto.GeneralOpenAIRequest) map[string]any { } return options } + +// toOllamaTools 将 OpenAI tools 转换为 Ollama tools（两者 function 格式一致）。 +func toOllamaTools(tools []dto.Tool) []dto.OllamaTool { + if len(tools) == 0 { + return nil + } + out := make([]dto.OllamaTool, 0, len(tools)) + for _, t := range tools { + if t.Type != "function" { + continue + } + out = append(out, dto.OllamaTool{ + Type: "function", + Function: dto.OllamaToolFunction{ + Name: t.Function.Name, + Description: t.Function.Description, + Parameters: t.Function.Parameters, + }, + }) + } + if len(out) == 0 { + return nil + } + return out +} + +// toOllamaToolCalls 将 OpenAI tool_calls 转换为 Ollama tool_calls， +// 把字符串编码的 arguments 解码为 JSON 对象。 +func toOllamaToolCalls(toolCalls []dto.ToolCall) ([]dto.OllamaToolCall, error) { + out := make([]dto.OllamaToolCall, 0, len(toolCalls)) + for _, tc := range toolCalls { + oc := dto.OllamaToolCall{ID: tc.ID} + oc.Function.Name = tc.Function.Name + var args any + if tc.Function.Arguments != "" { + if err := json.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil { + return nil, fmt.Errorf("parse tool call arguments: %w", err) + } + } + if args == nil { + args = map[string]any{} + } + oc.Function.Arguments = args + out = append(out, oc) + } + return out, nil +} + +// toOllamaThink 将 reasoning_effort 映射为 Ollama think 参数。 +// "none" 关闭思考；low/medium/high/max 作为思考档位原样透传； +// 未知取值直接跳过（不注入 think）而非报错。 +func toOllamaThink(effort string) (json.RawMessage, error) { + if effort == "" { + return nil, nil + } + var value any + switch effort { + case "none": + value = false + case "low", "medium", "high", "max": + value = effort + default: + return nil, nil + } + raw, err := json.Marshal(value) + if err != nil { + return nil, fmt.Errorf("marshal ollama think: %w", err) + } + return raw, nil +} + +// toOllamaResponseFormat 将 OpenAI response_format 映射为 Ollama format。 +func toOllamaResponseFormat(responseFormat *dto.ResponseFormat) (any, error) { + if responseFormat == nil { + return nil, nil + } + switch responseFormat.Type { + case "json", "json_object": + return "json", nil + case "json_schema": + return extractJSONSchema(responseFormat.JSONSchema), nil + default: + return nil, nil + } +} + +// extractJSONSchema 从 OpenAI json_schema 包装结构（{name, description, schema, strict}） +// 中提取裸 schema；Ollama 的 format 字段期望 schema 对象本身。 +func extractJSONSchema(v any) any { + if m, ok := v.(map[string]any); ok { + if schema, ok := m["schema"]; ok { + return schema + } + } + return v +} diff --git a/relaykit/types/error.go b/relaykit/types/error.go index 5cfc097..b53bdca 100644 --- a/relaykit/types/error.go +++ b/relaykit/types/error.go @@ -59,6 +59,10 @@ const ( ErrorCodeChannelAwsClientError ErrorCode = "channel:aws_client_error" ErrorCodeChannelInvalidKey ErrorCode = "channel:invalid_key" ErrorCodeChannelResponseTimeExceeded ErrorCode = "channel:response_time_exceeded" + // ErrorCodeChannelProtocolMismatch 渠道协议能力不匹配：如 responses 有状态请求 + // （previous_response_id）落在仅支持 chat 协议的渠道上，无法经转换等价表达。 + // 归类渠道级致命（channel: 前缀规则），驱动调度 FSM 换渠道。 + ErrorCodeChannelProtocolMismatch ErrorCode = "channel:protocol_mismatch" // 客户端请求错误 ErrorCodeReadRequestBodyFailed ErrorCode = "read_request_body_failed" diff --git a/web/admin/src/views/monitor/ModelPerformancePage.vue b/web/admin/src/views/monitor/ModelPerformancePage.vue index c11169b..e2e1197 100644 --- a/web/admin/src/views/monitor/ModelPerformancePage.vue +++ b/web/admin/src/views/monitor/ModelPerformancePage.vue @@ -1,6 +1,7 @@