From 4c212aeb040bfb4680959dbec0dcf216fd6ef362 Mon Sep 17 00:00:00 2001 From: chengqian Date: Tue, 15 Sep 2026 19:20:15 +0800 Subject: [PATCH] =?UTF-8?q?feat(ui):=E4=BC=98=E5=8C=96=E5=9C=A8=E7=BA=BF?= =?UTF-8?q?=E4=BD=93=E9=AA=8C=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 1 + internal/logic/common/audit.go | 15 +- internal/logic/common/inline_media.go | 97 +++ internal/logic/common/inline_media_test.go | 119 ++++ internal/middleware/content_filter.go | 15 +- relay/handler/relay_handler_test.go | 57 ++ web/tenant/src/components/common/Icon.vue | 4 + .../src/views/service/PlaygroundPage.vue | 106 ++- .../service/playground/AttachmentBar.vue | 187 ++++++ .../src/views/service/playground/ChatTab.vue | 622 ++++++++++++++---- .../views/service/playground/PromptInput.vue | 226 +++++++ .../src/views/service/playground/VideoTab.vue | 68 +- .../service/playground/attachmentParts.ts | 120 ++++ .../service/playground/useAttachments.ts | 205 ++++++ 14 files changed, 1703 insertions(+), 139 deletions(-) create mode 100644 internal/logic/common/inline_media.go create mode 100644 internal/logic/common/inline_media_test.go create mode 100644 web/tenant/src/views/service/playground/AttachmentBar.vue create mode 100644 web/tenant/src/views/service/playground/PromptInput.vue create mode 100644 web/tenant/src/views/service/playground/attachmentParts.ts create mode 100644 web/tenant/src/views/service/playground/useAttachments.ts diff --git a/CLAUDE.md b/CLAUDE.md index 8573818..141cb34 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -473,6 +473,7 @@ cmd(路由注册) | 对比分析 | `docs/new-api-vs-sub2api对比分析报告.md` | 两者差异 | | 大模型实施方案 | `docs/大模型完整实施方案-v2.1.md` | 大模型代理层设计 | | API 格式参考 | `docs/reference/api-format-reference.md` | JSON 示例、SSE 格式、错误映射、中间件差异 | +| 协议转换语料采集 | `docs/协议转换语料采集手册.md` | `corpus-extract` 命令用法:从渠道调试日志提取真实流量语料喂给协议转换测试 | | GoFrame 使用规范 | `docs/reference/goframe-conventions.md` | 框架使用规范 + 已修复的框架 bug 记录 | diff --git a/internal/logic/common/audit.go b/internal/logic/common/audit.go index 721f72e..a6124c6 100644 --- a/internal/logic/common/audit.go +++ b/internal/logic/common/audit.go @@ -100,13 +100,18 @@ func GetTenantAuditLevel(ctx context.Context, tenantID int64) string { // ApplyAuditLevel 根据审计级别处理请求体和响应体,返回处理后的内容。 func ApplyAuditLevel(level string, requestBody, responseBody string, isStream bool, path string) (req, resp string) { - req = requestBody - resp = responseBody + // 多模态请求(对话内联图片/音频、视频参考图)的 base64 载荷先剥离成长度占位: + // multipart 体已在 relay handler 侧被挡成 "[binary body omitted]",但内联在 JSON + // 里的 data URL 会原样落库,单条就能把 aud_request_logs 撑到几 MB。 + // 此处是对话与异步任务两条审计链路的唯一收口点(DataProviderImpl.RecordAudit), + // 在这里剥离即可同时覆盖。none 级别直接丢弃,无需白做一次剥离。 + if level == AuditLevelNone { + return "", "" + } + req = StripInlineMedia(requestBody) + resp = StripInlineMedia(responseBody) switch level { - case AuditLevelNone: - req = "" - resp = "" case AuditLevelQuestionOnly: resp = "" case AuditLevelMasked: diff --git a/internal/logic/common/inline_media.go b/internal/logic/common/inline_media.go new file mode 100644 index 0000000..c0823f0 --- /dev/null +++ b/internal/logic/common/inline_media.go @@ -0,0 +1,97 @@ +package common + +import ( + "bytes" + "strconv" + "strings" +) + +// 内联媒体(data URL)处理工具。 +// +// 多模态请求体里的图片/音频以 `data:;base64,<载荷>` 的形式内联在 JSON 中, +// 单个载荷动辄数 MB。这类载荷对三条链路都是纯负担: +// - 预扣估算按字节折算会把一张图估成几十万 token,直接冻穿租户余额 +// - 审计落库会把整段 base64 灌进 aud_request_logs +// - 敏感词匹配白扫几 MB,还可能在 base64 串里误命中 +// +// 这里提供统一的扫描与剥离能力,三处共用,避免各写一份。 + +// inlineMediaMarker data URL 中 base64 载荷的起始标记。 +const inlineMediaMarker = ";base64," + +// inlineMediaMinPayload 认定为内联媒体的最小载荷长度。 +// 真实的图片/音频 base64 远大于此,设下限是为了不误伤正文里恰好出现该标记的短文本。 +const inlineMediaMinPayload = 64 + +// ScanInlineMediaBytes 扫描内容中的 data URL base64 载荷,返回载荷总字节数与媒体个数。 +// +// 纯字节扫描不做 JSON 反序列化——调用点在请求热路径上,且请求体可能有数 MB。 +// base64 字母表不含 `"` 与 `\`,故以二者作为 JSON 字符串内载荷的终止符。 +func ScanInlineMediaBytes(b []byte) (payloadBytes, count int) { + marker := []byte(inlineMediaMarker) + rest := b + for { + idx := bytes.Index(rest, marker) + if idx < 0 { + return + } + rest = rest[idx+len(marker):] + end := bytes.IndexAny(rest, "\"\\") + if end < 0 { + end = len(rest) + } + if end >= inlineMediaMinPayload { + payloadBytes += end + count++ + } + rest = rest[end:] + } +} + +// StripInlineMediaBytes 是 StripInlineMedia 的字节切片版本。 +// 未命中内联媒体时原样返回入参切片,不产生任何拷贝——请求热路径上的绝大多数 +// 请求体是纯文本,这条快路径让它们零开销通过。 +func StripInlineMediaBytes(b []byte) []byte { + if !bytes.Contains(b, []byte(inlineMediaMarker)) { + return b + } + return []byte(StripInlineMedia(string(b))) +} + +// StripInlineMedia 把 base64 载荷替换成长度占位,保留 mime 前缀便于排查: +// +// data:image/png;base64,iVBORw0KGgo... → data:image/png;base64,<省略 2097152 字节> +// +// 未命中任何内联媒体时原样返回,不产生额外分配。 +func StripInlineMedia(s string) string { + if !strings.Contains(s, inlineMediaMarker) { + return s + } + + var sb strings.Builder + rest := s + for { + idx := strings.Index(rest, inlineMediaMarker) + if idx < 0 { + sb.WriteString(rest) + break + } + head := idx + len(inlineMediaMarker) + sb.WriteString(rest[:head]) + rest = rest[head:] + + end := strings.IndexAny(rest, "\"\\") + if end < 0 { + end = len(rest) + } + if end >= inlineMediaMinPayload { + sb.WriteString("<省略 ") + sb.WriteString(strconv.Itoa(end)) + sb.WriteString(" 字节>") + } else { + sb.WriteString(rest[:end]) + } + rest = rest[end:] + } + return sb.String() +} diff --git a/internal/logic/common/inline_media_test.go b/internal/logic/common/inline_media_test.go new file mode 100644 index 0000000..b5b6a8f --- /dev/null +++ b/internal/logic/common/inline_media_test.go @@ -0,0 +1,119 @@ +package common + +import ( + "strings" + "testing" +) + +// bigPayload 构造一段长度足够被认定为内联媒体的 base64 载荷。 +func bigPayload(n int) string { + return strings.Repeat("A", n) +} + +func TestScanInlineMediaBytes(t *testing.T) { + payload := bigPayload(400) + cases := []struct { + name string + body string + wantBytes int + wantCount int + }{ + {"纯文本", `{"model":"gpt-4o","messages":[{"role":"user","content":"你好"}]}`, 0, 0}, + {"空体", "", 0, 0}, + { + "单图", + `{"content":[{"type":"image_url","image_url":{"url":"data:image/png;base64,` + payload + `"}}]}`, + 400, 1, + }, + { + "双图", + `{"a":"data:image/png;base64,` + payload + `","b":"data:image/jpeg;base64,` + payload + `"}`, + 800, 2, + }, + { + "音频", + `{"input_audio":{"data":"data:audio/mp3;base64,` + payload + `","format":"mp3"}}`, + 400, 1, + }, + // 载荷过短:正文里恰好出现该标记,不应被当作媒体扣减 + {"短载荷不计入", `{"text":"写法是 ;base64,abc 这样"}`, 0, 0}, + // 无结束引号(截断的请求体):载荷取到末尾 + {"未闭合", `{"url":"data:image/png;base64,` + payload, 400, 1}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + gotBytes, gotCount := ScanInlineMediaBytes([]byte(c.body)) + if gotBytes != c.wantBytes || gotCount != c.wantCount { + t.Errorf("ScanInlineMediaBytes() = (%d, %d), want (%d, %d)", + gotBytes, gotCount, c.wantBytes, c.wantCount) + } + }) + } +} + +func TestStripInlineMedia(t *testing.T) { + payload := bigPayload(400) + + t.Run("剥离载荷保留 mime 前缀", func(t *testing.T) { + in := `{"url":"data:image/png;base64,` + payload + `","prompt":"描述这张图"}` + got := StripInlineMedia(in) + if strings.Contains(got, payload) { + t.Error("base64 载荷未被剥离") + } + if !strings.Contains(got, "data:image/png;base64,<省略 400 字节>") { + t.Errorf("占位文案不符: %s", got) + } + if !strings.Contains(got, `"prompt":"描述这张图"`) { + t.Error("载荷之外的内容被破坏") + } + }) + + t.Run("纯文本原样返回", func(t *testing.T) { + in := `{"model":"gpt-4o","messages":[{"role":"user","content":"你好"}]}` + if got := StripInlineMedia(in); got != in { + t.Errorf("StripInlineMedia() = %q, want 原样返回", got) + } + }) + + t.Run("多个载荷全部剥离", func(t *testing.T) { + in := `{"a":"data:image/png;base64,` + payload + `","b":"data:image/jpeg;base64,` + payload + `"}` + got := StripInlineMedia(in) + if strings.Contains(got, payload) { + t.Error("仍残留 base64 载荷") + } + if n := strings.Count(got, "<省略 400 字节>"); n != 2 { + t.Errorf("占位数量 = %d, want 2", n) + } + }) + + t.Run("短载荷保持原样", func(t *testing.T) { + in := `{"text":"写法是 ;base64,abc 这样"}` + if got := StripInlineMedia(in); got != in { + t.Errorf("StripInlineMedia() = %q, want 原样返回", got) + } + }) + + t.Run("剥离后可被字节版复用且无拷贝", func(t *testing.T) { + plain := []byte(`{"content":"纯文本"}`) + got := StripInlineMediaBytes(plain) + if &got[0] != &plain[0] { + t.Error("纯文本路径应原样返回入参切片,不做拷贝") + } + }) +} + +func TestApplyAuditLevelStripsInlineMedia(t *testing.T) { + payload := bigPayload(400) + body := `{"url":"data:image/png;base64,` + payload + `"}` + + req, _ := ApplyAuditLevel(AuditLevelFull, body, "", false, "/v1/chat/completions") + if strings.Contains(req, payload) { + t.Error("full 级别仍把 base64 载荷落库") + } + + req, resp := ApplyAuditLevel(AuditLevelNone, body, body, false, "/v1/chat/completions") + if req != "" || resp != "" { + t.Error("none 级别应返回空") + } +} diff --git a/internal/middleware/content_filter.go b/internal/middleware/content_filter.go index 458f63a..b86f645 100644 --- a/internal/middleware/content_filter.go +++ b/internal/middleware/content_filter.go @@ -94,7 +94,12 @@ func ContentFilter(r *ghttp.Request) { return } - result := common.ContentFilter().Check(body) + // 多模态请求体里的 base64 内联媒体先剥离再匹配:几 MB 的 base64 白扫一遍纯属浪费, + // 且 base64 字母表可能恰好拼出敏感词,导致正常的带图请求被误 block。 + // 纯文本请求体原样返回,不额外拷贝。 + checkBody := common.StripInlineMediaBytes(body) + + result := common.ContentFilter().Check(checkBody) if !result.Matched { r.Middleware.Next() return @@ -105,7 +110,7 @@ func ContentFilter(r *ghttp.Request) { r.SetCtxVar(CtxKeyContentFilterWords, result.MatchedWords) // Queue async log write - queueContentFilterLog(r, mode, result.MatchedWords, string(body)) + queueContentFilterLog(r, mode, result.MatchedWords, string(checkBody)) switch mode { case "log": @@ -115,8 +120,10 @@ func ContentFilter(r *ghttp.Request) { case "replace": g.Log().Infof(r.Context(), "[ContentFilter] replaced words: %v", result.MatchedWords) r.SetCtxVar(CtxKeyContentFilterReplaced, true) - r.SetCtxVar(CtxKeyContentFilterOriginal, string(body)) - r.SetCtxVar(CtxKeyContentFilterFiltered, result.FilteredText) + r.SetCtxVar(CtxKeyContentFilterOriginal, string(checkBody)) + // 替换后的请求体必须作用在原始 body 上——剥离版本丢了内联媒体, + // 一旦被下游拿去改写上游请求就会把图片/音频弄没。命中才多跑这一次,代价可接受。 + r.SetCtxVar(CtxKeyContentFilterFiltered, common.ContentFilter().Check(body).FilteredText) r.Middleware.Next() case "block": diff --git a/relay/handler/relay_handler_test.go b/relay/handler/relay_handler_test.go index 4b5b17c..aa58234 100644 --- a/relay/handler/relay_handler_test.go +++ b/relay/handler/relay_handler_test.go @@ -2,6 +2,7 @@ package handler import ( "context" + "strings" "testing" "github.com/qianfree/team-api/relay/common" @@ -60,3 +61,59 @@ func TestResolveRelayModel(t *testing.T) { } }) } + +// TestEstimateInputTokens 多模态请求体的预扣估算:base64 内联媒体必须按固定成本计入, +// 不能按字节折算——一张 2MB 的图按 len/4 会被估成约 70 万 token,预扣直接冻穿余额。 +func TestEstimateInputTokens(t *testing.T) { + // 模拟 2MB 的 base64 图片载荷 + payload := strings.Repeat("A", 2<<20) + + cases := []struct { + name string + body string + want int + }{ + { + "纯文本按字节折算", + strings.Repeat("x", 400), + 100, + }, + { + "空体", + "", + 0, + }, + { + // 文本骨架 40 字节 + 1 个媒体 + "单图不按字节折算", + `{"url":"data:image/png;base64,` + payload + `"}`, + (len(`{"url":"data:image/png;base64,`+`"}`))/4 + inlineMediaTokenCost, + }, + { + "双图按个数累加", + `{"a":"data:image/png;base64,` + payload + `","b":"data:image/png;base64,` + payload + `"}`, + (len(`{"a":"data:image/png;base64,`+`","b":"data:image/png;base64,`+`"}`))/4 + 2*inlineMediaTokenCost, + }, + { + // 畸形 data URL(无 base64 标记)退化为纯文本路径 + "畸形 data URL", + `{"url":"data:image/png,` + strings.Repeat("A", 400) + `"}`, + len(`{"url":"data:image/png,`+`"}`)/4 + 100, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := estimateInputTokens([]byte(c.body)); got != c.want { + t.Errorf("estimateInputTokens() = %d, want %d", got, c.want) + } + }) + } + + // 回归保护:2MB 图片的估算必须远低于按字节折算的量级 + naive := len(`{"url":"data:image/png;base64,`+payload+`"}`) / 4 + got := estimateInputTokens([]byte(`{"url":"data:image/png;base64,` + payload + `"}`)) + if got >= naive/10 { + t.Errorf("2MB 图片估算 %d tokens,与按字节折算的 %d 处于同一量级,预扣仍会冻穿余额", got, naive) + } +} diff --git a/web/tenant/src/components/common/Icon.vue b/web/tenant/src/components/common/Icon.vue index 2d347ca..815a98d 100644 --- a/web/tenant/src/components/common/Icon.vue +++ b/web/tenant/src/components/common/Icon.vue @@ -80,6 +80,10 @@ const icons: Record = { users: 'M15 19.128a9.38 9.38 0 002.625.372 9.337 9.337 0 004.121-.952 4.125 4.125 0 00-7.533-2.493M15 19.128v-.003c0-1.113-.285-2.16-.786-3.07M15 19.128v.106A12.318 12.318 0 018.624 21c-2.331 0-4.512-.645-6.374-1.766l-.001-.109a6.375 6.375 0 0111.964-3.07M12 6.375a3.375 3.375 0 11-6.75 0 3.375 3.375 0 016.75 0zm8.25 2.25a2.625 2.625 0 11-5.25 0 2.625 2.625 0 015.25 0z', // Files & Documents + photo: 'M2.25 15.75l5.159-5.159a2.25 2.25 0 013.182 0l5.159 5.159m-1.5-1.5l1.409-1.409a2.25 2.25 0 013.182 0l2.909 2.909m-18 3.75h16.5a1.5 1.5 0 001.5-1.5V6a1.5 1.5 0 00-1.5-1.5H3.75A1.5 1.5 0 002.25 6v12a1.5 1.5 0 001.5 1.5zm10.5-11.25h.008v.008h-.008V8.25zm.375 0a.375.375 0 11-.75 0 .375.375 0 01.75 0z', + paperclip: 'M18.375 12.739l-7.693 7.693a4.5 4.5 0 01-6.364-6.364l10.94-10.94A3 3 0 1119.5 7.372L8.552 18.32m.009-.01l-.01.01m5.699-9.941l-7.81 7.81a1.5 1.5 0 002.112 2.13', + musicalNote: 'M9 9l10.5-3m0 6.553v3.75a2.25 2.25 0 01-1.632 2.163l-1.32.377a1.803 1.803 0 11-.99-3.467l2.31-.66a2.25 2.25 0 001.632-2.163zm0 0V2.25L9 5.25v10.303m0 0v3.75a2.25 2.25 0 01-1.632 2.163l-1.32.377a1.803 1.803 0 01-.99-3.467l2.31-.66A2.25 2.25 0 009 15.553z', + film: 'M3.375 19.5h17.25m-17.25 0a1.125 1.125 0 01-1.125-1.125M3.375 19.5h7.5c.621 0 1.125-.504 1.125-1.125m-9.75 0V5.625m0 12.75v-1.5c0-.621.504-1.125 1.125-1.125m18.375 2.625V5.625m0 12.75c0 .621-.504 1.125-1.125 1.125m1.125-1.125v-1.5c0-.621-.504-1.125-1.125-1.125m0 3.75h-7.5A1.125 1.125 0 0112 18.375m9.75-12.75c0-.621-.504-1.125-1.125-1.125H3.375c-.621 0-1.125.504-1.125 1.125m19.5 0v1.5c0 .621-.504 1.125-1.125 1.125M2.25 5.625v1.5c0 .621.504 1.125 1.125 1.125m0 0h17.25m-17.25 0h7.5c.621 0 1.125.504 1.125 1.125M3.375 8.25c-.621 0-1.125.504-1.125 1.125v1.5c0 .621.504 1.125 1.125 1.125m17.25-3.75h-7.5c-.621 0-1.125.504-1.125 1.125m8.625-1.125c.621 0 1.125.504 1.125 1.125v1.5c0 .621-.504 1.125-1.125 1.125m-17.25 0h7.5m-7.5 0c-.621 0-1.125.504-1.125 1.125v1.5c0 .621.504 1.125 1.125 1.125M12 10.875v-1.5m0 1.5c0 .621-.504 1.125-1.125 1.125M12 10.875c0 .621.504 1.125 1.125 1.125m-2.25 0c.621 0 1.125.504 1.125 1.125M13.125 12h7.5m-7.5 0c-.621 0-1.125.504-1.125 1.125M20.625 12c.621 0 1.125.504 1.125 1.125v1.5c0 .621-.504 1.125-1.125 1.125m-17.25 0h7.5M12 14.625v-1.5m0 1.5c0 .621-.504 1.125-1.125 1.125M12 14.625c0 .621.504 1.125 1.125 1.125m-2.25 0c.621 0 1.125.504 1.125 1.125m0 1.5v-1.5m0 0c0-.621.504-1.125 1.125-1.125m0 0h7.5', document: 'M9 12h6m-6 4h6m2 5H7a2 2 0 01-2-2V5a2 2 0 012-2h5.586a1 1 0 01.707.293l5.414 5.414a1 1 0 01.293.707V19a2 2 0 01-2 2z', bookOpen: 'M12 6.042A8.967 8.967 0 006 3.75c-1.052 0-2.062.18-3 .512v14.25A8.987 8.987 0 016 18c2.305 0 4.408.867 6 2.292m0-14.25a8.966 8.966 0 016-2.292c1.052 0 2.062.18 3 .512v14.25A8.987 8.987 0 0018 18a8.967 8.967 0 00-6 2.292m0-14.25v14.25', clipboard: 'M9 12h3.75M9 15h3.75M9 18h3.75m3 .75H18a2.25 2.25 0 002.25-2.25V6.108c0-1.135-.845-2.098-1.976-2.192a48.424 48.424 0 00-1.123-.08m-5.801 0c-.065.21-.1.433-.1.664 0 .414.336.75.75.75h4.5a.75.75 0 00.75-.75 2.25 2.25 0 00-.1-.664m-5.8 0A2.251 2.251 0 0113.5 2.25H15c1.012 0 1.867.668 2.15 1.586m-5.8 0c-.376.023-.75.05-1.124.08C9.095 4.01 8.25 4.973 8.25 6.108V8.25m0 0H4.875c-.621 0-1.125.504-1.125 1.125v11.25c0 .621.504 1.125 1.125 1.125h9.75c.621 0 1.125-.504 1.125-1.125V9.375c0-.621-.504-1.125-1.125-1.125H8.25z', diff --git a/web/tenant/src/views/service/PlaygroundPage.vue b/web/tenant/src/views/service/PlaygroundPage.vue index 6b9c67f..4ff0494 100644 --- a/web/tenant/src/views/service/PlaygroundPage.vue +++ b/web/tenant/src/views/service/PlaygroundPage.vue @@ -23,15 +23,18 @@ interface ModelItem { async_image?: boolean // 图片模型同步端点是否可用(「仅异步」厂商为 false) image_sync_supported?: boolean + // 模型能力特性 JSON(如 {"vision":true,"audio_input":true}),用于门控附件上传 + capabilities?: string } +// 每个 Tab 的图标,用于分段控件左侧的视觉锚点 const tabs = [ - { key: 'chat', label: '对话' }, - { key: 'image', label: '图像' }, - { key: 'video', label: '视频' }, - { key: 'audio', label: '语音' }, - { key: 'embedding', label: '嵌入' }, - { key: 'rerank', label: '重排' }, + { key: 'chat', label: '对话', icon: 'chat' }, + { key: 'image', label: '图像', icon: 'photo' }, + { key: 'video', label: '视频', icon: 'film' }, + { key: 'audio', label: '语音', icon: 'musicalNote' }, + { key: 'embedding', label: '嵌入', icon: 'cube' }, + { key: 'rerank', label: '重排', icon: 'chart' }, ] const activeTab = ref('chat') @@ -59,6 +62,13 @@ const modelsLoading = ref(false) let loadingTimer: number | null = null let loadSeq = 0 +// 当前 Key 的显示名与掩码前缀,供工具栏的 Key 胶囊展示 +const selectedKeyLabel = computed(() => { + const key = apiKeys.value.find(k => k.id === selectedKeyId.value) + if (!key) return '' + return `${key.name} · ${key.key_prefix}···` +}) + // 按右上角选中的 API Key 拉取其可用模型,使左侧模型列表跟随 Key 切换。 // 切换期间保留旧列表避免空白闪烁;加载提示延迟 200ms 才出现,请求更快时 // 完全不显示,杜绝 spinner 瞬闪;loadSeq 丢弃过期响应,防止快速切换 Key 时旧结果覆盖。 @@ -93,15 +103,19 @@ watch(selectedKeyId, loadModels, { immediate: true })