diff --git a/internal/handler/relay/response_capture.go b/internal/handler/relay/response_capture.go index 09e3585..8bb6fb0 100644 --- a/internal/handler/relay/response_capture.go +++ b/internal/handler/relay/response_capture.go @@ -1,10 +1,12 @@ package relay import ( + "bytes" "net/http" "strings" "sync" "sync/atomic" + "unicode/utf8" ) const maxCaptureSize = 1 << 18 // 256KB total capture budget @@ -58,8 +60,15 @@ func (w *ResponseCaptureWriter) Write(b []byte) (int, error) { if len(b) <= remaining { w.buf = append(w.buf, b...) } else { - w.buf = append(w.buf, b[:remaining]...) - w.appendToTail(b[remaining:]) + // 切点回退到多字节字符的起始字节(UTF-8 最长 4 字节,最多回退 3 字节), + // 整字符归 tail,避免 head 末尾留下半个字符产生无效 UTF-8。 + // 跨 Write 到达的割裂字符无法在此对齐,由 Body() 出口清洗兜底。 + cut := remaining + for cut > 0 && remaining-cut < 3 && !utf8.RuneStart(b[cut]) { + cut-- + } + w.buf = append(w.buf, b[:cut]...) + w.appendToTail(b[cut:]) } } else { w.appendToTail(b) @@ -96,22 +105,31 @@ func (w *ResponseCaptureWriter) Flush() { // Body 返回已捕获的响应体内容。 // 短响应(<= headLimit)直接返回;长响应保留 head + truncation marker + tail。 +// 两个无效 UTF-8 来源都在出口兜底:分块到达的流可能在任意字节边界切开多字节字符 +// (跨 Write 的割裂无法在写入侧对齐),上游也可能直接输出非法字节;不清洗会导致 +// 审计日志 INSERT 被 UTF8 编码的 PostgreSQL 以 invalid byte sequence 拒绝、整条丢失。 func (w *ResponseCaptureWriter) Body() string { w.mu.Lock() defer w.mu.Unlock() if w.tailBuf == nil { - return string(w.buf) + return strings.ToValidUTF8(string(w.buf), string(utf8.RuneError)) } var sb strings.Builder sb.Write(w.buf) sb.WriteString("\n...[truncated]...\n") if w.tailWrapped { - sb.Write(w.tailBuf[w.tailPos:]) + // 环形缓冲区起点落在残行中间:残行前半已被覆盖、不可恢复, + // 跳到下一行行首,让 tail 从完整 SSE 行开始;找不到换行则原样输出。 + tail := w.tailBuf[w.tailPos:] + if idx := bytes.IndexByte(tail, '\n'); idx >= 0 { + tail = tail[idx+1:] + } + sb.Write(tail) sb.Write(w.tailBuf[:w.tailPos]) } else { sb.Write(w.tailBuf[:w.tailPos]) } - return sb.String() + return strings.ToValidUTF8(sb.String(), string(utf8.RuneError)) } // StatusCode 返回捕获的 HTTP 状态码 diff --git a/internal/handler/relay/response_capture_test.go b/internal/handler/relay/response_capture_test.go index 34f822c..04a93af 100644 --- a/internal/handler/relay/response_capture_test.go +++ b/internal/handler/relay/response_capture_test.go @@ -3,7 +3,9 @@ package relay import ( "net/http" "net/http/httptest" + "strings" "testing" + "unicode/utf8" ) func TestResponseCaptureWriterCommitted(t *testing.T) { @@ -46,3 +48,71 @@ func TestResponseCaptureWriterKeepsFirstStatus(t *testing.T) { t.Fatalf("status = %d, want first status %d", got, http.StatusAccepted) } } + +// 捕获体最终要写入 UTF8 编码的 PostgreSQL,任何无效 UTF-8 字节都会让审计 INSERT +// 整条被拒。以下用例覆盖合法短响应、上游脏字节、head/tail 分界、环形回绕等边界, +// 保证 Body() 恒返回合法 UTF-8。 +func TestResponseCaptureBodyRoundTrip(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte("data: 你好\n\n")) + if got := capture.Body(); got != "data: 你好\n\n" { + t.Fatalf("short valid body should round-trip, got %q", got) + } +} + +// 上游直接输出非法字节(供应商 bug、非 UTF-8 错误页),出口清洗兜底 +func TestResponseCaptureBodySanitizesInvalidUpstreamBytes(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte("ok \xe4\xb8")) // 半个“中” + got := capture.Body() + if !utf8.ValidString(got) { + t.Fatalf("body should be valid UTF-8, got %q", got) + } + if !strings.ContainsRune(got, utf8.RuneError) { + t.Fatalf("invalid sequence should be replaced with U+FFFD, got %q", got) + } +} + +// head/tail 分界点落在多字节字符中间时,整字符应归 tail,不留半个字符 +func TestResponseCaptureHeadTailSplitKeepsCharIntact(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte(strings.Repeat("a", headLimit-2))) + _, _ = capture.Write([]byte("中xyz")) + got := capture.Body() + if !utf8.ValidString(got) { + t.Fatal("body should be valid UTF-8 after boundary split") + } + if !strings.HasSuffix(got, "中xyz") { + t.Fatalf("char straddling the boundary should stay intact in tail, got %q", got[len(got)-20:]) + } +} + +// 环形缓冲区回绕后起点落在字符中间(且无换行可对齐),仅靠出口清洗兜底 +func TestResponseCaptureRingWrapBodyValid(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte(strings.Repeat("a", headLimit))) + // 21846 个“中”共 65538 字节,回绕后 tailPos=2,环起点落在“中”的最后一个字节上 + _, _ = capture.Write([]byte(strings.Repeat("中", 21846))) + if got := capture.Body(); !utf8.ValidString(got) { + t.Fatal("body should be valid UTF-8 after ring wrap") + } +} + +// 环形回绕起点落在残行中间时应跳到下一行行首,tail 从完整 SSE 行开始 +func TestResponseCaptureRingWrapAlignsToLineStart(t *testing.T) { + capture := NewResponseCaptureWriter(httptest.NewRecorder()) + _, _ = capture.Write([]byte(strings.Repeat("a", headLimit))) + // “d:中中\n”每行 9 字节,14564 行共 131076 字节,回绕后 tailPos=4(落在行首“中”中间) + _, _ = capture.Write([]byte(strings.Repeat("d:中中\n", 14564))) + body := capture.Body() + if !utf8.ValidString(body) { + t.Fatal("body should be valid UTF-8 after ring wrap") + } + prefix := strings.Repeat("a", headLimit) + "\n...[truncated]...\n" + if !strings.HasPrefix(body, prefix) { + t.Fatal("body should start with head + truncation marker") + } + if rest := body[len(prefix):]; !strings.HasPrefix(rest, "d:") { + t.Fatalf("tail should start at a complete line, got %q", rest) + } +} diff --git a/internal/logic/relay/provider.go b/internal/logic/relay/provider.go index a7f7e0c..f9fc2c7 100644 --- a/internal/logic/relay/provider.go +++ b/internal/logic/relay/provider.go @@ -640,6 +640,9 @@ func (p *DataProviderImpl) UpdateTaskAudit(ctx context.Context, record *common.A } func truncateBody(s string, maxLen int) string { + // 请求/响应体可能携带无效 UTF-8 字节(捕获侧按字节切割、上游/客户端原始输出), + // 落库前统一清洗为合法 UTF-8,避免 INSERT 被 UTF8 编码的 PostgreSQL 拒绝、整条记录丢失。 + s = strings.ToValidUTF8(s, string(utf8.RuneError)) // maxLen <= 0 表示不截断(配置了独立审计库时完整记录) if maxLen <= 0 || len(s) <= maxLen { return s