fix(core): use rune-based splitting in splitMessage to prevent invalid UTF-8
splitMessage used byte-length (len(s)) and byte-slicing (s[:n]) to split long messages into platform-sized chunks. When the text contains multi-byte UTF-8 characters (Chinese, Japanese, emoji, etc.), a byte-level split can cut in the middle of a character, producing invalid UTF-8 sequences that cause garbled output on messaging platforms. Switch to []rune-based length checks and slicing so splits always occur at character boundaries. Also fix the newline-boundary search to correctly convert byte offsets to rune offsets in the mixed-content case. Add regression tests covering CJK, emoji, and newline-split scenarios.
This commit is contained in:
1 parent
bd5ce073c8
commit
fc0e8e790f
2 files changed
+90
-11
No files matched your search
+16
-11
@@ -6476,28 +6476,33 @@ func truncateIf(s string, maxLen int) string {
|
||||
}
|
||||
|
||||
func splitMessage(text string, maxLen int) []string {
|
||||
if len(text) <= maxLen {
|
||||
runes := []rune(text)
|
||||
if len(runes) <= maxLen {
|
||||
return []string{text}
|
||||
}
|
||||
var chunks []string
|
||||
|
||||
for len(text) > 0 {
|
||||
if len(text) <= maxLen {
|
||||
chunks = append(chunks, text)
|
||||
for len(runes) > 0 {
|
||||
if len(runes) <= maxLen {
|
||||
chunks = append(chunks, string(runes))
|
||||
break
|
||||
}
|
||||
|
||||
end := maxLen
|
||||
|
||||
// Try to split at newline boundary
|
||||
if idx := strings.LastIndex(text[:end], "\n"); idx > 0 && idx >= end/2 {
|
||||
end = idx + 1
|
||||
// Try to split at newline boundary within the rune window.
|
||||
// Convert the candidate chunk back to a string for newline search.
|
||||
candidate := string(runes[:end])
|
||||
if idx := strings.LastIndex(candidate, "\n"); idx > 0 {
|
||||
// idx is a byte offset within candidate; convert to rune offset.
|
||||
runeIdx := utf8.RuneCountInString(candidate[:idx])
|
||||
if runeIdx >= end/2 {
|
||||
end = runeIdx + 1
|
||||
}
|
||||
}
|
||||
|
||||
chunk := text[:end]
|
||||
text = text[end:]
|
||||
|
||||
chunks = append(chunks, chunk)
|
||||
chunks = append(chunks, string(runes[:end]))
|
||||
runes = runes[end:]
|
||||
}
|
||||
return chunks
|
||||
}
|
||||
|
||||
@@ -2441,3 +2441,77 @@ func TestStaleGoroutineCleanup_RaceSimulation(t *testing.T) {
|
||||
t.Fatal("replacement agent session was killed by stale cleanup")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitMessageUTF8Safety(t *testing.T) {
|
||||
t.Run("ASCII short", func(t *testing.T) {
|
||||
result := splitMessage("hello", 10)
|
||||
if len(result) != 1 || result[0] != "hello" {
|
||||
t.Fatalf("expected single chunk 'hello', got %v", result)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CJK characters split at rune boundary", func(t *testing.T) {
|
||||
// 10 CJK characters (each 3 bytes in UTF-8), total 30 bytes
|
||||
input := "你好世界测试一二三四"
|
||||
if len([]rune(input)) != 10 {
|
||||
t.Fatalf("expected 10 runes, got %d", len([]rune(input)))
|
||||
}
|
||||
// maxLen=5 runes should split into 2 chunks of 5 runes each
|
||||
chunks := splitMessage(input, 5)
|
||||
if len(chunks) != 2 {
|
||||
t.Fatalf("expected 2 chunks, got %d: %v", len(chunks), chunks)
|
||||
}
|
||||
if chunks[0] != "你好世界测" {
|
||||
t.Errorf("chunk[0] = %q, want %q", chunks[0], "你好世界测")
|
||||
}
|
||||
if chunks[1] != "试一二三四" {
|
||||
t.Errorf("chunk[1] = %q, want %q", chunks[1], "试一二三四")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("emoji split at rune boundary", func(t *testing.T) {
|
||||
// Emoji: 4 bytes each in UTF-8
|
||||
input := "😀😁😂🤣😄😅"
|
||||
runes := []rune(input)
|
||||
if len(runes) != 6 {
|
||||
t.Fatalf("expected 6 runes, got %d", len(runes))
|
||||
}
|
||||
chunks := splitMessage(input, 3)
|
||||
if len(chunks) != 2 {
|
||||
t.Fatalf("expected 2 chunks, got %d: %v", len(chunks), chunks)
|
||||
}
|
||||
if chunks[0] != "😀😁😂" {
|
||||
t.Errorf("chunk[0] = %q, want %q", chunks[0], "😀😁😂")
|
||||
}
|
||||
if chunks[1] != "🤣😄😅" {
|
||||
t.Errorf("chunk[1] = %q, want %q", chunks[1], "🤣😄😅")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("prefers newline split", func(t *testing.T) {
|
||||
input := "abcde\nfghij"
|
||||
chunks := splitMessage(input, 8)
|
||||
if len(chunks) != 2 {
|
||||
t.Fatalf("expected 2 chunks, got %d: %v", len(chunks), chunks)
|
||||
}
|
||||
// Should split at newline (rune index 5), which is >= 8/2=4
|
||||
if chunks[0] != "abcde\n" {
|
||||
t.Errorf("chunk[0] = %q, want %q", chunks[0], "abcde\n")
|
||||
}
|
||||
if chunks[1] != "fghij" {
|
||||
t.Errorf("chunk[1] = %q, want %q", chunks[1], "fghij")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CJK with newline split", func(t *testing.T) {
|
||||
input := "你好\n世界测试一二三四"
|
||||
chunks := splitMessage(input, 5)
|
||||
if len(chunks) < 2 {
|
||||
t.Fatalf("expected at least 2 chunks, got %d: %v", len(chunks), chunks)
|
||||
}
|
||||
// First chunk should split at the newline
|
||||
if chunks[0] != "你好\n" {
|
||||
t.Errorf("chunk[0] = %q, want %q", chunks[0], "你好\n")
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in new issue
Block a user