fix(core): use rune-based splitting in splitMessage to prevent invalid UTF-8

splitMessage used byte-length (len(s)) and byte-slicing (s[:n]) to split
long messages into platform-sized chunks. When the text contains multi-byte
UTF-8 characters (Chinese, Japanese, emoji, etc.), a byte-level split can
cut in the middle of a character, producing invalid UTF-8 sequences that
cause garbled output on messaging platforms.

Switch to []rune-based length checks and slicing so splits always occur
at character boundaries. Also fix the newline-boundary search to correctly
convert byte offsets to rune offsets in the mixed-content case.

Add regression tests covering CJK, emoji, and newline-split scenarios.
This commit is contained in:
kevinWangSheng committed 2026-03-13 14:05:42 -07:00
1 parent bd5ce073c8
commit fc0e8e790f
2 files changed
+90 -11

No files matched your search

+16 -11
View File
@@ -6476,28 +6476,33 @@ func truncateIf(s string, maxLen int) string {
}
func splitMessage(text string, maxLen int) []string {
if len(text) <= maxLen {
runes := []rune(text)
if len(runes) <= maxLen {
return []string{text}
}
var chunks []string
for len(text) > 0 {
if len(text) <= maxLen {
chunks = append(chunks, text)
for len(runes) > 0 {
if len(runes) <= maxLen {
chunks = append(chunks, string(runes))
break
}
end := maxLen
// Try to split at newline boundary
if idx := strings.LastIndex(text[:end], "\n"); idx > 0 && idx >= end/2 {
end = idx + 1
// Try to split at newline boundary within the rune window.
// Convert the candidate chunk back to a string for newline search.
candidate := string(runes[:end])
if idx := strings.LastIndex(candidate, "\n"); idx > 0 {
// idx is a byte offset within candidate; convert to rune offset.
runeIdx := utf8.RuneCountInString(candidate[:idx])
if runeIdx >= end/2 {
end = runeIdx + 1
}
}
chunk := text[:end]
text = text[end:]
chunks = append(chunks, chunk)
chunks = append(chunks, string(runes[:end]))
runes = runes[end:]
}
return chunks
}
+74
View File
@@ -2441,3 +2441,77 @@ func TestStaleGoroutineCleanup_RaceSimulation(t *testing.T) {
t.Fatal("replacement agent session was killed by stale cleanup")
}
}
func TestSplitMessageUTF8Safety(t *testing.T) {
t.Run("ASCII short", func(t *testing.T) {
result := splitMessage("hello", 10)
if len(result) != 1 || result[0] != "hello" {
t.Fatalf("expected single chunk 'hello', got %v", result)
}
})
t.Run("CJK characters split at rune boundary", func(t *testing.T) {
// 10 CJK characters (each 3 bytes in UTF-8), total 30 bytes
input := "你好世界测试一二三四"
if len([]rune(input)) != 10 {
t.Fatalf("expected 10 runes, got %d", len([]rune(input)))
}
// maxLen=5 runes should split into 2 chunks of 5 runes each
chunks := splitMessage(input, 5)
if len(chunks) != 2 {
t.Fatalf("expected 2 chunks, got %d: %v", len(chunks), chunks)
}
if chunks[0] != "你好世界测" {
t.Errorf("chunk[0] = %q, want %q", chunks[0], "你好世界测")
}
if chunks[1] != "试一二三四" {
t.Errorf("chunk[1] = %q, want %q", chunks[1], "试一二三四")
}
})
t.Run("emoji split at rune boundary", func(t *testing.T) {
// Emoji: 4 bytes each in UTF-8
input := "😀😁😂🤣😄😅"
runes := []rune(input)
if len(runes) != 6 {
t.Fatalf("expected 6 runes, got %d", len(runes))
}
chunks := splitMessage(input, 3)
if len(chunks) != 2 {
t.Fatalf("expected 2 chunks, got %d: %v", len(chunks), chunks)
}
if chunks[0] != "😀😁😂" {
t.Errorf("chunk[0] = %q, want %q", chunks[0], "😀😁😂")
}
if chunks[1] != "🤣😄😅" {
t.Errorf("chunk[1] = %q, want %q", chunks[1], "🤣😄😅")
}
})
t.Run("prefers newline split", func(t *testing.T) {
input := "abcde\nfghij"
chunks := splitMessage(input, 8)
if len(chunks) != 2 {
t.Fatalf("expected 2 chunks, got %d: %v", len(chunks), chunks)
}
// Should split at newline (rune index 5), which is >= 8/2=4
if chunks[0] != "abcde\n" {
t.Errorf("chunk[0] = %q, want %q", chunks[0], "abcde\n")
}
if chunks[1] != "fghij" {
t.Errorf("chunk[1] = %q, want %q", chunks[1], "fghij")
}
})
t.Run("CJK with newline split", func(t *testing.T) {
input := "你好\n世界测试一二三四"
chunks := splitMessage(input, 5)
if len(chunks) < 2 {
t.Fatalf("expected at least 2 chunks, got %d: %v", len(chunks), chunks)
}
// First chunk should split at the newline
if chunks[0] != "你好\n" {
t.Errorf("chunk[0] = %q, want %q", chunks[0], "你好\n")
}
})
}