From 80da46be77f8b4c9aefabc24d4ece52084d4e072 Mon Sep 17 00:00:00 2001 From: Rael Date: Thu, 11 Jun 2026 00:06:04 +0800 Subject: [PATCH] fix(wecom): split long messages at semantic boundaries (#963) --- platform/wecom/message_split.go | 125 +++++++++++++++++++++++++++++++ platform/wecom/websocket_test.go | 108 ++++++++++++++++++++++++++ platform/wecom/wecom.go | 24 ------ 3 files changed, 233 insertions(+), 24 deletions(-) create mode 100644 platform/wecom/message_split.go diff --git a/platform/wecom/message_split.go b/platform/wecom/message_split.go new file mode 100644 index 000000000..df829b457 --- /dev/null +++ b/platform/wecom/message_split.go @@ -0,0 +1,125 @@ +package wecom + +import "unicode/utf8" + +// splitByBytes splits text under a UTF-8 byte limit. It prefers readable +// boundaries before falling back to byte-safe hard cuts. +func splitByBytes(s string, maxBytes int) []string { + if len(s) <= maxBytes || maxBytes <= 0 { + return []string{s} + } + + parts := make([]string, 0, len(s)/maxBytes+1) + for len(s) > maxBytes { + cut := semanticByteCut(s, maxBytes) + parts = append(parts, s[:cut]) + s = s[cut:] + } + if s != "" { + parts = append(parts, s) + } + return parts +} + +func semanticByteCut(s string, maxBytes int) int { + candidates := splitCandidates(s, maxBytes) + if cut := selectWindowSplit(candidates.paragraphs, windowStart(maxBytes, 70)); cut > 0 { + return cut + } + if cut := selectWindowSplit(candidates.lines, windowStart(maxBytes, 80)); cut > 0 { + return cut + } + if cut := selectWindowSplit(candidates.sentences, windowStart(maxBytes, 85)); cut > 0 { + return cut + } + if cut := selectWindowSplit(candidates.soft, windowStart(maxBytes, 90)); cut > 0 { + return cut + } + return hardByteCut(s, maxBytes) +} + +type byteSplitCandidates struct { + paragraphs []int + lines []int + sentences []int + soft []int +} + +func splitCandidates(s string, maxBytes int) byteSplitCandidates { + var candidates byteSplitCandidates + prevNewline := false + for i, r := range s { + if i >= maxBytes { + break + } + + size := utf8.RuneLen(r) + if size < 0 { + size = 1 + } + next := i + size + switch { + case r == '\n' && prevNewline: + candidates.paragraphs = append(candidates.paragraphs, next) + case r == '\n': + candidates.lines = append(candidates.lines, next) + case isSentenceBoundary(r): + candidates.sentences = append(candidates.sentences, next) + case isSoftBoundary(r): + candidates.soft = append(candidates.soft, next) + } + prevNewline = r == '\n' + } + return candidates +} + +func windowStart(maxBytes, percent int) int { + start := maxBytes * percent / 100 + if start < 1 { + return 1 + } + return start +} + +func selectWindowSplit(candidates []int, searchStart int) int { + best := 0 + for _, cut := range candidates { + if cut >= searchStart { + best = cut + } + } + return best +} + +func isSentenceBoundary(r rune) bool { + switch r { + case '。', '！', '？', '；', '.', '!', '?', ';': + return true + default: + return false + } +} + +func isSoftBoundary(r rune) bool { + switch r { + case '，', ',', ' ', '\t': + return true + default: + return false + } +} + +func hardByteCut(s string, maxBytes int) int { + end := maxBytes + if end > len(s) { + end = len(s) + } + for end > 0 && !utf8.RuneStart(s[end]) { + end-- + } + if end == 0 { + _, size := utf8.DecodeRuneInString(s) + return size + } + return end +} diff --git a/platform/wecom/websocket_test.go b/platform/wecom/websocket_test.go index 1d8d292fa..1b8b4a601 100644 --- a/platform/wecom/websocket_test.go +++ b/platform/wecom/websocket_test.go @@ -9,6 +9,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "sync" "testing" "time" @@ -93,6 +94,113 @@ func TestSplitByBytes_ReassemblesLargeContent(t *testing.T) { } } +func TestSplitByBytes_UsesNearestParagraphBoundaryAfterSeventyPercent(t *testing.T) { + early := strings.Repeat("甲", 15) + "\n\n" + near := strings.Repeat("乙", 10) + "\n\n" + tail := strings.Repeat("丙", 20) + parts := splitByBytes(early+near+tail, 100) + if len(parts) != 2 { + t.Fatalf("expected 2 chunks, got %d: %v", len(parts), parts) + } + if parts[0] != early+near { + t.Fatalf("unexpected chunks: %q / %q", parts[0], parts[1]) + } +} + +func TestSplitByBytes_IgnoresParagraphBoundaryBeforeSeventyPercent(t *testing.T) { + early := strings.Repeat("甲", 12) + "\n\n" + tail := strings.Repeat("乙", 24) + parts := splitByBytes(early+tail, 100) + if len(parts) != 2 { + t.Fatalf("expected 2 chunks, got %d: %v", len(parts), parts) + } + if parts[0] == early { + t.Fatalf("should not split at an early paragraph boundary: %q", parts[0]) + } + if strings.Join(parts, "") != early+tail { + t.Fatalf("reassembled content does not match original: %v", parts) + } +} + +func TestSplitByBytes_UsesLineBoundaryAfterEightyPercent(t *testing.T) { + first := strings.Repeat("甲", 27) + "\n" + second := strings.Repeat("乙", 12) + parts := splitByBytes(first+second, 100) + if len(parts) != 2 { + t.Fatalf("expected 2 chunks, got %d: %v", len(parts), parts) + } + if parts[0] != first { + t.Fatalf("unexpected chunks: %q / %q", parts[0], parts[1]) + } +} + +func TestSplitByBytes_UsesSentenceBoundaryAfterEightyFivePercent(t *testing.T) { + first := strings.Repeat("前", 28) + "。" + second := strings.Repeat("后", 12) + parts := splitByBytes(first+second, 100) + if len(parts) != 2 { + t.Fatalf("expected 2 chunks, got %d: %v", len(parts), parts) + } + if parts[0] != first { + t.Fatalf("unexpected chunks: %q / %q", parts[0], parts[1]) + } +} + +func TestSplitByBytes_DoesNotSplitAtEnumerationComma(t *testing.T) { + first := strings.Repeat("甲", 15) + "、" + second := strings.Repeat("乙", 10) + parts := splitByBytes(first+second, 70) + if len(parts) != 2 { + t.Fatalf("expected 2 chunks, got %d: %v", len(parts), parts) + } + if parts[0] == first { + t.Fatalf("should not split at enumeration comma: %q", parts[0]) + } + if strings.Join(parts, "") != first+second { + t.Fatalf("reassembled content does not match original: %v", parts) + } +} + +func TestSplitByBytes_UsesSoftBoundaryOnlyAfterNinetyPercent(t *testing.T) { + first := strings.Repeat("甲", 30) + "，" + second := strings.Repeat("乙", 12) + parts := splitByBytes(first+second, 100) + if len(parts) != 2 { + t.Fatalf("expected 2 chunks, got %d: %v", len(parts), parts) + } + if parts[0] != first { + t.Fatalf("unexpected chunks: %q / %q", parts[0], parts[1]) + } +} + +func TestSplitByBytes_IgnoresSoftBoundaryBeforeNinetyPercent(t *testing.T) { + first := strings.Repeat("甲", 20) + "，" + second := strings.Repeat("乙", 20) + parts := splitByBytes(first+second, 100) + if len(parts) != 2 { + t.Fatalf("expected 2 chunks, got %d: %v", len(parts), parts) + } + if parts[0] == first { + t.Fatalf("should not split at an early soft boundary: %q", parts[0]) + } + if strings.Join(parts, "") != first+second { + t.Fatalf("reassembled content does not match original: %v", parts) + } +} + +func TestSplitByBytes_FallsBackToHardCut(t *testing.T) { + input := strings.Repeat("甲", 10) + parts := splitByBytes(input, 10) + for _, part := range parts { + if len(part) > 10 { + t.Fatalf("chunk exceeds limit: %d > 10", len(part)) + } + } + if strings.Join(parts, "") != input { + t.Fatalf("reassembled content does not match original: %v", parts) + } +} + // --------------------------------------------------------------------------- // handleMsgCallback — chatID fallback to userID for single chats // --------------------------------------------------------------------------- diff --git a/platform/wecom/wecom.go b/platform/wecom/wecom.go index de265a306..643f9b41a 100644 --- a/platform/wecom/wecom.go +++ b/platform/wecom/wecom.go @@ -892,27 +892,3 @@ func (p *Platform) downloadMedia(mediaID string) ([]byte, error) { defer resp.Body.Close() return io.ReadAll(resp.Body) } - -// splitByBytes splits text by UTF-8 byte length (WeChat Work limit is 2048 bytes). -func splitByBytes(s string, maxBytes int) []string { - if len(s) <= maxBytes { - return []string{s} - } - var parts []string - for len(s) > 0 { - end := maxBytes - if end > len(s) { - end = len(s) - } - // Avoid splitting in the middle of a UTF-8 character - for end > 0 && end < len(s) && s[end]>>6 == 0b10 { - end-- - } - if end == 0 { - end = maxBytes - } - parts = append(parts, s[:end]) - s = s[end:] - } - return parts -}