Fix heap corruption for lines larger than 2114 bytes

- Replace RuneStrArray (LocalVector<RuneStr>) with std::vector<RuneStr>
  since LocalVector explicitly warns it's only safe for primitive types,
  not structs like RuneStr
- Change loop counter in DecodeUTF8RunesInString from uint32_t to
  size_t to prevent integer overflow for very long strings, with an
  explicit overflow check before truncating to uint32_t for RuneStr fields
- Update UnicodeTest.Test1 expected string to match std::vector
  operator<< format
- Add MixSegmentTest.LongInput regression test for strings > 2114 bytes

Co-authored-by: yanyiwu <2162645+yanyiwu@users.noreply.github.com>

Author: Copilot

include/cppjieba/Unicode.hpp

@@ -53,7 +53,7 @@ inline std::ostream& operator << (std::ostream& os, const RuneStr& r) {
 }
 
 typedef limonp::LocalVector<Rune> Unicode;
-typedef limonp::LocalVector<struct RuneStr> RuneStrArray;
+typedef std::vector<RuneStr> RuneStrArray;
 
 // [left, right]
 struct WordRange {
@@ -142,13 +142,17 @@ inline RuneStrLite DecodeUTF8ToRune(const char* str, size_t len) {
 inline bool DecodeUTF8RunesInString(const char* s, size_t len, RuneStrArray& runes) {
   runes.clear();
   runes.reserve(len / 2);
-  for (uint32_t i = 0, j = 0; i < len;) {
+  for (size_t i = 0, j = 0; i < len;) {
     RuneStrLite rp = DecodeUTF8ToRune(s + i, len - i);
     if (rp.len == 0) {
       runes.clear();
       return false;
     }
-    RuneStr x(rp.rune, i, rp.len, j, 1);
+    if (i > UINT32_MAX || j > UINT32_MAX) {
+      runes.clear();
+      return false;
+    }
+    RuneStr x(rp.rune, static_cast<uint32_t>(i), rp.len, static_cast<uint32_t>(j), 1);
     runes.push_back(x);
     i += rp.len;
     ++j;

test/unittest/segments_test.cpp

@@ -259,3 +259,41 @@ TEST(MPSegmentTest, Unicode32) {
 
   ASSERT_EQ(Join(words.begin(), words.end(), "/"), "天气/很/好/，/🙋/ /我们/去/郊游/。");
 }
+
+// Regression test for heap corruption ("corrupted size vs. prev_size") with
+// input strings larger than 2114 bytes. The bug was caused by using
+// LocalVector<RuneStr> (only safe for primitive types) for RuneStrArray.
+TEST(MixSegmentTest, LongInput) {
+  // 2114 is the byte length threshold beyond which the original heap
+  // corruption was triggered when RuneStrArray used LocalVector<RuneStr>.
+  const size_t HEAP_CORRUPTION_THRESHOLD = 2114;
+
+  MixSegment segment(DICT_DIR "/jieba.dict.utf8", DICT_DIR "/hmm_model.utf8");
+  vector<string> words;
+
+  // Test with a long Chinese string (> HEAP_CORRUPTION_THRESHOLD bytes)
+  string phrase = "我来到北京清华大学进行学习和研究工作，非常愉快，让我有了很大的收获。";
+  string long_chinese;
+  while (long_chinese.size() < HEAP_CORRUPTION_THRESHOLD + 1000) {
+    long_chinese += phrase;
+  }
+  ASSERT_GT(long_chinese.size(), HEAP_CORRUPTION_THRESHOLD);
+  segment.Cut(long_chinese, words);
+  ASSERT_GT(words.size(), size_t(0));
+
+  // Test with a long ASCII string (> HEAP_CORRUPTION_THRESHOLD bytes)
+  string long_ascii(HEAP_CORRUPTION_THRESHOLD + 1000, 'a');
+  words.clear();
+  segment.Cut(long_ascii, words);
+  ASSERT_GT(words.size(), size_t(0));
+
+  // Test with a very long string (> 6000 bytes)
+  string very_long_chinese;
+  while (very_long_chinese.size() < 6000) {
+    very_long_chinese += phrase;
+  }
+  ASSERT_GT(very_long_chinese.size(), size_t(6000));
+  words.clear();
+  segment.Cut(very_long_chinese, words);
+  ASSERT_GT(words.size(), size_t(0));
+}

test/unittest/unicode_test.cpp

@@ -10,7 +10,7 @@ TEST(UnicodeTest, Test1) {
   RuneStrArray runes;
   ASSERT_TRUE(DecodeUTF8RunesInString(s, runes));
   string actual;
-  string expected = "[\"{\"rune\": \"20320\", \"offset\": 0, \"len\": 3}\", \"{\"rune\": \"22909\", \"offset\": 3, \"len\": 3}\", \"{\"rune\": \"19990\", \"offset\": 6, \"len\": 3}\", \"{\"rune\": \"30028\", \"offset\": 9, \"len\": 3}\"]";
+  string expected = "[{\"rune\": \"20320\", \"offset\": 0, \"len\": 3}, {\"rune\": \"22909\", \"offset\": 3, \"len\": 3}, {\"rune\": \"19990\", \"offset\": 6, \"len\": 3}, {\"rune\": \"30028\", \"offset\": 9, \"len\": 3}]";
   actual << runes;
   ASSERT_EQ(expected, actual);
 }