Merge pull request #214 from yanyiwu/copilot/fix-fullsegment-cut-issue

Fix FullSegment skipping the rune after a null dict match

Author: yanyiwu

include/cppjieba/FullSegment.hpp

@@ -68,6 +68,7 @@ class FullSegment: public SegmentBase {
         assert(nextoffset < dags.size());
         const DictUnit* du = dags[i].nexts[j].second;
         if (du == NULL) {
+          wordLen = 1;
           if (dags[i].nexts.size() == 1 && maxIdx <= uIdx) {
             WordRange wr(begin + i, begin + nextoffset);
             res.push_back(wr);

test/unittest/segments_test.cpp

@@ -257,6 +257,14 @@ TEST(FullSegment, Test1) {
   ASSERT_EQ(s, "[\"上市\", \"公司\", \"C\", \"E\", \"O\"]");
 }
 
+TEST(FullSegment, NullDictUnitDoesNotSkipFollowingRune) {
+  FullSegment segment(DICT_DIR "/jieba.dict.utf8");
+  vector<string> words;
+
+  segment.Cut("崎岖的牙齿", words);
+  ASSERT_EQ("崎岖/的/牙齿", Join(words.begin(), words.end(), "/"));
+}
+
 TEST(QuerySegment, Test1) {
   QuerySegment segment(DICT_DIR "/jieba.dict.utf8", DICT_DIR "/hmm_model.utf8", "");
   vector<string> words;