There was an error while loading. Please reload this page.
1 parent 89b577d commit f662a72Copy full SHA for f662a72
1 file changed
src/tokenize.cxx
@@ -162,7 +162,8 @@ namespace Tokenizer {
162
const UnicodeString type_unknown = "UNKNOWN";
163
const UnicodeString type_unanalyzed = "UNANALYZED";
164
165
- UChar32 SPACE_PLACEHOLDER = U'Ž';
+ UChar32 SPACE_PLACEHOLDER = U'ǯ'; // LATIN SMALL LETTER EZH WITH CARON
166
+ // found in languages: sami4 [Skolt Sámi]; (sorry Finland)
167
168
UnicodeString filter_placeholder( const UnicodeString& in ){
169
/// filter out the SPACE_PACEHOLDER characters
0 commit comments