Add vietnamese

.gitignore

@@ -1,3 +1,4 @@
+.idea/
 coverage.out
 /nbproject/private/
 /nbproject/

README.md

@@ -36,6 +36,7 @@ It uses a curated list of the most frequent words used in these languages:
  * Swedish
  * Thai
  * Turkish
+ * Vietnamese
 
 If the function is used with an unsupported language, it doesn't fail, but will apply english filter to the content.
 

custom.go

@@ -19,7 +19,7 @@ import (
 func LoadStopWordsFromFile(filePath string, langCode string, sep string) {
 	b, err := ioutil.ReadFile(filePath)
 	if err != nil {
-			panic(err)
+		panic(err)
 	}
 	LoadStopWordsFromString(string(b), langCode, sep)
 }
@@ -41,137 +41,142 @@ func LoadStopWordsFromString(wordsList string, langCode string, sep string) {
 	case "ar":
 		arabic = make(map[string]string)
 		for _, word := range words {
-				arabic[word] = ""
+			arabic[word] = ""
 		}
 	case "bg":
 		bulgarian = make(map[string]string)
 		for _, word := range words {
-				bulgarian[word] = ""
+			bulgarian[word] = ""
 		}
 	case "cs":
 		czech = make(map[string]string)
 		for _, word := range words {
-				czech[word] = ""
+			czech[word] = ""
 		}
 	case "da":
 		danish = make(map[string]string)
 		for _, word := range words {
-				danish[word] = ""
+			danish[word] = ""
 		}
 	case "de":
 		german = make(map[string]string)
 		for _, word := range words {
-				german[word] = ""
+			german[word] = ""
 		}
 	case "el":
 		greek = make(map[string]string)
 		for _, word := range words {
-				greek[word] = ""
+			greek[word] = ""
 		}
 	case "en":
 		english = make(map[string]string)
 		for _, word := range words {
-				english[word] = ""
+			english[word] = ""
 		}
 	case "es":
 		spanish = make(map[string]string)
 		for _, word := range words {
-				spanish[word] = ""
+			spanish[word] = ""
 		}
 	case "fa":
 		persian = make(map[string]string)
 		for _, word := range words {
-				persian[word] = ""
+			persian[word] = ""
 		}
 	case "fr":
 		french = make(map[string]string)
 		for _, word := range words {
-				french[word] = ""
+			french[word] = ""
 		}
 	case "fi":
 		finnish = make(map[string]string)
 		for _, word := range words {
-				finnish[word] = ""
+			finnish[word] = ""
 		}
 	case "hu":
 		hungarian = make(map[string]string)
 		for _, word := range words {
-				hungarian[word] = ""
+			hungarian[word] = ""
 		}
 	case "id":
 		indonesian = make(map[string]string)
 		for _, word := range words {
-				indonesian[word] = ""
+			indonesian[word] = ""
 		}
 	case "it":
 		italian = make(map[string]string)
 		for _, word := range words {
-				italian[word] = ""
+			italian[word] = ""
 		}
 	case "ja":
 		japanese = make(map[string]string)
 		for _, word := range words {
-				japanese[word] = ""
+			japanese[word] = ""
 		}
 	case "km":
 		khmer = make(map[string]string)
 		for _, word := range words {
-				khmer[word] = ""
+			khmer[word] = ""
 		}
 	case "lv":
 		latvian = make(map[string]string)
 		for _, word := range words {
-				latvian[word] = ""
+			latvian[word] = ""
 		}
 	case "nl":
 		dutch = make(map[string]string)
 		for _, word := range words {
-				dutch[word] = ""
+			dutch[word] = ""
 		}
 	case "no":
 		norwegian = make(map[string]string)
 		for _, word := range words {
-				norwegian[word] = ""
+			norwegian[word] = ""
 		}
 	case "pl":
 		polish = make(map[string]string)
 		for _, word := range words {
-				polish[word] = ""
+			polish[word] = ""
 		}
 	case "pt":
 		portuguese = make(map[string]string)
 		for _, word := range words {
-				portuguese[word] = ""
+			portuguese[word] = ""
 		}
 	case "ro":
 		romanian = make(map[string]string)
 		for _, word := range words {
-				romanian[word] = ""
+			romanian[word] = ""
 		}
 	case "ru":
 		russian = make(map[string]string)
 		for _, word := range words {
-				russian[word] = ""
+			russian[word] = ""
 		}
 	case "sk":
 		slovak = make(map[string]string)
 		for _, word := range words {
-				slovak[word] = ""
+			slovak[word] = ""
 		}
 	case "sv":
 		swedish = make(map[string]string)
 		for _, word := range words {
-				swedish[word] = ""
+			swedish[word] = ""
 		}
 	case "th":
 		thai = make(map[string]string)
 		for _, word := range words {
-				thai[word] = ""
+			thai[word] = ""
 		}
 	case "tr":
 		turkish = make(map[string]string)
 		for _, word := range words {
-				turkish[word] = ""
+			turkish[word] = ""
+		}
+	case "vi":
+		vietnamese = make(map[string]string)
+		for _, word := range words {
+			vietnamese[word] = ""
 		}
 	}
 }

simhash.go

@@ -95,6 +95,8 @@ func Simhash(content []byte, langCode string, cleanHTML bool) uint64 {
 		hash = removeStopWordsAndHash(content, thai)
 	case "tr":
 		hash = removeStopWordsAndHash(content, turkish)
+	case "vi":
+		hash = removeStopWordsAndHash(content, vietnamese)
 	}
 
 	return hash

stopwords.go

@@ -8,7 +8,8 @@
 //
 // arabic, bulgarian, czech, danish, english, finnish, french, german,
 // hungarian, italian, japanese, latvian, norwegian, persian, polish,
-// portuguese, romanian, russian, slovak, spanish, swedish, turkish
+// portuguese, romanian, russian, slovak, spanish, swedish, turkish,
+// vietnamese
 
 // Package stopwords contains various algorithms of text comparison (Simhash, Levenshtein)
 package stopwords
@@ -23,8 +24,8 @@ import (
 )
 
 var (
-	remTags      = regexp.MustCompile(`<[^>]*>`)
-	oneSpace     = regexp.MustCompile(`\s{2,}`)
+	remTags       = regexp.MustCompile(`<[^>]*>`)
+	oneSpace      = regexp.MustCompile(`\s{2,}`)
 	wordSegmenter = regexp.MustCompile(`[\pL\p{Mc}\p{Mn}-_']+`)
 )
 
@@ -118,6 +119,8 @@ func Clean(content []byte, langCode string, cleanHTML bool) []byte {
 		content = removeStopWords(content, thai)
 	case "tr":
 		content = removeStopWords(content, turkish)
+	case "vi":
+		content = removeStopWords(content, vietnamese)
 	}
 
 	//Remove duplicated space characters