mirror of
https://github.com/wgh136/nysoure.git
synced 2025-09-27 04:17:23 +00:00
Check stopwords
This commit is contained in:
@@ -101,3 +101,14 @@ func SearchResource(keyword string) ([]uint, error) {
|
|||||||
|
|
||||||
return results, nil
|
return results, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func IsStopWord(word string) bool {
|
||||||
|
mapping := bleve.NewIndexMapping()
|
||||||
|
analyzerName := mapping.DefaultAnalyzer
|
||||||
|
analyzer := mapping.AnalyzerNamed(analyzerName)
|
||||||
|
if analyzer == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
tokens := analyzer.Analyze([]byte(word))
|
||||||
|
return len(tokens) == 0
|
||||||
|
}
|
||||||
|
@@ -163,3 +163,23 @@ func TestSearchResource(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestIsStopWord(t *testing.T) {
|
||||||
|
Init()
|
||||||
|
defer TearDown()
|
||||||
|
|
||||||
|
stopWords := []string{"the", "is", "at", "which", "on", "and", "a", "an", "in", "to", "of"}
|
||||||
|
nonStopWords := []string{"adventure", "mystery", "romance", "sci-fi", "comedy", "action", "horror"}
|
||||||
|
|
||||||
|
for _, word := range stopWords {
|
||||||
|
if !IsStopWord(word) {
|
||||||
|
t.Errorf("Expected '%s' to be identified as a stop word", word)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, word := range nonStopWords {
|
||||||
|
if IsStopWord(word) {
|
||||||
|
t.Errorf("Expected '%s' to not be identified as a stop word", word)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@@ -317,6 +317,9 @@ func SearchResource(query string, page int) ([]model.ResourceView, int, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, 0, err
|
return nil, 0, err
|
||||||
}
|
}
|
||||||
|
if len(res) == 0 && search.IsStopWord(keyword) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
if first {
|
if first {
|
||||||
temp = utils.RemoveDuplicate(res)
|
temp = utils.RemoveDuplicate(res)
|
||||||
first = false
|
first = false
|
||||||
|
Reference in New Issue
Block a user