From 2806f3abcf44a7606066b448540f9744ff194913 Mon Sep 17 00:00:00 2001 From: Kleissner Date: Tue, 14 Dec 2021 17:02:15 +0100 Subject: [PATCH] Search: support indexing of CamelCase words (breaking them up). --- search/Normalize.go | 4 +- search/Search Term.go | 8 ++-- search/Text 2 Hash.go | 102 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 103 insertions(+), 11 deletions(-) diff --git a/search/Normalize.go b/search/Normalize.go index 0e339e7..1d8f510 100644 --- a/search/Normalize.go +++ b/search/Normalize.go @@ -13,10 +13,10 @@ import ( "strings" ) +// sanitizeGeneric sanitizes the text. It intentionally does not lowercase the text so CamelCase can be detcted later. func sanitizeGeneric(filename string) string { - filename = strings.TrimSpace(filename) - filename = strings.ToLower(filename) filename = strings.ToValidUTF8(filename, "") + filename = strings.TrimSpace(filename) return filename } diff --git a/search/Search Term.go b/search/Search Term.go index 4f54df0..182a9ef 100644 --- a/search/Search Term.go +++ b/search/Search Term.go @@ -6,7 +6,9 @@ Author: Peter Kleissner package search -import "github.com/google/uuid" +import ( + "github.com/google/uuid" +) func (index *SearchIndexStore) Search(term string) (results []SearchIndexRecord) { termS, isExact, _ := sanitizeInputTerm(term) @@ -24,9 +26,9 @@ func (index *SearchIndexStore) Search(term string) (results []SearchIndexRecord) } // start with exact search - hashExact := hashWord(termS) + hashExact, wordH := hashWord(termS) if hashExact != nil { - index.LookupHash(SearchSelector{Hash: hashExact, Word: termS, ExactSearch: true}, resultMap) + index.LookupHash(SearchSelector{Hash: hashExact, Word: wordH, ExactSearch: true}, resultMap) } // exact search only? diff --git a/search/Text 2 Hash.go b/search/Text 2 Hash.go index 2517fb5..80491cb 100644 --- a/search/Text 2 Hash.go +++ b/search/Text 2 Hash.go @@ -16,6 +16,8 @@ import ( // Words must be the minimum length to be indexed. const wordMinLength = 3 +// text2Hashes creates hashes from words in the text. Text may be CamelCased. +// Text must be already validated for valid UTF8 when calling this function. func text2Hashes(text string, hashes map[[32]byte]string) { words := strings.FieldsFunc(text, func(char rune) bool { if unicode.IsSpace(char) { @@ -29,9 +31,17 @@ func text2Hashes(text string, hashes map[[32]byte]string) { word = strings.TrimPrefix(word, "#") hashWordMap(word, hashes) + + // CamelCase word detection + for _, word2 := range CamelCaseSplit(word) { + if word2 != word { + hashWordMap(word2, hashes) + } + } } } +// filename2Hashes creates hashes based on the filename and folder. func filename2Hashes(filename, folder string, hashes map[[32]byte]string) { if len(filename) < wordMinLength { return @@ -52,9 +62,9 @@ func filename2Hashes(filename, folder string, hashes map[[32]byte]string) { text2Hashes(folder, hashes) } -// hashWordMap hashes a word and stores it on the map. This immediately deduplicated hashes. +// hashWordMap hashes a word and stores it on the map. This immediately deduplicated hashes. It always lowercases the word. func hashWordMap(word string, hashes map[[32]byte]string) { - word = strings.TrimSpace(word) + word = strings.TrimSpace(strings.ToLower(word)) if len(word) < wordMinLength { return } @@ -74,13 +84,93 @@ func hashMapDelete(hash []byte, hashes map[[32]byte]string) { delete(hashes, hashB) } -// hashWord hashes a single word. It returns nil if not suitable. -func hashWord(word string) (hash []byte) { - word = strings.TrimSpace(word) +// hashWord hashes a single word. It returns nil if not suitable. It always lowercases the word. +func hashWord(word string) (hash []byte, wordHashed string) { + word = strings.TrimSpace(strings.ToLower(word)) if len(word) < wordMinLength { return } hashB := blake3.Sum256([]byte(word)) - return hashB[:] + return hashB[:], word +} + +// fork from https://github.com/fatih/camelcase/blob/master/camelcase.go + +// Split splits the camelcase word and returns a list of words. It also +// supports digits. Both lower camel case and upper camel case are supported. +// For more info please check: http://en.wikipedia.org/wiki/CamelCase +// +// Examples +// +// "" => [""] +// "lowercase" => ["lowercase"] +// "Class" => ["Class"] +// "MyClass" => ["My", "Class"] +// "MyC" => ["My", "C"] +// "HTML" => ["HTML"] +// "PDFLoader" => ["PDF", "Loader"] +// "AString" => ["A", "String"] +// "SimpleXMLParser" => ["Simple", "XML", "Parser"] +// "vimRPCPlugin" => ["vim", "RPC", "Plugin"] +// "GL11Version" => ["GL", "11", "Version"] +// "99Bottles" => ["99", "Bottles"] +// "May5" => ["May", "5"] +// "BFG9000" => ["BFG", "9000"] +// "BöseÜberraschung" => ["Böse", "Überraschung"] +// "Two spaces" => ["Two", " ", "spaces"] +// "BadUTF8\xe2\xe2\xa1" => ["BadUTF8\xe2\xe2\xa1"] +// +// Splitting rules +// +// 1) If string is not valid UTF-8, return it without splitting as -> removed in this fork. +// single item array. +// 2) Assign all unicode characters into one of 4 sets: lower case +// letters, upper case letters, numbers, and all other characters. +// 3) Iterate through characters of string, introducing splits +// between adjacent characters that belong to different sets. +// 4) Iterate through array of split strings, and if a given string +// is upper case: +// if subsequent string is lower case: +// move last character of upper case string to beginning of +// lower case string +func CamelCaseSplit(src string) (entries []string) { + entries = []string{} + var runes [][]rune + lastClass := 0 + class := 0 + // split into fields based on class of unicode character + for _, r := range src { + switch true { + case unicode.IsLower(r): + class = 1 + case unicode.IsUpper(r): + class = 2 + case unicode.IsDigit(r): + class = 3 + default: + class = 4 + } + if class == lastClass { + runes[len(runes)-1] = append(runes[len(runes)-1], r) + } else { + runes = append(runes, []rune{r}) + } + lastClass = class + } + // handle upper case -> lower case sequences, e.g. + // "PDFL", "oader" -> "PDF", "Loader" + for i := 0; i < len(runes)-1; i++ { + if unicode.IsUpper(runes[i][0]) && unicode.IsLower(runes[i+1][0]) { + runes[i+1] = append([]rune{runes[i][len(runes[i])-1]}, runes[i+1]...) + runes[i] = runes[i][:len(runes[i])-1] + } + } + // construct []string from results + for _, s := range runes { + if len(s) > 0 { + entries = append(entries, string(s)) + } + } + return }