Search: support indexing of CamelCase words (breaking them up).

search/Normalize.go

@@ -13,10 +13,10 @@ import (
 	"strings"
 )
 
+// sanitizeGeneric sanitizes the text. It intentionally does not lowercase the text so CamelCase can be detcted later.
 func sanitizeGeneric(filename string) string {
-	filename = strings.TrimSpace(filename)
-	filename = strings.ToLower(filename)
 	filename = strings.ToValidUTF8(filename, "")
+	filename = strings.TrimSpace(filename)
 
 	return filename
 }

search/Search Term.go

@@ -6,7 +6,9 @@ Author:     Peter Kleissner
 
 package search
 
-import "github.com/google/uuid"
+import (
+	"github.com/google/uuid"
+)
 
 func (index *SearchIndexStore) Search(term string) (results []SearchIndexRecord) {
 	termS, isExact, _ := sanitizeInputTerm(term)
@@ -24,9 +26,9 @@ func (index *SearchIndexStore) Search(term string) (results []SearchIndexRecord)
 	}
 
 	// start with exact search
-	hashExact := hashWord(termS)
+	hashExact, wordH := hashWord(termS)
 	if hashExact != nil {
-		index.LookupHash(SearchSelector{Hash: hashExact, Word: termS, ExactSearch: true}, resultMap)
+		index.LookupHash(SearchSelector{Hash: hashExact, Word: wordH, ExactSearch: true}, resultMap)
 	}
 
 	// exact search only?

search/Text 2 Hash.go

@@ -16,6 +16,8 @@ import (
 // Words must be the minimum length to be indexed.
 const wordMinLength = 3
 
+// text2Hashes creates hashes from words in the text. Text may be CamelCased.
+// Text must be already validated for valid UTF8 when calling this function.
 func text2Hashes(text string, hashes map[[32]byte]string) {
 	words := strings.FieldsFunc(text, func(char rune) bool {
 		if unicode.IsSpace(char) {
@@ -29,9 +31,17 @@ func text2Hashes(text string, hashes map[[32]byte]string) {
 		word = strings.TrimPrefix(word, "#")
 
 		hashWordMap(word, hashes)
+
+		// CamelCase word detection
+		for _, word2 := range CamelCaseSplit(word) {
+			if word2 != word {
+				hashWordMap(word2, hashes)
+			}
+		}
 	}
 }
 
+// filename2Hashes creates hashes based on the filename and folder.
 func filename2Hashes(filename, folder string, hashes map[[32]byte]string) {
 	if len(filename) < wordMinLength {
 		return
@@ -52,9 +62,9 @@ func filename2Hashes(filename, folder string, hashes map[[32]byte]string) {
 	text2Hashes(folder, hashes)
 }
 
-// hashWordMap hashes a word and stores it on the map. This immediately deduplicated hashes.
+// hashWordMap hashes a word and stores it on the map. This immediately deduplicated hashes. It always lowercases the word.
 func hashWordMap(word string, hashes map[[32]byte]string) {
-	word = strings.TrimSpace(word)
+	word = strings.TrimSpace(strings.ToLower(word))
 	if len(word) < wordMinLength {
 		return
 	}
@@ -74,13 +84,93 @@ func hashMapDelete(hash []byte, hashes map[[32]byte]string) {
 	delete(hashes, hashB)
 }
 
-// hashWord hashes a single word. It returns nil if not suitable.
-func hashWord(word string) (hash []byte) {
-	word = strings.TrimSpace(word)
+// hashWord hashes a single word. It returns nil if not suitable. It always lowercases the word.
+func hashWord(word string) (hash []byte, wordHashed string) {
+	word = strings.TrimSpace(strings.ToLower(word))
 	if len(word) < wordMinLength {
 		return
 	}
 
 	hashB := blake3.Sum256([]byte(word))
-	return hashB[:]
+	return hashB[:], word
+}
+
+// fork from https://github.com/fatih/camelcase/blob/master/camelcase.go
+
+// Split splits the camelcase word and returns a list of words. It also
+// supports digits. Both lower camel case and upper camel case are supported.
+// For more info please check: http://en.wikipedia.org/wiki/CamelCase
+//
+// Examples
+//
+//   "" =>                     [""]
+//   "lowercase" =>            ["lowercase"]
+//   "Class" =>                ["Class"]
+//   "MyClass" =>              ["My", "Class"]
+//   "MyC" =>                  ["My", "C"]
+//   "HTML" =>                 ["HTML"]
+//   "PDFLoader" =>            ["PDF", "Loader"]
+//   "AString" =>              ["A", "String"]
+//   "SimpleXMLParser" =>      ["Simple", "XML", "Parser"]
+//   "vimRPCPlugin" =>         ["vim", "RPC", "Plugin"]
+//   "GL11Version" =>          ["GL", "11", "Version"]
+//   "99Bottles" =>            ["99", "Bottles"]
+//   "May5" =>                 ["May", "5"]
+//   "BFG9000" =>              ["BFG", "9000"]
+//   "BöseÜberraschung" =>     ["Böse", "Überraschung"]
+//   "Two  spaces" =>          ["Two", "  ", "spaces"]
+//   "BadUTF8\xe2\xe2\xa1" =>  ["BadUTF8\xe2\xe2\xa1"]
+//
+// Splitting rules
+//
+//  1) If string is not valid UTF-8, return it without splitting as -> removed in this fork.
+//     single item array.
+//  2) Assign all unicode characters into one of 4 sets: lower case
+//     letters, upper case letters, numbers, and all other characters.
+//  3) Iterate through characters of string, introducing splits
+//     between adjacent characters that belong to different sets.
+//  4) Iterate through array of split strings, and if a given string
+//     is upper case:
+//       if subsequent string is lower case:
+//         move last character of upper case string to beginning of
+//         lower case string
+func CamelCaseSplit(src string) (entries []string) {
+	entries = []string{}
+	var runes [][]rune
+	lastClass := 0
+	class := 0
+	// split into fields based on class of unicode character
+	for _, r := range src {
+		switch true {
+		case unicode.IsLower(r):
+			class = 1
+		case unicode.IsUpper(r):
+			class = 2
+		case unicode.IsDigit(r):
+			class = 3
+		default:
+			class = 4
+		}
+		if class == lastClass {
+			runes[len(runes)-1] = append(runes[len(runes)-1], r)
+		} else {
+			runes = append(runes, []rune{r})
+		}
+		lastClass = class
+	}
+	// handle upper case -> lower case sequences, e.g.
+	// "PDFL", "oader" -> "PDF", "Loader"
+	for i := 0; i < len(runes)-1; i++ {
+		if unicode.IsUpper(runes[i][0]) && unicode.IsLower(runes[i+1][0]) {
+			runes[i+1] = append([]rune{runes[i][len(runes[i])-1]}, runes[i+1]...)
+			runes[i] = runes[i][:len(runes[i])-1]
+		}
+	}
+	// construct []string from results
+	for _, s := range runes {
+		if len(s) > 0 {
+			entries = append(entries, string(s))
+		}
+	}
+	return
 }