core/search/Text 2 Hash.go

/*
File Name:  Text 2 Hash.go
Copyright:  2021 Peernet s.r.o.
Author:     Peter Kleissner
*/

package search

import (
	"strings"
	"unicode"

	"lukechampine.com/blake3"
)

// Words must be the minimum length to be indexed.
const wordMinLength = 3

func text2Hashes(text string, hashes map[[32]byte]string) {
	words := strings.FieldsFunc(text, func(char rune) bool {
		if unicode.IsSpace(char) {
			return true
		}
		return strings.ContainsAny(string(char), "+-._()[],")
	})

	for _, word := range words {
		hashWordMap(word, hashes)
	}
}

func filename2Hashes(filename, folder string, hashes map[[32]byte]string) {
	if len(filename) < wordMinLength {
		return
	}

	// First hash is created on the entire filename including extension. This is done for perfect matches.
	hashWordMap(filename, hashes)

	// Hash the filename without extension (and proceed without extension)
	filename = filenameRemoveExtension(filename)
	hashWordMap(filename, hashes)

	// Hash each individual word of the filename and directory
	text2Hashes(filename, hashes)

	folder = strings.ReplaceAll(folder, "\\", " ")
	folder = strings.ReplaceAll(folder, "/", " ")
	text2Hashes(folder, hashes)
}

// hashWordMap hashes a word and stores it on the map. This immediately deduplicated hashes.
func hashWordMap(word string, hashes map[[32]byte]string) {
	word = strings.TrimSpace(word)
	if len(word) < wordMinLength {
		return
	}

	hashes[blake3.Sum256([]byte(word))] = word
}

// hashMapDelete deletes a hash from the list if the hash is valid
func hashMapDelete(hash []byte, hashes map[[32]byte]string) {
	if len(hash) == 0 {
		return
	}

	var hashB [32]byte
	copy(hashB[:], hash)

	delete(hashes, hashB)
}

// hashWord hashes a single word. It returns nil if not suitable.
func hashWord(word string) (hash []byte) {
	word = strings.TrimSpace(word)
	if len(word) < wordMinLength {
		return
	}

	hashB := blake3.Sum256([]byte(word))
	return hashB[:]
}