🔍 search algorithm! 🎉🎊🥳

Blockchain Cache Global.go

@@ -73,6 +73,8 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) {
 			}
 			cache.store.WriteBlock(peer.PublicKey, peer.BlockchainVersion, targetBlock.Offset, data)
 			header.ListBlocks = append(header.ListBlocks, targetBlock.Offset)
+
+			currentBackend.SearchIndex.IndexNewBlock(peer.PublicKey, targetBlock.Offset, data)
 		})
 
 		// only update the blockchain header if it changed
@@ -94,6 +96,10 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) {
 	case blockchain.MultiStatusInvalidRemote:
 		cache.store.DeleteBlockchain(peer.PublicKey, header)
 
+		for _, blockN := range header.ListBlocks {
+			currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, blockN)
+		}
+
 	case blockchain.MultiStatusHeaderNA:
 		if header, err = cache.store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil {
 			return
@@ -105,6 +111,10 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) {
 		// delete existing data first, then create it new
 		cache.store.DeleteBlockchain(peer.PublicKey, header)
 
+		for _, blockN := range header.ListBlocks {
+			currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, blockN)
+		}
+
 		if header, err = cache.store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil {
 			return
 		}

Config Default.yaml

@@ -3,6 +3,7 @@ LogFile:          "data/log.txt"                # Log file. It contains informat
 BlockchainMain:   "data/blockchain main/"       # Blockchain main stores the end-users blockchain data. It contains meta data of shared files, profile data, and social interactions.
 BlockchainGlobal: "data/blockchain global/"     # Blockchain global caches blockchain data from global users. Empty to disable.
 WarehouseMain:    "data/warehouse main/"        # Warehouse main stores the actual data of files shared by the end-user.
+SearchIndex:      "data/search index/"          # Local search index of blockchain records. Empty to disable.
 
 # Listen defines all IP:Port combinations to listen on. If empty, it will listen on all IPs automatically on available ports.
 # IPv6 must be in the form "[IPv6]:Port". This setting is only recommended to be set on servers.

Config.go

@@ -25,6 +25,7 @@ var config struct {
 	BlockchainMain   string `yaml:"BlockchainMain"`   // Blockchain main stores the end-users blockchain data. It contains meta data of shared files, profile data, and social interactions.
 	BlockchainGlobal string `yaml:"BlockchainGlobal"` // Blockchain global caches blockchain data from global users. Empty to disable.
 	WarehouseMain    string `yaml:"WarehouseMain"`    // Warehouse main stores the actual data of files shared by the end-user.
+	SearchIndex      string `yaml:"SearchIndex"`      // Local search index of blockchain records. Empty to disable.
 
 	// Listen settings
 	Listen        []string `yaml:"Listen"`        // IP:Port combinations

Peernet.go

@@ -6,6 +6,10 @@ Author:     Peter Kleissner
 
 package core
 
+import (
+	"github.com/PeernetOfficial/core/search"
+)
+
 var userAgent = "Peernet Core/0.1" // must be overwritten by the caller
 
 // Init initializes the client. The config must be loaded first!
@@ -15,6 +19,9 @@ func Init(UserAgent string) (backend *Backend) {
 		return
 	}
 
+	backend = &Backend{}
+	currentBackend = backend
+
 	initFilters()
 	initPeerID()
 	initUserBlockchain()
@@ -27,10 +34,14 @@ func Init(UserAgent string) (backend *Backend) {
 	initStore()
 	initNetwork()
 
-	backend = &Backend{}
-	backend.GlobalBlockchainCache = initBlockchainCache(config.BlockchainGlobal, config.CacheMaxBlockSize, config.CacheMaxBlockCount, config.LimitTotalRecords)
+	var err error
 
-	currentBackend = backend
+	backend.GlobalBlockchainCache = initBlockchainCache(config.BlockchainGlobal, config.CacheMaxBlockSize, config.CacheMaxBlockCount, config.LimitTotalRecords)
+	backend.SearchIndex, err = search.InitSearchIndexStore(config.SearchIndex)
+
+	if err != nil {
+		Filters.LogError("Init", "search index '%s' init: %s", config.SearchIndex, err.Error())
+	}
 
 	return backend
 }
@@ -49,7 +60,8 @@ func Connect() {
 // The Backend represents an instance of a Peernet client to be used by a frontend.
 // Global variables and init functions are to be merged.
 type Backend struct {
-	GlobalBlockchainCache *BlockchainCache // stores other peers blockchains
+	GlobalBlockchainCache *BlockchainCache         // Caches blockchains of other peers.
+	SearchIndex           *search.SearchIndexStore // Search index of blockchain records.
 }
 
 // This variable is to be replaced later by pointers in structures.

search/Normalize.go

@@ -0,0 +1,42 @@
+/*
+File Name:  Normalizing.go
+Copyright:  2021 Peernet s.r.o.
+Author:     Peter Kleissner
+
+Normalizing text so that it can be hashed.
+*/
+
+package search
+
+import (
+	"path"
+	"strings"
+)
+
+func sanitizeGeneric(filename string) string {
+	filename = strings.TrimSpace(filename)
+	filename = strings.ToLower(filename)
+	filename = strings.ToValidUTF8(filename, "")
+
+	return filename
+}
+
+// sanitizeInputTerm sanitizes a search term provided by the end user. It includes the generic rules that are done on indexing.
+func sanitizeInputTerm(inputTerm string) (outputTerm string, isExact, isWildcard bool) {
+	inputTerm = sanitizeGeneric(inputTerm)
+
+	// detect and remove quotes at the beginning and end
+	isExact = false
+	if len(inputTerm) >= 2 && ((strings.HasPrefix(inputTerm, "\"") && strings.HasSuffix(inputTerm, "\"")) || (strings.HasPrefix(inputTerm, "'") && strings.HasSuffix(inputTerm, "'"))) {
+		inputTerm = inputTerm[1 : len(inputTerm)-1]
+		isExact = true
+	}
+
+	isWildcard = strings.Contains(inputTerm, "*")
+
+	return inputTerm, isExact, isWildcard
+}
+
+func filenameRemoveExtension(filename string) string {
+	return strings.TrimSuffix(filename, path.Ext(filename))
+}

search/Search Index.go

@@ -0,0 +1,282 @@
+/*
+File Name:  Search Index.go
+Copyright:  2021 Peernet s.r.o.
+Author:     Peter Kleissner
+*/
+
+package search
+
+import (
+	"encoding/binary"
+	"errors"
+	"sync"
+
+	"github.com/PeernetOfficial/core/blockchain"
+	"github.com/PeernetOfficial/core/btcec"
+	"github.com/PeernetOfficial/core/store"
+	"github.com/google/uuid"
+)
+
+// SearchIndexRecord identifies a hash to a given file
+type SearchIndexRecord struct {
+	// input data
+	Word string
+	Hash []byte
+
+	// result data
+	FileID      uuid.UUID
+	PublicKey   *btcec.PublicKey
+	BlockNumber uint64
+}
+
+// This database stores hashes of keywords for file search.
+type SearchIndexStore struct {
+	Database store.Store // The database storing the blockchain.
+	sync.RWMutex
+}
+
+func InitSearchIndexStore(DatabaseDirectory string) (searchIndex *SearchIndexStore, err error) {
+	if DatabaseDirectory == "" {
+		return
+	}
+
+	searchIndex = &SearchIndexStore{}
+
+	if searchIndex.Database, err = store.NewPogrebStore(DatabaseDirectory); err != nil {
+		return nil, err
+	}
+
+	return searchIndex, nil
+}
+
+func (index *SearchIndexStore) IndexNewBlock(publicKey *btcec.PublicKey, blockNumber uint64, raw []byte) {
+	if index.Database == nil {
+		return
+	}
+
+	// decode all files from the block
+	decoded, status, err := blockchain.DecodeBlockRaw(raw)
+	if err != nil || status != blockchain.StatusOK {
+		return
+	}
+
+	for _, decodedR := range decoded.RecordsDecoded {
+		if file, ok := decodedR.(blockchain.BlockRecordFile); ok {
+			var filename, folder, description string
+			for _, tag := range file.Tags {
+				switch tag.Type {
+				case blockchain.TagName:
+					filename = sanitizeGeneric(tag.Text())
+				case blockchain.TagFolder:
+					folder = sanitizeGeneric(tag.Text())
+				case blockchain.TagDescription:
+					description = sanitizeGeneric(tag.Text())
+				}
+			}
+
+			hashes := make(map[[32]byte]string)
+			filename2Hashes(filename, folder, hashes)
+			text2Hashes(description, hashes)
+
+			for hash := range hashes {
+				index.IndexHash(publicKey, blockNumber, file.ID, hash[:])
+			}
+		}
+	}
+}
+
+func (index *SearchIndexStore) UnindexBlock(publicKey *btcec.PublicKey, blockNumber uint64) {
+	if index.Database == nil {
+		return
+	}
+
+	// get the reverse records
+	key := reverseIndexKey(publicKey, blockNumber)
+	raw, found := index.Database.Get(key)
+
+	if !found || len(raw)%reverseIndexRecordSize != 0 { // corrupt record
+		return
+	}
+
+	offset := 0
+
+	for n := 0; n < len(raw)/reverseIndexRecordSize; n++ {
+		var hash []byte
+		var fileID uuid.UUID
+
+		hash = raw[offset : offset+32]
+		copy(fileID[:], raw[offset+32:offset+32+16])
+
+		// delete the index record
+		index.UnindexHash(fileID, hash)
+	}
+
+	// delete the reverse record
+	index.Database.Delete(key)
+}
+
+// IndexHash indexes a new hash
+func (index *SearchIndexStore) IndexHash(publicKey *btcec.PublicKey, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) {
+	if index.Database == nil {
+		return
+	}
+
+	index.Lock()
+	defer index.Unlock()
+
+	// parse existing records, check if already stored
+	raw, found := index.Database.Get(hash)
+	if found && len(raw)%indexRecordSize == 0 { // check if record is corrupt
+		offset := 0
+		for n := 0; n < len(raw)/indexRecordSize; n++ {
+			if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil {
+				if fileID == record.FileID {
+					return errors.New("already indexed")
+				}
+			}
+
+			offset += indexRecordSize
+		}
+	}
+
+	raw = append(raw, encodeIndexRecord(publicKey, blockNumber, fileID)...)
+
+	// create the reverse record
+	index.createReverseIndexRecord(publicKey, blockNumber, fileID, hash)
+
+	return index.Database.Set(hash, raw)
+}
+
+// UnindexHash deletes a index record. If there are no more files associated with the hash, the entire hash record is deleted.
+func (index *SearchIndexStore) UnindexHash(fileID uuid.UUID, hash []byte) (err error) {
+	if index.Database == nil {
+		return
+	}
+
+	index.Lock()
+	defer index.Unlock()
+
+	var newRaw []byte
+
+	raw, found := index.Database.Get(hash)
+	if !found {
+		return errors.New("index record not found")
+	}
+
+	if len(raw)%indexRecordSize == 0 { // check if record is corrupt
+		offset := 0
+		for n := 0; n < len(raw)/indexRecordSize; n++ {
+			if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil {
+				if fileID != record.FileID {
+					newRaw = append(newRaw, raw[offset:offset+indexRecordSize]...)
+				}
+			}
+
+			offset += indexRecordSize
+		}
+	}
+
+	if len(newRaw) == 0 {
+		// delete the entire hash key
+		index.Database.Delete(hash)
+		return
+	}
+
+	return index.Database.Set(hash, newRaw)
+}
+
+// LookupHash returns all index records stored for the hash.
+func (index *SearchIndexStore) LookupHash(hash []byte) (records []SearchIndexRecord, err error) {
+	if index.Database == nil {
+		return
+	}
+
+	index.RLock()
+	defer index.RUnlock()
+
+	raw, found := index.Database.Get(hash)
+	if !found {
+		return nil, nil
+	} else if len(raw)%indexRecordSize != 0 { // check if record is corrupt
+		return nil, errors.New("corrupt index record")
+	}
+
+	for offset := 0; offset < len(raw); offset += indexRecordSize {
+		if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil {
+			record.Hash = hash
+			records = append(records, *record)
+		}
+	}
+
+	return records, nil
+}
+
+// ---- index and reverse index code ----
+
+/*
+Structure for each index record:
+
+Offset   Size    Info
+0        16      File ID
+16       33      Public Key compressed
+49       8       Block Number
+*/
+
+const indexRecordSize = 57
+
+// decodeIndexRecord decodes the index record and sets the fields File ID, Public Key, and Block Number.
+func decodeIndexRecord(raw []byte) (record *SearchIndexRecord) {
+	if len(raw) < indexRecordSize {
+		return
+	}
+
+	record = &SearchIndexRecord{}
+	copy(record.FileID[:], raw[0:16])
+
+	var err error
+	if record.PublicKey, err = btcec.ParsePubKey(raw[16:16+33], btcec.S256()); err != nil {
+		return nil
+	}
+
+	record.BlockNumber = binary.LittleEndian.Uint64(raw[49 : 49+8])
+
+	return record
+}
+
+func encodeIndexRecord(publicKey *btcec.PublicKey, blockNumber uint64, fileID uuid.UUID) (raw []byte) {
+	raw = make([]byte, indexRecordSize)
+
+	copy(raw[0:16], fileID[:])
+	copy(raw[16:16+33], publicKey.SerializeCompressed())
+	binary.LittleEndian.PutUint64(raw[49:49+8], blockNumber)
+
+	return raw
+}
+
+// This creates a reverse index record. It uses the blockchain and block number as key, and provides the hash and file ID as value.
+// This function must be called in a RW locked database state. The caller must ensure that this does not result in a duplicate.
+func (index *SearchIndexStore) createReverseIndexRecord(publicKey *btcec.PublicKey, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) {
+	key := reverseIndexKey(publicKey, blockNumber)
+	raw, _ := index.Database.Get(key)
+
+	// each record is only hash + file ID
+	reverseRecord := make([]byte, reverseIndexRecordSize)
+	copy(reverseRecord[0:32], hash)
+	copy(reverseRecord[32:32+16], fileID[:])
+
+	raw = append(raw, reverseRecord...)
+
+	return index.Database.Set(key, raw)
+}
+
+const reverseIndexRecordSize = 32 + 16
+
+func reverseIndexKey(publicKey *btcec.PublicKey, blockNumber uint64) (key []byte) {
+	key = publicKey.SerializeCompressed()
+
+	var temp [8]byte
+	binary.LittleEndian.PutUint64(temp[0:8], blockNumber)
+	key = append(key, temp[:]...)
+
+	return key
+}

search/Search Term.go

@@ -0,0 +1,51 @@
+/*
+File Name:  Search Term.go
+Copyright:  2021 Peernet s.r.o.
+Author:     Peter Kleissner
+*/
+
+package search
+
+func (index *SearchIndexStore) Search(term string) (results []SearchIndexRecord) {
+	termS, isExact, _ := sanitizeInputTerm(term)
+
+	if len(termS) < wordMinLength {
+		return
+	}
+
+	// start with exact search
+	hashExact := hashWord(termS)
+	if hashExact != nil {
+		records, _ := index.LookupHash(hashExact)
+		results = append(results, setSearchIndexFields(records, termS)...)
+	}
+
+	// exact search only?
+	if isExact {
+		return
+	}
+
+	// break up the term into hashes
+	hashes := make(map[[32]byte]string)
+
+	text2Hashes(termS, hashes)
+
+	// The exact search was already performed, exclude it.
+	hashMapDelete(hashExact, hashes)
+
+	// look up the hashes!
+	for hash, keyword := range hashes {
+		records, _ := index.LookupHash(hash[:])
+		results = append(results, setSearchIndexFields(records, keyword)...)
+	}
+
+	return
+}
+
+// setSearchIndexFields sets certain fields in the index record for adding context in the output
+func setSearchIndexFields(records []SearchIndexRecord, word string) []SearchIndexRecord {
+	for n := range records {
+		records[n].Word = word
+	}
+	return records
+}

search/Text 2 Hash.go

@@ -0,0 +1,83 @@
+/*
+File Name:  Text 2 Hash.go
+Copyright:  2021 Peernet s.r.o.
+Author:     Peter Kleissner
+*/
+
+package search
+
+import (
+	"strings"
+	"unicode"
+
+	"lukechampine.com/blake3"
+)
+
+// Words must be the minimum length to be indexed.
+const wordMinLength = 3
+
+func text2Hashes(text string, hashes map[[32]byte]string) {
+	words := strings.FieldsFunc(text, func(char rune) bool {
+		if unicode.IsSpace(char) {
+			return true
+		}
+		return strings.ContainsAny(string(char), "+-._()[],")
+	})
+
+	for _, word := range words {
+		hashWordMap(word, hashes)
+	}
+}
+
+func filename2Hashes(filename, folder string, hashes map[[32]byte]string) {
+	if len(filename) < wordMinLength {
+		return
+	}
+
+	// First hash is created on the entire filename including extension. This is done for perfect matches.
+	hashWordMap(filename, hashes)
+
+	// Hash the filename without extension (and proceed without extension)
+	filename = filenameRemoveExtension(filename)
+	hashWordMap(filename, hashes)
+
+	// Hash each individual word of the filename and directory
+	text2Hashes(filename, hashes)
+
+	folder = strings.ReplaceAll(folder, "\\", " ")
+	folder = strings.ReplaceAll(folder, "/", " ")
+	text2Hashes(folder, hashes)
+}
+
+// hashWordMap hashes a word and stores it on the map. This immediately deduplicated hashes.
+func hashWordMap(word string, hashes map[[32]byte]string) {
+	word = strings.TrimSpace(word)
+	if len(word) < wordMinLength {
+		return
+	}
+
+	hashes[blake3.Sum256([]byte(word))] = word
+}
+
+// hashMapDelete deletes a hash from the list if the hash is valid
+func hashMapDelete(hash []byte, hashes map[[32]byte]string) {
+	if len(hash) == 0 {
+		return
+	}
+
+	var hashB [32]byte
+	copy(hashB[:], hash)
+
+	delete(hashes, hashB)
+}
+
+// hashWord hashes a single word. It returns nil if not suitable.
+func hashWord(word string) (hash []byte) {
+	word = strings.TrimSpace(word)
+	if len(word) < wordMinLength {
+		return
+	}
+
+	hashB := blake3.Sum256([]byte(word))
+	return hashB[:]
+}

search/readme.md

@@ -0,0 +1,17 @@
+# Search Index
+
+## Search Term Normalization
+
+The user input search term undergoes normalization:
+1. Trim space
+2. Lowercase
+3. Remove invalid UTF-8 characters
+4. Detect and remove quotes in the form '" (activates exact search mode)
+
+Wildcards are not supported.
+
+## Generic Text Normalization
+
+1. Trim space
+2. Lowercase
+3. Remove invalid UTF-8 characters