diff --git a/Blockchain Cache Global.go b/Blockchain Cache Global.go index 1471ae3..b378975 100644 --- a/Blockchain Cache Global.go +++ b/Blockchain Cache Global.go @@ -73,6 +73,8 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) { } cache.store.WriteBlock(peer.PublicKey, peer.BlockchainVersion, targetBlock.Offset, data) header.ListBlocks = append(header.ListBlocks, targetBlock.Offset) + + currentBackend.SearchIndex.IndexNewBlock(peer.PublicKey, targetBlock.Offset, data) }) // only update the blockchain header if it changed @@ -94,6 +96,10 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) { case blockchain.MultiStatusInvalidRemote: cache.store.DeleteBlockchain(peer.PublicKey, header) + for _, blockN := range header.ListBlocks { + currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, blockN) + } + case blockchain.MultiStatusHeaderNA: if header, err = cache.store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { return @@ -105,6 +111,10 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) { // delete existing data first, then create it new cache.store.DeleteBlockchain(peer.PublicKey, header) + for _, blockN := range header.ListBlocks { + currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, blockN) + } + if header, err = cache.store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { return } diff --git a/Config Default.yaml b/Config Default.yaml index 9d151d1..00a95f2 100644 --- a/Config Default.yaml +++ b/Config Default.yaml @@ -3,6 +3,7 @@ LogFile: "data/log.txt" # Log file. It contains informat BlockchainMain: "data/blockchain main/" # Blockchain main stores the end-users blockchain data. It contains meta data of shared files, profile data, and social interactions. BlockchainGlobal: "data/blockchain global/" # Blockchain global caches blockchain data from global users. Empty to disable. WarehouseMain: "data/warehouse main/" # Warehouse main stores the actual data of files shared by the end-user. +SearchIndex: "data/search index/" # Local search index of blockchain records. Empty to disable. # Listen defines all IP:Port combinations to listen on. If empty, it will listen on all IPs automatically on available ports. # IPv6 must be in the form "[IPv6]:Port". This setting is only recommended to be set on servers. diff --git a/Config.go b/Config.go index c4bc8fd..ae70fb3 100644 --- a/Config.go +++ b/Config.go @@ -25,6 +25,7 @@ var config struct { BlockchainMain string `yaml:"BlockchainMain"` // Blockchain main stores the end-users blockchain data. It contains meta data of shared files, profile data, and social interactions. BlockchainGlobal string `yaml:"BlockchainGlobal"` // Blockchain global caches blockchain data from global users. Empty to disable. WarehouseMain string `yaml:"WarehouseMain"` // Warehouse main stores the actual data of files shared by the end-user. + SearchIndex string `yaml:"SearchIndex"` // Local search index of blockchain records. Empty to disable. // Listen settings Listen []string `yaml:"Listen"` // IP:Port combinations diff --git a/Peernet.go b/Peernet.go index 00647fe..1866b4a 100644 --- a/Peernet.go +++ b/Peernet.go @@ -6,6 +6,10 @@ Author: Peter Kleissner package core +import ( + "github.com/PeernetOfficial/core/search" +) + var userAgent = "Peernet Core/0.1" // must be overwritten by the caller // Init initializes the client. The config must be loaded first! @@ -15,6 +19,9 @@ func Init(UserAgent string) (backend *Backend) { return } + backend = &Backend{} + currentBackend = backend + initFilters() initPeerID() initUserBlockchain() @@ -27,10 +34,14 @@ func Init(UserAgent string) (backend *Backend) { initStore() initNetwork() - backend = &Backend{} - backend.GlobalBlockchainCache = initBlockchainCache(config.BlockchainGlobal, config.CacheMaxBlockSize, config.CacheMaxBlockCount, config.LimitTotalRecords) + var err error - currentBackend = backend + backend.GlobalBlockchainCache = initBlockchainCache(config.BlockchainGlobal, config.CacheMaxBlockSize, config.CacheMaxBlockCount, config.LimitTotalRecords) + backend.SearchIndex, err = search.InitSearchIndexStore(config.SearchIndex) + + if err != nil { + Filters.LogError("Init", "search index '%s' init: %s", config.SearchIndex, err.Error()) + } return backend } @@ -49,7 +60,8 @@ func Connect() { // The Backend represents an instance of a Peernet client to be used by a frontend. // Global variables and init functions are to be merged. type Backend struct { - GlobalBlockchainCache *BlockchainCache // stores other peers blockchains + GlobalBlockchainCache *BlockchainCache // Caches blockchains of other peers. + SearchIndex *search.SearchIndexStore // Search index of blockchain records. } // This variable is to be replaced later by pointers in structures. diff --git a/search/Normalize.go b/search/Normalize.go new file mode 100644 index 0000000..0e339e7 --- /dev/null +++ b/search/Normalize.go @@ -0,0 +1,42 @@ +/* +File Name: Normalizing.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Normalizing text so that it can be hashed. +*/ + +package search + +import ( + "path" + "strings" +) + +func sanitizeGeneric(filename string) string { + filename = strings.TrimSpace(filename) + filename = strings.ToLower(filename) + filename = strings.ToValidUTF8(filename, "") + + return filename +} + +// sanitizeInputTerm sanitizes a search term provided by the end user. It includes the generic rules that are done on indexing. +func sanitizeInputTerm(inputTerm string) (outputTerm string, isExact, isWildcard bool) { + inputTerm = sanitizeGeneric(inputTerm) + + // detect and remove quotes at the beginning and end + isExact = false + if len(inputTerm) >= 2 && ((strings.HasPrefix(inputTerm, "\"") && strings.HasSuffix(inputTerm, "\"")) || (strings.HasPrefix(inputTerm, "'") && strings.HasSuffix(inputTerm, "'"))) { + inputTerm = inputTerm[1 : len(inputTerm)-1] + isExact = true + } + + isWildcard = strings.Contains(inputTerm, "*") + + return inputTerm, isExact, isWildcard +} + +func filenameRemoveExtension(filename string) string { + return strings.TrimSuffix(filename, path.Ext(filename)) +} diff --git a/search/Search Index.go b/search/Search Index.go new file mode 100644 index 0000000..cdcbf56 --- /dev/null +++ b/search/Search Index.go @@ -0,0 +1,282 @@ +/* +File Name: Search Index.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package search + +import ( + "encoding/binary" + "errors" + "sync" + + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/store" + "github.com/google/uuid" +) + +// SearchIndexRecord identifies a hash to a given file +type SearchIndexRecord struct { + // input data + Word string + Hash []byte + + // result data + FileID uuid.UUID + PublicKey *btcec.PublicKey + BlockNumber uint64 +} + +// This database stores hashes of keywords for file search. +type SearchIndexStore struct { + Database store.Store // The database storing the blockchain. + sync.RWMutex +} + +func InitSearchIndexStore(DatabaseDirectory string) (searchIndex *SearchIndexStore, err error) { + if DatabaseDirectory == "" { + return + } + + searchIndex = &SearchIndexStore{} + + if searchIndex.Database, err = store.NewPogrebStore(DatabaseDirectory); err != nil { + return nil, err + } + + return searchIndex, nil +} + +func (index *SearchIndexStore) IndexNewBlock(publicKey *btcec.PublicKey, blockNumber uint64, raw []byte) { + if index.Database == nil { + return + } + + // decode all files from the block + decoded, status, err := blockchain.DecodeBlockRaw(raw) + if err != nil || status != blockchain.StatusOK { + return + } + + for _, decodedR := range decoded.RecordsDecoded { + if file, ok := decodedR.(blockchain.BlockRecordFile); ok { + var filename, folder, description string + for _, tag := range file.Tags { + switch tag.Type { + case blockchain.TagName: + filename = sanitizeGeneric(tag.Text()) + case blockchain.TagFolder: + folder = sanitizeGeneric(tag.Text()) + case blockchain.TagDescription: + description = sanitizeGeneric(tag.Text()) + } + } + + hashes := make(map[[32]byte]string) + filename2Hashes(filename, folder, hashes) + text2Hashes(description, hashes) + + for hash := range hashes { + index.IndexHash(publicKey, blockNumber, file.ID, hash[:]) + } + } + } +} + +func (index *SearchIndexStore) UnindexBlock(publicKey *btcec.PublicKey, blockNumber uint64) { + if index.Database == nil { + return + } + + // get the reverse records + key := reverseIndexKey(publicKey, blockNumber) + raw, found := index.Database.Get(key) + + if !found || len(raw)%reverseIndexRecordSize != 0 { // corrupt record + return + } + + offset := 0 + + for n := 0; n < len(raw)/reverseIndexRecordSize; n++ { + var hash []byte + var fileID uuid.UUID + + hash = raw[offset : offset+32] + copy(fileID[:], raw[offset+32:offset+32+16]) + + // delete the index record + index.UnindexHash(fileID, hash) + } + + // delete the reverse record + index.Database.Delete(key) +} + +// IndexHash indexes a new hash +func (index *SearchIndexStore) IndexHash(publicKey *btcec.PublicKey, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { + if index.Database == nil { + return + } + + index.Lock() + defer index.Unlock() + + // parse existing records, check if already stored + raw, found := index.Database.Get(hash) + if found && len(raw)%indexRecordSize == 0 { // check if record is corrupt + offset := 0 + for n := 0; n < len(raw)/indexRecordSize; n++ { + if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil { + if fileID == record.FileID { + return errors.New("already indexed") + } + } + + offset += indexRecordSize + } + } + + raw = append(raw, encodeIndexRecord(publicKey, blockNumber, fileID)...) + + // create the reverse record + index.createReverseIndexRecord(publicKey, blockNumber, fileID, hash) + + return index.Database.Set(hash, raw) +} + +// UnindexHash deletes a index record. If there are no more files associated with the hash, the entire hash record is deleted. +func (index *SearchIndexStore) UnindexHash(fileID uuid.UUID, hash []byte) (err error) { + if index.Database == nil { + return + } + + index.Lock() + defer index.Unlock() + + var newRaw []byte + + raw, found := index.Database.Get(hash) + if !found { + return errors.New("index record not found") + } + + if len(raw)%indexRecordSize == 0 { // check if record is corrupt + offset := 0 + for n := 0; n < len(raw)/indexRecordSize; n++ { + if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil { + if fileID != record.FileID { + newRaw = append(newRaw, raw[offset:offset+indexRecordSize]...) + } + } + + offset += indexRecordSize + } + } + + if len(newRaw) == 0 { + // delete the entire hash key + index.Database.Delete(hash) + return + } + + return index.Database.Set(hash, newRaw) +} + +// LookupHash returns all index records stored for the hash. +func (index *SearchIndexStore) LookupHash(hash []byte) (records []SearchIndexRecord, err error) { + if index.Database == nil { + return + } + + index.RLock() + defer index.RUnlock() + + raw, found := index.Database.Get(hash) + if !found { + return nil, nil + } else if len(raw)%indexRecordSize != 0 { // check if record is corrupt + return nil, errors.New("corrupt index record") + } + + for offset := 0; offset < len(raw); offset += indexRecordSize { + if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil { + record.Hash = hash + records = append(records, *record) + } + } + + return records, nil +} + +// ---- index and reverse index code ---- + +/* +Structure for each index record: + +Offset Size Info +0 16 File ID +16 33 Public Key compressed +49 8 Block Number +*/ + +const indexRecordSize = 57 + +// decodeIndexRecord decodes the index record and sets the fields File ID, Public Key, and Block Number. +func decodeIndexRecord(raw []byte) (record *SearchIndexRecord) { + if len(raw) < indexRecordSize { + return + } + + record = &SearchIndexRecord{} + copy(record.FileID[:], raw[0:16]) + + var err error + if record.PublicKey, err = btcec.ParsePubKey(raw[16:16+33], btcec.S256()); err != nil { + return nil + } + + record.BlockNumber = binary.LittleEndian.Uint64(raw[49 : 49+8]) + + return record +} + +func encodeIndexRecord(publicKey *btcec.PublicKey, blockNumber uint64, fileID uuid.UUID) (raw []byte) { + raw = make([]byte, indexRecordSize) + + copy(raw[0:16], fileID[:]) + copy(raw[16:16+33], publicKey.SerializeCompressed()) + binary.LittleEndian.PutUint64(raw[49:49+8], blockNumber) + + return raw +} + +// This creates a reverse index record. It uses the blockchain and block number as key, and provides the hash and file ID as value. +// This function must be called in a RW locked database state. The caller must ensure that this does not result in a duplicate. +func (index *SearchIndexStore) createReverseIndexRecord(publicKey *btcec.PublicKey, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { + key := reverseIndexKey(publicKey, blockNumber) + raw, _ := index.Database.Get(key) + + // each record is only hash + file ID + reverseRecord := make([]byte, reverseIndexRecordSize) + copy(reverseRecord[0:32], hash) + copy(reverseRecord[32:32+16], fileID[:]) + + raw = append(raw, reverseRecord...) + + return index.Database.Set(key, raw) +} + +const reverseIndexRecordSize = 32 + 16 + +func reverseIndexKey(publicKey *btcec.PublicKey, blockNumber uint64) (key []byte) { + key = publicKey.SerializeCompressed() + + var temp [8]byte + binary.LittleEndian.PutUint64(temp[0:8], blockNumber) + key = append(key, temp[:]...) + + return key +} diff --git a/search/Search Term.go b/search/Search Term.go new file mode 100644 index 0000000..d982fbf --- /dev/null +++ b/search/Search Term.go @@ -0,0 +1,51 @@ +/* +File Name: Search Term.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package search + +func (index *SearchIndexStore) Search(term string) (results []SearchIndexRecord) { + termS, isExact, _ := sanitizeInputTerm(term) + + if len(termS) < wordMinLength { + return + } + + // start with exact search + hashExact := hashWord(termS) + if hashExact != nil { + records, _ := index.LookupHash(hashExact) + results = append(results, setSearchIndexFields(records, termS)...) + } + + // exact search only? + if isExact { + return + } + + // break up the term into hashes + hashes := make(map[[32]byte]string) + + text2Hashes(termS, hashes) + + // The exact search was already performed, exclude it. + hashMapDelete(hashExact, hashes) + + // look up the hashes! + for hash, keyword := range hashes { + records, _ := index.LookupHash(hash[:]) + results = append(results, setSearchIndexFields(records, keyword)...) + } + + return +} + +// setSearchIndexFields sets certain fields in the index record for adding context in the output +func setSearchIndexFields(records []SearchIndexRecord, word string) []SearchIndexRecord { + for n := range records { + records[n].Word = word + } + return records +} diff --git a/search/Text 2 Hash.go b/search/Text 2 Hash.go new file mode 100644 index 0000000..db4ff09 --- /dev/null +++ b/search/Text 2 Hash.go @@ -0,0 +1,83 @@ +/* +File Name: Text 2 Hash.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package search + +import ( + "strings" + "unicode" + + "lukechampine.com/blake3" +) + +// Words must be the minimum length to be indexed. +const wordMinLength = 3 + +func text2Hashes(text string, hashes map[[32]byte]string) { + words := strings.FieldsFunc(text, func(char rune) bool { + if unicode.IsSpace(char) { + return true + } + return strings.ContainsAny(string(char), "+-._()[],") + }) + + for _, word := range words { + hashWordMap(word, hashes) + } +} + +func filename2Hashes(filename, folder string, hashes map[[32]byte]string) { + if len(filename) < wordMinLength { + return + } + + // First hash is created on the entire filename including extension. This is done for perfect matches. + hashWordMap(filename, hashes) + + // Hash the filename without extension (and proceed without extension) + filename = filenameRemoveExtension(filename) + hashWordMap(filename, hashes) + + // Hash each individual word of the filename and directory + text2Hashes(filename, hashes) + + folder = strings.ReplaceAll(folder, "\\", " ") + folder = strings.ReplaceAll(folder, "/", " ") + text2Hashes(folder, hashes) +} + +// hashWordMap hashes a word and stores it on the map. This immediately deduplicated hashes. +func hashWordMap(word string, hashes map[[32]byte]string) { + word = strings.TrimSpace(word) + if len(word) < wordMinLength { + return + } + + hashes[blake3.Sum256([]byte(word))] = word +} + +// hashMapDelete deletes a hash from the list if the hash is valid +func hashMapDelete(hash []byte, hashes map[[32]byte]string) { + if len(hash) == 0 { + return + } + + var hashB [32]byte + copy(hashB[:], hash) + + delete(hashes, hashB) +} + +// hashWord hashes a single word. It returns nil if not suitable. +func hashWord(word string) (hash []byte) { + word = strings.TrimSpace(word) + if len(word) < wordMinLength { + return + } + + hashB := blake3.Sum256([]byte(word)) + return hashB[:] +} diff --git a/search/readme.md b/search/readme.md new file mode 100644 index 0000000..7298431 --- /dev/null +++ b/search/readme.md @@ -0,0 +1,17 @@ +# Search Index + +## Search Term Normalization + +The user input search term undergoes normalization: +1. Trim space +2. Lowercase +3. Remove invalid UTF-8 characters +4. Detect and remove quotes in the form '" (activates exact search mode) + +Wildcards are not supported. + +## Generic Text Normalization + +1. Trim space +2. Lowercase +3. Remove invalid UTF-8 characters