From f9c661b0034a88323185f55a1a21c27bdfead9f9 Mon Sep 17 00:00:00 2001 From: Kleissner Date: Mon, 13 Dec 2021 00:05:54 +0100 Subject: [PATCH] Search index: Add blockchain version to the records. --- Blockchain Cache Global.go | 28 +++++++++++++++++++++--- search/Search Index.go | 45 +++++++++++++++++++++----------------- 2 files changed, 50 insertions(+), 23 deletions(-) diff --git a/Blockchain Cache Global.go b/Blockchain Cache Global.go index b378975..3c4a943 100644 --- a/Blockchain Cache Global.go +++ b/Blockchain Cache Global.go @@ -8,8 +8,10 @@ package core import ( "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/btcec" "github.com/PeernetOfficial/core/protocol" "github.com/enfipy/locker" + "github.com/google/uuid" ) // The blockchain cache stores blockchains. @@ -74,7 +76,7 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) { cache.store.WriteBlock(peer.PublicKey, peer.BlockchainVersion, targetBlock.Offset, data) header.ListBlocks = append(header.ListBlocks, targetBlock.Offset) - currentBackend.SearchIndex.IndexNewBlock(peer.PublicKey, targetBlock.Offset, data) + currentBackend.SearchIndex.IndexNewBlock(peer.PublicKey, peer.BlockchainVersion, targetBlock.Offset, data) }) // only update the blockchain header if it changed @@ -97,7 +99,7 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) { cache.store.DeleteBlockchain(peer.PublicKey, header) for _, blockN := range header.ListBlocks { - currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, blockN) + currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, header.Version, blockN) } case blockchain.MultiStatusHeaderNA: @@ -112,7 +114,7 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) { cache.store.DeleteBlockchain(peer.PublicKey, header) for _, blockN := range header.ListBlocks { - currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, blockN) + currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, header.Version, blockN) } if header, err = cache.store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { @@ -147,3 +149,23 @@ func (peer *PeerInfo) remoteBlockchainUpdate() { // TODO: This entire function should be instead a non-blocking message via a buffer channel. go currentBackend.GlobalBlockchainCache.SeenBlockchainVersion(peer) } + +func (cache *BlockchainCache) ReadFile(PublicKey *btcec.PublicKey, Version, BlockNumber uint64, FileID uuid.UUID) (file blockchain.BlockRecordFile, raw []byte, found bool, err error) { + if raw, found = cache.store.ReadBlock(PublicKey, Version, BlockNumber); !found { + return file, nil, false, nil + } + + // decode the entire block and find the file based on its ID + blockDecoded, status, err := blockchain.DecodeBlockRaw(raw) + if err != nil || status != blockchain.StatusOK { + return file, raw, false, err + } + + for _, decodedR := range blockDecoded.RecordsDecoded { + if file, ok := decodedR.(blockchain.BlockRecordFile); ok && file.ID == FileID { + return file, raw, true, nil + } + } + + return file, raw, false, nil +} diff --git a/search/Search Index.go b/search/Search Index.go index cdcbf56..38148f4 100644 --- a/search/Search Index.go +++ b/search/Search Index.go @@ -24,9 +24,10 @@ type SearchIndexRecord struct { Hash []byte // result data - FileID uuid.UUID - PublicKey *btcec.PublicKey - BlockNumber uint64 + FileID uuid.UUID + PublicKey *btcec.PublicKey + BlockchainVersion uint64 + BlockNumber uint64 } // This database stores hashes of keywords for file search. @@ -49,7 +50,7 @@ func InitSearchIndexStore(DatabaseDirectory string) (searchIndex *SearchIndexSto return searchIndex, nil } -func (index *SearchIndexStore) IndexNewBlock(publicKey *btcec.PublicKey, blockNumber uint64, raw []byte) { +func (index *SearchIndexStore) IndexNewBlock(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, raw []byte) { if index.Database == nil { return } @@ -79,19 +80,19 @@ func (index *SearchIndexStore) IndexNewBlock(publicKey *btcec.PublicKey, blockNu text2Hashes(description, hashes) for hash := range hashes { - index.IndexHash(publicKey, blockNumber, file.ID, hash[:]) + index.IndexHash(publicKey, blockchainVersion, blockNumber, file.ID, hash[:]) } } } } -func (index *SearchIndexStore) UnindexBlock(publicKey *btcec.PublicKey, blockNumber uint64) { +func (index *SearchIndexStore) UnindexBlock(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64) { if index.Database == nil { return } // get the reverse records - key := reverseIndexKey(publicKey, blockNumber) + key := reverseIndexKey(publicKey, blockchainVersion, blockNumber) raw, found := index.Database.Get(key) if !found || len(raw)%reverseIndexRecordSize != 0 { // corrupt record @@ -116,7 +117,7 @@ func (index *SearchIndexStore) UnindexBlock(publicKey *btcec.PublicKey, blockNum } // IndexHash indexes a new hash -func (index *SearchIndexStore) IndexHash(publicKey *btcec.PublicKey, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { +func (index *SearchIndexStore) IndexHash(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { if index.Database == nil { return } @@ -139,10 +140,10 @@ func (index *SearchIndexStore) IndexHash(publicKey *btcec.PublicKey, blockNumber } } - raw = append(raw, encodeIndexRecord(publicKey, blockNumber, fileID)...) + raw = append(raw, encodeIndexRecord(publicKey, blockchainVersion, blockNumber, fileID)...) // create the reverse record - index.createReverseIndexRecord(publicKey, blockNumber, fileID, hash) + index.createReverseIndexRecord(publicKey, blockchainVersion, blockNumber, fileID, hash) return index.Database.Set(hash, raw) } @@ -219,10 +220,11 @@ Structure for each index record: Offset Size Info 0 16 File ID 16 33 Public Key compressed -49 8 Block Number +49 8 Blockchain Version +57 8 Block Number */ -const indexRecordSize = 57 +const indexRecordSize = 65 // decodeIndexRecord decodes the index record and sets the fields File ID, Public Key, and Block Number. func decodeIndexRecord(raw []byte) (record *SearchIndexRecord) { @@ -238,25 +240,27 @@ func decodeIndexRecord(raw []byte) (record *SearchIndexRecord) { return nil } - record.BlockNumber = binary.LittleEndian.Uint64(raw[49 : 49+8]) + record.BlockchainVersion = binary.LittleEndian.Uint64(raw[49 : 49+8]) + record.BlockNumber = binary.LittleEndian.Uint64(raw[57 : 57+8]) return record } -func encodeIndexRecord(publicKey *btcec.PublicKey, blockNumber uint64, fileID uuid.UUID) (raw []byte) { +func encodeIndexRecord(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID) (raw []byte) { raw = make([]byte, indexRecordSize) copy(raw[0:16], fileID[:]) copy(raw[16:16+33], publicKey.SerializeCompressed()) - binary.LittleEndian.PutUint64(raw[49:49+8], blockNumber) + binary.LittleEndian.PutUint64(raw[49:49+8], blockchainVersion) + binary.LittleEndian.PutUint64(raw[57:57+8], blockNumber) return raw } // This creates a reverse index record. It uses the blockchain and block number as key, and provides the hash and file ID as value. // This function must be called in a RW locked database state. The caller must ensure that this does not result in a duplicate. -func (index *SearchIndexStore) createReverseIndexRecord(publicKey *btcec.PublicKey, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { - key := reverseIndexKey(publicKey, blockNumber) +func (index *SearchIndexStore) createReverseIndexRecord(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { + key := reverseIndexKey(publicKey, blockchainVersion, blockNumber) raw, _ := index.Database.Get(key) // each record is only hash + file ID @@ -271,11 +275,12 @@ func (index *SearchIndexStore) createReverseIndexRecord(publicKey *btcec.PublicK const reverseIndexRecordSize = 32 + 16 -func reverseIndexKey(publicKey *btcec.PublicKey, blockNumber uint64) (key []byte) { +func reverseIndexKey(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64) (key []byte) { key = publicKey.SerializeCompressed() - var temp [8]byte - binary.LittleEndian.PutUint64(temp[0:8], blockNumber) + var temp [16]byte + binary.LittleEndian.PutUint64(temp[0:8], blockchainVersion) + binary.LittleEndian.PutUint64(temp[8:16], blockNumber) key = append(key, temp[:]...) return key