Search index: Change unindexing algorithm on entire blockchain level instead of version+block number. Fixing bug in unindexing.

Blockchain Cache Global.go

@@ -98,9 +98,7 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) {
 	case blockchain.MultiStatusInvalidRemote:
 		cache.store.DeleteBlockchain(peer.PublicKey, header)
 
-		for _, blockN := range header.ListBlocks {
-			currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, header.Version, blockN)
-		}
+		currentBackend.SearchIndex.UnindexBlockchain(peer.PublicKey)
 
 	case blockchain.MultiStatusHeaderNA:
 		if header, err = cache.store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil {
@@ -113,9 +111,7 @@ func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) {
 		// delete existing data first, then create it new
 		cache.store.DeleteBlockchain(peer.PublicKey, header)
 
-		for _, blockN := range header.ListBlocks {
-			currentBackend.SearchIndex.UnindexBlock(peer.PublicKey, header.Version, blockN)
-		}
+		currentBackend.SearchIndex.UnindexBlockchain(peer.PublicKey)
 
 		if header, err = cache.store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil {
 			return
@@ -151,14 +147,9 @@ func (peer *PeerInfo) remoteBlockchainUpdate() {
 }
 
 func (cache *BlockchainCache) ReadFile(PublicKey *btcec.PublicKey, Version, BlockNumber uint64, FileID uuid.UUID) (file blockchain.BlockRecordFile, raw []byte, found bool, err error) {
-	if raw, found = cache.store.ReadBlock(PublicKey, Version, BlockNumber); !found {
-		return file, nil, false, nil
-	}
-
-	// decode the entire block and find the file based on its ID
-	blockDecoded, status, err := blockchain.DecodeBlockRaw(raw)
-	if err != nil || status != blockchain.StatusOK {
-		return file, raw, false, err
+	blockDecoded, raw, found, err := cache.ReadBlock(PublicKey, Version, BlockNumber)
+	if !found {
+		return file, raw, found, err
 	}
 
 	for _, decodedR := range blockDecoded.RecordsDecoded {
@@ -169,3 +160,18 @@ func (cache *BlockchainCache) ReadFile(PublicKey *btcec.PublicKey, Version, Bloc
 
 	return file, raw, false, nil
 }
+
+// ReadBlock reads a block and decodes the records.
+func (cache *BlockchainCache) ReadBlock(PublicKey *btcec.PublicKey, Version, BlockNumber uint64) (decoded *blockchain.BlockDecoded, raw []byte, found bool, err error) {
+	if raw, found = cache.store.ReadBlock(PublicKey, Version, BlockNumber); !found {
+		return nil, nil, false, nil
+	}
+
+	// decode the entire block
+	blockDecoded, status, err := blockchain.DecodeBlockRaw(raw)
+	if err != nil || status != blockchain.StatusOK {
+		return nil, raw, false, err
+	}
+
+	return blockDecoded, raw, true, nil
+}

search/Search Index.go

@@ -86,22 +86,21 @@ func (index *SearchIndexStore) IndexNewBlock(publicKey *btcec.PublicKey, blockch
 	}
 }
 
-func (index *SearchIndexStore) UnindexBlock(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64) {
+// UnindexBlockchain deletes all index for a given blockchain. This is intentionally not done on a version/block level, because it could easily lead to orphans.
+func (index *SearchIndexStore) UnindexBlockchain(publicKey *btcec.PublicKey) {
 	if index.Database == nil {
 		return
 	}
 
-	// get the reverse records
-	key := reverseIndexKey(publicKey, blockchainVersion, blockNumber)
+	// get the reverse record
+	key := publicKey.SerializeCompressed()
 	raw, found := index.Database.Get(key)
 
 	if !found || len(raw)%reverseIndexRecordSize != 0 { // corrupt record
 		return
 	}
 
-	offset := 0
-
-	for n := 0; n < len(raw)/reverseIndexRecordSize; n++ {
+	for offset := 0; offset < len(raw); offset += reverseIndexRecordSize {
 		var hash []byte
 		var fileID uuid.UUID
 
@@ -128,15 +127,12 @@ func (index *SearchIndexStore) IndexHash(publicKey *btcec.PublicKey, blockchainV
 	// parse existing records, check if already stored
 	raw, found := index.Database.Get(hash)
 	if found && len(raw)%indexRecordSize == 0 { // check if record is corrupt
-		offset := 0
-		for n := 0; n < len(raw)/indexRecordSize; n++ {
+		for offset := 0; offset < len(raw); offset += indexRecordSize {
 			if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil {
 				if fileID == record.FileID {
 					return errors.New("already indexed")
 				}
 			}
-
-			offset += indexRecordSize
 		}
 	}
 
@@ -165,15 +161,12 @@ func (index *SearchIndexStore) UnindexHash(fileID uuid.UUID, hash []byte) (err e
 	}
 
 	if len(raw)%indexRecordSize == 0 { // check if record is corrupt
-		offset := 0
-		for n := 0; n < len(raw)/indexRecordSize; n++ {
+		for offset := 0; offset < len(raw); offset += indexRecordSize {
 			if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil {
 				if fileID != record.FileID {
 					newRaw = append(newRaw, raw[offset:offset+indexRecordSize]...)
 				}
 			}
-
-			offset += indexRecordSize
 		}
 	}
 
@@ -257,10 +250,13 @@ func encodeIndexRecord(publicKey *btcec.PublicKey, blockchainVersion, blockNumbe
 	return raw
 }
 
+// Reverse index records keep track of all hashes and file IDs searchable for a given blockchain.
+const reverseIndexRecordSize = 32 + 16
+
 // This creates a reverse index record. It uses the blockchain and block number as key, and provides the hash and file ID as value.
 // This function must be called in a RW locked database state. The caller must ensure that this does not result in a duplicate.
 func (index *SearchIndexStore) createReverseIndexRecord(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) {
-	key := reverseIndexKey(publicKey, blockchainVersion, blockNumber)
+	key := publicKey.SerializeCompressed()
 	raw, _ := index.Database.Get(key)
 
 	// each record is only hash + file ID
@@ -272,16 +268,3 @@ func (index *SearchIndexStore) createReverseIndexRecord(publicKey *btcec.PublicK
 
 	return index.Database.Set(key, raw)
 }
-
-const reverseIndexRecordSize = 32 + 16
-
-func reverseIndexKey(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64) (key []byte) {
-	key = publicKey.SerializeCompressed()
-
-	var temp [16]byte
-	binary.LittleEndian.PutUint64(temp[0:8], blockchainVersion)
-	binary.LittleEndian.PutUint64(temp[8:16], blockNumber)
-	key = append(key, temp[:]...)
-
-	return key
-}