Major refactoring of blockchain code into new sub-package.

2026-07-20 11:57:51 +01:00 · 2021-10-11 01:21:38 +02:00
parent 7495b823e4
commit fbeb01e7ff
16 changed files with 818 additions and 744 deletions
--- a/blockchain/Blockchain.go
+++ b/blockchain/Blockchain.go
@@ -0,0 +1,471 @@
+/*
+File Name:  Blockchain.go
+Copyright:  2021 Peernet s.r.o.
+Author:     Peter Kleissner
+
+All blocks and the blockchain header are stored in a key/value database.
+The key for the blockchain header is keyHeader and for each block is the block number as 64-bit unsigned integer little endian.
+
+Encoding of the blockchain header:
+Offset  Size   Info
+0       8      Height of the blockchain
+8       8      Version of the blockchain
+16      2      Format of the blockchain. This provides backward compatibility.
+18      65     Signature
+
+*/
+
+package blockchain
+
+import (
+	"encoding/binary"
+	"errors"
+	"sync"
+
+	"github.com/PeernetOfficial/core/store"
+	"github.com/btcsuite/btcd/btcec"
+	"github.com/google/uuid"
+)
+
+// Blockchain stores the blockchain's header in memory. Any changes must be synced to disk!
+type Blockchain struct {
+	// header
+	height  uint64 // Height is exchanged as uint32 in the protocol, but stored as uint64.
+	version uint64 // Version is always uint64.
+	format  uint16 // Format is only locally used.
+
+	// internals
+	publicKey  *btcec.PublicKey  // Public Key of the owner. This must match the ones used on disk.
+	privateKey *btcec.PrivateKey // Private Key of the owner. This must match the ones used on disk.
+	path       string            // Path of the blockchain on disk. Depends on key-value store whether a filename or folder.
+	database   store.Store       // The database storing the blockchain.
+	sync.Mutex                   // synchronized access to the header
+}
+
+// HashFunction must be set by the caller to the hash function (blake3) that shall be used.
+var HashFunction func(data []byte) (hash []byte)
+
+// hashSize is the blake3 digest size in bytes
+const hashSize = 32
+
+// PublicKey2NodeID must be set by the caller to the function generating the node ID from the public key. Typically it would use the HashFunction.
+var PublicKey2NodeID func(publicKey *btcec.PublicKey) (nodeID []byte)
+
+// Init initializes the given blockchain. It creates the blockchain file if it does not exist already.
+func Init(privateKey *btcec.PrivateKey, path string) (blockchain *Blockchain, err error) {
+	blockchain = &Blockchain{privateKey: privateKey, path: path}
+	publicKey := privateKey.PubKey()
+
+	// open existing blockchain file or create new one
+	if blockchain.database, err = store.NewPogrebStore(path); err != nil {
+		return nil, err
+	}
+
+	// verify header
+	var found bool
+
+	found, err = blockchain.headerRead()
+	if err != nil {
+		return blockchain, err // likely corrupt blockchain database
+	} else if !found {
+		// First run: create header signature!
+		blockchain.publicKey = publicKey
+
+		if err := blockchain.headerWrite(0, 0); err != nil {
+			return blockchain, err
+		}
+	} else if !blockchain.publicKey.IsEqual(publicKey) {
+		return blockchain, errors.New("corrupt user blockchain database. Public key mismatch")
+	}
+
+	return blockchain, nil
+}
+
+// the key names in the key/value database are constant and must not collide with block numbers (i.e. they must be >64 bit)
+const keyHeader = "header blockchain"
+
+// headerRead reads the header from the blockchain and decodes it.
+func (blockchain *Blockchain) headerRead() (found bool, err error) {
+	buffer, found := blockchain.database.Get([]byte(keyHeader))
+	if !found {
+		return false, nil
+	}
+
+	if len(buffer) != 83 {
+		return true, errors.New("blockchain header size mismatch")
+	}
+
+	blockchain.height = binary.LittleEndian.Uint64(buffer[0:8])
+	blockchain.version = binary.LittleEndian.Uint64(buffer[8:16])
+	blockchain.format = binary.LittleEndian.Uint16(buffer[16:18])
+	signature := buffer[18 : 18+65]
+
+	if blockchain.format != 0 {
+		return true, errors.New("future blockchain format not supported. You must go back to the future!")
+	}
+
+	blockchain.publicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, HashFunction(buffer[0:18]))
+
+	return
+}
+
+// headerWrite writes the header to the blockchain and signs it.
+func (blockchain *Blockchain) headerWrite(height, version uint64) (err error) {
+	blockchain.height = height
+	blockchain.version = version
+
+	var buffer [83]byte
+	binary.LittleEndian.PutUint64(buffer[0:8], height)
+	binary.LittleEndian.PutUint64(buffer[8:16], version)
+	binary.LittleEndian.PutUint16(buffer[16:18], 0) // Current format is 0
+
+	signature, err := btcec.SignCompact(btcec.S256(), blockchain.privateKey, HashFunction(buffer[0:18]), true)
+
+	if err != nil {
+		return err
+	} else if len(signature) != 65 {
+		return errors.New("signature length invalid")
+	}
+
+	copy(buffer[18:18+65], signature)
+
+	err = blockchain.database.Set([]byte(keyHeader), buffer[:])
+
+	return err
+}
+
+// BlockchainStatusX provides information about the blockchain status. Some errors codes indicate a corruption.
+const (
+	BlockchainStatusOK                 = 0 // No problems in the blockchain detected.
+	BlockchainStatusBlockNotFound      = 1 // Missing block in the blockchain.
+	BlockchainStatusCorruptBlock       = 2 // Error block encoding
+	BlockchainStatusCorruptBlockRecord = 3 // Error block record encoding
+	BlockchainStatusDataNotFound       = 4 // Requested data not available in the blockchain
+)
+
+// blockNumberToKey returns the database key for the given block number
+func blockNumberToKey(number uint64) (key []byte) {
+	var target [8]byte
+	binary.LittleEndian.PutUint64(target[:], number)
+
+	return target[:]
+}
+
+// Iterate iterates over the blockchain. Status is BlockchainStatusX.
+// If the callback returns non-zero, the function aborts and returns the inner status code.
+func (blockchain *Blockchain) Iterate(callback func(block *Block) int) (status int) {
+	// read all blocks until height is reached
+	height := blockchain.height
+
+	for blockN := uint64(0); blockN < height; blockN++ {
+		blockRaw, found := blockchain.database.Get(blockNumberToKey(blockN))
+		if !found || len(blockRaw) == 0 {
+			return BlockchainStatusBlockNotFound
+		}
+
+		block, err := decodeBlock(blockRaw)
+		if err != nil {
+			return BlockchainStatusCorruptBlock
+		}
+
+		if statusI := callback(block); statusI != BlockchainStatusOK {
+			return statusI
+		}
+	}
+
+	return BlockchainStatusOK
+}
+
+// IterateDeleteRecord iterates over the blockchain to find records to delete. Status is BlockchainStatusX.
+// deleteAction is 0 = no action on record, 1 = delete record, 2 = replace record, 3 = error blockchain corrupt
+// If the callback returns true, the record will be deleted. The blockchain will be automatically refactored and height and version updated.
+func (blockchain *Blockchain) IterateDeleteRecord(callback func(record *BlockRecordRaw) (deleteAction int)) (newHeight, newVersion uint64, status int) {
+	blockchain.Lock()
+	defer blockchain.Unlock()
+
+	// New blockchain keeps track of the new blocks. If anything changes in the blockchain, it must be recalculated and the version number increased.
+	var blockchainNew []Block
+	refactorBlockchain := false
+	refactorVersion := blockchain.version + 1
+
+	// Read all blocks until height is reached. At the end the height and version might be different if blocks are deleted.
+	height := blockchain.height
+
+	for blockN := uint64(0); blockN < height; blockN++ {
+		blockRaw, found := blockchain.database.Get(blockNumberToKey(blockN))
+		if !found || len(blockRaw) == 0 {
+			return 0, 0, BlockchainStatusBlockNotFound
+		}
+
+		block, err := decodeBlock(blockRaw)
+		if err != nil {
+			return 0, 0, BlockchainStatusCorruptBlock
+		}
+
+		// loop through all records in this block
+		refactorBlock := false
+		var newRecordsRaw []BlockRecordRaw
+
+		for n := range block.RecordsRaw {
+			switch callback(&block.RecordsRaw[n]) {
+			case 0: // no action on record
+				newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n])
+
+			case 1: // delete record
+				refactorBlock = true
+				refactorBlockchain = true
+
+			case 2: // replace record
+				newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n])
+				refactorBlock = true
+				refactorBlockchain = true
+
+			case 3: // error blockchain corrupt
+				return 0, 0, BlockchainStatusCorruptBlockRecord
+			}
+		}
+
+		// If refactor, re-calculate the block. All later blocks need to be re-encoded due to change of previous block hash. The version number needs to change.
+		if refactorBlock {
+			if len(newRecordsRaw) > 0 {
+				blockchainNew = append(blockchainNew, Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: newRecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))})
+			}
+		} else {
+			blockchainNew = append(blockchainNew, Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: block.RecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))})
+		}
+	}
+
+	if refactorBlockchain {
+		var lastBlockHash []byte
+
+		for _, block := range blockchainNew {
+			block.LastBlockHash = lastBlockHash
+
+			raw, err := encodeBlock(&block, blockchain.privateKey)
+			if err != nil {
+				return 0, 0, BlockchainStatusCorruptBlock
+			}
+
+			// store the block
+			blockchain.database.Set(blockNumberToKey(block.Number), raw)
+
+			lastBlockHash = HashFunction(raw)
+		}
+
+		// update the blockchain header in the database
+		blockchain.headerWrite(uint64(len(blockchainNew)), refactorVersion)
+
+		// delete orphaned blocks
+		for n := blockchain.height; n < height; n++ {
+			blockchain.database.Delete(blockNumberToKey(n))
+		}
+	}
+
+	return blockchain.height, blockchain.version, BlockchainStatusOK
+}
+
+// ---- blockchain manipulation functions ----
+
+// Header returns the users blockchain header which stores the height and version number.
+func (blockchain *Blockchain) Header() (publicKey *btcec.PublicKey, height uint64, version uint64) {
+	blockchain.Lock()
+	defer blockchain.Unlock()
+
+	return blockchain.publicKey, blockchain.height, blockchain.version
+}
+
+// Append appends a new block to the blockchain based on the provided raw records.
+// Status: BlockchainStatusX (0-2): 0 = Success, 1 = Error block not found, 2 = Error block encoding
+func (blockchain *Blockchain) Append(RecordsRaw []BlockRecordRaw) (newHeight, newVersion uint64, status int) {
+	blockchain.Lock()
+	defer blockchain.Unlock()
+
+	block := &Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: RecordsRaw}
+
+	// set the last block hash first
+	if blockchain.height > 0 {
+		previousBlockRaw, found := blockchain.database.Get(blockNumberToKey(blockchain.height - 1))
+		if !found || len(previousBlockRaw) == 0 {
+			return 0, 0, BlockchainStatusBlockNotFound
+		}
+
+		block.LastBlockHash = HashFunction(previousBlockRaw)
+	}
+
+	block.Number = blockchain.height
+	block.BlockchainVersion = blockchain.version
+
+	raw, err := encodeBlock(block, blockchain.privateKey)
+	if err != nil {
+		return 0, 0, BlockchainStatusCorruptBlock
+	}
+
+	// store the block
+	blockchain.database.Set(blockNumberToKey(block.Number), raw)
+
+	// update the blockchain header in the database, increase blockchain height
+	blockchain.headerWrite(blockchain.height+1, blockchain.version)
+
+	return blockchain.height, blockchain.version, BlockchainStatusOK
+}
+
+// Read reads the block number from the blockchain. Status is BlockchainStatusX.
+func (blockchain *Blockchain) Read(number uint64) (decoded *BlockDecoded, status int, err error) {
+	if number >= blockchain.height {
+		return nil, BlockchainStatusBlockNotFound, errors.New("block number exceeds blockchain height")
+	}
+
+	blockRaw, found := blockchain.database.Get(blockNumberToKey(number))
+	if !found || len(blockRaw) == 0 {
+		return nil, BlockchainStatusBlockNotFound, errors.New("block not found")
+	}
+
+	block, err := decodeBlock(blockRaw)
+	if err != nil {
+		return nil, BlockchainStatusCorruptBlock, err
+	}
+
+	decoded, err = decodeBlockRecords(block)
+	if err != nil {
+		return nil, BlockchainStatusCorruptBlock, err
+	}
+
+	return decoded, BlockchainStatusOK, nil
+}
+
+// AddFiles adds files to the blockchain. Status is BlockchainStatusX.
+// It makes sense to group all files in the same directory into one call, since only one directory record will be created per unique directory per block.
+func (blockchain *Blockchain) AddFiles(files []BlockRecordFile) (newHeight, newVersion uint64, status int) {
+	encoded, err := encodeBlockRecordFiles(files)
+	if err != nil {
+		return 0, 0, BlockchainStatusCorruptBlockRecord
+	}
+
+	return blockchain.Append(encoded)
+}
+
+// ListFiles returns a list of all files. Status is BlockchainStatusX.
+// If there is a corruption in the blockchain it will stop reading but return the files parsed so far.
+func (blockchain *Blockchain) ListFiles() (files []BlockRecordFile, status int) {
+	status = blockchain.Iterate(func(block *Block) (statusI int) {
+		filesMore, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID)
+		if err != nil {
+			return BlockchainStatusCorruptBlockRecord
+		}
+		files = append(files, filesMore...)
+
+		return BlockchainStatusOK
+	})
+
+	return files, status
+}
+
+// ProfileReadField reads the specified profile field. See ProfileX for the list of recognized fields. The encoding depends on the field type. Status is BlockchainStatusX.
+func (blockchain *Blockchain) ProfileReadField(index uint16) (data []byte, status int) {
+	found := false
+
+	status = blockchain.Iterate(func(block *Block) (statusI int) {
+		fields, err := decodeBlockRecordProfile(block.RecordsRaw)
+		if err != nil {
+			return BlockchainStatusCorruptBlockRecord
+		} else if len(fields) == 0 {
+			return BlockchainStatusOK
+		}
+
+		// Check if the field is available in the profile record. If there are multiple records, only return the latest one.
+		for n := range fields {
+			if fields[n].Type == index {
+				data = fields[n].Data
+				found = true
+			}
+		}
+
+		return BlockchainStatusOK
+	})
+
+	if status != BlockchainStatusOK {
+		return nil, status
+	} else if !found {
+		return nil, BlockchainStatusDataNotFound
+	}
+
+	return data, BlockchainStatusOK
+}
+
+// ProfileList lists all profile fields. Status is BlockchainStatusX.
+func (blockchain *Blockchain) ProfileList() (fields []BlockRecordProfile, status int) {
+	uniqueFields := make(map[uint16][]byte)
+
+	status = blockchain.Iterate(func(block *Block) (statusI int) {
+		fields, err := decodeBlockRecordProfile(block.RecordsRaw)
+		if err != nil {
+			return BlockchainStatusCorruptBlockRecord
+		}
+
+		for n := range fields {
+			uniqueFields[fields[n].Type] = fields[n].Data
+		}
+
+		return BlockchainStatusOK
+	})
+
+	for key, value := range uniqueFields {
+		fields = append(fields, BlockRecordProfile{Type: key, Data: value})
+	}
+
+	return fields, status
+}
+
+// ProfileWrite writes profile fields and blobs to the blockchain. Status is BlockchainStatusX.
+func (blockchain *Blockchain) ProfileWrite(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) {
+	encoded, err := encodeBlockRecordProfile(fields)
+	if err != nil {
+		return 0, 0, BlockchainStatusCorruptBlockRecord
+	}
+
+	return blockchain.Append(encoded)
+}
+
+// ProfileDelete deletes fields and blobs from the blockchain. Status is BlockchainStatusX.
+func (blockchain *Blockchain) ProfileDelete(fields []uint16) (newHeight, newVersion uint64, status int) {
+	return blockchain.IterateDeleteRecord(func(record *BlockRecordRaw) (deleteAction int) {
+		if record.Type != RecordTypeProfile {
+			return 0 // no action
+		}
+
+		existingFields, err := decodeBlockRecordProfile([]BlockRecordRaw{*record})
+		if err != nil || len(existingFields) != 1 {
+			return 3 // error blockchain corrupt
+		}
+
+		for _, i := range fields {
+			if i == existingFields[0].Type { // found a file ID to delete?
+				return 1 // delete record
+			}
+		}
+
+		return 0 // no action on record
+	})
+}
+
+// DeleteFiles deletes files from the blockchain. Status is BlockchainStatusX.
+func (blockchain *Blockchain) DeleteFiles(IDs []uuid.UUID) (newHeight, newVersion uint64, status int) {
+	return blockchain.IterateDeleteRecord(func(record *BlockRecordRaw) (deleteAction int) {
+		if record.Type != RecordTypeFile {
+			return 0 // no action on record
+		}
+
+		filesDecoded, err := decodeBlockRecordFiles([]BlockRecordRaw{*record}, nil)
+		if err != nil || len(filesDecoded) != 1 {
+			return 3 // error blockchain corrupt
+		}
+
+		for _, id := range IDs {
+			if id == filesDecoded[0].ID { // found a file ID to delete?
+				return 1 // delete record
+			}
+		}
+
+		return 0 // no action on record
+	})
+}