From fbeb01e7fffc34ce715ec9beb86bc9bdc53b75af Mon Sep 17 00:00:00 2001 From: Kleissner Date: Mon, 11 Oct 2021 01:21:38 +0200 Subject: [PATCH] Major refactoring of blockchain code into new sub-package. --- .gitignore | 3 +- Blockchain.go | 453 +---------------- Config.go | 2 +- Message Encoding.go | 12 +- Test_test.go | 234 --------- .../Block Encoding.go | 6 +- .../Block Records.go | 2 +- blockchain/Blockchain.go | 471 ++++++++++++++++++ File Metadata.go => blockchain/File Tag.go | 4 +- Profile Data.go => blockchain/Profile Data.go | 2 +- blockchain/Test_test.go | 265 ++++++++++ webapi/Blockchain.go | 17 +- webapi/File.go | 45 +- webapi/Profile.go | 25 +- webapi/Search Job.go | 6 +- webapi/Search Temp.go | 15 +- 16 files changed, 818 insertions(+), 744 deletions(-) rename Block Encoding.go => blockchain/Block Encoding.go (95%) rename Block Records.go => blockchain/Block Records.go (96%) create mode 100644 blockchain/Blockchain.go rename File Metadata.go => blockchain/File Tag.go (95%) rename Profile Data.go => blockchain/Profile Data.go (95%) create mode 100644 blockchain/Test_test.go diff --git a/.gitignore b/.gitignore index d000ff3..092f84b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.exe log.txt *debug_bin -self.* \ No newline at end of file +self.* +test.* \ No newline at end of file diff --git a/Blockchain.go b/Blockchain.go index 737d809..e03d165 100644 --- a/Blockchain.go +++ b/Blockchain.go @@ -2,470 +2,33 @@ File Name: Blockchain.go Copyright: 2021 Peernet s.r.o. Author: Peter Kleissner - -All blocks and the blockchain header are stored in a key/value database. -The key for the blockchain header is keyHeader and for each block is the block number as 64-bit unsigned integer little endian. - -Encoding of the blockchain header: -Offset Size Info -0 8 Height of the blockchain -8 8 Version of the blockchain -16 2 Format of the blockchain. This provides backward compatibility. -18 65 Signature - */ package core import ( - "encoding/binary" - "encoding/hex" - "errors" "os" - "sync" - "github.com/PeernetOfficial/core/store" - "github.com/btcsuite/btcd/btcec" - "github.com/google/uuid" + "github.com/PeernetOfficial/core/blockchain" ) -// BlockchainHeight is the current count of blocks -var BlockchainHeight = uint32(0) - -// BlockchainVersion is the version of the blockchain -var BlockchainVersion = uint64(0) +// UserBlockchain is the user's blockchain and exports functions to directly read and write it +var UserBlockchain *blockchain.Blockchain // filenameUserBlockchain is the filename/folder of the user's blockchain const filenameUserBlockchain = "self.blockchain" -// the key names in the key/value database are constant and must not collide with block numbers (i.e. they must be >64 bit) -const keyHeader = "header blockchain" - -// userBlockchainHeader stores the users blockchain header in memory. Any changes must be synced to disk! -var userBlockchainHeader struct { - height uint64 - version uint64 - format uint16 - publicKey *btcec.PublicKey - sync.Mutex -} - -var userBlockchainDB store.Store - // initUserBlockchain initializes the users blockchain. It creates the blockchain file if it does not exist already. // If it is corrupted, it will log the error and exit the process. func initUserBlockchain() { - // open existing blockchain file or create new one + blockchain.HashFunction = hashData + blockchain.PublicKey2NodeID = PublicKey2NodeID + var err error - if userBlockchainDB, err = store.NewPogrebStore(filenameUserBlockchain); err != nil { - Filters.LogError("initUserBlockchain", "error opening user blockchain: %s\n", err.Error()) - os.Exit(1) - } + UserBlockchain, err = blockchain.Init(peerPrivateKey, filenameUserBlockchain) - // verify header - var found bool - userBlockchainHeader.publicKey, userBlockchainHeader.height, userBlockchainHeader.version, found, err = blockchainHeaderRead(userBlockchainDB) if err != nil { - Filters.LogError("initUserBlockchain", "corrupt user blockchain database: %s\n", err.Error()) - os.Exit(1) - } else if !found { - // First run: create header signature! - userBlockchainHeader.height = 0 - userBlockchainHeader.version = 0 - userBlockchainHeader.publicKey = peerPublicKey - - if err := blockchainHeaderWrite(userBlockchainDB, peerPrivateKey, userBlockchainHeader.height, userBlockchainHeader.version); err != nil { - Filters.LogError("initUserBlockchain", "initializing user blockchain: %s", err.Error()) - os.Exit(1) - } - } else if !userBlockchainHeader.publicKey.IsEqual(peerPublicKey) { - Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Public key mismatch. Height is '%d', version '%d'. Public key expected '%s' vs provided '%s'\n", userBlockchainHeader.height, userBlockchainHeader.version, hex.EncodeToString(peerPublicKey.SerializeCompressed()), hex.EncodeToString(userBlockchainHeader.publicKey.SerializeCompressed())) + Filters.LogError("initUserBlockchain", "error: %s\n", err.Error()) os.Exit(1) } } - -// blockchainHeaderRead reads the header from the blockchain and decodes it. -func blockchainHeaderRead(db store.Store) (publicKey *btcec.PublicKey, height, version uint64, found bool, err error) { - buffer, found := db.Get([]byte(keyHeader)) - if !found { - return nil, 0, 0, false, nil - } - - if len(buffer) != 83 { - return nil, 0, 0, true, errors.New("blockchain header size mismatch") - } - - height = binary.LittleEndian.Uint64(buffer[0:8]) - version = binary.LittleEndian.Uint64(buffer[8:16]) - format := binary.LittleEndian.Uint16(buffer[16:18]) - signature := buffer[18 : 18+65] - - if format != 0 { - return nil, 0, 0, true, errors.New("future blockchain format not supported. You must go back to the future!") - } - - publicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, hashData(buffer[0:18])) - - return -} - -// blockchainHeaderWrite writes the header to the blockchain and signs it. -func blockchainHeaderWrite(db store.Store, privateKey *btcec.PrivateKey, height, version uint64) (err error) { - var buffer [83]byte - binary.LittleEndian.PutUint64(buffer[0:8], height) - binary.LittleEndian.PutUint64(buffer[8:16], version) - binary.LittleEndian.PutUint16(buffer[16:18], 0) // Current format is 0 - - signature, err := btcec.SignCompact(btcec.S256(), privateKey, hashData(buffer[0:18]), true) - - if err != nil { - return err - } else if len(signature) != 65 { - return errors.New("signature length invalid") - } - - copy(buffer[18:18+65], signature) - - err = db.Set([]byte(keyHeader), buffer[:]) - - return err -} - -// BlockchainStatusX provides information about the blockchain status. Some errors codes indicate a corruption. -const ( - BlockchainStatusOK = 0 // No problems in the blockchain detected. - BlockchainStatusBlockNotFound = 1 // Missing block in the blockchain. - BlockchainStatusCorruptBlock = 2 // Error block encoding - BlockchainStatusCorruptBlockRecord = 3 // Error block record encoding - BlockchainStatusDataNotFound = 4 // Requested data not available in the blockchain -) - -// blockNumberToKey returns the database key for the given block number -func blockNumberToKey(number uint64) (key []byte) { - var target [8]byte - binary.LittleEndian.PutUint64(target[:], number) - - return target[:] -} - -// blockchainIterate iterates over the blockchain. Status is BlockchainStatusX. -// If the callback returns non-zero, the function aborts and returns the inner status code. -func blockchainIterate(callback func(block *Block) int) (status int) { - // read all blocks until height is reached - height := userBlockchainHeader.height - - for blockN := uint64(0); blockN < height; blockN++ { - blockRaw, found := userBlockchainDB.Get(blockNumberToKey(blockN)) - if !found || len(blockRaw) == 0 { - return BlockchainStatusBlockNotFound - } - - block, err := decodeBlock(blockRaw) - if err != nil { - return BlockchainStatusCorruptBlock - } - - if statusI := callback(block); statusI != BlockchainStatusOK { - return statusI - } - } - - return BlockchainStatusOK -} - -// blockchainIterateDeleteRecord iterates over the blockchain to find records to delete. Status is BlockchainStatusX. -// deleteAction is 0 = no action on record, 1 = delete record, 2 = replace record, 3 = error blockchain corrupt -// If the callback returns true, the record will be deleted. The blockchain will be automatically refactored and height and version updated. -func blockchainIterateDeleteRecord(callback func(record *BlockRecordRaw) (deleteAction int)) (newHeight, newVersion uint64, status int) { - userBlockchainHeader.Lock() - defer userBlockchainHeader.Unlock() - - // New blockchain keeps track of the new blocks. If anything changes in the blockchain, it must be recalculated and the version number increased. - var blockchainNew []Block - refactorBlockchain := false - refactorVersion := userBlockchainHeader.version + 1 - - // Read all blocks until height is reached. At the end the height and version might be different if blocks are deleted. - height := userBlockchainHeader.height - - for blockN := uint64(0); blockN < height; blockN++ { - blockRaw, found := userBlockchainDB.Get(blockNumberToKey(blockN)) - if !found || len(blockRaw) == 0 { - return 0, 0, BlockchainStatusBlockNotFound - } - - block, err := decodeBlock(blockRaw) - if err != nil { - return 0, 0, BlockchainStatusCorruptBlock - } - - // loop through all records in this block - refactorBlock := false - var newRecordsRaw []BlockRecordRaw - - for n := range block.RecordsRaw { - switch callback(&block.RecordsRaw[n]) { - case 0: // no action on record - newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) - - case 1: // delete record - refactorBlock = true - refactorBlockchain = true - - case 2: // replace record - newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) - refactorBlock = true - refactorBlockchain = true - - case 3: // error blockchain corrupt - return 0, 0, BlockchainStatusCorruptBlockRecord - } - } - - // If refactor, re-calculate the block. All later blocks need to be re-encoded due to change of previous block hash. The version number needs to change. - if refactorBlock { - if len(newRecordsRaw) > 0 { - blockchainNew = append(blockchainNew, Block{OwnerPublicKey: peerPublicKey, RecordsRaw: newRecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))}) - } - } else { - blockchainNew = append(blockchainNew, Block{OwnerPublicKey: peerPublicKey, RecordsRaw: block.RecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))}) - } - } - - if refactorBlockchain { - var lastBlockHash []byte - - for _, block := range blockchainNew { - block.LastBlockHash = lastBlockHash - - raw, err := encodeBlock(&block, peerPrivateKey) - if err != nil { - return 0, 0, BlockchainStatusCorruptBlock - } - - // store the block - userBlockchainDB.Set(blockNumberToKey(block.Number), raw) - - lastBlockHash = hashData(raw) - } - - userBlockchainHeader.height = uint64(len(blockchainNew)) - userBlockchainHeader.version = refactorVersion - - // update the blockchain header in the database - blockchainHeaderWrite(userBlockchainDB, peerPrivateKey, userBlockchainHeader.height, userBlockchainHeader.version) - - // delete orphaned blocks - for n := userBlockchainHeader.height; n < height; n++ { - userBlockchainDB.Delete(blockNumberToKey(n)) - } - } - - return userBlockchainHeader.height, userBlockchainHeader.version, BlockchainStatusOK -} - -// ---- blockchain manipulation functions ---- - -// UserBlockchainHeader returns the users blockchain header which stores the height and version number. -func UserBlockchainHeader() (publicKey *btcec.PublicKey, height uint64, version uint64) { - return userBlockchainHeader.publicKey, userBlockchainHeader.height, userBlockchainHeader.version -} - -// UserBlockchainAppend appends a new block to the blockchain based on the provided raw records. -// Status: BlockchainStatusX (0-2): 0 = Success, 1 = Error block not found, 2 = Error block encoding -func UserBlockchainAppend(RecordsRaw []BlockRecordRaw) (newHeight, newVersion uint64, status int) { - userBlockchainHeader.Lock() - defer userBlockchainHeader.Unlock() - - block := &Block{OwnerPublicKey: peerPublicKey, RecordsRaw: RecordsRaw} - - // set the last block hash first - if userBlockchainHeader.height > 0 { - previousBlockRaw, found := userBlockchainDB.Get(blockNumberToKey(userBlockchainHeader.height - 1)) - if !found || len(previousBlockRaw) == 0 { - return 0, 0, BlockchainStatusBlockNotFound - } - - block.LastBlockHash = hashData(previousBlockRaw) - } - - block.Number = userBlockchainHeader.height - block.BlockchainVersion = userBlockchainHeader.version - - raw, err := encodeBlock(block, peerPrivateKey) - if err != nil { - return 0, 0, BlockchainStatusCorruptBlock - } - - // increase blockchain height - userBlockchainHeader.height++ - - // store the block - userBlockchainDB.Set(blockNumberToKey(block.Number), raw) - - // update the blockchain header in the database - blockchainHeaderWrite(userBlockchainDB, peerPrivateKey, userBlockchainHeader.height, userBlockchainHeader.version) - - return userBlockchainHeader.height, userBlockchainHeader.version, BlockchainStatusOK -} - -// UserBlockchainRead reads the block number from the blockchain. Status is BlockchainStatusX. -func UserBlockchainRead(number uint64) (decoded *BlockDecoded, status int, err error) { - if number >= userBlockchainHeader.height { - return nil, BlockchainStatusBlockNotFound, errors.New("block number exceeds blockchain height") - } - - blockRaw, found := userBlockchainDB.Get(blockNumberToKey(number)) - if !found || len(blockRaw) == 0 { - return nil, BlockchainStatusBlockNotFound, errors.New("block not found") - } - - block, err := decodeBlock(blockRaw) - if err != nil { - return nil, BlockchainStatusCorruptBlock, err - } - - decoded, err = decodeBlockRecords(block) - if err != nil { - return nil, BlockchainStatusCorruptBlock, err - } - - return decoded, BlockchainStatusOK, nil -} - -// UserBlockchainAddFiles adds files to the blockchain. Status is BlockchainStatusX. -// It makes sense to group all files in the same directory into one call, since only one directory record will be created per unique directory per block. -func UserBlockchainAddFiles(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { - encoded, err := encodeBlockRecordFiles(files) - if err != nil { - return 0, 0, BlockchainStatusCorruptBlockRecord - } - - return UserBlockchainAppend(encoded) -} - -// UserBlockchainListFiles returns a list of all files. Status is BlockchainStatusX. -// If there is a corruption in the blockchain it will stop reading but return the files parsed so far. -func UserBlockchainListFiles() (files []BlockRecordFile, status int) { - status = blockchainIterate(func(block *Block) (statusI int) { - filesMore, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) - if err != nil { - return BlockchainStatusCorruptBlockRecord - } - files = append(files, filesMore...) - - return BlockchainStatusOK - }) - - return files, status -} - -// UserProfileReadField reads the specified profile field. See ProfileX for the list of recognized fields. The encoding depends on the field type. Status is BlockchainStatusX. -func UserProfileReadField(index uint16) (data []byte, status int) { - found := false - - status = blockchainIterate(func(block *Block) (statusI int) { - fields, err := decodeBlockRecordProfile(block.RecordsRaw) - if err != nil { - return BlockchainStatusCorruptBlockRecord - } else if len(fields) == 0 { - return BlockchainStatusOK - } - - // Check if the field is available in the profile record. If there are multiple records, only return the latest one. - for n := range fields { - if fields[n].Type == index { - data = fields[n].Data - found = true - } - } - - return BlockchainStatusOK - }) - - if status != BlockchainStatusOK { - return nil, status - } else if !found { - return nil, BlockchainStatusDataNotFound - } - - return data, BlockchainStatusOK -} - -// UserProfileList lists all profile fields. Status is BlockchainStatusX. -func UserProfileList() (fields []BlockRecordProfile, status int) { - uniqueFields := make(map[uint16][]byte) - - status = blockchainIterate(func(block *Block) (statusI int) { - fields, err := decodeBlockRecordProfile(block.RecordsRaw) - if err != nil { - return BlockchainStatusCorruptBlockRecord - } - - for n := range fields { - uniqueFields[fields[n].Type] = fields[n].Data - } - - return BlockchainStatusOK - }) - - for key, value := range uniqueFields { - fields = append(fields, BlockRecordProfile{Type: key, Data: value}) - } - - return fields, status -} - -// UserProfileWrite writes profile fields and blobs to the blockchain. Status is BlockchainStatusX. -func UserProfileWrite(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) { - encoded, err := encodeBlockRecordProfile(fields) - if err != nil { - return 0, 0, BlockchainStatusCorruptBlockRecord - } - - return UserBlockchainAppend(encoded) -} - -// UserProfileDelete deletes fields and blobs from the blockchain. Status is BlockchainStatusX. -func UserProfileDelete(fields []uint16) (newHeight, newVersion uint64, status int) { - return blockchainIterateDeleteRecord(func(record *BlockRecordRaw) (deleteAction int) { - if record.Type != RecordTypeProfile { - return 0 // no action - } - - existingFields, err := decodeBlockRecordProfile([]BlockRecordRaw{*record}) - if err != nil || len(existingFields) != 1 { - return 3 // error blockchain corrupt - } - - for _, i := range fields { - if i == existingFields[0].Type { // found a file ID to delete? - return 1 // delete record - } - } - - return 0 // no action on record - }) -} - -// UserBlockchainDeleteFiles deletes files from the blockchain. Status is BlockchainStatusX. -func UserBlockchainDeleteFiles(IDs []uuid.UUID) (newHeight, newVersion uint64, status int) { - return blockchainIterateDeleteRecord(func(record *BlockRecordRaw) (deleteAction int) { - if record.Type != RecordTypeFile { - return 0 // no action on record - } - - filesDecoded, err := decodeBlockRecordFiles([]BlockRecordRaw{*record}, nil) - if err != nil || len(filesDecoded) != 1 { - return 3 // error blockchain corrupt - } - - for _, id := range IDs { - if id == filesDecoded[0].ID { // found a file ID to delete? - return 1 // delete record - } - } - - return 0 // no action on record - }) -} diff --git a/Config.go b/Config.go index 2ebc425..bc756c2 100644 --- a/Config.go +++ b/Config.go @@ -16,7 +16,7 @@ import ( ) // Version is the current core library version -const Version = "Alpha 3/31.07.2021" +const Version = "Alpha 4/11.10.2021" var config struct { LogFile string `yaml:"LogFile"` // Log file diff --git a/Message Encoding.go b/Message Encoding.go index 2799bab..b6aa6da 100644 --- a/Message Encoding.go +++ b/Message Encoding.go @@ -531,8 +531,10 @@ createPacketLoop: raw[0] = byte(ProtocolVersion) // Protocol raw[1] = FeatureSupport() // Feature support //raw[2] = Actions // Action bit array - binary.LittleEndian.PutUint32(raw[3:7], BlockchainHeight) - binary.LittleEndian.PutUint64(raw[7:15], BlockchainVersion) + + _, blockchainHeight, blockchainVersion := UserBlockchain.Header() + binary.LittleEndian.PutUint32(raw[3:7], uint32(blockchainHeight)) + binary.LittleEndian.PutUint64(raw[7:15], blockchainVersion) // only on initial announcement the User Agent must be provided according to the protocol spec if sendUA { @@ -673,8 +675,10 @@ createPacketLoop: raw[0] = byte(ProtocolVersion) // Protocol raw[1] = FeatureSupport() // Feature support //raw[2] = Actions // Action bit array - binary.LittleEndian.PutUint32(raw[3:7], BlockchainHeight) - binary.LittleEndian.PutUint64(raw[7:15], BlockchainVersion) + + _, blockchainHeight, blockchainVersion := UserBlockchain.Header() + binary.LittleEndian.PutUint32(raw[3:7], uint32(blockchainHeight)) + binary.LittleEndian.PutUint64(raw[7:15], blockchainVersion) // only on initial response the User Agent must be provided according to the protocol spec if sendUA { diff --git a/Test_test.go b/Test_test.go index 2e166a1..37730ef 100644 --- a/Test_test.go +++ b/Test_test.go @@ -2,12 +2,8 @@ package core import ( - "encoding/hex" "fmt" "testing" - - "github.com/btcsuite/btcd/btcec" - "github.com/google/uuid" ) func TestMessageEncodingAnnouncement(t *testing.T) { @@ -83,233 +79,3 @@ func TestMessageEncodingResponse(t *testing.T) { fmt.Printf("Decode:\nUser Agent: %s\nHash2Peers: %v\nHashesNotFound: %v\nFiles embedded: %v\n", result.UserAgent, result.Hash2Peers, result.HashesNotFound, result.FilesEmbed) } - -func TestBlockEncoding(t *testing.T) { - privateKey, _, err := Secp256k1NewPrivateKey() - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - - encoded1, _ := encodeBlockRecordProfile([]BlockRecordProfile{ProfileFieldFromText(ProfileName, "Test User 1")}) - - file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, ID: uuid.New()} - file1.Tags = append(file1.Tags, TagFromText(TagName, "Filename 1.txt")) - file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder")) - - file2 := BlockRecordFile{Hash: hashData([]byte("Test data 2")), Type: TypeText, Format: FormatText, Size: 9, ID: uuid.New()} - file2.Tags = append(file2.Tags, TagFromText(TagName, "Filename 2.txt")) - file2.Tags = append(file2.Tags, TagFromText(TagFolder, "documents\\sub folder")) - - encodedFiles, _ := encodeBlockRecordFiles([]BlockRecordFile{file1, file2}) - - blockE := &Block{BlockchainVersion: 42, Number: 0} - blockE.RecordsRaw = append(blockE.RecordsRaw, encoded1...) - blockE.RecordsRaw = append(blockE.RecordsRaw, encodedFiles...) - - raw, err := encodeBlock(blockE, privateKey) - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - - block, err := decodeBlock(raw) - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - - decoded, err := decodeBlockRecords(block) - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - - // output the block details - fmt.Printf("Block details:\n----------------\nNumber: %d\nVersion: %d\nLast Hash: %s\nPublic Key: %s\n", block.Number, block.BlockchainVersion, hex.EncodeToString(block.LastBlockHash), hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())) - - for _, decodedR := range decoded.RecordsDecoded { - switch record := decodedR.(type) { - case BlockRecordFile: - printFile(record) - - case BlockRecordProfile: - printProfileField(record) - - } - - } -} - -func initTestPrivateKey() { - // use static test key, otherwise tests will be inconsistent (would otherwise fail to open blockchain database) - privateKeyTestA := "d65da474861d826edd29c1307f1250d79e9dbf84e3a2449022658445c8d8ed63" - privateKeyB, _ := hex.DecodeString(privateKeyTestA) - peerPrivateKey, peerPublicKey = btcec.PrivKeyFromBytes(btcec.S256(), privateKeyB) - nodeID = PublicKey2NodeID(peerPublicKey) - - fmt.Printf("Loaded public key: %s\n", hex.EncodeToString(peerPublicKey.SerializeCompressed())) -} - -func TestBlockchainAdd(t *testing.T) { - initTestPrivateKey() - initUserBlockchain() - - file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, ID: uuid.New()} - file1.Tags = append(file1.Tags, TagFromText(TagName, "Filename 1.txt")) - file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder")) - - newHeight, newVersion, status := UserBlockchainAddFiles([]BlockRecordFile{file1}) - - switch status { - case 0: - case 1: // Error previous block not found - fmt.Printf("Error adding file to blockchain: Previous block not found.\n") - case 2: // Error block encoding - fmt.Printf("Error adding file to blockchain: Error block encoding.\n") - case 3: // Error block record encoding - fmt.Printf("Error adding file to blockchain: Error block record encoding.\n") - default: - fmt.Printf("Error adding file to blockchain: Unknown status %d\n", status) - } - - if status != 0 { - return - } - - fmt.Printf("Success adding files to blockchain. New blockchain height %d version %d\n", newHeight, newVersion) -} - -func TestBlockchainRead(t *testing.T) { - initTestPrivateKey() - initFilters() - initUserBlockchain() - - blockNumber := uint64(0) - - decoded, status, err := UserBlockchainRead(blockNumber) - switch status { - case 0: - case 1: // Error block not found - fmt.Printf("Error reading block %d: Block not found.\n", blockNumber) - case 2: // Error block encoding - fmt.Printf("Error reading block %d: Block encoding corrupt: %s\n", blockNumber, err.Error()) - case 3: // Error block record encoding - fmt.Printf("Error reading block %d: Block record encoding corrupt.\n", blockNumber) - default: - fmt.Printf("Error reading block %d: Unknown status %d\n", blockNumber, status) - } - - if status != 0 { - return - } - - for _, decodedR := range decoded.RecordsDecoded { - if file, ok := decodedR.(BlockRecordFile); ok { - printFile(file) - } - } -} - -func printFile(file BlockRecordFile) { - fmt.Printf("* File %s\n", file.ID.String()) - fmt.Printf(" Size %d\n", file.Size) - fmt.Printf(" Type %d\n", file.Type) - fmt.Printf(" Format %d\n", file.Format) - fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) - - for _, tag := range file.Tags { - switch tag.Type { - case TagName: - fmt.Printf(" Name %s\n", tag.Text()) - case TagFolder: - fmt.Printf(" Folder %s\n", tag.Text()) - case TagDescription: - fmt.Printf(" Description %s\n", tag.Text()) - } - } -} - -func TestBlockchainDelete(t *testing.T) { - initTestPrivateKey() - initFilters() - initUserBlockchain() - - // test add file - file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, ID: uuid.New()} - file1.Tags = append(file1.Tags, TagFromText(TagName, "Test file to be deleted.txt")) - file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder")) - - newHeight, newVersion, status := UserBlockchainAddFiles([]BlockRecordFile{file1}) - fmt.Printf("Added file: Status %d height %d version %d\n", status, newHeight, newVersion) - - // list files - files, _ := UserBlockchainListFiles() - for _, file := range files { - printFile(file) - } - - fmt.Printf("----------------\n") - - // delete the file - newHeight, newVersion, status = UserBlockchainDeleteFiles([]uuid.UUID{file1.ID}) - fmt.Printf("Deleted file: Status %d height %d version %d\n", status, newHeight, newVersion) - - // list all files - files, _ = UserBlockchainListFiles() - for _, file := range files { - printFile(file) - } -} - -func TestBlockchainProfile(t *testing.T) { - initTestPrivateKey() - initFilters() - initUserBlockchain() - - // write some test profile data - newHeight, newVersion, status := UserProfileWrite([]BlockRecordProfile{ - ProfileFieldFromText(ProfileName, "Test User 1"), - ProfileFieldFromText(ProfileEmail, "test@test.com"), - {Type: 100, Data: []byte{0, 1, 2, 3}}}) - - fmt.Printf("Write profile data: Status %d height %d version %d\n", status, newHeight, newVersion) - - // list all profile info - printProfileData() - - fmt.Printf("----------------\n") - - // delete profile info - newHeight, newVersion, status = UserProfileDelete([]uint16{ProfileEmail}) - fmt.Printf("Deleted profile email: Status %d height %d version %d\n", status, newHeight, newVersion) - - printProfileData() -} - -func printProfileData() { - fields, status := UserProfileList() - if status != BlockchainStatusOK { - fmt.Printf("Reading profile data error status: %d\n", status) - return - } - - if len(fields) == 0 { - fmt.Printf("No profile data to visualize.\n") - return - } - - for _, field := range fields { - printProfileField(field) - } -} - -func printProfileField(field BlockRecordProfile) { - switch field.Type { - case ProfileName, ProfileEmail, ProfileWebsite, ProfileTwitter, ProfileYouTube, ProfileAddress: - fmt.Printf("* Field %d = %s\n", field.Type, string(field.Data)) - - default: - fmt.Printf("* Field %d = %s\n", field.Type, hex.EncodeToString(field.Data)) - } -} diff --git a/Block Encoding.go b/blockchain/Block Encoding.go similarity index 95% rename from Block Encoding.go rename to blockchain/Block Encoding.go index d1b1d82..162beb7 100644 --- a/Block Encoding.go +++ b/blockchain/Block Encoding.go @@ -21,7 +21,7 @@ Offset Size Info */ -package core +package blockchain import ( "bytes" @@ -63,7 +63,7 @@ func decodeBlock(raw []byte) (block *Block, err error) { signature := raw[0 : 0+65] - block.OwnerPublicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, hashData(raw[65:])) + block.OwnerPublicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, HashFunction(raw[65:])) if err != nil { return nil, err } @@ -158,7 +158,7 @@ func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, e binary.LittleEndian.PutUint16(raw[113:113+2], countRecords) // Count of records // signature is last - signature, err := btcec.SignCompact(btcec.S256(), ownerPrivateKey, hashData(raw[65:]), true) + signature, err := btcec.SignCompact(btcec.S256(), ownerPrivateKey, HashFunction(raw[65:]), true) if err != nil { return nil, err } diff --git a/Block Records.go b/blockchain/Block Records.go similarity index 96% rename from Block Records.go rename to blockchain/Block Records.go index a7a4edc..79f4c9d 100644 --- a/Block Records.go +++ b/blockchain/Block Records.go @@ -31,7 +31,7 @@ Offset Size Info */ -package core +package blockchain import ( "encoding/binary" diff --git a/blockchain/Blockchain.go b/blockchain/Blockchain.go new file mode 100644 index 0000000..bb696c5 --- /dev/null +++ b/blockchain/Blockchain.go @@ -0,0 +1,471 @@ +/* +File Name: Blockchain.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +All blocks and the blockchain header are stored in a key/value database. +The key for the blockchain header is keyHeader and for each block is the block number as 64-bit unsigned integer little endian. + +Encoding of the blockchain header: +Offset Size Info +0 8 Height of the blockchain +8 8 Version of the blockchain +16 2 Format of the blockchain. This provides backward compatibility. +18 65 Signature + +*/ + +package blockchain + +import ( + "encoding/binary" + "errors" + "sync" + + "github.com/PeernetOfficial/core/store" + "github.com/btcsuite/btcd/btcec" + "github.com/google/uuid" +) + +// Blockchain stores the blockchain's header in memory. Any changes must be synced to disk! +type Blockchain struct { + // header + height uint64 // Height is exchanged as uint32 in the protocol, but stored as uint64. + version uint64 // Version is always uint64. + format uint16 // Format is only locally used. + + // internals + publicKey *btcec.PublicKey // Public Key of the owner. This must match the ones used on disk. + privateKey *btcec.PrivateKey // Private Key of the owner. This must match the ones used on disk. + path string // Path of the blockchain on disk. Depends on key-value store whether a filename or folder. + database store.Store // The database storing the blockchain. + sync.Mutex // synchronized access to the header +} + +// HashFunction must be set by the caller to the hash function (blake3) that shall be used. +var HashFunction func(data []byte) (hash []byte) + +// hashSize is the blake3 digest size in bytes +const hashSize = 32 + +// PublicKey2NodeID must be set by the caller to the function generating the node ID from the public key. Typically it would use the HashFunction. +var PublicKey2NodeID func(publicKey *btcec.PublicKey) (nodeID []byte) + +// Init initializes the given blockchain. It creates the blockchain file if it does not exist already. +func Init(privateKey *btcec.PrivateKey, path string) (blockchain *Blockchain, err error) { + blockchain = &Blockchain{privateKey: privateKey, path: path} + publicKey := privateKey.PubKey() + + // open existing blockchain file or create new one + if blockchain.database, err = store.NewPogrebStore(path); err != nil { + return nil, err + } + + // verify header + var found bool + + found, err = blockchain.headerRead() + if err != nil { + return blockchain, err // likely corrupt blockchain database + } else if !found { + // First run: create header signature! + blockchain.publicKey = publicKey + + if err := blockchain.headerWrite(0, 0); err != nil { + return blockchain, err + } + } else if !blockchain.publicKey.IsEqual(publicKey) { + return blockchain, errors.New("corrupt user blockchain database. Public key mismatch") + } + + return blockchain, nil +} + +// the key names in the key/value database are constant and must not collide with block numbers (i.e. they must be >64 bit) +const keyHeader = "header blockchain" + +// headerRead reads the header from the blockchain and decodes it. +func (blockchain *Blockchain) headerRead() (found bool, err error) { + buffer, found := blockchain.database.Get([]byte(keyHeader)) + if !found { + return false, nil + } + + if len(buffer) != 83 { + return true, errors.New("blockchain header size mismatch") + } + + blockchain.height = binary.LittleEndian.Uint64(buffer[0:8]) + blockchain.version = binary.LittleEndian.Uint64(buffer[8:16]) + blockchain.format = binary.LittleEndian.Uint16(buffer[16:18]) + signature := buffer[18 : 18+65] + + if blockchain.format != 0 { + return true, errors.New("future blockchain format not supported. You must go back to the future!") + } + + blockchain.publicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, HashFunction(buffer[0:18])) + + return +} + +// headerWrite writes the header to the blockchain and signs it. +func (blockchain *Blockchain) headerWrite(height, version uint64) (err error) { + blockchain.height = height + blockchain.version = version + + var buffer [83]byte + binary.LittleEndian.PutUint64(buffer[0:8], height) + binary.LittleEndian.PutUint64(buffer[8:16], version) + binary.LittleEndian.PutUint16(buffer[16:18], 0) // Current format is 0 + + signature, err := btcec.SignCompact(btcec.S256(), blockchain.privateKey, HashFunction(buffer[0:18]), true) + + if err != nil { + return err + } else if len(signature) != 65 { + return errors.New("signature length invalid") + } + + copy(buffer[18:18+65], signature) + + err = blockchain.database.Set([]byte(keyHeader), buffer[:]) + + return err +} + +// BlockchainStatusX provides information about the blockchain status. Some errors codes indicate a corruption. +const ( + BlockchainStatusOK = 0 // No problems in the blockchain detected. + BlockchainStatusBlockNotFound = 1 // Missing block in the blockchain. + BlockchainStatusCorruptBlock = 2 // Error block encoding + BlockchainStatusCorruptBlockRecord = 3 // Error block record encoding + BlockchainStatusDataNotFound = 4 // Requested data not available in the blockchain +) + +// blockNumberToKey returns the database key for the given block number +func blockNumberToKey(number uint64) (key []byte) { + var target [8]byte + binary.LittleEndian.PutUint64(target[:], number) + + return target[:] +} + +// Iterate iterates over the blockchain. Status is BlockchainStatusX. +// If the callback returns non-zero, the function aborts and returns the inner status code. +func (blockchain *Blockchain) Iterate(callback func(block *Block) int) (status int) { + // read all blocks until height is reached + height := blockchain.height + + for blockN := uint64(0); blockN < height; blockN++ { + blockRaw, found := blockchain.database.Get(blockNumberToKey(blockN)) + if !found || len(blockRaw) == 0 { + return BlockchainStatusBlockNotFound + } + + block, err := decodeBlock(blockRaw) + if err != nil { + return BlockchainStatusCorruptBlock + } + + if statusI := callback(block); statusI != BlockchainStatusOK { + return statusI + } + } + + return BlockchainStatusOK +} + +// IterateDeleteRecord iterates over the blockchain to find records to delete. Status is BlockchainStatusX. +// deleteAction is 0 = no action on record, 1 = delete record, 2 = replace record, 3 = error blockchain corrupt +// If the callback returns true, the record will be deleted. The blockchain will be automatically refactored and height and version updated. +func (blockchain *Blockchain) IterateDeleteRecord(callback func(record *BlockRecordRaw) (deleteAction int)) (newHeight, newVersion uint64, status int) { + blockchain.Lock() + defer blockchain.Unlock() + + // New blockchain keeps track of the new blocks. If anything changes in the blockchain, it must be recalculated and the version number increased. + var blockchainNew []Block + refactorBlockchain := false + refactorVersion := blockchain.version + 1 + + // Read all blocks until height is reached. At the end the height and version might be different if blocks are deleted. + height := blockchain.height + + for blockN := uint64(0); blockN < height; blockN++ { + blockRaw, found := blockchain.database.Get(blockNumberToKey(blockN)) + if !found || len(blockRaw) == 0 { + return 0, 0, BlockchainStatusBlockNotFound + } + + block, err := decodeBlock(blockRaw) + if err != nil { + return 0, 0, BlockchainStatusCorruptBlock + } + + // loop through all records in this block + refactorBlock := false + var newRecordsRaw []BlockRecordRaw + + for n := range block.RecordsRaw { + switch callback(&block.RecordsRaw[n]) { + case 0: // no action on record + newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) + + case 1: // delete record + refactorBlock = true + refactorBlockchain = true + + case 2: // replace record + newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) + refactorBlock = true + refactorBlockchain = true + + case 3: // error blockchain corrupt + return 0, 0, BlockchainStatusCorruptBlockRecord + } + } + + // If refactor, re-calculate the block. All later blocks need to be re-encoded due to change of previous block hash. The version number needs to change. + if refactorBlock { + if len(newRecordsRaw) > 0 { + blockchainNew = append(blockchainNew, Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: newRecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))}) + } + } else { + blockchainNew = append(blockchainNew, Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: block.RecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))}) + } + } + + if refactorBlockchain { + var lastBlockHash []byte + + for _, block := range blockchainNew { + block.LastBlockHash = lastBlockHash + + raw, err := encodeBlock(&block, blockchain.privateKey) + if err != nil { + return 0, 0, BlockchainStatusCorruptBlock + } + + // store the block + blockchain.database.Set(blockNumberToKey(block.Number), raw) + + lastBlockHash = HashFunction(raw) + } + + // update the blockchain header in the database + blockchain.headerWrite(uint64(len(blockchainNew)), refactorVersion) + + // delete orphaned blocks + for n := blockchain.height; n < height; n++ { + blockchain.database.Delete(blockNumberToKey(n)) + } + } + + return blockchain.height, blockchain.version, BlockchainStatusOK +} + +// ---- blockchain manipulation functions ---- + +// Header returns the users blockchain header which stores the height and version number. +func (blockchain *Blockchain) Header() (publicKey *btcec.PublicKey, height uint64, version uint64) { + blockchain.Lock() + defer blockchain.Unlock() + + return blockchain.publicKey, blockchain.height, blockchain.version +} + +// Append appends a new block to the blockchain based on the provided raw records. +// Status: BlockchainStatusX (0-2): 0 = Success, 1 = Error block not found, 2 = Error block encoding +func (blockchain *Blockchain) Append(RecordsRaw []BlockRecordRaw) (newHeight, newVersion uint64, status int) { + blockchain.Lock() + defer blockchain.Unlock() + + block := &Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: RecordsRaw} + + // set the last block hash first + if blockchain.height > 0 { + previousBlockRaw, found := blockchain.database.Get(blockNumberToKey(blockchain.height - 1)) + if !found || len(previousBlockRaw) == 0 { + return 0, 0, BlockchainStatusBlockNotFound + } + + block.LastBlockHash = HashFunction(previousBlockRaw) + } + + block.Number = blockchain.height + block.BlockchainVersion = blockchain.version + + raw, err := encodeBlock(block, blockchain.privateKey) + if err != nil { + return 0, 0, BlockchainStatusCorruptBlock + } + + // store the block + blockchain.database.Set(blockNumberToKey(block.Number), raw) + + // update the blockchain header in the database, increase blockchain height + blockchain.headerWrite(blockchain.height+1, blockchain.version) + + return blockchain.height, blockchain.version, BlockchainStatusOK +} + +// Read reads the block number from the blockchain. Status is BlockchainStatusX. +func (blockchain *Blockchain) Read(number uint64) (decoded *BlockDecoded, status int, err error) { + if number >= blockchain.height { + return nil, BlockchainStatusBlockNotFound, errors.New("block number exceeds blockchain height") + } + + blockRaw, found := blockchain.database.Get(blockNumberToKey(number)) + if !found || len(blockRaw) == 0 { + return nil, BlockchainStatusBlockNotFound, errors.New("block not found") + } + + block, err := decodeBlock(blockRaw) + if err != nil { + return nil, BlockchainStatusCorruptBlock, err + } + + decoded, err = decodeBlockRecords(block) + if err != nil { + return nil, BlockchainStatusCorruptBlock, err + } + + return decoded, BlockchainStatusOK, nil +} + +// AddFiles adds files to the blockchain. Status is BlockchainStatusX. +// It makes sense to group all files in the same directory into one call, since only one directory record will be created per unique directory per block. +func (blockchain *Blockchain) AddFiles(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { + encoded, err := encodeBlockRecordFiles(files) + if err != nil { + return 0, 0, BlockchainStatusCorruptBlockRecord + } + + return blockchain.Append(encoded) +} + +// ListFiles returns a list of all files. Status is BlockchainStatusX. +// If there is a corruption in the blockchain it will stop reading but return the files parsed so far. +func (blockchain *Blockchain) ListFiles() (files []BlockRecordFile, status int) { + status = blockchain.Iterate(func(block *Block) (statusI int) { + filesMore, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) + if err != nil { + return BlockchainStatusCorruptBlockRecord + } + files = append(files, filesMore...) + + return BlockchainStatusOK + }) + + return files, status +} + +// ProfileReadField reads the specified profile field. See ProfileX for the list of recognized fields. The encoding depends on the field type. Status is BlockchainStatusX. +func (blockchain *Blockchain) ProfileReadField(index uint16) (data []byte, status int) { + found := false + + status = blockchain.Iterate(func(block *Block) (statusI int) { + fields, err := decodeBlockRecordProfile(block.RecordsRaw) + if err != nil { + return BlockchainStatusCorruptBlockRecord + } else if len(fields) == 0 { + return BlockchainStatusOK + } + + // Check if the field is available in the profile record. If there are multiple records, only return the latest one. + for n := range fields { + if fields[n].Type == index { + data = fields[n].Data + found = true + } + } + + return BlockchainStatusOK + }) + + if status != BlockchainStatusOK { + return nil, status + } else if !found { + return nil, BlockchainStatusDataNotFound + } + + return data, BlockchainStatusOK +} + +// ProfileList lists all profile fields. Status is BlockchainStatusX. +func (blockchain *Blockchain) ProfileList() (fields []BlockRecordProfile, status int) { + uniqueFields := make(map[uint16][]byte) + + status = blockchain.Iterate(func(block *Block) (statusI int) { + fields, err := decodeBlockRecordProfile(block.RecordsRaw) + if err != nil { + return BlockchainStatusCorruptBlockRecord + } + + for n := range fields { + uniqueFields[fields[n].Type] = fields[n].Data + } + + return BlockchainStatusOK + }) + + for key, value := range uniqueFields { + fields = append(fields, BlockRecordProfile{Type: key, Data: value}) + } + + return fields, status +} + +// ProfileWrite writes profile fields and blobs to the blockchain. Status is BlockchainStatusX. +func (blockchain *Blockchain) ProfileWrite(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) { + encoded, err := encodeBlockRecordProfile(fields) + if err != nil { + return 0, 0, BlockchainStatusCorruptBlockRecord + } + + return blockchain.Append(encoded) +} + +// ProfileDelete deletes fields and blobs from the blockchain. Status is BlockchainStatusX. +func (blockchain *Blockchain) ProfileDelete(fields []uint16) (newHeight, newVersion uint64, status int) { + return blockchain.IterateDeleteRecord(func(record *BlockRecordRaw) (deleteAction int) { + if record.Type != RecordTypeProfile { + return 0 // no action + } + + existingFields, err := decodeBlockRecordProfile([]BlockRecordRaw{*record}) + if err != nil || len(existingFields) != 1 { + return 3 // error blockchain corrupt + } + + for _, i := range fields { + if i == existingFields[0].Type { // found a file ID to delete? + return 1 // delete record + } + } + + return 0 // no action on record + }) +} + +// DeleteFiles deletes files from the blockchain. Status is BlockchainStatusX. +func (blockchain *Blockchain) DeleteFiles(IDs []uuid.UUID) (newHeight, newVersion uint64, status int) { + return blockchain.IterateDeleteRecord(func(record *BlockRecordRaw) (deleteAction int) { + if record.Type != RecordTypeFile { + return 0 // no action on record + } + + filesDecoded, err := decodeBlockRecordFiles([]BlockRecordRaw{*record}, nil) + if err != nil || len(filesDecoded) != 1 { + return 3 // error blockchain corrupt + } + + for _, id := range IDs { + if id == filesDecoded[0].ID { // found a file ID to delete? + return 1 // delete record + } + } + + return 0 // no action on record + }) +} diff --git a/File Metadata.go b/blockchain/File Tag.go similarity index 95% rename from File Metadata.go rename to blockchain/File Tag.go index 919d38e..61493c5 100644 --- a/File Metadata.go +++ b/blockchain/File Tag.go @@ -1,12 +1,12 @@ /* -File Name: File Metadata.go +File Name: File Tag.go Copyright: 2021 Peernet s.r.o. Author: Peter Kleissner Metadata tags provide meta information about files. */ -package core +package blockchain import ( "encoding/binary" diff --git a/Profile Data.go b/blockchain/Profile Data.go similarity index 95% rename from Profile Data.go rename to blockchain/Profile Data.go index 25bf199..d4d85ba 100644 --- a/Profile Data.go +++ b/blockchain/Profile Data.go @@ -4,7 +4,7 @@ Copyright: 2021 Peernet s.r.o. Author: Peter Kleissner */ -package core +package blockchain // List of recognized profile fields. const ( diff --git a/blockchain/Test_test.go b/blockchain/Test_test.go new file mode 100644 index 0000000..f23c0b3 --- /dev/null +++ b/blockchain/Test_test.go @@ -0,0 +1,265 @@ +package blockchain + +import ( + "encoding/hex" + "fmt" + "testing" + + "github.com/btcsuite/btcd/btcec" + "github.com/google/uuid" + "lukechampine.com/blake3" +) + +func TestBlockEncoding(t *testing.T) { + initRequiredFunctions() + privateKey, err := btcec.NewPrivateKey(btcec.S256()) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + encoded1, _ := encodeBlockRecordProfile([]BlockRecordProfile{ProfileFieldFromText(ProfileName, "Test User 1")}) + + file1 := BlockRecordFile{Hash: testHashData([]byte("Test data")), Type: testTypeText, Format: testFormatText, Size: 9, ID: uuid.New()} + file1.Tags = append(file1.Tags, TagFromText(TagName, "Filename 1.txt")) + file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder")) + + file2 := BlockRecordFile{Hash: testHashData([]byte("Test data 2!")), Type: testTypeText, Format: testFormatText, Size: 10, ID: uuid.New()} + file2.Tags = append(file2.Tags, TagFromText(TagName, "Filename 2.txt")) + file2.Tags = append(file2.Tags, TagFromText(TagFolder, "documents\\sub folder")) + + encodedFiles, _ := encodeBlockRecordFiles([]BlockRecordFile{file1, file2}) + + blockE := &Block{BlockchainVersion: 42, Number: 0} + blockE.RecordsRaw = append(blockE.RecordsRaw, encoded1...) + blockE.RecordsRaw = append(blockE.RecordsRaw, encodedFiles...) + + raw, err := encodeBlock(blockE, privateKey) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + block, err := decodeBlock(raw) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + decoded, err := decodeBlockRecords(block) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + // output the block details + fmt.Printf("Block details:\n----------------\nNumber: %d\nVersion: %d\nLast Hash: %s\nPublic Key: %s\n", block.Number, block.BlockchainVersion, hex.EncodeToString(block.LastBlockHash), hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())) + + for _, decodedR := range decoded.RecordsDecoded { + switch record := decodedR.(type) { + case BlockRecordFile: + printFile(record) + + case BlockRecordProfile: + printProfileField(record) + + } + + } +} + +func initRequiredFunctions() { + HashFunction = testHashData + PublicKey2NodeID = func(publicKey *btcec.PublicKey) (nodeID []byte) { + return testHashData(publicKey.SerializeCompressed()) + } +} + +func initTestPrivateKey() (blockchain *Blockchain, err error) { + // use static test key, otherwise tests will be inconsistent (would otherwise fail to open blockchain database) + privateKeyTestA := "d65da474861d826edd29c1307f1250d79e9dbf84e3a2449022658445c8d8ed63" + privateKeyB, _ := hex.DecodeString(privateKeyTestA) + peerPrivateKey, peerPublicKey := btcec.PrivKeyFromBytes(btcec.S256(), privateKeyB) + + fmt.Printf("Loaded public key: %s\n", hex.EncodeToString(peerPublicKey.SerializeCompressed())) + + initRequiredFunctions() + + return Init(peerPrivateKey, "test.blockchain") +} + +func TestBlockchainAdd(t *testing.T) { + blockchain, err := initTestPrivateKey() + if err != nil { + return + } + + file1 := BlockRecordFile{Hash: testHashData([]byte("Test data")), Type: testTypeText, Format: testFormatText, Size: 9, ID: uuid.New()} + file1.Tags = append(file1.Tags, TagFromText(TagName, "Filename 1.txt")) + file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder")) + + newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1}) + + switch status { + case 0: + case 1: // Error previous block not found + fmt.Printf("Error adding file to blockchain: Previous block not found.\n") + case 2: // Error block encoding + fmt.Printf("Error adding file to blockchain: Error block encoding.\n") + case 3: // Error block record encoding + fmt.Printf("Error adding file to blockchain: Error block record encoding.\n") + default: + fmt.Printf("Error adding file to blockchain: Unknown status %d\n", status) + } + + if status != 0 { + return + } + + fmt.Printf("Success adding files to blockchain. New blockchain height %d version %d\n", newHeight, newVersion) +} + +func TestBlockchainRead(t *testing.T) { + blockchain, err := initTestPrivateKey() + if err != nil { + return + } + + blockNumber := uint64(0) + + decoded, status, err := blockchain.Read(blockNumber) + switch status { + case 0: + case 1: // Error block not found + fmt.Printf("Error reading block %d: Block not found.\n", blockNumber) + case 2: // Error block encoding + fmt.Printf("Error reading block %d: Block encoding corrupt: %s\n", blockNumber, err.Error()) + case 3: // Error block record encoding + fmt.Printf("Error reading block %d: Block record encoding corrupt.\n", blockNumber) + default: + fmt.Printf("Error reading block %d: Unknown status %d\n", blockNumber, status) + } + + if status != 0 { + return + } + + for _, decodedR := range decoded.RecordsDecoded { + if file, ok := decodedR.(BlockRecordFile); ok { + printFile(file) + } + } +} + +func printFile(file BlockRecordFile) { + fmt.Printf("* File %s\n", file.ID.String()) + fmt.Printf(" Size %d\n", file.Size) + fmt.Printf(" Type %d\n", file.Type) + fmt.Printf(" Format %d\n", file.Format) + fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) + + for _, tag := range file.Tags { + switch tag.Type { + case TagName: + fmt.Printf(" Name %s\n", tag.Text()) + case TagFolder: + fmt.Printf(" Folder %s\n", tag.Text()) + case TagDescription: + fmt.Printf(" Description %s\n", tag.Text()) + } + } +} + +func TestBlockchainDelete(t *testing.T) { + blockchain, err := initTestPrivateKey() + if err != nil { + return + } + + // test add file + file1 := BlockRecordFile{Hash: testHashData([]byte("Test data")), Type: testTypeText, Format: testFormatText, Size: 9, ID: uuid.New()} + file1.Tags = append(file1.Tags, TagFromText(TagName, "Test file to be deleted.txt")) + file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder")) + + newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1}) + fmt.Printf("Added file: Status %d height %d version %d\n", status, newHeight, newVersion) + + // list files + files, _ := blockchain.ListFiles() + for _, file := range files { + printFile(file) + } + + fmt.Printf("----------------\n") + + // delete the file + newHeight, newVersion, status = blockchain.DeleteFiles([]uuid.UUID{file1.ID}) + fmt.Printf("Deleted file: Status %d height %d version %d\n", status, newHeight, newVersion) + + // list all files + files, _ = blockchain.ListFiles() + for _, file := range files { + printFile(file) + } +} + +func TestBlockchainProfile(t *testing.T) { + blockchain, err := initTestPrivateKey() + if err != nil { + return + } + + // write some test profile data + newHeight, newVersion, status := blockchain.ProfileWrite([]BlockRecordProfile{ + ProfileFieldFromText(ProfileName, "Test User 1"), + ProfileFieldFromText(ProfileEmail, "test@test.com"), + {Type: 100, Data: []byte{0, 1, 2, 3}}}) + + fmt.Printf("Write profile data: Status %d height %d version %d\n", status, newHeight, newVersion) + + // list all profile info + printProfileData(blockchain) + + fmt.Printf("----------------\n") + + // delete profile info + newHeight, newVersion, status = blockchain.ProfileDelete([]uint16{ProfileEmail}) + fmt.Printf("Deleted profile email: Status %d height %d version %d\n", status, newHeight, newVersion) + + printProfileData(blockchain) +} + +func printProfileData(blockchain *Blockchain) { + fields, status := blockchain.ProfileList() + if status != BlockchainStatusOK { + fmt.Printf("Reading profile data error status: %d\n", status) + return + } + + if len(fields) == 0 { + fmt.Printf("No profile data to visualize.\n") + return + } + + for _, field := range fields { + printProfileField(field) + } +} + +func printProfileField(field BlockRecordProfile) { + switch field.Type { + case ProfileName, ProfileEmail, ProfileWebsite, ProfileTwitter, ProfileYouTube, ProfileAddress: + fmt.Printf("* Field %d = %s\n", field.Type, string(field.Data)) + + default: + fmt.Printf("* Field %d = %s\n", field.Type, hex.EncodeToString(field.Data)) + } +} + +func testHashData(data []byte) (hash []byte) { + hash32 := blake3.Sum256(data) + return hash32[:] +} + +const testTypeText = 1 +const testFormatText = 10 diff --git a/webapi/Blockchain.go b/webapi/Blockchain.go index d0fb5c7..4a2a192 100644 --- a/webapi/Blockchain.go +++ b/webapi/Blockchain.go @@ -12,6 +12,7 @@ import ( "strconv" "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" ) type apiBlockchainHeader struct { @@ -27,7 +28,7 @@ Request: GET /blockchain/self/header Result: 200 with JSON structure apiResponsePeerSelf */ func apiBlockchainSelfHeader(w http.ResponseWriter, r *http.Request) { - publicKey, height, version := core.UserBlockchainHeader() + publicKey, height, version := core.UserBlockchain.Header() EncodeJSON(w, r, apiBlockchainHeader{Version: version, Height: height, PeerID: hex.EncodeToString(publicKey.SerializeCompressed())}) } @@ -43,7 +44,7 @@ type apiBlockchainBlockRaw struct { } type apiBlockchainBlockStatus struct { - Status int `json:"status"` // Status: 0 = Success, 1 = Error invalid data + Status int `json:"status"` // See BlockchainStatusX. Height uint64 `json:"height"` // Height of the blockchain (number of blocks). Version uint64 `json:"version"` // Version of the blockchain. } @@ -61,13 +62,13 @@ func apiBlockchainSelfAppend(w http.ResponseWriter, r *http.Request) { return } - var records []core.BlockRecordRaw + var records []blockchain.BlockRecordRaw for _, record := range input.Records { - records = append(records, core.BlockRecordRaw{Type: record.Type, Data: record.Data}) + records = append(records, blockchain.BlockRecordRaw{Type: record.Type, Data: record.Data}) } - newHeight, newVersion, status := core.UserBlockchainAppend(records) + newHeight, newVersion, status := core.UserBlockchain.Append(records) EncodeJSON(w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) } @@ -96,7 +97,7 @@ func apiBlockchainSelfRead(w http.ResponseWriter, r *http.Request) { return } - block, status, _ := core.UserBlockchainRead(uint64(blockN)) + block, status, _ := core.UserBlockchain.Read(uint64(blockN)) result := apiBlockchainBlock{Status: status} if status == 0 { @@ -108,10 +109,10 @@ func apiBlockchainSelfRead(w http.ResponseWriter, r *http.Request) { for _, record := range block.RecordsDecoded { switch v := record.(type) { - case core.BlockRecordFile: + case blockchain.BlockRecordFile: result.RecordsDecoded = append(result.RecordsDecoded, blockRecordFileToAPI(v)) - case core.BlockRecordProfile: + case blockchain.BlockRecordProfile: result.RecordsDecoded = append(result.RecordsDecoded, blockRecordProfileToAPI(v)) } diff --git a/webapi/File.go b/webapi/File.go index 0a0d5f8..1bdcf58 100644 --- a/webapi/File.go +++ b/webapi/File.go @@ -11,6 +11,7 @@ import ( "time" "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" "github.com/google/uuid" ) @@ -42,31 +43,31 @@ type apiFile struct { // --- conversion from core to API data --- -func blockRecordFileToAPI(input core.BlockRecordFile) (output apiFile) { +func blockRecordFileToAPI(input blockchain.BlockRecordFile) (output apiFile) { output = apiFile{ID: input.ID, Hash: input.Hash, NodeID: input.NodeID, Type: input.Type, Format: input.Format, Size: input.Size, Metadata: []apiFileMetadata{}} for _, tag := range input.Tags { switch tag.Type { - case core.TagName: + case blockchain.TagName: output.Name = tag.Text() - case core.TagFolder: + case blockchain.TagFolder: output.Folder = tag.Text() - case core.TagDescription: + case blockchain.TagDescription: output.Description = tag.Text() - case core.TagDateShared: + case blockchain.TagDateShared: output.Date, _ = tag.Date() - case core.TagDateCreated: + case blockchain.TagDateCreated: date, _ := tag.Date() output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Date Created", Date: date}) - case core.TagSharedByCount: + case blockchain.TagSharedByCount: output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By Count", Number: tag.Number()}) - case core.TagSharedByGeoIP: + case blockchain.TagSharedByGeoIP: output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By GeoIP", Text: tag.Text()}) default: @@ -77,32 +78,32 @@ func blockRecordFileToAPI(input core.BlockRecordFile) (output apiFile) { return output } -func blockRecordFileFromAPI(input apiFile) (output core.BlockRecordFile) { - output = core.BlockRecordFile{ID: input.ID, Hash: input.Hash, Type: input.Type, Format: input.Format, Size: input.Size} +func blockRecordFileFromAPI(input apiFile) (output blockchain.BlockRecordFile) { + output = blockchain.BlockRecordFile{ID: input.ID, Hash: input.Hash, Type: input.Type, Format: input.Format, Size: input.Size} if input.Name != "" { - output.Tags = append(output.Tags, core.TagFromText(core.TagName, input.Name)) + output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagName, input.Name)) } if input.Folder != "" { - output.Tags = append(output.Tags, core.TagFromText(core.TagFolder, input.Folder)) + output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagFolder, input.Folder)) } if input.Description != "" { - output.Tags = append(output.Tags, core.TagFromText(core.TagDescription, input.Description)) + output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagDescription, input.Description)) } for _, meta := range input.Metadata { - if core.IsTagVirtual(meta.Type) { // Virtual tags are not mapped back. They are read-only. + if blockchain.IsTagVirtual(meta.Type) { // Virtual tags are not mapped back. They are read-only. continue } switch meta.Type { - case core.TagName, core.TagFolder, core.TagDescription: // auto mapped tags + case blockchain.TagName, blockchain.TagFolder, blockchain.TagDescription: // auto mapped tags - case core.TagDateCreated: - output.Tags = append(output.Tags, core.TagFromDate(meta.Type, meta.Date)) + case blockchain.TagDateCreated: + output.Tags = append(output.Tags, blockchain.TagFromDate(meta.Type, meta.Date)) default: - output.Tags = append(output.Tags, core.BlockRecordFileTag{Type: meta.Type, Data: meta.Blob}) + output.Tags = append(output.Tags, blockchain.BlockRecordFileTag{Type: meta.Type, Data: meta.Blob}) } } @@ -129,7 +130,7 @@ func apiBlockchainSelfAddFile(w http.ResponseWriter, r *http.Request) { return } - var filesAdd []core.BlockRecordFile + var filesAdd []blockchain.BlockRecordFile for _, file := range input.Files { if file.ID == uuid.Nil { // if the ID is not provided by the caller, set it @@ -139,7 +140,7 @@ func apiBlockchainSelfAddFile(w http.ResponseWriter, r *http.Request) { filesAdd = append(filesAdd, blockRecordFileFromAPI(file)) } - newHeight, newVersion, status := core.UserBlockchainAddFiles(filesAdd) + newHeight, newVersion, status := core.UserBlockchain.AddFiles(filesAdd) EncodeJSON(w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) } @@ -151,7 +152,7 @@ Request: GET /blockchain/self/list/file Response: 200 with JSON structure apiBlockAddFiles */ func apiBlockchainSelfListFile(w http.ResponseWriter, r *http.Request) { - files, status := core.UserBlockchainListFiles() + files, status := core.UserBlockchain.ListFiles() var result apiBlockAddFiles @@ -182,7 +183,7 @@ func apiBlockchainSelfDeleteFile(w http.ResponseWriter, r *http.Request) { deleteIDs = append(deleteIDs, input.Files[n].ID) } - newHeight, newVersion, status := core.UserBlockchainDeleteFiles(deleteIDs) + newHeight, newVersion, status := core.UserBlockchain.DeleteFiles(deleteIDs) EncodeJSON(w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) } diff --git a/webapi/Profile.go b/webapi/Profile.go index 6208b01..0ed0586 100644 --- a/webapi/Profile.go +++ b/webapi/Profile.go @@ -11,6 +11,7 @@ import ( "strconv" "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" ) // apiProfileData contains profile metadata stored on the blockchain. Any data is treated as untrusted and unverified by default. @@ -35,7 +36,7 @@ Request: GET /profile/list Response: 200 with JSON structure apiProfileData */ func apiProfileList(w http.ResponseWriter, r *http.Request) { - fields, status := core.UserProfileList() + fields, status := core.UserBlockchain.ProfileList() result := apiProfileData{Status: status} for n := range fields { @@ -63,8 +64,8 @@ func apiProfileRead(w http.ResponseWriter, r *http.Request) { var result apiProfileData var data []byte - if data, result.Status = core.UserProfileReadField(uint16(fieldN)); result.Status == core.BlockchainStatusOK { - result.Fields = append(result.Fields, blockRecordProfileToAPI(core.BlockRecordProfile{Type: uint16(fieldN), Data: data})) + if data, result.Status = core.UserBlockchain.ProfileReadField(uint16(fieldN)); result.Status == blockchain.BlockchainStatusOK { + result.Fields = append(result.Fields, blockRecordProfileToAPI(blockchain.BlockRecordProfile{Type: uint16(fieldN), Data: data})) } EncodeJSON(w, r, result) @@ -82,13 +83,13 @@ func apiProfileWrite(w http.ResponseWriter, r *http.Request) { return } - var fields []core.BlockRecordProfile + var fields []blockchain.BlockRecordProfile for n := range input.Fields { fields = append(fields, blockRecordProfileFromAPI(input.Fields[n])) } - newHeight, newVersion, status := core.UserProfileWrite(fields) + newHeight, newVersion, status := core.UserBlockchain.ProfileWrite(fields) EncodeJSON(w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) } @@ -111,21 +112,21 @@ func apiProfileDelete(w http.ResponseWriter, r *http.Request) { fields = append(fields, input.Fields[n].Type) } - newHeight, newVersion, status := core.UserProfileDelete(fields) + newHeight, newVersion, status := core.UserBlockchain.ProfileDelete(fields) EncodeJSON(w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) } // --- conversion from core to API data --- -func blockRecordProfileToAPI(input core.BlockRecordProfile) (output apiBlockRecordProfile) { +func blockRecordProfileToAPI(input blockchain.BlockRecordProfile) (output apiBlockRecordProfile) { output.Type = input.Type switch input.Type { - case core.ProfileName, core.ProfileEmail, core.ProfileWebsite, core.ProfileTwitter, core.ProfileYouTube, core.ProfileAddress: + case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress: output.Text = input.Text() - case core.ProfilePicture: + case blockchain.ProfilePicture: output.Blob = input.Data default: @@ -135,14 +136,14 @@ func blockRecordProfileToAPI(input core.BlockRecordProfile) (output apiBlockReco return output } -func blockRecordProfileFromAPI(input apiBlockRecordProfile) (output core.BlockRecordProfile) { +func blockRecordProfileFromAPI(input apiBlockRecordProfile) (output blockchain.BlockRecordProfile) { output.Type = input.Type switch input.Type { - case core.ProfileName, core.ProfileEmail, core.ProfileWebsite, core.ProfileTwitter, core.ProfileYouTube, core.ProfileAddress: + case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress: output.Data = []byte(input.Text) - case core.ProfilePicture: + case blockchain.ProfilePicture: output.Data = input.Blob default: diff --git a/webapi/Search Job.go b/webapi/Search Job.go index 562f994..9d83502 100644 --- a/webapi/Search Job.go +++ b/webapi/Search Job.go @@ -11,7 +11,7 @@ import ( "sync" "time" - "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" "github.com/google/uuid" ) @@ -273,11 +273,11 @@ func SortFiles(files []*apiFile, Sort int) (sorted []*apiFile) { case SortSharedByCountAsc: sort.SliceStable(files, func(i, j int) bool { - return files[i].GetMetadata(core.TagSharedByCount).Number < files[j].GetMetadata(core.TagSharedByCount).Number + return files[i].GetMetadata(blockchain.TagSharedByCount).Number < files[j].GetMetadata(blockchain.TagSharedByCount).Number }) case SortSharedByCountDesc: sort.SliceStable(files, func(i, j int) bool { - return files[i].GetMetadata(core.TagSharedByCount).Number > files[j].GetMetadata(core.TagSharedByCount).Number + return files[i].GetMetadata(blockchain.TagSharedByCount).Number > files[j].GetMetadata(blockchain.TagSharedByCount).Number }) } diff --git a/webapi/Search Temp.go b/webapi/Search Temp.go index 21716ff..4a14fbe 100644 --- a/webapi/Search Temp.go +++ b/webapi/Search Temp.go @@ -15,6 +15,7 @@ import ( "time" "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" "github.com/google/uuid" ) @@ -58,7 +59,7 @@ func dispatchSearch(input SearchRequest) (job *SearchJob) { } // createTestResult creates a test file. fileType = -1 for any. -func createTestResult(fileType int) (file core.BlockRecordFile) { +func createTestResult(fileType int) (file blockchain.BlockRecordFile) { randomData := make([]byte, 10*1024) rand.Read(randomData) @@ -192,12 +193,12 @@ func createTestResult(fileType int) (file core.BlockRecordFile) { extension = ".bin" } - file.Tags = append(file.Tags, core.TagFromText(core.TagName, tempFileName("", extension))) - //file.Tags = append(file.Tags, core.TagFromText(core.TagFolder, "not set")) - file.Tags = append(file.Tags, core.TagFromDate(core.TagDateShared, time.Now().UTC())) + file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagName, tempFileName("", extension))) + //file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagFolder, "not set")) + file.Tags = append(file.Tags, blockchain.TagFromDate(blockchain.TagDateShared, time.Now().UTC())) sharedByCount := uint64(rand.Intn(10)) - file.Tags = append(file.Tags, core.TagFromNumber(core.TagSharedByCount, sharedByCount)) + file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, sharedByCount)) if sharedByCount > 0 { var sharedByGeoIP string @@ -210,7 +211,7 @@ func createTestResult(fileType int) (file core.BlockRecordFile) { sharedByGeoIP += fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) } - file.Tags = append(file.Tags, core.TagFromText(core.TagSharedByGeoIP, sharedByGeoIP)) + file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP)) } return @@ -235,7 +236,7 @@ func randomGeoIP() (latitude, longitude float32) { } // queryRecentShared returns recently shared files on the network. fileType = -1 for any. -func queryRecentShared(fileType, limit int) (files []*core.BlockRecordFile) { +func queryRecentShared(fileType, limit int) (files []*blockchain.BlockRecordFile) { for n := 0; n < limit; n++ { newFile := createTestResult(fileType) files = append(files, &newFile)