Peernet-Archive/core
1
0
Fork 0
mirror of https://github.com/PeernetOfficial/core.git synced 2026-07-16 18:37:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

Protocol change: Implement merkle tree for files and store root hash and fragment size in the blockchain.

Browse Source
This commit is contained in:
Kleissner
2021-11-28 21:59:52 +01:00
parent 05a31cf3af
commit e5c2d8d91d
2 changed files with 72 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
blockchain/Block Record File.go
View File

@@ -7,11 +7,13 @@ File records:
Offset Size Info
0 32 Hash blake3 of the file content
32 16 File ID
48 1 File Type
49 2 File Format
51 8 File Size
59 2 Count of Tags
61 ? Tags
48 32 Merkle Root Hash
80 8 Fragment Size
88 1 File Type
89 2 File Format
91 8 File Size
99 2 Count of Tags
101 ? Tags
Each file tag provides additional optional information:
Offset Size Info
@@ -37,13 +39,15 @@ import (
// BlockRecordFile is the metadata of a file published on the blockchain
type BlockRecordFile struct {
Hash []byte // Hash of the file data
ID uuid.UUID // ID of the file
Type uint8 // File Type
Format uint16 // File Format
Size uint64 // Size of the file data
NodeID []byte // Node ID, owner of the file
Tags []BlockRecordFileTag // Tags provide additional metadata
Hash []byte // Hash of the file data
ID uuid.UUID // ID of the file
MerkleRootHash []byte // Merkle Root Hash
FragmentSize uint64 // Fragment Size
Type uint8 // File Type
Format uint16 // File Format
Size uint64 // Size of the file data
NodeID []byte // Node ID, owner of the file
Tags []BlockRecordFileTag // Tags provide additional metadata
}
// BlockRecordFileTag provides metadata about the file.
@@ -60,7 +64,7 @@ func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files [
for i, record := range recordsRaw {
switch record.Type {
case RecordTypeFile:
if len(record.Data) < 61 {
if len(record.Data) < 101 {
return nil, errors.New("file record invalid size")
}
@@ -68,13 +72,18 @@ func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files [
file.Hash = make([]byte, protocol.HashSize)
copy(file.Hash, record.Data[0:0+protocol.HashSize])
copy(file.ID[:], record.Data[32:32+16])
file.Type = record.Data[48]
file.Format = binary.LittleEndian.Uint16(record.Data[49 : 49+2])
file.Size = binary.LittleEndian.Uint64(record.Data[51 : 51+8])
countTags := binary.LittleEndian.Uint16(record.Data[59 : 59+2])
file.MerkleRootHash = make([]byte, protocol.HashSize)
copy(file.MerkleRootHash, record.Data[48:48+protocol.HashSize])
file.FragmentSize = binary.LittleEndian.Uint64(record.Data[80 : 80+8])
index := 61
file.Type = record.Data[88]
file.Format = binary.LittleEndian.Uint16(record.Data[89 : 89+2])
file.Size = binary.LittleEndian.Uint64(record.Data[91 : 91+8])
countTags := binary.LittleEndian.Uint16(record.Data[99 : 99+2])
index := 101
for n := uint16(0); n < countTags; n++ {
if index+6 > len(record.Data) {
@@ -151,19 +160,23 @@ func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRa
// then encode all files as records
for n := range files {
data := make([]byte, 61)
data := make([]byte, 101)
if len(files[n].Hash) != protocol.HashSize {
return nil, errors.New("encodeBlockRecords invalid file hash")
} else if len(files[n].MerkleRootHash) != protocol.HashSize {
return nil, errors.New("encodeBlockRecords invalid merkle root hash")
}
copy(data[0:32], files[n].Hash[0:32])
copy(data[32:32+16], files[n].ID[:])
copy(data[48:48+32], files[n].MerkleRootHash[0:32])
binary.LittleEndian.PutUint64(data[80:80+8], files[n].FragmentSize)
data[48] = files[n].Type
binary.LittleEndian.PutUint16(data[49:49+2], files[n].Format)
binary.LittleEndian.PutUint64(data[51:51+8], files[n].Size)
binary.LittleEndian.PutUint16(data[59:59+2], uint16(len(files[n].Tags)))
data[88] = files[n].Type
binary.LittleEndian.PutUint16(data[89:89+2], files[n].Format)
binary.LittleEndian.PutUint64(data[91:91+8], files[n].Size)
binary.LittleEndian.PutUint16(data[99:99+2], uint16(len(files[n].Tags)))
for _, tag := range files[n].Tags {
// Some tags are virtual and never stored on the blockchain. If attempted to write, ignore.

58
blockchain/Test_test.go
View File

@@ -1,11 +1,13 @@
package blockchain
import (
"bytes"
"encoding/hex"
"fmt"
"testing"
"github.com/PeernetOfficial/core/btcec"
"github.com/PeernetOfficial/core/merkle"
"github.com/PeernetOfficial/core/protocol"
"github.com/google/uuid"
)
@@ -19,15 +21,14 @@ func TestBlockEncoding(t *testing.T) {
encoded1, _ := encodeBlockRecordProfile([]BlockRecordProfile{ProfileFieldFromText(ProfileName, "Test User 1")})
file1 := BlockRecordFile{Hash: protocol.HashData([]byte("Test data")), Type: testTypeText, Format: testFormatText, Size: 9, ID: uuid.New()}
file1.Tags = append(file1.Tags, TagFromText(TagName, "Filename 1.txt"))
file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder"))
file1, _ := createBlockRecordFile([]byte("Test data"), "Filename 1.txt", "documents\\sub folder")
file2, _ := createBlockRecordFile([]byte("Test data 2!"), "Filename 2.txt", "documents\\sub folder")
file2 := BlockRecordFile{Hash: protocol.HashData([]byte("Test data 2!")), Type: testTypeText, Format: testFormatText, Size: 10, ID: uuid.New()}
file2.Tags = append(file2.Tags, TagFromText(TagName, "Filename 2.txt"))
file2.Tags = append(file2.Tags, TagFromText(TagFolder, "documents\\sub folder"))
encodedFiles, _ := encodeBlockRecordFiles([]BlockRecordFile{file1, file2})
encodedFiles, err := encodeBlockRecordFiles([]BlockRecordFile{file1, file2})
if err != nil {
fmt.Printf("Error encoding files: %s\n", err.Error())
return
}
blockE := &Block{BlockchainVersion: 42, Number: 0}
blockE.RecordsRaw = append(blockE.RecordsRaw, encoded1...)
@@ -67,6 +68,21 @@ func TestBlockEncoding(t *testing.T) {
}
}
func createBlockRecordFile(data []byte, name, folder string) (file BlockRecordFile, err error) {
file = BlockRecordFile{Hash: protocol.HashData(data), Type: testTypeText, Format: testFormatText, Size: uint64(len(data)), ID: uuid.New()}
file.Tags = append(file.Tags, TagFromText(TagName, name))
file.Tags = append(file.Tags, TagFromText(TagFolder, folder))
file.FragmentSize = merkle.CalculateFragmentSize(file.Size)
tree, err := merkle.NewMerkleTree(file.Size, file.FragmentSize, bytes.NewBuffer(data))
if err != nil {
return file, err
}
file.MerkleRootHash = tree.RootHash
return file, nil
}
func initTestPrivateKey() (blockchain *Blockchain, err error) {
// use static test key, otherwise tests will be inconsistent (would otherwise fail to open blockchain database)
privateKeyTestA := "d65da474861d826edd29c1307f1250d79e9dbf84e3a2449022658445c8d8ed63"
@@ -84,9 +100,7 @@ func TestBlockchainAdd(t *testing.T) {
return
}
file1 := BlockRecordFile{Hash: protocol.HashData([]byte("Test data")), Type: testTypeText, Format: testFormatText, Size: 9, ID: uuid.New()}
file1.Tags = append(file1.Tags, TagFromText(TagName, "Filename 1.txt"))
file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder"))
file1, _ := createBlockRecordFile([]byte("Test data"), "Filename 1.txt", "documents\\sub folder")
newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1})
@@ -142,20 +156,22 @@ func TestBlockchainRead(t *testing.T) {
}
func printFile(file BlockRecordFile) {
fmt.Printf("* File %s\n", file.ID.String())
fmt.Printf(" Size %d\n", file.Size)
fmt.Printf(" Type %d\n", file.Type)
fmt.Printf(" Format %d\n", file.Format)
fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash))
fmt.Printf("* File %s\n", file.ID.String())
fmt.Printf(" Size %d\n", file.Size)
fmt.Printf(" Type %d\n", file.Type)
fmt.Printf(" Format %d\n", file.Format)
fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash))
fmt.Printf(" Merkle Root Hash %s\n", hex.EncodeToString(file.MerkleRootHash))
fmt.Printf(" Fragment Size %d\n", file.FragmentSize)
for _, tag := range file.Tags {
switch tag.Type {
case TagName:
fmt.Printf(" Name %s\n", tag.Text())
fmt.Printf(" Name %s\n", tag.Text())
case TagFolder:
fmt.Printf(" Folder %s\n", tag.Text())
fmt.Printf(" Folder %s\n", tag.Text())
case TagDescription:
fmt.Printf(" Description %s\n", tag.Text())
fmt.Printf(" Description %s\n", tag.Text())
}
}
}
@@ -167,9 +183,7 @@ func TestBlockchainDelete(t *testing.T) {
}
// test add file
file1 := BlockRecordFile{Hash: protocol.HashData([]byte("Test data")), Type: testTypeText, Format: testFormatText, Size: 9, ID: uuid.New()}
file1.Tags = append(file1.Tags, TagFromText(TagName, "Test file to be deleted.txt"))
file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder"))
file1, _ := createBlockRecordFile([]byte("Test data"), "Test file to be deleted.txt", "documents\\sub folder")
newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1})
fmt.Printf("Added file: Status %d height %d version %d\n", status, newHeight, newVersion)