Protocol change: Implement merkle tree for files and store root hash and fragment size in the blockchain.

blockchain/Block Record File.go

@@ -7,11 +7,13 @@ File records:
 Offset  Size    Info
 0       32      Hash blake3 of the file content
 32      16      File ID
-48      1       File Type
+48      32      Merkle Root Hash
-49      2       File Format
+80      8       Fragment Size
-51      8       File Size
+88      1       File Type
-59      2       Count of Tags
+89      2       File Format
-61      ?       Tags
+91      8       File Size
+99      2       Count of Tags
+101     ?       Tags
 
 Each file tag provides additional optional information:
 Offset  Size    Info
@@ -37,13 +39,15 @@ import (
 
 // BlockRecordFile is the metadata of a file published on the blockchain
 type BlockRecordFile struct {
-	Hash   []byte               // Hash of the file data
+	Hash           []byte               // Hash of the file data
-	ID     uuid.UUID            // ID of the file
+	ID             uuid.UUID            // ID of the file
-	Type   uint8                // File Type
+	MerkleRootHash []byte               // Merkle Root Hash
-	Format uint16               // File Format
+	FragmentSize   uint64               // Fragment Size
-	Size   uint64               // Size of the file data
+	Type           uint8                // File Type
-	NodeID []byte               // Node ID, owner of the file
+	Format         uint16               // File Format
-	Tags   []BlockRecordFileTag // Tags provide additional metadata
+	Size           uint64               // Size of the file data
+	NodeID         []byte               // Node ID, owner of the file
+	Tags           []BlockRecordFileTag // Tags provide additional metadata
 }
 
 // BlockRecordFileTag provides metadata about the file.
@@ -60,7 +64,7 @@ func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files [
 	for i, record := range recordsRaw {
 		switch record.Type {
 		case RecordTypeFile:
-			if len(record.Data) < 61 {
+			if len(record.Data) < 101 {
 				return nil, errors.New("file record invalid size")
 			}
 
@@ -68,13 +72,18 @@ func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files [
 			file.Hash = make([]byte, protocol.HashSize)
 			copy(file.Hash, record.Data[0:0+protocol.HashSize])
 			copy(file.ID[:], record.Data[32:32+16])
-			file.Type = record.Data[48]
-			file.Format = binary.LittleEndian.Uint16(record.Data[49 : 49+2])
-			file.Size = binary.LittleEndian.Uint64(record.Data[51 : 51+8])
 
-			countTags := binary.LittleEndian.Uint16(record.Data[59 : 59+2])
+			file.MerkleRootHash = make([]byte, protocol.HashSize)
+			copy(file.MerkleRootHash, record.Data[48:48+protocol.HashSize])
+			file.FragmentSize = binary.LittleEndian.Uint64(record.Data[80 : 80+8])
 
-			index := 61
+			file.Type = record.Data[88]
+			file.Format = binary.LittleEndian.Uint16(record.Data[89 : 89+2])
+			file.Size = binary.LittleEndian.Uint64(record.Data[91 : 91+8])
+
+			countTags := binary.LittleEndian.Uint16(record.Data[99 : 99+2])
+
+			index := 101
 
 			for n := uint16(0); n < countTags; n++ {
 				if index+6 > len(record.Data) {
@@ -151,19 +160,23 @@ func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRa
 
 	// then encode all files as records
 	for n := range files {
-		data := make([]byte, 61)
+		data := make([]byte, 101)
 
 		if len(files[n].Hash) != protocol.HashSize {
 			return nil, errors.New("encodeBlockRecords invalid file hash")
+		} else if len(files[n].MerkleRootHash) != protocol.HashSize {
+			return nil, errors.New("encodeBlockRecords invalid merkle root hash")
 		}
 
 		copy(data[0:32], files[n].Hash[0:32])
 		copy(data[32:32+16], files[n].ID[:])
+		copy(data[48:48+32], files[n].MerkleRootHash[0:32])
+		binary.LittleEndian.PutUint64(data[80:80+8], files[n].FragmentSize)
 
-		data[48] = files[n].Type
+		data[88] = files[n].Type
-		binary.LittleEndian.PutUint16(data[49:49+2], files[n].Format)
+		binary.LittleEndian.PutUint16(data[89:89+2], files[n].Format)
-		binary.LittleEndian.PutUint64(data[51:51+8], files[n].Size)
+		binary.LittleEndian.PutUint64(data[91:91+8], files[n].Size)
-		binary.LittleEndian.PutUint16(data[59:59+2], uint16(len(files[n].Tags)))
+		binary.LittleEndian.PutUint16(data[99:99+2], uint16(len(files[n].Tags)))
 
 		for _, tag := range files[n].Tags {
 			// Some tags are virtual and never stored on the blockchain. If attempted to write, ignore.

blockchain/Test_test.go

@@ -1,11 +1,13 @@
 package blockchain
 
 import (
+	"bytes"
 	"encoding/hex"
 	"fmt"
 	"testing"
 
 	"github.com/PeernetOfficial/core/btcec"
+	"github.com/PeernetOfficial/core/merkle"
 	"github.com/PeernetOfficial/core/protocol"
 	"github.com/google/uuid"
 )
@@ -19,15 +21,14 @@ func TestBlockEncoding(t *testing.T) {
 
 	encoded1, _ := encodeBlockRecordProfile([]BlockRecordProfile{ProfileFieldFromText(ProfileName, "Test User 1")})
 
-	file1 := BlockRecordFile{Hash: protocol.HashData([]byte("Test data")), Type: testTypeText, Format: testFormatText, Size: 9, ID: uuid.New()}
+	file1, _ := createBlockRecordFile([]byte("Test data"), "Filename 1.txt", "documents\\sub folder")
-	file1.Tags = append(file1.Tags, TagFromText(TagName, "Filename 1.txt"))
+	file2, _ := createBlockRecordFile([]byte("Test data 2!"), "Filename 2.txt", "documents\\sub folder")
-	file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder"))
 
-	file2 := BlockRecordFile{Hash: protocol.HashData([]byte("Test data 2!")), Type: testTypeText, Format: testFormatText, Size: 10, ID: uuid.New()}
+	encodedFiles, err := encodeBlockRecordFiles([]BlockRecordFile{file1, file2})
-	file2.Tags = append(file2.Tags, TagFromText(TagName, "Filename 2.txt"))
+	if err != nil {
-	file2.Tags = append(file2.Tags, TagFromText(TagFolder, "documents\\sub folder"))
+		fmt.Printf("Error encoding files: %s\n", err.Error())
-
+		return
-	encodedFiles, _ := encodeBlockRecordFiles([]BlockRecordFile{file1, file2})
+	}
 
 	blockE := &Block{BlockchainVersion: 42, Number: 0}
 	blockE.RecordsRaw = append(blockE.RecordsRaw, encoded1...)
@@ -67,6 +68,21 @@ func TestBlockEncoding(t *testing.T) {
 	}
 }
 
+func createBlockRecordFile(data []byte, name, folder string) (file BlockRecordFile, err error) {
+	file = BlockRecordFile{Hash: protocol.HashData(data), Type: testTypeText, Format: testFormatText, Size: uint64(len(data)), ID: uuid.New()}
+	file.Tags = append(file.Tags, TagFromText(TagName, name))
+	file.Tags = append(file.Tags, TagFromText(TagFolder, folder))
+
+	file.FragmentSize = merkle.CalculateFragmentSize(file.Size)
+	tree, err := merkle.NewMerkleTree(file.Size, file.FragmentSize, bytes.NewBuffer(data))
+	if err != nil {
+		return file, err
+	}
+	file.MerkleRootHash = tree.RootHash
+
+	return file, nil
+}
+
 func initTestPrivateKey() (blockchain *Blockchain, err error) {
 	// use static test key, otherwise tests will be inconsistent (would otherwise fail to open blockchain database)
 	privateKeyTestA := "d65da474861d826edd29c1307f1250d79e9dbf84e3a2449022658445c8d8ed63"
@@ -84,9 +100,7 @@ func TestBlockchainAdd(t *testing.T) {
 		return
 	}
 
-	file1 := BlockRecordFile{Hash: protocol.HashData([]byte("Test data")), Type: testTypeText, Format: testFormatText, Size: 9, ID: uuid.New()}
+	file1, _ := createBlockRecordFile([]byte("Test data"), "Filename 1.txt", "documents\\sub folder")
-	file1.Tags = append(file1.Tags, TagFromText(TagName, "Filename 1.txt"))
-	file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder"))
 
 	newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1})
 
@@ -142,20 +156,22 @@ func TestBlockchainRead(t *testing.T) {
 }
 
 func printFile(file BlockRecordFile) {
-	fmt.Printf("* File          %s\n", file.ID.String())
+	fmt.Printf("* File                %s\n", file.ID.String())
-	fmt.Printf("  Size          %d\n", file.Size)
+	fmt.Printf("  Size                %d\n", file.Size)
-	fmt.Printf("  Type          %d\n", file.Type)
+	fmt.Printf("  Type                %d\n", file.Type)
-	fmt.Printf("  Format        %d\n", file.Format)
+	fmt.Printf("  Format              %d\n", file.Format)
-	fmt.Printf("  Hash          %s\n", hex.EncodeToString(file.Hash))
+	fmt.Printf("  Hash                %s\n", hex.EncodeToString(file.Hash))
+	fmt.Printf("  Merkle Root Hash    %s\n", hex.EncodeToString(file.MerkleRootHash))
+	fmt.Printf("  Fragment Size       %d\n", file.FragmentSize)
 
 	for _, tag := range file.Tags {
 		switch tag.Type {
 		case TagName:
-			fmt.Printf("  Name          %s\n", tag.Text())
+			fmt.Printf("  Name                %s\n", tag.Text())
 		case TagFolder:
-			fmt.Printf("  Folder        %s\n", tag.Text())
+			fmt.Printf("  Folder              %s\n", tag.Text())
 		case TagDescription:
-			fmt.Printf("  Description   %s\n", tag.Text())
+			fmt.Printf("  Description         %s\n", tag.Text())
 		}
 	}
 }
@@ -167,9 +183,7 @@ func TestBlockchainDelete(t *testing.T) {
 	}
 
 	// test add file
-	file1 := BlockRecordFile{Hash: protocol.HashData([]byte("Test data")), Type: testTypeText, Format: testFormatText, Size: 9, ID: uuid.New()}
+	file1, _ := createBlockRecordFile([]byte("Test data"), "Test file to be deleted.txt", "documents\\sub folder")
-	file1.Tags = append(file1.Tags, TagFromText(TagName, "Test file to be deleted.txt"))
-	file1.Tags = append(file1.Tags, TagFromText(TagFolder, "documents\\sub folder"))
 
 	newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1})
 	fmt.Printf("Added file: Status %d height %d version %d\n", status, newHeight, newVersion)