From c8e0d1cfbe0c037653287596a7e85badbdc12ee4 Mon Sep 17 00:00:00 2001 From: Kleissner Date: Sat, 11 Dec 2021 18:07:48 +0100 Subject: [PATCH] Blockchain: New TargetBlockSize which is honored for files and profile fields. Close #80, close #75 --- blockchain/Block Record File.go | 24 +++++++++++++++++++--- blockchain/Block Record Profile.go | 5 +++++ blockchain/Blockchain.go | 7 +++++++ blockchain/File.go | 32 ++++++++++++++++++++++++++---- blockchain/Profile.go | 32 ++++++++++++++++++++++++++---- blockchain/readme.md | 7 ++++++- 6 files changed, 95 insertions(+), 12 deletions(-) diff --git a/blockchain/Block Record File.go b/blockchain/Block Record File.go index 870b738..a019fa3 100644 --- a/blockchain/Block Record File.go +++ b/blockchain/Block Record File.go @@ -59,12 +59,14 @@ type BlockRecordFileTag struct { // This is an embedded basic compression algorithm for repetitive tag. For example directory tags or album tags might be heavily repetitive among files. } +const blockRecordFileMinSize = 101 + // decodeBlockRecordFiles decodes only file records. Other records are ignored. func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files []BlockRecordFile, err error) { for i, record := range recordsRaw { switch record.Type { case RecordTypeFile: - if len(record.Data) < 101 { + if len(record.Data) < blockRecordFileMinSize { return nil, errors.New("file record invalid size") } @@ -83,7 +85,7 @@ func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files [ countTags := binary.LittleEndian.Uint16(record.Data[99 : 99+2]) - index := 101 + index := blockRecordFileMinSize for n := uint16(0); n < countTags; n++ { if index+6 > len(record.Data) { @@ -160,7 +162,7 @@ func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRa // then encode all files as records for n := range files { - data := make([]byte, 101) + data := make([]byte, blockRecordFileMinSize) if len(files[n].Hash) != protocol.HashSize { return nil, errors.New("encodeBlockRecords invalid file hash") @@ -226,3 +228,19 @@ func intToBytes(number int) (buffer []byte) { binary.LittleEndian.PutUint64(buffer[0:8], uint64(number)) return buffer[0:8] } + +// SizeInBlock returns the full size this file takes up in a single block. (i.e., the record size) +// If paired with other files in a single block, compression (via tag references) may reduce the actual size. +func (file *BlockRecordFile) SizeInBlock() (size uint64) { + size = blockRecordHeaderSize + blockRecordFileMinSize + + for _, tag := range file.Tags { + if tag.IsVirtual() { + continue + } + + size += 6 + uint64(len(tag.Data)) + } + + return size +} diff --git a/blockchain/Block Record Profile.go b/blockchain/Block Record Profile.go index eefff58..260e200 100644 --- a/blockchain/Block Record Profile.go +++ b/blockchain/Block Record Profile.go @@ -68,3 +68,8 @@ func encodeBlockRecordProfile(fields []BlockRecordProfile) (recordsRaw []BlockRe return recordsRaw, nil } + +// SizeInBlock returns the full size this file takes up in a single block. (i.e., the record size) +func (field *BlockRecordProfile) SizeInBlock() (size uint64) { + return blockRecordHeaderSize + 2 + uint64(len(field.Data)) +} diff --git a/blockchain/Blockchain.go b/blockchain/Blockchain.go index ec71e4e..89cd164 100644 --- a/blockchain/Blockchain.go +++ b/blockchain/Blockchain.go @@ -27,6 +27,13 @@ import ( "github.com/PeernetOfficial/core/store" ) +// TargetBlockSize is the target size that a generated block shall not exceed. This ensures the block will be transferred via blockchain exchange and cached in DHT. +// Large blocks may be ignored by clients for size and spam reasons, resulting in decreased discoverability. +var TargetBlockSize = uint64(4096) + +// MinAcceptableBlockSize is the minimum block size peers must accept. +const MinAcceptableBlockSize = uint64(1024) + // Blockchain stores the blockchain's header in memory. Any changes must be synced to disk! type Blockchain struct { // header diff --git a/blockchain/File.go b/blockchain/File.go index 218d95e..f7eb30e 100644 --- a/blockchain/File.go +++ b/blockchain/File.go @@ -17,12 +17,36 @@ import ( // AddFiles adds files to the blockchain. Status is StatusX. // It makes sense to group all files in the same directory into one call, since only one directory record will be created per unique directory per block. func (blockchain *Blockchain) AddFiles(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { - encoded, err := encodeBlockRecordFiles(files) - if err != nil { - return 0, 0, StatusCorruptBlockRecord + encodeFilesAppend := func(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { + encoded, err := encodeBlockRecordFiles(files) + if err != nil { + return 0, 0, StatusCorruptBlockRecord + } + + return blockchain.Append(encoded) } - return blockchain.Append(encoded) + blockSize := uint64(blockHeaderSize) + var recordFiles []BlockRecordFile + + for _, file := range files { + recordSize := file.SizeInBlock() + + // need to create a new block due to target block size? + if len(recordFiles) > 0 && blockSize+recordSize > TargetBlockSize { + if newHeight, newVersion, status = encodeFilesAppend(recordFiles); status != StatusOK { + return newHeight, newVersion, status + } + + blockSize = blockHeaderSize + recordFiles = nil + } + + blockSize += recordSize + recordFiles = append(recordFiles, file) + } + + return encodeFilesAppend(recordFiles) } // ListFiles returns a list of all files. Status is StatusX. diff --git a/blockchain/Profile.go b/blockchain/Profile.go index 7ab740e..1ce8c19 100644 --- a/blockchain/Profile.go +++ b/blockchain/Profile.go @@ -64,12 +64,36 @@ func (blockchain *Blockchain) ProfileList() (fields []BlockRecordProfile, status // ProfileWrite writes profile fields and blobs to the blockchain. Status is StatusX. func (blockchain *Blockchain) ProfileWrite(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) { - encoded, err := encodeBlockRecordProfile(fields) - if err != nil { - return 0, 0, StatusCorruptBlockRecord + encodeProfileAppend := func(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) { + encoded, err := encodeBlockRecordProfile(fields) + if err != nil { + return 0, 0, StatusCorruptBlockRecord + } + + return blockchain.Append(encoded) } - return blockchain.Append(encoded) + blockSize := uint64(blockHeaderSize) + var recordFields []BlockRecordProfile + + for _, field := range fields { + recordSize := field.SizeInBlock() + + // need to create a new block due to target block size? + if len(recordFields) > 0 && blockSize+recordSize > TargetBlockSize { + if newHeight, newVersion, status = encodeProfileAppend(recordFields); status != StatusOK { + return newHeight, newVersion, status + } + + blockSize = blockHeaderSize + recordFields = nil + } + + blockSize += recordSize + recordFields = append(recordFields, field) + } + + return encodeProfileAppend(recordFields) } // ProfileDelete deletes fields and blobs from the blockchain. Status is StatusX. diff --git a/blockchain/readme.md b/blockchain/readme.md index cd70f92..3bcddf0 100644 --- a/blockchain/readme.md +++ b/blockchain/readme.md @@ -48,7 +48,12 @@ Offset Size Info ## Block Size -The block size is currently recommended to be slightly below 64 KB (minus message header overhead), so that it fits within a single UDP packet. Having a block size smaller than the max. message size reduces complexity when exchanging individual blocks and increases performance for operations such as file search. +Peers must accept a minimum block size of 1 KB. + +The target block size (for generating new blocks) is defined via `TargetBlockSize`. If records cannot fit within that target size, they are added into a new block. + +Small block sizes ensure that the block will be transferred via blockchain exchange and cached in DHT. +Large blocks may be ignored by clients for size and spam reasons, resulting in decreased discoverability. ## Edge Cases