Blockchain: New TargetBlockSize which is honored for files and profile fields. Close #80, close #75

blockchain/Block Record File.go

@@ -59,12 +59,14 @@ type BlockRecordFileTag struct {
 	// This is an embedded basic compression algorithm for repetitive tag. For example directory tags or album tags might be heavily repetitive among files.
 }
 
+const blockRecordFileMinSize = 101
+
 // decodeBlockRecordFiles decodes only file records. Other records are ignored.
 func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files []BlockRecordFile, err error) {
 	for i, record := range recordsRaw {
 		switch record.Type {
 		case RecordTypeFile:
-			if len(record.Data) < 101 {
+			if len(record.Data) < blockRecordFileMinSize {
 				return nil, errors.New("file record invalid size")
 			}
 
@@ -83,7 +85,7 @@ func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files [
 
 			countTags := binary.LittleEndian.Uint16(record.Data[99 : 99+2])
 
-			index := 101
+			index := blockRecordFileMinSize
 
 			for n := uint16(0); n < countTags; n++ {
 				if index+6 > len(record.Data) {
@@ -160,7 +162,7 @@ func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRa
 
 	// then encode all files as records
 	for n := range files {
-		data := make([]byte, 101)
+		data := make([]byte, blockRecordFileMinSize)
 
 		if len(files[n].Hash) != protocol.HashSize {
 			return nil, errors.New("encodeBlockRecords invalid file hash")
@@ -226,3 +228,19 @@ func intToBytes(number int) (buffer []byte) {
 	binary.LittleEndian.PutUint64(buffer[0:8], uint64(number))
 	return buffer[0:8]
 }
+
+// SizeInBlock returns the full size this file takes up in a single block. (i.e., the record size)
+// If paired with other files in a single block, compression (via tag references) may reduce the actual size.
+func (file *BlockRecordFile) SizeInBlock() (size uint64) {
+	size = blockRecordHeaderSize + blockRecordFileMinSize
+
+	for _, tag := range file.Tags {
+		if tag.IsVirtual() {
+			continue
+		}
+
+		size += 6 + uint64(len(tag.Data))
+	}
+
+	return size
+}

blockchain/Block Record Profile.go

@@ -68,3 +68,8 @@ func encodeBlockRecordProfile(fields []BlockRecordProfile) (recordsRaw []BlockRe
 
 	return recordsRaw, nil
 }
+
+// SizeInBlock returns the full size this file takes up in a single block. (i.e., the record size)
+func (field *BlockRecordProfile) SizeInBlock() (size uint64) {
+	return blockRecordHeaderSize + 2 + uint64(len(field.Data))
+}

blockchain/Blockchain.go

@@ -27,6 +27,13 @@ import (
 	"github.com/PeernetOfficial/core/store"
 )
 
+// TargetBlockSize is the target size that a generated block shall not exceed. This ensures the block will be transferred via blockchain exchange and cached in DHT.
+// Large blocks may be ignored by clients for size and spam reasons, resulting in decreased discoverability.
+var TargetBlockSize = uint64(4096)
+
+// MinAcceptableBlockSize is the minimum block size peers must accept.
+const MinAcceptableBlockSize = uint64(1024)
+
 // Blockchain stores the blockchain's header in memory. Any changes must be synced to disk!
 type Blockchain struct {
 	// header

blockchain/File.go

@@ -17,12 +17,36 @@ import (
 // AddFiles adds files to the blockchain. Status is StatusX.
 // It makes sense to group all files in the same directory into one call, since only one directory record will be created per unique directory per block.
 func (blockchain *Blockchain) AddFiles(files []BlockRecordFile) (newHeight, newVersion uint64, status int) {
-	encoded, err := encodeBlockRecordFiles(files)
-	if err != nil {
-		return 0, 0, StatusCorruptBlockRecord
+	encodeFilesAppend := func(files []BlockRecordFile) (newHeight, newVersion uint64, status int) {
+		encoded, err := encodeBlockRecordFiles(files)
+		if err != nil {
+			return 0, 0, StatusCorruptBlockRecord
+		}
+
+		return blockchain.Append(encoded)
 	}
 
-	return blockchain.Append(encoded)
+	blockSize := uint64(blockHeaderSize)
+	var recordFiles []BlockRecordFile
+
+	for _, file := range files {
+		recordSize := file.SizeInBlock()
+
+		// need to create a new block due to target block size?
+		if len(recordFiles) > 0 && blockSize+recordSize > TargetBlockSize {
+			if newHeight, newVersion, status = encodeFilesAppend(recordFiles); status != StatusOK {
+				return newHeight, newVersion, status
+			}
+
+			blockSize = blockHeaderSize
+			recordFiles = nil
+		}
+
+		blockSize += recordSize
+		recordFiles = append(recordFiles, file)
+	}
+
+	return encodeFilesAppend(recordFiles)
 }
 
 // ListFiles returns a list of all files. Status is StatusX.

blockchain/Profile.go

@@ -64,12 +64,36 @@ func (blockchain *Blockchain) ProfileList() (fields []BlockRecordProfile, status
 
 // ProfileWrite writes profile fields and blobs to the blockchain. Status is StatusX.
 func (blockchain *Blockchain) ProfileWrite(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) {
-	encoded, err := encodeBlockRecordProfile(fields)
-	if err != nil {
-		return 0, 0, StatusCorruptBlockRecord
+	encodeProfileAppend := func(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) {
+		encoded, err := encodeBlockRecordProfile(fields)
+		if err != nil {
+			return 0, 0, StatusCorruptBlockRecord
+		}
+
+		return blockchain.Append(encoded)
 	}
 
-	return blockchain.Append(encoded)
+	blockSize := uint64(blockHeaderSize)
+	var recordFields []BlockRecordProfile
+
+	for _, field := range fields {
+		recordSize := field.SizeInBlock()
+
+		// need to create a new block due to target block size?
+		if len(recordFields) > 0 && blockSize+recordSize > TargetBlockSize {
+			if newHeight, newVersion, status = encodeProfileAppend(recordFields); status != StatusOK {
+				return newHeight, newVersion, status
+			}
+
+			blockSize = blockHeaderSize
+			recordFields = nil
+		}
+
+		blockSize += recordSize
+		recordFields = append(recordFields, field)
+	}
+
+	return encodeProfileAppend(recordFields)
 }
 
 // ProfileDelete deletes fields and blobs from the blockchain. Status is StatusX.

blockchain/readme.md

@@ -48,7 +48,12 @@ Offset  Size   Info
 
 ## Block Size
 
-The block size is currently recommended to be slightly below 64 KB (minus message header overhead), so that it fits within a single UDP packet. Having a block size smaller than the max. message size reduces complexity when exchanging individual blocks and increases performance for operations such as file search.
+Peers must accept a minimum block size of 1 KB.
+
+The target block size (for generating new blocks) is defined via `TargetBlockSize`. If records cannot fit within that target size, they are added into a new block.
+
+Small block sizes ensure that the block will be transferred via blockchain exchange and cached in DHT.
+Large blocks may be ignored by clients for size and spam reasons, resulting in decreased discoverability.
 
 ## Edge Cases