Lite Packet algorithm for data transfer. It bypasses the CPU expensive Peernet Protocol packet encoding (which uses public key signing).

2026-07-22 04:37:50 +01:00 · 2022-01-17 03:48:40 +01:00
parent 6a5312d834
commit f305e5b158
14 changed files with 403 additions and 64 deletions
--- a/protocol/Message
+++ b/protocol/Message
@@ -11,8 +11,9 @@ Offset  Size    Info
 Control = 0: Request Blocks
 34      8       Limit total count of blocks to transfer. The transfer will be terminated if the limit is reached.
 42      8       Limit of bytes per block to transfer max. Blocks exceeding this limit will not be transferred.
-50      2       Count of block ranges
-52      16 * ?  List of block ranges
+50      16      Transfer ID. This will identify lite packets.
+66      2       Count of block ranges
+68      16 * ?  List of block ranges

 Block range:
 0       8       Block number
@@ -41,6 +42,7 @@ import (
 	"io"

 	"github.com/PeernetOfficial/core/btcec"
+	"github.com/google/uuid"
 )

 const (
@@ -57,6 +59,9 @@ const (
 	GetBlockStatusSizeExceed   = 2
 )

+// Min size of header for Get Block control 0 message.
+const getBlockRequestHeaderSize = 68
+
 // MessageGetBlock is the decoded Get Block message.
 type MessageGetBlock struct {
 	*MessageRaw                          // Underlying raw message.
@@ -64,6 +69,7 @@ type MessageGetBlock struct {
 	BlockchainPublicKey *btcec.PublicKey // Peer ID of blockchain to transfer.

 	// fields valid only for GetBlockControlRequestStart
+	TransferID      uuid.UUID    // Transfer ID to identify lite packets.
 	LimitBlockCount uint64       // Limit total count of blocks to transfer
 	MaxBlockSize    uint64       // Limit of bytes per block to transfer max. Blocks exceeding this limit will not be transferred.
 	TargetBlocks    []BlockRange // Target list of block ranges to transfer.
@@ -96,21 +102,22 @@ func DecodeGetBlock(msg *MessageRaw) (result *MessageGetBlock, err error) {
 	}

 	if result.Control == GetBlockControlRequestStart {
-		if len(msg.Payload) < 52 {
+		if len(msg.Payload) < getBlockRequestHeaderSize {
 			return nil, errors.New("get block: invalid minimum length")
 		}

 		result.LimitBlockCount = binary.LittleEndian.Uint64(msg.Payload[34 : 34+8])
 		result.MaxBlockSize = binary.LittleEndian.Uint64(msg.Payload[42 : 42+8])
+		copy(result.TransferID[:], msg.Payload[50:50+16])

-		countBlockRanges := int(binary.LittleEndian.Uint16(msg.Payload[50:52]))
+		countBlockRanges := int(binary.LittleEndian.Uint16(msg.Payload[66 : 66+2]))
 		if countBlockRanges == 0 {
 			return nil, errors.New("get block: empty block range")
-		} else if len(msg.Payload) < 52+16*countBlockRanges {
+		} else if len(msg.Payload) < getBlockRequestHeaderSize+16*countBlockRanges {
 			return nil, errors.New("get block: cound block ranges exceeds length")
 		}

-		index := 52
+		index := getBlockRequestHeaderSize

 		for n := 0; n < countBlockRanges; n++ {
 			var target BlockRange
@@ -128,18 +135,18 @@ func DecodeGetBlock(msg *MessageRaw) (result *MessageGetBlock, err error) {
 }

 // EncodeGetBlock encodes a Get Block message. The embedded packet size must be smaller than TransferMaxEmbedSize.
-func EncodeGetBlock(senderPrivateKey *btcec.PrivateKey, data []byte, control uint8, blockchainPublicKey *btcec.PublicKey, limitBlockCount, maxBlockSize uint64, targetBlocks []BlockRange) (packetRaw []byte, err error) {
+func EncodeGetBlock(senderPrivateKey *btcec.PrivateKey, data []byte, control uint8, blockchainPublicKey *btcec.PublicKey, limitBlockCount, maxBlockSize uint64, targetBlocks []BlockRange, transferID uuid.UUID) (packetRaw []byte, err error) {
 	if control == GetBlockControlRequestStart && len(data) != 0 {
 		return nil, errors.New("get block encode: payload not allowed in start")
 	} else if isPacketSizeExceed(transferPayloadHeaderSize, len(data)) {
 		return nil, errors.New("get block encode: embedded packet too big")
-	} else if control == GetBlockControlRequestStart && isPacketSizeExceed(52, len(targetBlocks)*16) {
+	} else if control == GetBlockControlRequestStart && isPacketSizeExceed(getBlockRequestHeaderSize, len(targetBlocks)*16) {
 		return nil, errors.New("get block encode: too many target block ranges")
 	}

 	packetSize := transferPayloadHeaderSize
 	if control == GetBlockControlRequestStart {
-		packetSize = 52 + len(targetBlocks)*16
+		packetSize = getBlockRequestHeaderSize + len(targetBlocks)*16
 	} else if control == GetBlockControlActive {
 		packetSize += len(data)
 	}
@@ -153,9 +160,10 @@ func EncodeGetBlock(senderPrivateKey *btcec.PrivateKey, data []byte, control uin
 	if control == GetBlockControlRequestStart {
 		binary.LittleEndian.PutUint64(raw[34:34+8], limitBlockCount)
 		binary.LittleEndian.PutUint64(raw[42:42+8], maxBlockSize)
-		binary.LittleEndian.PutUint16(raw[50:50+2], uint16(len(targetBlocks)))
+		copy(raw[50:50+16], transferID[:])
+		binary.LittleEndian.PutUint16(raw[66:66+2], uint16(len(targetBlocks)))

-		index := 52
+		index := getBlockRequestHeaderSize
 		for _, target := range targetBlocks {
 			binary.LittleEndian.PutUint64(raw[index:index+8], target.Offset)
 			binary.LittleEndian.PutUint64(raw[index+8:index+16], target.Limit)
--- a/protocol/Message
+++ b/protocol/Message
@@ -12,11 +12,10 @@ Offset  Size   Info
 Control = 0: Request Start
 34      8      Offset to start reading in the file
 42      8      Limit of bytes to read at the offset
+50      16     Transfer ID. This will identify lite packets.

-Offset + limit must not exceed the file size.
-
-Control = 3: Active
-34      ?      Embedded protocol data
+Offset + limit must not exceed the file size. Actual data transfer should be sent via lite packets.
+The regular Peernet packets would be too CPU expensive and slow due to public key signing.

 */

@@ -27,18 +26,20 @@ import (
 	"errors"

 	"github.com/PeernetOfficial/core/btcec"
+	"github.com/google/uuid"
 )

 // MessageTransfer is the decoded transfer message.
 // It is sent to initiate a file transfer, and to send data as part of a file transfer. The actual file data is encapsulated via UDT.
 type MessageTransfer struct {
-	*MessageRaw             // Underlying raw message.
-	Control          uint8  // Control. See TransferControlX.
-	TransferProtocol uint8  // Embedded transfer protocol: 0 = UDT
-	Hash             []byte // Hash of the file to transfer.
-	Offset           uint64 // Offset to start reading at. Only TransferControlRequestStart.
-	Limit            uint64 // Limit (count of bytes) to read starting at the offset. Only TransferControlRequestStart.
-	Data             []byte // Embedded protocol data. Only TransferControlActive.
+	*MessageRaw                // Underlying raw message.
+	Control          uint8     // Control. See TransferControlX.
+	TransferProtocol uint8     // Embedded transfer protocol: 0 = UDT
+	Hash             []byte    // Hash of the file to transfer.
+	Offset           uint64    // Offset to start reading at. Only TransferControlRequestStart.
+	Limit            uint64    // Limit (count of bytes) to read starting at the offset. Only TransferControlRequestStart.
+	TransferID       uuid.UUID // Transfer ID to identify lite packets.
+	Data             []byte    // Embedded protocol data. Only TransferControlActive.
 }

 const (
@@ -48,7 +49,9 @@ const (
 	TransferControlTerminate    = 3 // Terminate
 )

-const TransferProtocolUDT = 0 // Indicates that UDT is used as embedded transfer protocol.
+const (
+	TransferProtocolUDT = 0 // UDT via lite packets. No encryption.
+)

 const transferPayloadHeaderSize = 34

@@ -76,9 +79,11 @@ func DecodeTransfer(msg *MessageRaw) (result *MessageTransfer, err error) {

 		result.Offset = binary.LittleEndian.Uint64(msg.Payload[34 : 34+8])
 		result.Limit = binary.LittleEndian.Uint64(msg.Payload[42 : 42+8])
+		copy(result.TransferID[:], msg.Payload[50:50+16])

 	case TransferControlActive:
-		result.Data = msg.Payload[34:]
+		// Data should be transferred via lite packets for performance reasons, but it is allowed to be encapsulated in Peernet packets.
+		result.Data = msg.Payload[transferPayloadHeaderSize:]

 	}

@@ -90,8 +95,11 @@ func DecodeTransfer(msg *MessageRaw) (result *MessageTransfer, err error) {
 // The caller may send bigger payloads but may risk that data packets are simply dropped and never arrive. A MTU negotiation or detection could pimp that.
 const TransferMaxEmbedSize = internetSafeMTU - PacketLengthMin - transferPayloadHeaderSize

+// Same as TransferMaxEmbedSize but for encoding via lite packets.
+const TransferMaxEmbedSizeLite = internetSafeMTU - PacketLiteSizeMin
+
 // EncodeTransfer encodes a transfer message. The embedded packet size must be smaller than TransferMaxEmbedSize.
-func EncodeTransfer(senderPrivateKey *btcec.PrivateKey, data []byte, control, transferProtocol uint8, hash []byte, offset, limit uint64) (packetRaw []byte, err error) {
+func EncodeTransfer(senderPrivateKey *btcec.PrivateKey, data []byte, control, transferProtocol uint8, hash []byte, offset, limit uint64, transferID uuid.UUID) (packetRaw []byte, err error) {
 	if control == TransferControlRequestStart && len(data) != 0 {
 		return nil, errors.New("transfer encode: payload not allowed in start")
 	} else if isPacketSizeExceed(transferPayloadHeaderSize, len(data)) {
@@ -100,7 +108,7 @@ func EncodeTransfer(senderPrivateKey *btcec.PrivateKey, data []byte, control, tr

 	packetSize := transferPayloadHeaderSize
 	if control == TransferControlRequestStart {
-		packetSize += 16
+		packetSize += 32
 	} else if control == TransferControlActive {
 		packetSize += len(data)
 	}
@@ -114,6 +122,7 @@ func EncodeTransfer(senderPrivateKey *btcec.PrivateKey, data []byte, control, tr
 	if control == TransferControlRequestStart {
 		binary.LittleEndian.PutUint64(raw[34:34+8], offset)
 		binary.LittleEndian.PutUint64(raw[42:42+8], limit)
+		copy(raw[50:50+16], transferID[:])
 	} else if control == TransferControlActive {
 		copy(raw[34:34+len(data)], data)
 	}
--- a/protocol/Packet
+++ b/protocol/Packet
@@ -0,0 +1,188 @@
+/*
+File Name:  Packet Lite.go
+Copyright:  2021 Peernet s.r.o.
+Author:     Peter Kleissner
+
+The lite packet header is used for encoding data transfer packets. The regular header is too expensive in terms of CPU consumption due to public key signing.
+Instead, a simple session ID will identify lite packets. The ID is randomized and only valid during the session.
+Unsolicited lite packets are therefore impossible; the receiver must have the ID already whitelisted for the packet to be recognized.
+
+Offset  Size   Info
+0       16     ID
+16      2      Size of data to follow
+
+*/
+
+package protocol
+
+import (
+	"encoding/binary"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+// PacketLiteRaw is a decrypted P2P lite packet
+type PacketLiteRaw struct {
+	ID      uuid.UUID // ID
+	Payload []byte    // Payload
+	Session *LiteID   // Session info
+}
+
+// Minimum packet size of lite packets.
+const PacketLiteSizeMin = 16 + 2
+
+// IsPacketLite identifies a lite packet based on its ID. If the ID is not recognized, it fails.
+func (router *LiteRouter) IsPacketLite(raw []byte) (isLite bool, err error) {
+	if len(raw) < PacketLiteSizeMin {
+		return false, errors.New("invalid packet size")
+	}
+
+	// Parse the ID and then look it up.
+	var id uuid.UUID
+	copy(id[:], raw[0:16])
+
+	return router.LookupLiteID(id) != nil, nil
+}
+
+// PacketLiteDecode a lite packet. It will identify the lite packet based on its ID. If the ID is not recognized (which is the case for regular Peernet packets), the function fails.
+// It does not perform any decryption.
+func (router *LiteRouter) PacketLiteDecode(raw []byte) (packet *PacketLiteRaw, err error) {
+	if len(raw) < PacketLiteSizeMin {
+		return nil, errors.New("invalid packet size")
+	}
+
+	// Parse the ID and look it up. It will contain information about the decryption algorithm to use.
+	var id uuid.UUID
+	copy(id[:], raw[0:16])
+
+	session := router.LookupLiteID(id)
+	if session == nil {
+		return nil, errors.New("packet ID not found")
+	}
+
+	// TODO: Decrypt the data if indicated by the session.
+
+	sizePayload := binary.LittleEndian.Uint16(raw[16 : 16+2])
+	if int(sizePayload) > len(raw)-PacketLiteSizeMin { // invalid size field?
+		return nil, errors.New("invalid packet size field")
+	}
+
+	// Valid packet received, extend expiration.
+	session.expires = time.Now().Add(session.timeout)
+
+	return &PacketLiteRaw{Payload: raw[PacketLiteSizeMin:], ID: id, Session: session}, nil
+}
+
+// Encodes a lite packet.
+func PacketLiteEncode(id uuid.UUID, data []byte) (raw []byte, err error) {
+	raw = make([]byte, PacketLiteSizeMin+len(data))
+
+	copy(raw[0:16], id[:])
+	binary.LittleEndian.PutUint16(raw[16:16+2], uint16(len(data)))
+	copy(raw[PacketLiteSizeMin:], data)
+
+	return raw, nil
+}
+
+// ---- Lite packet ID management. This is similar to packet sequences. ----
+
+// LiteRouter keeps track of accepted (expected) packet IDs.
+type LiteRouter struct {
+	// list of recognized IDs
+	ids map[uuid.UUID]*LiteID
+
+	sync.Mutex // synchronized access to the IDs
+}
+
+// LiteID contains session information for a bidirectional transfer of data
+type LiteID struct {
+	ID             uuid.UUID     // ID
+	created        time.Time     // When the ID was created.
+	expires        time.Time     // When the ID expires. This can be extended on the fly!
+	Data           interface{}   // Optional high-level data associated with the ID
+	timeout        time.Duration // Timeout for receiving the next message
+	invalidateFunc func()        // Called on expiration.
+}
+
+// Creates a new manager to keep track of accepted IDs.
+func NewLiteRouter() (router *LiteRouter) {
+	router = &LiteRouter{
+		ids: make(map[uuid.UUID]*LiteID),
+	}
+
+	go router.autoDeleteExpired()
+
+	return
+}
+
+// autoDeleteExpired deletes all IDs that are expired.
+func (router *LiteRouter) autoDeleteExpired() {
+	for {
+		time.Sleep(4 * time.Second)
+		now := time.Now()
+
+		router.Lock()
+		for id, info := range router.ids {
+			if info.expires.Before(now) {
+				delete(router.ids, id)
+
+				if info.invalidateFunc != nil {
+					go info.invalidateFunc()
+				}
+			}
+		}
+		router.Unlock()
+	}
+}
+
+func (router *LiteRouter) LookupLiteID(id uuid.UUID) (info *LiteID) {
+	router.Lock()
+	info = router.ids[id]
+	router.Unlock()
+
+	return info
+}
+
+// Returns a new lite ID to be used.
+func (router *LiteRouter) NewLiteID(data interface{}, timeout time.Duration, invalidateFunc func()) (info *LiteID) {
+	info = &LiteID{
+		created:        time.Now(),
+		expires:        time.Now().Add(timeout),
+		timeout:        timeout,
+		invalidateFunc: invalidateFunc,
+		Data:           data,
+		ID:             uuid.New(),
+	}
+
+	router.Lock()
+	router.ids[info.ID] = info
+	router.Unlock()
+
+	return
+}
+
+func (router *LiteRouter) RegisterLiteID(id uuid.UUID, data interface{}, timeout time.Duration, invalidateFunc func()) (info *LiteID) {
+	info = &LiteID{
+		ID:             id,
+		created:        time.Now(),
+		expires:        time.Now().Add(timeout),
+		timeout:        timeout,
+		invalidateFunc: invalidateFunc,
+		Data:           data,
+	}
+
+	router.Lock()
+	existingInfo := router.ids[info.ID]
+	router.ids[info.ID] = info
+	router.Unlock()
+
+	// Call the invalidate function if there is a collision. This should never happen.
+	if existingInfo != nil && existingInfo.invalidateFunc != nil {
+		go existingInfo.invalidateFunc()
+	}
+
+	return
+}