From 594c06e27655393eb66e98249138ba3ea7351987 Mon Sep 17 00:00:00 2001 From: Kleissner Date: Sun, 30 May 2021 15:28:11 +0200 Subject: [PATCH] Implemented block encoding according to Whitepaper 0.8.2. Blocks aren't yet shared. --- Block Encoding.go | 336 ++++++++++++++++++++++++++++++++++++++++++++++ File Formats.go | 44 ++++++ Test_test.go | 119 ++++++++++++++++ 3 files changed, 499 insertions(+) create mode 100644 Block Encoding.go create mode 100644 File Formats.go create mode 100644 Test_test.go diff --git a/Block Encoding.go b/Block Encoding.go new file mode 100644 index 0000000..1aff952 --- /dev/null +++ b/Block Encoding.go @@ -0,0 +1,336 @@ +/* +File Name: Block Encoding.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Block encoding in messages. +*/ + +package core + +import ( + "bytes" + "encoding/binary" + "errors" + "io" + "path" + "strings" + "unicode/utf8" + + "github.com/btcsuite/btcd/btcec" +) + +// Block is a single block containing a set of records (metadata). +// It has no upper size limit, although a soft limit of 64 KB - overhead is encouraged for efficiency. +type Block struct { + OwnerPublicKey *btcec.PublicKey // Owner Public Key, ECDSA (secp256k1) 257-bit + LastBlockHash []byte // Hash of the last block. Blake3. + BlockchainVersion uint64 // Blockchain version + Number uint32 // Block number + RecordsRaw []BlockRecordRaw // Block records raw + Files []BlockRecordFile // Files + User BlockRecordUser // User details + directories []BlockRecordDirectory // Internal list of directories for decoding files. +} + +// BlockRecordRaw is a single block record (not decoded) +type BlockRecordRaw struct { + Type uint8 // Record Type. See RecordTypeX. + Data []byte // Data according to the type +} + +const ( + RecordTypeUsername = 0 // Username. Arbitrary name defined by the user. + RecordTypeDirectory = 1 // Directory. Only valid in the context of the current block. + RecordTypeFile = 2 // File + RecordTypeContentRating = 3 // Content rating (positive). + RecordTypeContentReport = 4 // Content report (negative). + RecordTypeDelete = 5 // Delete previous record. +) + +// block record structures + +// BlockRecordUser specifies user information +type BlockRecordUser struct { + Valid bool // Whether the username is supplied + Name string // Arbitrary name of the user. + NameS string // Sanitized version of the name. +} + +// BlockRecordDirectory is a directory, only valid within the same block. +type BlockRecordDirectory struct { + ID uint16 // ID, only valid within the same block + Name string // Name of the directory. Slashes (both backward and forward) mark subdirectories. +} + +// BlockRecordFile is the metadata of a file published on the blockchain +type BlockRecordFile struct { + Hash []byte // Hash of the file data + Type uint8 // Type (low-level) + Format uint16 // Format (high-level) + Size uint64 // Size of the file + Directory string // Directory + Name string // File name + directoryID uint16 // Internal directory ID + // Tags todo +} + +// Tag structure to be defined + +const blockHeaderSize = 115 +const blockRecordHeaderSize = 5 + +// decodeBlock decodes a single block +func decodeBlock(raw []byte) (block *Block, err error) { + if len(raw) < blockHeaderSize { + return nil, errors.New("decodeBlock invalid block size") + } + + block = &Block{} + + signature := raw[0 : 0+65] + + block.OwnerPublicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, hashData(raw[65:])) + if err != nil { + return nil, err + } + + block.LastBlockHash = make([]byte, hashSize) + copy(block.LastBlockHash, raw[65:65+hashSize]) + + block.BlockchainVersion = binary.LittleEndian.Uint64(raw[97 : 97+8]) + block.Number = binary.LittleEndian.Uint32(raw[105 : 105+4]) + + blockSize := binary.LittleEndian.Uint32(raw[109 : 109+4]) + if blockSize != uint32(len(raw)) { + return nil, errors.New("decodeBlock invalid block size") + } + + countRecords := binary.LittleEndian.Uint16(raw[113 : 113+2]) + index := 115 + + for n := uint16(0); n < countRecords; n++ { + if index+blockRecordHeaderSize > len(raw) { + return nil, errors.New("decodeBlock block record exceeds block size") + } + + recordType := raw[index] + recordSize := binary.LittleEndian.Uint32(raw[index+1 : index+5]) + index += blockRecordHeaderSize + + if index+int(recordSize) > len(raw) { + return nil, errors.New("decodeBlock block record exceeds block size") + } + + block.RecordsRaw = append(block.RecordsRaw, BlockRecordRaw{Type: recordType, Data: raw[index : index+int(recordSize)]}) + + if err := decodeBlockRecord(raw[index:index+int(recordSize)], block, recordType); err != nil { + return nil, err + } + + index += int(recordSize) + } + + return block, nil +} + +// decodeBlockRecord decodes a single block and fills it into the provided block structure +func decodeBlockRecord(data []byte, block *Block, recordType uint8) (err error) { + switch recordType { + case RecordTypeUsername: + block.User.Name = string(data) + block.User.NameS = sanitizeUsername(block.User.Name) + block.User.Valid = true + + case RecordTypeDirectory: + if len(data) < 3 { + return errors.New("decodeBlockRecord directory record invalid size") + } + + directory := &BlockRecordDirectory{} + directory.ID = binary.LittleEndian.Uint16(data[0 : 0+2]) + directory.Name = string(data[2:]) + block.directories = append(block.directories, *directory) + + case RecordTypeFile: + if len(data) < 49 { + return errors.New("decodeBlockRecord file record invalid size") + } + + file := BlockRecordFile{} + file.Hash = make([]byte, hashSize) + copy(file.Hash, data[0:0+hashSize]) + file.Type = data[32] + file.Format = binary.LittleEndian.Uint16(data[33 : 33+2]) + file.Size = binary.LittleEndian.Uint64(data[35 : 35+8]) + directoryID := binary.LittleEndian.Uint16(data[43 : 43+2]) + filenameSize := binary.LittleEndian.Uint16(data[45 : 45+2]) + //countTags := binary.LittleEndian.Uint16(data[47 : 47+2]) // future implementation of tags + + if len(data) < 49+int(filenameSize) { + return errors.New("decodeBlockRecord file record invalid filename size") + } + file.Name = string(data[49 : 49+filenameSize]) + + for n := range block.directories { + if block.directories[n].ID == directoryID { + file.Directory = block.directories[n].Name + break + } + } + + block.Files = append(block.Files, file) + } + + return nil +} + +// sanitizeUsername returns the sanitized version of the username. +func sanitizeUsername(input string) string { + if !utf8.ValidString(input) { + return "" + } + + input = strings.TrimSpace(input) + input = strings.ReplaceAll(input, "\n", " ") + input = strings.ReplaceAll(input, "\r", "") + + // Max length for sanitized version is 36, resembling the limit from StackOverflow. + if len(input) > 36 { + input = input[:36] + } + + return input +} + +func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, err error) { + var buffer bytes.Buffer + buffer.Write(make([]byte, 65)) // Signature, filled at the end + + if block.Number > 0 && len(block.LastBlockHash) != hashSize { + return nil, errors.New("encodeBlock invalid last block hash") + } else if block.Number == 0 { // Block 0: Empty last hash + block.LastBlockHash = make([]byte, 32) + } + buffer.Write(block.LastBlockHash) + + var temp [8]byte + binary.LittleEndian.PutUint64(temp[0:8], block.BlockchainVersion) + buffer.Write(temp[:]) + + binary.LittleEndian.PutUint32(temp[0:4], block.Number) + buffer.Write(temp[:4]) + + buffer.Write(make([]byte, 4)) // Size of block, filled later + buffer.Write(make([]byte, 2)) // Count of records, filled later + + // write all records + countRecords, err := encodeBlockRecords(&buffer, block) + if err != nil { + return nil, err + } + + // finalize the block + raw = buffer.Bytes() + if len(raw) < blockHeaderSize { + return nil, errors.New("encodeBlock invalid block size") + } + + binary.LittleEndian.PutUint32(raw[109:109+4], uint32(len(raw))) // Size of block + binary.LittleEndian.PutUint16(raw[113:113+2], countRecords) // Count of records + + // signature is last + signature, err := btcec.SignCompact(btcec.S256(), ownerPrivateKey, hashData(raw[65:]), true) + if err != nil { + return nil, err + } + copy(raw[0:0+65], signature) + + return raw, nil +} + +// encodeBlockRecords writes all records of the block into the writer +func encodeBlockRecords(writer io.Writer, block *Block) (count uint16, err error) { + nextDirectoryID := uint16(1) // start as 1 to prevent collision with files without explicit directory + + writeBlockRecord := func(recordType uint8, data []byte) { + var temp [8]byte + binary.LittleEndian.PutUint32(temp[0:4], uint32(len(data))) + + writer.Write([]byte{recordType}) // Record Type + writer.Write(temp[:4]) // Size of data + writer.Write(data) // Data + + count++ + } + + // Username + if block.User.Valid { + writeBlockRecord(RecordTypeUsername, []byte(block.User.Name)) + } + + // First the directory records must be declared for any references by files +directoryCreateLoop: + for n := range block.Files { + block.Files[n].sanitizePath() + if block.Files[n].Directory == "" { + continue + } + + // already known directory ID? + for m := range block.directories { + if block.directories[m].Name == block.Files[n].Directory { + block.Files[n].directoryID = block.directories[m].ID + continue directoryCreateLoop + } + } + + // Create the new directory record + var directoryID [2]byte + binary.LittleEndian.PutUint16(directoryID[0:2], nextDirectoryID) + block.Files[n].directoryID = nextDirectoryID + nextDirectoryID++ + + writeBlockRecord(RecordTypeDirectory, append(directoryID[:], []byte(block.Files[n].Directory)...)) + } + + for n := range block.Files { + var data [49]byte + + if len(block.Files[n].Hash) != hashSize { + return 0, errors.New("encodeBlockRecords invalid file hash") + } + copy(data[0:32], block.Files[n].Hash[0:32]) + + data[32] = block.Files[n].Type + binary.LittleEndian.PutUint16(data[33:33+2], block.Files[n].Format) + binary.LittleEndian.PutUint64(data[35:35+8], block.Files[n].Size) + binary.LittleEndian.PutUint16(data[43:43+2], block.Files[n].directoryID) + + filenameB := []byte(block.Files[n].Name) + binary.LittleEndian.PutUint16(data[45:45+2], uint16(len(filenameB))) + binary.LittleEndian.PutUint16(data[47:47+2], uint16(0)) // Count of Tags (future use) + + writeBlockRecord(RecordTypeFile, append(data[:], filenameB...)) + } + + return count, nil +} + +const PATH_MAX_LENGTH = 32767 // Windows Maximum Path Length for UNC paths + +func (record *BlockRecordFile) sanitizePath() { + // Enforced forward slashes as directory separator and clean the path. + record.Directory = strings.ReplaceAll(record.Directory, "\\", "/") + record.Directory = path.Clean(record.Directory) + + // No slash at the beginning and end to save space. + record.Directory = strings.Trim(record.Directory, "/") + + // Slashes in filenames are not encouraged, but not removed. + + // Enforce max filename length. + if len(record.Name) > PATH_MAX_LENGTH { + record.Name = record.Name[:PATH_MAX_LENGTH] + } +} diff --git a/File Formats.go b/File Formats.go new file mode 100644 index 0000000..9837afa --- /dev/null +++ b/File Formats.go @@ -0,0 +1,44 @@ +/* +File Name: File Formats.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Definition of all recognized file formats. This file is likely being updated more frequently than regular code. +*/ + +package core + +// Low-level content types of file data. This is a copy from Intelligence X. +const ( + TypeBinary = iota // Binary/unspecified + TypeText // Plain text + TypePicture // Picture of any format + TypeVideo // Video + TypeAudio // Audio + TypeDocument // Any document file, including office documents, PDFs, power point, spreadsheets + TypeExecutable // Any executable file, OS independent + TypeContainer // Container files like ZIP, RAR, TAR + TypeCompressed // Compressed files like GZ, BZ +) + +// High-level file types. New ones may be added as required. +const ( + FormatUnspecified = iota // Unspecified + FormatPDF // PDF document + FormatWord // Word document + FormatExcel // Excel + FormatPowerpoint // Powerpoint + FormatPicture // Pictures (including GIF, excluding icons) + FormatAudio // Audio files + FormatVideo // Video files + FormatContainer // Compressed files including ZIP, RAR, TAR and others + FormatHTML // HTML file + FormatText // Text file + FormatEbook // Ebook file + FormatCompressed // Compressed file. + FormatDatabase // Database file + FormatEmail // Single email + FormatCSV // CSV file +) + +// Future tags to be defined for audio/video: Artist, Album, Title, Length, Bitrate, Codec diff --git a/Test_test.go b/Test_test.go new file mode 100644 index 0000000..bfa2ee3 --- /dev/null +++ b/Test_test.go @@ -0,0 +1,119 @@ +// Functions to manually debug encoding/decoding. No actual automated unit tests. +package core + +import ( + "encoding/hex" + "fmt" + "testing" +) + +func TestMessageEncodingAnnouncement(t *testing.T) { + _, publicKey, err := Secp256k1NewPrivateKey() + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + // encode and decode announcement + packetR := PacketRaw{Protocol: 0, Command: CommandAnnouncement, Sequence: 123} + + var findPeer []KeyHash + var findValue []KeyHash + var files []InfoStore + + hash1 := hashData([]byte("test")) + hash2 := hashData([]byte("test3")) + findPeer = append(findPeer, KeyHash{Hash: hash1}) + findValue = append(findValue, KeyHash{Hash: hash2}) + + packets := msgEncodeAnnouncement(true, true, findPeer, findValue, files) + + msg := &MessageRaw{PacketRaw: packetR, SenderPublicKey: publicKey} + msg.Payload = packets[0].raw + + result, err := msgDecodeAnnouncement(msg) + if err != nil { + fmt.Printf("Error msgDecodeAnnouncement: %s\n", err.Error()) + return + } + + fmt.Printf("Decode:\nUser Agent: %s\nFind Peer: %v\nFind Data: %v\n", result.UserAgent, result.FindPeerKeys, result.FindDataKeys) +} + +func TestMessageEncodingResponse(t *testing.T) { + _, publicKey, err := Secp256k1NewPrivateKey() + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + // encode and decode response + packetR := PacketRaw{Protocol: 0, Command: CommandResponse} + + var hash2Peers []Hash2Peer + var filesEmbed []EmbeddedFileData + var hashesNotFound [][]byte + + file1Data := []byte("test") + file2Data := []byte("test3") + file1 := EmbeddedFileData{ID: KeyHash{hashData(file1Data)}, Data: file1Data} + file2 := EmbeddedFileData{ID: KeyHash{hashData(file2Data)}, Data: file2Data} + filesEmbed = append(filesEmbed, file1) + filesEmbed = append(filesEmbed, file2) + + hashesNotFound = append(hashesNotFound, hashData([]byte("NA"))) + + packetsRaw, err := msgEncodeResponse(true, hash2Peers, filesEmbed, hashesNotFound) + if err != nil { + fmt.Printf("Error msgEncodeAnnouncement: %s\n", err.Error()) + return + } + + msg := &MessageRaw{PacketRaw: packetR, SenderPublicKey: publicKey} + msg.Payload = packetsRaw[0] + + result, err := msgDecodeResponse(msg) + if err != nil { + fmt.Printf("Error msgDecodeAnnouncement: %s\n", err.Error()) + return + } + + fmt.Printf("Decode:\nUser Agent: %s\nHash2Peers: %v\nHashesNotFound: %v\nFiles embedded: %v\n", result.UserAgent, result.Hash2Peers, result.HashesNotFound, result.FilesEmbed) +} + +func TestBlockEncoding(t *testing.T) { + privateKey, _, err := Secp256k1NewPrivateKey() + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, Name: "Filename 1.txt", Directory: "documents\\sub folder"} + + block := &Block{BlockchainVersion: 42, Number: 0, User: BlockRecordUser{Valid: true, Name: "Test User 1"}, Files: []BlockRecordFile{file1}} + + raw, err := encodeBlock(block, privateKey) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + block, err = decodeBlock(raw) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + // output the block details + fmt.Printf("Block details:\n----------------\nNumber: %d\nVersion: %d\nLast Hash: %s\nPublic Key: %s\n", block.Number, block.BlockchainVersion, hex.EncodeToString(block.LastBlockHash), hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())) + + for _, file := range block.Files { + fmt.Printf("* File %s\n", file.Name) + fmt.Printf(" Directory %s\n", file.Directory) + fmt.Printf(" Size %d\n", file.Size) + fmt.Printf(" Type %d\n", file.Type) + fmt.Printf(" Format %d\n", file.Format) + fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) + fmt.Printf(" Directory ID %d\n\n", file.directoryID) + } +}