From a2384ca627872bc27b4bfb45a8aca4c0b1e54c07 Mon Sep 17 00:00:00 2001 From: Kleissner Date: Fri, 27 Aug 2021 03:12:30 +0200 Subject: [PATCH] Blockchain: Change of block format. File records: * Add unique ID to file record. * Introducing tags which are flexible. Close #31 * Name and Directory fields are now tags. * Embedded basic compression by storing duplicate file tags as single record. Added date field to record structure. --- Block Encoding.go | 46 +++-- Block Records.go | 391 ++++++++++++++++++++++++++++++------------- Blockchain.go | 33 +--- Test_test.go | 75 ++++++--- go.mod | 1 + go.sum | 2 + sanitize/Sanitize.go | 16 +- 7 files changed, 387 insertions(+), 177 deletions(-) diff --git a/Block Encoding.go b/Block Encoding.go index 3119bfc..ca695a3 100644 --- a/Block Encoding.go +++ b/Block Encoding.go @@ -3,7 +3,22 @@ File Name: Block Encoding.go Copyright: 2021 Peernet s.r.o. Author: Peter Kleissner -Block encoding in messages and local storage. +Encoding of a block (it is the same stored in the database and shared in a message): +Offset Size Info +0 65 Signature of entire block +65 32 Hash (blake3) of last block. 0 for first one. +97 8 Blockchain version number +105 4 Block number +109 4 Size of entire block including this header +113 2 Count of records that follow + +Each record inside the block has this basic structure: +Offset Size Info +0 1 Record type +1 8 Date created. This remains the same in case of block refactoring. +9 4 Size of data +13 ? Data (encoding depends on record type) + */ package core @@ -12,6 +27,7 @@ import ( "bytes" "encoding/binary" "errors" + "time" "github.com/btcsuite/btcd/btcec" ) @@ -28,12 +44,13 @@ type Block struct { // BlockRecordRaw is a single block record (not decoded) type BlockRecordRaw struct { - Type uint8 // Record Type. See RecordTypeX. - Data []byte // Data according to the type + Type uint8 // Record Type. See RecordTypeX. + Date time.Time // Date created. This remains the same in case of block refactoring. + Data []byte // Data according to the type } const blockHeaderSize = 115 -const blockRecordHeaderSize = 5 +const blockRecordHeaderSize = 13 // decodeBlock decodes a single block func decodeBlock(raw []byte) (block *Block, err error) { @@ -67,18 +84,19 @@ func decodeBlock(raw []byte) (block *Block, err error) { for n := uint16(0); n < countRecords; n++ { if index+blockRecordHeaderSize > len(raw) { - return nil, errors.New("decodeBlock block record exceeds block size") + return nil, errors.New("decodeBlock record exceeds block size") } recordType := raw[index] - recordSize := binary.LittleEndian.Uint32(raw[index+1 : index+5]) + recordDate := int64(binary.LittleEndian.Uint64(raw[index+1 : index+9])) // Unix time int64, the number of seconds elapsed since January 1, 1970 UTC + recordSize := binary.LittleEndian.Uint32(raw[index+9 : index+9+4]) index += blockRecordHeaderSize if index+int(recordSize) > len(raw) { - return nil, errors.New("decodeBlock block record exceeds block size") + return nil, errors.New("decodeBlock record exceeds block size") } - block.RecordsRaw = append(block.RecordsRaw, BlockRecordRaw{Type: recordType, Data: raw[index : index+int(recordSize)]}) + block.RecordsRaw = append(block.RecordsRaw, BlockRecordRaw{Type: recordType, Data: raw[index : index+int(recordSize)], Date: time.Unix(recordDate, 0)}) index += int(recordSize) } @@ -111,11 +129,17 @@ func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, e countRecords := uint16(0) for _, record := range block.RecordsRaw { - var temp [8]byte - binary.LittleEndian.PutUint32(temp[0:4], uint32(len(record.Data))) + if record.Date == (time.Time{}) { // Always set date if not already set + record.Date = time.Now() + } + + var tempSize, tempDate [8]byte + binary.LittleEndian.PutUint32(tempSize[0:4], uint32(len(record.Data))) + binary.LittleEndian.PutUint64(tempDate[0:8], uint64(record.Date.UTC().Unix())) buffer.Write([]byte{record.Type}) // Record Type - buffer.Write(temp[:4]) // Size of data + buffer.Write(tempDate[:8]) // Date created + buffer.Write(tempSize[:4]) // Size of data buffer.Write(record.Data) // Data countRecords++ diff --git a/Block Records.go b/Block Records.go index 34598d1..3146651 100644 --- a/Block Records.go +++ b/Block Records.go @@ -3,24 +3,26 @@ File Name: Block Encoding.go Copyright: 2021 Peernet s.r.o. Author: Peter Kleissner -Encoding of records inside blocks. +This files defines the encoding of blocks and records within. File records: Offset Size Info 0 32 Hash blake3 of the file content -32 1 File type (low-level) -33 2 File format (high-level) -35 8 File size -43 2 Directory ID -45 2 Size of file name in bytes -47 2 Count of tags -49 ? File name -? ? Tags (TBD) +32 16 File ID +48 1 File Type (low-level) +49 2 File Format (high-level) +51 8 File Size +59 2 Count of Tags +61 ? Tags -Directory records: +Each file tag provides additional optional information: Offset Size Info -0 2 Directory ID -2 ? Directory name +0 2 Type +2 4 Size of data that follows +6 ? Data according to the tag type + +Tag data record contains only raw data and may be referenced by Tags in File records. +This is a basic embedded way of compression when tags are repetitive in multiple files within the same block. */ @@ -29,76 +31,90 @@ package core import ( "encoding/binary" "errors" + "math" + "time" "github.com/PeernetOfficial/core/sanitize" + "github.com/google/uuid" ) -// BlockDecoded contains the decoded records from a block -type BlockDecoded struct { - Block - Files []BlockRecordFile // Files - User BlockRecordUser // User details - directories []BlockRecordDirectory // Internal list of directories for decoding files. -} +// ---- Block record structures (decoded) ---- // RecordTypeX defines the type of the record const ( RecordTypeUsername = 0 // Username. Arbitrary name defined by the user. - RecordTypeDirectory = 1 // Directory. Only valid in the context of the current block. + RecordTypeTagData = 1 // Tag data record to be referenced by one or multiple tags. Only valid in the context of the current block. RecordTypeFile = 2 // File - RecordTypeContentRating = 3 // Content rating (positive). - RecordTypeContentReport = 4 // Content report (negative). - RecordTypeDelete = 5 // Delete previous record. + RecordTypeDelete = 3 // Delete previous record by ID. + RecordTypeCertificate = 4 // Certificate to certify provided information in the blockchain issued by a trusted 3rd party. + RecordTypeContentRating = 5 // Content rating (positive). + RecordTypeContentReport = 6 // Content report (negative). ) -// block record structures - // BlockRecordUser specifies user information type BlockRecordUser struct { - Valid bool // Whether the username is supplied - Name string // Arbitrary name of the user. - NameS string // Sanitized version of the name. -} - -// BlockRecordDirectory is a directory, only valid within the same block. -type BlockRecordDirectory struct { - ID uint16 // ID, only valid within the same block - Name string // Name of the directory. Slashes (both backward and forward) mark subdirectories. + Name string // Arbitrary name of the user (original input). + NameSanitized string // Sanitized version of the name. } // BlockRecordFile is the metadata of a file published on the blockchain type BlockRecordFile struct { - Hash []byte // Hash of the file data - Type uint8 // Type (low-level) - Format uint16 // Format (high-level) - Size uint64 // Size of the file - Directory string // Directory - Name string // File name - directoryID uint16 // Internal directory ID - // Tags todo + Hash []byte // Hash of the file data + ID uuid.UUID // ID + Type uint8 // Type (low-level) + Format uint16 // Format (high-level) + Size uint64 // Size of the file data + TagsRaw []BlockRecordFileTag // Tags to provide additional metadata + TagsDecoded []interface{} // Decoded tags. See FileTagX structures. } -// Tag structure to be defined +// ---- Tag structures ---- -// decodeBlockRecords decodes the raw records in the block and returns a high-level decoded structure -func decodeBlockRecords(block *Block) (decoded *BlockDecoded, err error) { - decoded = &BlockDecoded{Block: *block} +// BlockRecordFileTag provides additional metadata about the file. +// New tags can be defines in the future without breaking support. +type BlockRecordFileTag struct { + Type uint16 // Type (low-level). See TagTypeX constants. + Data []byte // Actual data of the tag, decoded into FileTagX structures. - if decoded.Files, decoded.directories, err = decodeBlockRecordFiles(block.RecordsRaw); err != nil { - return nil, err - } - if user := decodeBlockRecordUser(block.RecordsRaw); user != nil { - decoded.User = *user - } - - return decoded, nil + // If top bit of Type is set, then Data must be 2, 4, or 8 bytes representing the distance number (positive or negative) of raw record in the block that will be used as data. + // This is an embedded basic compression algorithm for repetitive tag. For example directory tags or album tags might be heavily repetitive among files. } +// TagTypeName defines the type of the tag +const ( + TagTypeName = 0 // File name + TagTypeDirectory = 1 // Directory + TagTypeDateCreated = 2 // Date when the file was originally created. This may differ from the date in the block record, which indicates when the file was shared. + TagTypeDescription = 3 // Arbitrary description of the file. May contain hashtags. +) + +// FileTagDirectory specifies in which directory the file is stored +type FileTagDirectory struct { + Directory string // Directory the file is stored at +} + +// FileTagName specifies in which directory the file is stored +type FileTagName struct { + Name string // Name of the file +} + +// FileTagDescription is an arbitrary of the description of the file. It may contain hashtags to help tagging and searching. +type FileTagDescription struct { + Description string +} + +// FileTagCreated is the date when the file was originally created +type FileTagCreated struct { + Date time.Time // Created time +} + +// ---- low-level encoding ---- + // decodeBlockRecordUser decodes the username, if available. Otherwise return nil. func decodeBlockRecordUser(recordsRaw []BlockRecordRaw) (user *BlockRecordUser) { for _, record := range recordsRaw { if record.Type == RecordTypeUsername { - user = &BlockRecordUser{Valid: true, Name: string(record.Data), NameS: sanitize.Username(string(record.Data))} + user = &BlockRecordUser{Name: string(record.Data), NameSanitized: sanitize.Username(string(record.Data))} // continue to seek for an overriding username record, even though that would be stupid within the same block } } @@ -107,108 +123,253 @@ func decodeBlockRecordUser(recordsRaw []BlockRecordRaw) (user *BlockRecordUser) // encodeBlockRecordUser encodes the username func encodeBlockRecordUser(user BlockRecordUser) (recordsRaw []BlockRecordRaw, err error) { - if user.Valid { - recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeUsername, Data: []byte(user.Name)}) - } + recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeUsername, Data: []byte(user.Name)}) return recordsRaw, nil } // decodeBlockRecordFiles decodes only file records. Other records are ignored -func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw) (files []BlockRecordFile, directories []BlockRecordDirectory, err error) { - for _, record := range recordsRaw { +func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw) (files []BlockRecordFile, err error) { + for i, record := range recordsRaw { switch record.Type { - case RecordTypeDirectory: - if len(record.Data) < 3 { - return nil, nil, errors.New("decodeBlockRecordFiles directory record invalid size") - } - - directory := &BlockRecordDirectory{} - directory.ID = binary.LittleEndian.Uint16(record.Data[0 : 0+2]) - directory.Name = string(record.Data[2:]) - directories = append(directories, *directory) - case RecordTypeFile: - if len(record.Data) < 49 { - return nil, nil, errors.New("decodeBlockRecordFiles file record invalid size") + if len(record.Data) < 61 { + return nil, errors.New("decodeBlockRecordFiles file record invalid size") } file := BlockRecordFile{} file.Hash = make([]byte, hashSize) copy(file.Hash, record.Data[0:0+hashSize]) - file.Type = record.Data[32] - file.Format = binary.LittleEndian.Uint16(record.Data[33 : 33+2]) - file.Size = binary.LittleEndian.Uint64(record.Data[35 : 35+8]) - directoryID := binary.LittleEndian.Uint16(record.Data[43 : 43+2]) - filenameSize := binary.LittleEndian.Uint16(record.Data[45 : 45+2]) - //countTags := binary.LittleEndian.Uint16(record.Data[47 : 47+2]) // future implementation of tags + copy(file.ID[:], record.Data[32:32+16]) + file.Type = record.Data[48] + file.Format = binary.LittleEndian.Uint16(record.Data[49 : 49+2]) + file.Size = binary.LittleEndian.Uint64(record.Data[51 : 51+8]) - if len(record.Data) < 49+int(filenameSize) { - return nil, nil, errors.New("decodeBlockRecordFiles file record invalid filename size") - } - file.Name = string(record.Data[49 : 49+filenameSize]) + countTags := binary.LittleEndian.Uint16(record.Data[59 : 59+2]) - for n := range directories { - if directories[n].ID == directoryID { - file.Directory = directories[n].Name - break + index := 61 + + for n := uint16(0); n < countTags; n++ { + if index+6 > len(record.Data) { + return nil, errors.New("decodeBlockRecordFiles file record tags invalid size") } + + tag := BlockRecordFileTag{} + tag.Type = binary.LittleEndian.Uint16(record.Data[index:index+2]) & 0x7FFF + tagSize := binary.LittleEndian.Uint32(record.Data[index+2 : index+2+4]) + isDataReference := record.Data[index+1]&0x80 != 0 + + if index+6+int(tagSize) > len(record.Data) { + return nil, errors.New("decodeBlockRecordFiles file record tag data invalid size") + } + + if isDataReference { // reference to RecordTypeTagData record? + var refRecordNumber int + if tagSize == 2 { + refRecordNumber = i + int(int16(binary.LittleEndian.Uint16(record.Data[index+6:index+6+2]))) + } else if tagSize == 4 { + refRecordNumber = i + int(int32(binary.LittleEndian.Uint32(record.Data[index+6:index+6+4]))) + } else if tagSize == 8 { + refRecordNumber = i + int(int64(binary.LittleEndian.Uint64(record.Data[index+6:index+6+8]))) + } else { + return nil, errors.New("decodeBlockRecordFiles file record tag reference invalid size") + } + + if refRecordNumber < 0 || refRecordNumber >= len(recordsRaw) { + return nil, errors.New("decodeBlockRecordFiles file record tag reference not available") + } else if recordsRaw[refRecordNumber].Type != RecordTypeTagData { + return nil, errors.New("decodeBlockRecordFiles file record tag reference invalid") + } + + tag.Data = recordsRaw[refRecordNumber].Data + + } else { + tag.Data = record.Data[index+6 : index+6+int(tagSize)] + } + + file.TagsRaw = append(file.TagsRaw, tag) + + if decoded, err := decodeFileTag(tag); err != nil { + return nil, err + } else if decoded != nil { + file.TagsDecoded = append(file.TagsDecoded, decoded) + } + + index += 6 + int(tagSize) } files = append(files, file) } } - return files, directories, err + return files, err +} + +// decodeFileTag decodes a file tag. If the tag type is not known, it returns nil. +func decodeFileTag(tag BlockRecordFileTag) (decoded interface{}, err error) { + switch tag.Type { + case TagTypeDirectory: + return FileTagDirectory{Directory: string(tag.Data)}, nil + + case TagTypeName: + return FileTagName{Name: string(tag.Data)}, nil + + case TagTypeDescription: + return FileTagDescription{Description: string(tag.Data)}, nil + + case TagTypeDateCreated: + if len(tag.Data) != 8 { + return nil, errors.New("decodeFileTag invalid date tag size") + } + + timeB := int64(binary.LittleEndian.Uint64(tag.Data[0:8])) + return FileTagCreated{Date: time.Unix(timeB, 0)}, nil + + } + + return nil, nil +} + +// encodeFileTag encodes a file tag. If the tag type is not known, it returns nil. +func encodeFileTag(decoded interface{}) (tag BlockRecordFileTag, err error) { + switch v := decoded.(type) { + case FileTagDirectory: + return BlockRecordFileTag{Type: TagTypeDirectory, Data: []byte(v.Directory)}, nil + + case FileTagName: + return BlockRecordFileTag{Type: TagTypeName, Data: []byte(v.Name)}, nil + + case FileTagDescription: + return BlockRecordFileTag{Type: TagTypeDescription, Data: []byte(v.Description)}, nil + + case FileTagCreated: + var tempDate [8]byte + binary.LittleEndian.PutUint64(tempDate[0:8], uint64(v.Date.UTC().Unix())) + return BlockRecordFileTag{Type: TagTypeDateCreated, Data: tempDate[:]}, nil + + } + + return tag, errors.New("encodeFileTag unknown tag type") } // encodeBlockRecordFiles encodes files into the block record data // This function should be called grouped with all files in the same directory. The directory name is deduplicated; only unique directory records will be returned. func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRaw, err error) { - // First the directory records must be declared for any references by files - nextDirectoryID := uint16(1) // start as 1 to prevent collision with files without explicit directory - directoryList := make(map[string]int) + uniqueTagDataMap := make(map[string]struct{}) + duplicateTagDataMap := make(map[string]int) // list of tag data that appeared twice. Number in recordsRaw. + // loop through all tags to encode them and create list of duplicates that will be replaced by references for n := range files { - files[n].Directory, files[n].Name = sanitize.Path(files[n].Directory, files[n].Name) + for m := range files[n].TagsDecoded { + tag, err := encodeFileTag(files[n].TagsDecoded[m]) + if err != nil { + return nil, err + } + files[n].TagsRaw = append(files[n].TagsRaw, tag) - if files[n].Directory == "" { - continue + if len(tag.Data) > 4 { + if _, ok := uniqueTagDataMap[string(tag.Data)]; !ok { + uniqueTagDataMap[string(tag.Data)] = struct{}{} + } else if _, ok := duplicateTagDataMap[string(tag.Data)]; !ok { + recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeTagData, Data: tag.Data}) + duplicateTagDataMap[string(tag.Data)] = len(recordsRaw) - 1 + } + } } - - if directoryID, ok := directoryList[files[n].Directory]; ok { - files[n].directoryID = uint16(directoryID) - continue - } - - // Create the new directory record - var directoryIDb [2]byte - binary.LittleEndian.PutUint16(directoryIDb[0:2], nextDirectoryID) - files[n].directoryID = nextDirectoryID - nextDirectoryID++ - - recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeDirectory, Data: append(directoryIDb[:], []byte(files[n].Directory)...)}) } + // then encode all files as records for n := range files { - var data [49]byte + data := make([]byte, 61) if len(files[n].Hash) != hashSize { return nil, errors.New("encodeBlockRecords invalid file hash") } + copy(data[0:32], files[n].Hash[0:32]) + copy(data[32:32+16], files[n].ID[:]) - data[32] = files[n].Type - binary.LittleEndian.PutUint16(data[33:33+2], files[n].Format) - binary.LittleEndian.PutUint64(data[35:35+8], files[n].Size) - binary.LittleEndian.PutUint16(data[43:43+2], files[n].directoryID) + data[48] = files[n].Type + binary.LittleEndian.PutUint16(data[49:49+2], files[n].Format) + binary.LittleEndian.PutUint64(data[51:51+8], files[n].Size) + binary.LittleEndian.PutUint16(data[59:59+2], uint16(len(files[n].TagsRaw))) - filenameB := []byte(files[n].Name) - binary.LittleEndian.PutUint16(data[45:45+2], uint16(len(filenameB))) - binary.LittleEndian.PutUint16(data[47:47+2], uint16(0)) // Count of Tags (future use) + for _, tagRaw := range files[n].TagsRaw { + if len(tagRaw.Data) > 4 { + if refNumber, ok := duplicateTagDataMap[string(tagRaw.Data)]; ok { + // In case the data is duplicated, use reference to the RecordTypeTagData instead + tagRaw.Type |= 0x8000 + tagRaw.Data = intToBytes(-(len(recordsRaw) - refNumber)) + } + } - recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeFile, Data: append(data[:], filenameB...)}) + var tempTag [6]byte + + binary.LittleEndian.PutUint16(tempTag[0:2], tagRaw.Type) + binary.LittleEndian.PutUint32(tempTag[2:2+4], uint32(len(tagRaw.Data))) + + data = append(data, tempTag[:]...) + data = append(data, tagRaw.Data...) + } + + recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeFile, Data: data}) } return recordsRaw, nil } + +// intToBytes encodes int to little endian byte array as it fits to 16, 32 or 64 bit. +func intToBytes(number int) (buffer []byte) { + buffer = make([]byte, 4) + + if number <= math.MaxInt16 && number >= math.MinInt16 { + binary.LittleEndian.PutUint16(buffer[0:2], uint16(number)) + return buffer[0:2] + } else if number <= math.MaxInt32 && number >= math.MinInt32 { + binary.LittleEndian.PutUint32(buffer[0:4], uint32(number)) + return buffer[0:4] + } + + binary.LittleEndian.PutUint64(buffer[0:8], uint64(number)) + return buffer[0:8] +} + +// TagRawToText returns the tag text of the given tag, if available. Empty if not. +func (file *BlockRecordFile) TagRawToText(tagType uint16) string { + for m := range file.TagsRaw { + if file.TagsRaw[m].Type == tagType { + return string(file.TagsRaw[m].Data) + } + } + + return "" +} + +// ---- high-level decoding ---- + +// BlockDecoded contains the decoded records from a block +type BlockDecoded struct { + Block + RecordsDecoded []interface{} // Decoded records. See BlockRecordX structures. +} + +// decodeBlockRecords decodes all raw records in the block and returns a high-level decoded structure +// Use decodeBlockRecordX instead for specific record decoding. +func decodeBlockRecords(block *Block) (decoded *BlockDecoded, err error) { + decoded = &BlockDecoded{Block: *block} + + files, err := decodeBlockRecordFiles(block.RecordsRaw) + if err != nil { + return nil, err + } + + for _, file := range files { + decoded.RecordsDecoded = append(decoded.RecordsDecoded, file) + } + + if user := decodeBlockRecordUser(block.RecordsRaw); user != nil { + decoded.RecordsDecoded = append(decoded.RecordsDecoded, user) + } + + return decoded, nil +} diff --git a/Blockchain.go b/Blockchain.go index b81f000..9b282b7 100644 --- a/Blockchain.go +++ b/Blockchain.go @@ -10,24 +10,9 @@ Encoding of the blockchain header: Offset Size Info 0 8 Height of the blockchain 8 8 Version of the blockchain -16 2 Format of the block data. 0 = Current format. This is for backward compatibility and supporting changes to the block structure in the future. +16 2 Format of the blockchain. This provides backward compatibility. 18 65 Signature -Encoding of each block (it is the same stored in the database and shared in a message): -Offset Size Info -0 65 Signature of entire block -65 32 Hash (blake3) of last block. 0 for first one. -97 8 Blockchain version number -105 4 Block number -109 4 Size of entire block including this header -113 2 Count of records that follow - -Each record inside the block has this basic structure: -Offset Size Info -0 1 Record Type -1 4 Size of data -5 ? Data (encoding depends on record type) - */ package core @@ -126,7 +111,7 @@ func blockchainHeaderWrite(db store.Store, privateKey *btcec.PrivateKey, height, var buffer [83]byte binary.LittleEndian.PutUint64(buffer[0:8], height) binary.LittleEndian.PutUint64(buffer[8:16], version) - binary.LittleEndian.PutUint16(buffer[16:18], 0) + binary.LittleEndian.PutUint16(buffer[16:18], 0) // Current format is 0 signature, err := btcec.SignCompact(btcec.S256(), privateKey, hashData(buffer[0:18]), true) @@ -195,29 +180,29 @@ func UserBlockchainAppend(RecordsRaw []BlockRecordRaw) (newHeight uint64, status // UserBlockchainRead reads the block number from the blockchain. // Status: 0 = Success, 1 = Error block not found, 2 = Error block encoding, 3 = Error block record encoding // Errors 2 and 3 indicate data corruption. -func UserBlockchainRead(number uint64) (decoded *BlockDecoded, status int) { +func UserBlockchainRead(number uint64) (decoded *BlockDecoded, status int, err error) { if number >= userBlockchainHeader.height { - return nil, 1 + return nil, 1, errors.New("block number exceeds blockchain height") } var target [8]byte binary.LittleEndian.PutUint64(target[:], userBlockchainHeader.height-1) blockRaw, found := userBlockchainDB.Get(target[:]) if !found || len(blockRaw) == 0 { - return nil, 1 + return nil, 1, errors.New("block not found") } block, err := decodeBlock(blockRaw) if err != nil { - return nil, 2 + return nil, 2, err } decoded, err = decodeBlockRecords(block) if err != nil { - return nil, 2 + return nil, 2, err } - return decoded, 0 + return decoded, 0, nil } // UserBlockchainAddFiles adds files to the blockchain @@ -253,7 +238,7 @@ func UserBlockchainListFiles() (files []BlockRecordFile, status int) { return files, 2 } - filesMore, _, err := decodeBlockRecordFiles(block.RecordsRaw) + filesMore, err := decodeBlockRecordFiles(block.RecordsRaw) if err != nil { return nil, 3 } diff --git a/Test_test.go b/Test_test.go index fcce082..756c164 100644 --- a/Test_test.go +++ b/Test_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/btcsuite/btcd/btcec" + "github.com/google/uuid" ) func TestMessageEncodingAnnouncement(t *testing.T) { @@ -90,13 +91,21 @@ func TestBlockEncoding(t *testing.T) { return } - file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, Name: "Filename 1.txt", Directory: "documents\\sub folder"} - encoded1, _ := encodeBlockRecordUser(BlockRecordUser{Valid: true, Name: "Test User 1"}) - encoded2, _ := encodeBlockRecordFiles([]BlockRecordFile{file1}) + encoded1, _ := encodeBlockRecordUser(BlockRecordUser{Name: "Test User 1"}) + + file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, ID: uuid.New()} + file1.TagsDecoded = append(file1.TagsDecoded, FileTagName{Name: "Filename 1.txt"}) + file1.TagsDecoded = append(file1.TagsDecoded, FileTagDirectory{Directory: "documents\\sub folder"}) + + file2 := BlockRecordFile{Hash: hashData([]byte("Test data 2")), Type: TypeText, Format: FormatText, Size: 9, ID: uuid.New()} + file2.TagsDecoded = append(file2.TagsDecoded, FileTagName{Name: "Filename 2.txt"}) + file2.TagsDecoded = append(file2.TagsDecoded, FileTagDirectory{Directory: "documents\\sub folder"}) + + encodedFiles, _ := encodeBlockRecordFiles([]BlockRecordFile{file1, file2}) blockE := &Block{BlockchainVersion: 42, Number: 0} blockE.RecordsRaw = append(blockE.RecordsRaw, encoded1...) - blockE.RecordsRaw = append(blockE.RecordsRaw, encoded2...) + blockE.RecordsRaw = append(blockE.RecordsRaw, encodedFiles...) raw, err := encodeBlock(blockE, privateKey) if err != nil { @@ -119,14 +128,23 @@ func TestBlockEncoding(t *testing.T) { // output the block details fmt.Printf("Block details:\n----------------\nNumber: %d\nVersion: %d\nLast Hash: %s\nPublic Key: %s\n", block.Number, block.BlockchainVersion, hex.EncodeToString(block.LastBlockHash), hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())) - for _, file := range decoded.Files { - fmt.Printf("* File %s\n", file.Name) - fmt.Printf(" Directory %s\n", file.Directory) - fmt.Printf(" Size %d\n", file.Size) - fmt.Printf(" Type %d\n", file.Type) - fmt.Printf(" Format %d\n", file.Format) - fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) - fmt.Printf(" Directory ID %d\n\n", file.directoryID) + for _, decodedR := range decoded.RecordsDecoded { + if file, ok := decodedR.(BlockRecordFile); ok { + fmt.Printf("* File %s\n", file.ID.String()) + fmt.Printf(" Size %d\n", file.Size) + fmt.Printf(" Type %d\n", file.Type) + fmt.Printf(" Format %d\n", file.Format) + fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) + + for _, decodedT := range file.TagsDecoded { + switch v := decodedT.(type) { + case FileTagName: + fmt.Printf(" Name %s\n", v.Name) + case FileTagDirectory: + fmt.Printf(" Directory %s\n", v.Directory) + } + } + } } } @@ -144,7 +162,9 @@ func TestBlockchainAdd(t *testing.T) { initTestPrivateKey() initUserBlockchain() - file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, Name: "Filename 1.txt", Directory: "documents\\sub folder"} + file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, ID: uuid.New()} + file1.TagsDecoded = append(file1.TagsDecoded, FileTagName{Name: "Filename 1.txt"}) + file1.TagsDecoded = append(file1.TagsDecoded, FileTagDirectory{Directory: "documents\\sub folder"}) newHeight, status := UserBlockchainAddFiles([]BlockRecordFile{file1}) @@ -174,13 +194,13 @@ func TestBlockchainRead(t *testing.T) { blockNumber := uint64(0) - decoded, status := UserBlockchainRead(blockNumber) + decoded, status, err := UserBlockchainRead(blockNumber) switch status { case 0: case 1: // Error block not found fmt.Printf("Error reading block %d: Block not found.\n", blockNumber) case 2: // Error block encoding - fmt.Printf("Error reading block %d: Block encoding corrupt.\n", blockNumber) + fmt.Printf("Error reading block %d: Block encoding corrupt: %s\n", blockNumber, err.Error()) case 3: // Error block record encoding fmt.Printf("Error reading block %d: Block record encoding corrupt.\n", blockNumber) default: @@ -191,13 +211,22 @@ func TestBlockchainRead(t *testing.T) { return } - for _, file := range decoded.Files { - fmt.Printf("* File %s\n", file.Name) - fmt.Printf(" Directory %s\n", file.Directory) - fmt.Printf(" Size %d\n", file.Size) - fmt.Printf(" Type %d\n", file.Type) - fmt.Printf(" Format %d\n", file.Format) - fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) - fmt.Printf(" Directory ID %d\n\n", file.directoryID) + for _, decodedR := range decoded.RecordsDecoded { + if file, ok := decodedR.(BlockRecordFile); ok { + fmt.Printf("* File %s\n", file.ID.String()) + fmt.Printf(" Size %d\n", file.Size) + fmt.Printf(" Type %d\n", file.Type) + fmt.Printf(" Format %d\n", file.Format) + fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) + + for _, decodedT := range file.TagsDecoded { + switch v := decodedT.(type) { + case FileTagName: + fmt.Printf(" Name %s\n", v.Name) + case FileTagDirectory: + fmt.Printf(" Directory %s\n", v.Directory) + } + } + } } } diff --git a/go.mod b/go.mod index 66e6faf..b96ef3e 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.16 require ( github.com/akrylysov/pogreb v0.10.1 github.com/btcsuite/btcd v0.21.0-beta.0.20210401013323-36a96f6a0025 + github.com/google/uuid v1.3.0 github.com/pkg/errors v0.9.1 golang.org/x/crypto v0.0.0-20210415154028-4f45737414dc golang.org/x/net v0.0.0-20210420210106-798c2154c571 diff --git a/go.sum b/go.sum index 3df8792..519acea 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= diff --git a/sanitize/Sanitize.go b/sanitize/Sanitize.go index 1c12c5f..3fc3f52 100644 --- a/sanitize/Sanitize.go +++ b/sanitize/Sanitize.go @@ -14,8 +14,8 @@ import ( const PATH_MAX_LENGTH = 32767 // Windows Maximum Path Length for UNC paths -// Path sanitizies the directory and filename. -func Path(directory, filename string) (string, string) { +// PathDirectory sanitizes a directory path (without filename) +func PathDirectory(directory string) string { // Enforced forward slashes as directory separator and clean the path. directory = strings.ReplaceAll(directory, "\\", "/") directory = path.Clean(directory) @@ -23,14 +23,22 @@ func Path(directory, filename string) (string, string) { // No slash at the beginning and end to save space. directory = strings.Trim(directory, "/") - // Slashes in filenames are not encouraged, but not removed. + // Enforce max length. + if len(directory) > PATH_MAX_LENGTH { + directory = directory[:PATH_MAX_LENGTH] + } + return directory +} + +// PathFile sanitizes the filename. +func PathFile(filename string) string { // Enforce max filename length. if len(filename) > PATH_MAX_LENGTH { filename = filename[:PATH_MAX_LENGTH] } - return directory, filename + return filename } // Username sanitizes the username.