diff --git a/.gitignore b/.gitignore index 3c503b3..d000ff3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.exe log.txt *debug_bin +self.* \ No newline at end of file diff --git a/Block Encoding.go b/Block Encoding.go index 1aff952..3119bfc 100644 --- a/Block Encoding.go +++ b/Block Encoding.go @@ -3,7 +3,7 @@ File Name: Block Encoding.go Copyright: 2021 Peernet s.r.o. Author: Peter Kleissner -Block encoding in messages. +Block encoding in messages and local storage. */ package core @@ -12,10 +12,6 @@ import ( "bytes" "encoding/binary" "errors" - "io" - "path" - "strings" - "unicode/utf8" "github.com/btcsuite/btcd/btcec" ) @@ -23,14 +19,11 @@ import ( // Block is a single block containing a set of records (metadata). // It has no upper size limit, although a soft limit of 64 KB - overhead is encouraged for efficiency. type Block struct { - OwnerPublicKey *btcec.PublicKey // Owner Public Key, ECDSA (secp256k1) 257-bit - LastBlockHash []byte // Hash of the last block. Blake3. - BlockchainVersion uint64 // Blockchain version - Number uint32 // Block number - RecordsRaw []BlockRecordRaw // Block records raw - Files []BlockRecordFile // Files - User BlockRecordUser // User details - directories []BlockRecordDirectory // Internal list of directories for decoding files. + OwnerPublicKey *btcec.PublicKey // Owner Public Key, ECDSA (secp256k1) 257-bit + LastBlockHash []byte // Hash of the last block. Blake3. + BlockchainVersion uint64 // Blockchain version + Number uint64 // Block number + RecordsRaw []BlockRecordRaw // Block records raw } // BlockRecordRaw is a single block record (not decoded) @@ -39,44 +32,6 @@ type BlockRecordRaw struct { Data []byte // Data according to the type } -const ( - RecordTypeUsername = 0 // Username. Arbitrary name defined by the user. - RecordTypeDirectory = 1 // Directory. Only valid in the context of the current block. - RecordTypeFile = 2 // File - RecordTypeContentRating = 3 // Content rating (positive). - RecordTypeContentReport = 4 // Content report (negative). - RecordTypeDelete = 5 // Delete previous record. -) - -// block record structures - -// BlockRecordUser specifies user information -type BlockRecordUser struct { - Valid bool // Whether the username is supplied - Name string // Arbitrary name of the user. - NameS string // Sanitized version of the name. -} - -// BlockRecordDirectory is a directory, only valid within the same block. -type BlockRecordDirectory struct { - ID uint16 // ID, only valid within the same block - Name string // Name of the directory. Slashes (both backward and forward) mark subdirectories. -} - -// BlockRecordFile is the metadata of a file published on the blockchain -type BlockRecordFile struct { - Hash []byte // Hash of the file data - Type uint8 // Type (low-level) - Format uint16 // Format (high-level) - Size uint64 // Size of the file - Directory string // Directory - Name string // File name - directoryID uint16 // Internal directory ID - // Tags todo -} - -// Tag structure to be defined - const blockHeaderSize = 115 const blockRecordHeaderSize = 5 @@ -99,13 +54,14 @@ func decodeBlock(raw []byte) (block *Block, err error) { copy(block.LastBlockHash, raw[65:65+hashSize]) block.BlockchainVersion = binary.LittleEndian.Uint64(raw[97 : 97+8]) - block.Number = binary.LittleEndian.Uint32(raw[105 : 105+4]) + block.Number = uint64(binary.LittleEndian.Uint32(raw[105 : 105+4])) // for now 32-bit in protocol blockSize := binary.LittleEndian.Uint32(raw[109 : 109+4]) if blockSize != uint32(len(raw)) { return nil, errors.New("decodeBlock invalid block size") } + // decode on a low-level all block records countRecords := binary.LittleEndian.Uint16(raw[113 : 113+2]) index := 115 @@ -124,85 +80,12 @@ func decodeBlock(raw []byte) (block *Block, err error) { block.RecordsRaw = append(block.RecordsRaw, BlockRecordRaw{Type: recordType, Data: raw[index : index+int(recordSize)]}) - if err := decodeBlockRecord(raw[index:index+int(recordSize)], block, recordType); err != nil { - return nil, err - } - index += int(recordSize) } return block, nil } -// decodeBlockRecord decodes a single block and fills it into the provided block structure -func decodeBlockRecord(data []byte, block *Block, recordType uint8) (err error) { - switch recordType { - case RecordTypeUsername: - block.User.Name = string(data) - block.User.NameS = sanitizeUsername(block.User.Name) - block.User.Valid = true - - case RecordTypeDirectory: - if len(data) < 3 { - return errors.New("decodeBlockRecord directory record invalid size") - } - - directory := &BlockRecordDirectory{} - directory.ID = binary.LittleEndian.Uint16(data[0 : 0+2]) - directory.Name = string(data[2:]) - block.directories = append(block.directories, *directory) - - case RecordTypeFile: - if len(data) < 49 { - return errors.New("decodeBlockRecord file record invalid size") - } - - file := BlockRecordFile{} - file.Hash = make([]byte, hashSize) - copy(file.Hash, data[0:0+hashSize]) - file.Type = data[32] - file.Format = binary.LittleEndian.Uint16(data[33 : 33+2]) - file.Size = binary.LittleEndian.Uint64(data[35 : 35+8]) - directoryID := binary.LittleEndian.Uint16(data[43 : 43+2]) - filenameSize := binary.LittleEndian.Uint16(data[45 : 45+2]) - //countTags := binary.LittleEndian.Uint16(data[47 : 47+2]) // future implementation of tags - - if len(data) < 49+int(filenameSize) { - return errors.New("decodeBlockRecord file record invalid filename size") - } - file.Name = string(data[49 : 49+filenameSize]) - - for n := range block.directories { - if block.directories[n].ID == directoryID { - file.Directory = block.directories[n].Name - break - } - } - - block.Files = append(block.Files, file) - } - - return nil -} - -// sanitizeUsername returns the sanitized version of the username. -func sanitizeUsername(input string) string { - if !utf8.ValidString(input) { - return "" - } - - input = strings.TrimSpace(input) - input = strings.ReplaceAll(input, "\n", " ") - input = strings.ReplaceAll(input, "\r", "") - - // Max length for sanitized version is 36, resembling the limit from StackOverflow. - if len(input) > 36 { - input = input[:36] - } - - return input -} - func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, err error) { var buffer bytes.Buffer buffer.Write(make([]byte, 65)) // Signature, filled at the end @@ -218,16 +101,24 @@ func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, e binary.LittleEndian.PutUint64(temp[0:8], block.BlockchainVersion) buffer.Write(temp[:]) - binary.LittleEndian.PutUint32(temp[0:4], block.Number) + binary.LittleEndian.PutUint32(temp[0:4], uint32(block.Number)) // for now 32-bit in protocol buffer.Write(temp[:4]) buffer.Write(make([]byte, 4)) // Size of block, filled later buffer.Write(make([]byte, 2)) // Count of records, filled later // write all records - countRecords, err := encodeBlockRecords(&buffer, block) - if err != nil { - return nil, err + countRecords := uint16(0) + + for _, record := range block.RecordsRaw { + var temp [8]byte + binary.LittleEndian.PutUint32(temp[0:4], uint32(len(record.Data))) + + buffer.Write([]byte{record.Type}) // Record Type + buffer.Write(temp[:4]) // Size of data + buffer.Write(record.Data) // Data + + countRecords++ } // finalize the block @@ -248,89 +139,3 @@ func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, e return raw, nil } - -// encodeBlockRecords writes all records of the block into the writer -func encodeBlockRecords(writer io.Writer, block *Block) (count uint16, err error) { - nextDirectoryID := uint16(1) // start as 1 to prevent collision with files without explicit directory - - writeBlockRecord := func(recordType uint8, data []byte) { - var temp [8]byte - binary.LittleEndian.PutUint32(temp[0:4], uint32(len(data))) - - writer.Write([]byte{recordType}) // Record Type - writer.Write(temp[:4]) // Size of data - writer.Write(data) // Data - - count++ - } - - // Username - if block.User.Valid { - writeBlockRecord(RecordTypeUsername, []byte(block.User.Name)) - } - - // First the directory records must be declared for any references by files -directoryCreateLoop: - for n := range block.Files { - block.Files[n].sanitizePath() - if block.Files[n].Directory == "" { - continue - } - - // already known directory ID? - for m := range block.directories { - if block.directories[m].Name == block.Files[n].Directory { - block.Files[n].directoryID = block.directories[m].ID - continue directoryCreateLoop - } - } - - // Create the new directory record - var directoryID [2]byte - binary.LittleEndian.PutUint16(directoryID[0:2], nextDirectoryID) - block.Files[n].directoryID = nextDirectoryID - nextDirectoryID++ - - writeBlockRecord(RecordTypeDirectory, append(directoryID[:], []byte(block.Files[n].Directory)...)) - } - - for n := range block.Files { - var data [49]byte - - if len(block.Files[n].Hash) != hashSize { - return 0, errors.New("encodeBlockRecords invalid file hash") - } - copy(data[0:32], block.Files[n].Hash[0:32]) - - data[32] = block.Files[n].Type - binary.LittleEndian.PutUint16(data[33:33+2], block.Files[n].Format) - binary.LittleEndian.PutUint64(data[35:35+8], block.Files[n].Size) - binary.LittleEndian.PutUint16(data[43:43+2], block.Files[n].directoryID) - - filenameB := []byte(block.Files[n].Name) - binary.LittleEndian.PutUint16(data[45:45+2], uint16(len(filenameB))) - binary.LittleEndian.PutUint16(data[47:47+2], uint16(0)) // Count of Tags (future use) - - writeBlockRecord(RecordTypeFile, append(data[:], filenameB...)) - } - - return count, nil -} - -const PATH_MAX_LENGTH = 32767 // Windows Maximum Path Length for UNC paths - -func (record *BlockRecordFile) sanitizePath() { - // Enforced forward slashes as directory separator and clean the path. - record.Directory = strings.ReplaceAll(record.Directory, "\\", "/") - record.Directory = path.Clean(record.Directory) - - // No slash at the beginning and end to save space. - record.Directory = strings.Trim(record.Directory, "/") - - // Slashes in filenames are not encouraged, but not removed. - - // Enforce max filename length. - if len(record.Name) > PATH_MAX_LENGTH { - record.Name = record.Name[:PATH_MAX_LENGTH] - } -} diff --git a/Block Records.go b/Block Records.go new file mode 100644 index 0000000..b65c54b --- /dev/null +++ b/Block Records.go @@ -0,0 +1,179 @@ +/* +File Name: Block Encoding.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Encoding of records inside blocks. +*/ + +package core + +import ( + "encoding/binary" + "errors" + + "github.com/PeernetOfficial/core/sanitize" +) + +// BlockDecoded contains the decoded records from a block +type BlockDecoded struct { + Block + Files []BlockRecordFile // Files + User BlockRecordUser // User details + directories []BlockRecordDirectory // Internal list of directories for decoding files. +} + +// RecordTypeX defines the type of the record +const ( + RecordTypeUsername = 0 // Username. Arbitrary name defined by the user. + RecordTypeDirectory = 1 // Directory. Only valid in the context of the current block. + RecordTypeFile = 2 // File + RecordTypeContentRating = 3 // Content rating (positive). + RecordTypeContentReport = 4 // Content report (negative). + RecordTypeDelete = 5 // Delete previous record. +) + +// block record structures + +// BlockRecordUser specifies user information +type BlockRecordUser struct { + Valid bool // Whether the username is supplied + Name string // Arbitrary name of the user. + NameS string // Sanitized version of the name. +} + +// BlockRecordDirectory is a directory, only valid within the same block. +type BlockRecordDirectory struct { + ID uint16 // ID, only valid within the same block + Name string // Name of the directory. Slashes (both backward and forward) mark subdirectories. +} + +// BlockRecordFile is the metadata of a file published on the blockchain +type BlockRecordFile struct { + Hash []byte // Hash of the file data + Type uint8 // Type (low-level) + Format uint16 // Format (high-level) + Size uint64 // Size of the file + Directory string // Directory + Name string // File name + directoryID uint16 // Internal directory ID + // Tags todo +} + +// Tag structure to be defined + +// decodeBlockRecords decodes the raw records in the block and returns a high-level decoded structure +func decodeBlockRecords(block *Block) (decoded *BlockDecoded, err error) { + decoded = &BlockDecoded{Block: *block} + + for _, record := range block.RecordsRaw { + + switch record.Type { + case RecordTypeUsername: + decoded.User.Name = string(record.Data) + decoded.User.NameS = sanitize.Username(decoded.User.Name) + decoded.User.Valid = true + + case RecordTypeDirectory: + if len(record.Data) < 3 { + return nil, errors.New("decodeBlockRecord directory record invalid size") + } + + directory := &BlockRecordDirectory{} + directory.ID = binary.LittleEndian.Uint16(record.Data[0 : 0+2]) + directory.Name = string(record.Data[2:]) + decoded.directories = append(decoded.directories, *directory) + + case RecordTypeFile: + if len(record.Data) < 49 { + return nil, errors.New("decodeBlockRecord file record invalid size") + } + + file := BlockRecordFile{} + file.Hash = make([]byte, hashSize) + copy(file.Hash, record.Data[0:0+hashSize]) + file.Type = record.Data[32] + file.Format = binary.LittleEndian.Uint16(record.Data[33 : 33+2]) + file.Size = binary.LittleEndian.Uint64(record.Data[35 : 35+8]) + directoryID := binary.LittleEndian.Uint16(record.Data[43 : 43+2]) + filenameSize := binary.LittleEndian.Uint16(record.Data[45 : 45+2]) + //countTags := binary.LittleEndian.Uint16(record.Data[47 : 47+2]) // future implementation of tags + + if len(record.Data) < 49+int(filenameSize) { + return nil, errors.New("decodeBlockRecord file record invalid filename size") + } + file.Name = string(record.Data[49 : 49+filenameSize]) + + for n := range decoded.directories { + if decoded.directories[n].ID == directoryID { + file.Directory = decoded.directories[n].Name + break + } + } + + decoded.Files = append(decoded.Files, file) + } + } + + return decoded, nil +} + +// encodeBlockRecordUser encodes the username +func encodeBlockRecordUser(user BlockRecordUser) (recordsRaw []BlockRecordRaw, err error) { + if user.Valid { + recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeUsername, Data: []byte(user.Name)}) + } + + return recordsRaw, nil +} + +// encodeBlockRecordFiles encodes files into the block record data +// This function should be called grouped with all files in the same directory. The directory name is deduplicated; only unique directory records will be returned. +func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRaw, err error) { + // First the directory records must be declared for any references by files + nextDirectoryID := uint16(1) // start as 1 to prevent collision with files without explicit directory + directoryList := make(map[string]int) + + for n := range files { + files[n].Directory, files[n].Name = sanitize.Path(files[n].Directory, files[n].Name) + + if files[n].Directory == "" { + continue + } + + if directoryID, ok := directoryList[files[n].Directory]; ok { + files[n].directoryID = uint16(directoryID) + continue + } + + // Create the new directory record + var directoryIDb [2]byte + binary.LittleEndian.PutUint16(directoryIDb[0:2], nextDirectoryID) + files[n].directoryID = nextDirectoryID + nextDirectoryID++ + + recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeDirectory, Data: append(directoryIDb[:], []byte(files[n].Directory)...)}) + } + + for n := range files { + var data [49]byte + + if len(files[n].Hash) != hashSize { + return nil, errors.New("encodeBlockRecords invalid file hash") + } + copy(data[0:32], files[n].Hash[0:32]) + + data[32] = files[n].Type + binary.LittleEndian.PutUint16(data[33:33+2], files[n].Format) + binary.LittleEndian.PutUint64(data[35:35+8], files[n].Size) + binary.LittleEndian.PutUint16(data[43:43+2], files[n].directoryID) + + filenameB := []byte(files[n].Name) + binary.LittleEndian.PutUint16(data[45:45+2], uint16(len(filenameB))) + binary.LittleEndian.PutUint16(data[47:47+2], uint16(0)) // Count of Tags (future use) + + recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeFile, Data: append(data[:], filenameB...)}) + } + + return recordsRaw, nil +} diff --git a/Blockchain.go b/Blockchain.go index cea844a..2dbded5 100644 --- a/Blockchain.go +++ b/Blockchain.go @@ -2,6 +2,31 @@ File Name: Blockchain.go Copyright: 2021 Peernet s.r.o. Author: Peter Kleissner + +All blocks and the blockchain header are stored in a key/value database. +The key for the blockchain header is keyHeader and for each block is the block number as 64-bit unsigned integer little endian. + +Encoding of the blockchain header: +Offset Size Info +0 8 Height +8 8 Version +16 65 Signature + +Encoding of each block (it is the same stored in the database and shared in a message): +Offset Size Info +0 65 Signature of entire block +65 32 Hash (blake3) of last block. 0 for first one. +97 8 Blockchain version number +105 4 Block number +109 4 Size of entire block including this header +113 2 Count of records that follow + +Each record inside the block has this basic structure: +Offset Size Info +0 1 Record Type +1 4 Size of data +5 ? Data (encoding depends on record type) + */ package core @@ -9,8 +34,9 @@ package core import ( "encoding/binary" "encoding/hex" + "errors" "os" - "strconv" + "sync" "github.com/PeernetOfficial/core/store" "github.com/btcsuite/btcd/btcec" @@ -26,96 +52,174 @@ var BlockchainVersion = uint64(0) const filenameUserBlockchain = "self.blockchain" // the key names in the key/value database are constant and must not collide with block numbers (i.e. they must be >64 bit) -const keyHeaderHeight = "header.height" -const keyHeaderVersion = "header.version" -const keyHeaderSignature = "header.signature" +const keyHeader = "header blockchain" -// BlockchainHeader is the virtual blockchain header. The data is not stored as single header record, but split up. -type BlockchainHeader struct { - Height uint64 - Version uint64 - PublicKey *btcec.PublicKey +// userBlockchainHeader stores the users blockchain header in memory. Any changes must be synced to disk! +var userBlockchainHeader struct { + height uint64 + version uint64 + publicKey *btcec.PublicKey + sync.Mutex } var userBlockchainDB store.Store -var userBlockchainHeader BlockchainHeader // initUserBlockchain initializes the users blockchain. It creates the blockchain file if it does not exist already. // If it is corrupted, it will log the error and exit the process. func initUserBlockchain() { // open existing blockchain file or create new one - blockchain, err := store.NewPogrebStore(filenameUserBlockchain) - if err != nil { + var err error + if userBlockchainDB, err = store.NewPogrebStore(filenameUserBlockchain); err != nil { Filters.LogError("initUserBlockchain", "error opening user blockchain: %s\n", err.Error()) os.Exit(1) } // verify header - var height, version uint64 - heightB, found1 := blockchain.Get([]byte(keyHeaderHeight)) - versionB, found2 := blockchain.Get([]byte(keyHeaderVersion)) - headerSignature, isSignature := blockchain.Get([]byte(keyHeaderSignature)) - - if found1 && len(heightB) != 8 || found2 && len(versionB) != 8 || isSignature && len(headerSignature) != 65 { - Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Invalid header length. Height is '%s', version '%s' and signature '%s'.\n", hex.EncodeToString(heightB), hex.EncodeToString(versionB), hex.EncodeToString(headerSignature)) + var found bool + userBlockchainHeader.publicKey, userBlockchainHeader.height, userBlockchainHeader.version, found, err = blockchainHeaderRead(userBlockchainDB) + if err != nil { + Filters.LogError("initUserBlockchain", "corrupt user blockchain database: %s\n", err.Error()) os.Exit(1) - } - - if found1 && len(heightB) == 8 { - height = binary.BigEndian.Uint64(heightB) - } - if found2 && len(heightB) == 8 { - version = binary.BigEndian.Uint64(versionB) - } - - if isSignature { - // validate header signature - headerA := strconv.FormatUint(height, 10) + "/" + strconv.FormatUint(version, 10) - headerPublicKey, _, err := btcec.RecoverCompact(btcec.S256(), headerSignature[:], hashData([]byte(headerA))) - if err != nil { - Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Error decoding signature. Height is '%s', version '%s', signature '%s'. %s\n", hex.EncodeToString(heightB), hex.EncodeToString(versionB), hex.EncodeToString(headerSignature), err.Error()) - os.Exit(1) - } - if !headerPublicKey.IsEqual(peerPublicKey) { - Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Signature key mismatch. Height is '%s', version '%s', signature '%s'.\n", hex.EncodeToString(heightB), hex.EncodeToString(versionB), hex.EncodeToString(headerSignature)) - os.Exit(1) - } - - userBlockchainHeader = BlockchainHeader{Height: height, Version: version, PublicKey: headerPublicKey} - - } else if !found1 && !found2 && !isSignature { + } else if !found { // First run: create header signature! - height = 0 - version = 0 - signature, _ := blockchainCreateHeaderSignature(height, version) - if err := blockchain.Set([]byte(keyHeaderSignature), signature); err != nil { + userBlockchainHeader.height = 0 + userBlockchainHeader.version = 0 + if err := blockchainHeaderWrite(userBlockchainDB, peerPrivateKey, userBlockchainHeader.height, userBlockchainHeader.version); err != nil { Filters.LogError("initUserBlockchain", "initializing user blockchain: %s", err.Error()) os.Exit(1) } - - } else { - // header signature not present, but height/version is -> corrupt data. - Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Height is '%s', version '%s' and signature '%s'.\n", hex.EncodeToString(heightB), hex.EncodeToString(versionB), hex.EncodeToString(headerSignature)) + } else if !userBlockchainHeader.publicKey.IsEqual(peerPublicKey) { + Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Public key mismatch. Height is '%d', version '%d'. Public key expected '%s' vs provided '%s'\n", userBlockchainHeader.height, userBlockchainHeader.version, hex.EncodeToString(peerPublicKey.SerializeCompressed()), hex.EncodeToString(userBlockchainHeader.publicKey.SerializeCompressed())) os.Exit(1) } - - userBlockchainDB = blockchain } -func blockchainCreateHeaderSignature(height, version uint64) (signature []byte, err error) { - headerA := strconv.FormatUint(height, 10) + "/" + strconv.FormatUint(version, 10) - signature, err = btcec.SignCompact(btcec.S256(), peerPrivateKey, hashData([]byte(headerA)), true) - - if err != nil { - Filters.LogError("blockchainCreateHeaderSignature", "creating signature of '%s': %s\n", headerA, err.Error()) +// blockchainHeaderRead reads the header from the blockchain and decodes it. +func blockchainHeaderRead(db store.Store) (publicKey *btcec.PublicKey, height, version uint64, found bool, err error) { + buffer, found := db.Get([]byte(keyHeader)) + if !found { + return nil, 0, 0, false, nil } - return signature, err + if len(buffer) != 81 { + return nil, 0, 0, true, errors.New("blockchain header size mismatch") + } + + height = binary.LittleEndian.Uint64(buffer[0:8]) + version = binary.LittleEndian.Uint64(buffer[8:16]) + signature := buffer[16 : 16+65] + + publicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, hashData(buffer[0:16])) + + return +} + +// blockchainHeaderWrite writes the header to the blockchain and signs it. +func blockchainHeaderWrite(db store.Store, privateKey *btcec.PrivateKey, height, version uint64) (err error) { + var buffer [81]byte + binary.LittleEndian.PutUint64(buffer[0:8], height) + binary.LittleEndian.PutUint64(buffer[8:16], version) + + signature, err := btcec.SignCompact(btcec.S256(), privateKey, hashData(buffer[0:16]), true) + + if err != nil { + return err + } else if len(signature) != 65 { + return errors.New("signature length invalid") + } + + copy(buffer[16:16+65], signature) + + err = db.Set([]byte(keyHeader), buffer[:]) + + return err } // UserBlockchainHeader returns the users blockchain header which stores the height and version number. -func UserBlockchainHeader() (header BlockchainHeader) { - return userBlockchainHeader +func UserBlockchainHeader() (publicKey *btcec.PublicKey, height uint64, version uint64) { + return userBlockchainHeader.publicKey, userBlockchainHeader.height, userBlockchainHeader.version } // ---- low-level blockchain manipulation functions ---- + +// UserBlockchainAppend appends a new block to the blockchain based on the provided raw records. +// Status: 0 = Success, 1 = Error previous block not found, 2 = Error block encoding +func UserBlockchainAppend(RecordsRaw []BlockRecordRaw) (newHeight uint64, status int) { + userBlockchainHeader.Lock() + defer userBlockchainHeader.Unlock() + + block := &Block{OwnerPublicKey: peerPublicKey, RecordsRaw: RecordsRaw} + + // set the last block hash first + if userBlockchainHeader.height > 0 { + var target [8]byte + binary.LittleEndian.PutUint64(target[:], userBlockchainHeader.height-1) + previousBlockRaw, found := userBlockchainDB.Get(target[:]) + if !found || len(previousBlockRaw) == 0 { + return 0, 1 + } + + block.LastBlockHash = hashData(previousBlockRaw) + } + + block.Number = userBlockchainHeader.height + block.BlockchainVersion = userBlockchainHeader.version + + raw, err := encodeBlock(block, peerPrivateKey) + if err != nil { + return 0, 2 + } + + // increase blockchain height + userBlockchainHeader.height++ + + // store the block + var numberB [8]byte + binary.LittleEndian.PutUint64(numberB[:], block.Number) + userBlockchainDB.Set(numberB[:], raw) + + // update the blockchain header in the database + blockchainHeaderWrite(userBlockchainDB, peerPrivateKey, userBlockchainHeader.height, userBlockchainHeader.version) + + return userBlockchainHeader.height, 0 +} + +// UserBlockchainRead reads the block number from the blockchain. +// Status: 0 = Success, 1 = Error block not found, 2 = Error block encoding, 3 = Error block record encoding +// Errors 2 and 3 indicate data corruption. +func UserBlockchainRead(number uint64) (decoded *BlockDecoded, status int) { + if number >= userBlockchainHeader.height { + return nil, 1 + } + + var target [8]byte + binary.LittleEndian.PutUint64(target[:], userBlockchainHeader.height-1) + blockRaw, found := userBlockchainDB.Get(target[:]) + if !found || len(blockRaw) == 0 { + return nil, 1 + } + + block, err := decodeBlock(blockRaw) + if err != nil { + return nil, 2 + } + + decoded, err = decodeBlockRecords(block) + if err != nil { + return nil, 2 + } + + return decoded, 0 +} + +// UserBlockchainAddFiles adds files to the blockchain +// Status: 0 = Success, 1 = Error previous block not found, 2 = Error block encoding, 3 = Error block record encoding +// It makes sense to group all files in the same directory into one call, since only one directory record will be created per unique directory per block. +func UserBlockchainAddFiles(files []BlockRecordFile) (newHeight uint64, status int) { + encoded, err := encodeBlockRecordFiles(files) + if err != nil { + return 0, 3 + } + + return UserBlockchainAppend(encoded) +} diff --git a/Test_test.go b/Test_test.go index bfa2ee3..fcce082 100644 --- a/Test_test.go +++ b/Test_test.go @@ -5,6 +5,8 @@ import ( "encoding/hex" "fmt" "testing" + + "github.com/btcsuite/btcd/btcec" ) func TestMessageEncodingAnnouncement(t *testing.T) { @@ -89,16 +91,26 @@ func TestBlockEncoding(t *testing.T) { } file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, Name: "Filename 1.txt", Directory: "documents\\sub folder"} + encoded1, _ := encodeBlockRecordUser(BlockRecordUser{Valid: true, Name: "Test User 1"}) + encoded2, _ := encodeBlockRecordFiles([]BlockRecordFile{file1}) - block := &Block{BlockchainVersion: 42, Number: 0, User: BlockRecordUser{Valid: true, Name: "Test User 1"}, Files: []BlockRecordFile{file1}} + blockE := &Block{BlockchainVersion: 42, Number: 0} + blockE.RecordsRaw = append(blockE.RecordsRaw, encoded1...) + blockE.RecordsRaw = append(blockE.RecordsRaw, encoded2...) - raw, err := encodeBlock(block, privateKey) + raw, err := encodeBlock(blockE, privateKey) if err != nil { fmt.Printf("Error: %s\n", err.Error()) return } - block, err = decodeBlock(raw) + block, err := decodeBlock(raw) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + decoded, err := decodeBlockRecords(block) if err != nil { fmt.Printf("Error: %s\n", err.Error()) return @@ -107,7 +119,79 @@ func TestBlockEncoding(t *testing.T) { // output the block details fmt.Printf("Block details:\n----------------\nNumber: %d\nVersion: %d\nLast Hash: %s\nPublic Key: %s\n", block.Number, block.BlockchainVersion, hex.EncodeToString(block.LastBlockHash), hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())) - for _, file := range block.Files { + for _, file := range decoded.Files { + fmt.Printf("* File %s\n", file.Name) + fmt.Printf(" Directory %s\n", file.Directory) + fmt.Printf(" Size %d\n", file.Size) + fmt.Printf(" Type %d\n", file.Type) + fmt.Printf(" Format %d\n", file.Format) + fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) + fmt.Printf(" Directory ID %d\n\n", file.directoryID) + } +} + +func initTestPrivateKey() { + // use static test key, otherwise tests will be inconsistent (would otherwise fail to open blockchain database) + privateKeyTestA := "d65da474861d826edd29c1307f1250d79e9dbf84e3a2449022658445c8d8ed63" + privateKeyB, _ := hex.DecodeString(privateKeyTestA) + peerPrivateKey, peerPublicKey = btcec.PrivKeyFromBytes(btcec.S256(), privateKeyB) + nodeID = PublicKey2NodeID(peerPublicKey) + + fmt.Printf("Loaded public key: %s\n", hex.EncodeToString(peerPublicKey.SerializeCompressed())) +} + +func TestBlockchainAdd(t *testing.T) { + initTestPrivateKey() + initUserBlockchain() + + file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, Name: "Filename 1.txt", Directory: "documents\\sub folder"} + + newHeight, status := UserBlockchainAddFiles([]BlockRecordFile{file1}) + + switch status { + case 0: + case 1: // Error previous block not found + fmt.Printf("Error adding file to blockchain: Previous block not found.\n") + case 2: // Error block encoding + fmt.Printf("Error adding file to blockchain: Error block encoding.\n") + case 3: // Error block record encoding + fmt.Printf("Error adding file to blockchain: Error block record encoding.\n") + default: + fmt.Printf("Error adding file to blockchain: Unknown status %d\n", status) + } + + if status != 0 { + return + } + + fmt.Printf("Success adding files to blockchain. New blockchain height: %d\n", newHeight) +} + +func TestBlockchainRead(t *testing.T) { + initTestPrivateKey() + initFilters() + initUserBlockchain() + + blockNumber := uint64(0) + + decoded, status := UserBlockchainRead(blockNumber) + switch status { + case 0: + case 1: // Error block not found + fmt.Printf("Error reading block %d: Block not found.\n", blockNumber) + case 2: // Error block encoding + fmt.Printf("Error reading block %d: Block encoding corrupt.\n", blockNumber) + case 3: // Error block record encoding + fmt.Printf("Error reading block %d: Block record encoding corrupt.\n", blockNumber) + default: + fmt.Printf("Error reading block %d: Unknown status %d\n", blockNumber, status) + } + + if status != 0 { + return + } + + for _, file := range decoded.Files { fmt.Printf("* File %s\n", file.Name) fmt.Printf(" Directory %s\n", file.Directory) fmt.Printf(" Size %d\n", file.Size) diff --git a/sanitize/Sanitize.go b/sanitize/Sanitize.go new file mode 100644 index 0000000..1c12c5f --- /dev/null +++ b/sanitize/Sanitize.go @@ -0,0 +1,52 @@ +/* +File Name: Sanitize.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package sanitize + +import ( + "path" + "strings" + "unicode/utf8" +) + +const PATH_MAX_LENGTH = 32767 // Windows Maximum Path Length for UNC paths + +// Path sanitizies the directory and filename. +func Path(directory, filename string) (string, string) { + // Enforced forward slashes as directory separator and clean the path. + directory = strings.ReplaceAll(directory, "\\", "/") + directory = path.Clean(directory) + + // No slash at the beginning and end to save space. + directory = strings.Trim(directory, "/") + + // Slashes in filenames are not encouraged, but not removed. + + // Enforce max filename length. + if len(filename) > PATH_MAX_LENGTH { + filename = filename[:PATH_MAX_LENGTH] + } + + return directory, filename +} + +// Username sanitizes the username. +func Username(input string) string { + if !utf8.ValidString(input) { + return "" + } + + input = strings.TrimSpace(input) + input = strings.ReplaceAll(input, "\n", " ") + input = strings.ReplaceAll(input, "\r", "") + + // Max length for sanitized version is 36, resembling the limit from StackOverflow. + if len(input) > 36 { + input = input[:36] + } + + return input +}