mirror of
https://github.com/PeernetOfficial/core.git
synced 2026-07-17 02:47:51 +01:00
Store user blockchain data in file. Close #30
Refactoring sanitize code into separate sub-package Refactoring block and block record decoding code
This commit is contained in:
Kleissner
1
.gitignore
vendored
1
.gitignore
vendored
@@ -1,3 +1,4 @@
|
||||
*.exe
|
||||
log.txt
|
||||
*debug_bin
|
||||
self.*
|
||||
@@ -3,7 +3,7 @@ File Name: Block Encoding.go
|
||||
Copyright: 2021 Peernet s.r.o.
|
||||
Author: Peter Kleissner
|
||||
|
||||
Block encoding in messages.
|
||||
Block encoding in messages and local storage.
|
||||
*/
|
||||
|
||||
package core
|
||||
@@ -12,10 +12,6 @@ import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/btcsuite/btcd/btcec"
|
||||
)
|
||||
@@ -23,14 +19,11 @@ import (
|
||||
// Block is a single block containing a set of records (metadata).
|
||||
// It has no upper size limit, although a soft limit of 64 KB - overhead is encouraged for efficiency.
|
||||
type Block struct {
|
||||
OwnerPublicKey *btcec.PublicKey // Owner Public Key, ECDSA (secp256k1) 257-bit
|
||||
LastBlockHash []byte // Hash of the last block. Blake3.
|
||||
BlockchainVersion uint64 // Blockchain version
|
||||
Number uint32 // Block number
|
||||
RecordsRaw []BlockRecordRaw // Block records raw
|
||||
Files []BlockRecordFile // Files
|
||||
User BlockRecordUser // User details
|
||||
directories []BlockRecordDirectory // Internal list of directories for decoding files.
|
||||
OwnerPublicKey *btcec.PublicKey // Owner Public Key, ECDSA (secp256k1) 257-bit
|
||||
LastBlockHash []byte // Hash of the last block. Blake3.
|
||||
BlockchainVersion uint64 // Blockchain version
|
||||
Number uint64 // Block number
|
||||
RecordsRaw []BlockRecordRaw // Block records raw
|
||||
}
|
||||
|
||||
// BlockRecordRaw is a single block record (not decoded)
|
||||
@@ -39,44 +32,6 @@ type BlockRecordRaw struct {
|
||||
Data []byte // Data according to the type
|
||||
}
|
||||
|
||||
const (
|
||||
RecordTypeUsername = 0 // Username. Arbitrary name defined by the user.
|
||||
RecordTypeDirectory = 1 // Directory. Only valid in the context of the current block.
|
||||
RecordTypeFile = 2 // File
|
||||
RecordTypeContentRating = 3 // Content rating (positive).
|
||||
RecordTypeContentReport = 4 // Content report (negative).
|
||||
RecordTypeDelete = 5 // Delete previous record.
|
||||
)
|
||||
|
||||
// block record structures
|
||||
|
||||
// BlockRecordUser specifies user information
|
||||
type BlockRecordUser struct {
|
||||
Valid bool // Whether the username is supplied
|
||||
Name string // Arbitrary name of the user.
|
||||
NameS string // Sanitized version of the name.
|
||||
}
|
||||
|
||||
// BlockRecordDirectory is a directory, only valid within the same block.
|
||||
type BlockRecordDirectory struct {
|
||||
ID uint16 // ID, only valid within the same block
|
||||
Name string // Name of the directory. Slashes (both backward and forward) mark subdirectories.
|
||||
}
|
||||
|
||||
// BlockRecordFile is the metadata of a file published on the blockchain
|
||||
type BlockRecordFile struct {
|
||||
Hash []byte // Hash of the file data
|
||||
Type uint8 // Type (low-level)
|
||||
Format uint16 // Format (high-level)
|
||||
Size uint64 // Size of the file
|
||||
Directory string // Directory
|
||||
Name string // File name
|
||||
directoryID uint16 // Internal directory ID
|
||||
// Tags todo
|
||||
}
|
||||
|
||||
// Tag structure to be defined
|
||||
|
||||
const blockHeaderSize = 115
|
||||
const blockRecordHeaderSize = 5
|
||||
|
||||
@@ -99,13 +54,14 @@ func decodeBlock(raw []byte) (block *Block, err error) {
|
||||
copy(block.LastBlockHash, raw[65:65+hashSize])
|
||||
|
||||
block.BlockchainVersion = binary.LittleEndian.Uint64(raw[97 : 97+8])
|
||||
block.Number = binary.LittleEndian.Uint32(raw[105 : 105+4])
|
||||
block.Number = uint64(binary.LittleEndian.Uint32(raw[105 : 105+4])) // for now 32-bit in protocol
|
||||
|
||||
blockSize := binary.LittleEndian.Uint32(raw[109 : 109+4])
|
||||
if blockSize != uint32(len(raw)) {
|
||||
return nil, errors.New("decodeBlock invalid block size")
|
||||
}
|
||||
|
||||
// decode on a low-level all block records
|
||||
countRecords := binary.LittleEndian.Uint16(raw[113 : 113+2])
|
||||
index := 115
|
||||
|
||||
@@ -124,85 +80,12 @@ func decodeBlock(raw []byte) (block *Block, err error) {
|
||||
|
||||
block.RecordsRaw = append(block.RecordsRaw, BlockRecordRaw{Type: recordType, Data: raw[index : index+int(recordSize)]})
|
||||
|
||||
if err := decodeBlockRecord(raw[index:index+int(recordSize)], block, recordType); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
index += int(recordSize)
|
||||
}
|
||||
|
||||
return block, nil
|
||||
}
|
||||
|
||||
// decodeBlockRecord decodes a single block and fills it into the provided block structure
|
||||
func decodeBlockRecord(data []byte, block *Block, recordType uint8) (err error) {
|
||||
switch recordType {
|
||||
case RecordTypeUsername:
|
||||
block.User.Name = string(data)
|
||||
block.User.NameS = sanitizeUsername(block.User.Name)
|
||||
block.User.Valid = true
|
||||
|
||||
case RecordTypeDirectory:
|
||||
if len(data) < 3 {
|
||||
return errors.New("decodeBlockRecord directory record invalid size")
|
||||
}
|
||||
|
||||
directory := &BlockRecordDirectory{}
|
||||
directory.ID = binary.LittleEndian.Uint16(data[0 : 0+2])
|
||||
directory.Name = string(data[2:])
|
||||
block.directories = append(block.directories, *directory)
|
||||
|
||||
case RecordTypeFile:
|
||||
if len(data) < 49 {
|
||||
return errors.New("decodeBlockRecord file record invalid size")
|
||||
}
|
||||
|
||||
file := BlockRecordFile{}
|
||||
file.Hash = make([]byte, hashSize)
|
||||
copy(file.Hash, data[0:0+hashSize])
|
||||
file.Type = data[32]
|
||||
file.Format = binary.LittleEndian.Uint16(data[33 : 33+2])
|
||||
file.Size = binary.LittleEndian.Uint64(data[35 : 35+8])
|
||||
directoryID := binary.LittleEndian.Uint16(data[43 : 43+2])
|
||||
filenameSize := binary.LittleEndian.Uint16(data[45 : 45+2])
|
||||
//countTags := binary.LittleEndian.Uint16(data[47 : 47+2]) // future implementation of tags
|
||||
|
||||
if len(data) < 49+int(filenameSize) {
|
||||
return errors.New("decodeBlockRecord file record invalid filename size")
|
||||
}
|
||||
file.Name = string(data[49 : 49+filenameSize])
|
||||
|
||||
for n := range block.directories {
|
||||
if block.directories[n].ID == directoryID {
|
||||
file.Directory = block.directories[n].Name
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
block.Files = append(block.Files, file)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// sanitizeUsername returns the sanitized version of the username.
|
||||
func sanitizeUsername(input string) string {
|
||||
if !utf8.ValidString(input) {
|
||||
return "<invalid encoding>"
|
||||
}
|
||||
|
||||
input = strings.TrimSpace(input)
|
||||
input = strings.ReplaceAll(input, "\n", " ")
|
||||
input = strings.ReplaceAll(input, "\r", "")
|
||||
|
||||
// Max length for sanitized version is 36, resembling the limit from StackOverflow.
|
||||
if len(input) > 36 {
|
||||
input = input[:36]
|
||||
}
|
||||
|
||||
return input
|
||||
}
|
||||
|
||||
func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, err error) {
|
||||
var buffer bytes.Buffer
|
||||
buffer.Write(make([]byte, 65)) // Signature, filled at the end
|
||||
@@ -218,16 +101,24 @@ func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, e
|
||||
binary.LittleEndian.PutUint64(temp[0:8], block.BlockchainVersion)
|
||||
buffer.Write(temp[:])
|
||||
|
||||
binary.LittleEndian.PutUint32(temp[0:4], block.Number)
|
||||
binary.LittleEndian.PutUint32(temp[0:4], uint32(block.Number)) // for now 32-bit in protocol
|
||||
buffer.Write(temp[:4])
|
||||
|
||||
buffer.Write(make([]byte, 4)) // Size of block, filled later
|
||||
buffer.Write(make([]byte, 2)) // Count of records, filled later
|
||||
|
||||
// write all records
|
||||
countRecords, err := encodeBlockRecords(&buffer, block)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
countRecords := uint16(0)
|
||||
|
||||
for _, record := range block.RecordsRaw {
|
||||
var temp [8]byte
|
||||
binary.LittleEndian.PutUint32(temp[0:4], uint32(len(record.Data)))
|
||||
|
||||
buffer.Write([]byte{record.Type}) // Record Type
|
||||
buffer.Write(temp[:4]) // Size of data
|
||||
buffer.Write(record.Data) // Data
|
||||
|
||||
countRecords++
|
||||
}
|
||||
|
||||
// finalize the block
|
||||
@@ -248,89 +139,3 @@ func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, e
|
||||
|
||||
return raw, nil
|
||||
}
|
||||
|
||||
// encodeBlockRecords writes all records of the block into the writer
|
||||
func encodeBlockRecords(writer io.Writer, block *Block) (count uint16, err error) {
|
||||
nextDirectoryID := uint16(1) // start as 1 to prevent collision with files without explicit directory
|
||||
|
||||
writeBlockRecord := func(recordType uint8, data []byte) {
|
||||
var temp [8]byte
|
||||
binary.LittleEndian.PutUint32(temp[0:4], uint32(len(data)))
|
||||
|
||||
writer.Write([]byte{recordType}) // Record Type
|
||||
writer.Write(temp[:4]) // Size of data
|
||||
writer.Write(data) // Data
|
||||
|
||||
count++
|
||||
}
|
||||
|
||||
// Username
|
||||
if block.User.Valid {
|
||||
writeBlockRecord(RecordTypeUsername, []byte(block.User.Name))
|
||||
}
|
||||
|
||||
// First the directory records must be declared for any references by files
|
||||
directoryCreateLoop:
|
||||
for n := range block.Files {
|
||||
block.Files[n].sanitizePath()
|
||||
if block.Files[n].Directory == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// already known directory ID?
|
||||
for m := range block.directories {
|
||||
if block.directories[m].Name == block.Files[n].Directory {
|
||||
block.Files[n].directoryID = block.directories[m].ID
|
||||
continue directoryCreateLoop
|
||||
}
|
||||
}
|
||||
|
||||
// Create the new directory record
|
||||
var directoryID [2]byte
|
||||
binary.LittleEndian.PutUint16(directoryID[0:2], nextDirectoryID)
|
||||
block.Files[n].directoryID = nextDirectoryID
|
||||
nextDirectoryID++
|
||||
|
||||
writeBlockRecord(RecordTypeDirectory, append(directoryID[:], []byte(block.Files[n].Directory)...))
|
||||
}
|
||||
|
||||
for n := range block.Files {
|
||||
var data [49]byte
|
||||
|
||||
if len(block.Files[n].Hash) != hashSize {
|
||||
return 0, errors.New("encodeBlockRecords invalid file hash")
|
||||
}
|
||||
copy(data[0:32], block.Files[n].Hash[0:32])
|
||||
|
||||
data[32] = block.Files[n].Type
|
||||
binary.LittleEndian.PutUint16(data[33:33+2], block.Files[n].Format)
|
||||
binary.LittleEndian.PutUint64(data[35:35+8], block.Files[n].Size)
|
||||
binary.LittleEndian.PutUint16(data[43:43+2], block.Files[n].directoryID)
|
||||
|
||||
filenameB := []byte(block.Files[n].Name)
|
||||
binary.LittleEndian.PutUint16(data[45:45+2], uint16(len(filenameB)))
|
||||
binary.LittleEndian.PutUint16(data[47:47+2], uint16(0)) // Count of Tags (future use)
|
||||
|
||||
writeBlockRecord(RecordTypeFile, append(data[:], filenameB...))
|
||||
}
|
||||
|
||||
return count, nil
|
||||
}
|
||||
|
||||
const PATH_MAX_LENGTH = 32767 // Windows Maximum Path Length for UNC paths
|
||||
|
||||
func (record *BlockRecordFile) sanitizePath() {
|
||||
// Enforced forward slashes as directory separator and clean the path.
|
||||
record.Directory = strings.ReplaceAll(record.Directory, "\\", "/")
|
||||
record.Directory = path.Clean(record.Directory)
|
||||
|
||||
// No slash at the beginning and end to save space.
|
||||
record.Directory = strings.Trim(record.Directory, "/")
|
||||
|
||||
// Slashes in filenames are not encouraged, but not removed.
|
||||
|
||||
// Enforce max filename length.
|
||||
if len(record.Name) > PATH_MAX_LENGTH {
|
||||
record.Name = record.Name[:PATH_MAX_LENGTH]
|
||||
}
|
||||
}
|
||||
|
||||
179
Block Records.go
Normal file
179
Block Records.go
Normal file
@@ -0,0 +1,179 @@
|
||||
/*
|
||||
File Name: Block Encoding.go
|
||||
Copyright: 2021 Peernet s.r.o.
|
||||
Author: Peter Kleissner
|
||||
|
||||
Encoding of records inside blocks.
|
||||
*/
|
||||
|
||||
package core
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"github.com/PeernetOfficial/core/sanitize"
|
||||
)
|
||||
|
||||
// BlockDecoded contains the decoded records from a block
|
||||
type BlockDecoded struct {
|
||||
Block
|
||||
Files []BlockRecordFile // Files
|
||||
User BlockRecordUser // User details
|
||||
directories []BlockRecordDirectory // Internal list of directories for decoding files.
|
||||
}
|
||||
|
||||
// RecordTypeX defines the type of the record
|
||||
const (
|
||||
RecordTypeUsername = 0 // Username. Arbitrary name defined by the user.
|
||||
RecordTypeDirectory = 1 // Directory. Only valid in the context of the current block.
|
||||
RecordTypeFile = 2 // File
|
||||
RecordTypeContentRating = 3 // Content rating (positive).
|
||||
RecordTypeContentReport = 4 // Content report (negative).
|
||||
RecordTypeDelete = 5 // Delete previous record.
|
||||
)
|
||||
|
||||
// block record structures
|
||||
|
||||
// BlockRecordUser specifies user information
|
||||
type BlockRecordUser struct {
|
||||
Valid bool // Whether the username is supplied
|
||||
Name string // Arbitrary name of the user.
|
||||
NameS string // Sanitized version of the name.
|
||||
}
|
||||
|
||||
// BlockRecordDirectory is a directory, only valid within the same block.
|
||||
type BlockRecordDirectory struct {
|
||||
ID uint16 // ID, only valid within the same block
|
||||
Name string // Name of the directory. Slashes (both backward and forward) mark subdirectories.
|
||||
}
|
||||
|
||||
// BlockRecordFile is the metadata of a file published on the blockchain
|
||||
type BlockRecordFile struct {
|
||||
Hash []byte // Hash of the file data
|
||||
Type uint8 // Type (low-level)
|
||||
Format uint16 // Format (high-level)
|
||||
Size uint64 // Size of the file
|
||||
Directory string // Directory
|
||||
Name string // File name
|
||||
directoryID uint16 // Internal directory ID
|
||||
// Tags todo
|
||||
}
|
||||
|
||||
// Tag structure to be defined
|
||||
|
||||
// decodeBlockRecords decodes the raw records in the block and returns a high-level decoded structure
|
||||
func decodeBlockRecords(block *Block) (decoded *BlockDecoded, err error) {
|
||||
decoded = &BlockDecoded{Block: *block}
|
||||
|
||||
for _, record := range block.RecordsRaw {
|
||||
|
||||
switch record.Type {
|
||||
case RecordTypeUsername:
|
||||
decoded.User.Name = string(record.Data)
|
||||
decoded.User.NameS = sanitize.Username(decoded.User.Name)
|
||||
decoded.User.Valid = true
|
||||
|
||||
case RecordTypeDirectory:
|
||||
if len(record.Data) < 3 {
|
||||
return nil, errors.New("decodeBlockRecord directory record invalid size")
|
||||
}
|
||||
|
||||
directory := &BlockRecordDirectory{}
|
||||
directory.ID = binary.LittleEndian.Uint16(record.Data[0 : 0+2])
|
||||
directory.Name = string(record.Data[2:])
|
||||
decoded.directories = append(decoded.directories, *directory)
|
||||
|
||||
case RecordTypeFile:
|
||||
if len(record.Data) < 49 {
|
||||
return nil, errors.New("decodeBlockRecord file record invalid size")
|
||||
}
|
||||
|
||||
file := BlockRecordFile{}
|
||||
file.Hash = make([]byte, hashSize)
|
||||
copy(file.Hash, record.Data[0:0+hashSize])
|
||||
file.Type = record.Data[32]
|
||||
file.Format = binary.LittleEndian.Uint16(record.Data[33 : 33+2])
|
||||
file.Size = binary.LittleEndian.Uint64(record.Data[35 : 35+8])
|
||||
directoryID := binary.LittleEndian.Uint16(record.Data[43 : 43+2])
|
||||
filenameSize := binary.LittleEndian.Uint16(record.Data[45 : 45+2])
|
||||
//countTags := binary.LittleEndian.Uint16(record.Data[47 : 47+2]) // future implementation of tags
|
||||
|
||||
if len(record.Data) < 49+int(filenameSize) {
|
||||
return nil, errors.New("decodeBlockRecord file record invalid filename size")
|
||||
}
|
||||
file.Name = string(record.Data[49 : 49+filenameSize])
|
||||
|
||||
for n := range decoded.directories {
|
||||
if decoded.directories[n].ID == directoryID {
|
||||
file.Directory = decoded.directories[n].Name
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
decoded.Files = append(decoded.Files, file)
|
||||
}
|
||||
}
|
||||
|
||||
return decoded, nil
|
||||
}
|
||||
|
||||
// encodeBlockRecordUser encodes the username
|
||||
func encodeBlockRecordUser(user BlockRecordUser) (recordsRaw []BlockRecordRaw, err error) {
|
||||
if user.Valid {
|
||||
recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeUsername, Data: []byte(user.Name)})
|
||||
}
|
||||
|
||||
return recordsRaw, nil
|
||||
}
|
||||
|
||||
// encodeBlockRecordFiles encodes files into the block record data
|
||||
// This function should be called grouped with all files in the same directory. The directory name is deduplicated; only unique directory records will be returned.
|
||||
func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRaw, err error) {
|
||||
// First the directory records must be declared for any references by files
|
||||
nextDirectoryID := uint16(1) // start as 1 to prevent collision with files without explicit directory
|
||||
directoryList := make(map[string]int)
|
||||
|
||||
for n := range files {
|
||||
files[n].Directory, files[n].Name = sanitize.Path(files[n].Directory, files[n].Name)
|
||||
|
||||
if files[n].Directory == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if directoryID, ok := directoryList[files[n].Directory]; ok {
|
||||
files[n].directoryID = uint16(directoryID)
|
||||
continue
|
||||
}
|
||||
|
||||
// Create the new directory record
|
||||
var directoryIDb [2]byte
|
||||
binary.LittleEndian.PutUint16(directoryIDb[0:2], nextDirectoryID)
|
||||
files[n].directoryID = nextDirectoryID
|
||||
nextDirectoryID++
|
||||
|
||||
recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeDirectory, Data: append(directoryIDb[:], []byte(files[n].Directory)...)})
|
||||
}
|
||||
|
||||
for n := range files {
|
||||
var data [49]byte
|
||||
|
||||
if len(files[n].Hash) != hashSize {
|
||||
return nil, errors.New("encodeBlockRecords invalid file hash")
|
||||
}
|
||||
copy(data[0:32], files[n].Hash[0:32])
|
||||
|
||||
data[32] = files[n].Type
|
||||
binary.LittleEndian.PutUint16(data[33:33+2], files[n].Format)
|
||||
binary.LittleEndian.PutUint64(data[35:35+8], files[n].Size)
|
||||
binary.LittleEndian.PutUint16(data[43:43+2], files[n].directoryID)
|
||||
|
||||
filenameB := []byte(files[n].Name)
|
||||
binary.LittleEndian.PutUint16(data[45:45+2], uint16(len(filenameB)))
|
||||
binary.LittleEndian.PutUint16(data[47:47+2], uint16(0)) // Count of Tags (future use)
|
||||
|
||||
recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeFile, Data: append(data[:], filenameB...)})
|
||||
}
|
||||
|
||||
return recordsRaw, nil
|
||||
}
|
||||
230
Blockchain.go
230
Blockchain.go
@@ -2,6 +2,31 @@
|
||||
File Name: Blockchain.go
|
||||
Copyright: 2021 Peernet s.r.o.
|
||||
Author: Peter Kleissner
|
||||
|
||||
All blocks and the blockchain header are stored in a key/value database.
|
||||
The key for the blockchain header is keyHeader and for each block is the block number as 64-bit unsigned integer little endian.
|
||||
|
||||
Encoding of the blockchain header:
|
||||
Offset Size Info
|
||||
0 8 Height
|
||||
8 8 Version
|
||||
16 65 Signature
|
||||
|
||||
Encoding of each block (it is the same stored in the database and shared in a message):
|
||||
Offset Size Info
|
||||
0 65 Signature of entire block
|
||||
65 32 Hash (blake3) of last block. 0 for first one.
|
||||
97 8 Blockchain version number
|
||||
105 4 Block number
|
||||
109 4 Size of entire block including this header
|
||||
113 2 Count of records that follow
|
||||
|
||||
Each record inside the block has this basic structure:
|
||||
Offset Size Info
|
||||
0 1 Record Type
|
||||
1 4 Size of data
|
||||
5 ? Data (encoding depends on record type)
|
||||
|
||||
*/
|
||||
|
||||
package core
|
||||
@@ -9,8 +34,9 @@ package core
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"github.com/PeernetOfficial/core/store"
|
||||
"github.com/btcsuite/btcd/btcec"
|
||||
@@ -26,96 +52,174 @@ var BlockchainVersion = uint64(0)
|
||||
const filenameUserBlockchain = "self.blockchain"
|
||||
|
||||
// the key names in the key/value database are constant and must not collide with block numbers (i.e. they must be >64 bit)
|
||||
const keyHeaderHeight = "header.height"
|
||||
const keyHeaderVersion = "header.version"
|
||||
const keyHeaderSignature = "header.signature"
|
||||
const keyHeader = "header blockchain"
|
||||
|
||||
// BlockchainHeader is the virtual blockchain header. The data is not stored as single header record, but split up.
|
||||
type BlockchainHeader struct {
|
||||
Height uint64
|
||||
Version uint64
|
||||
PublicKey *btcec.PublicKey
|
||||
// userBlockchainHeader stores the users blockchain header in memory. Any changes must be synced to disk!
|
||||
var userBlockchainHeader struct {
|
||||
height uint64
|
||||
version uint64
|
||||
publicKey *btcec.PublicKey
|
||||
sync.Mutex
|
||||
}
|
||||
|
||||
var userBlockchainDB store.Store
|
||||
var userBlockchainHeader BlockchainHeader
|
||||
|
||||
// initUserBlockchain initializes the users blockchain. It creates the blockchain file if it does not exist already.
|
||||
// If it is corrupted, it will log the error and exit the process.
|
||||
func initUserBlockchain() {
|
||||
// open existing blockchain file or create new one
|
||||
blockchain, err := store.NewPogrebStore(filenameUserBlockchain)
|
||||
if err != nil {
|
||||
var err error
|
||||
if userBlockchainDB, err = store.NewPogrebStore(filenameUserBlockchain); err != nil {
|
||||
Filters.LogError("initUserBlockchain", "error opening user blockchain: %s\n", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// verify header
|
||||
var height, version uint64
|
||||
heightB, found1 := blockchain.Get([]byte(keyHeaderHeight))
|
||||
versionB, found2 := blockchain.Get([]byte(keyHeaderVersion))
|
||||
headerSignature, isSignature := blockchain.Get([]byte(keyHeaderSignature))
|
||||
|
||||
if found1 && len(heightB) != 8 || found2 && len(versionB) != 8 || isSignature && len(headerSignature) != 65 {
|
||||
Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Invalid header length. Height is '%s', version '%s' and signature '%s'.\n", hex.EncodeToString(heightB), hex.EncodeToString(versionB), hex.EncodeToString(headerSignature))
|
||||
var found bool
|
||||
userBlockchainHeader.publicKey, userBlockchainHeader.height, userBlockchainHeader.version, found, err = blockchainHeaderRead(userBlockchainDB)
|
||||
if err != nil {
|
||||
Filters.LogError("initUserBlockchain", "corrupt user blockchain database: %s\n", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if found1 && len(heightB) == 8 {
|
||||
height = binary.BigEndian.Uint64(heightB)
|
||||
}
|
||||
if found2 && len(heightB) == 8 {
|
||||
version = binary.BigEndian.Uint64(versionB)
|
||||
}
|
||||
|
||||
if isSignature {
|
||||
// validate header signature
|
||||
headerA := strconv.FormatUint(height, 10) + "/" + strconv.FormatUint(version, 10)
|
||||
headerPublicKey, _, err := btcec.RecoverCompact(btcec.S256(), headerSignature[:], hashData([]byte(headerA)))
|
||||
if err != nil {
|
||||
Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Error decoding signature. Height is '%s', version '%s', signature '%s'. %s\n", hex.EncodeToString(heightB), hex.EncodeToString(versionB), hex.EncodeToString(headerSignature), err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
if !headerPublicKey.IsEqual(peerPublicKey) {
|
||||
Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Signature key mismatch. Height is '%s', version '%s', signature '%s'.\n", hex.EncodeToString(heightB), hex.EncodeToString(versionB), hex.EncodeToString(headerSignature))
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
userBlockchainHeader = BlockchainHeader{Height: height, Version: version, PublicKey: headerPublicKey}
|
||||
|
||||
} else if !found1 && !found2 && !isSignature {
|
||||
} else if !found {
|
||||
// First run: create header signature!
|
||||
height = 0
|
||||
version = 0
|
||||
signature, _ := blockchainCreateHeaderSignature(height, version)
|
||||
if err := blockchain.Set([]byte(keyHeaderSignature), signature); err != nil {
|
||||
userBlockchainHeader.height = 0
|
||||
userBlockchainHeader.version = 0
|
||||
if err := blockchainHeaderWrite(userBlockchainDB, peerPrivateKey, userBlockchainHeader.height, userBlockchainHeader.version); err != nil {
|
||||
Filters.LogError("initUserBlockchain", "initializing user blockchain: %s", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
} else {
|
||||
// header signature not present, but height/version is -> corrupt data.
|
||||
Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Height is '%s', version '%s' and signature '%s'.\n", hex.EncodeToString(heightB), hex.EncodeToString(versionB), hex.EncodeToString(headerSignature))
|
||||
} else if !userBlockchainHeader.publicKey.IsEqual(peerPublicKey) {
|
||||
Filters.LogError("initUserBlockchain", "corrupt user blockchain database. Public key mismatch. Height is '%d', version '%d'. Public key expected '%s' vs provided '%s'\n", userBlockchainHeader.height, userBlockchainHeader.version, hex.EncodeToString(peerPublicKey.SerializeCompressed()), hex.EncodeToString(userBlockchainHeader.publicKey.SerializeCompressed()))
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
userBlockchainDB = blockchain
|
||||
}
|
||||
|
||||
func blockchainCreateHeaderSignature(height, version uint64) (signature []byte, err error) {
|
||||
headerA := strconv.FormatUint(height, 10) + "/" + strconv.FormatUint(version, 10)
|
||||
signature, err = btcec.SignCompact(btcec.S256(), peerPrivateKey, hashData([]byte(headerA)), true)
|
||||
|
||||
if err != nil {
|
||||
Filters.LogError("blockchainCreateHeaderSignature", "creating signature of '%s': %s\n", headerA, err.Error())
|
||||
// blockchainHeaderRead reads the header from the blockchain and decodes it.
|
||||
func blockchainHeaderRead(db store.Store) (publicKey *btcec.PublicKey, height, version uint64, found bool, err error) {
|
||||
buffer, found := db.Get([]byte(keyHeader))
|
||||
if !found {
|
||||
return nil, 0, 0, false, nil
|
||||
}
|
||||
|
||||
return signature, err
|
||||
if len(buffer) != 81 {
|
||||
return nil, 0, 0, true, errors.New("blockchain header size mismatch")
|
||||
}
|
||||
|
||||
height = binary.LittleEndian.Uint64(buffer[0:8])
|
||||
version = binary.LittleEndian.Uint64(buffer[8:16])
|
||||
signature := buffer[16 : 16+65]
|
||||
|
||||
publicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, hashData(buffer[0:16]))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// blockchainHeaderWrite writes the header to the blockchain and signs it.
|
||||
func blockchainHeaderWrite(db store.Store, privateKey *btcec.PrivateKey, height, version uint64) (err error) {
|
||||
var buffer [81]byte
|
||||
binary.LittleEndian.PutUint64(buffer[0:8], height)
|
||||
binary.LittleEndian.PutUint64(buffer[8:16], version)
|
||||
|
||||
signature, err := btcec.SignCompact(btcec.S256(), privateKey, hashData(buffer[0:16]), true)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
} else if len(signature) != 65 {
|
||||
return errors.New("signature length invalid")
|
||||
}
|
||||
|
||||
copy(buffer[16:16+65], signature)
|
||||
|
||||
err = db.Set([]byte(keyHeader), buffer[:])
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// UserBlockchainHeader returns the users blockchain header which stores the height and version number.
|
||||
func UserBlockchainHeader() (header BlockchainHeader) {
|
||||
return userBlockchainHeader
|
||||
func UserBlockchainHeader() (publicKey *btcec.PublicKey, height uint64, version uint64) {
|
||||
return userBlockchainHeader.publicKey, userBlockchainHeader.height, userBlockchainHeader.version
|
||||
}
|
||||
|
||||
// ---- low-level blockchain manipulation functions ----
|
||||
|
||||
// UserBlockchainAppend appends a new block to the blockchain based on the provided raw records.
|
||||
// Status: 0 = Success, 1 = Error previous block not found, 2 = Error block encoding
|
||||
func UserBlockchainAppend(RecordsRaw []BlockRecordRaw) (newHeight uint64, status int) {
|
||||
userBlockchainHeader.Lock()
|
||||
defer userBlockchainHeader.Unlock()
|
||||
|
||||
block := &Block{OwnerPublicKey: peerPublicKey, RecordsRaw: RecordsRaw}
|
||||
|
||||
// set the last block hash first
|
||||
if userBlockchainHeader.height > 0 {
|
||||
var target [8]byte
|
||||
binary.LittleEndian.PutUint64(target[:], userBlockchainHeader.height-1)
|
||||
previousBlockRaw, found := userBlockchainDB.Get(target[:])
|
||||
if !found || len(previousBlockRaw) == 0 {
|
||||
return 0, 1
|
||||
}
|
||||
|
||||
block.LastBlockHash = hashData(previousBlockRaw)
|
||||
}
|
||||
|
||||
block.Number = userBlockchainHeader.height
|
||||
block.BlockchainVersion = userBlockchainHeader.version
|
||||
|
||||
raw, err := encodeBlock(block, peerPrivateKey)
|
||||
if err != nil {
|
||||
return 0, 2
|
||||
}
|
||||
|
||||
// increase blockchain height
|
||||
userBlockchainHeader.height++
|
||||
|
||||
// store the block
|
||||
var numberB [8]byte
|
||||
binary.LittleEndian.PutUint64(numberB[:], block.Number)
|
||||
userBlockchainDB.Set(numberB[:], raw)
|
||||
|
||||
// update the blockchain header in the database
|
||||
blockchainHeaderWrite(userBlockchainDB, peerPrivateKey, userBlockchainHeader.height, userBlockchainHeader.version)
|
||||
|
||||
return userBlockchainHeader.height, 0
|
||||
}
|
||||
|
||||
// UserBlockchainRead reads the block number from the blockchain.
|
||||
// Status: 0 = Success, 1 = Error block not found, 2 = Error block encoding, 3 = Error block record encoding
|
||||
// Errors 2 and 3 indicate data corruption.
|
||||
func UserBlockchainRead(number uint64) (decoded *BlockDecoded, status int) {
|
||||
if number >= userBlockchainHeader.height {
|
||||
return nil, 1
|
||||
}
|
||||
|
||||
var target [8]byte
|
||||
binary.LittleEndian.PutUint64(target[:], userBlockchainHeader.height-1)
|
||||
blockRaw, found := userBlockchainDB.Get(target[:])
|
||||
if !found || len(blockRaw) == 0 {
|
||||
return nil, 1
|
||||
}
|
||||
|
||||
block, err := decodeBlock(blockRaw)
|
||||
if err != nil {
|
||||
return nil, 2
|
||||
}
|
||||
|
||||
decoded, err = decodeBlockRecords(block)
|
||||
if err != nil {
|
||||
return nil, 2
|
||||
}
|
||||
|
||||
return decoded, 0
|
||||
}
|
||||
|
||||
// UserBlockchainAddFiles adds files to the blockchain
|
||||
// Status: 0 = Success, 1 = Error previous block not found, 2 = Error block encoding, 3 = Error block record encoding
|
||||
// It makes sense to group all files in the same directory into one call, since only one directory record will be created per unique directory per block.
|
||||
func UserBlockchainAddFiles(files []BlockRecordFile) (newHeight uint64, status int) {
|
||||
encoded, err := encodeBlockRecordFiles(files)
|
||||
if err != nil {
|
||||
return 0, 3
|
||||
}
|
||||
|
||||
return UserBlockchainAppend(encoded)
|
||||
}
|
||||
|
||||
92
Test_test.go
92
Test_test.go
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/btcsuite/btcd/btcec"
|
||||
)
|
||||
|
||||
func TestMessageEncodingAnnouncement(t *testing.T) {
|
||||
@@ -89,16 +91,26 @@ func TestBlockEncoding(t *testing.T) {
|
||||
}
|
||||
|
||||
file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, Name: "Filename 1.txt", Directory: "documents\\sub folder"}
|
||||
encoded1, _ := encodeBlockRecordUser(BlockRecordUser{Valid: true, Name: "Test User 1"})
|
||||
encoded2, _ := encodeBlockRecordFiles([]BlockRecordFile{file1})
|
||||
|
||||
block := &Block{BlockchainVersion: 42, Number: 0, User: BlockRecordUser{Valid: true, Name: "Test User 1"}, Files: []BlockRecordFile{file1}}
|
||||
blockE := &Block{BlockchainVersion: 42, Number: 0}
|
||||
blockE.RecordsRaw = append(blockE.RecordsRaw, encoded1...)
|
||||
blockE.RecordsRaw = append(blockE.RecordsRaw, encoded2...)
|
||||
|
||||
raw, err := encodeBlock(block, privateKey)
|
||||
raw, err := encodeBlock(blockE, privateKey)
|
||||
if err != nil {
|
||||
fmt.Printf("Error: %s\n", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
block, err = decodeBlock(raw)
|
||||
block, err := decodeBlock(raw)
|
||||
if err != nil {
|
||||
fmt.Printf("Error: %s\n", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
decoded, err := decodeBlockRecords(block)
|
||||
if err != nil {
|
||||
fmt.Printf("Error: %s\n", err.Error())
|
||||
return
|
||||
@@ -107,7 +119,79 @@ func TestBlockEncoding(t *testing.T) {
|
||||
// output the block details
|
||||
fmt.Printf("Block details:\n----------------\nNumber: %d\nVersion: %d\nLast Hash: %s\nPublic Key: %s\n", block.Number, block.BlockchainVersion, hex.EncodeToString(block.LastBlockHash), hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed()))
|
||||
|
||||
for _, file := range block.Files {
|
||||
for _, file := range decoded.Files {
|
||||
fmt.Printf("* File %s\n", file.Name)
|
||||
fmt.Printf(" Directory %s\n", file.Directory)
|
||||
fmt.Printf(" Size %d\n", file.Size)
|
||||
fmt.Printf(" Type %d\n", file.Type)
|
||||
fmt.Printf(" Format %d\n", file.Format)
|
||||
fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash))
|
||||
fmt.Printf(" Directory ID %d\n\n", file.directoryID)
|
||||
}
|
||||
}
|
||||
|
||||
func initTestPrivateKey() {
|
||||
// use static test key, otherwise tests will be inconsistent (would otherwise fail to open blockchain database)
|
||||
privateKeyTestA := "d65da474861d826edd29c1307f1250d79e9dbf84e3a2449022658445c8d8ed63"
|
||||
privateKeyB, _ := hex.DecodeString(privateKeyTestA)
|
||||
peerPrivateKey, peerPublicKey = btcec.PrivKeyFromBytes(btcec.S256(), privateKeyB)
|
||||
nodeID = PublicKey2NodeID(peerPublicKey)
|
||||
|
||||
fmt.Printf("Loaded public key: %s\n", hex.EncodeToString(peerPublicKey.SerializeCompressed()))
|
||||
}
|
||||
|
||||
func TestBlockchainAdd(t *testing.T) {
|
||||
initTestPrivateKey()
|
||||
initUserBlockchain()
|
||||
|
||||
file1 := BlockRecordFile{Hash: hashData([]byte("Test data")), Type: TypeText, Format: FormatText, Size: 9, Name: "Filename 1.txt", Directory: "documents\\sub folder"}
|
||||
|
||||
newHeight, status := UserBlockchainAddFiles([]BlockRecordFile{file1})
|
||||
|
||||
switch status {
|
||||
case 0:
|
||||
case 1: // Error previous block not found
|
||||
fmt.Printf("Error adding file to blockchain: Previous block not found.\n")
|
||||
case 2: // Error block encoding
|
||||
fmt.Printf("Error adding file to blockchain: Error block encoding.\n")
|
||||
case 3: // Error block record encoding
|
||||
fmt.Printf("Error adding file to blockchain: Error block record encoding.\n")
|
||||
default:
|
||||
fmt.Printf("Error adding file to blockchain: Unknown status %d\n", status)
|
||||
}
|
||||
|
||||
if status != 0 {
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Printf("Success adding files to blockchain. New blockchain height: %d\n", newHeight)
|
||||
}
|
||||
|
||||
func TestBlockchainRead(t *testing.T) {
|
||||
initTestPrivateKey()
|
||||
initFilters()
|
||||
initUserBlockchain()
|
||||
|
||||
blockNumber := uint64(0)
|
||||
|
||||
decoded, status := UserBlockchainRead(blockNumber)
|
||||
switch status {
|
||||
case 0:
|
||||
case 1: // Error block not found
|
||||
fmt.Printf("Error reading block %d: Block not found.\n", blockNumber)
|
||||
case 2: // Error block encoding
|
||||
fmt.Printf("Error reading block %d: Block encoding corrupt.\n", blockNumber)
|
||||
case 3: // Error block record encoding
|
||||
fmt.Printf("Error reading block %d: Block record encoding corrupt.\n", blockNumber)
|
||||
default:
|
||||
fmt.Printf("Error reading block %d: Unknown status %d\n", blockNumber, status)
|
||||
}
|
||||
|
||||
if status != 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, file := range decoded.Files {
|
||||
fmt.Printf("* File %s\n", file.Name)
|
||||
fmt.Printf(" Directory %s\n", file.Directory)
|
||||
fmt.Printf(" Size %d\n", file.Size)
|
||||
|
||||
52
sanitize/Sanitize.go
Normal file
52
sanitize/Sanitize.go
Normal file
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
File Name: Sanitize.go
|
||||
Copyright: 2021 Peernet s.r.o.
|
||||
Author: Peter Kleissner
|
||||
*/
|
||||
|
||||
package sanitize
|
||||
|
||||
import (
|
||||
"path"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const PATH_MAX_LENGTH = 32767 // Windows Maximum Path Length for UNC paths
|
||||
|
||||
// Path sanitizies the directory and filename.
|
||||
func Path(directory, filename string) (string, string) {
|
||||
// Enforced forward slashes as directory separator and clean the path.
|
||||
directory = strings.ReplaceAll(directory, "\\", "/")
|
||||
directory = path.Clean(directory)
|
||||
|
||||
// No slash at the beginning and end to save space.
|
||||
directory = strings.Trim(directory, "/")
|
||||
|
||||
// Slashes in filenames are not encouraged, but not removed.
|
||||
|
||||
// Enforce max filename length.
|
||||
if len(filename) > PATH_MAX_LENGTH {
|
||||
filename = filename[:PATH_MAX_LENGTH]
|
||||
}
|
||||
|
||||
return directory, filename
|
||||
}
|
||||
|
||||
// Username sanitizes the username.
|
||||
func Username(input string) string {
|
||||
if !utf8.ValidString(input) {
|
||||
return "<invalid encoding>"
|
||||
}
|
||||
|
||||
input = strings.TrimSpace(input)
|
||||
input = strings.ReplaceAll(input, "\n", " ")
|
||||
input = strings.ReplaceAll(input, "\r", "")
|
||||
|
||||
// Max length for sanitized version is 36, resembling the limit from StackOverflow.
|
||||
if len(input) > 36 {
|
||||
input = input[:36]
|
||||
}
|
||||
|
||||
return input
|
||||
}
|
||||
Reference in New Issue
Block a user