From 05a31cf3afe4c91670894c2a1fe81490e5fd67c5 Mon Sep 17 00:00:00 2001 From: Kleissner Date: Sun, 28 Nov 2021 21:52:30 +0100 Subject: [PATCH] Warehouse: Automatically create .merkle companion files to store the entire merkle tree. Close #48 Pregenerating this means that fragments can be serve immediately with 0 hash calculation needed. --- merkle/Merkle Tree.go | 8 +-- warehouse/Merkle.go | 120 ++++++++++++++++++++++++++++++++++++++++++ warehouse/Store.go | 28 ++++++++-- 3 files changed, 148 insertions(+), 8 deletions(-) create mode 100644 warehouse/Merkle.go diff --git a/merkle/Merkle Tree.go b/merkle/Merkle Tree.go index 479d3e5..c5dd74b 100644 --- a/merkle/Merkle Tree.go +++ b/merkle/Merkle Tree.go @@ -198,7 +198,7 @@ Offset Size Info */ -const merkleTreeFileHeaderSize = 8 + 8 + 32 +const MerkleTreeFileHeaderSize = 8 + 8 + 32 // calculateTotalHashCount returns the total number of fragment and middle hashes needed for the given count of fragments func calculateTotalHashCount(fragmentCount uint64) (count uint64) { @@ -226,7 +226,7 @@ func calculateTotalHashCount(fragmentCount uint64) (count uint64) { // Export stores the tree as blob func (tree *MerkleTree) Export() (data []byte) { - data = make([]byte, merkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32) + data = make([]byte, MerkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32) // header binary.LittleEndian.PutUint64(data[0:8], tree.FileSize) @@ -258,7 +258,7 @@ func ImportMerkleTree(data []byte) (tree *MerkleTree) { } // verify size - if uint64(len(data)) < merkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32 { + if uint64(len(data)) < MerkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32 { return nil } @@ -296,7 +296,7 @@ func ImportMerkleTree(data []byte) (tree *MerkleTree) { // ReadMerkleTreeHeader reads the merkle tree header. Fragment and middle hashes are not loaded. func ReadMerkleTreeHeader(data []byte) (tree *MerkleTree) { // Read the header. Enforce the minimum size. - if len(data) < 8+8+32 { + if len(data) < MerkleTreeFileHeaderSize { return nil } diff --git a/warehouse/Merkle.go b/warehouse/Merkle.go new file mode 100644 index 0000000..4d1f9da --- /dev/null +++ b/warehouse/Merkle.go @@ -0,0 +1,120 @@ +/* +File Name: Merkle.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package warehouse + +import ( + "errors" + "io" + "os" + "path/filepath" + + "github.com/PeernetOfficial/core/merkle" +) + +// Merkle companion files are created to store the entire merkle tree for files that are bigger than the minimum fragment size. +const merkleCompanionExt = ".merkle" + +// MerkleFileExists checks if the merkle companion file exists. It returns StatusInvalidHash, StatusFileNotFound, or StatusOK. +func (wh *Warehouse) MerkleFileExists(hash []byte) (path string, fileInfo os.FileInfo, status int, err error) { + hashA, err := ValidateHash(hash) + if err != nil { + return "", nil, StatusInvalidHash, err + } + + a, b := buildPath(wh.Directory, hashA) + path = filepath.Join(a, b) + merkleCompanionExt + + if fileInfo, err := os.Stat(path); err == nil { + // file exists + return path, fileInfo, StatusOK, nil + } + + return "", nil, StatusFileNotFound, os.ErrNotExist +} + +// createMerkleCompanionFile creates a merkle companion file. If the merkle companion file already exists, it is overwritten. +// dataFilePath is the full path to the data file in the warehouse. +func (wh *Warehouse) createMerkleCompanionFile(dataFilePath string) (status int, err error) { + // open the data file + dataFile, err := os.Open(dataFilePath) + if err != nil && os.IsNotExist(err) { + return StatusFileNotFound, err + } else if err != nil { + return StatusErrorOpenFile, err + } + defer dataFile.Close() + + var fileSize uint64 + stat, err := dataFile.Stat() + if err != nil { + return StatusErrorOpenFile, err + } + fileSize = uint64(stat.Size()) + + // Merkle files are only created if merkle trees are actually used, which means the file must be bigger than the minimum fragment size. + // Otherwise the merkle root hash will be just the file hash and the merkle overhead provides no advantage whatsoever. + if fileSize <= merkle.MinimumFragmentSize { + return StatusOK, nil + } + + // Create a new merkle file. If one exists, overwrite. + merkleFile := dataFilePath + merkleCompanionExt + + fileM, err := os.OpenFile(merkleFile, os.O_WRONLY|os.O_CREATE, 0666) // 666 = All uses can read/write + if err != nil { + return StatusErrorCreateTarget, err + } + defer fileM.Close() + + // create the merkle tree and write it to the companion file + fragmentSize := merkle.CalculateFragmentSize(fileSize) + tree, err := merkle.NewMerkleTree(fileSize, fragmentSize, dataFile) + if err != nil { + return StatusErrorCreateMerkle, err + } + + fileM.Write(tree.Export()) + + return StatusOK, nil +} + +// ReadMerkleTree reads the merkle tree from the companion file associated with the hash. +// It is the callers responsibility to first check if a merkle tree file is to be expected (files smaller or equal than the minimum fragment size do not use a merkle tree). +func (wh *Warehouse) ReadMerkleTree(hash []byte, headerOnly bool) (tree *merkle.MerkleTree, status int, err error) { + path, _, status, err := wh.MerkleFileExists(hash) + if status != StatusOK { // file does not exist or invalid hash + return nil, status, err + } + + fileM, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return nil, StatusErrorOpenFile, err + } + defer fileM.Close() + + if headerOnly { + data := make([]byte, merkle.MerkleTreeFileHeaderSize) + if _, err = io.ReadFull(fileM, data); err != nil { + return nil, StatusErrorReadFile, err + } + + if tree = merkle.ReadMerkleTreeHeader(data); tree == nil { + return nil, StatusErrorMerkleTreeFile, errors.New("invalid merkle tree file header") + } + } else { + data, err := io.ReadAll(fileM) + if err != nil { + return nil, StatusErrorReadFile, err + } + + if tree = merkle.ImportMerkleTree(data); tree == nil { + return nil, StatusErrorMerkleTreeFile, errors.New("invalid merkle tree file header") + } + } + + return tree, StatusOK, nil +} diff --git a/warehouse/Store.go b/warehouse/Store.go index 3f55106..61a1c15 100644 --- a/warehouse/Store.go +++ b/warehouse/Store.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/PeernetOfficial/core/merkle" "lukechampine.com/blake3" ) @@ -31,10 +32,13 @@ const ( StatusErrorSeekFile = 12 // Error seeking to position in file. StatusErrorTargetExists = 13 // Target file already exists. StatusErrorCreateTarget = 14 // Error creating target file. + StatusErrorCreateMerkle = 15 // Error creating merkle tree. + StatusErrorMerkleTreeFile = 16 // Invalid merkle tree companion file. ) // CreateFile creates a new file in the warehouse -func (wh *Warehouse) CreateFile(data io.Reader) (hash []byte, status int, err error) { +// If fileSize is provided, creating the merkle tree is significantly faster as it will be created on the fly. If the file size is unknown, set the size to 0. +func (wh *Warehouse) CreateFile(data io.Reader, fileSize uint64) (hash []byte, status int, err error) { // create a temporary file to hold the body content tmpFile, err := wh.tempFile() if err != nil { @@ -43,6 +47,11 @@ func (wh *Warehouse) CreateFile(data io.Reader) (hash []byte, status int, err er tmpFileName := tmpFile.Name() + // create merkle tree in parallel if the file size is known (which means the fragment size can be calculated) + if fileSize > 0 { + // TODO + } + // create the hash-writer hashWriter := blake3.New(hashSize, nil) @@ -93,6 +102,13 @@ func (wh *Warehouse) CreateFile(data io.Reader) (hash []byte, status int, err er return nil, StatusErrorRenameTempFile, err } } + + // create the merkle tree companion file + if fileSize == 0 || fileSize > merkle.MinimumFragmentSize { + if status, err = wh.createMerkleCompanionFile(pathFull); status != StatusOK { + return hash, status, err + } + } } return hash, StatusOK, nil @@ -108,11 +124,15 @@ func (wh *Warehouse) CreateFileFromPath(file string) (hash []byte, status int, e // cannot open file return nil, StatusErrorOpenFile, err } - defer fileHandle.Close() + var fileSize uint64 + if stat, err := fileHandle.Stat(); err == nil { + fileSize = uint64(stat.Size()) + } + // create the file using the opened file - return wh.CreateFile(fileHandle) + return wh.CreateFile(fileHandle, fileSize) } // ReadFile reads a file from the warehouse and outputs it to the writer @@ -218,7 +238,7 @@ func (wh *Warehouse) ReadFileToDisk(hash []byte, offset, limit int64, fileTarget } // create the target file - fileT, err := os.OpenFile(fileTarget, os.O_WRONLY|os.O_CREATE, 0644) + fileT, err := os.OpenFile(fileTarget, os.O_WRONLY|os.O_CREATE, 0666) // 666 = All uses can read/write if err != nil { return StatusErrorCreateTarget, 0, err }