Warehouse: Automatically create .merkle companion files to store the entire merkle tree. Close #48

Pregenerating this means that fragments can be serve immediately with 0 hash calculation needed.

merkle/Merkle Tree.go

@@ -198,7 +198,7 @@ Offset  Size        Info
 
 */
 
-const merkleTreeFileHeaderSize = 8 + 8 + 32
+const MerkleTreeFileHeaderSize = 8 + 8 + 32
 
 // calculateTotalHashCount returns the total number of fragment and middle hashes needed for the given count of fragments
 func calculateTotalHashCount(fragmentCount uint64) (count uint64) {
@@ -226,7 +226,7 @@ func calculateTotalHashCount(fragmentCount uint64) (count uint64) {
 
 // Export stores the tree as blob
 func (tree *MerkleTree) Export() (data []byte) {
-	data = make([]byte, merkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32)
+	data = make([]byte, MerkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32)
 
 	// header
 	binary.LittleEndian.PutUint64(data[0:8], tree.FileSize)
@@ -258,7 +258,7 @@ func ImportMerkleTree(data []byte) (tree *MerkleTree) {
 	}
 
 	// verify size
-	if uint64(len(data)) < merkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32 {
+	if uint64(len(data)) < MerkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32 {
 		return nil
 	}
 
@@ -296,7 +296,7 @@ func ImportMerkleTree(data []byte) (tree *MerkleTree) {
 // ReadMerkleTreeHeader reads the merkle tree header. Fragment and middle hashes are not loaded.
 func ReadMerkleTreeHeader(data []byte) (tree *MerkleTree) {
 	// Read the header. Enforce the minimum size.
-	if len(data) < 8+8+32 {
+	if len(data) < MerkleTreeFileHeaderSize {
 		return nil
 	}
 

warehouse/Merkle.go

@@ -0,0 +1,120 @@
+/*
+File Name:  Merkle.go
+Copyright:  2021 Peernet Foundation s.r.o.
+Author:     Peter Kleissner
+*/
+
+package warehouse
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/PeernetOfficial/core/merkle"
+)
+
+// Merkle companion files are created to store the entire merkle tree for files that are bigger than the minimum fragment size.
+const merkleCompanionExt = ".merkle"
+
+// MerkleFileExists checks if the merkle companion file exists. It returns StatusInvalidHash, StatusFileNotFound, or StatusOK.
+func (wh *Warehouse) MerkleFileExists(hash []byte) (path string, fileInfo os.FileInfo, status int, err error) {
+	hashA, err := ValidateHash(hash)
+	if err != nil {
+		return "", nil, StatusInvalidHash, err
+	}
+
+	a, b := buildPath(wh.Directory, hashA)
+	path = filepath.Join(a, b) + merkleCompanionExt
+
+	if fileInfo, err := os.Stat(path); err == nil {
+		// file exists
+		return path, fileInfo, StatusOK, nil
+	}
+
+	return "", nil, StatusFileNotFound, os.ErrNotExist
+}
+
+// createMerkleCompanionFile creates a merkle companion file. If the merkle companion file already exists, it is overwritten.
+// dataFilePath is the full path to the data file in the warehouse.
+func (wh *Warehouse) createMerkleCompanionFile(dataFilePath string) (status int, err error) {
+	// open the data file
+	dataFile, err := os.Open(dataFilePath)
+	if err != nil && os.IsNotExist(err) {
+		return StatusFileNotFound, err
+	} else if err != nil {
+		return StatusErrorOpenFile, err
+	}
+	defer dataFile.Close()
+
+	var fileSize uint64
+	stat, err := dataFile.Stat()
+	if err != nil {
+		return StatusErrorOpenFile, err
+	}
+	fileSize = uint64(stat.Size())
+
+	// Merkle files are only created if merkle trees are actually used, which means the file must be bigger than the minimum fragment size.
+	// Otherwise the merkle root hash will be just the file hash and the merkle overhead provides no advantage whatsoever.
+	if fileSize <= merkle.MinimumFragmentSize {
+		return StatusOK, nil
+	}
+
+	// Create a new merkle file. If one exists, overwrite.
+	merkleFile := dataFilePath + merkleCompanionExt
+
+	fileM, err := os.OpenFile(merkleFile, os.O_WRONLY|os.O_CREATE, 0666) // 666 = All uses can read/write
+	if err != nil {
+		return StatusErrorCreateTarget, err
+	}
+	defer fileM.Close()
+
+	// create the merkle tree and write it to the companion file
+	fragmentSize := merkle.CalculateFragmentSize(fileSize)
+	tree, err := merkle.NewMerkleTree(fileSize, fragmentSize, dataFile)
+	if err != nil {
+		return StatusErrorCreateMerkle, err
+	}
+
+	fileM.Write(tree.Export())
+
+	return StatusOK, nil
+}
+
+// ReadMerkleTree reads the merkle tree from the companion file associated with the hash.
+// It is the callers responsibility to first check if a merkle tree file is to be expected (files smaller or equal than the minimum fragment size do not use a merkle tree).
+func (wh *Warehouse) ReadMerkleTree(hash []byte, headerOnly bool) (tree *merkle.MerkleTree, status int, err error) {
+	path, _, status, err := wh.MerkleFileExists(hash)
+	if status != StatusOK { // file does not exist or invalid hash
+		return nil, status, err
+	}
+
+	fileM, err := os.OpenFile(path, os.O_RDONLY, 0)
+	if err != nil {
+		return nil, StatusErrorOpenFile, err
+	}
+	defer fileM.Close()
+
+	if headerOnly {
+		data := make([]byte, merkle.MerkleTreeFileHeaderSize)
+		if _, err = io.ReadFull(fileM, data); err != nil {
+			return nil, StatusErrorReadFile, err
+		}
+
+		if tree = merkle.ReadMerkleTreeHeader(data); tree == nil {
+			return nil, StatusErrorMerkleTreeFile, errors.New("invalid merkle tree file header")
+		}
+	} else {
+		data, err := io.ReadAll(fileM)
+		if err != nil {
+			return nil, StatusErrorReadFile, err
+		}
+
+		if tree = merkle.ImportMerkleTree(data); tree == nil {
+			return nil, StatusErrorMerkleTreeFile, errors.New("invalid merkle tree file header")
+		}
+	}
+
+	return tree, StatusOK, nil
+}

warehouse/Store.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/PeernetOfficial/core/merkle"
 	"lukechampine.com/blake3"
 )
 
@@ -31,10 +32,13 @@ const (
 	StatusErrorSeekFile       = 12 // Error seeking to position in file.
 	StatusErrorTargetExists   = 13 // Target file already exists.
 	StatusErrorCreateTarget   = 14 // Error creating target file.
+	StatusErrorCreateMerkle   = 15 // Error creating merkle tree.
+	StatusErrorMerkleTreeFile = 16 // Invalid merkle tree companion file.
 )
 
 // CreateFile creates a new file in the warehouse
-func (wh *Warehouse) CreateFile(data io.Reader) (hash []byte, status int, err error) {
+// If fileSize is provided, creating the merkle tree is significantly faster as it will be created on the fly. If the file size is unknown, set the size to 0.
+func (wh *Warehouse) CreateFile(data io.Reader, fileSize uint64) (hash []byte, status int, err error) {
 	// create a temporary file to hold the body content
 	tmpFile, err := wh.tempFile()
 	if err != nil {
@@ -43,6 +47,11 @@ func (wh *Warehouse) CreateFile(data io.Reader) (hash []byte, status int, err er
 
 	tmpFileName := tmpFile.Name()
 
+	// create merkle tree in parallel if the file size is known (which means the fragment size can be calculated)
+	if fileSize > 0 {
+		// TODO
+	}
+
 	// create the hash-writer
 	hashWriter := blake3.New(hashSize, nil)
 
@@ -93,6 +102,13 @@ func (wh *Warehouse) CreateFile(data io.Reader) (hash []byte, status int, err er
 				return nil, StatusErrorRenameTempFile, err
 			}
 		}
+
+		// create the merkle tree companion file
+		if fileSize == 0 || fileSize > merkle.MinimumFragmentSize {
+			if status, err = wh.createMerkleCompanionFile(pathFull); status != StatusOK {
+				return hash, status, err
+			}
+		}
 	}
 
 	return hash, StatusOK, nil
@@ -108,11 +124,15 @@ func (wh *Warehouse) CreateFileFromPath(file string) (hash []byte, status int, e
 		// cannot open file
 		return nil, StatusErrorOpenFile, err
 	}
-
 	defer fileHandle.Close()
 
+	var fileSize uint64
+	if stat, err := fileHandle.Stat(); err == nil {
+		fileSize = uint64(stat.Size())
+	}
+
 	// create the file using the opened file
-	return wh.CreateFile(fileHandle)
+	return wh.CreateFile(fileHandle, fileSize)
 }
 
 // ReadFile reads a file from the warehouse and outputs it to the writer
@@ -218,7 +238,7 @@ func (wh *Warehouse) ReadFileToDisk(hash []byte, offset, limit int64, fileTarget
 	}
 
 	// create the target file
-	fileT, err := os.OpenFile(fileTarget, os.O_WRONLY|os.O_CREATE, 0644)
+	fileT, err := os.OpenFile(fileTarget, os.O_WRONLY|os.O_CREATE, 0666) // 666 = All uses can read/write
 	if err != nil {
 		return StatusErrorCreateTarget, 0, err
 	}