/blockchain/self/add/file checks now if the file exists in the Warehouse and aborts with StatusCorruptBlockRecord if not.

blockchain/Blockchain.go

@@ -3,7 +3,7 @@ File Name:  Blockchain.go
 Copyright:  2021 Peernet s.r.o.
 Author:     Peter Kleissner
 
-All blocks and the blockchain header are stored in a key/value database.
+All blocks and the blockchain header are stored in a key-value database.
 The key for the blockchain header is keyHeader and for each block is the block number as 64-bit unsigned integer little endian.
 
 Encoding of the blockchain header:
@@ -81,7 +81,7 @@ func Init(privateKey *btcec.PrivateKey, path string) (blockchain *Blockchain, er
 	return blockchain, nil
 }
 
-// the key names in the key/value database are constant and must not collide with block numbers (i.e. they must be >64 bit)
+// the key names in the key-value database are constant and must not collide with block numbers (i.e. they must be >64 bit)
 const keyHeader = "header blockchain"
 
 // headerRead reads the header from the blockchain and decodes it.
@@ -141,6 +141,7 @@ const (
 	BlockchainStatusCorruptBlock       = 2 // Error block encoding
 	BlockchainStatusCorruptBlockRecord = 3 // Error block record encoding
 	BlockchainStatusDataNotFound       = 4 // Requested data not available in the blockchain
+	BlockchainStatusNotInWarehouse     = 5 // File to be added to blockchain does not exist in the Warehouse
 )
 
 // blockNumberToKey returns the database key for the given block number

store/readme.md

@@ -0,0 +1,7 @@
+# Key-value Store
+
+This package provides a wrapper for a simple key-value store. The underlying database may be changed later.
+
+Tested key-value packages:
+* Pebble: Has many dependencies and increases the binary file size by ~6 MB.
+* Pogreb: Currently used. Limited to 4 billion records due to 32-bit uint used as index.

warehouse/Store.go

@@ -117,7 +117,7 @@ func (wh *Warehouse) CreateFileFromPath(file string) (hash []byte, status int, e
 // ReadFile reads a file from the warehouse and outputs it to the writer
 // Offset is the position in the file to start reading. Limit (0 = not used) defines how many bytes to read starting at the offset.
 func (wh *Warehouse) ReadFile(hash []byte, offset, limit int64, writer io.Writer) (status int, err error) {
-	hashA, err := validateHash(hash)
+	hashA, err := ValidateHash(hash)
 	if err != nil {
 		return StatusInvalidHash, err
 	}
@@ -174,7 +174,7 @@ retryOpenFile:
 
 // DeleteFile deletes a file from the warehouse
 func (wh *Warehouse) DeleteFile(hash []byte) (status int, err error) {
-	hashA, err := validateHash(hash)
+	hashA, err := ValidateHash(hash)
 	if err != nil {
 		return StatusInvalidHash, err
 	}

warehouse/Warehouse.go

@@ -70,7 +70,7 @@ func (wh *Warehouse) FileExists(hash string) (path string, fileInfo os.FileInfo,
 
 // ---- hash functions ----
 
-func validateHash(hash []byte) (hashA string, err error) {
+func ValidateHash(hash []byte) (hashA string, err error) {
 	if len(hash) != hashSize {
 		return "", os.ErrInvalid
 	}

webapi/File.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/PeernetOfficial/core"
 	"github.com/PeernetOfficial/core/blockchain"
+	"github.com/PeernetOfficial/core/warehouse"
 	"github.com/google/uuid"
 )
 
@@ -120,9 +121,14 @@ type apiBlockAddFiles struct {
 
 /*
 apiBlockchainSelfAddFile adds a file with the provided information to the blockchain.
+Each file must be already stored in the Warehouse (virtual folders are exempt).
+If any file is not stored in the Warehouse, the function aborts with the status code StatusNotInWarehouse.
+If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord.
+In case the function aborts, the blockchain remains unchanged.
 
 Request:    POST /blockchain/self/add/file with JSON structure apiBlockAddFiles
 Response:   200 with JSON structure apiBlockchainBlockStatus
+			400 if invalid input
 */
 func apiBlockchainSelfAddFile(w http.ResponseWriter, r *http.Request) {
 	var input apiBlockAddFiles
@@ -137,6 +143,17 @@ func apiBlockchainSelfAddFile(w http.ResponseWriter, r *http.Request) {
 			file.ID = uuid.New()
 		}
 
+		// Verify that the file exists in the warehouse. Folders are exempt from this check as they are only virtual.
+		if !file.IsVirtualFolder() {
+			if hashA, err := warehouse.ValidateHash(file.Hash); err != nil {
+				http.Error(w, "", http.StatusBadRequest)
+				return
+			} else if _, _, valid := core.UserWarehouse.FileExists(hashA); !valid {
+				EncodeJSON(w, r, apiBlockchainBlockStatus{Status: blockchain.BlockchainStatusNotInWarehouse})
+				return
+			}
+		}
+
 		filesAdd = append(filesAdd, blockRecordFileFromAPI(file))
 	}
 
@@ -209,3 +226,8 @@ func (info *apiFileMetadata) GetNumber() uint64 {
 
 	return info.Number
 }
+
+// IsVirtualFolder returns true if the file is a virtual folder
+func (file *apiFile) IsVirtualFolder() bool {
+	return file.Type == core.TypeFolder && file.Format == core.FormatFolder
+}

webapi/readme.md

@@ -115,6 +115,7 @@ const (
     BlockchainStatusCorruptBlock       = 2 // Error block encoding
     BlockchainStatusCorruptBlockRecord = 3 // Error block record encoding
     BlockchainStatusDataNotFound       = 4 // Requested data not available in the blockchain
+	BlockchainStatusNotInWarehouse     = 5 // File to be added to blockchain does not exist in the Warehouse
 )
 ```
 
@@ -235,6 +236,10 @@ This adds a file with the provided information to the blockchain. The date field
 
 Any file added is publicly accessible. The user should be informed about this fact in advance. The user is responsible and liable for any files shared.
 
+Each file must be already stored in the Warehouse (virtual folders are exempt). If any file is not stored in the Warehouse, the function aborts with the status code StatusNotInWarehouse. Files can be added to the Warehouse via `/warehouse/create` and `/warehouse/create/path`.
+
+If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord. In case the function aborts, the blockchain remains unchanged.
+
 ```
 Request:    POST /blockchain/self/add/file with JSON structure apiBlockAddFiles
 Response:   200 with JSON structure apiBlockchainBlockStatus
@@ -295,6 +300,8 @@ Request:    GET /blockchain/self/list/file
 Response:   200 with JSON structure apiBlockAddFiles
 ```
 
+Example request: `http://127.0.0.1:112/blockchain/self/list/file`
+
 Example response:
 
 ```json
@@ -867,7 +874,7 @@ Example response:
 The Warehouse stores the actual files that are shared by the user. The blockchain only stores the metadata information. The Warehouse and the blockchain must be kept in sync.
 
 * Files are identified (and adressed) by their hash.
-* Before using `/blockchain/self/add/file`, you must store the file in the Warehouse using `/warehouse/create` or `/warehouse/create/path`. The blockchain function verifies if the file exists in the Warehouse and fails if it does not.
+* Before using `/blockchain/self/add/file`, you must store the file in the Warehouse using `/warehouse/create` or `/warehouse/create/path`. The blockchain add file function verifies if the file exists in the Warehouse and fails if it does not.
 * When deleting a file from the blockchain via `/blockchain/self/delete/file`, it will automatically delete the file from the warehouse if there are no other files on the blockchain referencing it.
 * Because files are addressed using their hash, they are automatically deduplicated. If the user shares the exact same file data under different file names, it is only stored once.