From 335f83d79b6c89953e3c3253e1bf6334ca8fee74 Mon Sep 17 00:00:00 2001 From: Kleissner Date: Wed, 13 Oct 2021 02:42:04 +0200 Subject: [PATCH] /blockchain/self/add/file checks now if the file exists in the Warehouse and aborts with StatusCorruptBlockRecord if not. --- blockchain/Blockchain.go | 5 +++-- store/readme.md | 7 +++++++ warehouse/Store.go | 4 ++-- warehouse/Warehouse.go | 2 +- webapi/File.go | 22 ++++++++++++++++++++++ webapi/readme.md | 9 ++++++++- 6 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 store/readme.md diff --git a/blockchain/Blockchain.go b/blockchain/Blockchain.go index bb696c5..e644c68 100644 --- a/blockchain/Blockchain.go +++ b/blockchain/Blockchain.go @@ -3,7 +3,7 @@ File Name: Blockchain.go Copyright: 2021 Peernet s.r.o. Author: Peter Kleissner -All blocks and the blockchain header are stored in a key/value database. +All blocks and the blockchain header are stored in a key-value database. The key for the blockchain header is keyHeader and for each block is the block number as 64-bit unsigned integer little endian. Encoding of the blockchain header: @@ -81,7 +81,7 @@ func Init(privateKey *btcec.PrivateKey, path string) (blockchain *Blockchain, er return blockchain, nil } -// the key names in the key/value database are constant and must not collide with block numbers (i.e. they must be >64 bit) +// the key names in the key-value database are constant and must not collide with block numbers (i.e. they must be >64 bit) const keyHeader = "header blockchain" // headerRead reads the header from the blockchain and decodes it. @@ -141,6 +141,7 @@ const ( BlockchainStatusCorruptBlock = 2 // Error block encoding BlockchainStatusCorruptBlockRecord = 3 // Error block record encoding BlockchainStatusDataNotFound = 4 // Requested data not available in the blockchain + BlockchainStatusNotInWarehouse = 5 // File to be added to blockchain does not exist in the Warehouse ) // blockNumberToKey returns the database key for the given block number diff --git a/store/readme.md b/store/readme.md new file mode 100644 index 0000000..2c6744d --- /dev/null +++ b/store/readme.md @@ -0,0 +1,7 @@ +# Key-value Store + +This package provides a wrapper for a simple key-value store. The underlying database may be changed later. + +Tested key-value packages: +* Pebble: Has many dependencies and increases the binary file size by ~6 MB. +* Pogreb: Currently used. Limited to 4 billion records due to 32-bit uint used as index. diff --git a/warehouse/Store.go b/warehouse/Store.go index eec3b2e..33187e2 100644 --- a/warehouse/Store.go +++ b/warehouse/Store.go @@ -117,7 +117,7 @@ func (wh *Warehouse) CreateFileFromPath(file string) (hash []byte, status int, e // ReadFile reads a file from the warehouse and outputs it to the writer // Offset is the position in the file to start reading. Limit (0 = not used) defines how many bytes to read starting at the offset. func (wh *Warehouse) ReadFile(hash []byte, offset, limit int64, writer io.Writer) (status int, err error) { - hashA, err := validateHash(hash) + hashA, err := ValidateHash(hash) if err != nil { return StatusInvalidHash, err } @@ -174,7 +174,7 @@ retryOpenFile: // DeleteFile deletes a file from the warehouse func (wh *Warehouse) DeleteFile(hash []byte) (status int, err error) { - hashA, err := validateHash(hash) + hashA, err := ValidateHash(hash) if err != nil { return StatusInvalidHash, err } diff --git a/warehouse/Warehouse.go b/warehouse/Warehouse.go index 44fd55f..e5b765f 100644 --- a/warehouse/Warehouse.go +++ b/warehouse/Warehouse.go @@ -70,7 +70,7 @@ func (wh *Warehouse) FileExists(hash string) (path string, fileInfo os.FileInfo, // ---- hash functions ---- -func validateHash(hash []byte) (hashA string, err error) { +func ValidateHash(hash []byte) (hashA string, err error) { if len(hash) != hashSize { return "", os.ErrInvalid } diff --git a/webapi/File.go b/webapi/File.go index 1bdcf58..be1b7bb 100644 --- a/webapi/File.go +++ b/webapi/File.go @@ -12,6 +12,7 @@ import ( "github.com/PeernetOfficial/core" "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/warehouse" "github.com/google/uuid" ) @@ -120,9 +121,14 @@ type apiBlockAddFiles struct { /* apiBlockchainSelfAddFile adds a file with the provided information to the blockchain. +Each file must be already stored in the Warehouse (virtual folders are exempt). +If any file is not stored in the Warehouse, the function aborts with the status code StatusNotInWarehouse. +If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord. +In case the function aborts, the blockchain remains unchanged. Request: POST /blockchain/self/add/file with JSON structure apiBlockAddFiles Response: 200 with JSON structure apiBlockchainBlockStatus + 400 if invalid input */ func apiBlockchainSelfAddFile(w http.ResponseWriter, r *http.Request) { var input apiBlockAddFiles @@ -137,6 +143,17 @@ func apiBlockchainSelfAddFile(w http.ResponseWriter, r *http.Request) { file.ID = uuid.New() } + // Verify that the file exists in the warehouse. Folders are exempt from this check as they are only virtual. + if !file.IsVirtualFolder() { + if hashA, err := warehouse.ValidateHash(file.Hash); err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } else if _, _, valid := core.UserWarehouse.FileExists(hashA); !valid { + EncodeJSON(w, r, apiBlockchainBlockStatus{Status: blockchain.BlockchainStatusNotInWarehouse}) + return + } + } + filesAdd = append(filesAdd, blockRecordFileFromAPI(file)) } @@ -209,3 +226,8 @@ func (info *apiFileMetadata) GetNumber() uint64 { return info.Number } + +// IsVirtualFolder returns true if the file is a virtual folder +func (file *apiFile) IsVirtualFolder() bool { + return file.Type == core.TypeFolder && file.Format == core.FormatFolder +} diff --git a/webapi/readme.md b/webapi/readme.md index eea8d9f..c762130 100644 --- a/webapi/readme.md +++ b/webapi/readme.md @@ -115,6 +115,7 @@ const ( BlockchainStatusCorruptBlock = 2 // Error block encoding BlockchainStatusCorruptBlockRecord = 3 // Error block record encoding BlockchainStatusDataNotFound = 4 // Requested data not available in the blockchain + BlockchainStatusNotInWarehouse = 5 // File to be added to blockchain does not exist in the Warehouse ) ``` @@ -235,6 +236,10 @@ This adds a file with the provided information to the blockchain. The date field Any file added is publicly accessible. The user should be informed about this fact in advance. The user is responsible and liable for any files shared. +Each file must be already stored in the Warehouse (virtual folders are exempt). If any file is not stored in the Warehouse, the function aborts with the status code StatusNotInWarehouse. Files can be added to the Warehouse via `/warehouse/create` and `/warehouse/create/path`. + +If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord. In case the function aborts, the blockchain remains unchanged. + ``` Request: POST /blockchain/self/add/file with JSON structure apiBlockAddFiles Response: 200 with JSON structure apiBlockchainBlockStatus @@ -295,6 +300,8 @@ Request: GET /blockchain/self/list/file Response: 200 with JSON structure apiBlockAddFiles ``` +Example request: `http://127.0.0.1:112/blockchain/self/list/file` + Example response: ```json @@ -867,7 +874,7 @@ Example response: The Warehouse stores the actual files that are shared by the user. The blockchain only stores the metadata information. The Warehouse and the blockchain must be kept in sync. * Files are identified (and adressed) by their hash. -* Before using `/blockchain/self/add/file`, you must store the file in the Warehouse using `/warehouse/create` or `/warehouse/create/path`. The blockchain function verifies if the file exists in the Warehouse and fails if it does not. +* Before using `/blockchain/self/add/file`, you must store the file in the Warehouse using `/warehouse/create` or `/warehouse/create/path`. The blockchain add file function verifies if the file exists in the Warehouse and fails if it does not. * When deleting a file from the blockchain via `/blockchain/self/delete/file`, it will automatically delete the file from the warehouse if there are no other files on the blockchain referencing it. * Because files are addressed using their hash, they are automatically deduplicated. If the user shares the exact same file data under different file names, it is only stored once.