diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..26f04a5 Binary files /dev/null and b/.DS_Store differ diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/Abstraction.iml b/.idea/Abstraction.iml new file mode 100644 index 0000000..5e764c4 --- /dev/null +++ b/.idea/Abstraction.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/dictionaries b/.idea/dictionaries new file mode 100644 index 0000000..b2e5525 --- /dev/null +++ b/.idea/dictionaries @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..b9ecc2d --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/files.go b/files.go new file mode 100644 index 0000000..cccd6aa --- /dev/null +++ b/files.go @@ -0,0 +1,144 @@ +/* +File Name: abstractions.go +Copyright: 2021 Peernet s.r.o. +Authors: Peter Kleissner, Akilan Selvacoumar +*/ +package Abstrations + +import ( + "encoding/hex" + "errors" + "github.com/PeernetOfficial/Abstraction/webapi" + "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/warehouse" + "github.com/google/uuid" + "path/filepath" + "time" +) + +/* +Library description +to about abstracted function to easily add and remove files. +*/ + +type TouchReturn struct { + BlockchainHeight uint64 + BlockchainVersion uint64 +} + +// Touch abstracted function that creates a file +// and adds the file to the warehouse and +// blockchain +// returns blockchain version and height +func Touch(b *core.Backend, filePath string) (*TouchReturn, error) { + // Creates a File in the warehouse + hash, _, err := b.UserWarehouse.CreateFileFromPath(filePath) + if err != nil { + return nil, err + } + + // Add the File to the local blockchain + var input webapi.ApiBlockAddFiles + var inputFiles []webapi.ApiFile + var inputFile webapi.ApiFile + + // Write File information to the input File + inputFile.Date = time.Now() + // Folder and File name + dir, file := filepath.Split(filePath) + inputFile.Folder = dir + inputFile.Name = file + inputFile.ID = uuid.New() + inputFile.Hash = hash + + // Get the public key of the current node + _, publicKey := b.ExportPrivateKey() + inputFile.NodeID = []byte(hex.EncodeToString(publicKey.SerializeCompressed())) + + inputFiles = append(inputFiles, inputFile) + + input.Files = inputFiles + + var filesAdd []blockchain.BlockRecordFile + + for _, File := range input.Files { + if len(File.Hash) != protocol.HashSize { + return nil, errors.New("bad request") + } + if File.ID == uuid.Nil { // if the ID is not provided by the caller, set it + File.ID = uuid.New() + } + + // Verify that the File exists in the warehouse. Folders are exempt from this check as they are only virtual. + if !File.IsVirtualFolder() { + if _, err := warehouse.ValidateHash(File.Hash); err != nil { + return nil, errors.New("bad request when validating hash") + } else if _, fileInfo, status, _ := b.UserWarehouse.FileExists(File.Hash); status != warehouse.StatusOK { + //EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return nil, errors.New("file not in warehouse") + } else { + File.Size = fileInfo + } + } else { + File.Hash = protocol.HashData(nil) + File.Size = 0 + } + + blockRecord := webapi.BlockRecordFileFromAPI(File) + + // Set the merkle tree info as appropriate. + if !webapi.SetFileMerkleInfo(b, &blockRecord) { + return nil, errors.New("merkle information not set") + } + + filesAdd = append(filesAdd, blockRecord) + } + + newHeight, newVersion, _ := b.UserBlockchain.AddFiles(filesAdd) + + // Creating object for custom return type + var touchReturn TouchReturn + touchReturn.BlockchainHeight = newHeight + touchReturn.BlockchainVersion = newVersion + + return &touchReturn, nil +} + +// Rm abstracted function that removes a file +func Rm(b *core.Backend, hashStr string) error { + // Remove file from warehouse + err := RmWarehouse(b, hashStr) + if err != nil { + return err + } + + return nil +} + +// RmWarehouse abstracted function that removes a file +// from the warehouse +func RmWarehouse(b *core.Backend, hashStr string) error { + // Remove file from warehouse + hash, valid1 := webapi.DecodeBlake3Hash(hashStr) + if !valid1 { + //http.Error(w, "", http.StatusBadRequest) + return errors.New("hash not valid") + } + + status, err := b.UserWarehouse.DeleteFile(hash) + + if err != nil { + b.LogError("warehouse.DeleteFile", "status %d error: %v", status, err) + return err + } + + return nil +} + +// RmBlockchain abstracted function that removes a file +// metadata from the blockchain +//func RmBlockchain(b *core.Backend, hashStr string) error { +// +//} diff --git a/go.mod b/go.mod index 9cc47ec..f3a02d7 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,22 @@ module github.com/PeernetOfficial/Abstraction go 1.19 + +require ( + github.com/PeernetOfficial/core v0.0.0-20221101165801-6989ef4a19c5 + github.com/google/uuid v1.3.0 +) + +require ( + github.com/IncSW/geoip2 v0.1.2 // indirect + github.com/akrylysov/pogreb v0.10.1 // indirect + github.com/enfipy/locker v1.1.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/gorilla/websocket v1.5.0 // indirect + github.com/klauspost/cpuid/v2 v2.1.2 // indirect + golang.org/x/crypto v0.0.0-20221012134737-56aed061732a // indirect + golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect + golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.1.7 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8219872 --- /dev/null +++ b/go.sum @@ -0,0 +1,30 @@ +github.com/IncSW/geoip2 v0.1.2 h1:v7iAyDiNZjHES45P1JPM3SMvkw0VNeJtz0XSVxkRwOY= +github.com/IncSW/geoip2 v0.1.2/go.mod h1:adcasR40vXiUBjtzdaTTKL/6wSf+fgO4M8Gve/XzPUk= +github.com/PeernetOfficial/core v0.0.0-20221101165801-6989ef4a19c5 h1:9AQcZ535+vpMZm8vky527b/G4jyCuO1FSjoGld0ufSk= +github.com/PeernetOfficial/core v0.0.0-20221101165801-6989ef4a19c5/go.mod h1:UGZqseyO3H9uA4Mm8OQDHP6Gs4z+q7R9VF7R+WzzJbY= +github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w= +github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI= +github.com/enfipy/locker v1.1.0 h1:2zVJ0ky7cS1Vjs0x6OQWFiT2dSEiHrI5/O2KCz1fgGc= +github.com/enfipy/locker v1.1.0/go.mod h1:uuj+dvWHECshK8rkHcw+ZOb9SLo16yc0Em/JGUqRqko= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.1.2 h1:XhdX4fqAJUA0yj+kUwMavO0hHrSPAecYdYf1ZmxHvak= +github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +golang.org/x/crypto v0.0.0-20221012134737-56aed061732a h1:NmSIgad6KjE6VvHciPZuNRTKxGhlPfD6OA87W/PLkqg= +golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU= +golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 h1:OK7RB6t2WQX54srQQYSXMW8dF5C6/8+oA/s5QBmmto4= +golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= +lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff --git a/webapi/API.go b/webapi/API.go new file mode 100644 index 0000000..dec0d8d --- /dev/null +++ b/webapi/API.go @@ -0,0 +1,197 @@ +/* +File Name: API.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "crypto/tls" + "encoding/json" + "errors" + "net/http" + "sync" + "time" + + "github.com/IncSW/geoip2" + "github.com/PeernetOfficial/core" + "github.com/google/uuid" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" +) + +type WebapiInstance struct { + backend *core.Backend + geoipCityReader *geoip2.CityReader + + // Router can be used to register additional API functions + Router *mux.Router + AllowKeyInParam []string // List of paths that accept the API key as &k= parameter + + // search jobs + allJobs map[uuid.UUID]*SearchJob + allJobsMutex sync.RWMutex + + // download info + downloads map[uuid.UUID]*downloadInfo + downloadsMutex sync.RWMutex +} + +// WSUpgrader is used for websocket functionality. It allows all requests. +var WSUpgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { + // allow all connections by default + return true + }, +} + +// Start starts the API. ListenAddresses is a list of IP:Ports. +// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. +// The API key may be uuid.Nil to disable it although this is not recommended for security reasons. +func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKey string, TimeoutRead, TimeoutWrite time.Duration, APIKey uuid.UUID) (api *WebapiInstance) { + if len(ListenAddresses) == 0 { + return nil + } + + api = &WebapiInstance{ + backend: Backend, + Router: mux.NewRouter(), + AllowKeyInParam: []string{"/file/read", "/file/view"}, + allJobs: make(map[uuid.UUID]*SearchJob), + downloads: make(map[uuid.UUID]*downloadInfo), + } + + if APIKey != uuid.Nil { + api.Router.Use(api.authenticateMiddleware(APIKey)) + } + + api.Router.HandleFunc("/test", apiTest).Methods("GET") + api.Router.HandleFunc("/status", api.apiStatus).Methods("GET") + api.Router.HandleFunc("/status/peers", api.apiStatusPeers).Methods("GET") + api.Router.HandleFunc("/account/info", api.apiAccountInfo).Methods("GET") + api.Router.HandleFunc("/account/delete", api.apiAccountDelete).Methods("GET") + api.Router.HandleFunc("/blockchain/header", api.apiBlockchainHeaderFunc).Methods("GET") + api.Router.HandleFunc("/blockchain/append", api.apiBlockchainAppend).Methods("POST") + api.Router.HandleFunc("/blockchain/read", api.apiBlockchainRead).Methods("GET") + api.Router.HandleFunc("/blockchain/file/add", api.apiBlockchainFileAdd).Methods("POST") + api.Router.HandleFunc("/blockchain/file/list", api.apiBlockchainFileList).Methods("GET") + api.Router.HandleFunc("/blockchain/file/delete", api.apiBlockchainFileDelete).Methods("POST") + api.Router.HandleFunc("/blockchain/file/update", api.apiBlockchainFileUpdate).Methods("POST") + api.Router.HandleFunc("/profile/list", api.apiProfileList).Methods("GET") + api.Router.HandleFunc("/profile/read", api.apiProfileRead).Methods("GET") + api.Router.HandleFunc("/profile/write", api.apiProfileWrite).Methods("POST") + api.Router.HandleFunc("/profile/delete", api.apiProfileDelete).Methods("POST") + api.Router.HandleFunc("/search", api.apiSearch).Methods("POST") + api.Router.HandleFunc("/search/result", api.apiSearchResult).Methods("GET") + api.Router.HandleFunc("/search/result/ws", api.apiSearchResultStream).Methods("GET") + api.Router.HandleFunc("/search/statistic", api.apiSearchStatistic).Methods("GET") + api.Router.HandleFunc("/search/terminate", api.apiSearchTerminate).Methods("GET") + api.Router.HandleFunc("/explore", api.apiExplore).Methods("GET") + api.Router.HandleFunc("/file/format", api.apiFileFormat).Methods("GET") + api.Router.HandleFunc("/download/start", api.apiDownloadStart).Methods("GET") + api.Router.HandleFunc("/download/status", api.apiDownloadStatus).Methods("GET") + api.Router.HandleFunc("/download/action", api.apiDownloadAction).Methods("GET") + api.Router.HandleFunc("/warehouse/create", api.apiWarehouseCreateFile).Methods("POST") + api.Router.HandleFunc("/warehouse/create/path", api.apiWarehouseCreateFilePath).Methods("GET") + api.Router.HandleFunc("/warehouse/read", api.apiWarehouseReadFile).Methods("GET") + api.Router.HandleFunc("/warehouse/read/path", api.apiWarehouseReadFilePath).Methods("GET") + api.Router.HandleFunc("/warehouse/delete", api.apiWarehouseDeleteFile).Methods("GET") + api.Router.HandleFunc("/file/read", api.apiFileRead).Methods("GET") + api.Router.HandleFunc("/file/view", api.apiFileView).Methods("GET") + + for _, listen := range ListenAddresses { + go startWebAPI(Backend, listen, UseSSL, CertificateFile, CertificateKey, api.Router, "API", TimeoutRead, TimeoutWrite) + } + + return api +} + +// startWebAPI starts a web-server with given parameters and logs the status. If may block forever and only returns if there is an error. +// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. +func startWebAPI(Backend *core.Backend, WebListen string, UseSSL bool, CertificateFile, CertificateKey string, Handler http.Handler, Info string, ReadTimeout, WriteTimeout time.Duration) { + Backend.LogError("startWebAPI", "Start API at '%s'\n", WebListen) + + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} // for security reasons disable TLS 1.0/1.1 + + server := &http.Server{ + Addr: WebListen, + Handler: Handler, + ReadTimeout: ReadTimeout, // ReadTimeout is the maximum duration for reading the entire request, including the body. + WriteTimeout: WriteTimeout, // WriteTimeout is the maximum duration before timing out writes of the response. This includes processing time and is therefore the max time any HTTP function may take. + //IdleTimeout: IdleTimeout, // IdleTimeout is the maximum amount of time to wait for the next request when keep-alives are enabled. + TLSConfig: tlsConfig, + } + + if UseSSL { + // HTTPS + if err := server.ListenAndServeTLS(CertificateFile, CertificateKey); err != nil { + Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) + } + } else { + // HTTP + if err := server.ListenAndServe(); err != nil { + Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) + } + } +} + +// EncodeJSON encodes the data as JSON +func EncodeJSON(Backend *core.Backend, w http.ResponseWriter, r *http.Request, data interface{}) (err error) { + w.Header().Set("Content-Type", "application/json") + + err = json.NewEncoder(w).Encode(data) + if err != nil { + Backend.LogError("EncodeJSON", "Error writing data for route '%s': %v\n", r.URL.Path, err) + } + + return err +} + +// DecodeJSON decodes input JSON data server side sent either via GET or POST. It does not limit the maximum amount to read. +// In case of error it will automatically send an error to the client. +func DecodeJSON(w http.ResponseWriter, r *http.Request, data interface{}) (err error) { + if r.Body == nil { + http.Error(w, "", http.StatusBadRequest) + return errors.New("no data") + } + + err = json.NewDecoder(r.Body).Decode(data) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return err + } + + return nil +} + +// authenticateMiddleware returns a middleware function to be used with mux.Router.Use(). It handles all authentication functionality. +func (api *WebapiInstance) authenticateMiddleware(APIKey uuid.UUID) func(http.Handler) http.Handler { + return (func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + keyID, err := uuid.Parse(r.Header.Get("x-api-key")) + if err != nil { // special case for some paths + for _, exceptPath := range api.AllowKeyInParam { + if exceptPath == r.URL.Path { + r.ParseForm() + keyID, err = uuid.Parse(r.Form.Get("k")) + break + } + } + } + if err != nil { // Invalid key format + w.WriteHeader(http.StatusUnauthorized) + return + } + + if keyID != APIKey { + w.WriteHeader(http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r) + }) + }) +} diff --git a/webapi/Blockchain.go b/webapi/Blockchain.go new file mode 100644 index 0000000..0d6a362 --- /dev/null +++ b/webapi/Blockchain.go @@ -0,0 +1,122 @@ +/* +File Name: Blockchain.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "encoding/hex" + "net/http" + "strconv" + + "github.com/PeernetOfficial/core/blockchain" +) + +type apiBlockchainHeader struct { + PeerID string `json:"peerid"` // Peer ID hex encoded. + Version uint64 `json:"version"` // Current version number of the blockchain. + Height uint64 `json:"height"` // Height of the blockchain (number of blocks). If 0, no data exists. +} + +/* +apiBlockchainHeaderFunc returns the current blockchain header information + +Request: GET /blockchain/header +Result: 200 with JSON structure apiResponsePeerSelf +*/ +func (api *WebapiInstance) apiBlockchainHeaderFunc(w http.ResponseWriter, r *http.Request) { + publicKey, height, version := api.backend.UserBlockchain.Header() + + EncodeJSON(api.backend, w, r, apiBlockchainHeader{Version: version, Height: height, PeerID: hex.EncodeToString(publicKey.SerializeCompressed())}) +} + +type apiBlockRecordRaw struct { + Type uint8 `json:"type"` // Record Type. See core.RecordTypeX. + Data []byte `json:"data"` // Data according to the type. +} + +// apiBlockchainBlockRaw contains a raw block of the blockchain via API +type apiBlockchainBlockRaw struct { + Records []apiBlockRecordRaw `json:"records"` // Block records in encoded raw format. +} + +type apiBlockchainBlockStatus struct { + Status int `json:"status"` // See blockchain.StatusX. + Height uint64 `json:"height"` // Height of the blockchain (number of blocks). + Version uint64 `json:"version"` // Version of the blockchain. +} + +/* +apiBlockchainAppend appends a block to the blockchain. This is a low-level function for already encoded blocks. +Do not use this function. Adding invalid data to the blockchain may corrupt it which might result in blacklisting by other peers. + +Request: POST /blockchain/append with JSON structure apiBlockchainBlockRaw +Response: 200 with JSON structure apiBlockchainBlockStatus +*/ +func (api *WebapiInstance) apiBlockchainAppend(w http.ResponseWriter, r *http.Request) { + var input apiBlockchainBlockRaw + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var records []blockchain.BlockRecordRaw + + for _, record := range input.Records { + records = append(records, blockchain.BlockRecordRaw{Type: record.Type, Data: record.Data}) + } + + newHeight, newVersion, status := api.backend.UserBlockchain.Append(records) + + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +type apiBlockchainBlock struct { + Status int `json:"status"` // See blockchain.StatusX. + PeerID string `json:"peerid"` // Peer ID hex encoded. + LastBlockHash []byte `json:"lastblockhash"` // Hash of the last block. Blake3. + BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version + Number uint64 `json:"blocknumber"` // Block number + RecordsRaw []apiBlockRecordRaw `json:"recordsraw"` // Records raw. Successfully decoded records are parsed into the below fields. + RecordsDecoded []interface{} `json:"recordsdecoded"` // Records decoded. The encoding for each record depends on its type. +} + +/* +apiBlockchainRead reads a block and returns the decoded information. + +Request: GET /blockchain/read?block=[number] +Result: 200 with JSON structure apiBlockchainBlock +*/ +func (api *WebapiInstance) apiBlockchainRead(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + blockN, err := strconv.Atoi(r.Form.Get("block")) + if err != nil || blockN < 0 { + http.Error(w, "", http.StatusBadRequest) + return + } + + block, status, _ := api.backend.UserBlockchain.Read(uint64(blockN)) + result := apiBlockchainBlock{Status: status} + + if status == 0 { + for _, record := range block.RecordsRaw { + result.RecordsRaw = append(result.RecordsRaw, apiBlockRecordRaw{Type: record.Type, Data: record.Data}) + } + + result.PeerID = hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed()) + + for _, record := range block.RecordsDecoded { + switch v := record.(type) { + case blockchain.BlockRecordFile: + result.RecordsDecoded = append(result.RecordsDecoded, blockRecordFileToAPI(v)) + + case blockchain.BlockRecordProfile: + result.RecordsDecoded = append(result.RecordsDecoded, blockRecordProfileToAPI(v)) + + } + } + } + + EncodeJSON(api.backend, w, r, result) +} diff --git a/webapi/Download Transfer.go b/webapi/Download Transfer.go new file mode 100644 index 0000000..b3a59d2 --- /dev/null +++ b/webapi/Download Transfer.go @@ -0,0 +1,202 @@ +/* +File Name: Download Transfer.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner + +Temporary download code to provide dummy results for testing. To be replaced! +*/ + +package webapi + +import ( + "bytes" + "os" + "time" + + "github.com/PeernetOfficial/core/warehouse" +) + +// Starts the download. +func (info *downloadInfo) Start() { + // current user? + if bytes.Equal(info.nodeID, info.backend.SelfNodeID()) { + info.DownloadSelf() + return + } + + for n := 0; n < 3 && info.peer == nil; n++ { + _, info.peer, _ = info.backend.FindNode(info.nodeID, time.Second*5) + + if info.status == DownloadCanceled { + return + } + } + + if info.peer != nil { + info.Download() + } else { + info.status = DownloadCanceled + } +} + +func (info *downloadInfo) Download() { + //fmt.Printf("Download start of %s\n", hex.EncodeToString(info.hash)) + + // try to download the entire file + reader, fileSize, transferSize, err := FileStartReader(info.peer, info.hash, 0, 0, nil) + if reader != nil { + defer reader.Close() + } + if err != nil { + info.status = DownloadCanceled + return + } else if fileSize != transferSize { + info.status = DownloadCanceled + return + } + + info.file.Size = fileSize + info.status = DownloadActive + + // download in a loop + var fileOffset, totalRead uint64 + dataRemaining := fileSize + readSize := uint64(4096) + + for dataRemaining > 0 { + //fmt.Printf("data remaining: downloaded %d from total %d = %d %%\n", totalRead, fileSize, totalRead*100/fileSize) + if dataRemaining < readSize { + readSize = dataRemaining + } + + data := make([]byte, readSize) + n, err := reader.Read(data) + + totalRead += uint64(n) + dataRemaining -= uint64(n) + data = data[:n] + + if err != nil { + info.status = DownloadCanceled + return + } + + info.storeDownloadData(data[:n], fileOffset) + + fileOffset += uint64(n) + } + + //fmt.Printf("data finished: downloaded %d from total %d = %d %%\n", totalRead, fileSize, totalRead*100/fileSize) + + info.Finish() + info.DeleteDefer(time.Hour * 1) // cache the details for 1 hour before removing +} + +// Pause pauses the download. Status is DownloadResponseX. +func (info *downloadInfo) Pause() (status int) { + info.Lock() + defer info.Unlock() + + if info.status != DownloadActive { // The download must be active to be paused. + return DownloadResponseActionInvalid + } + + info.status = DownloadPause + + return DownloadResponseSuccess +} + +// Resume resumes the download. Status is DownloadResponseX. +func (info *downloadInfo) Resume() (status int) { + info.Lock() + defer info.Unlock() + + if info.status != DownloadPause { // The download must be paused to resume. + return DownloadResponseActionInvalid + } + + info.status = DownloadActive + + return DownloadResponseSuccess +} + +// Cancel cancels the download. Status is DownloadResponseX. +func (info *downloadInfo) Cancel() (status int) { + info.Lock() + defer info.Unlock() + + if info.status >= DownloadCanceled { // The download must not be already canceled or finished. + return DownloadResponseActionInvalid + } + + info.status = DownloadCanceled + info.DiskFile.Handle.Close() + + return DownloadResponseSuccess +} + +// Finish marks the download as finished. +func (info *downloadInfo) Finish() (status int) { + info.Lock() + defer info.Unlock() + + if info.status != DownloadActive { // The download must be active. + return DownloadResponseActionInvalid + } + + info.status = DownloadFinished + info.DiskFile.Handle.Close() + + return DownloadResponseSuccess +} + +// initDiskFile creates the target file +func (info *downloadInfo) initDiskFile(path string) (err error) { + info.DiskFile.Name = path + info.DiskFile.Handle, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) // 666 : All uses can read/write + + return err +} + +// storeDownloadData stores downloaded data. It does not change the download status. +func (info *downloadInfo) storeDownloadData(data []byte, offset uint64) (status int) { + info.Lock() + defer info.Unlock() + + if info.status != DownloadActive { // The download must be active. + return DownloadResponseActionInvalid + } + + if _, err := info.DiskFile.Handle.WriteAt(data, int64(offset)); err != nil { + return DownloadResponseFileWrite + } + + info.DiskFile.StoredSize += uint64(len(data)) + + return DownloadResponseSuccess +} + +func (info *downloadInfo) DownloadSelf() { + // Check if the file is available in the local warehouse. + _, fileSize, status, _ := info.backend.UserWarehouse.FileExists(info.hash) + if status != warehouse.StatusOK { + info.status = DownloadCanceled + return + } + + info.file.Size = fileSize + info.status = DownloadActive + + // read the file + status, bytesRead, _ := info.backend.UserWarehouse.ReadFile(info.hash, 0, int64(info.file.Size), info.DiskFile.Handle) + + info.DiskFile.StoredSize = uint64(bytesRead) + + if status != warehouse.StatusOK { + info.status = DownloadCanceled + return + } + + info.Finish() + info.DeleteDefer(time.Hour * 1) // cache the details for 1 hour before removing} +} diff --git a/webapi/Download.go b/webapi/Download.go new file mode 100644 index 0000000..e4101a2 --- /dev/null +++ b/webapi/Download.go @@ -0,0 +1,243 @@ +/* +File Name: Download.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "encoding/hex" + "math" + "net/http" + "os" + "strconv" + "sync" + "time" + + "github.com/PeernetOfficial/core" + "github.com/google/uuid" +) + +type apiResponseDownloadStatus struct { + APIStatus int `json:"apistatus"` // Status of the API call. See DownloadResponseX. + ID uuid.UUID `json:"id"` // Download ID. This can be used to query the latest status and take actions. + DownloadStatus int `json:"downloadstatus"` // Status of the download. See DownloadX. + File ApiFile `json:"file"` // File information. Only available for status >= DownloadWaitSwarm. + Progress struct { + TotalSize uint64 `json:"totalsize"` // Total size in bytes. + DownloadedSize uint64 `json:"downloadedsize"` // Count of bytes download so far. + Percentage float64 `json:"percentage"` // Percentage downloaded. Rounded to 2 decimal points. Between 0.00 and 100.00. + } `json:"progress"` // Progress of the download. Only valid for status >= DownloadWaitSwarm. + Swarm struct { + CountPeers uint64 `json:"countpeers"` // Count of peers participating in the swarm. + } `json:"swarm"` // Information about the swarm. Only valid for status >= DownloadActive. +} + +const ( + DownloadResponseSuccess = 0 // Success + DownloadResponseIDNotFound = 1 // Error: Download ID not found. + DownloadResponseFileInvalid = 2 // Error: Target file cannot be used. For example, permissions denied to create it. + DownloadResponseActionInvalid = 4 // Error: Invalid action. Pausing a non-active download, resuming a non-paused download, or canceling already canceled or finished download. + DownloadResponseFileWrite = 5 // Error writing file. +) + +// Download status list +const ( + DownloadWaitMetadata = 0 // Wait for file metadata. + DownloadWaitSwarm = 1 // Wait to join swarm. + DownloadActive = 2 // Active downloading. It could still be stuck at any percentage (including 0%) if no seeders are available. + DownloadPause = 3 // Paused by the user. + DownloadCanceled = 4 // Canceled by the user before the download finished. Once canceled, a new download has to be started if the file shall be downloaded. + DownloadFinished = 5 // Download finished 100%. +) + +/* +apiDownloadStart starts the download of a file. The path is the full path on disk to store the file. +The hash parameter identifies the file to download. The node ID identifies the blockchain (i.e., the "owner" of the file). + +Request: GET /download/start?path=[target path on disk]&hash=[file hash to download]&node=[node ID] +Result: 200 with JSON structure apiResponseDownloadStatus +*/ +func (api *WebapiInstance) apiDownloadStart(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + + // validate hashes, must be blake3 + hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) + if !valid1 || !valid2 { + http.Error(w, "", http.StatusBadRequest) + return + } + + filePath := r.Form.Get("path") + if filePath == "" { + http.Error(w, "", http.StatusBadRequest) + return + } + + info := &downloadInfo{backend: api.backend, api: api, id: uuid.New(), created: time.Now(), hash: hash, nodeID: nodeID} + + // create the file immediately + if info.initDiskFile(filePath) != nil { + EncodeJSON(api.backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseFileInvalid}) + return + } + + // add the download to the list + api.downloadAdd(info) + + // start the download! + go info.Start() + + EncodeJSON(api.backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseSuccess, ID: info.id, DownloadStatus: DownloadWaitMetadata}) +} + +/* +apiDownloadStatus returns the status of an active download. + +Request: GET /download/status?id=[download ID] +Result: 200 with JSON structure apiResponseDownloadStatus +*/ +func (api *WebapiInstance) apiDownloadStatus(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + id, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + info := api.downloadLookup(id) + if info == nil { + EncodeJSON(api.backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseIDNotFound}) + return + } + + info.RLock() + + response := apiResponseDownloadStatus{APIStatus: DownloadResponseSuccess, ID: info.id, DownloadStatus: info.status} + + if info.status >= DownloadWaitSwarm { + response.File = info.file + + response.Progress.TotalSize = info.file.Size + response.Progress.DownloadedSize = info.DiskFile.StoredSize + + response.Progress.Percentage = math.Round(float64(info.DiskFile.StoredSize)/float64(info.file.Size)*100*100) / 100 + } + + if info.status >= DownloadActive { + response.Swarm.CountPeers = info.Swarm.CountPeers + } + + info.RUnlock() + + EncodeJSON(api.backend, w, r, response) +} + +/* +apiDownloadAction pauses, resumes, and cancels a download. Once canceled, a new download has to be started if the file shall be downloaded. +Only active downloads can be paused. While a download is in discovery phase (querying metadata, joining swarm), it can only be canceled. +Action: 0 = Pause, 1 = Resume, 2 = Cancel. + +Request: GET /download/action?id=[download ID]&action=[action] +Result: 200 with JSON structure apiResponseDownloadStatus (using APIStatus and DownloadStatus) +*/ +func (api *WebapiInstance) apiDownloadAction(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + id, err := uuid.Parse(r.Form.Get("id")) + action, err2 := strconv.Atoi(r.Form.Get("action")) + if err != nil || err2 != nil || action < 0 || action > 2 { + http.Error(w, "", http.StatusBadRequest) + return + } + + info := api.downloadLookup(id) + if info == nil { + EncodeJSON(api.backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseIDNotFound}) + return + } + + apiStatus := 0 + + switch action { + case 0: // Pause + apiStatus = info.Pause() + + case 1: // Resume + apiStatus = info.Resume() + + case 2: // Cancel + apiStatus = info.Cancel() + } + + EncodeJSON(api.backend, w, r, apiResponseDownloadStatus{APIStatus: apiStatus, ID: info.id, DownloadStatus: info.status}) +} + +// ---- download tracking ---- + +type downloadInfo struct { + id uuid.UUID // Download ID + status int // Current status. See DownloadX. + sync.RWMutex // Mutext for changing the status + + // input + hash []byte // File hash + nodeID []byte // Node ID of the owner + + // runtime data + created time.Time // When the download was created. + ended time.Time // When the download was finished (only status = DownloadFinished). + + file ApiFile // File metadata (only status >= DownloadWaitSwarm) + + DiskFile struct { // Target file on disk to store downloaded data + Name string // File name + Handle *os.File // Target file (on disk) to store downloaded data + StoredSize uint64 // Count of bytes downloaded and stored in the file + } + + Swarm struct { // Information about the swarm. Only valid for status >= DownloadActive. + CountPeers uint64 // Count of peers participating in the swarm. + } + + // live connections, to be changed + peer *core.PeerInfo + + api *WebapiInstance + backend *core.Backend +} + +func (api *WebapiInstance) downloadAdd(info *downloadInfo) { + api.downloadsMutex.Lock() + api.downloads[info.id] = info + api.downloadsMutex.Unlock() +} + +func (api *WebapiInstance) downloadDelete(id uuid.UUID) { + api.downloadsMutex.Lock() + delete(api.downloads, id) + api.downloadsMutex.Unlock() +} + +func (api *WebapiInstance) downloadLookup(id uuid.UUID) (info *downloadInfo) { + api.downloadsMutex.Lock() + info = api.downloads[id] + api.downloadsMutex.Unlock() + return info +} + +// DeleteDefer deletes the download from the downloads list after the given duration. +// It does not wait for the download to be finished. +func (info *downloadInfo) DeleteDefer(Duration time.Duration) { + go func() { + <-time.After(Duration) + info.api.downloadDelete(info.id) + }() +} + +// DecodeBlake3Hash decodes a blake3 hash that is hex encoded +func DecodeBlake3Hash(text string) (hash []byte, valid bool) { + hash, err := hex.DecodeString(text) + return hash, err == nil && len(hash) == 256/8 +} diff --git a/webapi/File Detection.go b/webapi/File Detection.go new file mode 100644 index 0000000..321dfed --- /dev/null +++ b/webapi/File Detection.go @@ -0,0 +1,221 @@ +/* +File Name: File Detection.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "net/http" + "os" + "path" + "strings" + + "github.com/PeernetOfficial/core" +) + +// PathToExtension translates a path to a file extension, if possible. It also returns the second file extension if there is one (relevant for files like "test.tar.gz"). +func PathToExtension(Path string) (extension, extension2 string, valid bool) { + _, fileA := path.Split(Path) + + parts := strings.Split(fileA, ".") + if len(parts) <= 1 { + return "", "", false + } + extension = parts[len(parts)-1] + extension = strings.ToLower(extension) + + if len(parts) >= 3 { + extension2 = parts[len(parts)-2] + extension2 = strings.ToLower(extension2) + } + + return extension, extension2, true +} + +// FileTranslateExtension translates the extension to a File Type and File Format. If invalid, types are 0. +func FileTranslateExtension(extension string) (fileType, fileFormat uint16) { + switch extension { + case "txt", "log", "ini", "json", "md": + return core.TypeText, core.FormatText + + case "csv", "tsv": + return core.TypeText, core.FormatCSV + + case "html", "htm": + return core.TypeText, core.FormatHTML + + case "doc", "docx", "rtf", "odt": + return core.TypeDocument, core.FormatWord + + case "pdf": + return core.TypeDocument, core.FormatPDF + + case "xls", "xlsx", "ods": + return core.TypeDocument, core.FormatExcel + + case "gif", "jpg", "jpeg", "png", "svg", "bmp", "tif", "tiff", "jfif", "webp": + return core.TypePicture, core.FormatPicture + + case "mp4", "flv", "avi", "mov", "mpg", "mpeg", "h264", "3g2", "3gp", "mkv", "wmv", "webm", "ts": + return core.TypeVideo, core.FormatVideo + + case "mp3", "ogg", "flac": + return core.TypeAudio, core.FormatAudio + + case "zip", "rar", "7z", "tar": + return core.TypeContainer, core.FormatContainer + + case "ppt", "pptx", "odp": + return core.TypeDocument, core.FormatPowerpoint + + case "epub", "mobi", "prc": + return core.TypeEbook, core.FormatEbook + + case "gz", "bz", "bz2", "xz": + return core.TypeCompressed, core.FormatCompressed + + case "sql": + return core.TypeText, core.FormatDatabase + + case "eml", "mbox": + return core.TypeText, core.FormatEmail + + case "exe", "sys", "dll", "cmd", "bat": + return core.TypeExecutable, core.FormatExecutable + + case "msi": + return core.TypeExecutable, core.FormatInstaller + + case "apk": + return core.TypeExecutable, core.FormatAPK + + case "iso": + return core.TypeContainer, core.FormatISO + + default: + return core.TypeBinary, core.FormatBinary + } +} + +// HTTPContentTypeToCore translates the HTTP content type to the File Type and File Format used by the core package. +func HTTPContentTypeToCore(httpContentType string) (fileType, fileFormat uint16) { + switch httpContentType { + case "text/html", "application/xhtml+xml", "application/xml": + return core.TypeText, core.FormatHTML + + case "text/plain": + return core.TypeText, core.FormatText + + case "text/csv", "text/tsv", "text/tab-separated-values", "text/x-csv", "application/csv", "application/x-csv", "text/x-comma-separated-values": + return core.TypeText, core.FormatCSV + + case "application/pdf", "application/x-pdf": + return core.TypeDocument, core.FormatPDF + + case "application/msword", "application/rtf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.oasis.opendocument.text": + return core.TypeDocument, core.FormatWord + + case "application/excel", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.oasis.opendocument.spreadsheet": + return core.TypeDocument, core.FormatExcel + + case "application/vnd.ms-powerpoint", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.oasis.opendocument.presentation": + return core.TypeDocument, core.FormatPowerpoint + + case "image/png", "image/jpeg", "image/gif", "image/svg+xml", "image/tiff", "image/webp", "image/bmp", "image/x-bmp", "image/x-windows-bmp": // image/x-icon excluded + return core.TypePicture, core.FormatPicture + + case "audio/aac", "audio/midi", "audio/ogg", "audio/x-wav", "audio/webm", "audio/3gpp", "audio/3gpp2", "audio/mpeg", "audio/vorbis": + return core.TypeAudio, core.FormatAudio + + case "video/x-msvideo", "video/x-ms-wmv", "video/mpeg", "video/ogg", "video/webm", "video/3gpp", "video/3gpp2", "video/x-flv", "video/mp4": + return core.TypeVideo, core.FormatVideo + + case "application/zip", "application/x-rar-compressed", "application/x-tar", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed": + return core.TypeContainer, core.FormatContainer + + case "application/epub+zip", "application/x-mobipocket-ebook": + return core.TypeEbook, core.FormatEbook + + default: + return core.TypeBinary, core.FormatBinary + } +} + +// FileDataToHTTPContentType returns the HTTP content type based on the initial file data. It reads the first 512 bytes of the file. +func FileDataToHTTPContentType(Path string) (httpContentType string, err error) { + file, err := os.Open(Path) + if err != nil { + return "", err + } + + // Read up to 512 bytes. This specific number comes from http.DetectContentType which specifies it as constant "sniffLen". + buff := make([]byte, 512) + + if _, err := file.Read(buff); err != nil { + return "", err + } + + if err := file.Close(); err != nil { + return "", err + } + + httpContentType = http.DetectContentType(buff) + + // sanitize it first + httpContentType = strings.ToLower(strings.TrimSpace(httpContentType)) + if indexD := strings.IndexAny(httpContentType, ";"); indexD >= 0 { + httpContentType = httpContentType[:indexD] + } + + return httpContentType, nil +} + +// FileDetectType detects the File Type and File Format of a file. It uses the extension if available, otherwise the file data, for detection. +func FileDetectType(Path string) (fileType, fileFormat uint16, err error) { + // If a file extension is available, use that to detect the file type and file format. + // Otherwise, use the initial file data for detection. + if ext1, _, valid := PathToExtension(Path); valid { + fileType, fileFormat = FileTranslateExtension(ext1) + return fileType, fileFormat, nil + } + + httpContentType, err := FileDataToHTTPContentType(Path) + if err != nil { + return core.TypeBinary, core.FormatBinary, err + } + + fileType, fileFormat = HTTPContentTypeToCore(httpContentType) + return fileType, fileFormat, nil +} + +type apiResponseFileFormat struct { + Status int `json:"status"` // Status: 0 = Success, 1 = Error reading file + FileType uint16 `json:"filetype"` // File Type. + FileFormat uint16 `json:"fileformat"` // File Format. +} + +/* +apiFileFormat detects the file type and file format of the specified file. +It will primarily use the file extension for detection. If unavailable, it uses the first 512 bytes of the file data to detect the type. + +Request: GET /file/format?path=[file path on disk] +Result: 200 with JSON structure apiResponseFileFormat +*/ +func (api *WebapiInstance) apiFileFormat(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + filePath := r.Form.Get("path") + if filePath == "" { + http.Error(w, "", http.StatusBadRequest) + return + } + + fileType, fileFormat, err := FileDetectType(filePath) + if err != nil { + EncodeJSON(api.backend, w, r, apiResponseFileFormat{Status: 1}) + return + } + + EncodeJSON(api.backend, w, r, apiResponseFileFormat{Status: 0, FileType: fileType, FileFormat: fileFormat}) +} diff --git a/webapi/File IO.go b/webapi/File IO.go new file mode 100644 index 0000000..434d382 --- /dev/null +++ b/webapi/File IO.go @@ -0,0 +1,319 @@ +/* +File Name: File IO.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "errors" + "io" + "net/http" + "strconv" + "time" + + "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/warehouse" +) + +/* +apiFileRead reads a file immediately from a remote peer. Use the /download functions to download a file. +This endpoint supports the Range, Content-Range and Content-Length headers. Multipart ranges are not supported and result in HTTP 400. +Instead of providing the node ID, the peer ID is also accepted in the &node= parameter. +The default timeout for connecting to the peer is 10 seconds. + +Request: GET /file/read?hash=[hash]&node=[node ID] + Optional: &offset=[offset]&limit=[limit] or via Range header. + Optional: &timeout=[seconds] +Response: 200 with the content + 206 with partial content + 400 if the parameters are invalid + 404 if the file was not found or other error on transfer initiate + 502 if unable to find or connect to the remote peer in time +*/ +func (api *WebapiInstance) apiFileRead(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + var err error + + // validate hashes (must be blake3) and other input + fileHash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) + publicKey, err3 := core.PublicKeyFromPeerID(r.Form.Get("node")) + if !valid1 || (!valid2 && err3 != nil) { + http.Error(w, "", http.StatusBadRequest) + return + } + + timeoutSeconds, _ := strconv.Atoi(r.Form.Get("timeout")) + if timeoutSeconds == 0 { + timeoutSeconds = 10 + } + timeout := time.Duration(timeoutSeconds) * time.Second + + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, _ := strconv.Atoi(r.Form.Get("limit")) + + // Range header? + var ranges []HTTPRange + if ranges, err = ParseRangeHeader(r.Header.Get("Range"), -1, true); err != nil || len(ranges) > 1 { + http.Error(w, "", http.StatusBadRequest) + return + } else if len(ranges) == 1 { + if ranges[0].length != -1 { // if length is not specified, limit remains 0 which is maximum + limit = ranges[0].length + } + offset = ranges[0].start + } + + // Is the file available in the local warehouse? In that case requesting it from the remote is unnecessary. + if serveFileFromWarehouse(api.backend, w, fileHash, uint64(offset), uint64(limit), ranges) { + return + } + + // try connecting via node ID or peer ID? + var peer *core.PeerInfo + + if valid2 { + peer, err = PeerConnectNode(api.backend, nodeID, timeout) + } else if err3 == nil { + peer, err = PeerConnectPublicKey(api.backend, publicKey, timeout) + } + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + + // Start the reader. If this HTTP request is canceled, r.Context().Done() acts as cancellation signal to the underlying UDT connection. + reader, fileSize, transferSize, err := FileStartReader(peer, fileHash, uint64(offset), uint64(limit), r.Context().Done()) + if reader != nil { + defer reader.Close() + } + if err != nil || reader == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + // set the right headers + setContentLengthRangeHeader(w, uint64(offset), transferSize, fileSize, ranges) + + // Start sending the data! + io.Copy(w, io.LimitReader(reader, int64(transferSize))) +} + +// serveFileFromWarehouse serves the file from the warehouse. If it is not available, it returns false and does not use the writer. +// Limit is optional, 0 means the entire file. +func serveFileFromWarehouse(backend *core.Backend, w http.ResponseWriter, fileHash []byte, offset, limit uint64, ranges []HTTPRange) (valid bool) { + // Check if the file is available in the local warehouse. + _, fileSize, status, _ := backend.UserWarehouse.FileExists(fileHash) + if status != warehouse.StatusOK { + return false + } + + // validate offset and limit + if limit > 0 && offset+limit > fileSize { + http.Error(w, "invalid limit", http.StatusBadRequest) + return true + } else if offset > fileSize { + http.Error(w, "invalid offset", http.StatusBadRequest) + return true + } else if limit == 0 { + limit = fileSize - offset + } + + setContentLengthRangeHeader(w, offset, limit, fileSize, ranges) + + status, _, _ = backend.UserWarehouse.ReadFile(fileHash, int64(offset), int64(limit), w) + + // StatusErrorReadFile must be considered success, since parts of the file may have been transferred already and recovery is not possible. + return status == warehouse.StatusErrorReadFile || status == warehouse.StatusOK +} + +/* +apiFileView is similar to /file/read but but provides a format parameter. It sets the Content-Type and Accept-Ranges headers. +This endpoint supports the Range, Content-Range and Content-Length headers. Multipart ranges are not supported and result in HTTP 400. +Instead of providing the node ID, the peer ID is also accepted in the &node= parameter. +The default timeout for connecting to the peer is 10 seconds. +Formats: 14 = Video + +Request: GET /file/view?hash=[hash]&node=[node ID]&format=[format] + Optional: &offset=[offset]&limit=[limit] or via Range header. + Optional: &timeout=[seconds] +Response: 200 with the content + 206 with partial content + 400 if the parameters are invalid + 404 if the file was not found or other error on transfer initiate + 502 if unable to find or connect to the remote peer in time +*/ +func (api *WebapiInstance) apiFileView(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + var err error + + // validate hashes (must be blake3) and other input + fileHash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) + publicKey, err3 := core.PublicKeyFromPeerID(r.Form.Get("node")) + if !valid1 || (!valid2 && err3 != nil) { + http.Error(w, "", http.StatusBadRequest) + return + } + + timeoutSeconds, _ := strconv.Atoi(r.Form.Get("timeout")) + if timeoutSeconds == 0 { + timeoutSeconds = 10 + } + timeout := time.Duration(timeoutSeconds) * time.Second + + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, _ := strconv.Atoi(r.Form.Get("limit")) + format, _ := strconv.Atoi(r.Form.Get("format")) + localCacheDisable, _ := strconv.ParseBool(r.Form.Get("nocache")) + + // Range header? + var ranges []HTTPRange + if ranges, err = ParseRangeHeader(r.Header.Get("Range"), -1, true); err != nil || len(ranges) > 1 { + http.Error(w, "", http.StatusBadRequest) + return + } else if len(ranges) == 1 { + if ranges[0].length != -1 { // if length is not specified, limit remains 0 which is maximum + limit = ranges[0].length + } + offset = ranges[0].start + } + + w.Header().Set("Accept-Ranges", "bytes") // always indicate accepting of Range header + + switch format { + case 14: + // Video: Indicate MP4 always. There are tons of other MIME types that could be used. + w.Header().Set("Content-Type", "video/mp4") + } + + // Is the file available in the local warehouse? In that case requesting it from the remote is unnecessary. + if !localCacheDisable { + if serveFileFromWarehouse(api.backend, w, fileHash, uint64(offset), uint64(limit), ranges) { + return + } + } + + // try connecting via node ID or peer ID? + var peer *core.PeerInfo + + if valid2 { + peer, err = PeerConnectNode(api.backend, nodeID, timeout) + } else if err3 == nil { + peer, err = PeerConnectPublicKey(api.backend, publicKey, timeout) + } + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + + // start the reader + reader, fileSize, transferSize, err := FileStartReader(peer, fileHash, uint64(offset), uint64(limit), r.Context().Done()) + if reader != nil { + defer reader.Close() + } + if err != nil || reader == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + // set the right headers + setContentLengthRangeHeader(w, uint64(offset), transferSize, fileSize, ranges) + + // Start sending the data! + io.Copy(w, io.LimitReader(reader, int64(transferSize))) +} + +// PeerConnectPublicKey attempts to connect to the peer specified by its public key (= peer ID). +func PeerConnectPublicKey(backend *core.Backend, publicKey *btcec.PublicKey, timeout time.Duration) (peer *core.PeerInfo, err error) { + if publicKey == nil { + return nil, errors.New("invalid public key") + } + + // First look up in the peer list. + if peer = backend.PeerlistLookup(publicKey); peer != nil { + return peer, nil + } + + // Try to connect via DHT. + nodeID := protocol.PublicKey2NodeID(publicKey) + if _, peer, _ = backend.FindNode(nodeID, timeout); peer != nil { + return peer, nil + } + + // otherwise not found :( + return nil, errors.New("peer not found") +} + +// PeerConnectNode tries to connect via the node ID +func PeerConnectNode(backend *core.Backend, nodeID []byte, timeout time.Duration) (peer *core.PeerInfo, err error) { + if len(nodeID) != 256/8 { + return nil, errors.New("invalid node ID") + } + + // Try to connect via DHT. + if _, peer, _ = backend.FindNode(nodeID, timeout); peer != nil { + return peer, nil + } + + // otherwise not found :( + return nil, errors.New("peer not found") +} + +// FileStartReader providers a reader to a remote file. The reader must be closed by the caller. +// File Size is the full file size reported by the remote peer, regardless of the requested offset and limit. Limit is optional (0 means the entire file). +// Transfer Size is the size in bytes that is actually going to be transferred. The reader should be closed after reading that amount. +// The optional cancelChan can be used to stop the file transfer at any point. +func FileStartReader(peer *core.PeerInfo, hash []byte, offset, limit uint64, cancelChan <-chan struct{}) (reader io.ReadCloser, fileSize, transferSize uint64, err error) { + if peer == nil { + return nil, 0, 0, errors.New("peer not provided") + } else if !peer.IsConnectionActive() { + return nil, 0, 0, errors.New("no valid connection to peer") + } + + udtConn, virtualConn, err := peer.FileTransferRequestUDT(hash, offset, limit) + if err != nil { + return nil, 0, 0, err + } + + if cancelChan != nil { + go func() { + <-cancelChan + udtConn.Close() + }() + } + + fileSize, transferSize, err = protocol.FileTransferReadHeader(udtConn) + if err != nil { + udtConn.Close() + return nil, 0, 0, err + } + + virtualConn.Stats.(*core.FileTransferStats).FileSize = fileSize + + return udtConn, fileSize, transferSize, nil +} + +// FileReadAll downloads the file from the peer. +// This function should only be used for testing or as a basis to fork. The caller should develop a custom download function that handles timeouts and excessive file sizes. +// It allocates whatever size is reported by the remote peer. This could lead to an out of memory crash. +// This function is blocking and may take a long time depending on the remote peer and the network connection. +func FileReadAll(peer *core.PeerInfo, hash []byte) (data []byte, err error) { + reader, _, transferSize, err := FileStartReader(peer, hash, 0, 0, nil) + if err != nil { + return nil, err + } + defer reader.Close() + + // read all data + data = make([]byte, transferSize) // Warning: This could lead to an out of memory crash. + _, err = reader.Read(data) + + // Note: This function does not verify if the returned data matches the hash and expected size. + + return data, err +} diff --git a/webapi/File.go b/webapi/File.go new file mode 100644 index 0000000..1f4ceb4 --- /dev/null +++ b/webapi/File.go @@ -0,0 +1,338 @@ +/* +File Name: File.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "net/http" + "time" + + "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/merkle" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/warehouse" + "github.com/google/uuid" +) + +// ApiFileMetadata contains metadata information. +type ApiFileMetadata struct { + Type uint16 `json:"type"` // See core.TagX constants. + Name string `json:"name"` // User friendly name of the metadata type. Use the Type fields to identify the metadata as this name may change. + // Depending on the exact type, one of the below fields is used for proper encoding: + Text string `json:"text"` // Text value. UTF-8 encoding. + Blob []byte `json:"blob"` // Binary data + Date time.Time `json:"date"` // Date + Number uint64 `json:"number"` // Number +} + +// ApiFile is the metadata of a file published on the blockchain +type ApiFile struct { + ID uuid.UUID `json:"id"` // Unique ID. + Hash []byte `json:"hash"` // Blake3 hash of the file data + Type uint8 `json:"type"` // File Type. For example audio or document. See TypeX. + Format uint16 `json:"format"` // File Format. This is more granular, for example PDF or Word file. See FormatX. + Size uint64 `json:"size"` // Size of the file + Folder string `json:"folder"` // Folder, optional + Name string `json:"name"` // Name of the file + Description string `json:"description"` // Description. This is expected to be multiline and contain hashtags! + Date time.Time `json:"date"` // Date shared + NodeID []byte `json:"nodeid"` // Node ID, owner of the file. Read only. + Metadata []ApiFileMetadata `json:"metadata"` // Additional metadata. +} + +// --- conversion from core to API data --- + +func blockRecordFileToAPI(input blockchain.BlockRecordFile) (output ApiFile) { + output = ApiFile{ID: input.ID, Hash: input.Hash, NodeID: input.NodeID, Type: input.Type, Format: input.Format, Size: input.Size, Metadata: []ApiFileMetadata{}} + + for _, tag := range input.Tags { + switch tag.Type { + case blockchain.TagName: + output.Name = tag.Text() + + case blockchain.TagFolder: + output.Folder = tag.Text() + + case blockchain.TagDescription: + output.Description = tag.Text() + + case blockchain.TagDateShared: + output.Date, _ = tag.Date() + + case blockchain.TagDateCreated: + date, _ := tag.Date() + output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Date Created", Date: date}) + + case blockchain.TagSharedByCount: + output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Shared By Count", Number: tag.Number()}) + + case blockchain.TagSharedByGeoIP: + output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Shared By GeoIP", Text: tag.Text()}) + + default: + output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Blob: tag.Data}) + } + } + + return output +} + +func BlockRecordFileFromAPI(input ApiFile) (output blockchain.BlockRecordFile) { + output = blockchain.BlockRecordFile{ID: input.ID, Hash: input.Hash, Type: input.Type, Format: input.Format, Size: input.Size} + + if input.Name != "" { + output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagName, input.Name)) + } + if input.Folder != "" { + output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagFolder, input.Folder)) + } + if input.Description != "" { + output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagDescription, input.Description)) + } + + for _, meta := range input.Metadata { + if blockchain.IsTagVirtual(meta.Type) { // Virtual tags are not mapped back. They are read-only. + continue + } + + switch meta.Type { + case blockchain.TagName, blockchain.TagFolder, blockchain.TagDescription: // auto mapped tags + + case blockchain.TagDateCreated: + output.Tags = append(output.Tags, blockchain.TagFromDate(meta.Type, meta.Date)) + + default: + output.Tags = append(output.Tags, blockchain.BlockRecordFileTag{Type: meta.Type, Data: meta.Blob}) + } + } + + return output +} + +// --- File API --- + +// ApiBlockAddFiles contains a list of files from the blockchain +type ApiBlockAddFiles struct { + Files []ApiFile `json:"files"` // List of files + Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API. +} + +/* +apiBlockchainFileAdd adds a file with the provided information to the blockchain. +Each file must be already stored in the Warehouse (virtual folders are exempt). +If any file is not stored in the Warehouse, the function aborts with the status code StatusNotInWarehouse. +If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord. +In case the function aborts, the blockchain remains unchanged. + +Request: POST /blockchain/file/add with JSON structure ApiBlockAddFiles +Response: 200 with JSON structure apiBlockchainBlockStatus + 400 if invalid input +*/ +func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.Request) { + var input ApiBlockAddFiles + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var filesAdd []blockchain.BlockRecordFile + + for _, file := range input.Files { + if len(file.Hash) != protocol.HashSize { + http.Error(w, "", http.StatusBadRequest) + return + } + if file.ID == uuid.Nil { // if the ID is not provided by the caller, set it + file.ID = uuid.New() + } + + // Verify that the file exists in the warehouse. Folders are exempt from this check as they are only virtual. + if !file.IsVirtualFolder() { + if _, err := warehouse.ValidateHash(file.Hash); err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } else if _, fileSize, status, _ := api.backend.UserWarehouse.FileExists(file.Hash); status != warehouse.StatusOK { + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return + } else { + file.Size = fileSize + } + } else { + file.Hash = protocol.HashData(nil) + file.Size = 0 + } + + blockRecord := BlockRecordFileFromAPI(file) + + // Set the merkle tree info as appropriate. + if !SetFileMerkleInfo(api.backend, &blockRecord) { + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return + } + + filesAdd = append(filesAdd, blockRecord) + } + + newHeight, newVersion, status := api.backend.UserBlockchain.AddFiles(filesAdd) + + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +/* +apiBlockchainFileList lists all files stored on the blockchain. + +Request: GET /blockchain/file/list +Response: 200 with JSON structure ApiBlockAddFiles +*/ +func (api *WebapiInstance) apiBlockchainFileList(w http.ResponseWriter, r *http.Request) { + files, status := api.backend.UserBlockchain.ListFiles() + + var result ApiBlockAddFiles + + for _, file := range files { + result.Files = append(result.Files, blockRecordFileToAPI(file)) + } + + result.Status = status + + EncodeJSON(api.backend, w, r, result) +} + +/* +apiBlockchainFileDelete deletes files with the provided IDs. Other fields are ignored. +It will automatically delete the file in the Warehouse if there are no other references. + +Request: POST /blockchain/file/delete with JSON structure ApiBlockAddFiles +Response: 200 with JSON structure apiBlockchainBlockStatus +*/ +func (api *WebapiInstance) apiBlockchainFileDelete(w http.ResponseWriter, r *http.Request) { + var input ApiBlockAddFiles + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var deleteIDs []uuid.UUID + + for n := range input.Files { + deleteIDs = append(deleteIDs, input.Files[n].ID) + } + + newHeight, newVersion, deletedFiles, status := api.backend.UserBlockchain.DeleteFiles(deleteIDs) + + // If successfully deleted from the blockchain, delete from the Warehouse in case there are no other references. + if status == blockchain.StatusOK { + for n := range deletedFiles { + if files, status := api.backend.UserBlockchain.FileExists(deletedFiles[n].Hash); status == blockchain.StatusOK && len(files) == 0 { + api.backend.UserWarehouse.DeleteFile(deletedFiles[n].Hash) + } + } + } + + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +/* +apiBlockchainSelfUpdateFile updates files that are already published on the blockchain. + +Request: POST /blockchain/file/update with JSON structure ApiBlockAddFiles +Response: 200 with JSON structure apiBlockchainBlockStatus + 400 if invalid input +*/ +func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *http.Request) { + var input ApiBlockAddFiles + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var filesAdd []blockchain.BlockRecordFile + + for _, file := range input.Files { + if len(file.Hash) != protocol.HashSize { + http.Error(w, "", http.StatusBadRequest) + return + } else if file.ID == uuid.Nil { // if the ID is not provided by the caller, abort + http.Error(w, "", http.StatusBadRequest) + return + } + + // Verify that the file exists in the warehouse. Folders are exempt from this check as they are only virtual. + if !file.IsVirtualFolder() { + if _, err := warehouse.ValidateHash(file.Hash); err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } else if _, fileSize, status, _ := api.backend.UserWarehouse.FileExists(file.Hash); status != warehouse.StatusOK { + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return + } else { + file.Size = fileSize + } + } else { + file.Hash = protocol.HashData(nil) + file.Size = 0 + } + + blockRecord := BlockRecordFileFromAPI(file) + + // Set the merkle tree info as appropriate. + if !SetFileMerkleInfo(api.backend, &blockRecord) { + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return + } + + filesAdd = append(filesAdd, blockRecord) + } + + newHeight, newVersion, status := api.backend.UserBlockchain.ReplaceFiles(filesAdd) + + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +// ---- metadata functions ---- + +// GetMetadata returns the specified metadata or nil if not available. +func (file *ApiFile) GetMetadata(Type uint16) (info *ApiFileMetadata) { + for n := range file.Metadata { + if file.Metadata[n].Type == Type { + return &file.Metadata[n] + } + } + + return nil +} + +// GetNumber returns the data as number. 0 if not available. +func (info *ApiFileMetadata) GetNumber() uint64 { + if info == nil { + return 0 + } + + return info.Number +} + +// IsVirtualFolder returns true if the file is a virtual folder +func (file *ApiFile) IsVirtualFolder() bool { + return file.Type == core.TypeFolder && file.Format == core.FormatFolder +} + +// SetFileMerkleInfo sets the merkle fields in the BlockRecordFile +func SetFileMerkleInfo(backend *core.Backend, file *blockchain.BlockRecordFile) (valid bool) { + if file.Size <= merkle.MinimumFragmentSize { + // If smaller or equal than the minimum fragment size, the merkle tree is not used. + file.MerkleRootHash = file.Hash + file.FragmentSize = merkle.MinimumFragmentSize + } else { + // Get the information from the Warehouse .merkle companion file. + tree, status, _ := backend.UserWarehouse.ReadMerkleTree(file.Hash, true) + if status != warehouse.StatusOK { + return false + } + + file.MerkleRootHash = tree.RootHash + file.FragmentSize = tree.FragmentSize + } + + return true +} diff --git a/webapi/GeoIP.go b/webapi/GeoIP.go new file mode 100644 index 0000000..aa551fb --- /dev/null +++ b/webapi/GeoIP.go @@ -0,0 +1,50 @@ +/* +File Name: GeoIP.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner + +Support for the free MaxMind database 'GeoLite2 City'. +Information about the database: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data + +Potential libraries: +* https://github.com/IncSW/geoip2 +* https://github.com/oschwald/maxminddb-golang + +The IncSW lib was chosen because it has 0 dependencies - awesome! + +*/ + +package webapi + +import ( + "net" + + "github.com/IncSW/geoip2" + "github.com/PeernetOfficial/core" +) + +func (api *WebapiInstance) InitGeoIPDatabase(filename string) (err error) { + api.geoipCityReader, err = geoip2.NewCityReaderFromFile(filename) + return err +} + +func (api *WebapiInstance) GeoIPLocation(IP net.IP) (latitude, longitude float64, valid bool) { + if api.geoipCityReader == nil { + return 0, 0, false + } + + record, err := api.geoipCityReader.Lookup(IP) + if err != nil { + return 0, 0, false + } + + return record.Location.Latitude, record.Location.Longitude, true +} + +func (api *WebapiInstance) Peer2GeoIP(peer *core.PeerInfo) (latitude, longitude float64, valid bool) { + if connection := peer.GetConnection2Share(false, true, true); connection != nil { + return api.GeoIPLocation(connection.Address.IP) + } + + return 0, 0, false +} diff --git a/webapi/HTTP Range.go b/webapi/HTTP Range.go new file mode 100644 index 0000000..6dbf473 --- /dev/null +++ b/webapi/HTTP Range.go @@ -0,0 +1,109 @@ +/* +File Name: HTTP Range.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "errors" + "net/http" + "net/textproto" + "strconv" + "strings" +) + +// Fork from https://golang.org/src/net/http/fs.go + +// HTTPRange represents an HTTP range +type HTTPRange struct { + start, length int +} + +// ParseRangeHeader parses a Range header string as per RFC 7233. +func ParseRangeHeader(s string, size int, noMultiRange bool) ([]HTTPRange, error) { + if s == "" { + return nil, nil // header not present + } + const b = "bytes=" + if !strings.HasPrefix(s, b) { + return nil, errors.New("invalid range") + } + var ranges []HTTPRange + for _, ra := range strings.Split(s[len(b):], ",") { + ra = textproto.TrimString(ra) + if ra == "" { + continue + } + i := strings.Index(ra, "-") + if i < 0 { + return nil, errors.New("invalid range") + } + start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:]) + var r HTTPRange + if start == "" { + if size < 0 { + return nil, errors.New("range start relative to end not supported") + } + // If no start is specified, end specifies the + // range start relative to the end of the file. + i, err := strconv.ParseInt(end, 10, 64) + if err != nil { + return nil, errors.New("invalid range") + } + if int(i) > size { + i = int64(size) + } + r.start = size - int(i) + r.length = size - r.start + } else { + i, err := strconv.ParseInt(start, 10, 64) + if err != nil || i < 0 { + return nil, errors.New("invalid range") + } + if size > 0 && int(i) >= size { + // If the range begins after the size of the content, then it does not overlap. -> always return error. + return nil, errors.New("start out of range") + } + r.start = int(i) + if end == "" { + // If no end is specified, range extends to end of the file. + if size < 0 { + //return nil, errors.New("open range not supported") + r.length = -1 + } else { + r.length = size - r.start + } + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || r.start > int(i) { + return nil, errors.New("invalid range") + } + if size > 0 && int(i) >= size { + i = int64(size - 1) + } + r.length = int(i) - r.start + 1 + } + } + ranges = append(ranges, r) + if noMultiRange && len(ranges) > 1 { + return nil, errors.New("multiple ranges not supported") + } + } + return ranges, nil +} + +// setContentLengthRangeHeader sets the appropriate Content-Length and Content-Range headers +func setContentLengthRangeHeader(w http.ResponseWriter, offset, transferSize, fileSize uint64, ranges []HTTPRange) { + // Set the Content-Length header, always to the actual size of transferred data. + w.Header().Set("Content-Length", strconv.FormatUint(transferSize, 10)) + + // Set the Content-Range header if needed. + if len(ranges) == 1 { + w.Header().Set("Content-Range", "bytes "+strconv.FormatUint(offset, 10)+"-"+strconv.FormatUint(offset+transferSize-1, 10)+"/"+strconv.FormatUint(fileSize, 10)) + w.WriteHeader(http.StatusPartialContent) + } else { + w.WriteHeader(http.StatusOK) + } +} diff --git a/webapi/Profile.go b/webapi/Profile.go new file mode 100644 index 0000000..024ded0 --- /dev/null +++ b/webapi/Profile.go @@ -0,0 +1,153 @@ +/* +File Name: Profile.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "net/http" + "strconv" + + "github.com/PeernetOfficial/core/blockchain" +) + +// apiProfileData contains profile metadata stored on the blockchain. Any data is treated as untrusted and unverified by default. +type apiProfileData struct { + Fields []apiBlockRecordProfile `json:"fields"` // All fields + Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API. See blockchain.StatusX. +} + +// apiBlockRecordProfile provides information about the end user. Note that all profile data is arbitrary and shall be considered untrusted and unverified. +// To establish trust, the user must load Certificates into the blockchain that validate certain data. +type apiBlockRecordProfile struct { + Type uint16 `json:"type"` // See ProfileX constants. + // Depending on the exact type, one of the below fields is used for proper encoding: + Text string `json:"text"` // Text value. UTF-8 encoding. + Blob []byte `json:"blob"` // Binary data +} + +/* +apiProfileList lists all users profile fields. + +Request: GET /profile/list +Response: 200 with JSON structure apiProfileData +*/ +func (api *WebapiInstance) apiProfileList(w http.ResponseWriter, r *http.Request) { + fields, status := api.backend.UserBlockchain.ProfileList() + + result := apiProfileData{Status: status} + for n := range fields { + result.Fields = append(result.Fields, blockRecordProfileToAPI(fields[n])) + } + + EncodeJSON(api.backend, w, r, result) +} + +/* +apiProfileRead reads a specific users profile field. See core.ProfileX for recognized fields. + +Request: GET /profile/read?field=[index] +Response: 200 with JSON structure apiProfileData +*/ +func (api *WebapiInstance) apiProfileRead(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + fieldN, err1 := strconv.Atoi(r.Form.Get("field")) + + if err1 != nil || fieldN < 0 { + http.Error(w, "", http.StatusBadRequest) + return + } + + var result apiProfileData + + var data []byte + if data, result.Status = api.backend.UserBlockchain.ProfileReadField(uint16(fieldN)); result.Status == blockchain.StatusOK { + result.Fields = append(result.Fields, blockRecordProfileToAPI(blockchain.BlockRecordProfile{Type: uint16(fieldN), Data: data})) + } + + EncodeJSON(api.backend, w, r, result) +} + +/* +apiProfileWrite writes profile fields. See core.ProfileX for recognized fields. + +Request: POST /profile/write with JSON structure apiProfileData +Response: 200 with JSON structure apiBlockchainBlockStatus +*/ +func (api *WebapiInstance) apiProfileWrite(w http.ResponseWriter, r *http.Request) { + var input apiProfileData + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var fields []blockchain.BlockRecordProfile + + for n := range input.Fields { + fields = append(fields, blockRecordProfileFromAPI(input.Fields[n])) + } + + newHeight, newVersion, status := api.backend.UserBlockchain.ProfileWrite(fields) + + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +/* +apiProfileDelete deletes profile fields identified by the types. See core.ProfileX for recognized fields. + +Request: POST /profile/delete with JSON structure apiProfileData +Response: 200 with JSON structure apiBlockchainBlockStatus +*/ +func (api *WebapiInstance) apiProfileDelete(w http.ResponseWriter, r *http.Request) { + var input apiProfileData + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var fields []uint16 + + for n := range input.Fields { + fields = append(fields, input.Fields[n].Type) + } + + newHeight, newVersion, status := api.backend.UserBlockchain.ProfileDelete(fields) + + EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +// --- conversion from core to API data --- + +func blockRecordProfileToAPI(input blockchain.BlockRecordProfile) (output apiBlockRecordProfile) { + output.Type = input.Type + + switch input.Type { + case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress: + output.Text = input.Text() + + case blockchain.ProfilePicture: + output.Blob = input.Data + + default: + output.Blob = input.Data + } + + return output +} + +func blockRecordProfileFromAPI(input apiBlockRecordProfile) (output blockchain.BlockRecordProfile) { + output.Type = input.Type + + switch input.Type { + case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress: + output.Data = []byte(input.Text) + + case blockchain.ProfilePicture: + output.Data = input.Blob + + default: + output.Data = input.Blob + } + + return output +} diff --git a/webapi/Search Dispatch.go b/webapi/Search Dispatch.go new file mode 100644 index 0000000..3107bfa --- /dev/null +++ b/webapi/Search Dispatch.go @@ -0,0 +1,83 @@ +/* +File Name: Search Dispatch.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "bytes" + "fmt" + "time" + + "github.com/PeernetOfficial/core/blockchain" +) + +func (api *WebapiInstance) dispatchSearch(input SearchRequest) (job *SearchJob) { + Timeout := input.Parse() + Filter := input.ToSearchFilter() + + // create the search job + job = api.CreateSearchJob(Timeout, input.MaxResults, Filter) + + // todo: create actual search clients! + job.Status = SearchStatusLive + + go job.localSearch(api, input.Term) + + api.RemoveJobDefer(job, job.timeout+time.Minute*10) + + return job +} + +func (job *SearchJob) localSearch(api *WebapiInstance, term string) { + if api.backend.SearchIndex == nil { + job.Status = SearchStatusNoIndex + return + } + + results := api.backend.SearchIndex.Search(term) + + job.ResultSync.Lock() + +resultLoop: + for _, result := range results { + file, _, found, err := api.backend.ReadFile(result.PublicKey, result.BlockchainVersion, result.BlockNumber, result.FileID) + if err != nil || !found { + continue + } + + // Deduplicate based on file hash from the same peer. + for n := range job.AllFiles { + if bytes.Equal(job.AllFiles[n].Hash, file.Hash) && bytes.Equal(job.AllFiles[n].NodeID, file.NodeID) { + continue resultLoop + } + } + + if bytes.Equal(file.NodeID, api.backend.SelfNodeID()) { + // Indicates data from the current user. + file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1)) + } else if peer := api.backend.NodelistLookup(file.NodeID); peer != nil { + // add the tags 'Shared By Count' and 'Shared By GeoIP' + file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1)) + if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { + sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) + file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP)) + } + } + + // new result + newFile := blockRecordFileToAPI(file) + + job.Files = append(job.Files, &newFile) + job.AllFiles = append(job.AllFiles, &newFile) + job.requireSort = true + job.statsAdd(&newFile) + } + + job.Status = SearchStatusTerminated + + job.ResultSync.Unlock() + job.Terminate() +} diff --git a/webapi/Search Job.go b/webapi/Search Job.go new file mode 100644 index 0000000..e0d39d3 --- /dev/null +++ b/webapi/Search Job.go @@ -0,0 +1,469 @@ +/* +File Name: Search Job.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "sort" + "sync" + "time" + + "github.com/PeernetOfficial/core/blockchain" + "github.com/google/uuid" +) + +// SearchFilter allows to filter search results based on the criteria. +type SearchFilter struct { + IsDates bool // Whether the from/to dates are valid, both are required. + DateFrom time.Time // Optional date from + DateTo time.Time // Optional date to + FileType int // File type such as binary, text document etc. See core.TypeX. -1 = not used. + FileFormat int // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. + Sort int // Sort order. See SortX. + SizeMin int // Min file size in bytes. -1 = not used. + SizeMax int // Max file size in bytes. -1 = not used. +} + +// SearchJob is a collection of search jobs +type SearchJob struct { + // input settings + id uuid.UUID // The job id + timeout time.Duration // timeout set for all searches + maxResult int // max results user-facing. + + filtersStart SearchFilter // Filters when starting the search. They cannot be changed later on. Any incoming file is checked against them, even if there are different runtime filters. + filtersRuntime SearchFilter // Runtime Filters. They allow filtering results after they were received. + + // File statistics (filters are ignored) of returned results. Map value is always count of files. + stats struct { + sync.RWMutex // Synced access to maps + date map[time.Time]int // Files per day (rounded down to midnight) + fileType map[uint8]int // Files per File Type + fileFormat map[uint16]int // Files per File Format + total int // Total count of files + } + + // -- result data -- + + // Status indicates the overall search status. This will be removed later when relying on search clients. + Status int + + // runtime data + //clients []*SearchClient // all search clients + clientsMutex sync.Mutex // mutex for manipulating client list + + // List of files found but not yet returned via API to the caller. They are subject to sorting. + Files []*ApiFile + requireSort bool // if Files requires sort before returning the results + + // FreezeFiles is a list of files that were already finally delivered via the API. They may NOT change in sorting. + FreezeFiles []*ApiFile + + // List of all files. Does not change based on sorting or runtime filters. This list only gets expanded. + AllFiles []*ApiFile + + ResultSync sync.Mutex // ResultSync ensures unique access to the file results + + currentOffset int // for always getting the next results +} + +const ( + SearchStatusNotStarted = iota // Search was not yet started + SearchStatusLive // Search running + SearchStatusTerminated // Search is terminated. No more results are expected. + SearchStatusNoIndex // Search is terminated. No search index to use. +) + +// CreateSearchJob creates a new search job and adds it to the lookup list. +// Timeout and MaxResults must be set and must not be 0. +func (api *WebapiInstance) CreateSearchJob(Timeout time.Duration, MaxResults int, Filter SearchFilter) (job *SearchJob) { + job = &SearchJob{} + job.Status = SearchStatusNotStarted + job.id = uuid.New() + job.timeout = Timeout + job.maxResult = MaxResults + job.filtersStart = Filter + job.filtersRuntime = Filter // initialize the runtime filters as the same + + job.stats.date = make(map[time.Time]int) + job.stats.fileType = make(map[uint8]int) + job.stats.fileFormat = make(map[uint16]int) + + // add to the list of jobs + api.allJobsMutex.Lock() + api.allJobs[job.id] = job + api.allJobsMutex.Unlock() + + return +} + +// ReturnResult returns the selected results. +func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*ApiFile) { + if Limit == 0 { + return Result + } + + job.ResultSync.Lock() + defer job.ResultSync.Unlock() + + // serve files from frozen list? + if Offset < len(job.FreezeFiles) { + countCopy := len(job.FreezeFiles) - Offset + if countCopy > Limit { + countCopy = Limit + } + Result = job.FreezeFiles[Offset : Offset+countCopy] + Limit -= countCopy + Offset = 0 + } else { + Offset -= len(job.FreezeFiles) + } + + if Limit == 0 { + return Result + } + + // go through the live results and fill the list + if Offset >= len(job.Files) { // offset wants to skip entire queue? + job.FreezeFiles = append(job.FreezeFiles, job.Files...) + job.Files = nil + return Result + } + + // check if a sort is required before using this queue + if job.requireSort { + job.requireSort = false + + job.Files = SortFiles(job.Files, job.filtersRuntime.Sort) + } + + // set the amount of files to copy + countCopy := len(job.Files) - Offset + if countCopy > Limit { + countCopy = Limit + } + + // copy the results and freeze them + Result = append(Result, job.Files[Offset:Offset+countCopy]...) + + // note that freeze disregards the offset, it has to freeze any elements before! + job.FreezeFiles = append(job.FreezeFiles, job.Files[:Offset+countCopy]...) + job.Files = job.Files[Offset+countCopy:] + + //Limit -= countCopy + + return Result +} + +// ReturnNext returns the next results. Call must be serialized. +func (job *SearchJob) ReturnNext(Limit int) (Result []*ApiFile) { + Result = job.ReturnResult(job.currentOffset, Limit) + job.currentOffset += len(Result) + + return +} + +// PeekResult returns the selected results but will not change any frozen files or impact auto offset +func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*ApiFile) { + job.ResultSync.Lock() + defer job.ResultSync.Unlock() + + // serve files from frozen list? + if Offset < len(job.FreezeFiles) { + countCopy := len(job.FreezeFiles) - Offset + if countCopy > Limit { + countCopy = Limit + } + Result = job.FreezeFiles[Offset : Offset+countCopy] + Limit -= countCopy + Offset = 0 + } else { + Offset -= len(job.FreezeFiles) + } + + if Limit == 0 || Offset >= len(job.Files) { // offset wants to skip entire queue? + return Result + } + + // check if a sort is required before using this queue + if job.requireSort { + job.requireSort = false + + job.Files = SortFiles(job.Files, job.filtersRuntime.Sort) + } + + countCopy := len(job.Files) - Offset + if countCopy > Limit { + countCopy = Limit + } + + // copy the results + Result = append(Result, job.Files[Offset:Offset+countCopy]...) + + return Result +} + +// RuntimeFilter allows to apply filters at runtime to search jobs that already started. To remove the filters, call this function without the filters set. +func (job *SearchJob) RuntimeFilter(Filter SearchFilter) { + job.ResultSync.Lock() + defer job.ResultSync.Unlock() + + job.filtersRuntime = Filter + + // FreezeFiles and current offset is reset + job.FreezeFiles = nil + job.currentOffset = 0 + + // files remain in AllFiles, but Files needs to be filtered based on the new filter + job.Files = []*ApiFile{} + job.requireSort = false // Sorting is done immediately below + + // set Files based on AllFiles with the filter + for m := range job.AllFiles { + // only append if filter is matching + if job.isFileFiltered(job.AllFiles[m]) { + job.Files = append(job.Files, job.AllFiles[m]) + } + + // sort, if a sort order is defined + if job.filtersRuntime.Sort > 0 { + job.Files = SortFiles(job.Files, job.filtersRuntime.Sort) + } + } +} + +// isFileFiltered returns true if the file conforms to the runtime filter. If there is no runtime filter, it always returns true. +func (job *SearchJob) isFileFiltered(file *ApiFile) bool { + if job.filtersRuntime.FileType >= 0 && file.Type != uint8(job.filtersRuntime.FileType) { + return false + } + + if job.filtersRuntime.FileFormat >= 0 && file.Format != uint16(job.filtersRuntime.FileFormat) { + return false + } + + // Note: If the date is not available in the file, it will be filtered out. Since this is the mapped Shared Date this should normally not occur though. + if job.filtersRuntime.IsDates && (file.Date.IsZero() || file.Date.Before(job.filtersRuntime.DateFrom) || file.Date.After(job.filtersRuntime.DateTo)) { + return false + } + + if job.filtersRuntime.SizeMin >= 0 && file.Size < uint64(job.filtersRuntime.SizeMin) || job.filtersRuntime.SizeMax >= 0 && file.Size > uint64(job.filtersRuntime.SizeMax) { + return false + } + + return true +} + +// SortFiles sorts a list of files. It returns a sorted list. 0 = no sorting, 1 = Relevance ASC, 2 = Relevance DESC, 3 = Date ASC, 4 = Date DESC, 5 = Name ASC, 6 = Name DESC +func SortFiles(files []*ApiFile, Sort int) (sorted []*ApiFile) { + switch Sort { + case SortRelevanceAsc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) }) // first as date for secondary sorting + //sort.SliceStable(files, func(i, j int) bool { return files[i].Score < files[j].Score }) // TODO + case SortRelevanceDec: + sort.SliceStable(files, func(i, j int) bool { return files[j].Date.Before(files[i].Date) }) // first as date for secondary sorting + //sort.SliceStable(files, func(i, j int) bool { return files[i].Score > files[j].Score }) // TODO + + case SortDateAsc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) }) + case SortDateDesc: + sort.SliceStable(files, func(i, j int) bool { return files[j].Date.Before(files[i].Date) }) + + case SortNameAsc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Name < files[j].Name }) + case SortNameDesc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Name > files[j].Name }) + + case SortSizeAsc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Size < files[j].Size }) + case SortSizeDesc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Size > files[j].Size }) + + case SortSharedByCountAsc: + sort.SliceStable(files, func(i, j int) bool { + return files[i].GetMetadata(blockchain.TagSharedByCount).Number < files[j].GetMetadata(blockchain.TagSharedByCount).Number + }) + case SortSharedByCountDesc: + sort.SliceStable(files, func(i, j int) bool { + return files[i].GetMetadata(blockchain.TagSharedByCount).Number > files[j].GetMetadata(blockchain.TagSharedByCount).Number + }) + + } + + return files +} + +// IsSearchResults checks if search results may be expected (either files are in queue or a search is running) +func (job *SearchJob) IsSearchResults() bool { + // check for any available results. Do not use any lock here as this is read only. + return len(job.Files) > 0 || !job.IsTerminated() +} + +// isFileReceived checks if a file was already received, preventing double results +func (job *SearchJob) isFileReceived(id uuid.UUID) (exists bool) { + // Future: A map would be likely faster than iterating over all results. + for m := range job.AllFiles { + if id == job.AllFiles[m].ID { + return true + } + } + + return false +} + +// ---- job list management ---- + +// RemoveJob removes the job structure from the list. Terminate should be called before. Unless the search is manually removed, it stays forever in the list. +func (api *WebapiInstance) RemoveJob(job *SearchJob) { + api.allJobsMutex.Lock() + delete(api.allJobs, job.id) // delete is safe to call multiple times, so auto-removal and manual one are fine and need no syncing + api.allJobsMutex.Unlock() +} + +// RemoveDefer removes the search job after a given time after all searches are terminated. This can be used for automated time delayed removal. Do not create additional search clients after deferal removing. +func (api *WebapiInstance) RemoveJobDefer(job *SearchJob, Duration time.Duration) { + go func() { + // for _, client := range job.clients { + // <-client.TerminateSignal + // } + + <-time.After(Duration) + api.RemoveJob(job) + }() +} + +// JobLookup looks up a job. Returns nil if not found. +func (api *WebapiInstance) JobLookup(id uuid.UUID) (job *SearchJob) { + api.allJobsMutex.RLock() + job = api.allJobs[id] + api.allJobsMutex.RUnlock() + + return job +} + +// IsTerminated checks if all searches are finished. The job itself does not record a termination signal. +func (job *SearchJob) IsTerminated() bool { + job.clientsMutex.Lock() + defer job.clientsMutex.Unlock() + + // for n := range job.clients { + // if !job.clients[n].IsTerminated { + // return false + // } + // } + + return job.Status == SearchStatusTerminated || job.Status == SearchStatusNoIndex +} + +// Terminate terminates all searches +func (job *SearchJob) Terminate() { + job.clientsMutex.Lock() + defer job.clientsMutex.Unlock() + + // for n := range job.clients { + // if !job.clients[n].IsTerminated { + // job.clients[n].Terminate(true) + // } + // } +} + +// WaitTerminate waits until all search clients are terminated. Do not create additional search clients after calling this function. +func (job *SearchJob) WaitTerminate() { + //for _, client := range job.clients { + // <-client.TerminateSignal + //} +} + +// ---- statistics ---- + +// SearchStatisticRecordDay is a single record containing date info. +type SearchStatisticRecordDay struct { + Date time.Time `json:"date"` // The day (which covers the full 24 hours). Always rounded down to midnight. + Count int `json:"count"` // Count of files. +} + +// SearchStatisticRecord is a single record. +type SearchStatisticRecord struct { + Key int `json:"key"` // Key index. The exact meaning depends on where this structure is used. + Count int `json:"count"` // Count of files for the given key +} + +// SearchStatisticData contains statistics on search results. +type SearchStatisticData struct { + Date []SearchStatisticRecordDay `json:"date"` // Files per date + FileType []SearchStatisticRecord `json:"filetype"` // Files per file type + FileFormat []SearchStatisticRecord `json:"fileformat"` // Files per file format + Total int `json:"total"` // Total count of files +} + +// Statistics generates statistics on all results, regardless of runtime filters. +func (job *SearchJob) Statistics() (result SearchStatisticData) { + job.stats.RLock() + defer job.stats.RUnlock() + + result.Total = job.stats.total + + // Files per date. Sort dates to date ASC. + for key, value := range job.stats.date { + result.Date = append(result.Date, SearchStatisticRecordDay{Date: key, Count: value}) + } + sort.SliceStable(result.Date, func(i, j int) bool { return result.Date[i].Date.Before(result.Date[j].Date) }) + + // File Type and Format + for key, value := range job.stats.fileType { + result.FileType = append(result.FileType, SearchStatisticRecord{Key: int(key), Count: value}) + } + for key, value := range job.stats.fileFormat { + result.FileFormat = append(result.FileFormat, SearchStatisticRecord{Key: int(key), Count: value}) + } + + return +} + +// statsAdd counts the files in the statistics +func (job *SearchJob) statsAdd(files ...*ApiFile) { + job.stats.Lock() + defer job.stats.Unlock() + + for _, file := range files { + // Use file's Date field if available. + if !file.Date.IsZero() { + // Files per day + date := file.Date.Truncate(24 * time.Hour) + countDate := job.stats.date[date] + countDate++ + job.stats.date[date] = countDate + } + + // File Type and Format + countType := job.stats.fileType[file.Type] + countType++ + job.stats.fileType[file.Type] = countType + + countFormat := job.stats.fileFormat[file.Format] + countFormat++ + job.stats.fileFormat[file.Format] = countFormat + } + + job.stats.total += len(files) +} + +// ---- actual search & retrieving results ---- + +// appendSearchClient appends a search client to a job + +func (job *SearchJob) SearchAway() { + job.clientsMutex.Lock() + defer job.clientsMutex.Unlock() + + // TODO +} + +//func (job *SearchJob) appendSearchClient(search *dht.SearchClient) { +//} + +//func (job *SearchJob) receiveClientResults(client *dht.SearchClient) { +//} diff --git a/webapi/Search.go b/webapi/Search.go new file mode 100644 index 0000000..44a8c5b --- /dev/null +++ b/webapi/Search.go @@ -0,0 +1,417 @@ +/* +File Name: Search.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner + +/search Submit a search request +/search/result Return search results +/search/terminate Terminate a search +/search/result/ws Websocket to receive results as stream +/search/statistic Statistics about the results + +/explore List recently shared files + +*/ + +package webapi + +import ( + "net/http" + "strconv" + "time" + + "github.com/google/uuid" +) + +// SearchRequest is the information from the end-user for the search. Filters and sort order may be applied when starting the search, or at runtime when getting the results. +type SearchRequest struct { + Term string `json:"term"` // Search term. + Timeout int `json:"timeout"` // Timeout in seconds. 0 means default. This is the entire time the search may take. Found results are still available after this timeout. + MaxResults int `json:"maxresults"` // Total number of max results. 0 means default. + DateFrom string `json:"datefrom"` // Date from, both from/to are required if set. Format "2006-01-02 15:04:05". + DateTo string `json:"dateto"` // Date to, both from/to are required if set. Format "2006-01-02 15:04:05". + Sort int `json:"sort"` // See SortX. + TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate. + FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. -1 = not used. + FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. + SizeMin int `json:"sizemin"` // Min file size in bytes. -1 = not used. + SizeMax int `json:"sizemax"` // Max file size in bytes. -1 = not used. +} + +// Sort orders +const ( + SortNone = 0 // No sorting. Results are returned as they come in. + SortRelevanceAsc = 1 // Least relevant results first. + SortRelevanceDec = 2 // Most relevant results first. + SortDateAsc = 3 // Oldest first. + SortDateDesc = 4 // Newest first. + SortNameAsc = 5 // File name ascending. The folder name is not used for sorting. + SortNameDesc = 6 // File name descending. The folder name is not used for sorting. + SortSizeAsc = 7 // File size ascending. Smallest files first. + SortSizeDesc = 8 // File size descending. Largest files first. + SortSharedByCountAsc = 9 // Shared by count ascending. Files that are shared by the least count of peers first. + SortSharedByCountDesc = 10 // Shared by count descending. Files that are shared by the most count of peers first. +) + +// SearchRequestResponse is the result to the initial search request +type SearchRequestResponse struct { + ID uuid.UUID `json:"id"` // ID of the search job. This is used to get the results. + Status int `json:"status"` // Status of the search: 0 = Success (ID valid), 1 = Invalid Term, 2 = Error Max Concurrent Searches +} + +// SearchResult contains the search results. +type SearchResult struct { + Status int `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying + Files []ApiFile `json:"files"` // List of files found + Statistic interface{} `json:"statistic"` // Statistics of all results (independent from applied filters), if requested. Only set if files are returned (= if statistics changed). See SearchStatisticData. +} + +// SearchStatistic contains statistics on search results. Statistics are always calculated over all results, regardless of any applied runtime filters. +type SearchStatistic struct { + SearchStatisticData + Status int `json:"status"` // Status: 0 = Success + IsTerminated bool `json:"terminated"` // Whether the search is terminated, meaning that statistics won't change +} + +const apiDateFormat = "2006-01-02 15:04:05" + +/* +apiSearch submits a search request + +Request: POST /search with JSON SearchRequest +Result: 200 on success with JSON SearchRequestResponse + 400 on invalid JSON +*/ +func (api *WebapiInstance) apiSearch(w http.ResponseWriter, r *http.Request) { + + var input SearchRequest + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + if input.Timeout <= 0 { + input.Timeout = 20 + } + if input.MaxResults <= 0 { + input.MaxResults = 200 + } + + // Terminate previous searches, if their IDs were supplied. This allows terminating the old search immediately without making a separate /search/terminate request. + for _, terminate := range input.TerminateID { + if job := api.JobLookup(terminate); job != nil { + job.Terminate() + api.RemoveJob(job) + } + } + + job := api.dispatchSearch(input) + + EncodeJSON(api.backend, w, r, SearchRequestResponse{Status: 0, ID: job.id}) +} + +/* +apiSearchResult returns results. The default limit is 100. +If reset is set, all results will be filtered and sorted according to the settings. This means that the new first result will be returned again and internal result offset is set to 0. + +Request: GET /search/result?id=[UUID]&limit=[max records] +Optional parameters: + &reset=[0|1] to reset the filters or sort orders with any of the below parameters (all required): + &filetype=[File Type] + &fileformat=[File Format] + &from=[Date From]&to=[Date To] + &sizemin=[Minimum file size] + &sizemax=[Maximum file size] + &sort=[sort order] + &offset=[absolute offset] with &limit=[records] to get items pagination style. Returned items (and ones before) are automatically frozen. +Result: 200 with JSON structure SearchResult. Check the field status. +*/ +func (api *WebapiInstance) apiSearchResult(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + limit, err := strconv.Atoi(r.Form.Get("limit")) + if err != nil { + limit = 100 + } + offset, errOffset := strconv.Atoi(r.Form.Get("offset")) + + // find the job ID + job := api.JobLookup(jobID) + if job == nil { + EncodeJSON(api.backend, w, r, SearchResult{Status: 2}) + return + } + + // filters and sort parameter + if filterReset, _ := strconv.ParseBool(r.Form.Get("reset")); filterReset { + fileType, _ := strconv.Atoi(r.Form.Get("filetype")) + fileFormat, _ := strconv.Atoi(r.Form.Get("fileformat")) + dateFrom := r.Form.Get("from") + dateTo := r.Form.Get("to") + sort, _ := strconv.Atoi(r.Form.Get("sort")) + sizeMin, _ := strconv.Atoi(r.Form.Get("sizemin")) + sizeMax, _ := strconv.Atoi(r.Form.Get("sizemax")) + + filter := inputToSearchFilter(sort, fileType, fileFormat, dateFrom, dateTo, sizeMin, sizeMax) + + job.RuntimeFilter(filter) + } + + // query all results + var resultFiles []*ApiFile + if errOffset == nil { + resultFiles = job.ReturnResult(offset, limit) + } else { + resultFiles = job.ReturnNext(limit) + } + + var result SearchResult + result.Files = []ApiFile{} + + // loop over results + for n := range resultFiles { + result.Files = append(result.Files, *resultFiles[n]) + } + + // set the status + if len(result.Files) > 0 { + if job.IsSearchResults() { + result.Status = 0 // 0 = Success with results + } else { + result.Status = 1 // No more results to expect + } + } else { + switch job.Status { + case SearchStatusLive: + result.Status = 3 // No results yet available keep trying + + case SearchStatusTerminated: + result.Status = 1 // No more results to expect + + default: // SearchStatusNoIndex, SearchStatusNotStarted + result.Status = 1 // No more results to expect + } + } + + // embedded statistics? + if returnStats, _ := strconv.ParseBool(r.Form.Get("stats")); returnStats { + result.Statistic = job.Statistics() + } + + EncodeJSON(api.backend, w, r, result) +} + +/* +apiSearchResultStream provides a websocket to receive results as stream. + +Request: GET /search/result/ws?id=[UUID]&limit=[optional max records] +Result: If successful, upgrades to a websocket and sends JSON structure SearchResult messages. + Limit is optional. Not used if ommitted or 0. +*/ +func (api *WebapiInstance) apiSearchResultStream(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + limit, err := strconv.Atoi(r.Form.Get("limit")) + useLimit := err == nil + + // look up the job + job := api.JobLookup(jobID) + if job == nil { + EncodeJSON(api.backend, w, r, SearchResult{Status: 2}) + return + } + + // upgrade to websocket + conn, err := WSUpgrader.Upgrade(w, r, nil) + if err != nil { + // gorilla will automatically respond with "400 Bad Request", no other response is therefore necessary + return + } + + defer conn.Close() + + // loop to get new results and send out via the web socket. + // Only exit if limit is reached if used, otherwise only if there are no result or the connection breaks. + for { + // query all results + var resultFiles []*ApiFile + + queryCount := 1 + if useLimit { + queryCount = limit + } + resultFiles = job.ReturnNext(queryCount) + + if useLimit { + limit -= len(resultFiles) + } + + // loop over results + var result SearchResult + result.Files = []ApiFile{} + + for n := range resultFiles { + result.Files = append(result.Files, *resultFiles[n]) + } + + if !job.IsSearchResults() { + result.Status = 1 // No more results to expect + + if len(result.Files) == 0 { + conn.WriteJSON(result) // final message + return + } + } + + // if no results, stall + if len(result.Files) == 0 { + time.Sleep(time.Millisecond * 100) + continue + } + + // send out the results via the websocket + if err := conn.WriteJSON(result); err != nil { + return + } + + // Check whether to continue. If the limit is used break once all done. + if (useLimit && limit <= 0) || result.Status == 1 { + break + } + } +} + +/* +apiSearchTerminate terminates a search + +Request: GET /search/terminate?id=[UUID] +Response: 204 Empty + 400 Invalid input + 404 ID not found +*/ +func (api *WebapiInstance) apiSearchTerminate(w http.ResponseWriter, r *http.Request) { + + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + // look up the job + job := api.JobLookup(jobID) + if job == nil { + http.Error(w, "", http.StatusNotFound) + return + } + + // terminate and remove it from the list + job.Terminate() + api.RemoveJob(job) + + w.WriteHeader(http.StatusNoContent) +} + +/* +apiSearchStatistic returns search result statistics. Statistics are always calculated over all results, regardless of any applied runtime filters. + +Request: GET /search/result?id=[UUID] +Result: 200 with JSON structure SearchStatistic. Check the field status (0 = Success, 2 = ID not found). +*/ +func (api *WebapiInstance) apiSearchStatistic(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + // find the job ID + job := api.JobLookup(jobID) + if job == nil { + EncodeJSON(api.backend, w, r, SearchStatistic{Status: 2}) + return + } + + stats := job.Statistics() + + EncodeJSON(api.backend, w, r, SearchStatistic{SearchStatisticData: stats, Status: 0, IsTerminated: job.IsTerminated()}) +} + +/* +apiExplore returns recently shared files in Peernet. Results are returned in real-time. The file type is an optional filter. See TypeX. +Special type -2 = Binary, Compressed, Container, Executable. This special type includes everything except Documents, Video, Audio, Ebooks, Picture, Text. + +Request: GET /explore?limit=[max records]&type=[file type]&offset=[offset] +Result: 200 with JSON structure SearchResult. Check the field status. +*/ +func (api *WebapiInstance) apiExplore(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, err := strconv.Atoi(r.Form.Get("limit")) + if err != nil { + limit = 100 + } + fileType, err := strconv.Atoi(r.Form.Get("type")) + if err != nil { + fileType = -1 + } + + resultFiles := api.queryRecentShared(api.backend, fileType, uint64(limit*20/100), uint64(offset), uint64(limit)) + + var result SearchResult + result.Files = []ApiFile{} + + // loop over results + for n := range resultFiles { + result.Files = append(result.Files, blockRecordFileToAPI(resultFiles[n])) + } + + if len(result.Files) == 0 { + result.Status = 3 // No results yet available keep trying + } + + result.Status = 1 // No more results to expect + + EncodeJSON(api.backend, w, r, result) +} + +func (input *SearchRequest) Parse() (Timeout time.Duration) { + if input.Timeout == 0 { + Timeout = time.Second * 20 // default timeout: 20 seconds + } else { + Timeout = time.Duration(input.Timeout) * time.Second + } + + return +} + +// ToSearchFilter converts the user input to a valid search filter +func (input *SearchRequest) ToSearchFilter() (output SearchFilter) { + return inputToSearchFilter(input.Sort, input.FileType, input.FileFormat, input.DateFrom, input.DateTo, input.SizeMin, input.SizeMax) +} + +func inputToSearchFilter(Sort, FileType, FileFormat int, DateFrom, DateTo string, SizeMin, SizeMax int) (output SearchFilter) { + output.Sort = Sort + output.FileType = FileType + output.FileFormat = FileFormat + output.SizeMin = SizeMin + output.SizeMax = SizeMax + + dateFrom, errFrom := time.Parse(apiDateFormat, DateFrom) + dateTo, errTo := time.Parse(apiDateFormat, DateTo) + if errFrom == nil && errTo == nil && dateFrom.Before(dateTo) { + output.DateFrom = dateFrom + output.DateTo = dateTo + output.IsDates = true + } + + return +} diff --git a/webapi/Shared Recent.go b/webapi/Shared Recent.go new file mode 100644 index 0000000..73cb9f7 --- /dev/null +++ b/webapi/Shared Recent.go @@ -0,0 +1,89 @@ +/* +File Name: Shared Recent.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ +package webapi + +import ( + "fmt" + + "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" +) + +// queryRecentShared returns recently shared files on the network from random peers until the limit is reached. +func (api *WebapiInstance) queryRecentShared(backend *core.Backend, fileType int, limitPeer, offsetTotal, limitTotal uint64) (files []blockchain.BlockRecordFile) { + if limitPeer == 0 { + limitPeer = 1 + } + + // Use the peer list to know about active peers. Random order! + peerList := api.backend.PeerlistGet() + + // Files from peers exceeding the limit. It is used if from all peers the total limit is not reached. + var filesSeconday []blockchain.BlockRecordFile + + for _, peer := range peerList { + if peer.BlockchainHeight == 0 { + continue + } + + var filesFromPeer uint64 + + // decode blocks from top down + blockLoop: + for blockN := peer.BlockchainHeight - 1; blockN > 0; blockN-- { + blockDecoded, _, found, _ := backend.ReadBlock(peer.PublicKey, peer.BlockchainVersion, blockN) + if !found { + continue + } + + for _, record := range blockDecoded.RecordsDecoded { + if file, ok := record.(blockchain.BlockRecordFile); ok && isFileTypeMatchBlock(&file, fileType) { + // add the tags 'Shared By Count' and 'Shared By GeoIP' + file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1)) + if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { + sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) + file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP)) + } + + // found a new file! append. + if filesFromPeer < limitPeer { + filesFromPeer++ + + if offsetTotal > 0 { + offsetTotal-- + continue + } + + files = append(files, file) + + if uint64(len(files)) >= limitTotal { + return + } + } else if uint64(len(filesSeconday)) < limitTotal-uint64(len(files)) { + filesSeconday = append(filesSeconday, file) + } else { + break blockLoop + } + } + } + } + } + + files = append(files, filesSeconday...) + + return +} + +// isFileTypeMatchBlock checks if the file type matches. -1 = accept any. -2 = core.TypeBinary, core.TypeCompressed, core.TypeContainer, core.TypeExecutable. +func isFileTypeMatchBlock(file *blockchain.BlockRecordFile, fileType int) bool { + if fileType == -1 { + return true + } else if fileType == -2 { + return file.Type == core.TypeBinary || file.Type == core.TypeCompressed || file.Type == core.TypeContainer || file.Type == core.TypeExecutable + } + + return file.Type == uint8(fileType) +} diff --git a/webapi/Status.go b/webapi/Status.go new file mode 100644 index 0000000..a316e4d --- /dev/null +++ b/webapi/Status.go @@ -0,0 +1,126 @@ +/* +File Name: Status.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "encoding/hex" + "fmt" + "net/http" + "strconv" +) + +func apiTest(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) +} + +type apiResponseStatus struct { + Status int `json:"status"` // Status code: 0 = Ok. + IsConnected bool `json:"isconnected"` // Whether connected to Peernet. + CountPeerList int `json:"countpeerlist"` // Count of peers in the peer list. Note that this contains peers that are considered inactive, but have not yet been removed from the list. + CountNetwork int `json:"countnetwork"` // Count of total peers in the network. + // This is usually a higher number than CountPeerList, which just represents the current number of connected peers. + // The CountNetwork number is going to be queried from root peers which may or may not have a limited view. +} + +/* +apiStatus returns the current connectivity status to the network +Request: GET /status +Result: 200 with JSON structure Status +*/ +func (api *WebapiInstance) apiStatus(w http.ResponseWriter, r *http.Request) { + status := apiResponseStatus{Status: 0, CountPeerList: api.backend.PeerlistCount()} + status.CountNetwork = status.CountPeerList // For now always same as CountPeerList, until native Statistics message to root peers is available. + + // Connected: If at leat 2 peers. + // This metric needs to be improved in the future, as root peers never disconnect. + // Instead, the core should keep a count of "active peers". + status.IsConnected = status.CountPeerList >= 2 + + EncodeJSON(api.backend, w, r, status) +} + +type apiResponsePeerSelf struct { + PeerID string `json:"peerid"` // Peer ID. This is derived from the public in compressed form. + NodeID string `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. +} + +/* +apiAccountInfo provides information about the current account. +Request: GET /account/info +Result: 200 with JSON structure apiResponsePeerSelf +*/ +func (api *WebapiInstance) apiAccountInfo(w http.ResponseWriter, r *http.Request) { + response := apiResponsePeerSelf{} + response.NodeID = hex.EncodeToString(api.backend.SelfNodeID()) + + _, publicKey := api.backend.ExportPrivateKey() + response.PeerID = hex.EncodeToString(publicKey.SerializeCompressed()) + + EncodeJSON(api.backend, w, r, response) +} + +/* +apiAccountDelete deletes the current account. The confirm parameter must include the user's choice. +Request: GET /account/delete?confirm=[0 or 1] +Result: 204 if the user choses not to delete the account + + 200 if successfully deleted +*/ +func (api *WebapiInstance) apiAccountDelete(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + if confirm, _ := strconv.ParseBool(r.Form.Get("confirm")); !confirm { + w.WriteHeader(http.StatusNoContent) + return + } + + api.backend.DeleteAccount() + + w.WriteHeader(http.StatusOK) +} + +/* +apiStatusPeers returns the information about peers currently connected. +The GeoIP information may not alawys be available, for example if the GeoIP file is not available or the mapping from IP address to location is not available. +Peers that are connected only via local network will not have a geo location. + +Request: GET /status/peers +Result: 200 with JSON array apiResponsePeerInfo +*/ +func (api *WebapiInstance) apiStatusPeers(w http.ResponseWriter, r *http.Request) { + var peers []apiResponsePeerInfo + + // query all nodes + for _, peer := range api.backend.PeerlistGet() { + peerInfo := apiResponsePeerInfo{ + PeerID: peer.PublicKey.SerializeCompressed(), + NodeID: peer.NodeID, + UserAgent: peer.UserAgent, + IsRoot: peer.IsRootPeer, + BlockchainHeight: peer.BlockchainHeight, + BlockchainVersion: peer.BlockchainVersion, + } + + if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { + peerInfo.GeoIP = fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) + } + + peers = append(peers, peerInfo) + } + + EncodeJSON(api.backend, w, r, peers) +} + +type apiResponsePeerInfo struct { + PeerID []byte `json:"peerid"` // Peer ID. This is derived from the public in compressed form. + NodeID []byte `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. + GeoIP string `json:"geoip"` // GeoIP location as "Latitude,Longitude" CSV format. Empty if location not available. + UserAgent string `json:"useragent"` // User Agent. + IsRoot bool `json:"isroot"` // If the peer is a root peer. + BlockchainHeight uint64 `json:"blockchainheight"` // Blockchain height + BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version +} diff --git a/webapi/Warehouse.go b/webapi/Warehouse.go new file mode 100644 index 0000000..0717611 --- /dev/null +++ b/webapi/Warehouse.go @@ -0,0 +1,151 @@ +/* +File Name: Warehouse.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "net/http" + "strconv" + + "github.com/PeernetOfficial/core/warehouse" +) + +// WarehouseResult is the response to creating a new file in the warehouse +type WarehouseResult struct { + Status int `json:"status"` // See warehouse.StatusX. + Hash []byte `json:"hash"` // Hash of the file. +} + +/* +apiWarehouseCreateFile creates a file in the warehouse. + +Request: POST /warehouse/create with raw data to create as new file +Response: 200 with JSON structure WarehouseResult +*/ +func (api *WebapiInstance) apiWarehouseCreateFile(w http.ResponseWriter, r *http.Request) { + hash, status, err := api.backend.UserWarehouse.CreateFile(r.Body, 0) + + if err != nil { + api.backend.LogError("warehouse.CreateFile", "status %d error: %v", status, err) + } + + EncodeJSON(api.backend, w, r, WarehouseResult{Status: status, Hash: hash}) +} + +/* +apiWarehouseCreateFilePath creates a file in the warehouse by copying it from an existing file. +Warning: An attacker could supply any local file using this function, put them into storage and read them! No input path verification or limitation is done. +In the future the API should be secured using a random API key and setting the CORS header prohibiting regular browsers to access the API. + +Request: GET /warehouse/create/path?path=[target path on disk] +Response: 200 with JSON structure WarehouseResult +*/ +func (api *WebapiInstance) apiWarehouseCreateFilePath(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + filePath := r.Form.Get("path") + if filePath == "" { + http.Error(w, "", http.StatusBadRequest) + return + } + + hash, status, err := api.backend.UserWarehouse.CreateFileFromPath(filePath) + + if err != nil { + api.backend.LogError("warehouse.CreateFile", "status %d error: %v", status, err) + } + + EncodeJSON(api.backend, w, r, WarehouseResult{Status: status, Hash: hash}) +} + +/* +apiWarehouseReadFile reads a file in the warehouse. + +Request: GET /warehouse/read?hash=[hash] + Optional parameters &offset=[file offset]&limit=[read limit in bytes] +Response: 200 with the raw file data + 404 if file was not found + 500 in case of internal error opening the file +*/ +func (api *WebapiInstance) apiWarehouseReadFile(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + if !valid1 { + http.Error(w, "", http.StatusBadRequest) + return + } + + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, _ := strconv.Atoi(r.Form.Get("limit")) + + status, bytesRead, err := api.backend.UserWarehouse.ReadFile(hash, int64(offset), int64(limit), w) + + switch status { + case warehouse.StatusFileNotFound: + w.WriteHeader(http.StatusNotFound) + return + case warehouse.StatusInvalidHash, warehouse.StatusErrorOpenFile, warehouse.StatusErrorSeekFile: + w.WriteHeader(http.StatusInternalServerError) + return + // Cannot catch warehouse.StatusErrorReadFile since data may have been already returned. + // In the future a special header indicating the expected file length could be sent (would require a callback in ReadFile), although the caller should already know the file size based on metadata. + } + + if err != nil { + api.backend.LogError("warehouse.ReadFile", "status %d read %d error: %v", status, bytesRead, err) + } +} + +/* +apiWarehouseDeleteFile deletes a file in the warehouse. + +Request: GET /warehouse/delete?hash=[hash] +Response: 200 with JSON structure WarehouseResult +*/ +func (api *WebapiInstance) apiWarehouseDeleteFile(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + if !valid1 { + http.Error(w, "", http.StatusBadRequest) + return + } + + status, err := api.backend.UserWarehouse.DeleteFile(hash) + + if err != nil { + api.backend.LogError("warehouse.DeleteFile", "status %d error: %v", status, err) + } + + EncodeJSON(api.backend, w, r, WarehouseResult{Status: status, Hash: hash}) +} + +/* +apiWarehouseReadFilePath reads a file from the warehouse and stores it to the target file. It fails with StatusErrorTargetExists if the target file already exists. +The path must include the full directory and file name. + +Request: GET /warehouse/read/path?hash=[hash]&path=[target path on disk] + Optional parameters &offset=[file offset]&limit=[read limit in bytes] +Response: 200 with JSON structure WarehouseResult +*/ +func (api *WebapiInstance) apiWarehouseReadFilePath(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + if !valid1 { + http.Error(w, "", http.StatusBadRequest) + return + } + + targetFile := r.Form.Get("path") + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, _ := strconv.Atoi(r.Form.Get("limit")) + + status, bytesRead, err := api.backend.UserWarehouse.ReadFileToDisk(hash, int64(offset), int64(limit), targetFile) + + if err != nil { + api.backend.LogError("warehouse.ReadFileToDisk", "status %d read %d error: %v", status, bytesRead, err) + } + + EncodeJSON(api.backend, w, r, WarehouseResult{Status: status, Hash: hash}) +} diff --git a/webapi/readme.md b/webapi/readme.md new file mode 100644 index 0000000..68c1427 --- /dev/null +++ b/webapi/readme.md @@ -0,0 +1,1088 @@ +# Web API + +The web API provides access to core functions via HTTP. + +It can be used by local client software to connect to the Peernet network and use functions such as share, search, and download files. + +## Use Considerations (when not to use it) + +* Do not expose this API to the internet or local network. The API provides direct access to the users blockchain. It provides sensitive actions such as deleting the account (including the private key). If the use of an API key is disabled, an unauthenticated attacker could abuse the API to add any local file to the user's blockchain and then read it. +* The API shall only run on a loopback IP such as `127.0.0.1` or `::1`. +* The API is not supposed to be used by regular web browsers. CORS HTTP headers are intentionally not set. +* You should use the API key functionality, which enforces the API key in every call using the HTTP header `x-api-key`. + +## Deployment + +The API must be initialized and started before use. The last parameter is the API key (in this example no API key is used). For security reasons it is recommended to use a random local port and provide a randomly generated API key. + +```go +webapi.Start([]string{"127.0.0.1:112"}, false, "", "", 10*time.Second, 10*time.Second, uuid.Nil) + +// To register an additional API endpoint: +webapi.Router.HandleFunc("/newfunction", newFunction).Methods("GET") +``` + +## API Key + +Each API instance should use a random UUID as API key. Subsequently, that UUID must be provided by the client in every API call in the `x-api-key` HTTP header. Failure to provide the API key in calls results in HTTP status 401 Unauthorized. + +This effectively secures the API against unauthenticated attackers, including other software running on the same machine, malicious websites using a DNS rebinding attack, and accidental link opening by the user. + +To disable the use of API keys a null UUID (= `00000000-0000-0000-0000-000000000000`) can be provided when starting the API. This may be useful for development purposes, but should never be used in production. + +# Available Functions + +These are the functions provided by the API: + +``` +/status Provide current connectivity status to the network + +/account/info Information about the current account +/account/delete Delete account + +/blockchain/header Header of the blockchain +/blockchain/append Append a block to the blockchain +/blockchain/read Read a block of the blockchain +/blockchain/file/add Add file to the blockchain +/blockchain/file/list List all files stored on the blockchain +/blockchain/file/delete Delete files from the blockchain +/blockchain/file/update Updates files on the blockchain + +/profile/list List all profile fields +/profile/read Read a profile field +/profile/write Write profile fields +/profile/delete Delete profile fields + +/search Submit a search request +/search/result Return search results +/search/result/ws Websocket to receive results +/search/terminate Terminate a search +/search/statistic Search result statistics + +/download/start Start the download of a file +/download/status Get the status of a download +/download/action Pause, resume, and cancel a download + +/explore List recently shared files + +/file/format Detect file type and format + +/warehouse/create Create a file in the warehouse +/warehouse/create/path Create a file in the warehouse via copy +/warehouse/read Read a file in the warehouse +/warehouse/read/path Read a file in the warehouse to disk +/warehouse/delete Delete a file in the warehouse +``` + +# API Documentation + +All times used by the API (both input and output) are UTC based. It is the frontend's responsibility to convert the times to the local time zone for visualization to the end user where appropriate. + +## Informational Functions + +### Status + +This function informs about the current connection status of the client to the network. Additional fields will be added in the future. + +``` +Request: GET /status +Response: 200 with JSON structure apiResponseStatus +``` + +```go +type apiResponseStatus struct { + Status int `json:"status"` // Status code: 0 = Ok. + IsConnected bool `json:"isconnected"` // Whether connected to Peernet. + CountPeerList int `json:"countpeerlist"` // Count of peers in the peer list. Note that this contains peers that are considered inactive, but have not yet been removed from the list. + CountNetwork int `json:"countnetwork"` // Count of total peers in the network. + // This is usually a higher number than CountPeerList, which just represents the current number of connected peers. + // The CountNetwork number is going to be queried from root peers which may or may not have a limited view into the network. +} +``` + +## Account API + +### Information + +This function returns information about the current peer. + +``` +Request: GET /account/info +Response: 200 with JSON structure apiResponsePeerSelf +``` + +The peer and node IDs are encoded as hex encoded strings. + +```go +type apiResponsePeerSelf struct { + PeerID string `json:"peerid"` // Peer ID. This is derived from the public in compressed form. + NodeID string `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. +} +``` + +### Delete + +This deletes the account. This action is irreversible. After deleting the account, the backend shall no longer be used. + +Note that it currently does not send a termination message to other peers. As a result, other peers may retain data or metadata. + +``` +Request: GET /account/delete?confirm=[0 or 1] +Result: 204 if the user choses not to delete the account + 200 if successfully deleted +``` + +## Blockchain Functions + +Common status codes returned by various endpoints in the `blockchain` package: + +| Status | Constant | Info | +| ------ | ------------------------ | --------------------------------------------------------------- | +| 0 | StatusOK | Successful operation. | +| 1 | StatusBlockNotFound | Missing block in the blockchain. | +| 2 | StatusCorruptBlock | Error block encoding. | +| 3 | StatusCorruptBlockRecord | Error block record encoding. | +| 4 | StatusDataNotFound | Requested data not available in the blockchain. | +| 5 | StatusNotInWarehouse | File to be added to blockchain does not exist in the Warehouse. | + +### Blockchain Header + +This function returns information about the current peer. It is not required that a peer has a blockchain. If no data is shared, there are no blocks. The blockchain does not formally have a header as each block has the same structure. + +``` +Request: GET /blockchain/header +Response: 200 with JSON structure apiBlockchainHeader +``` + +```go +type apiBlockchainHeader struct { + PeerID string `json:"peerid"` // Peer ID hex encoded. + Version uint64 `json:"version"` // Current version number of the blockchain. + Height uint64 `json:"height"` // Height of the blockchain (number of blocks). If 0, no data exists. +} +``` + +### Blockchain Append Block + +This appends a block to the blockchain. This is a low-level function for already encoded blocks. +Do not use this function. Adding invalid data to the blockchain may corrupt it which subsequently might result in blacklisting by other peers. + +``` +Request: POST /blockchain/append with JSON structure apiBlockchainBlockRaw +Response: 200 with JSON structure apiBlockchainBlockStatus +``` + +```go +type apiBlockRecordRaw struct { + Type uint8 `json:"type"` // Record Type. See core.RecordTypeX. + Data []byte `json:"data"` // Data according to the type. +} + +type apiBlockchainBlockRaw struct { + Records []apiBlockRecordRaw `json:"records"` // Block records in encoded raw format. +} + +type apiBlockchainBlockStatus struct { + Status int `json:"status"` // See blockchain.StatusX. + Height uint64 `json:"height"` // Height of the blockchain (number of blocks). + Version uint64 `json:"version"` // Version of the blockchain. +} +``` + +### Blockchain Read Block + +This reads a block of the current peer. + +``` +Request: GET /blockchain/read?block=[number] +Response: 200 with JSON structure apiBlockchainBlock +``` + +```go +type apiBlockchainBlock struct { + Status int `json:"status"` // See blockchain.StatusX. + PeerID string `json:"peerid"` // Peer ID hex encoded. + LastBlockHash []byte `json:"lastblockhash"` // Hash of the last block. Blake3. + BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version + Number uint64 `json:"blocknumber"` // Block number + RecordsRaw []apiBlockRecordRaw `json:"recordsraw"` // Records raw. Successfully decoded records are parsed into the below fields. + RecordsDecoded []interface{} `json:"recordsdecoded"` // Records decoded. The encoding for each record depends on its type. +} +``` + +The array `RecordsDecoded` will contain any present record of the following: +* Profile records, see `apiBlockRecordProfile` +* File records, see `apiFile` + +## File Functions + +These functions allow adding, deleting, and listing files stored on the users blockchain. Only metadata is actually stored on the blockchain. To download a remote file both the file hash and the node ID are required. The node ID specifies the owner of the file. + +```go +type apiFile struct { + ID uuid.UUID `json:"id"` // Unique ID. + Hash []byte `json:"hash"` // Blake3 hash of the file data + Type uint8 `json:"type"` // File Type. For example audio or document. See TypeX. + Format uint16 `json:"format"` // File Format. This is more granular, for example PDF or Word file. See FormatX. + Size uint64 `json:"size"` // Size of the file + Folder string `json:"folder"` // Folder, optional + Name string `json:"name"` // Name of the file + Description string `json:"description"` // Description. This is expected to be multiline and contain hashtags! + Date time.Time `json:"date"` // Date shared + NodeID []byte `json:"nodeid"` // Node ID, owner of the file. Read only. + Metadata []apiFileMetadata `json:"metadata"` // Additional metadata. +} + +type apiFileMetadata struct { + Type uint16 `json:"type"` // See core.TagX constants. + Name string `json:"name"` // User friendly name of the metadata type. Use the Type fields to identify the metadata as this name may change. + // Depending on the exact type, one of the below fields is used for proper encoding: + Text string `json:"text"` // Text value. UTF-8 encoding. + Blob []byte `json:"blob"` // Binary data + Date time.Time `json:"date"` // Date + Number uint64 `json:"number"` // Number +} +``` + +Below is the list of defined metadata types. Undefined types may be used by clients, but are always mapped into the `blob` field. Virtual tags are generated at runtime and are read-only. They cannot be stored on the blockchain. + +| Type | Constant | Encoding | Virtual | Info | +| ---- | ---------------- | -------- | ------- | -------------------------------------------------------------------------------------------- | +| 0 | TagName | Text | | Mapped into Name field. Name of file. | +| 1 | TagFolder | Text | | Mapped into Folder field. Folder name. | +| 2 | TagDescription | Text | | Mapped into Description field. Arbitrary description of the file. May contain hashtags. | +| 3 | TagDateShared | Date | x | Mapped into Date field. When the file was published on the blockchain. | +| 4 | TagDateCreated | Date | | Date when the file was originally created. | +| 5 | TagSharedByCount | Number | x | Count of peers that share the file. | +| 6 | TagSharedByGeoIP | Text/CSV | x | GeoIP data of peers that are sharing the file. CSV encoded with header "latitude,longitude". | + +The file type is an indication what type of content the file's data is: + +| Type | Constant | Info | +| ---- | -------------- | ------------------------------------------------------------------------------ | +| 0 | TypeBinary | Binary/unspecified | +| 1 | TypeText | Plain text | +| 2 | TypePicture | Picture of any format | +| 3 | TypeVideo | Video | +| 4 | TypeAudio | Audio | +| 5 | TypeDocument | Any document file, including office documents, PDFs, power point, spreadsheets | +| 6 | TypeExecutable | Any executable file, OS independent | +| 7 | TypeContainer | Container files like ZIP, RAR, TAR, ISO | +| 8 | TypeCompressed | Compressed files like GZ, BZ | +| 9 | TypeFolder | Virtual folder | +| 10 | TypeEbook | Ebook | + +The file format is a more granular indicator about the content of a file: + +| Type | Constant | Info | +| ---- | ---------------- | --------------------------------------------------- | +| 0 | FormatBinary | Binary/unspecified | +| 1 | FormatPDF | PDF document | +| 2 | FormatWord | Word document | +| 3 | FormatExcel | Excel | +| 4 | FormatPowerpoint | Powerpoint | +| 5 | FormatPicture | Pictures (including GIF, excluding icons) | +| 6 | FormatAudio | Audio files | +| 7 | FormatVideo | Video files | +| 8 | FormatContainer | Compressed files including ZIP, RAR, TAR and others | +| 9 | FormatHTML | HTML file | +| 10 | FormatText | Text file | +| 11 | FormatEbook | Ebook file | +| 12 | FormatCompressed | Compressed file | +| 13 | FormatDatabase | Database file | +| 14 | FormatEmail | Single email | +| 15 | FormatCSV | CSV file | +| 16 | FormatFolder | Virtual folder | +| 17 | FormatExecutable | Executable file | +| 18 | FormatInstaller | Installer | +| 19 | FormatAPK | APK | +| 20 | FormatISO | ISO | + +### Add File + +This adds a file with the provided information to the blockchain. The date field cannot be set by the caller and is ignored. If the ID field is left empty, a random UUID is automatically assigned. The size field is ignored; it will be automatically set to the file size identified by the hash (via the Warehouse). The format and type fields need to be set by the caller; `/file/format` can be used to detect them. + +Any file added is publicly accessible. The user should be informed about this fact in advance. The user is responsible and liable for any files shared. + +Each file must be already stored in the Warehouse (virtual folders are exempt). Files in the Warehouse are identified using the hash. +If any file is not stored in the Warehouse, the function aborts with the status code StatusNotInWarehouse. Files can be added to the Warehouse via `/warehouse/create` and `/warehouse/create/path`. + +If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord. In case the function aborts, the blockchain remains unchanged. + +Do not add the same file with the same ID multiple times. Doing so will create double entries. This function does not check if the file is already stored on the blockchain. Storing multiple files with the same file hash, but different IDs, is perfectly fine. + +``` +Request: POST /blockchain/file/add with JSON structure apiBlockAddFiles +Response: 200 with JSON structure apiBlockchainBlockStatus +``` + +```go +type apiBlockAddFiles struct { + Files []apiFile `json:"files"` // List of files + Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API. +} +``` + +Example POST request to `http://127.0.0.1:112/blockchain/file/add`: + +```json +{ + "files": [{ + "id": "236de31d-f402-4389-bdd1-56463abdc309", + "hash": "aFad3zRACbk44dsOw5sVGxYmz+Rqh8ORDcGJNqIz+Ss=", + "type": 1, + "format": 10, + "size": 4, + "name": "Test.txt", + "folder": "sample directory/sub folder", + "description": "", + "metadata": [] + }] +} +``` + +Another payload example to create a new file but with a new arbitrary tag with type number 100 set to "test" and setting the metadata field "Date Created" (which is type 2 = `core.TagTypeDateCreated`): + +```json +{ + "files": [{ + "id": "bc32cbae-011d-4f0b-80a8-281ca93692e7", + "hash": "aFad3zRACbk44dsOw5sVGxYmz+Rqh8ORDcGJNqIz+Ss=", + "type": 1, + "format": 10, + "size": 4, + "name": "Test.txt", + "folder": "sample directory/sub folder", + "description": "Example description\nThis can be any text #newfile #2021.", + "metadata": [{ + "type": 2, + "date": "2021-08-28T00:00:00Z" + }] + }] +} +``` + +### List Files + +This lists all files stored on the blockchain. + +``` +Request: GET /blockchain/file/list +Response: 200 with JSON structure apiBlockAddFiles +``` + +Example request: `http://127.0.0.1:112/blockchain/file/list` + +Example response: + +```json +{ + "files": [{ + "id": "a59b6465-fe8c-4a61-9fcc-fe37cf711fd4", + "hash": "aFad3zRACbk44dsOw5sVGxYmz+Rqh8ORDcGJNqIz+Ss=", + "type": 1, + "format": 10, + "size": 4, + "folder": "sample directory/sub folder", + "name": "Test.txt", + "description": "", + "date": "2021-08-27T14:59:13Z", + "nodeid": "0Zo9QHCF06Nrbxgg9s4Q4wYpcHzsQhSMsmftQqjanVI=", + "metadata": [] + }, { + "id": "bc32cbae-011d-4f0b-80a8-281ca9369211", + "hash": "aFad3zRACbk44dsOw5sVGxYmz+Rqh8ORDcGJNqIz+Ss=", + "type": 1, + "format": 10, + "size": 4, + "folder": "sample directory/sub folder", + "name": "Test 2.txt", + "description": "Example description\nThis can be any text #newfile #2021.", + "date": "2021-09-27T23:33:37Z", + "nodeid": "0Zo9QHCF06Nrbxgg9s4Q4wYpcHzsQhSMsmftQqjanVI=", + "metadata": [{ + "type": 2, + "name": "Date Created", + "text": "", + "blob": null, + "date": "2021-08-28T00:00:00Z" + }] + }], + "status": 0 +} +``` + +### Delete File + +This deletes files from the blockchain with the provided IDs. The blockchain will be refactored, which means it is recalculated without the specified files. The blockchains version number might be increased. + +It will automatically delete the file in the Warehouse if there are no other references. + +``` +Request: POST /blockchain/file/delete with JSON structure apiBlockAddFiles +Response: 200 with JSON structure apiBlockchainBlockStatus +``` + +Example POST request to `http://127.0.0.1:112/blockchain/file/delete`: + +```json +{ + "files": [{ + "id": "236de31d-f402-4389-bdd1-56463abdc309" + }] +} +``` + +Example response indicating success: + +```json +{ + "status": 0, + "height": 7, + "version": 1 +} +``` + +### Update File + +This updates files that are already published on the blockchain. This is useful for example when changing a file name or description. +Just like with the add file function, the file must be already stored in the Warehouse, otherwise this function fails. + +The files are identified by their IDs. If an ID is not set, this function fails with HTTP 400. The size field is ignored; it will be automatically set to the file size identified by the hash (via the Warehouse). + +Note as this replaces the previous file record on the blockchain, all details (including special metadata fields) must be included. + +``` +Request: POST /blockchain/file/update with JSON structure apiBlockAddFiles +Response: 200 with JSON structure apiBlockchainBlockStatus +``` + +## Profile Functions + +User profile data such as the username, email address, and picture are stored on the blockchain. Profile fields are text (UTF-8) or binary encoded, depending on the type. + +Note that all profile data is arbitrary and shall be considered untrusted and unverified. To establish trust, the user must load Certificates into the blockchain that validate certain data. + +Below is the list of well known profile information. Clients may define additional fields. The purpose of this defined list is to provide a common mapping across different client software. Undefined types are always mapped into the `blob` field. + +| Type | Constant | Encoding | Info | +| ---- | -------------- | -------- | ----------------------------- | +| 0 | ProfileName | Text | Arbitrary username | +| 1 | ProfileEmail | Text | Email address | +| 2 | ProfileWebsite | Text | Website address | +| 3 | ProfileTwitter | Text | Twitter account without the @ | +| 4 | ProfileYouTube | Text | YouTube channel URL | +| 5 | ProfileAddress | Text | Physical address | +| 6 | ProfilePicture | Blob | Profile picture | + +### Profile List + +This lists all profile fields. + +``` +Request: GET /profile/list +Response: 200 with JSON structure apiProfileData +``` + +```go +type apiProfileData struct { + Fields []apiBlockRecordProfile `json:"fields"` // All fields + Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API. See blockchain.StatusX. +} + +type apiBlockRecordProfile struct { + Type uint16 `json:"type"` // See ProfileX constants. + // Depending on the exact type, one of the below fields is used for proper encoding: + Text string `json:"text"` // Text value. UTF-8 encoding. + Blob []byte `json:"blob"` // Binary data +} +``` + +Example request: `http://127.0.0.1:112/profile/list` + +Example response: + +```json +{ + "fields": [{ + "type": 0, + "text": "Test Username 2021", + "blob": null + }, { + "type": 1, + "text": "test@example.com", + "blob": null + }], + "status": 0 +} +``` + +### Profile Read + +This reads a specific profile field. See ProfileX for recognized fields. + +``` +Request: GET /profile/read?field=[index] +Response: 200 with JSON structure apiProfileData +``` + +Example request to read the users username: `http://127.0.0.1:112/profile/read?field=0` + +Example response: + +```json +{ + "fields": [{ + "type": 0, + "text": "Test Username 2021", + "blob": null + }], + "status": 0 +} +``` + +### Profile Write + +This writes profile fields. It can write multiple fields at once. See ProfileX for recognized fields. + +``` +Request: POST /profile/write with JSON structure apiProfileData +Response: 200 with JSON structure apiBlockchainBlockStatus +``` + +Example POST request to `http://127.0.0.1:112/profile/write`: + +```json +{ + "fields": [{ + "type": 0, + "text": "Test Username 2021" + }] +} +``` + +Example response: + +```json +{ + "status": 0, + "height": 1, + "version": 0 +} +``` + +### Profile Delete + +This function allows to delete profile fields. Only the type number is required. Multiple fields can be deleted at the same time. + +``` +Request: POST /profile/delete with JSON structure apiProfileData +Response: 200 with JSON structure apiBlockchainBlockStatus +``` + +Example POST request to `http://127.0.0.1:112/profile/delete` (deleting the profile name): + +```json +{ + "fields": [{ + "type": 0 + }] +} +``` + +## Search API + +The search API provides a high-level function to search for files in Peernet. Searching is always asynchronous. `/search` returns an UUID which is used to loop over `/search/result` until the search is terminated. + +The current implementation of the underlying search algorithm only searches file names. + +Filters and sort order may be applied when starting the search at `/search`, or at runtime when returning the results at `/search/result`. + +These are the available sort options: + +| Sort | Constant | Info | +| ---- | --------------------- | ----------------------------------------------------------------------------------- | +| 0 | SortNone | No sorting. Results are returned as they come in. | +| 1 | SortRelevanceAsc | Least relevant results first. | +| 2 | SortRelevanceDec | Most relevant results first. | +| 3 | SortDateAsc | Oldest first. | +| 4 | SortDateDesc | Newest first. | +| 5 | SortNameAsc | File name ascending. The folder name is not used for sorting. | +| 6 | SortNameDesc | File name descending. The folder name is not used for sorting. | +| 7 | SortSizeAsc | File size ascending. Smallest files first. | +| 8 | SortSizeDesc | File size descending. Largest files first. | +| 9 | SortSharedByCountAsc | Shared by count ascending. Files that are shared by the least count of peers first. | +| 10 | SortSharedByCountDesc | Shared by count descending. Files that are shared by the most count of peers first. | + +The following filters are supported: + +* Filter by date from and to. Both dates are required. The inclusion check for the 'from date' is >= and 'to date' <. +* File type such as binary, text document etc. See core.TypeX. +* File format (which is more granular) such as PDF, Word, Ebook, etc. See core.FormatX. + +### Submitting a Search Request + +This starts a search request and returns an ID that can be used to collect the results asynchronously. Note that some of the filters described below (such as `filetype`) must be set to -1 if they are not used. + +``` +Request: POST /search with JSON SearchRequest +Response: 200 on success with JSON SearchRequestResponse +``` + +```go +type SearchRequest struct { + Term string `json:"term"` // Search term. + Timeout int `json:"timeout"` // Timeout in seconds. 0 means default. This is the entire time the search may take. Found results are still available after this timeout. + MaxResults int `json:"maxresults"` // Total number of max results. 0 means default. + DateFrom string `json:"datefrom"` // Date from, both from/to are required if set. Format "2006-01-02 15:04:05". + DateTo string `json:"dateto"` // Date to, both from/to are required if set. Format "2006-01-02 15:04:05". + Sort int `json:"sort"` // See SortX. + TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate. + FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. -1 = not used. + FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. + SizeMin int `json:"sizemin"` // Min file size in bytes. -1 = not used. + SizeMax int `json:"sizemax"` // Max file size in bytes. -1 = not used. +} + +type SearchRequestResponse struct { + ID uuid.UUID `json:"id"` // ID of the search job. This is used to get the results. + Status int `json:"status"` // Status of the search: 0 = Success (ID valid), 1 = Invalid Term, 2 = Error Max Concurrent Searches +} +``` + +Note that the date format for the `datefrom` and `dateto` fields is "2006-01-02 15:04:05" which is different to native JSON time encoding used elsewhere. The time zone is UTC. + +Example POST request to `http://127.0.0.1:112/search`: + +```json +{ + "term": "Test Search", + "timeout": 10, + "maxresults": 1000, + "sort": 0, + "filetype": -1, + "fileformat": -1, + "sizemin": -1, + "sizemax": -1 +} +``` + +Example response: + +```json +{ + "id": "ac5efa64-d403-4a57-8259-c7b7dfb09667", + "status": 0 +} +``` + +### Returning Search Results + +This function returns search results. The default limit is 100. + +If reset is set, all results will be filtered and sorted according to the provided parameters. This means that the new first result will be returned again and internal result offset is set to 0. Note that most filters must be set to -1 if they are not used (see the field comments in the `SearchRequest` structure in `/search` above). + +The statistics of all results (regardless of applied runtime filters) can be returned immediately in the `statistics` field by specifying `&stats=1`. The returned statistics is the `SearchStatisticData` structure and matches with what is returned by `/search/statistic`. + +Note that the date format for the `&from=` and `&to=` parameters is "2006-01-02 15:04:05" which is different to native JSON time encoding used elsewhere. The time zone is UTC. + +``` +Request: GET /search/result?id=[UUID]&limit=[max records] +Optional parameters: + &reset=[0|1] to reset the filters or sort orders with any of the below parameters (all required): + &filetype=[File Type] + &fileformat=[File Format] + &from=[Date From]&to=[Date To] + &sizemin=[Minimum file size] + &sizemax=[Maximum file size] + &sort=[sort order] + &offset=[absolute offset] with &limit=[records] to get items pagination style. Returned items (and ones before) are automatically frozen. +Result: 200 with JSON structure SearchResult. Check the field status. +``` + +```go +type SearchResult struct { + Status int `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying + Files []apiFile `json:"files"` // List of files found + Statistic interface{} `json:"statistic"` // Statistics of all results (independent from applied filters), if requested. Only set if files are returned (= if statistics changed). See SearchStatisticData. +} +``` + +Example request: `http://127.0.0.1:112/search/result?id=ac5efa64-d403-4a57-8259-c7b7dfb09667&limit=10` + +Example response with dummy data: + +```json +{ + "status": 1, + "files": [{ + "id": "b5b0706c-817c-492f-8203-5005c59f110c", + "hash": "Mv6O773ytkJ5jSjLoy2EvHQaM5KfVppJHeTppMc7alA=", + "type": 1, + "format": 14, + "size": 10, + "folder": "", + "name": "88d8cc57d5c2a5fea881ceea09503ee4.txt", + "description": "", + "date": "2021-09-23T00:00:00Z", + "nodeid": "j4yHzmCXiXqg4DPhowj0DIOuuyJxQflo2QSNG3yhCK8=", + "metadata": [{ + "type": 5, + "name": "Shared By Count", + "text": "", + "blob": null, + "date": "0001-01-01T00:00:00Z", + "number": 7 + }, { + "type": 6, + "name": "Shared By GeoIP", + "text": "25.7766,-178.1275\n-46.4041,8.0066\n84.4478,8.2417\n14.1721,-9.7539\n-67.2364,127.6007\n-75.1604,106.7583\n70.5132,-133.4146", + "blob": null, + "date": "0001-01-01T00:00:00Z", + "number": 0 + }] + }] +} +``` + +### Search Result Statistics + +This returns search result statistics. Statistics are always calculated over all results, regardless of any applied runtime filters. + +``` +Request: GET /search/statistic?id=[UUID] +Result: 200 with JSON structure SearchStatistic. Check the field status (0 = Success, 2 = ID not found). +``` + +```go +type SearchStatistic struct { + SearchStatisticData + Status int `json:"status"` // Status: 0 = Success + IsTerminated bool `json:"terminated"` // Whether the search is terminated, meaning that statistics won't change +} + +type SearchStatisticData struct { + Date []SearchStatisticRecordDay `json:"date"` // Files per date + FileType []SearchStatisticRecord `json:"filetype"` // Files per file type + FileFormat []SearchStatisticRecord `json:"fileformat"` // Files per file format + Total int `json:"total"` // Total count of files +} + +type SearchStatisticRecordDay struct { + Date time.Time `json:"date"` // The day (which covers the full 24 hours). Always rounded down to midnight. + Count int `json:"count"` // Count of files. +} + +type SearchStatisticRecord struct { + Key int `json:"key"` // Key index. The exact meaning depends on where this structure is used. + Count int `json:"count"` // Count of files for the given key +} +``` + +### Receiving Search Results via Websocket + +This provides a websocket to receive results as stream. It does not support changing runtime filters and returning statistics. + +``` +Request: GET /search/result/ws?id=[UUID]&limit=[optional max records] +Result: If successful, upgrades to a websocket and sends JSON structure SearchResult messages. + Limit is optional. Not used if ommitted or 0. +``` + +Example socket URL: `ws://127.0.0.1:112/search/result/ws?id=08ab3469-cd0e-4219-998f-bfdf496351eb` + +### Terminating a Search + +The user can terminate a search early using this function. This helps save system resources and should be considered best practice once a search is no longer needed (for example when the user closes the tab or window that shows the results). + +``` +Request: GET /search/terminate?id=[UUID] +Response: 204 Empty +``` + +## Download API + +Downloads can have these status types: + +| Status | Constant | Info | +| ------ | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| 0 | DownloadWaitMetadata | Wait for file metadata. | +| 1 | DownloadWaitSwarm | Wait to join swarm. | +| 2 | DownloadActive | Active downloading. It could still be stuck at any percentage (including 0%) if no seeders are available. | +| 3 | DownloadPause | Paused by the user. | +| 4 | DownloadCanceled | Canceled by the user before the download finished. Once canceled, a new download has to be started if the file shall be downloaded. | +| 5 | DownloadFinished | Download finished 100%. | + +The API response codes for download functions are: + +| Status | Constant | Info | +| ------ | ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| 0 | DownloadResponseSuccess | Success | +| 1 | DownloadResponseIDNotFound | Error: Download ID not found. | +| 2 | DownloadResponseFileInvalid | Error: Target file cannot be used. For example, permissions denied to create it. | +| 3 | DownloadResponseActionInvalid | Error: Invalid action. Pausing a non-active download, resuming a non-paused download, or canceling already canceled or finished download. | +| 4 | DownloadResponseFileWrite | Error writing file. | + +### Start Download + +This starts the download of a file. The path is the full path on disk to store the file. +The hash parameter identifies the file to download. The node ID identifies the blockchain (i.e., the "owner" of the file). The hash and node must be hex-encoded. + +``` +Request: GET /download/start?path=[target path on disk]&hash=[file hash to download]&node=[node ID] +Result: 200 with JSON structure apiResponseDownloadStatus +``` + +```go +type apiResponseDownloadStatus struct { + APIStatus int `json:"apistatus"` // Status of the API call. See DownloadResponseX. + ID uuid.UUID `json:"id"` // Download ID. This can be used to query the latest status and take actions. + DownloadStatus int `json:"downloadstatus"` // Status of the download. See DownloadX. + File apiFile `json:"file"` // File information. Only available for status >= DownloadWaitSwarm. + Progress struct { + TotalSize uint64 `json:"totalsize"` // Total size in bytes. + DownloadedSize uint64 `json:"downloadedsize"` // Count of bytes download so far. + Percentage float64 `json:"percentage"` // Percentage downloaded. Rounded to 2 decimal points. Between 0.00 and 100.00. + } `json:"progress"` // Progress of the download. Only valid for status >= DownloadWaitSwarm. + Swarm struct { + CountPeers uint64 `json:"countpeers"` // Count of peers participating in the swarm. + } `json:"swarm"` // Information about the swarm. Only valid for status >= DownloadActive. +} +``` + +Example request: `http://127.0.0.1:112/download/start?path=test.bin&hash=cde13a55f41e387480391c47238acfe9c0136dd56bf365b01416aec03eec7dc4&node=5a0f712822ddc49633d27df6009d3efa27f19cb371319837f04160bdbda38544` + +Example response (only apistatus, id, and downloadstatus are used): + +```json +{ + "apistatus": 0, + "id": "a6107122-9e31-42d3-b663-0df64263c6bc", + "downloadstatus": 0 +} +``` + +### Get Download Status + +This returns the status of an active download. + +``` +Request: GET /download/status?id=[download ID] +Result: 200 with JSON structure apiResponseDownloadStatus +``` + +Example request: `http://127.0.0.1:112/download/status?id=a6107122-9e31-42d3-b663-0df64263c6bc` + +```json +{ + "apistatus": 0, + "id": "950316e8-23b4-49c7-83dd-c021e793129e", + "downloadstatus": 5, + "file": { + "id": "78ac46dc-6731-4f3d-a9d4-22c9a4eb5fb9", + "hash": "LiQUdqPD78+e6j1eS+0VmSUdCgUXVDN74ELVTRcgmWc=", + "type": 0, + "format": 13, + "size": 10240, + "folder": "", + "name": "a96dc7b6a4a7a401c48f93c442f01de9.bin", + "description": "", + "date": "2021-10-04T04:37:17Z", + "nodeid": "lMP3/nYMjoE/PfGKRDZi+ms5h7jWUrdIZaKSvLAAq6A=", + "metadata": [] + }, + "progress": { + "totalsize": 10240, + "downloadedsize": 1024, + "percentage": 10 + }, + "swarm": { + "countpeers": 0 + } +} +``` + +### Pause, Resume, and Cancel a Download + +This pauses, resumes, and cancels a download. Once canceled, a new download has to be started if the file shall be downloaded. +Only active downloads can be paused. While a download is in discovery phase (querying metadata, joining swarm), it can only be canceled. +Action: 0 = Pause, 1 = Resume, 2 = Cancel. + +``` +Request: GET /download/action?id=[download ID]&action=[action] +Result: 200 with JSON structure apiResponseDownloadStatus (using APIStatus and DownloadStatus) +``` + +## Explore + +### List Recently Shared Files + +This returns recently shared files in Peernet. Results are returned in real-time. The file type is an optional filter. + +``` +Request: GET /explore?limit=[max records]&type=[file type]&offset=[offset] +Result: 200 with JSON structure SearchResult. Check the field status. +``` + +Example request to list 20 recently shared files (all file types): `http://127.0.0.1:112/explore&limit=20` + +Example request to list 10 recent documents: `http://127.0.0.1:112/explore?type=5&limit=10` + +## Helper Functions + +These helper functions are usually not needed, but can be useful in special cases. + +### Detect file type and file format + +This function detects the file type and file format of the specified file. It will primarily use the file extension for detection. If unavailable, it uses the first 512 bytes of the file data to detect the type. The path is the full file path (including directory) on disk. + +``` +Request: GET /file/format?path=[file path on disk] +Result: 200 with JSON structure apiResponseFileFormat +``` + +```go +type apiResponseFileFormat struct { + Status int `json:"status"` // Status: 0 = Success, 1 = Error reading file + FileType uint16 `json:"filetype"` // File Type. + FileFormat uint16 `json:"fileformat"` // File Format. +} +``` + +Example request: `http://127.0.0.1:112/file/format?path=test.txt` + +Example response: + +```json +{ + "status": 0, + "filetype": 1, + "fileformat": 10 +} +``` + +## Warehouse + +The Warehouse stores the actual files that are shared by the user. The blockchain only stores the metadata information. The Warehouse and the blockchain must be kept in sync. + +* Files are identified (and adressed) by their hash. +* Before using `/blockchain/file/add`, you must store the file in the Warehouse using `/warehouse/create` or `/warehouse/create/path`. The blockchain add file function verifies if the file exists in the Warehouse and fails if it does not. +* When deleting a file from the blockchain via `/blockchain/file/delete`, it will automatically delete the file from the warehouse if there are no other files on the blockchain referencing it. +* Because files are addressed using their hash, they are automatically deduplicated. If the user shares the exact same file data under different file names, it is only stored once. + +Note: The Warehouse does NOT store files downloaded from other users. It strictly only stores files that the user choses to publish. + +Status codes: + +| Status | Constant | Info | +| ------ | ------------------------- | ------------------------------------------------- | +| 0 | StatusOK | Success | +| 1 | StatusErrorCreateTempFile | Error creating a temporary file. | +| 2 | StatusErrorWriteTempFile | Error writing temporary file. | +| 3 | StatusErrorCloseTempFile | Error closing temporary file. | +| 4 | StatusErrorRenameTempFile | Error renaming temporary file. | +| 5 | StatusErrorCreatePath | Error creating path for target file in warehouse. | +| 7 | StatusErrorOpenFile | Error opening file. | +| 8 | StatusInvalidHash | Invalid hash. | +| 9 | StatusFileNotFound | File not found. | +| 10 | StatusErrorDeleteFile | Error deleting file. | +| 11 | StatusErrorReadFile | Error reading file. | +| 12 | StatusErrorSeekFile | Error seeking to position in file. | +| 13 | StatusErrorTargetExists | Target file already exists. | +| 14 | StatusErrorCreateTarget | Error creating target file. | +| 15 | StatusErrorCreateMerkle | Error creating merkle tree. | +| 16 | StatusErrorMerkleTreeFile | Invalid merkle tree companion file. | + +### Create File + +This creates a file in the warehouse. The payload data is the file data to store. It returns the hash of the stored file. If the file already exists it does not return an error. + +``` +Request: POST /warehouse/create with raw data to create as new file +Response: 200 with JSON structure WarehouseResult +``` + +```go +type WarehouseResult struct { + Status int `json:"status"` // See warehouse.StatusX. + Hash []byte `json:"hash"` // Hash of the file. +} +``` + +Example POST request to `http://127.0.0.1:112/warehouse/create`: + +``` +Test file. +``` + +Example response: + +```json +{ + "status": 0, + "hash": "2/NE8j54ICYTKYg64m9kkpp8mXdUkAHSjcQMkgLXZR4=" +} +``` + +### Create File by Copy + +This creates a file in the warehouse by copying it from an existing local file. + +Warning: An attacker could supply any local file using this function, put them into storage and read them! No input path verification or limitation is done. +In the future the API should be secured using a random API key and setting the CORS header prohibiting regular browsers to access the API. + +``` +Request: GET /warehouse/create/path?path=[target path on disk] +Response: 200 with JSON structure WarehouseResult +``` + +Example request to add the local file "C:\Test File 1.txt": `http://127.0.0.1:112/warehouse/create/path?path=C%3A%5CTest%20File%201.txt` + +Example response in case the file does not exist (returning StatusFileNotFound): + +```json +{ + "status": 9, + "hash": null +} +``` + +### Read File + +This reads a file in the warehouse. The offset and limit parameter are optional. The hash must be hex encoded. + +``` +Request: GET /warehouse/read?hash=[hash] + Optional parameters &offset=[file offset]&limit=[read limit in bytes] +Response: 200 with the raw file data + 404 if file was not found + 500 in case of internal error opening the file +``` + +Example request: `http://127.0.0.1:112/warehouse/read?hash=dbf344f23e7820261329883ae26f64929a7c9977549001d28dc40c9202d7651e` + +### Read File To Disk + +This reads a file from the warehouse and stores it to the target file. It fails with StatusErrorTargetExists if the target file already exists. +The path must include the full directory and file name. + +``` +Request: GET /warehouse/read/path?hash=[hash]&path=[target path on disk] + Optional parameters &offset=[file offset]&limit=[read limit in bytes] +Response: 200 with JSON structure WarehouseResult +``` + +Example request: `http://127.0.0.1:112/warehouse/read/path?hash=dbf344f23e7820261329883ae26f64929a7c9977549001d28dc40c9202d7651e&path=C%3A%5CTest%20File%202.bin` + +### Delete File + +This deletes a file in the warehouse. This is normally not needed, since `/blockchain/file/delete` will automatically delete files in the Warehouse if there are no active references. + +Warning: Deleting files from the warehouse but not the blockchain creates orphans. Peers might blacklist other peers who advertise files via their blockchain, but fail to provide them for transfer. + +``` +Request: GET /warehouse/delete?hash=[hash] +Response: 200 with JSON structure WarehouseResult +``` + +Example request: `http://127.0.0.1:112/warehouse/delete?hash=dbf344f23e7820261329883ae26f64929a7c9977549001d28dc40c9202d7651e`