From c54bd187ea95439cc06cad0ad43d4551bd5b39fd Mon Sep 17 00:00:00 2001 From: Akilan Date: Sat, 30 Apr 2022 18:15:06 +0400 Subject: [PATCH] adding prototype follow feature --- webapi/API.go | 393 ++++++++++++++++++++++++------------------------ webapi/Clone.go | 130 ++++++++++++++++ 2 files changed, 327 insertions(+), 196 deletions(-) create mode 100644 webapi/Clone.go diff --git a/webapi/API.go b/webapi/API.go index f0b0abe..f5ab3f4 100644 --- a/webapi/API.go +++ b/webapi/API.go @@ -1,196 +1,197 @@ -/* -File Name: API.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "crypto/tls" - "encoding/json" - "errors" - "net/http" - "sync" - "time" - - "github.com/IncSW/geoip2" - "github.com/PeernetOfficial/core" - "github.com/google/uuid" - "github.com/gorilla/mux" - "github.com/gorilla/websocket" -) - -type WebapiInstance struct { - backend *core.Backend - geoipCityReader *geoip2.CityReader - - // Router can be used to register additional API functions - Router *mux.Router - AllowKeyInParam []string // List of paths that accept the API key as &k= parameter - - // search jobs - allJobs map[uuid.UUID]*SearchJob - allJobsMutex sync.RWMutex - - // download info - downloads map[uuid.UUID]*downloadInfo - downloadsMutex sync.RWMutex -} - -// WSUpgrader is used for websocket functionality. It allows all requests. -var WSUpgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, - CheckOrigin: func(r *http.Request) bool { - // allow all connections by default - return true - }, -} - -// Start starts the API. ListenAddresses is a list of IP:Ports. -// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. -// The API key may be uuid.Nil to disable it although this is not recommended for security reasons. -func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKey string, TimeoutRead, TimeoutWrite time.Duration, APIKey uuid.UUID) (api *WebapiInstance) { - if len(ListenAddresses) == 0 { - return nil - } - - api = &WebapiInstance{ - backend: Backend, - Router: mux.NewRouter(), - AllowKeyInParam: []string{"/file/read", "/file/view"}, - allJobs: make(map[uuid.UUID]*SearchJob), - downloads: make(map[uuid.UUID]*downloadInfo), - } - - if APIKey != uuid.Nil { - api.Router.Use(api.authenticateMiddleware(APIKey)) - } - - api.Router.HandleFunc("/test", apiTest).Methods("GET") - api.Router.HandleFunc("/status", api.apiStatus).Methods("GET") - api.Router.HandleFunc("/account/info", api.apiAccountInfo).Methods("GET") - api.Router.HandleFunc("/account/delete", api.apiAccountDelete).Methods("GET") - api.Router.HandleFunc("/blockchain/header", api.apiBlockchainHeaderFunc).Methods("GET") - api.Router.HandleFunc("/blockchain/append", api.apiBlockchainAppend).Methods("POST") - api.Router.HandleFunc("/blockchain/read", api.apiBlockchainRead).Methods("GET") - api.Router.HandleFunc("/blockchain/file/add", api.apiBlockchainFileAdd).Methods("POST") - api.Router.HandleFunc("/blockchain/file/list", api.apiBlockchainFileList).Methods("GET") - api.Router.HandleFunc("/blockchain/file/delete", api.apiBlockchainFileDelete).Methods("POST") - api.Router.HandleFunc("/blockchain/file/update", api.apiBlockchainFileUpdate).Methods("POST") - api.Router.HandleFunc("/profile/list", api.apiProfileList).Methods("GET") - api.Router.HandleFunc("/profile/read", api.apiProfileRead).Methods("GET") - api.Router.HandleFunc("/profile/write", api.apiProfileWrite).Methods("POST") - api.Router.HandleFunc("/profile/delete", api.apiProfileDelete).Methods("POST") - api.Router.HandleFunc("/search", api.apiSearch).Methods("POST") - api.Router.HandleFunc("/search/result", api.apiSearchResult).Methods("GET") - api.Router.HandleFunc("/search/result/ws", api.apiSearchResultStream).Methods("GET") - api.Router.HandleFunc("/search/statistic", api.apiSearchStatistic).Methods("GET") - api.Router.HandleFunc("/search/terminate", api.apiSearchTerminate).Methods("GET") - api.Router.HandleFunc("/explore", api.apiExplore).Methods("GET") - api.Router.HandleFunc("/file/format", api.apiFileFormat).Methods("GET") - api.Router.HandleFunc("/download/start", api.apiDownloadStart).Methods("GET") - api.Router.HandleFunc("/download/status", api.apiDownloadStatus).Methods("GET") - api.Router.HandleFunc("/download/action", api.apiDownloadAction).Methods("GET") - api.Router.HandleFunc("/warehouse/create", api.apiWarehouseCreateFile).Methods("POST") - api.Router.HandleFunc("/warehouse/create/path", api.apiWarehouseCreateFilePath).Methods("GET") - api.Router.HandleFunc("/warehouse/read", api.apiWarehouseReadFile).Methods("GET") - api.Router.HandleFunc("/warehouse/read/path", api.apiWarehouseReadFilePath).Methods("GET") - api.Router.HandleFunc("/warehouse/delete", api.apiWarehouseDeleteFile).Methods("GET") - api.Router.HandleFunc("/file/read", api.apiFileRead).Methods("GET") - api.Router.HandleFunc("/file/view", api.apiFileView).Methods("GET") - - for _, listen := range ListenAddresses { - go startWebAPI(Backend, listen, UseSSL, CertificateFile, CertificateKey, api.Router, "API", TimeoutRead, TimeoutWrite) - } - - return api -} - -// startWebAPI starts a web-server with given parameters and logs the status. If may block forever and only returns if there is an error. -// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. -func startWebAPI(Backend *core.Backend, WebListen string, UseSSL bool, CertificateFile, CertificateKey string, Handler http.Handler, Info string, ReadTimeout, WriteTimeout time.Duration) { - Backend.LogError("startWebAPI", "Start API at '%s'\n", WebListen) - - tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} // for security reasons disable TLS 1.0/1.1 - - server := &http.Server{ - Addr: WebListen, - Handler: Handler, - ReadTimeout: ReadTimeout, // ReadTimeout is the maximum duration for reading the entire request, including the body. - WriteTimeout: WriteTimeout, // WriteTimeout is the maximum duration before timing out writes of the response. This includes processing time and is therefore the max time any HTTP function may take. - //IdleTimeout: IdleTimeout, // IdleTimeout is the maximum amount of time to wait for the next request when keep-alives are enabled. - TLSConfig: tlsConfig, - } - - if UseSSL { - // HTTPS - if err := server.ListenAndServeTLS(CertificateFile, CertificateKey); err != nil { - Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) - } - } else { - // HTTP - if err := server.ListenAndServe(); err != nil { - Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) - } - } -} - -// EncodeJSON encodes the data as JSON -func EncodeJSON(Backend *core.Backend, w http.ResponseWriter, r *http.Request, data interface{}) (err error) { - w.Header().Set("Content-Type", "application/json") - - err = json.NewEncoder(w).Encode(data) - if err != nil { - Backend.LogError("EncodeJSON", "Error writing data for route '%s': %v\n", r.URL.Path, err) - } - - return err -} - -// DecodeJSON decodes input JSON data server side sent either via GET or POST. It does not limit the maximum amount to read. -// In case of error it will automatically send an error to the client. -func DecodeJSON(w http.ResponseWriter, r *http.Request, data interface{}) (err error) { - if r.Body == nil { - http.Error(w, "", http.StatusBadRequest) - return errors.New("no data") - } - - err = json.NewDecoder(r.Body).Decode(data) - if err != nil { - http.Error(w, "", http.StatusBadRequest) - return err - } - - return nil -} - -// authenticateMiddleware returns a middleware function to be used with mux.Router.Use(). It handles all authentication functionality. -func (api *WebapiInstance) authenticateMiddleware(APIKey uuid.UUID) func(http.Handler) http.Handler { - return (func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - keyID, err := uuid.Parse(r.Header.Get("x-api-key")) - if err != nil { // special case for some paths - for _, exceptPath := range api.AllowKeyInParam { - if exceptPath == r.URL.Path { - r.ParseForm() - keyID, err = uuid.Parse(r.Form.Get("k")) - break - } - } - } - if err != nil { // Invalid key format - w.WriteHeader(http.StatusUnauthorized) - return - } - - if keyID != APIKey { - w.WriteHeader(http.StatusUnauthorized) - return - } - - next.ServeHTTP(w, r) - }) - }) -} +/* +File Name: API.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "crypto/tls" + "encoding/json" + "errors" + "net/http" + "sync" + "time" + + "github.com/IncSW/geoip2" + "github.com/PeernetOfficial/core" + "github.com/google/uuid" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" +) + +type WebapiInstance struct { + backend *core.Backend + geoipCityReader *geoip2.CityReader + + // Router can be used to register additional API functions + Router *mux.Router + AllowKeyInParam []string // List of paths that accept the API key as &k= parameter + + // search jobs + allJobs map[uuid.UUID]*SearchJob + allJobsMutex sync.RWMutex + + // download info + downloads map[uuid.UUID]*downloadInfo + downloadsMutex sync.RWMutex +} + +// WSUpgrader is used for websocket functionality. It allows all requests. +var WSUpgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { + // allow all connections by default + return true + }, +} + +// Start starts the API. ListenAddresses is a list of IP:Ports. +// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. +// The API key may be uuid.Nil to disable it although this is not recommended for security reasons. +func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKey string, TimeoutRead, TimeoutWrite time.Duration, APIKey uuid.UUID) (api *WebapiInstance) { + if len(ListenAddresses) == 0 { + return nil + } + + api = &WebapiInstance{ + backend: Backend, + Router: mux.NewRouter(), + AllowKeyInParam: []string{"/file/read", "/file/view"}, + allJobs: make(map[uuid.UUID]*SearchJob), + downloads: make(map[uuid.UUID]*downloadInfo), + } + + if APIKey != uuid.Nil { + api.Router.Use(api.authenticateMiddleware(APIKey)) + } + + api.Router.HandleFunc("/test", apiTest).Methods("GET") + api.Router.HandleFunc("/status", api.apiStatus).Methods("GET") + api.Router.HandleFunc("/account/info", api.apiAccountInfo).Methods("GET") + api.Router.HandleFunc("/account/delete", api.apiAccountDelete).Methods("GET") + api.Router.HandleFunc("/blockchain/header", api.apiBlockchainHeaderFunc).Methods("GET") + api.Router.HandleFunc("/blockchain/append", api.apiBlockchainAppend).Methods("POST") + api.Router.HandleFunc("/blockchain/read", api.apiBlockchainRead).Methods("GET") + api.Router.HandleFunc("/blockchain/file/add", api.apiBlockchainFileAdd).Methods("POST") + api.Router.HandleFunc("/blockchain/file/list", api.apiBlockchainFileList).Methods("GET") + api.Router.HandleFunc("/blockchain/file/delete", api.apiBlockchainFileDelete).Methods("POST") + api.Router.HandleFunc("/blockchain/file/update", api.apiBlockchainFileUpdate).Methods("POST") + api.Router.HandleFunc("/profile/list", api.apiProfileList).Methods("GET") + api.Router.HandleFunc("/profile/read", api.apiProfileRead).Methods("GET") + api.Router.HandleFunc("/profile/write", api.apiProfileWrite).Methods("POST") + api.Router.HandleFunc("/profile/delete", api.apiProfileDelete).Methods("POST") + api.Router.HandleFunc("/search", api.apiSearch).Methods("POST") + api.Router.HandleFunc("/search/result", api.apiSearchResult).Methods("GET") + api.Router.HandleFunc("/search/result/ws", api.apiSearchResultStream).Methods("GET") + api.Router.HandleFunc("/search/statistic", api.apiSearchStatistic).Methods("GET") + api.Router.HandleFunc("/search/terminate", api.apiSearchTerminate).Methods("GET") + api.Router.HandleFunc("/explore", api.apiExplore).Methods("GET") + api.Router.HandleFunc("/file/format", api.apiFileFormat).Methods("GET") + api.Router.HandleFunc("/download/start", api.apiDownloadStart).Methods("GET") + api.Router.HandleFunc("/download/status", api.apiDownloadStatus).Methods("GET") + api.Router.HandleFunc("/download/action", api.apiDownloadAction).Methods("GET") + api.Router.HandleFunc("/follow/file", api.apiHostFile).Methods("GET") + api.Router.HandleFunc("/warehouse/create", api.apiWarehouseCreateFile).Methods("POST") + api.Router.HandleFunc("/warehouse/create/path", api.apiWarehouseCreateFilePath).Methods("GET") + api.Router.HandleFunc("/warehouse/read", api.apiWarehouseReadFile).Methods("GET") + api.Router.HandleFunc("/warehouse/read/path", api.apiWarehouseReadFilePath).Methods("GET") + api.Router.HandleFunc("/warehouse/delete", api.apiWarehouseDeleteFile).Methods("GET") + api.Router.HandleFunc("/file/read", api.apiFileRead).Methods("GET") + api.Router.HandleFunc("/file/view", api.apiFileView).Methods("GET") + + for _, listen := range ListenAddresses { + go startWebAPI(Backend, listen, UseSSL, CertificateFile, CertificateKey, api.Router, "API", TimeoutRead, TimeoutWrite) + } + + return api +} + +// startWebAPI starts a web-server with given parameters and logs the status. If may block forever and only returns if there is an error. +// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. +func startWebAPI(Backend *core.Backend, WebListen string, UseSSL bool, CertificateFile, CertificateKey string, Handler http.Handler, Info string, ReadTimeout, WriteTimeout time.Duration) { + Backend.LogError("startWebAPI", "Start API at '%s'\n", WebListen) + + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} // for security reasons disable TLS 1.0/1.1 + + server := &http.Server{ + Addr: WebListen, + Handler: Handler, + ReadTimeout: ReadTimeout, // ReadTimeout is the maximum duration for reading the entire request, including the body. + WriteTimeout: WriteTimeout, // WriteTimeout is the maximum duration before timing out writes of the response. This includes processing time and is therefore the max time any HTTP function may take. + //IdleTimeout: IdleTimeout, // IdleTimeout is the maximum amount of time to wait for the next request when keep-alives are enabled. + TLSConfig: tlsConfig, + } + + if UseSSL { + // HTTPS + if err := server.ListenAndServeTLS(CertificateFile, CertificateKey); err != nil { + Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) + } + } else { + // HTTP + if err := server.ListenAndServe(); err != nil { + Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) + } + } +} + +// EncodeJSON encodes the data as JSON +func EncodeJSON(Backend *core.Backend, w http.ResponseWriter, r *http.Request, data interface{}) (err error) { + w.Header().Set("Content-Type", "application/json") + + err = json.NewEncoder(w).Encode(data) + if err != nil { + Backend.LogError("EncodeJSON", "Error writing data for route '%s': %v\n", r.URL.Path, err) + } + + return err +} + +// DecodeJSON decodes input JSON data server side sent either via GET or POST. It does not limit the maximum amount to read. +// In case of error it will automatically send an error to the client. +func DecodeJSON(w http.ResponseWriter, r *http.Request, data interface{}) (err error) { + if r.Body == nil { + http.Error(w, "", http.StatusBadRequest) + return errors.New("no data") + } + + err = json.NewDecoder(r.Body).Decode(data) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return err + } + + return nil +} + +// authenticateMiddleware returns a middleware function to be used with mux.Router.Use(). It handles all authentication functionality. +func (api *WebapiInstance) authenticateMiddleware(APIKey uuid.UUID) func(http.Handler) http.Handler { + return (func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + keyID, err := uuid.Parse(r.Header.Get("x-api-key")) + if err != nil { // special case for some paths + for _, exceptPath := range api.AllowKeyInParam { + if exceptPath == r.URL.Path { + r.ParseForm() + keyID, err = uuid.Parse(r.Form.Get("k")) + break + } + } + } + if err != nil { // Invalid key format + w.WriteHeader(http.StatusUnauthorized) + return + } + + if keyID != APIKey { + w.WriteHeader(http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r) + }) + }) +} diff --git a/webapi/Clone.go b/webapi/Clone.go new file mode 100644 index 0000000..3223750 --- /dev/null +++ b/webapi/Clone.go @@ -0,0 +1,130 @@ +/* +File Name: Download.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "bytes" + "errors" + "github.com/PeernetOfficial/core/blockchain" + "net/http" + "time" + + "github.com/google/uuid" +) + +type hostInfo struct { + api *WebapiInstance + info *downloadInfo + filepath string +} + +/* +apiCloneFile starts the download of a file, adds it the current warehouse and adds it +to your blockchain. The path is the full path on disk to store the file. +The hash parameter identifies the file to download. The node ID identifies the blockchain (i.e., the "owner" of the file). + +The following is a prototype implementation + +Request: GET /follow/file?path=[target path on disk]&hash=[file hash to download]&node=[node ID] +Result: 200 with JSON structure apiResponseDownloadStatus +*/ +func (api *WebapiInstance) apiHostFile(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + + // validate hashes, must be blake3 + hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) + if !valid1 || !valid2 { + http.Error(w, "node ID or hash not valid", http.StatusBadRequest) + return + } + + filePath := r.Form.Get("path") + if filePath == "" { + http.Error(w, "file path provided is blank", http.StatusBadRequest) + return + } + + info := &downloadInfo{backend: api.backend, api: api, id: uuid.New(), created: time.Now(), hash: hash, nodeID: nodeID} + + // create the file immediately + if info.initDiskFile(filePath) != nil { + EncodeJSON(api.backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseFileInvalid}) + return + } + + // add the download to the list + api.downloadAdd(info) + + // Setting host information + var HostInfo hostInfo + HostInfo.api = api + HostInfo.info = info + HostInfo.filepath = filePath + + // start the download! + go HostInfo.StartHostDownload() + + EncodeJSON(api.backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseSuccess, ID: info.id, DownloadStatus: DownloadWaitMetadata}) +} + +// StartHostDownload Starts the download, adds the downloaded file to the warehouse and then the following file is added to the user blockchain +func (info *hostInfo) StartHostDownload() error { + // current user? + if bytes.Equal(info.info.nodeID, info.info.backend.SelfNodeID()) { + info.info.DownloadSelf() + return nil + } + + for n := 0; n < 3 && info.info.peer == nil; n++ { + _, info.info.peer, _ = info.info.backend.FindNode(info.info.nodeID, time.Second*5) + + if info.info.status == DownloadCanceled { + return errors.New("DownloadCanceled") + } + } + + if info.info.peer != nil { + info.info.Download() + + // Adding the following file the warehouse + if info.filepath == "" { + return errors.New("File Path empty. ") + } + + hash, status, err := info.api.backend.UserWarehouse.CreateFileFromPath(info.filepath) + + if err != nil { + info.api.backend.LogError("warehouse.CreateFile", "status %d error: %v", status, err) + } + + // Adding file to the blockchain + var fileMetaData blockchain.BlockRecordFile + + // Add the following file to the blockchain + fileMetaData.ID = uuid.New() + fileMetaData.Hash = hash + + _, fileSize, _, _ := info.api.backend.UserWarehouse.FileExists(fileMetaData.Hash) + // setting file size + fileMetaData.Size = fileSize + + // create block based on file information provided + block := blockRecordFileFromAPI(info.info.file) + + var filesAdd []blockchain.BlockRecordFile + // appending the block to the file added + filesAdd = append(filesAdd, block) + + info.api.backend.UserBlockchain.AddFiles(filesAdd) + + } else { + info.info.status = DownloadCanceled + } + + return nil +}