From 23397e3d4a928852729da4b87079fe137796a890 Mon Sep 17 00:00:00 2001 From: Akilan Date: Thu, 27 Jul 2023 18:41:57 +0100 Subject: [PATCH] added new REST API for config file --- Blockchain Cache Global.go | 195 +++++++++++++++++-------------------- Config Default.yaml | 2 +- webapi/API.go | 1 + webapi/Status.go | 132 ++++++++++++++----------- 4 files changed, 167 insertions(+), 163 deletions(-) diff --git a/Blockchain Cache Global.go b/Blockchain Cache Global.go index bddbf77..664b381 100644 --- a/Blockchain Cache Global.go +++ b/Blockchain Cache Global.go @@ -7,151 +7,136 @@ Author: Peter Kleissner package core import ( - "github.com/PeernetOfficial/core/blockchain" - "github.com/PeernetOfficial/core/protocol" - "github.com/enfipy/locker" + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/protocol" + "github.com/enfipy/locker" ) // The blockchain cache stores blockchains. type BlockchainCache struct { - BlockchainDirectory string // The directory for storing blockchains in a key-value store. - MaxBlockSize uint64 // Max block size to accept. - MaxBlockCount uint64 // Max block count to cache per peer. - LimitTotalRecords uint64 // Max count of blocks and header in total to keep across all blockchains. 0 = unlimited. Max Records * Max Block Size = Size Limit. - ReadOnly bool // Whether the cache is read only. + BlockchainDirectory string // The directory for storing blockchains in a key-value store. + MaxBlockSize uint64 // Max block size to accept. + MaxBlockCount uint64 // Max block count to cache per peer. + LimitTotalRecords uint64 // Max count of blocks and header in total to keep across all blockchains. 0 = unlimited. Max Records * Max Block Size = Size Limit. + ReadOnly bool // Whether the cache is read only. - Store *blockchain.MultiStore - peerLock *locker.Locker + Store *blockchain.MultiStore + peerLock *locker.Locker - backend *Backend + backend *Backend } func (backend *Backend) initBlockchainCache() { - if backend.Config.BlockchainGlobal == "" { - return - } + if backend.Config.BlockchainGlobal == "" { + return + } - backend.GlobalBlockchainCache = &BlockchainCache{ - backend: backend, - BlockchainDirectory: backend.Config.BlockchainGlobal, - MaxBlockSize: backend.Config.CacheMaxBlockSize, - MaxBlockCount: backend.Config.CacheMaxBlockCount, - LimitTotalRecords: backend.Config.LimitTotalRecords, - } + backend.GlobalBlockchainCache = &BlockchainCache{ + backend: backend, + BlockchainDirectory: backend.Config.BlockchainGlobal, + MaxBlockSize: backend.Config.CacheMaxBlockSize, + MaxBlockCount: backend.Config.CacheMaxBlockCount, + LimitTotalRecords: backend.Config.LimitTotalRecords, + } - var err error - backend.GlobalBlockchainCache.Store, err = blockchain.InitMultiStore(backend.Config.BlockchainGlobal) - if err != nil { - backend.LogError("initBlockchainCache", "initializing database '%s': %s", backend.Config.BlockchainGlobal, err.Error()) - return - } + var err error + backend.GlobalBlockchainCache.Store, err = blockchain.InitMultiStore(backend.Config.BlockchainGlobal) + if err != nil { + backend.LogError("initBlockchainCache", "initializing database '%s': %s", backend.Config.BlockchainGlobal, err.Error()) + return + } - backend.GlobalBlockchainCache.peerLock = locker.Initialize() + backend.GlobalBlockchainCache.peerLock = locker.Initialize() - // Set the blockchain cache to read-only if the record limit is reached. - if backend.Config.LimitTotalRecords > 0 && backend.GlobalBlockchainCache.Store.Database.Count() >= backend.Config.LimitTotalRecords { - backend.GlobalBlockchainCache.ReadOnly = true - } + // Set the blockchain cache to read-only if the record limit is reached. + if backend.Config.LimitTotalRecords > 0 && backend.GlobalBlockchainCache.Store.Database.Count() >= backend.Config.LimitTotalRecords { + backend.GlobalBlockchainCache.ReadOnly = true + } - backend.GlobalBlockchainCache.Store.FilterStatisticUpdate = backend.Filters.GlobalBlockchainCacheStatistic - backend.GlobalBlockchainCache.Store.FilterBlockchainDelete = backend.Filters.GlobalBlockchainCacheDelete + backend.GlobalBlockchainCache.Store.FilterStatisticUpdate = backend.Filters.GlobalBlockchainCacheStatistic + backend.GlobalBlockchainCache.Store.FilterBlockchainDelete = backend.Filters.GlobalBlockchainCacheDelete } // SeenBlockchainVersion shall be called with information about another peer's blockchain. // If the reported version number is newer, all existing blocks are immediately deleted. func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) { - cache.peerLock.Lock(string(peer.PublicKey.SerializeCompressed())) - defer cache.peerLock.Unlock(string(peer.PublicKey.SerializeCompressed())) + cache.peerLock.Lock(string(peer.PublicKey.SerializeCompressed())) + defer cache.peerLock.Unlock(string(peer.PublicKey.SerializeCompressed())) - // intermediate function to download and process blocks - downloadAndProcessBlocks := func(peer *PeerInfo, header *blockchain.MultiBlockchainHeader, offset, limit uint64) { - if limit > cache.MaxBlockCount { - limit = cache.MaxBlockCount - } + // intermediate function to download and process blocks + downloadAndProcessBlocks := func(peer *PeerInfo, header *blockchain.MultiBlockchainHeader, offset, limit uint64) { + if limit > cache.MaxBlockCount { + limit = cache.MaxBlockCount + } - peer.BlockDownload(peer.PublicKey, cache.MaxBlockCount, cache.MaxBlockSize, []protocol.BlockRange{{Offset: offset, Limit: limit}}, func(data []byte, targetBlock protocol.BlockRange, blockSize uint64, availability uint8) { - if availability != protocol.GetBlockStatusAvailable { - return - } + peer.BlockDownload(peer.PublicKey, cache.MaxBlockCount, cache.MaxBlockSize, []protocol.BlockRange{{Offset: offset, Limit: limit}}, func(data []byte, targetBlock protocol.BlockRange, blockSize uint64, availability uint8) { + if availability != protocol.GetBlockStatusAvailable { + return + } - if decoded, _ := cache.Store.IngestBlock(header, targetBlock.Offset, data, true); decoded != nil { - // index it for search - cache.backend.SearchIndex.IndexNewBlockDecoded(peer.PublicKey, peer.BlockchainVersion, targetBlock.Offset, decoded.RecordsDecoded) - } - }) - } + if decoded, _ := cache.Store.IngestBlock(header, targetBlock.Offset, data, true); decoded != nil { + // index it for search + cache.backend.SearchIndex.IndexNewBlockDecoded(peer.PublicKey, peer.BlockchainVersion, targetBlock.Offset, decoded.RecordsDecoded) + } + }) + } - // get the old header - header, status, err := cache.Store.AssessBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight) - if err != nil { - return - } + // get the old header + header, status, err := cache.Store.AssessBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight) + if err != nil { + return + } - switch status { - case blockchain.MultiStatusEqual: - return + switch status { + case blockchain.MultiStatusEqual: + return - case blockchain.MultiStatusInvalidRemote: - cache.Store.DeleteBlockchain(header) + case blockchain.MultiStatusInvalidRemote: + cache.Store.DeleteBlockchain(header) - cache.backend.SearchIndex.UnindexBlockchain(peer.PublicKey) + cache.backend.SearchIndex.UnindexBlockchain(peer.PublicKey) - case blockchain.MultiStatusHeaderNA: - if header, err = cache.Store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { - return - } + case blockchain.MultiStatusHeaderNA: + if header, err = cache.Store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { + return + } - downloadAndProcessBlocks(peer, header, 0, peer.BlockchainHeight) + downloadAndProcessBlocks(peer, header, 0, peer.BlockchainHeight) - case blockchain.MultiStatusNewVersion: - // delete existing data first, then create it new - cache.Store.DeleteBlockchain(header) + case blockchain.MultiStatusNewVersion: + // delete existing data first, then create it new + cache.Store.DeleteBlockchain(header) - cache.backend.SearchIndex.UnindexBlockchain(peer.PublicKey) + cache.backend.SearchIndex.UnindexBlockchain(peer.PublicKey) - if header, err = cache.Store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { - return - } + if header, err = cache.Store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { + return + } - downloadAndProcessBlocks(peer, header, 0, peer.BlockchainHeight) + downloadAndProcessBlocks(peer, header, 0, peer.BlockchainHeight) - case blockchain.MultiStatusNewBlocks: - //offset := header.Height - //limit := peer.BlockchainHeight - header.Height - // - //header.Height = peer.BlockchainHeight - // - //fmt.Println("new peer height") - // - //downloadAndProcessBlocks(peer, header, offset, limit) - // delete existing data first, then create it new - // Temporary Fix to force update the public key - cache.Store.DeleteBlockchain(header) + case blockchain.MultiStatusNewBlocks: + offset := header.Height + limit := peer.BlockchainHeight - header.Height + header.Height = peer.BlockchainHeight + downloadAndProcessBlocks(peer, header, offset, limit) - cache.backend.SearchIndex.UnindexBlockchain(peer.PublicKey) + } - if header, err = cache.Store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { - return - } - - downloadAndProcessBlocks(peer, header, 0, peer.BlockchainHeight) - - } - - if cache.LimitTotalRecords > 0 { - // Bug: This code is currently never reached if ReadOnly is true. - cache.ReadOnly = cache.Store.Database.Count() >= cache.LimitTotalRecords - } + if cache.LimitTotalRecords > 0 { + // Bug: This code is currently never reached if ReadOnly is true. + cache.ReadOnly = cache.Store.Database.Count() >= cache.LimitTotalRecords + } } // remoteBlockchainUpdate shall be called to indicate a potential update of the remotes blockchain. // It will use the blockchain version and height to update the data lake as appropriate. // This function is called in the Go routine of the packet worker and therefore must not stall. func (peer *PeerInfo) remoteBlockchainUpdate() { - if peer.Backend.GlobalBlockchainCache == nil || peer.Backend.GlobalBlockchainCache.ReadOnly || peer.BlockchainVersion == 0 && peer.BlockchainHeight == 0 { - return - } + if peer.Backend.GlobalBlockchainCache == nil || peer.Backend.GlobalBlockchainCache.ReadOnly || peer.BlockchainVersion == 0 && peer.BlockchainHeight == 0 { + return + } - // TODO: This entire function should be instead a non-blocking message via a buffer channel. - go peer.Backend.GlobalBlockchainCache.SeenBlockchainVersion(peer) + // TODO: This entire function should be instead a non-blocking message via a buffer channel. + go peer.Backend.GlobalBlockchainCache.SeenBlockchainVersion(peer) } diff --git a/Config Default.yaml b/Config Default.yaml index d76f920..e457e0d 100644 --- a/Config Default.yaml +++ b/Config Default.yaml @@ -43,6 +43,6 @@ LocalFirewall: false # Indicates that a local firewall may drop unsolicited i PortForward: 0 # Default not set. # Global blockchain cache limits -CacheMaxBlockSize: 4096 # Max block size to accept in bytes. +CacheMaxBlockSize: 50096 # Max block size to accept in bytes. CacheMaxBlockCount: 256 # Max block count to cache per peer. LimitTotalRecords: 0 # Record count limit. 0 = unlimited. Max Records * Max Block Size = Size Limit. diff --git a/webapi/API.go b/webapi/API.go index 1e553d9..e904e44 100644 --- a/webapi/API.go +++ b/webapi/API.go @@ -83,6 +83,7 @@ func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, Certifi api.Router.HandleFunc("/test", apiTest).Methods("GET") api.Router.HandleFunc("/status", api.apiStatus).Methods("GET") api.Router.HandleFunc("/status/peers", api.apiStatusPeers).Methods("GET") + api.Router.HandleFunc("/status/config", api.apiStatusConfig).Methods("GET") api.Router.HandleFunc("/account/info", api.apiAccountInfo).Methods("GET") api.Router.HandleFunc("/account/delete", api.apiAccountDelete).Methods("GET") api.Router.HandleFunc("/blockchain/header", api.apiBlockchainHeaderFunc).Methods("GET") diff --git a/webapi/Status.go b/webapi/Status.go index 37cbdc5..89dccd8 100644 --- a/webapi/Status.go +++ b/webapi/Status.go @@ -7,24 +7,24 @@ Author: Peter Kleissner package webapi import ( - "encoding/hex" - "fmt" - "net/http" - "strconv" + "encoding/hex" + "fmt" + "net/http" + "strconv" ) func apiTest(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) } type apiResponseStatus struct { - Status int `json:"status"` // Status code: 0 = Ok. - IsConnected bool `json:"isconnected"` // Whether connected to Peernet. - CountPeerList int `json:"countpeerlist"` // Count of peers in the peer list. Note that this contains peers that are considered inactive, but have not yet been removed from the list. - CountNetwork int `json:"countnetwork"` // Count of total peers in the network. - // This is usually a higher number than CountPeerList, which just represents the current number of connected peers. - // The CountNetwork number is going to be queried from root peers which may or may not have a limited view. + Status int `json:"status"` // Status code: 0 = Ok. + IsConnected bool `json:"isconnected"` // Whether connected to Peernet. + CountPeerList int `json:"countpeerlist"` // Count of peers in the peer list. Note that this contains peers that are considered inactive, but have not yet been removed from the list. + CountNetwork int `json:"countnetwork"` // Count of total peers in the network. + // This is usually a higher number than CountPeerList, which just represents the current number of connected peers. + // The CountNetwork number is going to be queried from root peers which may or may not have a limited view. } /* @@ -33,20 +33,20 @@ Request: GET /status Result: 200 with JSON structure Status */ func (api *WebapiInstance) apiStatus(w http.ResponseWriter, r *http.Request) { - status := apiResponseStatus{Status: 0, CountPeerList: api.Backend.PeerlistCount()} - status.CountNetwork = status.CountPeerList // For now always same as CountPeerList, until native Statistics message to root peers is available. + status := apiResponseStatus{Status: 0, CountPeerList: api.Backend.PeerlistCount()} + status.CountNetwork = status.CountPeerList // For now always same as CountPeerList, until native Statistics message to root peers is available. - // Connected: If at leat 2 peers. - // This metric needs to be improved in the future, as root peers never disconnect. - // Instead, the core should keep a count of "active peers". - status.IsConnected = status.CountPeerList >= 2 + // Connected: If at leat 2 peers. + // This metric needs to be improved in the future, as root peers never disconnect. + // Instead, the core should keep a count of "active peers". + status.IsConnected = status.CountPeerList >= 2 - EncodeJSON(api.Backend, w, r, status) + EncodeJSON(api.Backend, w, r, status) } type apiResponsePeerSelf struct { - PeerID string `json:"peerid"` // Peer ID. This is derived from the public in compressed form. - NodeID string `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. + PeerID string `json:"peerid"` // Peer ID. This is derived from the public in compressed form. + NodeID string `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. } /* @@ -55,13 +55,13 @@ Request: GET /account/info Result: 200 with JSON structure apiResponsePeerSelf */ func (api *WebapiInstance) apiAccountInfo(w http.ResponseWriter, r *http.Request) { - response := apiResponsePeerSelf{} - response.NodeID = hex.EncodeToString(api.Backend.SelfNodeID()) + response := apiResponsePeerSelf{} + response.NodeID = hex.EncodeToString(api.Backend.SelfNodeID()) - _, publicKey := api.Backend.ExportPrivateKey() - response.PeerID = hex.EncodeToString(publicKey.SerializeCompressed()) + _, publicKey := api.Backend.ExportPrivateKey() + response.PeerID = hex.EncodeToString(publicKey.SerializeCompressed()) - EncodeJSON(api.Backend, w, r, response) + EncodeJSON(api.Backend, w, r, response) } /* @@ -72,15 +72,15 @@ Result: 204 if the user choses not to delete the account 200 if successfully deleted */ func (api *WebapiInstance) apiAccountDelete(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - if confirm, _ := strconv.ParseBool(r.Form.Get("confirm")); !confirm { - w.WriteHeader(http.StatusNoContent) - return - } + r.ParseForm() + if confirm, _ := strconv.ParseBool(r.Form.Get("confirm")); !confirm { + w.WriteHeader(http.StatusNoContent) + return + } - api.Backend.DeleteAccount() + api.Backend.DeleteAccount() - w.WriteHeader(http.StatusOK) + w.WriteHeader(http.StatusOK) } /* @@ -92,35 +92,53 @@ Request: GET /status/peers Result: 200 with JSON array apiResponsePeerInfo */ func (api *WebapiInstance) apiStatusPeers(w http.ResponseWriter, r *http.Request) { - var peers []apiResponsePeerInfo + var peers []apiResponsePeerInfo - // query all nodes - for _, peer := range api.Backend.PeerlistGet() { - peerInfo := apiResponsePeerInfo{ - PeerID: peer.PublicKey.SerializeCompressed(), - NodeID: peer.NodeID, - UserAgent: peer.UserAgent, - IsRoot: peer.IsRootPeer, - BlockchainHeight: peer.BlockchainHeight, - BlockchainVersion: peer.BlockchainVersion, - } + // query all nodes + for _, peer := range api.Backend.PeerlistGet() { + peerInfo := apiResponsePeerInfo{ + PeerID: peer.PublicKey.SerializeCompressed(), + NodeID: peer.NodeID, + UserAgent: peer.UserAgent, + IsRoot: peer.IsRootPeer, + BlockchainHeight: peer.BlockchainHeight, + BlockchainVersion: peer.BlockchainVersion, + } - if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { - peerInfo.GeoIP = fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) - } + if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { + peerInfo.GeoIP = fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) + } - peers = append(peers, peerInfo) - } + peers = append(peers, peerInfo) + } - EncodeJSON(api.Backend, w, r, peers) + EncodeJSON(api.Backend, w, r, peers) } type apiResponsePeerInfo struct { - PeerID []byte `json:"peerid"` // Peer ID. This is derived from the public in compressed form. - NodeID []byte `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. - GeoIP string `json:"geoip"` // GeoIP location as "Latitude,Longitude" CSV format. Empty if location not available. - UserAgent string `json:"useragent"` // User Agent. - IsRoot bool `json:"isroot"` // If the peer is a root peer. - BlockchainHeight uint64 `json:"blockchainheight"` // Blockchain height - BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version + PeerID []byte `json:"peerid"` // Peer ID. This is derived from the public in compressed form. + NodeID []byte `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. + GeoIP string `json:"geoip"` // GeoIP location as "Latitude,Longitude" CSV format. Empty if location not available. + UserAgent string `json:"useragent"` // User Agent. + IsRoot bool `json:"isroot"` // If the peer is a root peer. + BlockchainHeight uint64 `json:"blockchainheight"` // Blockchain height + BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version +} + +/* +apiConfigPeers return the appropriate config information of the current Peer +connected. + +Request: GET /status/config +Result: 200 with JSON apiResponseConfig +*/ +func (api *WebapiInstance) apiStatusConfig(w http.ResponseWriter, r *http.Request) { + var peer apiResponseConfig + peer.BlockSize = api.Backend.Config.CacheMaxBlockSize + + EncodeJSON(api.Backend, w, r, peer) +} + +type apiResponseConfig struct { + BlockSize uint64 `json:"blockSize"` }