diff --git a/Blockchain Cache Global.go b/Blockchain Cache Global.go index d3f0048..2b8926a 100644 --- a/Blockchain Cache Global.go +++ b/Blockchain Cache Global.go @@ -1,144 +1,144 @@ -/* -File Name: Blockchain Cache Global.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "github.com/PeernetOfficial/core/blockchain" - "github.com/PeernetOfficial/core/protocol" - "github.com/enfipy/locker" -) - -// The blockchain cache stores blockchains. -type BlockchainCache struct { - BlockchainDirectory string // The directory for storing blockchains in a key-value store. - MaxBlockSize uint64 // Max block size to accept. - MaxBlockCount uint64 // Max block count to cache per peer. - LimitTotalRecords uint64 // Max count of blocks and header in total to keep across all blockchains. 0 = unlimited. Max Records * Max Block Size = Size Limit. - ReadOnly bool // Whether the cache is read only. - - Store *blockchain.MultiStore - peerLock *locker.Locker - - backend *Backend -} - -func (backend *Backend) initBlockchainCache() { - if backend.Config.BlockchainGlobal == "" { - return - } - - backend.GlobalBlockchainCache = &BlockchainCache{ - backend: backend, - BlockchainDirectory: backend.Config.BlockchainGlobal, - MaxBlockSize: backend.Config.CacheMaxBlockSize, - MaxBlockCount: backend.Config.CacheMaxBlockCount, - LimitTotalRecords: backend.Config.LimitTotalRecords, - } - - var err error - backend.GlobalBlockchainCache.Store, err = blockchain.InitMultiStore(backend.Config.BlockchainGlobal) - if err != nil { - backend.LogError("initBlockchainCache", "initializing database '%s': %s", backend.Config.BlockchainGlobal, err.Error()) - return - } - - backend.GlobalBlockchainCache.peerLock = locker.Initialize() - - // Set the blockchain cache to read-only if the record limit is reached. - if backend.Config.LimitTotalRecords > 0 && backend.GlobalBlockchainCache.Store.Database.Count() >= backend.Config.LimitTotalRecords { - backend.GlobalBlockchainCache.ReadOnly = true - } - - backend.GlobalBlockchainCache.Store.FilterStatisticUpdate = backend.Filters.GlobalBlockchainCacheStatistic - backend.GlobalBlockchainCache.Store.FilterBlockchainDelete = backend.Filters.GlobalBlockchainCacheDelete -} - -// SeenBlockchainVersion shall be called with information about another peer's blockchain. -// If the reported version number is newer, all existing blocks are immediately deleted. -func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) { - cache.peerLock.Lock(string(peer.PublicKey.SerializeCompressed())) - defer cache.peerLock.Unlock(string(peer.PublicKey.SerializeCompressed())) - - // intermediate function to download and process blocks - downloadAndProcessBlocks := func(peer *PeerInfo, header *blockchain.MultiBlockchainHeader, offset, limit uint64) { - if limit > cache.MaxBlockCount { - limit = cache.MaxBlockCount - } - - peer.BlockDownload(peer.PublicKey, cache.MaxBlockCount, cache.MaxBlockSize, []protocol.BlockRange{{Offset: offset, Limit: limit}}, func(data []byte, targetBlock protocol.BlockRange, blockSize uint64, availability uint8) { - if availability != protocol.GetBlockStatusAvailable { - return - } - - if decoded, _ := cache.Store.IngestBlock(header, targetBlock.Offset, data, true); decoded != nil { - // index it for search - cache.backend.SearchIndex.IndexNewBlockDecoded(peer.PublicKey, peer.BlockchainVersion, targetBlock.Offset, decoded.RecordsDecoded) - } - }) - } - - // get the old header - header, status, err := cache.Store.AssessBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight) - if err != nil { - return - } - - switch status { - case blockchain.MultiStatusEqual: - return - - case blockchain.MultiStatusInvalidRemote: - cache.Store.DeleteBlockchain(header) - - cache.backend.SearchIndex.UnindexBlockchain(peer.PublicKey) - - case blockchain.MultiStatusHeaderNA: - if header, err = cache.Store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { - return - } - - downloadAndProcessBlocks(peer, header, 0, peer.BlockchainHeight) - - case blockchain.MultiStatusNewVersion: - // delete existing data first, then create it new - cache.Store.DeleteBlockchain(header) - - cache.backend.SearchIndex.UnindexBlockchain(peer.PublicKey) - - if header, err = cache.Store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { - return - } - - downloadAndProcessBlocks(peer, header, 0, peer.BlockchainHeight) - - case blockchain.MultiStatusNewBlocks: - offset := header.Height - limit := peer.BlockchainHeight - header.Height - - header.Height = peer.BlockchainHeight - - downloadAndProcessBlocks(peer, header, offset, limit) - - } - - if cache.LimitTotalRecords > 0 { - // Bug: This code is currently never reached if ReadOnly is true. - cache.ReadOnly = cache.Store.Database.Count() >= cache.LimitTotalRecords - } -} - -// remoteBlockchainUpdate shall be called to indicate a potential update of the remotes blockchain. -// It will use the blockchain version and height to update the data lake as appropriate. -// This function is called in the Go routine of the packet worker and therefore must not stall. -func (peer *PeerInfo) remoteBlockchainUpdate() { - if peer.Backend.GlobalBlockchainCache == nil || peer.Backend.GlobalBlockchainCache.ReadOnly || peer.BlockchainVersion == 0 && peer.BlockchainHeight == 0 { - return - } - - // TODO: This entire function should be instead a non-blocking message via a buffer channel. - go peer.Backend.GlobalBlockchainCache.SeenBlockchainVersion(peer) -} +/* +File Name: Blockchain Cache Global.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/protocol" + "github.com/enfipy/locker" +) + +// The blockchain cache stores blockchains. +type BlockchainCache struct { + BlockchainDirectory string // The directory for storing blockchains in a key-value store. + MaxBlockSize uint64 // Max block size to accept. + MaxBlockCount uint64 // Max block count to cache per peer. + LimitTotalRecords uint64 // Max count of blocks and header in total to keep across all blockchains. 0 = unlimited. Max Records * Max Block Size = Size Limit. + ReadOnly bool // Whether the cache is read only. + + Store *blockchain.MultiStore + peerLock *locker.Locker + + backend *Backend +} + +func (backend *Backend) initBlockchainCache() { + if backend.Config.BlockchainGlobal == "" { + return + } + + backend.GlobalBlockchainCache = &BlockchainCache{ + backend: backend, + BlockchainDirectory: backend.Config.BlockchainGlobal, + MaxBlockSize: backend.Config.CacheMaxBlockSize, + MaxBlockCount: backend.Config.CacheMaxBlockCount, + LimitTotalRecords: backend.Config.LimitTotalRecords, + } + + var err error + backend.GlobalBlockchainCache.Store, err = blockchain.InitMultiStore(backend.Config.BlockchainGlobal) + if err != nil { + backend.LogError("initBlockchainCache", "initializing database '%s': %s", backend.Config.BlockchainGlobal, err.Error()) + return + } + + backend.GlobalBlockchainCache.peerLock = locker.Initialize() + + // Set the blockchain cache to read-only if the record limit is reached. + if backend.Config.LimitTotalRecords > 0 && backend.GlobalBlockchainCache.Store.Database.Count() >= backend.Config.LimitTotalRecords { + backend.GlobalBlockchainCache.ReadOnly = true + } + + backend.GlobalBlockchainCache.Store.FilterStatisticUpdate = backend.Filters.GlobalBlockchainCacheStatistic + backend.GlobalBlockchainCache.Store.FilterBlockchainDelete = backend.Filters.GlobalBlockchainCacheDelete +} + +// SeenBlockchainVersion shall be called with information about another peer's blockchain. +// If the reported version number is newer, all existing blocks are immediately deleted. +func (cache *BlockchainCache) SeenBlockchainVersion(peer *PeerInfo) { + cache.peerLock.Lock(string(peer.PublicKey.SerializeCompressed())) + defer cache.peerLock.Unlock(string(peer.PublicKey.SerializeCompressed())) + + // intermediate function to download and process blocks + downloadAndProcessBlocks := func(peer *PeerInfo, header *blockchain.MultiBlockchainHeader, offset, limit uint64) { + if limit > cache.MaxBlockCount { + limit = cache.MaxBlockCount + } + + peer.BlockDownload(peer.PublicKey, cache.MaxBlockCount, cache.MaxBlockSize, []protocol.BlockRange{{Offset: offset, Limit: limit}}, func(data []byte, targetBlock protocol.BlockRange, blockSize uint64, availability uint8) { + if availability != protocol.GetBlockStatusAvailable { + return + } + + if decoded, _ := cache.Store.IngestBlock(header, targetBlock.Offset, data, true); decoded != nil { + // index it for search + cache.backend.SearchIndex.IndexNewBlockDecoded(peer.PublicKey, peer.BlockchainVersion, targetBlock.Offset, decoded.RecordsDecoded) + } + }) + } + + // get the old header + header, status, err := cache.Store.AssessBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight) + if err != nil { + return + } + + switch status { + case blockchain.MultiStatusEqual: + return + + case blockchain.MultiStatusInvalidRemote: + cache.Store.DeleteBlockchain(header) + + cache.backend.SearchIndex.UnindexBlockchain(peer.PublicKey) + + case blockchain.MultiStatusHeaderNA: + if header, err = cache.Store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { + return + } + + downloadAndProcessBlocks(peer, header, 0, peer.BlockchainHeight) + + case blockchain.MultiStatusNewVersion: + // delete existing data first, then create it new + cache.Store.DeleteBlockchain(header) + + cache.backend.SearchIndex.UnindexBlockchain(peer.PublicKey) + + if header, err = cache.Store.NewBlockchainHeader(peer.PublicKey, peer.BlockchainVersion, peer.BlockchainHeight); err != nil { + return + } + + downloadAndProcessBlocks(peer, header, 0, peer.BlockchainHeight) + + case blockchain.MultiStatusNewBlocks: + offset := header.Height + limit := peer.BlockchainHeight - header.Height + + header.Height = peer.BlockchainHeight + + downloadAndProcessBlocks(peer, header, offset, limit) + + } + + if cache.LimitTotalRecords > 0 { + // Bug: This code is currently never reached if ReadOnly is true. + cache.ReadOnly = cache.Store.Database.Count() >= cache.LimitTotalRecords + } +} + +// remoteBlockchainUpdate shall be called to indicate a potential update of the remotes blockchain. +// It will use the blockchain version and height to update the data lake as appropriate. +// This function is called in the Go routine of the packet worker and therefore must not stall. +func (peer *PeerInfo) remoteBlockchainUpdate() { + if peer.Backend.GlobalBlockchainCache == nil || peer.Backend.GlobalBlockchainCache.ReadOnly || peer.BlockchainVersion == 0 && peer.BlockchainHeight == 0 { + return + } + + // TODO: This entire function should be instead a non-blocking message via a buffer channel. + go peer.Backend.GlobalBlockchainCache.SeenBlockchainVersion(peer) +} diff --git a/Blockchain User.go b/Blockchain User.go index af12eb6..81af81b 100644 --- a/Blockchain User.go +++ b/Blockchain User.go @@ -1,110 +1,110 @@ -/* -File Name: Blockchain User.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "os" - - "github.com/PeernetOfficial/core/blockchain" - "github.com/PeernetOfficial/core/btcec" - "github.com/google/uuid" -) - -// initUserBlockchain initializes the users blockchain. It creates the blockchain file if it does not exist already. -// If it is corrupted, it will log the error and exit the process. -func (backend *Backend) initUserBlockchain() { - var err error - backend.UserBlockchain, err = blockchain.Init(backend.PeerPrivateKey, backend.Config.BlockchainMain) - - if err != nil { - backend.LogError("initUserBlockchain", "error: %s\n", err.Error()) - os.Exit(ExitBlockchainCorrupt) - } -} - -// Index the user's blockchain each time there is an update. -func (backend *Backend) userBlockchainUpdateSearchIndex() { - backend.UserBlockchain.BlockchainUpdate = func(blockchainU *blockchain.Blockchain, oldHeight, oldVersion, newHeight, newVersion uint64) { - - if newVersion != oldVersion || newHeight < oldHeight { - // invalidate search index data for the user's blockchain - backend.SearchIndex.UnindexBlockchain(backend.PeerPublicKey) - - // reindex everything - for blockN := uint64(0); blockN < newHeight; blockN++ { - raw, status, err := blockchainU.GetBlockRaw(blockN) - if err != nil || status != blockchain.StatusOK { - continue - } - - backend.SearchIndex.IndexNewBlock(backend.PeerPublicKey, newVersion, blockN, raw) - } - - return - } - - if newVersion == oldVersion && newHeight > oldHeight { - // index the new blocks - for blockN := oldHeight; blockN < newHeight; blockN++ { - raw, status, err := blockchainU.GetBlockRaw(blockN) - if err != nil || status != blockchain.StatusOK { - continue - } - - backend.SearchIndex.IndexNewBlock(backend.PeerPublicKey, newVersion, blockN, raw) - } - } - } -} - -// ReadBlock reads a block and decodes the records. This may be a block of the user's blockchain, or any other that is cached in the global blockchain cache. -func (backend *Backend) ReadBlock(PublicKey *btcec.PublicKey, Version, BlockNumber uint64) (decoded *blockchain.BlockDecoded, raw []byte, found bool, err error) { - // requesting a block from the user's blockchain? - if PublicKey.IsEqual(backend.PeerPublicKey) { - _, _, version := backend.UserBlockchain.Header() - if Version != version { - return nil, nil, false, nil - } - - var status int - raw, status, err = backend.UserBlockchain.GetBlockRaw(BlockNumber) - if err != nil || status != blockchain.StatusOK { - return nil, raw, false, err - } - } else if backend.GlobalBlockchainCache != nil { - // read from the cache - if raw, found = backend.GlobalBlockchainCache.Store.ReadBlock(PublicKey, Version, BlockNumber); !found { - return nil, nil, false, nil - } - } else { - return nil, nil, false, nil - } - - // decode the entire block - blockDecoded, status, err := blockchain.DecodeBlockRaw(raw) - if err != nil || status != blockchain.StatusOK { - return nil, raw, false, err - } - - return blockDecoded, raw, true, nil -} - -// ReadFile decodes a file from the given blockchain (the user or any other cached). The block number must be provided. -func (backend *Backend) ReadFile(PublicKey *btcec.PublicKey, Version, BlockNumber uint64, FileID uuid.UUID) (file blockchain.BlockRecordFile, raw []byte, found bool, err error) { - blockDecoded, raw, found, err := backend.ReadBlock(PublicKey, Version, BlockNumber) - if !found { - return file, raw, found, err - } - - for _, decodedR := range blockDecoded.RecordsDecoded { - if file, ok := decodedR.(blockchain.BlockRecordFile); ok && file.ID == FileID { - return file, raw, true, nil - } - } - - return file, raw, false, nil -} +/* +File Name: Blockchain User.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "os" + + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/btcec" + "github.com/google/uuid" +) + +// initUserBlockchain initializes the users blockchain. It creates the blockchain file if it does not exist already. +// If it is corrupted, it will log the error and exit the process. +func (backend *Backend) initUserBlockchain() { + var err error + backend.UserBlockchain, err = blockchain.Init(backend.PeerPrivateKey, backend.Config.BlockchainMain) + + if err != nil { + backend.LogError("initUserBlockchain", "error: %s\n", err.Error()) + os.Exit(ExitBlockchainCorrupt) + } +} + +// Index the user's blockchain each time there is an update. +func (backend *Backend) userBlockchainUpdateSearchIndex() { + backend.UserBlockchain.BlockchainUpdate = func(blockchainU *blockchain.Blockchain, oldHeight, oldVersion, newHeight, newVersion uint64) { + + if newVersion != oldVersion || newHeight < oldHeight { + // invalidate search index data for the user's blockchain + backend.SearchIndex.UnindexBlockchain(backend.PeerPublicKey) + + // reindex everything + for blockN := uint64(0); blockN < newHeight; blockN++ { + raw, status, err := blockchainU.GetBlockRaw(blockN) + if err != nil || status != blockchain.StatusOK { + continue + } + + backend.SearchIndex.IndexNewBlock(backend.PeerPublicKey, newVersion, blockN, raw) + } + + return + } + + if newVersion == oldVersion && newHeight > oldHeight { + // index the new blocks + for blockN := oldHeight; blockN < newHeight; blockN++ { + raw, status, err := blockchainU.GetBlockRaw(blockN) + if err != nil || status != blockchain.StatusOK { + continue + } + + backend.SearchIndex.IndexNewBlock(backend.PeerPublicKey, newVersion, blockN, raw) + } + } + } +} + +// ReadBlock reads a block and decodes the records. This may be a block of the user's blockchain, or any other that is cached in the global blockchain cache. +func (backend *Backend) ReadBlock(PublicKey *btcec.PublicKey, Version, BlockNumber uint64) (decoded *blockchain.BlockDecoded, raw []byte, found bool, err error) { + // requesting a block from the user's blockchain? + if PublicKey.IsEqual(backend.PeerPublicKey) { + _, _, version := backend.UserBlockchain.Header() + if Version != version { + return nil, nil, false, nil + } + + var status int + raw, status, err = backend.UserBlockchain.GetBlockRaw(BlockNumber) + if err != nil || status != blockchain.StatusOK { + return nil, raw, false, err + } + } else if backend.GlobalBlockchainCache != nil { + // read from the cache + if raw, found = backend.GlobalBlockchainCache.Store.ReadBlock(PublicKey, Version, BlockNumber); !found { + return nil, nil, false, nil + } + } else { + return nil, nil, false, nil + } + + // decode the entire block + blockDecoded, status, err := blockchain.DecodeBlockRaw(raw) + if err != nil || status != blockchain.StatusOK { + return nil, raw, false, err + } + + return blockDecoded, raw, true, nil +} + +// ReadFile decodes a file from the given blockchain (the user or any other cached). The block number must be provided. +func (backend *Backend) ReadFile(PublicKey *btcec.PublicKey, Version, BlockNumber uint64, FileID uuid.UUID) (file blockchain.BlockRecordFile, raw []byte, found bool, err error) { + blockDecoded, raw, found, err := backend.ReadBlock(PublicKey, Version, BlockNumber) + if !found { + return file, raw, found, err + } + + for _, decodedR := range blockDecoded.RecordsDecoded { + if file, ok := decodedR.(blockchain.BlockRecordFile); ok && file.ID == FileID { + return file, raw, true, nil + } + } + + return file, raw, false, nil +} diff --git a/Bootstrap.go b/Bootstrap.go index 3fc14a9..7adebd6 100644 --- a/Bootstrap.go +++ b/Bootstrap.go @@ -1,395 +1,395 @@ -/* -File Name: Bootstrap.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Strategy for sending our IPv6 Multicast and IPv4 Broadcast messages: -* During bootstrap: Immediately at the beginning, then every 10 seconds until there is at least 1 peer. -* Every 10 minutes during regular operation. -* Each time a network adapter / IP change is detected. - -*/ - -package core - -import ( - "encoding/hex" - "errors" - "net" - "strconv" - "sync" - "time" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/protocol" -) - -// rootPeer is a single root peer info -type rootPeer struct { - peer *PeerInfo // loaded PeerInfo - publicKey *btcec.PublicKey // Public key - addresses []*net.UDPAddr // IP:Port addresses - backend *Backend -} - -var rootPeers map[[btcec.PubKeyBytesLenCompressed]byte]*rootPeer - -// initSeedList loads the seed list from the config -// Note: This should be called before any network listening function so that incoming root peers are properly recognized. -func (backend *Backend) initSeedList() { - rootPeers = make(map[[btcec.PubKeyBytesLenCompressed]byte]*rootPeer) - recentContacts = make(map[[btcec.PubKeyBytesLenCompressed]byte]*recentContactInfo) - -loopSeedList: - for _, seed := range backend.Config.SeedList { - peer := &rootPeer{backend: backend} - - // parse the Public Key - publicKeyB, err := hex.DecodeString(seed.PublicKey) - if err != nil { - backend.LogError("initSeedList", "public key '%s': %v\n", seed.PublicKey, err.Error()) - continue - } - - if peer.publicKey, err = btcec.ParsePubKey(publicKeyB, btcec.S256()); err != nil { - backend.LogError("initSeedList", "public key '%s': %v\n", seed.PublicKey, err.Error()) - continue - } - - if peer.publicKey.IsEqual(backend.PeerPublicKey) { // skip if self - continue - } - - // parse all IP addresses - for _, addressA := range seed.Address { - address, err := parseAddress(addressA) - if err != nil { - backend.LogError("initSeedList", "public key '%s' address '%s': %v\n", seed.PublicKey, addressA, err.Error()) - continue loopSeedList - } - - peer.addresses = append(peer.addresses, address) - } - - rootPeers[publicKey2Compressed(peer.publicKey)] = peer - } -} - -// parseAddress parses an input peer address in the form "IP:Port". -func parseAddress(Address string) (remote *net.UDPAddr, err error) { - host, portA, err := net.SplitHostPort(Address) - if err != nil { - return nil, err - } - - portI, err := strconv.Atoi(portA) - if err != nil { - return nil, err - } else if portI <= 0 || portI > 65535 { - return nil, errors.New("invalid port number") - } - - ip := net.ParseIP(host) - if ip == nil { - return nil, errors.New("invalid input IP") - } - - return &net.UDPAddr{IP: ip, Port: portI}, err -} - -// contact tries to contact the root peer on all networks -func (peer *rootPeer) contact() { - // If already in peer list, no need to contact. - if peer.backend.PeerlistLookup(peer.publicKey) != nil { - return - } - - for _, address := range peer.addresses { - // Port internal is always set to 0 for root peers. It disables NAT detection and will not send out a Traverse message. - peer.backend.contactArbitraryPeer(peer.publicKey, address, 0, false) - } -} - -// bootstrap connects to the initial set of peers. -func (backend *Backend) bootstrap() { - go resetRecentContacts() - - if len(rootPeers) == 0 { - backend.LogError("bootstrap", "warning: Empty list of root peers. Connectivity relies on local peer discovery and incoming connections.\n") - return - } - - contactRootPeers := func() { - for _, peer := range rootPeers { - if peer.peer == nil { - peer.contact() - } - } - } - - countConnectedRootPeers := func() (connectedCount, total int) { - for _, peer := range rootPeers { - if peer.peer != nil { - connectedCount++ - } else if peer.peer = peer.backend.PeerlistLookup(peer.publicKey); peer.peer != nil { - connectedCount++ - } - } - return connectedCount, len(rootPeers) - } - - // initial contact to all root peer - contactRootPeers() - - // Phase 1: First 10 minutes. Try every 7 seconds to connect to all root peers until at least 2 peers connected. - for n := 0; n < 10*60/7; n++ { - time.Sleep(time.Second * 7) - - if connected, total := countConnectedRootPeers(); connected == total || connected >= 2 { - return - } - - contactRootPeers() - } - - // Phase 2: After that (if not 2 peers), try every 5 minutes to connect to remaining root peers for a maximum of 1 hour. - for n := 0; n < 1*60/5; n++ { - time.Sleep(time.Minute * 5) - - contactRootPeers() - - if connected, total := countConnectedRootPeers(); connected == total || connected >= 2 { - return - } - } - - backend.LogError("bootstrap", "unable to connect to at least 2 root peers, aborting\n") -} - -func (nets *Networks) autoMulticastBroadcast() { - sendMulticastBroadcast := func() { - nets.RLock() - defer nets.RUnlock() - - for _, network := range nets.networks6 { - if err := network.MulticastIPv6Send(); err != nil { - nets.backend.LogError("autoMulticastBroadcast", "multicast from network address '%s': %v\n", network.address.IP.String(), err.Error()) - } - } - - for _, network := range nets.networks4 { - if err := network.BroadcastIPv4Send(); err != nil { - nets.backend.LogError("autoMulticastBroadcast", "broadcast from network address '%s': %v\n", network.address.IP.String(), err.Error()) - } - } - } - - // Send out multicast/broadcast immediately. - sendMulticastBroadcast() - - // Phase 1: Resend every 10 seconds until at least 1 peer in the peer list. - for { - time.Sleep(time.Second * 10) - - if nets.backend.PeerlistCount() >= 1 { - break - } - - sendMulticastBroadcast() - } - - // Phase 2: Every 10 minutes. - for { - time.Sleep(time.Minute * 10) - sendMulticastBroadcast() - } -} - -// contactArbitraryPeer contacts a new arbitrary peer for the first time. -func (backend *Backend) contactArbitraryPeer(publicKey *btcec.PublicKey, address *net.UDPAddr, receiverPortInternal uint16, receiverFirewall bool) (contacted bool) { - findSelf := ShouldSendFindSelf() - _, blockchainHeight, blockchainVersion := backend.UserBlockchain.Header() - packets := protocol.EncodeAnnouncement(true, findSelf, nil, nil, nil, backend.FeatureSupport(), blockchainHeight, blockchainVersion, backend.userAgent) - if len(packets) == 0 { - return false - } - raw := &protocol.PacketRaw{Command: protocol.CommandAnnouncement, Payload: packets[0]} - - backend.Filters.MessageOutAnnouncement(publicKey, nil, raw, findSelf, nil, nil, nil) - - backend.networks.sendAllNetworks(publicKey, raw, address, receiverPortInternal, receiverFirewall, nil, &bootstrapFindSelf{}) - - return true -} - -// bootstrapFindSelf is a dummy structure assigned to sequences when sending the Announcement message. -// When receiving the Response message, it will know that it was a legitimate bootstrap request. -type bootstrapFindSelf struct { -} - -// bootstrapAcceptContacts is the maximum count of contacts considered. It limits the impact of fake peers. -const bootstrapAcceptContacts = 5 - -// cmdResponseBootstrapFindSelf processes FIND_SELF responses -func (peer *PeerInfo) cmdResponseBootstrapFindSelf(msg *protocol.MessageResponse, closest []protocol.PeerRecord) { - if len(closest) > bootstrapAcceptContacts { - closest = closest[:bootstrapAcceptContacts] - } - - for _, closePeer := range closest { - if peer.Backend.isReturnedPeerBadQuality(&closePeer) { - continue - } - - // If the peer is already in the peer list, no need to contact it again. - if peer.Backend.PeerlistLookup(closePeer.PublicKey) != nil { - continue - } - - // Check if the reported peer was recently contacted (in connection with the origin peer) for bootstrapping. This makes sure inactive peers are not contacted over and over again. - recent, blacklisted := isReturnedPeerRecent(&closePeer, peer.NodeID) - if blacklisted { - continue - } - - for _, address := range peerRecordToAddresses(&closePeer) { - // Check if the specific IP:Port was already contacted in the last 5-10 minutes. - if recent.IsAddressContacted(address) { - continue - } - - // Initiate contact. Once a response comes back, the peer will be actually added to the peer list. - peer.Backend.contactArbitraryPeer(closePeer.PublicKey, &net.UDPAddr{IP: address.IP, Port: int(address.Port)}, address.PortInternal, closePeer.Features&(1< 0) - } - } -} - -// ShouldSendFindSelf checks if FIND_SELF should be send -func ShouldSendFindSelf() bool { - // TODO - return true -} - -// isReturnedPeerBadQuality checks if the returned peer record is bad quality and should be discarded -func (backend *Backend) isReturnedPeerBadQuality(record *protocol.PeerRecord) bool { - isIPv4 := record.IPv4 != nil && !record.IPv4.IsUnspecified() - isIPv6 := record.IPv6 != nil && !record.IPv6.IsUnspecified() - - // At least one IP must be provided. - if !isIPv4 && !isIPv6 { - return true - } - - // Internal port must be provided. Otherwise the external port is likely not provided either, and checking the NAT and port forwarded status is not possible. - if isIPv4 && record.IPv4PortReportedInternal == 0 || isIPv6 && record.IPv6PortReportedInternal == 0 { - //fmt.Printf("IsReturnedPeerBadQuality port internal not available for target %s port %d, peer %s\n", record.IP.String(), record.Port, hex.EncodeToString(record.PublicKey.SerializeCompressed())) - return true - } - - // Must not be self. There is no point that a remote peer would return self - if record.PublicKey.IsEqual(backend.PeerPublicKey) { - //fmt.Printf("IsReturnedPeerBadQuality received self peer\n") - return true - } - - return false -} - -// peerRecordToAddresses returns the addresses in a usable way -func peerRecordToAddresses(record *protocol.PeerRecord) (addresses []*peerAddress) { - // IPv4 - ipv4Port := record.IPv4Port - if record.IPv4PortReportedExternal > 0 { // Use the external port if available - ipv4Port = record.IPv4PortReportedExternal - } - if record.IPv4 != nil && !record.IPv4.IsUnspecified() { - addresses = append(addresses, &peerAddress{IP: record.IPv4, Port: ipv4Port, PortInternal: record.IPv4PortReportedInternal}) - } - - // IPv6 - ipv6Port := record.IPv6Port - if record.IPv6PortReportedExternal > 0 { // Use the external port if available - ipv6Port = record.IPv6PortReportedExternal - } - if record.IPv6 != nil && !record.IPv6.IsUnspecified() { - addresses = append(addresses, &peerAddress{IP: record.IPv6, Port: ipv6Port, PortInternal: record.IPv6PortReportedInternal}) - } - - return addresses -} - -// ---- bootstrap cache of contacted peers to prevent flooding ---- - -// bootstrapRecentContact is the time in seconds when a peer will not be contacted again for bootstrapping. -// This prevents unnecessary flooding and prevents some attacks. Especially in small networks it will be the case that the same peer is returned multiple times. -const bootstrapRecentContact = 5 * 60 // 5-10 minutes - -type recentContactInfo struct { - added time.Time // When the peer was added to the list - addresses []*peerAddress // List of contacted addresses in IP:Port format - origin map[string]struct{} // List of node IDs who reported this contact - sync.RWMutex -} - -var ( - recentContacts map[[btcec.PubKeyBytesLenCompressed]byte]*recentContactInfo - recentContactsMutex sync.RWMutex -) - -func resetRecentContacts() { - for { - time.Sleep(bootstrapRecentContact * time.Second) - threshold := time.Now().Add(-bootstrapRecentContact * time.Second) - - recentContactsMutex.Lock() - - for key, recent := range recentContacts { - if recent.added.Before(threshold) { - delete(recentContacts, key) - } - } - - recentContactsMutex.Unlock() - } -} - -// isReturnedPeerRecent checks if the peer is blacklisted related to the origin peer due to recent contact. It will create a "recent contact" if none exists. -func isReturnedPeerRecent(record *protocol.PeerRecord, originNodeID []byte) (recent *recentContactInfo, blacklisted bool) { - key := publicKey2Compressed(record.PublicKey) - - recentContactsMutex.Lock() - defer recentContactsMutex.Unlock() - - if recent = recentContacts[key]; recent == nil { - recent = &recentContactInfo{added: time.Now(), origin: make(map[string]struct{})} - recent.origin[string(originNodeID)] = struct{}{} - - recentContacts[key] = recent - } else { - if _, blacklisted = recent.origin[string(originNodeID)]; !blacklisted { - recent.origin[string(originNodeID)] = struct{}{} - - // Here we could add an additional check: If number of recent.addresses (i.e. unique IP:Port tried) exceeds a threshold. - // However, this is currently not done due to risk of peer isolation. This could happen if enough peers would gang up to report false addresses for a given peer (such peer could still establish an inbound connection to this peer, however). - // Rather, those peers who report inactive peers should be blacklisted after a given threshold of garbage responses. - } - } - - return recent, blacklisted -} - -// IsAddressContacted checks if the address was contacted recently -func (recent *recentContactInfo) IsAddressContacted(address *peerAddress) bool { - recent.Lock() - defer recent.Unlock() - - for _, addressE := range recent.addresses { - if addressE.IP.Equal(address.IP) && addressE.Port == address.Port { - return true - } - } - - recent.addresses = append(recent.addresses, address) - - return false -} +/* +File Name: Bootstrap.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Strategy for sending our IPv6 Multicast and IPv4 Broadcast messages: +* During bootstrap: Immediately at the beginning, then every 10 seconds until there is at least 1 peer. +* Every 10 minutes during regular operation. +* Each time a network adapter / IP change is detected. + +*/ + +package core + +import ( + "encoding/hex" + "errors" + "net" + "strconv" + "sync" + "time" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" +) + +// rootPeer is a single root peer info +type rootPeer struct { + peer *PeerInfo // loaded PeerInfo + publicKey *btcec.PublicKey // Public key + addresses []*net.UDPAddr // IP:Port addresses + backend *Backend +} + +var rootPeers map[[btcec.PubKeyBytesLenCompressed]byte]*rootPeer + +// initSeedList loads the seed list from the config +// Note: This should be called before any network listening function so that incoming root peers are properly recognized. +func (backend *Backend) initSeedList() { + rootPeers = make(map[[btcec.PubKeyBytesLenCompressed]byte]*rootPeer) + recentContacts = make(map[[btcec.PubKeyBytesLenCompressed]byte]*recentContactInfo) + +loopSeedList: + for _, seed := range backend.Config.SeedList { + peer := &rootPeer{backend: backend} + + // parse the Public Key + publicKeyB, err := hex.DecodeString(seed.PublicKey) + if err != nil { + backend.LogError("initSeedList", "public key '%s': %v\n", seed.PublicKey, err.Error()) + continue + } + + if peer.publicKey, err = btcec.ParsePubKey(publicKeyB, btcec.S256()); err != nil { + backend.LogError("initSeedList", "public key '%s': %v\n", seed.PublicKey, err.Error()) + continue + } + + if peer.publicKey.IsEqual(backend.PeerPublicKey) { // skip if self + continue + } + + // parse all IP addresses + for _, addressA := range seed.Address { + address, err := parseAddress(addressA) + if err != nil { + backend.LogError("initSeedList", "public key '%s' address '%s': %v\n", seed.PublicKey, addressA, err.Error()) + continue loopSeedList + } + + peer.addresses = append(peer.addresses, address) + } + + rootPeers[publicKey2Compressed(peer.publicKey)] = peer + } +} + +// parseAddress parses an input peer address in the form "IP:Port". +func parseAddress(Address string) (remote *net.UDPAddr, err error) { + host, portA, err := net.SplitHostPort(Address) + if err != nil { + return nil, err + } + + portI, err := strconv.Atoi(portA) + if err != nil { + return nil, err + } else if portI <= 0 || portI > 65535 { + return nil, errors.New("invalid port number") + } + + ip := net.ParseIP(host) + if ip == nil { + return nil, errors.New("invalid input IP") + } + + return &net.UDPAddr{IP: ip, Port: portI}, err +} + +// contact tries to contact the root peer on all networks +func (peer *rootPeer) contact() { + // If already in peer list, no need to contact. + if peer.backend.PeerlistLookup(peer.publicKey) != nil { + return + } + + for _, address := range peer.addresses { + // Port internal is always set to 0 for root peers. It disables NAT detection and will not send out a Traverse message. + peer.backend.contactArbitraryPeer(peer.publicKey, address, 0, false) + } +} + +// bootstrap connects to the initial set of peers. +func (backend *Backend) bootstrap() { + go resetRecentContacts() + + if len(rootPeers) == 0 { + backend.LogError("bootstrap", "warning: Empty list of root peers. Connectivity relies on local peer discovery and incoming connections.\n") + return + } + + contactRootPeers := func() { + for _, peer := range rootPeers { + if peer.peer == nil { + peer.contact() + } + } + } + + countConnectedRootPeers := func() (connectedCount, total int) { + for _, peer := range rootPeers { + if peer.peer != nil { + connectedCount++ + } else if peer.peer = peer.backend.PeerlistLookup(peer.publicKey); peer.peer != nil { + connectedCount++ + } + } + return connectedCount, len(rootPeers) + } + + // initial contact to all root peer + contactRootPeers() + + // Phase 1: First 10 minutes. Try every 7 seconds to connect to all root peers until at least 2 peers connected. + for n := 0; n < 10*60/7; n++ { + time.Sleep(time.Second * 7) + + if connected, total := countConnectedRootPeers(); connected == total || connected >= 2 { + return + } + + contactRootPeers() + } + + // Phase 2: After that (if not 2 peers), try every 5 minutes to connect to remaining root peers for a maximum of 1 hour. + for n := 0; n < 1*60/5; n++ { + time.Sleep(time.Minute * 5) + + contactRootPeers() + + if connected, total := countConnectedRootPeers(); connected == total || connected >= 2 { + return + } + } + + backend.LogError("bootstrap", "unable to connect to at least 2 root peers, aborting\n") +} + +func (nets *Networks) autoMulticastBroadcast() { + sendMulticastBroadcast := func() { + nets.RLock() + defer nets.RUnlock() + + for _, network := range nets.networks6 { + if err := network.MulticastIPv6Send(); err != nil { + nets.backend.LogError("autoMulticastBroadcast", "multicast from network address '%s': %v\n", network.address.IP.String(), err.Error()) + } + } + + for _, network := range nets.networks4 { + if err := network.BroadcastIPv4Send(); err != nil { + nets.backend.LogError("autoMulticastBroadcast", "broadcast from network address '%s': %v\n", network.address.IP.String(), err.Error()) + } + } + } + + // Send out multicast/broadcast immediately. + sendMulticastBroadcast() + + // Phase 1: Resend every 10 seconds until at least 1 peer in the peer list. + for { + time.Sleep(time.Second * 10) + + if nets.backend.PeerlistCount() >= 1 { + break + } + + sendMulticastBroadcast() + } + + // Phase 2: Every 10 minutes. + for { + time.Sleep(time.Minute * 10) + sendMulticastBroadcast() + } +} + +// contactArbitraryPeer contacts a new arbitrary peer for the first time. +func (backend *Backend) contactArbitraryPeer(publicKey *btcec.PublicKey, address *net.UDPAddr, receiverPortInternal uint16, receiverFirewall bool) (contacted bool) { + findSelf := ShouldSendFindSelf() + _, blockchainHeight, blockchainVersion := backend.UserBlockchain.Header() + packets := protocol.EncodeAnnouncement(true, findSelf, nil, nil, nil, backend.FeatureSupport(), blockchainHeight, blockchainVersion, backend.userAgent) + if len(packets) == 0 { + return false + } + raw := &protocol.PacketRaw{Command: protocol.CommandAnnouncement, Payload: packets[0]} + + backend.Filters.MessageOutAnnouncement(publicKey, nil, raw, findSelf, nil, nil, nil) + + backend.networks.sendAllNetworks(publicKey, raw, address, receiverPortInternal, receiverFirewall, nil, &bootstrapFindSelf{}) + + return true +} + +// bootstrapFindSelf is a dummy structure assigned to sequences when sending the Announcement message. +// When receiving the Response message, it will know that it was a legitimate bootstrap request. +type bootstrapFindSelf struct { +} + +// bootstrapAcceptContacts is the maximum count of contacts considered. It limits the impact of fake peers. +const bootstrapAcceptContacts = 5 + +// cmdResponseBootstrapFindSelf processes FIND_SELF responses +func (peer *PeerInfo) cmdResponseBootstrapFindSelf(msg *protocol.MessageResponse, closest []protocol.PeerRecord) { + if len(closest) > bootstrapAcceptContacts { + closest = closest[:bootstrapAcceptContacts] + } + + for _, closePeer := range closest { + if peer.Backend.isReturnedPeerBadQuality(&closePeer) { + continue + } + + // If the peer is already in the peer list, no need to contact it again. + if peer.Backend.PeerlistLookup(closePeer.PublicKey) != nil { + continue + } + + // Check if the reported peer was recently contacted (in connection with the origin peer) for bootstrapping. This makes sure inactive peers are not contacted over and over again. + recent, blacklisted := isReturnedPeerRecent(&closePeer, peer.NodeID) + if blacklisted { + continue + } + + for _, address := range peerRecordToAddresses(&closePeer) { + // Check if the specific IP:Port was already contacted in the last 5-10 minutes. + if recent.IsAddressContacted(address) { + continue + } + + // Initiate contact. Once a response comes back, the peer will be actually added to the peer list. + peer.Backend.contactArbitraryPeer(closePeer.PublicKey, &net.UDPAddr{IP: address.IP, Port: int(address.Port)}, address.PortInternal, closePeer.Features&(1< 0) + } + } +} + +// ShouldSendFindSelf checks if FIND_SELF should be send +func ShouldSendFindSelf() bool { + // TODO + return true +} + +// isReturnedPeerBadQuality checks if the returned peer record is bad quality and should be discarded +func (backend *Backend) isReturnedPeerBadQuality(record *protocol.PeerRecord) bool { + isIPv4 := record.IPv4 != nil && !record.IPv4.IsUnspecified() + isIPv6 := record.IPv6 != nil && !record.IPv6.IsUnspecified() + + // At least one IP must be provided. + if !isIPv4 && !isIPv6 { + return true + } + + // Internal port must be provided. Otherwise the external port is likely not provided either, and checking the NAT and port forwarded status is not possible. + if isIPv4 && record.IPv4PortReportedInternal == 0 || isIPv6 && record.IPv6PortReportedInternal == 0 { + //fmt.Printf("IsReturnedPeerBadQuality port internal not available for target %s port %d, peer %s\n", record.IP.String(), record.Port, hex.EncodeToString(record.PublicKey.SerializeCompressed())) + return true + } + + // Must not be self. There is no point that a remote peer would return self + if record.PublicKey.IsEqual(backend.PeerPublicKey) { + //fmt.Printf("IsReturnedPeerBadQuality received self peer\n") + return true + } + + return false +} + +// peerRecordToAddresses returns the addresses in a usable way +func peerRecordToAddresses(record *protocol.PeerRecord) (addresses []*peerAddress) { + // IPv4 + ipv4Port := record.IPv4Port + if record.IPv4PortReportedExternal > 0 { // Use the external port if available + ipv4Port = record.IPv4PortReportedExternal + } + if record.IPv4 != nil && !record.IPv4.IsUnspecified() { + addresses = append(addresses, &peerAddress{IP: record.IPv4, Port: ipv4Port, PortInternal: record.IPv4PortReportedInternal}) + } + + // IPv6 + ipv6Port := record.IPv6Port + if record.IPv6PortReportedExternal > 0 { // Use the external port if available + ipv6Port = record.IPv6PortReportedExternal + } + if record.IPv6 != nil && !record.IPv6.IsUnspecified() { + addresses = append(addresses, &peerAddress{IP: record.IPv6, Port: ipv6Port, PortInternal: record.IPv6PortReportedInternal}) + } + + return addresses +} + +// ---- bootstrap cache of contacted peers to prevent flooding ---- + +// bootstrapRecentContact is the time in seconds when a peer will not be contacted again for bootstrapping. +// This prevents unnecessary flooding and prevents some attacks. Especially in small networks it will be the case that the same peer is returned multiple times. +const bootstrapRecentContact = 5 * 60 // 5-10 minutes + +type recentContactInfo struct { + added time.Time // When the peer was added to the list + addresses []*peerAddress // List of contacted addresses in IP:Port format + origin map[string]struct{} // List of node IDs who reported this contact + sync.RWMutex +} + +var ( + recentContacts map[[btcec.PubKeyBytesLenCompressed]byte]*recentContactInfo + recentContactsMutex sync.RWMutex +) + +func resetRecentContacts() { + for { + time.Sleep(bootstrapRecentContact * time.Second) + threshold := time.Now().Add(-bootstrapRecentContact * time.Second) + + recentContactsMutex.Lock() + + for key, recent := range recentContacts { + if recent.added.Before(threshold) { + delete(recentContacts, key) + } + } + + recentContactsMutex.Unlock() + } +} + +// isReturnedPeerRecent checks if the peer is blacklisted related to the origin peer due to recent contact. It will create a "recent contact" if none exists. +func isReturnedPeerRecent(record *protocol.PeerRecord, originNodeID []byte) (recent *recentContactInfo, blacklisted bool) { + key := publicKey2Compressed(record.PublicKey) + + recentContactsMutex.Lock() + defer recentContactsMutex.Unlock() + + if recent = recentContacts[key]; recent == nil { + recent = &recentContactInfo{added: time.Now(), origin: make(map[string]struct{})} + recent.origin[string(originNodeID)] = struct{}{} + + recentContacts[key] = recent + } else { + if _, blacklisted = recent.origin[string(originNodeID)]; !blacklisted { + recent.origin[string(originNodeID)] = struct{}{} + + // Here we could add an additional check: If number of recent.addresses (i.e. unique IP:Port tried) exceeds a threshold. + // However, this is currently not done due to risk of peer isolation. This could happen if enough peers would gang up to report false addresses for a given peer (such peer could still establish an inbound connection to this peer, however). + // Rather, those peers who report inactive peers should be blacklisted after a given threshold of garbage responses. + } + } + + return recent, blacklisted +} + +// IsAddressContacted checks if the address was contacted recently +func (recent *recentContactInfo) IsAddressContacted(address *peerAddress) bool { + recent.Lock() + defer recent.Unlock() + + for _, addressE := range recent.addresses { + if addressE.IP.Equal(address.IP) && addressE.Port == address.Port { + return true + } + } + + recent.addresses = append(recent.addresses, address) + + return false +} diff --git a/Command Traverse.go b/Command Traverse.go index 6c4a18c..afc68d7 100644 --- a/Command Traverse.go +++ b/Command Traverse.go @@ -1,134 +1,134 @@ -/* -File Name: Command Traverse.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "math/rand" - "net" - "time" - - "github.com/PeernetOfficial/core/protocol" -) - -// cmdTraverseForward handles an incoming traverse message that should be forwarded to another peer -func (peer *PeerInfo) cmdTraverseForward(msg *protocol.MessageTraverse) { - // Verify the signature. This makes sure that a fowarded message cannot be replayed by others. - if !msg.SignerPublicKey.IsEqual(peer.PublicKey) || !msg.SignerPublicKey.IsEqual(msg.SenderPublicKey) { - return - } - - // Check expiration - if msg.Expires.Before(time.Now()) { - return - } - - // Check if the target peer is known in the peer list. If not, nothing will be done. - // The original sender should only send the Traverse message as answer to a Response that contains a reported peer that is behind a NAT. - // In that case the target peer should be still in this peers' peer list. - peerTarget := peer.Backend.PeerlistLookup(msg.TargetPeer) - if peerTarget == nil { - return - } - - // Get the right IP:Port of the original sender to share to the target peer. - allowIPv4 := peerTarget.Features&(1< 0 - allowIPv6 := peerTarget.Features&(1< 0 - connectionIPv4 := peer.GetConnection2Share(false, allowIPv4, false) - connectionIPv6 := peer.GetConnection2Share(false, false, allowIPv6) - - if connectionIPv4 == nil && connectionIPv6 == nil { - return - } - - // get the individual fields - var IPv4, IPv6 net.IP - var PortIPv4, PortIPv4ReportedExternal, PortIPv6, PortIPv6ReportedExternal uint16 - if connectionIPv4 != nil { - IPv4 = connectionIPv4.Address.IP - PortIPv4 = uint16(connectionIPv4.Address.Port) - PortIPv4ReportedExternal = connectionIPv4.PortExternal - } - if connectionIPv6 != nil { - IPv6 = connectionIPv6.Address.IP - PortIPv6 = uint16(connectionIPv6.Address.Port) - PortIPv6ReportedExternal = connectionIPv6.PortExternal - } - - if err := protocol.EncodeTraverseSetAddress(msg.Payload, IPv4, PortIPv4, PortIPv4ReportedExternal, IPv6, PortIPv6, PortIPv6ReportedExternal); err != nil { - return - } - - peerTarget.send(&protocol.PacketRaw{Command: protocol.CommandTraverse, Payload: msg.Payload}) -} - -func (peer *PeerInfo) cmdTraverseReceive(msg *protocol.MessageTraverse) { - if msg.Expires.Before(time.Now()) { - return - } - - // Already an active connection established? The relayed message should not be needed in this case. - // This could be changed in the future if it turns out that there are 1-way connection issues. - if peerTarget := peer.Backend.PeerlistLookup(msg.SignerPublicKey); peerTarget != nil { - return - } - - // parse IP addresses of the original sender - var addresses []*peerAddress - - if !msg.IPv4.IsUnspecified() { - port := msg.PortIPv4 - if msg.PortIPv4ReportedExternal > 0 { - port = msg.PortIPv4ReportedExternal - } - addresses = append(addresses, &peerAddress{IP: msg.IPv4, Port: port, PortInternal: 0}) - } - if !msg.IPv6.IsUnspecified() { - port := msg.PortIPv6 - if msg.PortIPv6ReportedExternal > 0 { - port = msg.PortIPv6ReportedExternal - } - addresses = append(addresses, &peerAddress{IP: msg.IPv6, Port: port, PortInternal: 0}) - } - if len(addresses) == 0 { - return - } - - // ---- fork packetWorker to decode and validate embedded packet --- - // Due to missing connection and other embedded details in the message (such as ports), the packet is not just simply queued to rawPacketsIncoming. - decoded, senderPublicKey, err := protocol.PacketDecrypt(msg.EmbeddedPacketRaw, peer.Backend.PeerPublicKey) - if err != nil { - return - } - if !senderPublicKey.IsEqual(msg.SignerPublicKey) { - return - } else if senderPublicKey.IsEqual(peer.Backend.PeerPublicKey) { - return - } else if decoded.Protocol != 0 { - return - } else if decoded.Command != protocol.CommandAnnouncement { - return - } - - // process the packet and create a virtual peer - raw := &protocol.MessageRaw{SenderPublicKey: senderPublicKey, PacketRaw: *decoded} - peerV := &PeerInfo{Backend: peer.Backend, PublicKey: senderPublicKey, connectionActive: nil, connectionLatest: nil, NodeID: protocol.PublicKey2NodeID(senderPublicKey), messageSequence: rand.Uint32(), isVirtual: true, targetAddresses: addresses} - - // process it! - switch decoded.Command { - case protocol.CommandAnnouncement: // Announce - if announce, _ := protocol.DecodeAnnouncement(raw); announce != nil { - if len(announce.UserAgent) > 0 { - peerV.UserAgent = announce.UserAgent - } - peerV.Features = announce.Features - - peerV.cmdAnouncement(announce, nil) - } - - default: - } -} +/* +File Name: Command Traverse.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "math/rand" + "net" + "time" + + "github.com/PeernetOfficial/core/protocol" +) + +// cmdTraverseForward handles an incoming traverse message that should be forwarded to another peer +func (peer *PeerInfo) cmdTraverseForward(msg *protocol.MessageTraverse) { + // Verify the signature. This makes sure that a fowarded message cannot be replayed by others. + if !msg.SignerPublicKey.IsEqual(peer.PublicKey) || !msg.SignerPublicKey.IsEqual(msg.SenderPublicKey) { + return + } + + // Check expiration + if msg.Expires.Before(time.Now()) { + return + } + + // Check if the target peer is known in the peer list. If not, nothing will be done. + // The original sender should only send the Traverse message as answer to a Response that contains a reported peer that is behind a NAT. + // In that case the target peer should be still in this peers' peer list. + peerTarget := peer.Backend.PeerlistLookup(msg.TargetPeer) + if peerTarget == nil { + return + } + + // Get the right IP:Port of the original sender to share to the target peer. + allowIPv4 := peerTarget.Features&(1< 0 + allowIPv6 := peerTarget.Features&(1< 0 + connectionIPv4 := peer.GetConnection2Share(false, allowIPv4, false) + connectionIPv6 := peer.GetConnection2Share(false, false, allowIPv6) + + if connectionIPv4 == nil && connectionIPv6 == nil { + return + } + + // get the individual fields + var IPv4, IPv6 net.IP + var PortIPv4, PortIPv4ReportedExternal, PortIPv6, PortIPv6ReportedExternal uint16 + if connectionIPv4 != nil { + IPv4 = connectionIPv4.Address.IP + PortIPv4 = uint16(connectionIPv4.Address.Port) + PortIPv4ReportedExternal = connectionIPv4.PortExternal + } + if connectionIPv6 != nil { + IPv6 = connectionIPv6.Address.IP + PortIPv6 = uint16(connectionIPv6.Address.Port) + PortIPv6ReportedExternal = connectionIPv6.PortExternal + } + + if err := protocol.EncodeTraverseSetAddress(msg.Payload, IPv4, PortIPv4, PortIPv4ReportedExternal, IPv6, PortIPv6, PortIPv6ReportedExternal); err != nil { + return + } + + peerTarget.send(&protocol.PacketRaw{Command: protocol.CommandTraverse, Payload: msg.Payload}) +} + +func (peer *PeerInfo) cmdTraverseReceive(msg *protocol.MessageTraverse) { + if msg.Expires.Before(time.Now()) { + return + } + + // Already an active connection established? The relayed message should not be needed in this case. + // This could be changed in the future if it turns out that there are 1-way connection issues. + if peerTarget := peer.Backend.PeerlistLookup(msg.SignerPublicKey); peerTarget != nil { + return + } + + // parse IP addresses of the original sender + var addresses []*peerAddress + + if !msg.IPv4.IsUnspecified() { + port := msg.PortIPv4 + if msg.PortIPv4ReportedExternal > 0 { + port = msg.PortIPv4ReportedExternal + } + addresses = append(addresses, &peerAddress{IP: msg.IPv4, Port: port, PortInternal: 0}) + } + if !msg.IPv6.IsUnspecified() { + port := msg.PortIPv6 + if msg.PortIPv6ReportedExternal > 0 { + port = msg.PortIPv6ReportedExternal + } + addresses = append(addresses, &peerAddress{IP: msg.IPv6, Port: port, PortInternal: 0}) + } + if len(addresses) == 0 { + return + } + + // ---- fork packetWorker to decode and validate embedded packet --- + // Due to missing connection and other embedded details in the message (such as ports), the packet is not just simply queued to rawPacketsIncoming. + decoded, senderPublicKey, err := protocol.PacketDecrypt(msg.EmbeddedPacketRaw, peer.Backend.PeerPublicKey) + if err != nil { + return + } + if !senderPublicKey.IsEqual(msg.SignerPublicKey) { + return + } else if senderPublicKey.IsEqual(peer.Backend.PeerPublicKey) { + return + } else if decoded.Protocol != 0 { + return + } else if decoded.Command != protocol.CommandAnnouncement { + return + } + + // process the packet and create a virtual peer + raw := &protocol.MessageRaw{SenderPublicKey: senderPublicKey, PacketRaw: *decoded} + peerV := &PeerInfo{Backend: peer.Backend, PublicKey: senderPublicKey, connectionActive: nil, connectionLatest: nil, NodeID: protocol.PublicKey2NodeID(senderPublicKey), messageSequence: rand.Uint32(), isVirtual: true, targetAddresses: addresses} + + // process it! + switch decoded.Command { + case protocol.CommandAnnouncement: // Announce + if announce, _ := protocol.DecodeAnnouncement(raw); announce != nil { + if len(announce.UserAgent) > 0 { + peerV.UserAgent = announce.UserAgent + } + peerV.Features = announce.Features + + peerV.cmdAnouncement(announce, nil) + } + + default: + } +} diff --git a/Commands.go b/Commands.go index 0e8846a..6edb990 100644 --- a/Commands.go +++ b/Commands.go @@ -1,330 +1,330 @@ -/* -File Name: Commands.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "bytes" - "encoding/hex" - "fmt" - - "github.com/PeernetOfficial/core/dht" - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/warehouse" - "github.com/google/uuid" -) - -// respondClosesContactsCount is the number of closest contact to respond. -// Each peer record will take 70 bytes. Overhead is 77 + 20 payload header + UA length + 6 + 34 = 137 bytes without UA. -// It makes sense to stay below 508 bytes (no fragmentation). Reporting back 5 contacts for FIND_SELF requests should do the magic. -const respondClosesContactsCount = 5 - -// cmdAnouncement handles an incoming announcement. Connection may be nil for traverse relayed messages. -func (peer *PeerInfo) cmdAnouncement(msg *protocol.MessageAnnouncement, connection *Connection) { - // Filter function to only share peers that are "connectable" to the remote one. It checks IPv4, IPv6, and local connection. - filterFunc := func(allowLocal, allowIPv4, allowIPv6 bool) dht.NodeFilterFunc { - return func(node *dht.Node) (accept bool) { - return node.Info.(*PeerInfo).IsConnectable(allowLocal, allowIPv4, allowIPv6) - } - } - - allowIPv4 := msg.Features&(1< 0 - allowIPv6 := msg.Features&(1< 0 - - var hash2Peers []protocol.Hash2Peer - var hashesNotFound [][]byte - var filesEmbed []protocol.EmbeddedFileData - - // FIND_SELF: Requesting peers close to the sender? - if msg.Actions&(1< 0 { - peer.Backend.Filters.IncomingRequest(peer, protocol.ActionFindSelf, peer.NodeID, nil) - - selfD := protocol.Hash2Peer{ID: protocol.KeyHash{Hash: peer.NodeID}} - - // do not respond the caller's own peer (add to ignore list) - for _, node := range peer.Backend.nodesDHT.GetClosestContacts(respondClosesContactsCount, peer.NodeID, filterFunc(connection.IsLocal(), allowIPv4, allowIPv6), peer.NodeID) { - if info := node.Info.(*PeerInfo).peer2Record(connection.IsLocal(), allowIPv4, allowIPv6); info != nil { - selfD.Closest = append(selfD.Closest, *info) - } - } - - if len(selfD.Closest) > 0 { - hash2Peers = append(hash2Peers, selfD) - } else { - hashesNotFound = append(hashesNotFound, peer.NodeID) - } - } - - // FIND_PEER: Find a different peer? - if msg.Actions&(1< 0 && len(msg.FindPeerKeys) > 0 { - for _, findPeer := range msg.FindPeerKeys { - peer.Backend.Filters.IncomingRequest(peer, protocol.ActionFindPeer, findPeer.Hash, nil) - - details := protocol.Hash2Peer{ID: findPeer} - - // Same as before, put self as ignoredNodes. - for _, node := range peer.Backend.nodesDHT.GetClosestContacts(respondClosesContactsCount, findPeer.Hash, filterFunc(connection.IsLocal(), allowIPv4, allowIPv6), peer.NodeID) { - if info := node.Info.(*PeerInfo).peer2Record(connection.IsLocal(), allowIPv4, allowIPv6); info != nil { - details.Closest = append(details.Closest, *info) - } - } - - if len(details.Closest) > 0 { - hash2Peers = append(hash2Peers, details) - } else { - hashesNotFound = append(hashesNotFound, findPeer.Hash) - } - } - } - - // Find a value? - if msg.Actions&(1< 0 { - for _, findHash := range msg.FindDataKeys { - peer.Backend.Filters.IncomingRequest(peer, protocol.ActionFindValue, findHash.Hash, nil) - - stored, data := peer.announcementGetData(findHash.Hash) - if stored && len(data) > 0 { - filesEmbed = append(filesEmbed, protocol.EmbeddedFileData{ID: findHash, Data: data}) - } else if stored { - selfRecord := peer.Backend.selfPeerRecord() - hash2Peers = append(hash2Peers, protocol.Hash2Peer{ID: findHash, Storing: []protocol.PeerRecord{selfRecord}}) - } else { - hashesNotFound = append(hashesNotFound, findHash.Hash) - } - } - } - - // Information about files stored by the sender? - if msg.Actions&(1< 0 && len(msg.InfoStoreFiles) > 0 { - for n := range msg.InfoStoreFiles { - peer.Backend.Filters.IncomingRequest(peer, protocol.ActionInfoStore, msg.InfoStoreFiles[n].ID.Hash, &msg.InfoStoreFiles[n]) - } - - peer.announcementStore(msg.InfoStoreFiles) - } - - sendUA := msg.UserAgent != "" // Send user agent if one was provided. Per protocol the first announcement message must have the User Agent set. - peer.sendResponse(msg.Sequence, sendUA, hash2Peers, filesEmbed, hashesNotFound) -} - -func (peer *PeerInfo) peer2Record(allowLocal, allowIPv4, allowIPv6 bool) (result *protocol.PeerRecord) { - connectionIPv4 := peer.GetConnection2Share(allowLocal, allowIPv4, false) - connectionIPv6 := peer.GetConnection2Share(allowLocal, false, allowIPv6) - if connectionIPv4 == nil && connectionIPv6 == nil { - return nil - } - - result = &protocol.PeerRecord{ - PublicKey: peer.PublicKey, - NodeID: peer.NodeID, - Features: peer.Features, - } - - if connectionIPv4 != nil { - result.IPv4 = connectionIPv4.Address.IP - result.IPv4Port = uint16(connectionIPv4.Address.Port) - result.IPv4PortReportedInternal = connectionIPv4.PortInternal - result.IPv4PortReportedExternal = connectionIPv4.PortExternal - } - - if connectionIPv6 != nil { - result.IPv6 = connectionIPv6.Address.IP - result.IPv6Port = uint16(connectionIPv6.Address.Port) - result.IPv6PortReportedInternal = connectionIPv6.PortInternal - result.IPv6PortReportedExternal = connectionIPv6.PortExternal - } - - return result -} - -// cmdResponse handles the response to the announcement -func (peer *PeerInfo) cmdResponse(msg *protocol.MessageResponse, connection *Connection) { - // The sequence data is used to correlate this response with the announcement. - if msg.SequenceInfo == nil || msg.SequenceInfo.Data == nil { - // If there is no sequence data but there were results returned, it means we received unsolicited response data. It will be rejected. - if len(msg.HashesNotFound) > 0 || len(msg.Hash2Peers) > 0 || len(msg.FilesEmbed) > 0 { - peer.Backend.LogError("cmdResponse", "unsolicited response data received from %s\n", connection.Address.String()) - } - - return - } - - // bootstrap FIND_SELF? - if _, ok := msg.SequenceInfo.Data.(*bootstrapFindSelf); ok { - for _, hash2Peer := range msg.Hash2Peers { - // Make sure no garbage is returned. The key must be self and only Closest is expected. - if !bytes.Equal(hash2Peer.ID.Hash, peer.Backend.nodeID) || len(hash2Peer.Closest) == 0 { - peer.Backend.LogError("cmdResponse", "incoming response to bootstrap FIND_SELF contains invalid data from %s\n", connection.Address.String()) - return - } - - peer.cmdResponseBootstrapFindSelf(msg, hash2Peer.Closest) - } - - return - } - - // Response to an information request? - if _, ok := msg.SequenceInfo.Data.(*dht.InformationRequest); ok { - // Future: Once multiple information requests are pooled (multiplexed) into one or multiple Announcement sequences (messages), the responses need to be de-pooled. - // A simple multiplex structure linked via the sequence containing a map (hash 2 IR) could simplify this. - info := msg.SequenceInfo.Data.(*dht.InformationRequest) - - if len(msg.HashesNotFound) > 0 { - info.Done() - } - - for _, hash2Peer := range msg.Hash2Peers { - info.QueueResult(&dht.NodeMessage{SenderID: peer.NodeID, Closest: peer.records2Nodes(hash2Peer.Closest), Storing: peer.records2Nodes(hash2Peer.Storing)}) - - if hash2Peer.IsLast { - info.Done() - } - } - - for _, file := range msg.FilesEmbed { - info.QueueResult(&dht.NodeMessage{SenderID: peer.NodeID, Data: file.Data}) - - info.Done() - info.Terminate() // file was found, terminate the request. - } - } -} - -// cmdPing handles an incoming ping message -func (peer *PeerInfo) cmdPing(msg *protocol.MessageRaw, connection *Connection) { - // If PortInternal is 0, it means no incoming announcement or response message was received on that connection. - // This means the ping is unexpected. In that case for security reasons the remote peer is not asked for FIND_SELF. - if connection.PortInternal == 0 { - peer.sendAnnouncement(true, false, nil, nil, nil, nil) - return - } - - raw := &protocol.PacketRaw{Command: protocol.CommandPong, Sequence: msg.Sequence} - - peer.Backend.Filters.MessageOutPong(peer, raw) - - peer.send(raw) -} - -// cmdPong handles an incoming pong message -func (peer *PeerInfo) cmdPong(msg *protocol.MessageRaw, connection *Connection) { -} - -// cmdChat handles a chat message [debug] -func (peer *PeerInfo) cmdChat(msg *protocol.MessageRaw, connection *Connection) { - fmt.Fprintf(peer.Backend.Stdout, "Chat from %s '%s': %s\n", hex.EncodeToString(peer.PublicKey.SerializeCompressed()), connection.Address.String(), string(msg.PacketRaw.Payload)) -} - -// cmdLocalDiscovery handles an incoming announcement via local discovery -func (peer *PeerInfo) cmdLocalDiscovery(msg *protocol.MessageAnnouncement, connection *Connection) { - // 21.04.2021 update: Local peer discovery from public IPv4s is possible in datacenter situations. Keep it enabled for now. - // only accept local discovery message from private IPs for IPv4 - // IPv6 DHCP routers typically assign public IPv6s and they can join multicast in the local network. - //if connection.IsIPv4() && !connection.IsLocal() { - // LogError("cmdLocalDiscovery", "message received from non-local IP %s peer ID %s\n", connection.Address.String(), hex.EncodeToString(msg.SenderPublicKey.SerializeCompressed())) - // return - //} - - peer.sendAnnouncement(true, ShouldSendFindSelf(), nil, nil, nil, &bootstrapFindSelf{}) -} - -// SendChatAll sends a text message to all peers -func (backend *Backend) SendChatAll(text string) { - for _, peer := range backend.PeerlistGet() { - peer.Chat(text) - } -} - -// cmdTransfer handles an incoming transfer message -func (peer *PeerInfo) cmdTransfer(msg *protocol.MessageTransfer, connection *Connection) { - // Only UDT protocol is currently supported for file transfer. - if msg.TransferProtocol != protocol.TransferProtocolUDT { - return - } - - switch msg.Control { - case protocol.TransferControlRequestStart: - // First check if the file available in the warehouse. - _, fileSize, status, _ := peer.Backend.UserWarehouse.FileExists(msg.Hash) - if status != warehouse.StatusOK { - // File not available. - peer.sendTransfer(nil, protocol.TransferControlNotAvailable, msg.TransferProtocol, msg.Hash, 0, 0, msg.Sequence, uuid.UUID{}, false) - return - } else if msg.Limit > 0 && fileSize < msg.Offset+msg.Limit { - // If the read limit is out of bounds, this request is considered invalid and silently discarded. - return - } - - // Create a local UDT client to connect to the remote UDT server and serve the file! - go peer.startFileTransferUDT(msg.Hash, fileSize, msg.Offset, msg.Limit, msg.Sequence, msg.TransferID, msg.TransferProtocol) - - case protocol.TransferControlActive: - if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { - go v.receiveData(msg.Data) - return - } - - case protocol.TransferControlNotAvailable: - if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { - v.Terminate(404) - return - } - - case protocol.TransferControlTerminate: - if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { - v.Terminate(2) - return - } - - } -} - -// cmdGetBlock handles an incoming block message -func (peer *PeerInfo) cmdGetBlock(msg *protocol.MessageGetBlock, connection *Connection) { - switch msg.Control { - case protocol.GetBlockControlRequestStart: - // Currently only support the local blockchain. - if !msg.BlockchainPublicKey.IsEqual(peer.Backend.PeerPublicKey) { - peer.sendGetBlock(nil, protocol.GetBlockControlNotAvailable, msg.BlockchainPublicKey, 0, 0, nil, msg.Sequence, uuid.UUID{}, false) - return - } else if _, height, _ := peer.Backend.UserBlockchain.Header(); height == 0 { - peer.sendGetBlock(nil, protocol.GetBlockControlEmpty, msg.BlockchainPublicKey, 0, 0, nil, msg.Sequence, uuid.UUID{}, false) - return - } else if msg.LimitBlockCount == 0 { - peer.sendGetBlock(nil, protocol.GetBlockControlTerminate, msg.BlockchainPublicKey, 0, 0, nil, msg.Sequence, uuid.UUID{}, false) - return - } - - // Create a local UDT client to connect to the remote UDT server and serve the blocks! - go peer.startBlockTransfer(msg.BlockchainPublicKey, msg.LimitBlockCount, msg.MaxBlockSize, msg.TargetBlocks, msg.Sequence, msg.TransferID) - - case protocol.GetBlockControlActive: - if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { - go v.receiveData(msg.Data) - return - } - - case protocol.GetBlockControlNotAvailable: - if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { - v.Terminate(404) - return - } - - case protocol.GetBlockControlEmpty: - if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { - v.Terminate(410) - return - } - - case protocol.GetBlockControlTerminate: - if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { - v.Terminate(2) - return - } - - } -} +/* +File Name: Commands.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "bytes" + "encoding/hex" + "fmt" + + "github.com/PeernetOfficial/core/dht" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/warehouse" + "github.com/google/uuid" +) + +// respondClosesContactsCount is the number of closest contact to respond. +// Each peer record will take 70 bytes. Overhead is 77 + 20 payload header + UA length + 6 + 34 = 137 bytes without UA. +// It makes sense to stay below 508 bytes (no fragmentation). Reporting back 5 contacts for FIND_SELF requests should do the magic. +const respondClosesContactsCount = 5 + +// cmdAnouncement handles an incoming announcement. Connection may be nil for traverse relayed messages. +func (peer *PeerInfo) cmdAnouncement(msg *protocol.MessageAnnouncement, connection *Connection) { + // Filter function to only share peers that are "connectable" to the remote one. It checks IPv4, IPv6, and local connection. + filterFunc := func(allowLocal, allowIPv4, allowIPv6 bool) dht.NodeFilterFunc { + return func(node *dht.Node) (accept bool) { + return node.Info.(*PeerInfo).IsConnectable(allowLocal, allowIPv4, allowIPv6) + } + } + + allowIPv4 := msg.Features&(1< 0 + allowIPv6 := msg.Features&(1< 0 + + var hash2Peers []protocol.Hash2Peer + var hashesNotFound [][]byte + var filesEmbed []protocol.EmbeddedFileData + + // FIND_SELF: Requesting peers close to the sender? + if msg.Actions&(1< 0 { + peer.Backend.Filters.IncomingRequest(peer, protocol.ActionFindSelf, peer.NodeID, nil) + + selfD := protocol.Hash2Peer{ID: protocol.KeyHash{Hash: peer.NodeID}} + + // do not respond the caller's own peer (add to ignore list) + for _, node := range peer.Backend.nodesDHT.GetClosestContacts(respondClosesContactsCount, peer.NodeID, filterFunc(connection.IsLocal(), allowIPv4, allowIPv6), peer.NodeID) { + if info := node.Info.(*PeerInfo).peer2Record(connection.IsLocal(), allowIPv4, allowIPv6); info != nil { + selfD.Closest = append(selfD.Closest, *info) + } + } + + if len(selfD.Closest) > 0 { + hash2Peers = append(hash2Peers, selfD) + } else { + hashesNotFound = append(hashesNotFound, peer.NodeID) + } + } + + // FIND_PEER: Find a different peer? + if msg.Actions&(1< 0 && len(msg.FindPeerKeys) > 0 { + for _, findPeer := range msg.FindPeerKeys { + peer.Backend.Filters.IncomingRequest(peer, protocol.ActionFindPeer, findPeer.Hash, nil) + + details := protocol.Hash2Peer{ID: findPeer} + + // Same as before, put self as ignoredNodes. + for _, node := range peer.Backend.nodesDHT.GetClosestContacts(respondClosesContactsCount, findPeer.Hash, filterFunc(connection.IsLocal(), allowIPv4, allowIPv6), peer.NodeID) { + if info := node.Info.(*PeerInfo).peer2Record(connection.IsLocal(), allowIPv4, allowIPv6); info != nil { + details.Closest = append(details.Closest, *info) + } + } + + if len(details.Closest) > 0 { + hash2Peers = append(hash2Peers, details) + } else { + hashesNotFound = append(hashesNotFound, findPeer.Hash) + } + } + } + + // Find a value? + if msg.Actions&(1< 0 { + for _, findHash := range msg.FindDataKeys { + peer.Backend.Filters.IncomingRequest(peer, protocol.ActionFindValue, findHash.Hash, nil) + + stored, data := peer.announcementGetData(findHash.Hash) + if stored && len(data) > 0 { + filesEmbed = append(filesEmbed, protocol.EmbeddedFileData{ID: findHash, Data: data}) + } else if stored { + selfRecord := peer.Backend.selfPeerRecord() + hash2Peers = append(hash2Peers, protocol.Hash2Peer{ID: findHash, Storing: []protocol.PeerRecord{selfRecord}}) + } else { + hashesNotFound = append(hashesNotFound, findHash.Hash) + } + } + } + + // Information about files stored by the sender? + if msg.Actions&(1< 0 && len(msg.InfoStoreFiles) > 0 { + for n := range msg.InfoStoreFiles { + peer.Backend.Filters.IncomingRequest(peer, protocol.ActionInfoStore, msg.InfoStoreFiles[n].ID.Hash, &msg.InfoStoreFiles[n]) + } + + peer.announcementStore(msg.InfoStoreFiles) + } + + sendUA := msg.UserAgent != "" // Send user agent if one was provided. Per protocol the first announcement message must have the User Agent set. + peer.sendResponse(msg.Sequence, sendUA, hash2Peers, filesEmbed, hashesNotFound) +} + +func (peer *PeerInfo) peer2Record(allowLocal, allowIPv4, allowIPv6 bool) (result *protocol.PeerRecord) { + connectionIPv4 := peer.GetConnection2Share(allowLocal, allowIPv4, false) + connectionIPv6 := peer.GetConnection2Share(allowLocal, false, allowIPv6) + if connectionIPv4 == nil && connectionIPv6 == nil { + return nil + } + + result = &protocol.PeerRecord{ + PublicKey: peer.PublicKey, + NodeID: peer.NodeID, + Features: peer.Features, + } + + if connectionIPv4 != nil { + result.IPv4 = connectionIPv4.Address.IP + result.IPv4Port = uint16(connectionIPv4.Address.Port) + result.IPv4PortReportedInternal = connectionIPv4.PortInternal + result.IPv4PortReportedExternal = connectionIPv4.PortExternal + } + + if connectionIPv6 != nil { + result.IPv6 = connectionIPv6.Address.IP + result.IPv6Port = uint16(connectionIPv6.Address.Port) + result.IPv6PortReportedInternal = connectionIPv6.PortInternal + result.IPv6PortReportedExternal = connectionIPv6.PortExternal + } + + return result +} + +// cmdResponse handles the response to the announcement +func (peer *PeerInfo) cmdResponse(msg *protocol.MessageResponse, connection *Connection) { + // The sequence data is used to correlate this response with the announcement. + if msg.SequenceInfo == nil || msg.SequenceInfo.Data == nil { + // If there is no sequence data but there were results returned, it means we received unsolicited response data. It will be rejected. + if len(msg.HashesNotFound) > 0 || len(msg.Hash2Peers) > 0 || len(msg.FilesEmbed) > 0 { + peer.Backend.LogError("cmdResponse", "unsolicited response data received from %s\n", connection.Address.String()) + } + + return + } + + // bootstrap FIND_SELF? + if _, ok := msg.SequenceInfo.Data.(*bootstrapFindSelf); ok { + for _, hash2Peer := range msg.Hash2Peers { + // Make sure no garbage is returned. The key must be self and only Closest is expected. + if !bytes.Equal(hash2Peer.ID.Hash, peer.Backend.nodeID) || len(hash2Peer.Closest) == 0 { + peer.Backend.LogError("cmdResponse", "incoming response to bootstrap FIND_SELF contains invalid data from %s\n", connection.Address.String()) + return + } + + peer.cmdResponseBootstrapFindSelf(msg, hash2Peer.Closest) + } + + return + } + + // Response to an information request? + if _, ok := msg.SequenceInfo.Data.(*dht.InformationRequest); ok { + // Future: Once multiple information requests are pooled (multiplexed) into one or multiple Announcement sequences (messages), the responses need to be de-pooled. + // A simple multiplex structure linked via the sequence containing a map (hash 2 IR) could simplify this. + info := msg.SequenceInfo.Data.(*dht.InformationRequest) + + if len(msg.HashesNotFound) > 0 { + info.Done() + } + + for _, hash2Peer := range msg.Hash2Peers { + info.QueueResult(&dht.NodeMessage{SenderID: peer.NodeID, Closest: peer.records2Nodes(hash2Peer.Closest), Storing: peer.records2Nodes(hash2Peer.Storing)}) + + if hash2Peer.IsLast { + info.Done() + } + } + + for _, file := range msg.FilesEmbed { + info.QueueResult(&dht.NodeMessage{SenderID: peer.NodeID, Data: file.Data}) + + info.Done() + info.Terminate() // file was found, terminate the request. + } + } +} + +// cmdPing handles an incoming ping message +func (peer *PeerInfo) cmdPing(msg *protocol.MessageRaw, connection *Connection) { + // If PortInternal is 0, it means no incoming announcement or response message was received on that connection. + // This means the ping is unexpected. In that case for security reasons the remote peer is not asked for FIND_SELF. + if connection.PortInternal == 0 { + peer.sendAnnouncement(true, false, nil, nil, nil, nil) + return + } + + raw := &protocol.PacketRaw{Command: protocol.CommandPong, Sequence: msg.Sequence} + + peer.Backend.Filters.MessageOutPong(peer, raw) + + peer.send(raw) +} + +// cmdPong handles an incoming pong message +func (peer *PeerInfo) cmdPong(msg *protocol.MessageRaw, connection *Connection) { +} + +// cmdChat handles a chat message [debug] +func (peer *PeerInfo) cmdChat(msg *protocol.MessageRaw, connection *Connection) { + fmt.Fprintf(peer.Backend.Stdout, "Chat from %s '%s': %s\n", hex.EncodeToString(peer.PublicKey.SerializeCompressed()), connection.Address.String(), string(msg.PacketRaw.Payload)) +} + +// cmdLocalDiscovery handles an incoming announcement via local discovery +func (peer *PeerInfo) cmdLocalDiscovery(msg *protocol.MessageAnnouncement, connection *Connection) { + // 21.04.2021 update: Local peer discovery from public IPv4s is possible in datacenter situations. Keep it enabled for now. + // only accept local discovery message from private IPs for IPv4 + // IPv6 DHCP routers typically assign public IPv6s and they can join multicast in the local network. + //if connection.IsIPv4() && !connection.IsLocal() { + // LogError("cmdLocalDiscovery", "message received from non-local IP %s peer ID %s\n", connection.Address.String(), hex.EncodeToString(msg.SenderPublicKey.SerializeCompressed())) + // return + //} + + peer.sendAnnouncement(true, ShouldSendFindSelf(), nil, nil, nil, &bootstrapFindSelf{}) +} + +// SendChatAll sends a text message to all peers +func (backend *Backend) SendChatAll(text string) { + for _, peer := range backend.PeerlistGet() { + peer.Chat(text) + } +} + +// cmdTransfer handles an incoming transfer message +func (peer *PeerInfo) cmdTransfer(msg *protocol.MessageTransfer, connection *Connection) { + // Only UDT protocol is currently supported for file transfer. + if msg.TransferProtocol != protocol.TransferProtocolUDT { + return + } + + switch msg.Control { + case protocol.TransferControlRequestStart: + // First check if the file available in the warehouse. + _, fileSize, status, _ := peer.Backend.UserWarehouse.FileExists(msg.Hash) + if status != warehouse.StatusOK { + // File not available. + peer.sendTransfer(nil, protocol.TransferControlNotAvailable, msg.TransferProtocol, msg.Hash, 0, 0, msg.Sequence, uuid.UUID{}, false) + return + } else if msg.Limit > 0 && fileSize < msg.Offset+msg.Limit { + // If the read limit is out of bounds, this request is considered invalid and silently discarded. + return + } + + // Create a local UDT client to connect to the remote UDT server and serve the file! + go peer.startFileTransferUDT(msg.Hash, fileSize, msg.Offset, msg.Limit, msg.Sequence, msg.TransferID, msg.TransferProtocol) + + case protocol.TransferControlActive: + if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { + go v.receiveData(msg.Data) + return + } + + case protocol.TransferControlNotAvailable: + if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { + v.Terminate(404) + return + } + + case protocol.TransferControlTerminate: + if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { + v.Terminate(2) + return + } + + } +} + +// cmdGetBlock handles an incoming block message +func (peer *PeerInfo) cmdGetBlock(msg *protocol.MessageGetBlock, connection *Connection) { + switch msg.Control { + case protocol.GetBlockControlRequestStart: + // Currently only support the local blockchain. + if !msg.BlockchainPublicKey.IsEqual(peer.Backend.PeerPublicKey) { + peer.sendGetBlock(nil, protocol.GetBlockControlNotAvailable, msg.BlockchainPublicKey, 0, 0, nil, msg.Sequence, uuid.UUID{}, false) + return + } else if _, height, _ := peer.Backend.UserBlockchain.Header(); height == 0 { + peer.sendGetBlock(nil, protocol.GetBlockControlEmpty, msg.BlockchainPublicKey, 0, 0, nil, msg.Sequence, uuid.UUID{}, false) + return + } else if msg.LimitBlockCount == 0 { + peer.sendGetBlock(nil, protocol.GetBlockControlTerminate, msg.BlockchainPublicKey, 0, 0, nil, msg.Sequence, uuid.UUID{}, false) + return + } + + // Create a local UDT client to connect to the remote UDT server and serve the blocks! + go peer.startBlockTransfer(msg.BlockchainPublicKey, msg.LimitBlockCount, msg.MaxBlockSize, msg.TargetBlocks, msg.Sequence, msg.TransferID) + + case protocol.GetBlockControlActive: + if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { + go v.receiveData(msg.Data) + return + } + + case protocol.GetBlockControlNotAvailable: + if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { + v.Terminate(404) + return + } + + case protocol.GetBlockControlEmpty: + if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { + v.Terminate(410) + return + } + + case protocol.GetBlockControlTerminate: + if v, ok := msg.SequenceInfo.Data.(*VirtualPacketConn); ok { + v.Terminate(2) + return + } + + } +} diff --git a/Config.go b/Config.go index 8b0abfe..0562196 100644 --- a/Config.go +++ b/Config.go @@ -1,195 +1,195 @@ -/* -File Name: Settings.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - _ "embed" // Required for embedding default Config file - "fmt" - "io/ioutil" - "log" - "os" - "path" - - "gopkg.in/yaml.v3" -) - -// Version is the current core library version -const Version = "Alpha 9/01.11.2022" - -// Config defines the minimum required config for a Peernet client. -type Config struct { - // Locations of important files and folders - LogFile string `yaml:"LogFile"` // Log file. It contains informational and error messages. - BlockchainMain string `yaml:"BlockchainMain"` // Blockchain main stores the end-users blockchain data. It contains meta data of shared files, profile data, and social interactions. - BlockchainGlobal string `yaml:"BlockchainGlobal"` // Blockchain global caches blockchain data from global users. Empty to disable. - WarehouseMain string `yaml:"WarehouseMain"` // Warehouse main stores the actual data of files shared by the end-user. - SearchIndex string `yaml:"SearchIndex"` // Local search index of blockchain records. Empty to disable. - GeoIPDatabase string `yaml:"GeoIPDatabase"` // GeoLite2 City database to provide GeoIP information. - DataFolder string `yaml:"DataFolder"` // Data folder. - - // Target for the log messages: 0 = Log file, 1 = Stdout, 2 = Log file + Stdout, 3 = None - LogTarget int `yaml:"LogTarget"` - - // Listen settings - Listen []string `yaml:"Listen"` // IP:Port combinations - ListenWorkers int `yaml:"ListenWorkers"` // Count of workers to process incoming raw packets. Default 2. - ListenWorkersLite int `yaml:"ListenWorkersLite"` // Count of workers to process incoming lite packets. Default 2. - - // User specific settings - PrivateKey string `yaml:"PrivateKey"` // The Private Key, hex encoded so it can be copied manually - - // Initial peer seed list - SeedList []PeerSeed `yaml:"SeedList"` - AutoUpdateSeedList bool `yaml:"AutoUpdateSeedList"` - SeedListVersion int `yaml:"SeedListVersion"` - - // Connection settings - EnableUPnP bool `yaml:"EnableUPnP"` // Enables support for UPnP. - LocalFirewall bool `yaml:"LocalFirewall"` // Indicates that a local firewall may drop unsolicited incoming packets. - - // PortForward specifies an external port that was manually forwarded by the user. All listening IPs must have that same port number forwarded! - // If this setting is invalid, it will prohibit other peers from connecting. If set, it automatically disables UPnP. - PortForward uint16 `yaml:"PortForward"` - - // Global blockchain cache limits - CacheMaxBlockSize uint64 `yaml:"CacheMaxBlockSize"` // Max block size to accept in bytes. - CacheMaxBlockCount uint64 `yaml:"CacheMaxBlockCount"` // Max block count to cache per peer. - LimitTotalRecords uint64 `yaml:"LimitTotalRecords"` // Record count limit. 0 = unlimited. Max Records * Max Block Size = Size Limit. -} - -// PeerSeed is a singl peer entry from the config's seed list -type PeerSeed struct { - PublicKey string `yaml:"PublicKey"` // Public key = peer ID. Hex encoded. - Address []string `yaml:"Address"` // IP:Port -} - -//go:embed "Config Default.yaml" -var ConfigDefault []byte - -// LoadConfig reads the YAML configuration file and unmarshals it into the provided structure. -// If the config file does not exist or is empty, it will fall back to the default config which is hardcoded. -// Status is of type ExitX. -func LoadConfig(Filename string, ConfigOut interface{}) (status int, err error) { - var configData []byte - - // check if the file is non existent or empty - stats, err := os.Stat(Filename) - if err != nil && os.IsNotExist(err) || err == nil && stats.Size() == 0 { - configData = ConfigDefault - } else if err != nil { - return ExitErrorConfigAccess, err - } else if configData, err = ioutil.ReadFile(Filename); err != nil { - return ExitErrorConfigRead, err - } - - // parse the config - err = yaml.Unmarshal(configData, ConfigOut) - if err != nil { - return ExitErrorConfigParse, err - } - - return ExitSuccess, nil -} - -// SaveConfig stores the config. -func SaveConfig(Filename string, Config interface{}) (err error) { - data, err := yaml.Marshal(Config) - if err != nil { - return err - } - - return ioutil.WriteFile(Filename, data, 0666) -} - -// SaveConfigs stores the multiple configurations into a single file. They are essentially merged. -// It is the callers responsibility to make sure no field collision ensue. -func SaveConfigs(Filename string, Configs ...interface{}) (err error) { - var dataOut []byte - - for n, config := range Configs { - if n > 0 { - dataOut = append(dataOut, []byte("\n\n")...) - } - - data, err := yaml.Marshal(config) - if err != nil { - return err - } - - dataOut = append(dataOut, data...) - } - - return ioutil.WriteFile(Filename, dataOut, 0666) -} - -// SaveConfig stores the current runtime config to file. Any foreign settings not present in the Config structure will be deleted. -func (backend *Backend) SaveConfig() { - if backend.ConfigClient != nil { - if err := SaveConfigs(backend.ConfigFilename, *backend.Config, backend.ConfigClient); err != nil { - backend.LogError("SaveConfig", "writing config '%s': %v\n", backend.ConfigFilename, err.Error()) - } - return - } - - if err := SaveConfig(backend.ConfigFilename, *backend.Config); err != nil { - backend.LogError("SaveConfig", "writing config '%s': %v\n", backend.ConfigFilename, err.Error()) - } -} - -func (backend *Backend) configUpdateSeedList() { - // parse the embedded config - var configD Config - if err := yaml.Unmarshal(ConfigDefault, &configD); err != nil { - return - } - - // check if the seed list needs an update - if backend.Config.SeedListVersion < configD.SeedListVersion { - backend.Config.SeedList = configD.SeedList - backend.Config.SeedListVersion = configD.SeedListVersion - backend.SaveConfig() - } -} - -// InitLog redirects subsequent log messages into the default log file specified in the configuration -func (backend *Backend) initLog() (err error) { - // create the directory to the log file if specified - if directory, _ := path.Split(backend.Config.LogFile); directory != "" { - os.MkdirAll(directory, os.ModePerm) - } - - logFile, err := os.OpenFile(backend.Config.LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) // 666 : All uses can read/write - if err != nil { - return err - } - //defer logFile.Close() // has to remain open until program closes - - log.SetOutput(logFile) - log.Printf("---- Peernet Command-Line Client " + Version + " ----\n") - - return nil -} - -// Logs an error message. -func (backend *Backend) LogError(function, format string, v ...interface{}) { - switch backend.Config.LogTarget { - case 0: - log.Printf("["+function+"] "+format, v...) - - case 1: - fmt.Fprintf(backend.Stdout, "["+function+"] "+format, v...) - - case 2: - log.Printf("["+function+"] "+format, v...) - - fmt.Fprintf(backend.Stdout, "["+function+"] "+format, v...) - - case 3: // None - } - - backend.Filters.LogError(function, format, v) -} +/* +File Name: Settings.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + _ "embed" // Required for embedding default Config file + "fmt" + "io/ioutil" + "log" + "os" + "path" + + "gopkg.in/yaml.v3" +) + +// Version is the current core library version +const Version = "Alpha 9/01.11.2022" + +// Config defines the minimum required config for a Peernet client. +type Config struct { + // Locations of important files and folders + LogFile string `yaml:"LogFile"` // Log file. It contains informational and error messages. + BlockchainMain string `yaml:"BlockchainMain"` // Blockchain main stores the end-users blockchain data. It contains meta data of shared files, profile data, and social interactions. + BlockchainGlobal string `yaml:"BlockchainGlobal"` // Blockchain global caches blockchain data from global users. Empty to disable. + WarehouseMain string `yaml:"WarehouseMain"` // Warehouse main stores the actual data of files shared by the end-user. + SearchIndex string `yaml:"SearchIndex"` // Local search index of blockchain records. Empty to disable. + GeoIPDatabase string `yaml:"GeoIPDatabase"` // GeoLite2 City database to provide GeoIP information. + DataFolder string `yaml:"DataFolder"` // Data folder. + + // Target for the log messages: 0 = Log file, 1 = Stdout, 2 = Log file + Stdout, 3 = None + LogTarget int `yaml:"LogTarget"` + + // Listen settings + Listen []string `yaml:"Listen"` // IP:Port combinations + ListenWorkers int `yaml:"ListenWorkers"` // Count of workers to process incoming raw packets. Default 2. + ListenWorkersLite int `yaml:"ListenWorkersLite"` // Count of workers to process incoming lite packets. Default 2. + + // User specific settings + PrivateKey string `yaml:"PrivateKey"` // The Private Key, hex encoded so it can be copied manually + + // Initial peer seed list + SeedList []PeerSeed `yaml:"SeedList"` + AutoUpdateSeedList bool `yaml:"AutoUpdateSeedList"` + SeedListVersion int `yaml:"SeedListVersion"` + + // Connection settings + EnableUPnP bool `yaml:"EnableUPnP"` // Enables support for UPnP. + LocalFirewall bool `yaml:"LocalFirewall"` // Indicates that a local firewall may drop unsolicited incoming packets. + + // PortForward specifies an external port that was manually forwarded by the user. All listening IPs must have that same port number forwarded! + // If this setting is invalid, it will prohibit other peers from connecting. If set, it automatically disables UPnP. + PortForward uint16 `yaml:"PortForward"` + + // Global blockchain cache limits + CacheMaxBlockSize uint64 `yaml:"CacheMaxBlockSize"` // Max block size to accept in bytes. + CacheMaxBlockCount uint64 `yaml:"CacheMaxBlockCount"` // Max block count to cache per peer. + LimitTotalRecords uint64 `yaml:"LimitTotalRecords"` // Record count limit. 0 = unlimited. Max Records * Max Block Size = Size Limit. +} + +// PeerSeed is a singl peer entry from the config's seed list +type PeerSeed struct { + PublicKey string `yaml:"PublicKey"` // Public key = peer ID. Hex encoded. + Address []string `yaml:"Address"` // IP:Port +} + +//go:embed "Config Default.yaml" +var ConfigDefault []byte + +// LoadConfig reads the YAML configuration file and unmarshals it into the provided structure. +// If the config file does not exist or is empty, it will fall back to the default config which is hardcoded. +// Status is of type ExitX. +func LoadConfig(Filename string, ConfigOut interface{}) (status int, err error) { + var configData []byte + + // check if the file is non existent or empty + stats, err := os.Stat(Filename) + if err != nil && os.IsNotExist(err) || err == nil && stats.Size() == 0 { + configData = ConfigDefault + } else if err != nil { + return ExitErrorConfigAccess, err + } else if configData, err = ioutil.ReadFile(Filename); err != nil { + return ExitErrorConfigRead, err + } + + // parse the config + err = yaml.Unmarshal(configData, ConfigOut) + if err != nil { + return ExitErrorConfigParse, err + } + + return ExitSuccess, nil +} + +// SaveConfig stores the config. +func SaveConfig(Filename string, Config interface{}) (err error) { + data, err := yaml.Marshal(Config) + if err != nil { + return err + } + + return ioutil.WriteFile(Filename, data, 0666) +} + +// SaveConfigs stores the multiple configurations into a single file. They are essentially merged. +// It is the callers responsibility to make sure no field collision ensue. +func SaveConfigs(Filename string, Configs ...interface{}) (err error) { + var dataOut []byte + + for n, config := range Configs { + if n > 0 { + dataOut = append(dataOut, []byte("\n\n")...) + } + + data, err := yaml.Marshal(config) + if err != nil { + return err + } + + dataOut = append(dataOut, data...) + } + + return ioutil.WriteFile(Filename, dataOut, 0666) +} + +// SaveConfig stores the current runtime config to file. Any foreign settings not present in the Config structure will be deleted. +func (backend *Backend) SaveConfig() { + if backend.ConfigClient != nil { + if err := SaveConfigs(backend.ConfigFilename, *backend.Config, backend.ConfigClient); err != nil { + backend.LogError("SaveConfig", "writing config '%s': %v\n", backend.ConfigFilename, err.Error()) + } + return + } + + if err := SaveConfig(backend.ConfigFilename, *backend.Config); err != nil { + backend.LogError("SaveConfig", "writing config '%s': %v\n", backend.ConfigFilename, err.Error()) + } +} + +func (backend *Backend) configUpdateSeedList() { + // parse the embedded config + var configD Config + if err := yaml.Unmarshal(ConfigDefault, &configD); err != nil { + return + } + + // check if the seed list needs an update + if backend.Config.SeedListVersion < configD.SeedListVersion { + backend.Config.SeedList = configD.SeedList + backend.Config.SeedListVersion = configD.SeedListVersion + backend.SaveConfig() + } +} + +// InitLog redirects subsequent log messages into the default log file specified in the configuration +func (backend *Backend) initLog() (err error) { + // create the directory to the log file if specified + if directory, _ := path.Split(backend.Config.LogFile); directory != "" { + os.MkdirAll(directory, os.ModePerm) + } + + logFile, err := os.OpenFile(backend.Config.LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) // 666 : All uses can read/write + if err != nil { + return err + } + //defer logFile.Close() // has to remain open until program closes + + log.SetOutput(logFile) + log.Printf("---- Peernet Command-Line Client " + Version + " ----\n") + + return nil +} + +// Logs an error message. +func (backend *Backend) LogError(function, format string, v ...interface{}) { + switch backend.Config.LogTarget { + case 0: + log.Printf("["+function+"] "+format, v...) + + case 1: + fmt.Fprintf(backend.Stdout, "["+function+"] "+format, v...) + + case 2: + log.Printf("["+function+"] "+format, v...) + + fmt.Fprintf(backend.Stdout, "["+function+"] "+format, v...) + + case 3: // None + } + + backend.Filters.LogError(function, format, v) +} diff --git a/Connection.go b/Connection.go index ca0c1e9..a7a41da 100644 --- a/Connection.go +++ b/Connection.go @@ -1,499 +1,499 @@ -/* -File Name: Connection.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "errors" - "net" - "sync/atomic" - "time" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/protocol" -) - -// Connection is an established connection between a remote IP address and a local network adapter. -// New connections may only be created in case of successful INCOMING packets. -type Connection struct { - Network *Network // Network which received the packet. - Address *net.UDPAddr // Address of the remote peer. - PortInternal uint16 // Internal listening port reported by remote peer. 0 if no Announcement/Response message was yet received. - PortExternal uint16 // External listening port reported by remote peer. 0 if not known by the peer. - LastPacketIn time.Time // Last time an incoming packet was received. - LastPacketOut time.Time // Last time an outgoing packet was attempted to send. - LastPingOut time.Time // Last ping out. - Expires time.Time // Inactive connections only: Expiry date. If it does not become active by that date, it will be considered expired and removed. - Status int // 0 = Active established connection, 1 = Inactive, 2 = Removed, 3 = Redundant - RoundTripTime time.Duration // Full round-trip time of last reply. - Firewall bool // Whether the remote peer indicates a potential firewall. This means a Traverse message shall be sent to establish a connection. - traversePeer *PeerInfo // Temporary peer that may act as proxy for a Traverse message used for the first packet. This is used to establish this Connection to a peer that is behind a NAT or firewall. - backend *Backend -} - -// Connection status -const ( - ConnectionActive = iota - ConnectionInactive - ConnectionRemoved - ConnectionRedundant // Same as active. Incoming packets are accepted. Outgoing use only for redundancy. Reduces ping overhead. -) - -// Equal checks if the connection was established other the same network adapter using the same IP address. Port is intentionally not checked. -func (c *Connection) Equal(other *Connection) bool { - return c.Address.IP.Equal(other.Address.IP) && c.Network.address.IP.Equal(other.Network.address.IP) -} - -// IsLocal checks if the connection is a local network one (LAN) -func (c *Connection) IsLocal() bool { - return c != nil && IsIPLocal(c.Address.IP) -} - -// IsIPv4 checks if the connection is using IPv4 -func (c *Connection) IsIPv4() bool { - return IsIPv4(c.Address.IP) -} - -// IsIPv6 checks if the connection is using IPv6 -func (c *Connection) IsIPv6() bool { - return IsIPv6(c.Address.IP) -} - -// IsBehindNAT checks if the remote peer on the connection is likely behind a NAT -func (c *Connection) IsBehindNAT() bool { - return c.PortInternal > 0 && c.Address.Port != int(c.PortInternal) -} - -// IsPortForward checks if the remote peer uses port forwarding on the connection -func (c *Connection) IsPortForward() bool { - return c.PortExternal > 0 -} - -// IsVirtual returns true if the peer has not been connected yet. This is the case if another peer responds with peer details, and that peer shall be contacted. -func (peer *PeerInfo) IsVirtual() bool { - return peer.isVirtual -} - -// GetConnections returns the list of connections -func (peer *PeerInfo) GetConnections(active bool) (connections []*Connection) { - peer.RLock() - defer peer.RUnlock() - - if active { - return peer.connectionActive - } - return peer.connectionInactive -} - -// IsConnectionActive checks if the peer has an active connection that can be used to send and receive messages -func (peer *PeerInfo) IsConnectionActive() bool { - peer.RLock() - defer peer.RUnlock() - - return len(peer.connectionActive) > 0 -} - -// IsConnectable checks if the peer is connectable to the given IP parameters. -func (peer *PeerInfo) IsConnectable(allowLocal, allowIPv4, allowIPv6 bool) bool { - peer.RLock() - defer peer.RUnlock() - - // Only 1 active connection must be allowed for being connectable. - for _, connection := range peer.connectionActive { - // If the internal port is not known, which happens if no Announcement or Response was returned, do not share the peer details. - // This can happen if only other messages such as Ping/Pong were received, or the protocol implementation is not compatible. The external port is also likely not available. - // In this case sharing the peer would be bad, since the receiving peer could not use internal/external port to detemine the NAT status and port forwarding. - if connection.PortInternal == 0 { - continue - } - - if IsIPv4(connection.Address.IP) && allowIPv4 || IsIPv6(connection.Address.IP) && allowIPv6 { - if !(!allowLocal && connection.IsLocal()) { - return true - } - } - } - - return false -} - -// GetConnection2Share returns a connection to share. Nil if none. -// allowLocal specifies whether it is OK to return local IPs. -func (peer *PeerInfo) GetConnection2Share(allowLocal, allowIPv4, allowIPv6 bool) (connection *Connection) { - if !allowLocal && !allowIPv4 && !allowIPv6 { - return nil - } - - peer.RLock() - defer peer.RUnlock() - - if peer.connectionLatest != nil && !(!allowLocal && peer.connectionLatest.IsLocal()) && - (IsIPv4(peer.connectionLatest.Address.IP) && allowIPv4 || IsIPv6(peer.connectionLatest.Address.IP) && allowIPv6) && peer.connectionLatest.PortInternal > 0 { - return peer.connectionLatest - } - - for _, connection := range peer.connectionActive { - if (IsIPv4(connection.Address.IP) && allowIPv4 || IsIPv6(connection.Address.IP) && allowIPv6) && !(!allowLocal && connection.IsLocal()) && connection.PortInternal > 0 { - return connection - } - } - - return nil -} - -// registerConnection registers an incoming connection for an existing peer. If new, it will add to the list. If previously inactive, it will elevate. -func (peer *PeerInfo) registerConnection(incoming *Connection) (result *Connection) { - peer.Lock() - defer peer.Unlock() - - // first check if already an active connection to the same IP - for _, connection := range peer.connectionActive { - if connection.Equal(incoming) { - // Connection already established. Verify port and update if necessary. - // Some NATs may rotate ports. Some mobile phone providers even rotate IPs which is not detected here. - if connection.Address.Port != incoming.Address.Port { - connection.Address.Port = incoming.Address.Port - } - - connection.Status = ConnectionActive - peer.setConnectionLatest(connection) - return connection - } - } - - // if an inactive connection, elevate it to activated one - for n, connection := range peer.connectionInactive { - if connection.Equal(incoming) { - if connection.Address.Port != incoming.Address.Port { - connection.Address.Port = incoming.Address.Port - } - - // elevate by adding to active and mark as latest active - connection.Status = ConnectionActive - peer.connectionActive = append(peer.connectionActive, connection) - peer.setConnectionLatest(connection) - - // remove from inactive - inactiveNew := peer.connectionInactive[:n] - if n < len(peer.connectionInactive)-1 { - inactiveNew = append(inactiveNew, peer.connectionInactive[n+1:]...) - } - peer.connectionInactive = inactiveNew - - return connection - } - } - - // otherwise it is a new connection! - peer.connectionActive = append(peer.connectionActive, incoming) - peer.setConnectionLatest(incoming) - - peer.Backend.Filters.NewPeerConnection(peer, incoming) - - return incoming -} - -// setConnectionLatest updates the latest valid connection to use for sending. All other connections will be changed to redundant, which reduces ping overhead. -func (peer *PeerInfo) setConnectionLatest(latest *Connection) { - if peer.connectionLatest == latest { - return - } - - peer.connectionLatest = latest - - for _, connection := range peer.connectionActive { - if connection == latest { - continue - } - connection.Status = ConnectionRedundant - } -} - -// invalidateActiveConnection invalidates an active connection -func (peer *PeerInfo) invalidateActiveConnection(input *Connection) { - peer.Lock() - defer peer.Unlock() - - // Change the status to inactive and start the expiration. If the connection does not become valid by that date, it will be removed. - input.Status = ConnectionInactive - input.Expires = time.Now().Add(connectionRemove * time.Second) - - // remove from connectionLatest if selected so it won't be used by standard send function - if peer.connectionLatest == input { - peer.connectionLatest = nil - } - - for n, connection := range peer.connectionActive { - if connection == input { - // add to list of inactive connections - peer.connectionInactive = append(peer.connectionInactive, connection) - - // remove from active - activeNew := peer.connectionActive[:n] - if n < len(peer.connectionActive)-1 { - activeNew = append(activeNew, peer.connectionActive[n+1:]...) - } - peer.connectionActive = activeNew - - break - } - } -} - -// removeInactiveConnection removes an inactive connection. -func (peer *PeerInfo) removeInactiveConnection(input *Connection) { - peer.Lock() - defer peer.Unlock() - - input.Status = ConnectionRemoved - - for n, connection := range peer.connectionInactive { - if connection == input { - - // remove from inactive - inactiveNew := peer.connectionInactive[:n] - if n < len(peer.connectionInactive)-1 { - inactiveNew = append(inactiveNew, peer.connectionInactive[n+1:]...) - } - peer.connectionInactive = inactiveNew - - return - } - } -} - -// GetRTT returns the round-trip time for the most recent active connection. 0 if not available. -func (peer *PeerInfo) GetRTT() (rtt time.Duration) { - peer.Lock() - defer peer.Unlock() - - if peer.connectionLatest != nil && peer.connectionLatest.RoundTripTime > 0 { - return peer.connectionLatest.RoundTripTime - } - - for _, connection := range peer.connectionActive { - if connection.RoundTripTime > 0 { - return connection.RoundTripTime - } - } - - return 0 -} - -// IsBehindNAT checks if the peer is behind NAT -func (peer *PeerInfo) IsBehindNAT() (result bool) { - peer.Lock() - defer peer.Unlock() - - // Default is no. Only if a public network reports different connected port vs internal one, NAT is assumed. - // This also assumes that all 3rd party clients bind their connection to the outgoing port. - // PortInternal is 0 if no Announcement or Response message was received. - - for _, connection := range peer.connectionActive { - if connection.IsBehindNAT() { - return true - } - } - - for _, connection := range peer.connectionInactive { - if connection.IsBehindNAT() { - return true - } - } - - return false -} - -// IsPortForward checks if the peer uses port forwarding -func (peer *PeerInfo) IsPortForward() (result bool) { - peer.Lock() - defer peer.Unlock() - - for _, connection := range peer.connectionActive { - if connection.IsPortForward() { - return true - } - } - - for _, connection := range peer.connectionInactive { - if connection.IsPortForward() { - return true - } - } - - return false -} - -// IsFirewallReported checks if the peer reported to be behind a firewall -func (peer *PeerInfo) IsFirewallReported() (result bool) { - return peer.Features&(1< 0 -} - -// ---- sending code ---- - -// send sends the packet to the peer on the connection -// isFirstPacket indicates whether this is the first packet to an uncontacted peer. -func (c *Connection) send(packet *protocol.PacketRaw, receiverPublicKey *btcec.PublicKey, isFirstPacket bool) (err error) { - if c == nil { - return errors.New("invalid connection") - } - - packet.Protocol = protocol.ProtocolVersion - packet.SetSelfReportedPorts(c.Network.SelfReportedPorts()) - - c.backend.Filters.PacketOut(packet, receiverPublicKey, c) - - raw, err := protocol.PacketEncrypt(c.backend.PeerPrivateKey, receiverPublicKey, packet) - if err != nil { - return err - } - - c.LastPacketOut = time.Now() - - err = c.Network.send(c.Address.IP, c.Address.Port, raw) - - // Send Traverse message if the peer is behind a NAT or firewall and this is the first message. Only for Announcement. - if err == nil && isFirstPacket && (c.IsBehindNAT() || c.Firewall) && c.traversePeer != nil && packet.Command == protocol.CommandAnnouncement { - c.traversePeer.sendTraverse(packet, receiverPublicKey) - } - - return err -} - -// send sends a raw packet to the peer. Only uses active connections. -func (peer *PeerInfo) send(packet *protocol.PacketRaw) (err error) { - if peer.isVirtual { // special case for peers that were not contacted before - for _, address := range peer.targetAddresses { - peer.Backend.networks.sendAllNetworks(peer.PublicKey, packet, &net.UDPAddr{IP: address.IP, Port: int(address.Port)}, address.PortInternal, peer.Features&(1< 0, peer.traversePeer, nil) - } - return - } - if len(peer.connectionActive) == 0 { - return errors.New("no valid connection to peer") - } - - // For Traverse: check if no packet has been sent, and none received (i.e. initial contact). - // If a packet was already received directly (note: not via incoming traversed message), a valid connection is already established. - isFirstPacketOut := atomic.LoadUint64(&peer.StatsPacketSent) == 0 && atomic.LoadUint64(&peer.StatsPacketReceived) == 0 - - // always count as one sent packet even if sent via broadcast - atomic.AddUint64(&peer.StatsPacketSent, 1) - - // Send out the wire. Use connectionLatest if available. - // Failover: If sending fails and there are other connections available, try those. Automatically update connectionLatest if one is successful. - // Windows: This works great in case the adapter gets disabled, however, does not detect if the network cable is unplugged. - cLatest := peer.connectionLatest - if cLatest != nil { - if err := cLatest.send(packet, peer.PublicKey, isFirstPacketOut); err == nil { - return nil - } else if IsNetworkErrorFatal(err) { - // Invalid connection, immediately invalidate. Fallback to broadcast to all other active ones. - // Windows: A common error when the network adapter is disabled is "wsasendto: The requested address is not valid in its context". - peer.invalidateActiveConnection(cLatest) - } - } - - // If no latest connection available, broadcast on all other available connections. - // This might be noisy, but if no latest connection is available it means the last established connection is already considered dead. - // The receiver is responsible for incoming deduplication of packets. - activeConnections := peer.GetConnections(true) - for _, c := range activeConnections { - if c == cLatest { - continue - } - - if err := c.send(packet, peer.PublicKey, isFirstPacketOut); err != nil && IsNetworkErrorFatal(err) { - peer.invalidateActiveConnection(c) - } - } - - return nil // on broadcast no error is known and returned -} - -// sendConnection sends a packet to the peer using the specific connection -func (peer *PeerInfo) sendConnection(packet *protocol.PacketRaw, connection *Connection) (err error) { - isFirstPacketOut := atomic.LoadUint64(&peer.StatsPacketSent) == 0 && atomic.LoadUint64(&peer.StatsPacketReceived) == 0 - atomic.AddUint64(&peer.StatsPacketSent, 1) - - return connection.send(packet, peer.PublicKey, isFirstPacketOut) -} - -// sendAllNetworks sends a raw packet via all networks. It assigns a new sequence for each sent packet. -// receiverPortInternal is important for NAT detection and sending the traverse message. Firewall indicates whether the remote peer was reported to be behind a firewall. -func (nets *Networks) sendAllNetworks(receiverPublicKey *btcec.PublicKey, packet *protocol.PacketRaw, remote *net.UDPAddr, receiverPortInternal uint16, receiverFirewall bool, traversePeer *PeerInfo, sequenceData interface{}) (err error) { - nets.RLock() - defer nets.RUnlock() - - networksTarget := nets.networks4 - if IsIPv6(remote.IP.To16()) { - networksTarget = nets.networks6 - } - - successCount := 0 - isFirstPacket := true - - for _, network := range networksTarget { - // Do not mix link-local unicast targets with non link-local networks (only when iface is known, i.e. not catch all local) - if network.iface != nil && remote.IP.IsLinkLocalUnicast() != network.address.IP.IsLinkLocalUnicast() { - continue - } - - if sequenceData != nil { - packet.Sequence = nets.Sequences.ArbitrarySequence(receiverPublicKey, sequenceData).SequenceNumber - } - err = (&Connection{backend: nets.backend, Network: network, Address: remote, PortInternal: receiverPortInternal, traversePeer: traversePeer, Firewall: receiverFirewall}).send(packet, receiverPublicKey, isFirstPacket) - isFirstPacket = false - - if err == nil { - successCount++ - } - } - - if successCount == 0 { - return errors.New("no successful send") - } - - return nil -} - -// send sends a raw packet to the peer. Only uses active connections. -func (peer *PeerInfo) sendLite(raw []byte) (err error) { - if peer.isVirtual { // special case for peers that were not contacted before - return errors.New("cannot send lite packet to virtual peer") - } else if len(peer.connectionActive) == 0 { - return errors.New("no valid connection to peer") - } else if atomic.LoadUint64(&peer.StatsPacketSent) == 0 && atomic.LoadUint64(&peer.StatsPacketReceived) == 0 { - return errors.New("uncontacted peer") // A valid connection must have been established. - } - - // always count as one sent packet even if sent via broadcast - atomic.AddUint64(&peer.StatsPacketSent, 1) - - // Send out the wire. Use connectionLatest if available. - cLatest := peer.connectionLatest - if cLatest != nil { - if err := cLatest.Network.send(cLatest.Address.IP, cLatest.Address.Port, raw); err == nil { - return nil - } else if IsNetworkErrorFatal(err) { - // Invalid connection, immediately invalidate. Fallback to broadcast to all other active ones. - // Windows: A common error when the network adapter is disabled is "wsasendto: The requested address is not valid in its context". - peer.invalidateActiveConnection(cLatest) - } - } - - // If no latest connection available, broadcast on all other available connections. - for _, c := range peer.GetConnections(true) { - if c == cLatest { - continue - } - - if err := c.Network.send(c.Address.IP, c.Address.Port, raw); err != nil && IsNetworkErrorFatal(err) { - peer.invalidateActiveConnection(c) - } - } - - return nil // on broadcast no error is known and returned -} +/* +File Name: Connection.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "errors" + "net" + "sync/atomic" + "time" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" +) + +// Connection is an established connection between a remote IP address and a local network adapter. +// New connections may only be created in case of successful INCOMING packets. +type Connection struct { + Network *Network // Network which received the packet. + Address *net.UDPAddr // Address of the remote peer. + PortInternal uint16 // Internal listening port reported by remote peer. 0 if no Announcement/Response message was yet received. + PortExternal uint16 // External listening port reported by remote peer. 0 if not known by the peer. + LastPacketIn time.Time // Last time an incoming packet was received. + LastPacketOut time.Time // Last time an outgoing packet was attempted to send. + LastPingOut time.Time // Last ping out. + Expires time.Time // Inactive connections only: Expiry date. If it does not become active by that date, it will be considered expired and removed. + Status int // 0 = Active established connection, 1 = Inactive, 2 = Removed, 3 = Redundant + RoundTripTime time.Duration // Full round-trip time of last reply. + Firewall bool // Whether the remote peer indicates a potential firewall. This means a Traverse message shall be sent to establish a connection. + traversePeer *PeerInfo // Temporary peer that may act as proxy for a Traverse message used for the first packet. This is used to establish this Connection to a peer that is behind a NAT or firewall. + backend *Backend +} + +// Connection status +const ( + ConnectionActive = iota + ConnectionInactive + ConnectionRemoved + ConnectionRedundant // Same as active. Incoming packets are accepted. Outgoing use only for redundancy. Reduces ping overhead. +) + +// Equal checks if the connection was established other the same network adapter using the same IP address. Port is intentionally not checked. +func (c *Connection) Equal(other *Connection) bool { + return c.Address.IP.Equal(other.Address.IP) && c.Network.address.IP.Equal(other.Network.address.IP) +} + +// IsLocal checks if the connection is a local network one (LAN) +func (c *Connection) IsLocal() bool { + return c != nil && IsIPLocal(c.Address.IP) +} + +// IsIPv4 checks if the connection is using IPv4 +func (c *Connection) IsIPv4() bool { + return IsIPv4(c.Address.IP) +} + +// IsIPv6 checks if the connection is using IPv6 +func (c *Connection) IsIPv6() bool { + return IsIPv6(c.Address.IP) +} + +// IsBehindNAT checks if the remote peer on the connection is likely behind a NAT +func (c *Connection) IsBehindNAT() bool { + return c.PortInternal > 0 && c.Address.Port != int(c.PortInternal) +} + +// IsPortForward checks if the remote peer uses port forwarding on the connection +func (c *Connection) IsPortForward() bool { + return c.PortExternal > 0 +} + +// IsVirtual returns true if the peer has not been connected yet. This is the case if another peer responds with peer details, and that peer shall be contacted. +func (peer *PeerInfo) IsVirtual() bool { + return peer.isVirtual +} + +// GetConnections returns the list of connections +func (peer *PeerInfo) GetConnections(active bool) (connections []*Connection) { + peer.RLock() + defer peer.RUnlock() + + if active { + return peer.connectionActive + } + return peer.connectionInactive +} + +// IsConnectionActive checks if the peer has an active connection that can be used to send and receive messages +func (peer *PeerInfo) IsConnectionActive() bool { + peer.RLock() + defer peer.RUnlock() + + return len(peer.connectionActive) > 0 +} + +// IsConnectable checks if the peer is connectable to the given IP parameters. +func (peer *PeerInfo) IsConnectable(allowLocal, allowIPv4, allowIPv6 bool) bool { + peer.RLock() + defer peer.RUnlock() + + // Only 1 active connection must be allowed for being connectable. + for _, connection := range peer.connectionActive { + // If the internal port is not known, which happens if no Announcement or Response was returned, do not share the peer details. + // This can happen if only other messages such as Ping/Pong were received, or the protocol implementation is not compatible. The external port is also likely not available. + // In this case sharing the peer would be bad, since the receiving peer could not use internal/external port to detemine the NAT status and port forwarding. + if connection.PortInternal == 0 { + continue + } + + if IsIPv4(connection.Address.IP) && allowIPv4 || IsIPv6(connection.Address.IP) && allowIPv6 { + if !(!allowLocal && connection.IsLocal()) { + return true + } + } + } + + return false +} + +// GetConnection2Share returns a connection to share. Nil if none. +// allowLocal specifies whether it is OK to return local IPs. +func (peer *PeerInfo) GetConnection2Share(allowLocal, allowIPv4, allowIPv6 bool) (connection *Connection) { + if !allowLocal && !allowIPv4 && !allowIPv6 { + return nil + } + + peer.RLock() + defer peer.RUnlock() + + if peer.connectionLatest != nil && !(!allowLocal && peer.connectionLatest.IsLocal()) && + (IsIPv4(peer.connectionLatest.Address.IP) && allowIPv4 || IsIPv6(peer.connectionLatest.Address.IP) && allowIPv6) && peer.connectionLatest.PortInternal > 0 { + return peer.connectionLatest + } + + for _, connection := range peer.connectionActive { + if (IsIPv4(connection.Address.IP) && allowIPv4 || IsIPv6(connection.Address.IP) && allowIPv6) && !(!allowLocal && connection.IsLocal()) && connection.PortInternal > 0 { + return connection + } + } + + return nil +} + +// registerConnection registers an incoming connection for an existing peer. If new, it will add to the list. If previously inactive, it will elevate. +func (peer *PeerInfo) registerConnection(incoming *Connection) (result *Connection) { + peer.Lock() + defer peer.Unlock() + + // first check if already an active connection to the same IP + for _, connection := range peer.connectionActive { + if connection.Equal(incoming) { + // Connection already established. Verify port and update if necessary. + // Some NATs may rotate ports. Some mobile phone providers even rotate IPs which is not detected here. + if connection.Address.Port != incoming.Address.Port { + connection.Address.Port = incoming.Address.Port + } + + connection.Status = ConnectionActive + peer.setConnectionLatest(connection) + return connection + } + } + + // if an inactive connection, elevate it to activated one + for n, connection := range peer.connectionInactive { + if connection.Equal(incoming) { + if connection.Address.Port != incoming.Address.Port { + connection.Address.Port = incoming.Address.Port + } + + // elevate by adding to active and mark as latest active + connection.Status = ConnectionActive + peer.connectionActive = append(peer.connectionActive, connection) + peer.setConnectionLatest(connection) + + // remove from inactive + inactiveNew := peer.connectionInactive[:n] + if n < len(peer.connectionInactive)-1 { + inactiveNew = append(inactiveNew, peer.connectionInactive[n+1:]...) + } + peer.connectionInactive = inactiveNew + + return connection + } + } + + // otherwise it is a new connection! + peer.connectionActive = append(peer.connectionActive, incoming) + peer.setConnectionLatest(incoming) + + peer.Backend.Filters.NewPeerConnection(peer, incoming) + + return incoming +} + +// setConnectionLatest updates the latest valid connection to use for sending. All other connections will be changed to redundant, which reduces ping overhead. +func (peer *PeerInfo) setConnectionLatest(latest *Connection) { + if peer.connectionLatest == latest { + return + } + + peer.connectionLatest = latest + + for _, connection := range peer.connectionActive { + if connection == latest { + continue + } + connection.Status = ConnectionRedundant + } +} + +// invalidateActiveConnection invalidates an active connection +func (peer *PeerInfo) invalidateActiveConnection(input *Connection) { + peer.Lock() + defer peer.Unlock() + + // Change the status to inactive and start the expiration. If the connection does not become valid by that date, it will be removed. + input.Status = ConnectionInactive + input.Expires = time.Now().Add(connectionRemove * time.Second) + + // remove from connectionLatest if selected so it won't be used by standard send function + if peer.connectionLatest == input { + peer.connectionLatest = nil + } + + for n, connection := range peer.connectionActive { + if connection == input { + // add to list of inactive connections + peer.connectionInactive = append(peer.connectionInactive, connection) + + // remove from active + activeNew := peer.connectionActive[:n] + if n < len(peer.connectionActive)-1 { + activeNew = append(activeNew, peer.connectionActive[n+1:]...) + } + peer.connectionActive = activeNew + + break + } + } +} + +// removeInactiveConnection removes an inactive connection. +func (peer *PeerInfo) removeInactiveConnection(input *Connection) { + peer.Lock() + defer peer.Unlock() + + input.Status = ConnectionRemoved + + for n, connection := range peer.connectionInactive { + if connection == input { + + // remove from inactive + inactiveNew := peer.connectionInactive[:n] + if n < len(peer.connectionInactive)-1 { + inactiveNew = append(inactiveNew, peer.connectionInactive[n+1:]...) + } + peer.connectionInactive = inactiveNew + + return + } + } +} + +// GetRTT returns the round-trip time for the most recent active connection. 0 if not available. +func (peer *PeerInfo) GetRTT() (rtt time.Duration) { + peer.Lock() + defer peer.Unlock() + + if peer.connectionLatest != nil && peer.connectionLatest.RoundTripTime > 0 { + return peer.connectionLatest.RoundTripTime + } + + for _, connection := range peer.connectionActive { + if connection.RoundTripTime > 0 { + return connection.RoundTripTime + } + } + + return 0 +} + +// IsBehindNAT checks if the peer is behind NAT +func (peer *PeerInfo) IsBehindNAT() (result bool) { + peer.Lock() + defer peer.Unlock() + + // Default is no. Only if a public network reports different connected port vs internal one, NAT is assumed. + // This also assumes that all 3rd party clients bind their connection to the outgoing port. + // PortInternal is 0 if no Announcement or Response message was received. + + for _, connection := range peer.connectionActive { + if connection.IsBehindNAT() { + return true + } + } + + for _, connection := range peer.connectionInactive { + if connection.IsBehindNAT() { + return true + } + } + + return false +} + +// IsPortForward checks if the peer uses port forwarding +func (peer *PeerInfo) IsPortForward() (result bool) { + peer.Lock() + defer peer.Unlock() + + for _, connection := range peer.connectionActive { + if connection.IsPortForward() { + return true + } + } + + for _, connection := range peer.connectionInactive { + if connection.IsPortForward() { + return true + } + } + + return false +} + +// IsFirewallReported checks if the peer reported to be behind a firewall +func (peer *PeerInfo) IsFirewallReported() (result bool) { + return peer.Features&(1< 0 +} + +// ---- sending code ---- + +// send sends the packet to the peer on the connection +// isFirstPacket indicates whether this is the first packet to an uncontacted peer. +func (c *Connection) send(packet *protocol.PacketRaw, receiverPublicKey *btcec.PublicKey, isFirstPacket bool) (err error) { + if c == nil { + return errors.New("invalid connection") + } + + packet.Protocol = protocol.ProtocolVersion + packet.SetSelfReportedPorts(c.Network.SelfReportedPorts()) + + c.backend.Filters.PacketOut(packet, receiverPublicKey, c) + + raw, err := protocol.PacketEncrypt(c.backend.PeerPrivateKey, receiverPublicKey, packet) + if err != nil { + return err + } + + c.LastPacketOut = time.Now() + + err = c.Network.send(c.Address.IP, c.Address.Port, raw) + + // Send Traverse message if the peer is behind a NAT or firewall and this is the first message. Only for Announcement. + if err == nil && isFirstPacket && (c.IsBehindNAT() || c.Firewall) && c.traversePeer != nil && packet.Command == protocol.CommandAnnouncement { + c.traversePeer.sendTraverse(packet, receiverPublicKey) + } + + return err +} + +// send sends a raw packet to the peer. Only uses active connections. +func (peer *PeerInfo) send(packet *protocol.PacketRaw) (err error) { + if peer.isVirtual { // special case for peers that were not contacted before + for _, address := range peer.targetAddresses { + peer.Backend.networks.sendAllNetworks(peer.PublicKey, packet, &net.UDPAddr{IP: address.IP, Port: int(address.Port)}, address.PortInternal, peer.Features&(1< 0, peer.traversePeer, nil) + } + return + } + if len(peer.connectionActive) == 0 { + return errors.New("no valid connection to peer") + } + + // For Traverse: check if no packet has been sent, and none received (i.e. initial contact). + // If a packet was already received directly (note: not via incoming traversed message), a valid connection is already established. + isFirstPacketOut := atomic.LoadUint64(&peer.StatsPacketSent) == 0 && atomic.LoadUint64(&peer.StatsPacketReceived) == 0 + + // always count as one sent packet even if sent via broadcast + atomic.AddUint64(&peer.StatsPacketSent, 1) + + // Send out the wire. Use connectionLatest if available. + // Failover: If sending fails and there are other connections available, try those. Automatically update connectionLatest if one is successful. + // Windows: This works great in case the adapter gets disabled, however, does not detect if the network cable is unplugged. + cLatest := peer.connectionLatest + if cLatest != nil { + if err := cLatest.send(packet, peer.PublicKey, isFirstPacketOut); err == nil { + return nil + } else if IsNetworkErrorFatal(err) { + // Invalid connection, immediately invalidate. Fallback to broadcast to all other active ones. + // Windows: A common error when the network adapter is disabled is "wsasendto: The requested address is not valid in its context". + peer.invalidateActiveConnection(cLatest) + } + } + + // If no latest connection available, broadcast on all other available connections. + // This might be noisy, but if no latest connection is available it means the last established connection is already considered dead. + // The receiver is responsible for incoming deduplication of packets. + activeConnections := peer.GetConnections(true) + for _, c := range activeConnections { + if c == cLatest { + continue + } + + if err := c.send(packet, peer.PublicKey, isFirstPacketOut); err != nil && IsNetworkErrorFatal(err) { + peer.invalidateActiveConnection(c) + } + } + + return nil // on broadcast no error is known and returned +} + +// sendConnection sends a packet to the peer using the specific connection +func (peer *PeerInfo) sendConnection(packet *protocol.PacketRaw, connection *Connection) (err error) { + isFirstPacketOut := atomic.LoadUint64(&peer.StatsPacketSent) == 0 && atomic.LoadUint64(&peer.StatsPacketReceived) == 0 + atomic.AddUint64(&peer.StatsPacketSent, 1) + + return connection.send(packet, peer.PublicKey, isFirstPacketOut) +} + +// sendAllNetworks sends a raw packet via all networks. It assigns a new sequence for each sent packet. +// receiverPortInternal is important for NAT detection and sending the traverse message. Firewall indicates whether the remote peer was reported to be behind a firewall. +func (nets *Networks) sendAllNetworks(receiverPublicKey *btcec.PublicKey, packet *protocol.PacketRaw, remote *net.UDPAddr, receiverPortInternal uint16, receiverFirewall bool, traversePeer *PeerInfo, sequenceData interface{}) (err error) { + nets.RLock() + defer nets.RUnlock() + + networksTarget := nets.networks4 + if IsIPv6(remote.IP.To16()) { + networksTarget = nets.networks6 + } + + successCount := 0 + isFirstPacket := true + + for _, network := range networksTarget { + // Do not mix link-local unicast targets with non link-local networks (only when iface is known, i.e. not catch all local) + if network.iface != nil && remote.IP.IsLinkLocalUnicast() != network.address.IP.IsLinkLocalUnicast() { + continue + } + + if sequenceData != nil { + packet.Sequence = nets.Sequences.ArbitrarySequence(receiverPublicKey, sequenceData).SequenceNumber + } + err = (&Connection{backend: nets.backend, Network: network, Address: remote, PortInternal: receiverPortInternal, traversePeer: traversePeer, Firewall: receiverFirewall}).send(packet, receiverPublicKey, isFirstPacket) + isFirstPacket = false + + if err == nil { + successCount++ + } + } + + if successCount == 0 { + return errors.New("no successful send") + } + + return nil +} + +// send sends a raw packet to the peer. Only uses active connections. +func (peer *PeerInfo) sendLite(raw []byte) (err error) { + if peer.isVirtual { // special case for peers that were not contacted before + return errors.New("cannot send lite packet to virtual peer") + } else if len(peer.connectionActive) == 0 { + return errors.New("no valid connection to peer") + } else if atomic.LoadUint64(&peer.StatsPacketSent) == 0 && atomic.LoadUint64(&peer.StatsPacketReceived) == 0 { + return errors.New("uncontacted peer") // A valid connection must have been established. + } + + // always count as one sent packet even if sent via broadcast + atomic.AddUint64(&peer.StatsPacketSent, 1) + + // Send out the wire. Use connectionLatest if available. + cLatest := peer.connectionLatest + if cLatest != nil { + if err := cLatest.Network.send(cLatest.Address.IP, cLatest.Address.Port, raw); err == nil { + return nil + } else if IsNetworkErrorFatal(err) { + // Invalid connection, immediately invalidate. Fallback to broadcast to all other active ones. + // Windows: A common error when the network adapter is disabled is "wsasendto: The requested address is not valid in its context". + peer.invalidateActiveConnection(cLatest) + } + } + + // If no latest connection available, broadcast on all other available connections. + for _, c := range peer.GetConnections(true) { + if c == cLatest { + continue + } + + if err := c.Network.send(c.Address.IP, c.Address.Port, raw); err != nil && IsNetworkErrorFatal(err) { + peer.invalidateActiveConnection(c) + } + } + + return nil // on broadcast no error is known and returned +} diff --git a/DHT Store.go b/DHT Store.go index be44d62..d420772 100644 --- a/DHT Store.go +++ b/DHT Store.go @@ -1,48 +1,48 @@ -/* -File Name: DHT Store.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/store" -) - -// TODO: Via descriptors, files stored by other peers - -func (backend *Backend) initStore() { - backend.dhtStore = store.NewMemoryStore() -} - -// announcementGetData returns data for an announcement -func (peer *PeerInfo) announcementGetData(hash []byte) (stored bool, data []byte) { - // TODO: Create RetrieveIfSize to prevent files larger than EmbeddedFileSizeMax from being loaded - data, found := peer.Backend.dhtStore.Get(hash) - if !found { - return false, nil - } - - if len(data) <= protocol.EmbeddedFileSizeMax { - return true, data - } - - return true, nil -} - -// announcementStore handles an incoming announcement by another peer about storing data -func (peer *PeerInfo) announcementStore(records []protocol.InfoStore) { - // TODO: Only store the other peers data if certain conditions are met: - // - enough storage available - // - not exceeding record count per peer - // - not exceeding total record count limit - // - not exceeding record count per CIDR - //for _, record := range records { - //fmt.Printf("Remote node %s stores hash %s (size %d type %d)\n", hex.EncodeToString(peer.NodeID), hex.EncodeToString(record.ID.Hash), record.Size, record.Type) - //Warehouse.Store(record.ID.Hash, ) - // TODO: Request data from remote node. - //peer.sendAnnouncement(false, false, nil, []KeyHash{record.ID}, nil, nil) - //} -} +/* +File Name: DHT Store.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/store" +) + +// TODO: Via descriptors, files stored by other peers + +func (backend *Backend) initStore() { + backend.dhtStore = store.NewMemoryStore() +} + +// announcementGetData returns data for an announcement +func (peer *PeerInfo) announcementGetData(hash []byte) (stored bool, data []byte) { + // TODO: Create RetrieveIfSize to prevent files larger than EmbeddedFileSizeMax from being loaded + data, found := peer.Backend.dhtStore.Get(hash) + if !found { + return false, nil + } + + if len(data) <= protocol.EmbeddedFileSizeMax { + return true, data + } + + return true, nil +} + +// announcementStore handles an incoming announcement by another peer about storing data +func (peer *PeerInfo) announcementStore(records []protocol.InfoStore) { + // TODO: Only store the other peers data if certain conditions are met: + // - enough storage available + // - not exceeding record count per peer + // - not exceeding total record count limit + // - not exceeding record count per CIDR + //for _, record := range records { + //fmt.Printf("Remote node %s stores hash %s (size %d type %d)\n", hex.EncodeToString(peer.NodeID), hex.EncodeToString(record.ID.Hash), record.Size, record.Type) + //Warehouse.Store(record.ID.Hash, ) + // TODO: Request data from remote node. + //peer.sendAnnouncement(false, false, nil, []KeyHash{record.ID}, nil, nil) + //} +} diff --git a/Exit.go b/Exit.go index 727091e..ae59168 100644 --- a/Exit.go +++ b/Exit.go @@ -1,24 +1,24 @@ -/* -File Name: Exit.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -// Exit codes signal why the application exited. These are universal between clients developed by the Peernet organization. -// Clients are encouraged to log additional details in a log file. 3rd party clients may define additional exit codes. -const ( - ExitSuccess = 0 // This is actually never used. - ExitErrorConfigAccess = 1 // Error accessing the config file. - ExitErrorConfigRead = 2 // Error reading the config file. - ExitErrorConfigParse = 3 // Error parsing the config file. - ExitErrorLogInit = 4 // Error initializing log file. - ExitParamWebapiInvalid = 5 // Parameter for webapi is invalid. - ExitPrivateKeyCorrupt = 6 // Private key is corrupt. - ExitPrivateKeyCreate = 7 // Cannot create a new private key. - ExitBlockchainCorrupt = 8 // Blockchain is corrupt. - ExitGraceful = 9 // Graceful shutdown. - ExitParamApiKeyInvalid = 10 // API key parameter is invalid. - STATUS_CONTROL_C_EXIT = 0xC000013A // The application terminated as a result of a CTRL+C. This is a Windows NTSTATUS value. -) +/* +File Name: Exit.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +// Exit codes signal why the application exited. These are universal between clients developed by the Peernet organization. +// Clients are encouraged to log additional details in a log file. 3rd party clients may define additional exit codes. +const ( + ExitSuccess = 0 // This is actually never used. + ExitErrorConfigAccess = 1 // Error accessing the config file. + ExitErrorConfigRead = 2 // Error reading the config file. + ExitErrorConfigParse = 3 // Error parsing the config file. + ExitErrorLogInit = 4 // Error initializing log file. + ExitParamWebapiInvalid = 5 // Parameter for webapi is invalid. + ExitPrivateKeyCorrupt = 6 // Private key is corrupt. + ExitPrivateKeyCreate = 7 // Cannot create a new private key. + ExitBlockchainCorrupt = 8 // Blockchain is corrupt. + ExitGraceful = 9 // Graceful shutdown. + ExitParamApiKeyInvalid = 10 // API key parameter is invalid. + STATUS_CONTROL_C_EXIT = 0xC000013A // The application terminated as a result of a CTRL+C. This is a Windows NTSTATUS value. +) diff --git a/File Formats.go b/File Formats.go index 69638dd..3d717dd 100644 --- a/File Formats.go +++ b/File Formats.go @@ -1,49 +1,49 @@ -/* -File Name: File Formats.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Definition of all recognized file formats. This file is likely being updated more frequently than regular code. -*/ - -package core - -// General content types of data. -const ( - TypeBinary = iota // Binary/unspecified - TypeText // Plain text - TypePicture // Picture of any format - TypeVideo // Video - TypeAudio // Audio - TypeDocument // Any document file, including office documents, PDFs, power point, spreadsheets - TypeExecutable // Any executable file, OS independent - TypeContainer // Container files like ZIP, RAR, TAR, ISO - TypeCompressed // Compressed files like GZ, BZ - TypeFolder // Virtual folder - TypeEbook // Ebook -) - -// File formats. New ones may be added to the list as required. -const ( - FormatBinary = iota // Binary/unspecified - FormatPDF // PDF document - FormatWord // Word document - FormatExcel // Excel - FormatPowerpoint // Powerpoint - FormatPicture // Pictures (including GIF, excluding icons) - FormatAudio // Audio files - FormatVideo // Video files - FormatContainer // Compressed files including ZIP, RAR, TAR and others - FormatHTML // HTML file - FormatText // Text file - FormatEbook // Ebook file - FormatCompressed // Compressed file - FormatDatabase // Database file - FormatEmail // Single email - FormatCSV // CSV file - FormatFolder // Virtual folder - FormatExecutable // Executable file - FormatInstaller // Installer - FormatAPK // APK - FormatISO // ISO -) +/* +File Name: File Formats.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Definition of all recognized file formats. This file is likely being updated more frequently than regular code. +*/ + +package core + +// General content types of data. +const ( + TypeBinary = iota // Binary/unspecified + TypeText // Plain text + TypePicture // Picture of any format + TypeVideo // Video + TypeAudio // Audio + TypeDocument // Any document file, including office documents, PDFs, power point, spreadsheets + TypeExecutable // Any executable file, OS independent + TypeContainer // Container files like ZIP, RAR, TAR, ISO + TypeCompressed // Compressed files like GZ, BZ + TypeFolder // Virtual folder + TypeEbook // Ebook +) + +// File formats. New ones may be added to the list as required. +const ( + FormatBinary = iota // Binary/unspecified + FormatPDF // PDF document + FormatWord // Word document + FormatExcel // Excel + FormatPowerpoint // Powerpoint + FormatPicture // Pictures (including GIF, excluding icons) + FormatAudio // Audio files + FormatVideo // Video files + FormatContainer // Compressed files including ZIP, RAR, TAR and others + FormatHTML // HTML file + FormatText // Text file + FormatEbook // Ebook file + FormatCompressed // Compressed file + FormatDatabase // Database file + FormatEmail // Single email + FormatCSV // CSV file + FormatFolder // Virtual folder + FormatExecutable // Executable file + FormatInstaller // Installer + FormatAPK // APK + FormatISO // ISO +) diff --git a/Filter.go b/Filter.go index e825b8b..5a4cf57 100644 --- a/Filter.go +++ b/Filter.go @@ -1,164 +1,164 @@ -/* -File Name: Filter.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Filters allow the caller to intercept events. The filter functions must not modify any data. -*/ - -package core - -import ( - "io" - "sync" - - "github.com/PeernetOfficial/core/blockchain" - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/dht" - "github.com/PeernetOfficial/core/protocol" - "github.com/google/uuid" -) - -// Filters contains all functions to install the hook. Use nil for unused. -// The functions are called sequentially and block execution; if the filter takes a long time it should start a Go routine. -type Filters struct { - // NewPeer is called every time a new peer, that is one that is not currently in the peer list. - // Note that peers might be removed from peer lists and reappear quickly, i.e. this function may be called multiple times for the same peers. - // The filter must maintain its own map of unique peer IDs if actual uniqueness of new peers is desired. - NewPeer func(peer *PeerInfo, connection *Connection) - - // NewPeerConnection is called for each new established connection to a peer. Note that connections might be dropped and reconnected at anytime. - NewPeerConnection func(peer *PeerInfo, connection *Connection) - - // LogError is called for any error. - LogError func(function, format string, v ...interface{}) - - // DHTSearchStatus is called with updates of searches in the DHT. It allows to see the live progress of searches. - DHTSearchStatus func(client *dht.SearchClient, function, format string, v ...interface{}) - - // IncomingRequest receives all incoming information requests. The action field is set accordingly. - IncomingRequest func(peer *PeerInfo, Action int, Key []byte, Info interface{}) - - // PacketIn is a low-level filter for incoming packets after they are decrypted. - // Traverse messages are not covered. - PacketIn func(packet *protocol.PacketRaw, senderPublicKey *btcec.PublicKey, connection *Connection) - - // PacketOut is a low-level filter for outgoing packets before they are encrypted. - // IPv4 broadcast, IPv6 multicast, and Traverse messages are not covered. - PacketOut func(packet *protocol.PacketRaw, receiverPublicKey *btcec.PublicKey, connection *Connection) - - // MessageIn is a high-level filter for decoded incoming messages. message is of type nil, MessageAnnouncement, MessageResponse, or MessageTraverse - MessageIn func(peer *PeerInfo, raw *protocol.MessageRaw, message interface{}) - - // MessageOutAnnouncement is a high-level filter for outgoing announcements. Peer is nil on first contact. - // Broadcast and Multicast messages are not covered. - MessageOutAnnouncement func(receiverPublicKey *btcec.PublicKey, peer *PeerInfo, packet *protocol.PacketRaw, findSelf bool, findPeer []protocol.KeyHash, findValue []protocol.KeyHash, files []protocol.InfoStore) - - // MessageOutResponse is a high-level filter for outgoing responses. - MessageOutResponse func(peer *PeerInfo, packet *protocol.PacketRaw, hash2Peers []protocol.Hash2Peer, filesEmbed []protocol.EmbeddedFileData, hashesNotFound [][]byte) - - // MessageOutTraverse is a high-level filter for outgoing traverse messages. - MessageOutTraverse func(peer *PeerInfo, packet *protocol.PacketRaw, embeddedPacket *protocol.PacketRaw, receiverEnd *btcec.PublicKey) - - // MessageOutPing is a high-level filter for outgoing pings. - MessageOutPing func(peer *PeerInfo, packet *protocol.PacketRaw, connection *Connection) - - // MessageOutPong is a high-level filter for outgoing pongs. - MessageOutPong func(peer *PeerInfo, packet *protocol.PacketRaw) - - // Called when the statistics change of a single blockchain in the cache. Must be set on init. - GlobalBlockchainCacheStatistic func(multi *blockchain.MultiStore, header *blockchain.MultiBlockchainHeader, statsOld blockchain.BlockchainStats) - - // Called after a blockchain is deleted from the blockchain cache. The header reflects the status before deletion. Must be set on init. - GlobalBlockchainCacheDelete func(multi *blockchain.MultiStore, header *blockchain.MultiBlockchainHeader) -} - -func (backend *Backend) initFilters() { - // Set default filters to blank functions so they can be safely called without constant nil checks. - // Only if not already set before init. - - if backend.Filters.NewPeer == nil { - backend.Filters.NewPeer = func(peer *PeerInfo, connection *Connection) {} - } - if backend.Filters.NewPeerConnection == nil { - backend.Filters.NewPeerConnection = func(peer *PeerInfo, connection *Connection) {} - } - if backend.Filters.DHTSearchStatus == nil { - backend.Filters.DHTSearchStatus = func(client *dht.SearchClient, function, format string, v ...interface{}) {} - } - if backend.Filters.LogError == nil { - backend.Filters.LogError = func(function, format string, v ...interface{}) {} - } - if backend.Filters.IncomingRequest == nil { - backend.Filters.IncomingRequest = func(peer *PeerInfo, Action int, Key []byte, Info interface{}) {} - } - if backend.Filters.PacketIn == nil { - backend.Filters.PacketIn = func(packet *protocol.PacketRaw, senderPublicKey *btcec.PublicKey, c *Connection) {} - } - if backend.Filters.PacketOut == nil { - backend.Filters.PacketOut = func(packet *protocol.PacketRaw, receiverPublicKey *btcec.PublicKey, c *Connection) {} - } - if backend.Filters.MessageIn == nil { - backend.Filters.MessageIn = func(peer *PeerInfo, raw *protocol.MessageRaw, message interface{}) {} - } - if backend.Filters.MessageOutAnnouncement == nil { - backend.Filters.MessageOutAnnouncement = func(receiverPublicKey *btcec.PublicKey, peer *PeerInfo, packet *protocol.PacketRaw, findSelf bool, findPeer []protocol.KeyHash, findValue []protocol.KeyHash, files []protocol.InfoStore) { - } - } - if backend.Filters.MessageOutResponse == nil { - backend.Filters.MessageOutResponse = func(peer *PeerInfo, packet *protocol.PacketRaw, hash2Peers []protocol.Hash2Peer, filesEmbed []protocol.EmbeddedFileData, hashesNotFound [][]byte) { - } - } - if backend.Filters.MessageOutTraverse == nil { - backend.Filters.MessageOutTraverse = func(peer *PeerInfo, packet *protocol.PacketRaw, embeddedPacket *protocol.PacketRaw, receiverEnd *btcec.PublicKey) { - } - } - if backend.Filters.MessageOutPing == nil { - backend.Filters.MessageOutPing = func(peer *PeerInfo, packet *protocol.PacketRaw, connection *Connection) {} - } - if backend.Filters.MessageOutPong == nil { - backend.Filters.MessageOutPong = func(peer *PeerInfo, packet *protocol.PacketRaw) {} - } -} - -// MultiWriter code that allows to subscribe/unsubscribe. -type multiWriter struct { - writers map[uuid.UUID]io.Writer - sync.Mutex -} - -// Creates a new writer that duplicates its writes to all the subscribed writers. -// Each write is written to each subscribed writer, one at a time. If any writer returns an error, the entire write operation continues. -func newMultiWriter() *multiWriter { - return &multiWriter{writers: make(map[uuid.UUID]io.Writer)} -} - -// Subscribe a new writer to the list of writers -func (m *multiWriter) Subscribe(writer io.Writer) (id uuid.UUID) { - m.Lock() - defer m.Unlock() - - id = uuid.New() - m.writers[id] = writer - - return id -} - -// Unsubscribe a writer from the list of writers -func (m *multiWriter) Unsubscribe(id uuid.UUID) { - m.Lock() - defer m.Unlock() - - delete(m.writers, id) -} - -// Write a slice of byte to each of the subscribed writers. It will not return any errors. -func (m *multiWriter) Write(p []byte) (n int, err error) { - m.Lock() - defer m.Unlock() - - for _, w := range m.writers { - w.Write(p) - } - return len(p), nil -} +/* +File Name: Filter.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Filters allow the caller to intercept events. The filter functions must not modify any data. +*/ + +package core + +import ( + "io" + "sync" + + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/dht" + "github.com/PeernetOfficial/core/protocol" + "github.com/google/uuid" +) + +// Filters contains all functions to install the hook. Use nil for unused. +// The functions are called sequentially and block execution; if the filter takes a long time it should start a Go routine. +type Filters struct { + // NewPeer is called every time a new peer, that is one that is not currently in the peer list. + // Note that peers might be removed from peer lists and reappear quickly, i.e. this function may be called multiple times for the same peers. + // The filter must maintain its own map of unique peer IDs if actual uniqueness of new peers is desired. + NewPeer func(peer *PeerInfo, connection *Connection) + + // NewPeerConnection is called for each new established connection to a peer. Note that connections might be dropped and reconnected at anytime. + NewPeerConnection func(peer *PeerInfo, connection *Connection) + + // LogError is called for any error. + LogError func(function, format string, v ...interface{}) + + // DHTSearchStatus is called with updates of searches in the DHT. It allows to see the live progress of searches. + DHTSearchStatus func(client *dht.SearchClient, function, format string, v ...interface{}) + + // IncomingRequest receives all incoming information requests. The action field is set accordingly. + IncomingRequest func(peer *PeerInfo, Action int, Key []byte, Info interface{}) + + // PacketIn is a low-level filter for incoming packets after they are decrypted. + // Traverse messages are not covered. + PacketIn func(packet *protocol.PacketRaw, senderPublicKey *btcec.PublicKey, connection *Connection) + + // PacketOut is a low-level filter for outgoing packets before they are encrypted. + // IPv4 broadcast, IPv6 multicast, and Traverse messages are not covered. + PacketOut func(packet *protocol.PacketRaw, receiverPublicKey *btcec.PublicKey, connection *Connection) + + // MessageIn is a high-level filter for decoded incoming messages. message is of type nil, MessageAnnouncement, MessageResponse, or MessageTraverse + MessageIn func(peer *PeerInfo, raw *protocol.MessageRaw, message interface{}) + + // MessageOutAnnouncement is a high-level filter for outgoing announcements. Peer is nil on first contact. + // Broadcast and Multicast messages are not covered. + MessageOutAnnouncement func(receiverPublicKey *btcec.PublicKey, peer *PeerInfo, packet *protocol.PacketRaw, findSelf bool, findPeer []protocol.KeyHash, findValue []protocol.KeyHash, files []protocol.InfoStore) + + // MessageOutResponse is a high-level filter for outgoing responses. + MessageOutResponse func(peer *PeerInfo, packet *protocol.PacketRaw, hash2Peers []protocol.Hash2Peer, filesEmbed []protocol.EmbeddedFileData, hashesNotFound [][]byte) + + // MessageOutTraverse is a high-level filter for outgoing traverse messages. + MessageOutTraverse func(peer *PeerInfo, packet *protocol.PacketRaw, embeddedPacket *protocol.PacketRaw, receiverEnd *btcec.PublicKey) + + // MessageOutPing is a high-level filter for outgoing pings. + MessageOutPing func(peer *PeerInfo, packet *protocol.PacketRaw, connection *Connection) + + // MessageOutPong is a high-level filter for outgoing pongs. + MessageOutPong func(peer *PeerInfo, packet *protocol.PacketRaw) + + // Called when the statistics change of a single blockchain in the cache. Must be set on init. + GlobalBlockchainCacheStatistic func(multi *blockchain.MultiStore, header *blockchain.MultiBlockchainHeader, statsOld blockchain.BlockchainStats) + + // Called after a blockchain is deleted from the blockchain cache. The header reflects the status before deletion. Must be set on init. + GlobalBlockchainCacheDelete func(multi *blockchain.MultiStore, header *blockchain.MultiBlockchainHeader) +} + +func (backend *Backend) initFilters() { + // Set default filters to blank functions so they can be safely called without constant nil checks. + // Only if not already set before init. + + if backend.Filters.NewPeer == nil { + backend.Filters.NewPeer = func(peer *PeerInfo, connection *Connection) {} + } + if backend.Filters.NewPeerConnection == nil { + backend.Filters.NewPeerConnection = func(peer *PeerInfo, connection *Connection) {} + } + if backend.Filters.DHTSearchStatus == nil { + backend.Filters.DHTSearchStatus = func(client *dht.SearchClient, function, format string, v ...interface{}) {} + } + if backend.Filters.LogError == nil { + backend.Filters.LogError = func(function, format string, v ...interface{}) {} + } + if backend.Filters.IncomingRequest == nil { + backend.Filters.IncomingRequest = func(peer *PeerInfo, Action int, Key []byte, Info interface{}) {} + } + if backend.Filters.PacketIn == nil { + backend.Filters.PacketIn = func(packet *protocol.PacketRaw, senderPublicKey *btcec.PublicKey, c *Connection) {} + } + if backend.Filters.PacketOut == nil { + backend.Filters.PacketOut = func(packet *protocol.PacketRaw, receiverPublicKey *btcec.PublicKey, c *Connection) {} + } + if backend.Filters.MessageIn == nil { + backend.Filters.MessageIn = func(peer *PeerInfo, raw *protocol.MessageRaw, message interface{}) {} + } + if backend.Filters.MessageOutAnnouncement == nil { + backend.Filters.MessageOutAnnouncement = func(receiverPublicKey *btcec.PublicKey, peer *PeerInfo, packet *protocol.PacketRaw, findSelf bool, findPeer []protocol.KeyHash, findValue []protocol.KeyHash, files []protocol.InfoStore) { + } + } + if backend.Filters.MessageOutResponse == nil { + backend.Filters.MessageOutResponse = func(peer *PeerInfo, packet *protocol.PacketRaw, hash2Peers []protocol.Hash2Peer, filesEmbed []protocol.EmbeddedFileData, hashesNotFound [][]byte) { + } + } + if backend.Filters.MessageOutTraverse == nil { + backend.Filters.MessageOutTraverse = func(peer *PeerInfo, packet *protocol.PacketRaw, embeddedPacket *protocol.PacketRaw, receiverEnd *btcec.PublicKey) { + } + } + if backend.Filters.MessageOutPing == nil { + backend.Filters.MessageOutPing = func(peer *PeerInfo, packet *protocol.PacketRaw, connection *Connection) {} + } + if backend.Filters.MessageOutPong == nil { + backend.Filters.MessageOutPong = func(peer *PeerInfo, packet *protocol.PacketRaw) {} + } +} + +// MultiWriter code that allows to subscribe/unsubscribe. +type multiWriter struct { + writers map[uuid.UUID]io.Writer + sync.Mutex +} + +// Creates a new writer that duplicates its writes to all the subscribed writers. +// Each write is written to each subscribed writer, one at a time. If any writer returns an error, the entire write operation continues. +func newMultiWriter() *multiWriter { + return &multiWriter{writers: make(map[uuid.UUID]io.Writer)} +} + +// Subscribe a new writer to the list of writers +func (m *multiWriter) Subscribe(writer io.Writer) (id uuid.UUID) { + m.Lock() + defer m.Unlock() + + id = uuid.New() + m.writers[id] = writer + + return id +} + +// Unsubscribe a writer from the list of writers +func (m *multiWriter) Unsubscribe(id uuid.UUID) { + m.Lock() + defer m.Unlock() + + delete(m.writers, id) +} + +// Write a slice of byte to each of the subscribed writers. It will not return any errors. +func (m *multiWriter) Write(p []byte) (n int, err error) { + m.Lock() + defer m.Unlock() + + for _, w := range m.writers { + w.Write(p) + } + return len(p), nil +} diff --git a/Kademlia.go b/Kademlia.go index 4ad87f8..e8a377b 100644 --- a/Kademlia.go +++ b/Kademlia.go @@ -1,203 +1,203 @@ -/* -File Name: Kademlia.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "bytes" - "time" - - "github.com/PeernetOfficial/core/dht" - "github.com/PeernetOfficial/core/protocol" -) - -const alpha = 5 // Count of nodes to be contacted in parallel for finding a key -const bucketSize = 20 // Count of nodes per bucket - -func (backend *Backend) initKademlia() { - backend.nodesDHT = dht.NewDHT(&dht.Node{ID: backend.nodeID}, 256, bucketSize, alpha) - - // ShouldEvict determines whether node 1 shall be evicted in favor of node 2 - backend.nodesDHT.ShouldEvict = func(node1, node2 *dht.Node) bool { - rttOld := node1.Info.(*PeerInfo).GetRTT() - rttNew := node2.Info.(*PeerInfo).GetRTT() - - // evict the old node if the new one has a faster ping time - if rttOld == 0 { // old one has no recent RTT (happens if all connections are inactive)? - return true - } else if rttNew > 0 { - // If new RTT is smaller, evict old one. - return rttNew < rttOld - } - - // If here, none has a RTT. Keep the closer (by distance) one. - return backend.nodesDHT.IsNodeCloser(node1.ID, node2.ID) - } - - // SendRequestStore sends a store message to the remote node. I.e. asking it to store the given key-value - backend.nodesDHT.SendRequestStore = func(node *dht.Node, key []byte, dataSize uint64) { - node.Info.(*PeerInfo).sendAnnouncementStore(key, dataSize) - } - - // SendRequestFindNode sends an information request to find a particular node. nodes are the nodes to send the request to. - backend.nodesDHT.SendRequestFindNode = func(request *dht.InformationRequest) { - for _, node := range request.Nodes { - node.Info.(*PeerInfo).sendAnnouncementFindNode(request) - } - } - - // SendRequestFindValue sends an information request to find data. nodes are the nodes to send the request to. - backend.nodesDHT.SendRequestFindValue = func(request *dht.InformationRequest) { - for _, node := range request.Nodes { - node.Info.(*PeerInfo).sendAnnouncementFindValue(request) - } - } - - backend.nodesDHT.FilterSearchStatus = backend.Filters.DHTSearchStatus -} - -// autoBucketRefresh refreshes buckets every 5 minutes to meet the alpha nodes per bucket target. Force full refresh every hour. -func (backend *Backend) autoBucketRefresh() { - for minute := 5; ; minute += 5 { - time.Sleep(time.Minute * 5) - - target := alpha - if minute%60 == 0 { - target = 0 - } - - backend.nodesDHT.RefreshBuckets(target) - } -} - -// bootstrapKademlia bootstraps the Kademlia bucket list -func (backend *Backend) bootstrapKademlia() { - monitor := make(chan *PeerInfo) - backend.registerPeerMonitor(monitor) - - // Wait until there are at least 2 peers connected. - for { - <-monitor - if backend.nodesDHT.NumNodes() >= 2 { - break - } - } - - backend.unregisterPeerMonitor(monitor) - - // Refresh every 10 seconds 3 times - for n := 0; n < 3; n++ { - backend.nodesDHT.RefreshBuckets(alpha) - - time.Sleep(time.Second) - } -} - -// Future sendAnnouncementX: If it detects that announcements are sent out to the same peer within 50ms it should activate a wait-and-group scheme. - -func (peer *PeerInfo) sendAnnouncementFindNode(request *dht.InformationRequest) { - // If the key is self, send it as FIND_SELF - if bytes.Equal(request.Key, peer.Backend.nodeID) { - peer.sendAnnouncement(false, true, nil, nil, nil, request) - } else { - peer.sendAnnouncement(false, false, []protocol.KeyHash{{Hash: request.Key}}, nil, nil, request) - } -} - -func (peer *PeerInfo) sendAnnouncementFindValue(request *dht.InformationRequest) { - - findSelf := false - var findPeer []protocol.KeyHash - var findValue []protocol.KeyHash - - findValue = append(findValue, protocol.KeyHash{Hash: request.Key}) - - peer.sendAnnouncement(false, findSelf, findPeer, findValue, nil, request) -} - -func (peer *PeerInfo) sendAnnouncementStore(fileHash []byte, fileSize uint64) { - peer.sendAnnouncement(false, false, nil, nil, []protocol.InfoStore{{ID: protocol.KeyHash{Hash: fileHash}, Size: fileSize, Type: 0}}, nil) -} - -// ---- CORE DATA FUNCTIONS ---- - -// Data2Hash returns the hash for the data -func Data2Hash(data []byte) (hash []byte) { - return protocol.HashData(data) -} - -// GetData returns the requested data. It checks first the local store and then tries via DHT. -func (backend *Backend) GetData(hash []byte) (data []byte, senderNodeID []byte, found bool) { - if data, found = backend.GetDataLocal(hash); found { - return data, backend.nodeID, found - } - - return backend.GetDataDHT(hash) -} - -// GetDataLocal returns data from the local warehouse. -func (backend *Backend) GetDataLocal(hash []byte) (data []byte, found bool) { - return backend.dhtStore.Get(hash) -} - -// GetDataDHT requests data via DHT -func (backend *Backend) GetDataDHT(hash []byte) (data []byte, senderNodeID []byte, found bool) { - data, senderNodeID, found, _ = backend.nodesDHT.Get(hash) - return data, senderNodeID, found -} - -// StoreDataLocal stores data into the local warehouse. -func (backend *Backend) StoreDataLocal(data []byte) error { - key := protocol.HashData(data) - return backend.dhtStore.Set(key, data) -} - -// StoreDataDHT stores data locally and informs closestCount peers in the DHT about it. -// Remote peers may choose to keep a record (in case another peers asks) or mirror the full data. -func (backend *Backend) StoreDataDHT(data []byte, closestCount int) error { - key := protocol.HashData(data) - if err := backend.dhtStore.Set(key, data); err != nil { - return err - } - return backend.nodesDHT.Store(key, uint64(len(data)), closestCount) -} - -// ---- NODE FUNCTIONS ---- - -// IsNodeContact checks if the node is a contact in the local DHT routing table -func (backend *Backend) IsNodeContact(nodeID []byte) (node *dht.Node, peer *PeerInfo) { - node = backend.nodesDHT.IsNodeContact(nodeID) - if node == nil { - return nil, nil - } - - return node, node.Info.(*PeerInfo) -} - -// FindNode finds a node via the DHT -func (backend *Backend) FindNode(nodeID []byte, Timeout time.Duration) (node *dht.Node, peer *PeerInfo, err error) { - // first check if in mirrored node list - if peer = backend.NodelistLookup(nodeID); peer != nil { - return nil, peer, nil - } - - // Search the node via DHT. - node, err = backend.nodesDHT.FindNode(nodeID) - if node == nil { - return nil, nil, err - } - - return node, node.Info.(*PeerInfo), err -} - -// ---- Asynchronous Search ---- - -// AsyncSearch creates an async search for the given key in the DHT. -// Timeout is the total time the search may take, covering all information requests. TimeoutIR is the time an information request may take. -// Alpha is the number of concurrent requests that will be performed. -func (backend *Backend) AsyncSearch(Action int, Key []byte, Timeout, TimeoutIR time.Duration, Alpha int) (client *dht.SearchClient) { - return backend.nodesDHT.NewSearch(Action, Key, Timeout, TimeoutIR, Alpha) -} +/* +File Name: Kademlia.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "bytes" + "time" + + "github.com/PeernetOfficial/core/dht" + "github.com/PeernetOfficial/core/protocol" +) + +const alpha = 5 // Count of nodes to be contacted in parallel for finding a key +const bucketSize = 20 // Count of nodes per bucket + +func (backend *Backend) initKademlia() { + backend.nodesDHT = dht.NewDHT(&dht.Node{ID: backend.nodeID}, 256, bucketSize, alpha) + + // ShouldEvict determines whether node 1 shall be evicted in favor of node 2 + backend.nodesDHT.ShouldEvict = func(node1, node2 *dht.Node) bool { + rttOld := node1.Info.(*PeerInfo).GetRTT() + rttNew := node2.Info.(*PeerInfo).GetRTT() + + // evict the old node if the new one has a faster ping time + if rttOld == 0 { // old one has no recent RTT (happens if all connections are inactive)? + return true + } else if rttNew > 0 { + // If new RTT is smaller, evict old one. + return rttNew < rttOld + } + + // If here, none has a RTT. Keep the closer (by distance) one. + return backend.nodesDHT.IsNodeCloser(node1.ID, node2.ID) + } + + // SendRequestStore sends a store message to the remote node. I.e. asking it to store the given key-value + backend.nodesDHT.SendRequestStore = func(node *dht.Node, key []byte, dataSize uint64) { + node.Info.(*PeerInfo).sendAnnouncementStore(key, dataSize) + } + + // SendRequestFindNode sends an information request to find a particular node. nodes are the nodes to send the request to. + backend.nodesDHT.SendRequestFindNode = func(request *dht.InformationRequest) { + for _, node := range request.Nodes { + node.Info.(*PeerInfo).sendAnnouncementFindNode(request) + } + } + + // SendRequestFindValue sends an information request to find data. nodes are the nodes to send the request to. + backend.nodesDHT.SendRequestFindValue = func(request *dht.InformationRequest) { + for _, node := range request.Nodes { + node.Info.(*PeerInfo).sendAnnouncementFindValue(request) + } + } + + backend.nodesDHT.FilterSearchStatus = backend.Filters.DHTSearchStatus +} + +// autoBucketRefresh refreshes buckets every 5 minutes to meet the alpha nodes per bucket target. Force full refresh every hour. +func (backend *Backend) autoBucketRefresh() { + for minute := 5; ; minute += 5 { + time.Sleep(time.Minute * 5) + + target := alpha + if minute%60 == 0 { + target = 0 + } + + backend.nodesDHT.RefreshBuckets(target) + } +} + +// bootstrapKademlia bootstraps the Kademlia bucket list +func (backend *Backend) bootstrapKademlia() { + monitor := make(chan *PeerInfo) + backend.registerPeerMonitor(monitor) + + // Wait until there are at least 2 peers connected. + for { + <-monitor + if backend.nodesDHT.NumNodes() >= 2 { + break + } + } + + backend.unregisterPeerMonitor(monitor) + + // Refresh every 10 seconds 3 times + for n := 0; n < 3; n++ { + backend.nodesDHT.RefreshBuckets(alpha) + + time.Sleep(time.Second) + } +} + +// Future sendAnnouncementX: If it detects that announcements are sent out to the same peer within 50ms it should activate a wait-and-group scheme. + +func (peer *PeerInfo) sendAnnouncementFindNode(request *dht.InformationRequest) { + // If the key is self, send it as FIND_SELF + if bytes.Equal(request.Key, peer.Backend.nodeID) { + peer.sendAnnouncement(false, true, nil, nil, nil, request) + } else { + peer.sendAnnouncement(false, false, []protocol.KeyHash{{Hash: request.Key}}, nil, nil, request) + } +} + +func (peer *PeerInfo) sendAnnouncementFindValue(request *dht.InformationRequest) { + + findSelf := false + var findPeer []protocol.KeyHash + var findValue []protocol.KeyHash + + findValue = append(findValue, protocol.KeyHash{Hash: request.Key}) + + peer.sendAnnouncement(false, findSelf, findPeer, findValue, nil, request) +} + +func (peer *PeerInfo) sendAnnouncementStore(fileHash []byte, fileSize uint64) { + peer.sendAnnouncement(false, false, nil, nil, []protocol.InfoStore{{ID: protocol.KeyHash{Hash: fileHash}, Size: fileSize, Type: 0}}, nil) +} + +// ---- CORE DATA FUNCTIONS ---- + +// Data2Hash returns the hash for the data +func Data2Hash(data []byte) (hash []byte) { + return protocol.HashData(data) +} + +// GetData returns the requested data. It checks first the local store and then tries via DHT. +func (backend *Backend) GetData(hash []byte) (data []byte, senderNodeID []byte, found bool) { + if data, found = backend.GetDataLocal(hash); found { + return data, backend.nodeID, found + } + + return backend.GetDataDHT(hash) +} + +// GetDataLocal returns data from the local warehouse. +func (backend *Backend) GetDataLocal(hash []byte) (data []byte, found bool) { + return backend.dhtStore.Get(hash) +} + +// GetDataDHT requests data via DHT +func (backend *Backend) GetDataDHT(hash []byte) (data []byte, senderNodeID []byte, found bool) { + data, senderNodeID, found, _ = backend.nodesDHT.Get(hash) + return data, senderNodeID, found +} + +// StoreDataLocal stores data into the local warehouse. +func (backend *Backend) StoreDataLocal(data []byte) error { + key := protocol.HashData(data) + return backend.dhtStore.Set(key, data) +} + +// StoreDataDHT stores data locally and informs closestCount peers in the DHT about it. +// Remote peers may choose to keep a record (in case another peers asks) or mirror the full data. +func (backend *Backend) StoreDataDHT(data []byte, closestCount int) error { + key := protocol.HashData(data) + if err := backend.dhtStore.Set(key, data); err != nil { + return err + } + return backend.nodesDHT.Store(key, uint64(len(data)), closestCount) +} + +// ---- NODE FUNCTIONS ---- + +// IsNodeContact checks if the node is a contact in the local DHT routing table +func (backend *Backend) IsNodeContact(nodeID []byte) (node *dht.Node, peer *PeerInfo) { + node = backend.nodesDHT.IsNodeContact(nodeID) + if node == nil { + return nil, nil + } + + return node, node.Info.(*PeerInfo) +} + +// FindNode finds a node via the DHT +func (backend *Backend) FindNode(nodeID []byte, Timeout time.Duration) (node *dht.Node, peer *PeerInfo, err error) { + // first check if in mirrored node list + if peer = backend.NodelistLookup(nodeID); peer != nil { + return nil, peer, nil + } + + // Search the node via DHT. + node, err = backend.nodesDHT.FindNode(nodeID) + if node == nil { + return nil, nil, err + } + + return node, node.Info.(*PeerInfo), err +} + +// ---- Asynchronous Search ---- + +// AsyncSearch creates an async search for the given key in the DHT. +// Timeout is the total time the search may take, covering all information requests. TimeoutIR is the time an information request may take. +// Alpha is the number of concurrent requests that will be performed. +func (backend *Backend) AsyncSearch(Action int, Key []byte, Timeout, TimeoutIR time.Duration, Alpha int) (client *dht.SearchClient) { + return backend.nodesDHT.NewSearch(Action, Key, Timeout, TimeoutIR, Alpha) +} diff --git a/Message Send.go b/Message Send.go index 4e79b7a..f2e4e19 100644 --- a/Message Send.go +++ b/Message Send.go @@ -1,154 +1,154 @@ -/* -File Name: Message Send.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "time" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/protocol" - "github.com/google/uuid" -) - -// pingConnection sends a ping to the target peer via the specified connection -func (peer *PeerInfo) pingConnection(connection *Connection) { - raw := &protocol.PacketRaw{Command: protocol.CommandPing, Sequence: peer.Backend.networks.Sequences.NewSequence(peer.PublicKey, &peer.messageSequence, nil).SequenceNumber} - peer.Backend.Filters.MessageOutPing(peer, raw, connection) - - err := peer.sendConnection(raw, connection) - connection.LastPingOut = time.Now() - - if (connection.Status == ConnectionActive || connection.Status == ConnectionRedundant) && IsNetworkErrorFatal(err) { - peer.invalidateActiveConnection(connection) - } -} - -// pingConnectionAnnouncement sends an empty announcement via a particular connection. -// It has the same effect as ping, but returns the blockchain version and height of the other peer in the Response message, which may be useful for keeping the global blockchain cache up to date. -func (peer *PeerInfo) pingConnectionAnnouncement(connection *Connection) { - _, blockchainHeight, blockchainVersion := peer.Backend.UserBlockchain.Header() - packets := protocol.EncodeAnnouncement(false, false, nil, nil, nil, peer.Backend.FeatureSupport(), blockchainHeight, blockchainVersion, peer.Backend.userAgent) - if len(packets) != 1 { - return - } - - raw := &protocol.PacketRaw{Command: protocol.CommandAnnouncement, Payload: packets[0], Sequence: peer.Backend.networks.Sequences.NewSequence(peer.PublicKey, &peer.messageSequence, nil).SequenceNumber} - peer.Backend.Filters.MessageOutAnnouncement(peer.PublicKey, peer, raw, false, nil, nil, nil) - - err := peer.sendConnection(raw, connection) - connection.LastPingOut = time.Now() - - if (connection.Status == ConnectionActive || connection.Status == ConnectionRedundant) && IsNetworkErrorFatal(err) { - peer.invalidateActiveConnection(connection) - } -} - -// Ping sends a ping. This function exists only for debugging purposes, it should not be used normally. -// This ping is not used for uptime detection and the LastPingOut time in connections is not set. -func (peer *PeerInfo) Ping() { - peer.send(&protocol.PacketRaw{Command: protocol.CommandPing, Sequence: peer.Backend.networks.Sequences.NewSequence(peer.PublicKey, &peer.messageSequence, nil).SequenceNumber}) -} - -// Chat sends a text message -func (peer *PeerInfo) Chat(text string) { - peer.send(&protocol.PacketRaw{Command: protocol.CommandChat, Payload: []byte(text)}) -} - -// sendAnnouncement sends the announcement message. It acquires a new sequence for each message. -func (peer *PeerInfo) sendAnnouncement(sendUA, findSelf bool, findPeer []protocol.KeyHash, findValue []protocol.KeyHash, files []protocol.InfoStore, sequenceData interface{}) { - _, blockchainHeight, blockchainVersion := peer.Backend.UserBlockchain.Header() - packets := protocol.EncodeAnnouncement(sendUA, findSelf, findPeer, findValue, files, peer.Backend.FeatureSupport(), blockchainHeight, blockchainVersion, peer.Backend.userAgent) - - for _, packet := range packets { - raw := &protocol.PacketRaw{Command: protocol.CommandAnnouncement, Payload: packet, Sequence: peer.Backend.networks.Sequences.NewSequence(peer.PublicKey, &peer.messageSequence, sequenceData).SequenceNumber} - peer.Backend.Filters.MessageOutAnnouncement(peer.PublicKey, peer, raw, findSelf, findPeer, findValue, files) - peer.send(raw) - } -} - -// sendResponse sends the response message -func (peer *PeerInfo) sendResponse(sequence uint32, sendUA bool, hash2Peers []protocol.Hash2Peer, filesEmbed []protocol.EmbeddedFileData, hashesNotFound [][]byte) (err error) { - _, blockchainHeight, blockchainVersion := peer.Backend.UserBlockchain.Header() - packets, err := protocol.EncodeResponse(sendUA, hash2Peers, filesEmbed, hashesNotFound, peer.Backend.FeatureSupport(), blockchainHeight, blockchainVersion, peer.Backend.userAgent) - - for _, packet := range packets { - raw := &protocol.PacketRaw{Command: protocol.CommandResponse, Payload: packet, Sequence: sequence} - peer.Backend.Filters.MessageOutResponse(peer, raw, hash2Peers, filesEmbed, hashesNotFound) - peer.send(raw) - } - - return err -} - -// sendTraverse sends a traverse message -func (peer *PeerInfo) sendTraverse(packet *protocol.PacketRaw, receiverEnd *btcec.PublicKey) (err error) { - packet.Protocol = protocol.ProtocolVersion - // self-reported ports are not set, as this isn't sent via a specific network but a relay - //packet.SetSelfReportedPorts(c.Network.SelfReportedPorts()) - - embeddedPacketRaw, err := protocol.PacketEncrypt(peer.Backend.PeerPrivateKey, receiverEnd, packet) - if err != nil { - return err - } - - packetRaw, err := protocol.EncodeTraverse(peer.Backend.PeerPrivateKey, embeddedPacketRaw, receiverEnd, peer.PublicKey) - if err != nil { - return err - } - - raw := &protocol.PacketRaw{Command: protocol.CommandTraverse, Payload: packetRaw} - - peer.Backend.Filters.MessageOutTraverse(peer, raw, packet, receiverEnd) - - return peer.send(raw) -} - -// sendTransfer sends a transfer message -func (peer *PeerInfo) sendTransfer(data []byte, control, transferProtocol uint8, hash []byte, offset, limit uint64, sequenceNumber uint32, transferID uuid.UUID, isLite bool) (err error) { - // Send optionally as lite packet. This bypasses the signing overhead of regular Peernet packets which is CPU intensive and a bottleneck. - if control == protocol.TransferControlActive && isLite { - raw, err := protocol.PacketLiteEncode(transferID, data) - if err != nil { - return err - } - return peer.sendLite(raw) - } - - packetRaw, err := protocol.EncodeTransfer(peer.Backend.PeerPrivateKey, data, control, transferProtocol, hash, offset, limit, transferID) - if err != nil { - return err - } - - raw := &protocol.PacketRaw{Command: protocol.CommandTransfer, Payload: packetRaw, Sequence: sequenceNumber} - - //Filters.MessageOutTransfer(peer, raw, control, transferProtocol, hash, offset, limit) - - return peer.send(raw) -} - -// sendGetBlock sends a get block message -func (peer *PeerInfo) sendGetBlock(data []byte, control uint8, blockchainPublicKey *btcec.PublicKey, limitBlockCount, maxBlockSize uint64, targetBlocks []protocol.BlockRange, sequenceNumber uint32, transferID uuid.UUID, isLite bool) (err error) { - // Send optionally as lite packet. This bypasses the signing overhead of regular Peernet packets which is CPU intensive and a bottleneck. - if control == protocol.GetBlockControlActive && isLite { - raw, err := protocol.PacketLiteEncode(transferID, data) - if err != nil { - return err - } - return peer.sendLite(raw) - } - - packetRaw, err := protocol.EncodeGetBlock(peer.Backend.PeerPrivateKey, data, control, blockchainPublicKey, limitBlockCount, maxBlockSize, targetBlocks, transferID) - if err != nil { - return err - } - - raw := &protocol.PacketRaw{Command: protocol.CommandGetBlock, Payload: packetRaw, Sequence: sequenceNumber} - - //Filters.MessageOutGetBlock(peer, raw, control, ) - - return peer.send(raw) -} +/* +File Name: Message Send.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "time" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" + "github.com/google/uuid" +) + +// pingConnection sends a ping to the target peer via the specified connection +func (peer *PeerInfo) pingConnection(connection *Connection) { + raw := &protocol.PacketRaw{Command: protocol.CommandPing, Sequence: peer.Backend.networks.Sequences.NewSequence(peer.PublicKey, &peer.messageSequence, nil).SequenceNumber} + peer.Backend.Filters.MessageOutPing(peer, raw, connection) + + err := peer.sendConnection(raw, connection) + connection.LastPingOut = time.Now() + + if (connection.Status == ConnectionActive || connection.Status == ConnectionRedundant) && IsNetworkErrorFatal(err) { + peer.invalidateActiveConnection(connection) + } +} + +// pingConnectionAnnouncement sends an empty announcement via a particular connection. +// It has the same effect as ping, but returns the blockchain version and height of the other peer in the Response message, which may be useful for keeping the global blockchain cache up to date. +func (peer *PeerInfo) pingConnectionAnnouncement(connection *Connection) { + _, blockchainHeight, blockchainVersion := peer.Backend.UserBlockchain.Header() + packets := protocol.EncodeAnnouncement(false, false, nil, nil, nil, peer.Backend.FeatureSupport(), blockchainHeight, blockchainVersion, peer.Backend.userAgent) + if len(packets) != 1 { + return + } + + raw := &protocol.PacketRaw{Command: protocol.CommandAnnouncement, Payload: packets[0], Sequence: peer.Backend.networks.Sequences.NewSequence(peer.PublicKey, &peer.messageSequence, nil).SequenceNumber} + peer.Backend.Filters.MessageOutAnnouncement(peer.PublicKey, peer, raw, false, nil, nil, nil) + + err := peer.sendConnection(raw, connection) + connection.LastPingOut = time.Now() + + if (connection.Status == ConnectionActive || connection.Status == ConnectionRedundant) && IsNetworkErrorFatal(err) { + peer.invalidateActiveConnection(connection) + } +} + +// Ping sends a ping. This function exists only for debugging purposes, it should not be used normally. +// This ping is not used for uptime detection and the LastPingOut time in connections is not set. +func (peer *PeerInfo) Ping() { + peer.send(&protocol.PacketRaw{Command: protocol.CommandPing, Sequence: peer.Backend.networks.Sequences.NewSequence(peer.PublicKey, &peer.messageSequence, nil).SequenceNumber}) +} + +// Chat sends a text message +func (peer *PeerInfo) Chat(text string) { + peer.send(&protocol.PacketRaw{Command: protocol.CommandChat, Payload: []byte(text)}) +} + +// sendAnnouncement sends the announcement message. It acquires a new sequence for each message. +func (peer *PeerInfo) sendAnnouncement(sendUA, findSelf bool, findPeer []protocol.KeyHash, findValue []protocol.KeyHash, files []protocol.InfoStore, sequenceData interface{}) { + _, blockchainHeight, blockchainVersion := peer.Backend.UserBlockchain.Header() + packets := protocol.EncodeAnnouncement(sendUA, findSelf, findPeer, findValue, files, peer.Backend.FeatureSupport(), blockchainHeight, blockchainVersion, peer.Backend.userAgent) + + for _, packet := range packets { + raw := &protocol.PacketRaw{Command: protocol.CommandAnnouncement, Payload: packet, Sequence: peer.Backend.networks.Sequences.NewSequence(peer.PublicKey, &peer.messageSequence, sequenceData).SequenceNumber} + peer.Backend.Filters.MessageOutAnnouncement(peer.PublicKey, peer, raw, findSelf, findPeer, findValue, files) + peer.send(raw) + } +} + +// sendResponse sends the response message +func (peer *PeerInfo) sendResponse(sequence uint32, sendUA bool, hash2Peers []protocol.Hash2Peer, filesEmbed []protocol.EmbeddedFileData, hashesNotFound [][]byte) (err error) { + _, blockchainHeight, blockchainVersion := peer.Backend.UserBlockchain.Header() + packets, err := protocol.EncodeResponse(sendUA, hash2Peers, filesEmbed, hashesNotFound, peer.Backend.FeatureSupport(), blockchainHeight, blockchainVersion, peer.Backend.userAgent) + + for _, packet := range packets { + raw := &protocol.PacketRaw{Command: protocol.CommandResponse, Payload: packet, Sequence: sequence} + peer.Backend.Filters.MessageOutResponse(peer, raw, hash2Peers, filesEmbed, hashesNotFound) + peer.send(raw) + } + + return err +} + +// sendTraverse sends a traverse message +func (peer *PeerInfo) sendTraverse(packet *protocol.PacketRaw, receiverEnd *btcec.PublicKey) (err error) { + packet.Protocol = protocol.ProtocolVersion + // self-reported ports are not set, as this isn't sent via a specific network but a relay + //packet.SetSelfReportedPorts(c.Network.SelfReportedPorts()) + + embeddedPacketRaw, err := protocol.PacketEncrypt(peer.Backend.PeerPrivateKey, receiverEnd, packet) + if err != nil { + return err + } + + packetRaw, err := protocol.EncodeTraverse(peer.Backend.PeerPrivateKey, embeddedPacketRaw, receiverEnd, peer.PublicKey) + if err != nil { + return err + } + + raw := &protocol.PacketRaw{Command: protocol.CommandTraverse, Payload: packetRaw} + + peer.Backend.Filters.MessageOutTraverse(peer, raw, packet, receiverEnd) + + return peer.send(raw) +} + +// sendTransfer sends a transfer message +func (peer *PeerInfo) sendTransfer(data []byte, control, transferProtocol uint8, hash []byte, offset, limit uint64, sequenceNumber uint32, transferID uuid.UUID, isLite bool) (err error) { + // Send optionally as lite packet. This bypasses the signing overhead of regular Peernet packets which is CPU intensive and a bottleneck. + if control == protocol.TransferControlActive && isLite { + raw, err := protocol.PacketLiteEncode(transferID, data) + if err != nil { + return err + } + return peer.sendLite(raw) + } + + packetRaw, err := protocol.EncodeTransfer(peer.Backend.PeerPrivateKey, data, control, transferProtocol, hash, offset, limit, transferID) + if err != nil { + return err + } + + raw := &protocol.PacketRaw{Command: protocol.CommandTransfer, Payload: packetRaw, Sequence: sequenceNumber} + + //Filters.MessageOutTransfer(peer, raw, control, transferProtocol, hash, offset, limit) + + return peer.send(raw) +} + +// sendGetBlock sends a get block message +func (peer *PeerInfo) sendGetBlock(data []byte, control uint8, blockchainPublicKey *btcec.PublicKey, limitBlockCount, maxBlockSize uint64, targetBlocks []protocol.BlockRange, sequenceNumber uint32, transferID uuid.UUID, isLite bool) (err error) { + // Send optionally as lite packet. This bypasses the signing overhead of regular Peernet packets which is CPU intensive and a bottleneck. + if control == protocol.GetBlockControlActive && isLite { + raw, err := protocol.PacketLiteEncode(transferID, data) + if err != nil { + return err + } + return peer.sendLite(raw) + } + + packetRaw, err := protocol.EncodeGetBlock(peer.Backend.PeerPrivateKey, data, control, blockchainPublicKey, limitBlockCount, maxBlockSize, targetBlocks, transferID) + if err != nil { + return err + } + + raw := &protocol.PacketRaw{Command: protocol.CommandGetBlock, Payload: packetRaw, Sequence: sequenceNumber} + + //Filters.MessageOutGetBlock(peer, raw, control, ) + + return peer.send(raw) +} diff --git a/Network Detection.go b/Network Detection.go index 6c80b81..29f52b9 100644 --- a/Network Detection.go +++ b/Network Detection.go @@ -1,275 +1,275 @@ -/* -File Name: Network Detection.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "net" - "strings" - "time" -) - -// FindInterfaceByIP finds an interface based on the IP. The IP must be available at the interface. -func FindInterfaceByIP(ip net.IP) (iface *net.Interface, ipnet *net.IPNet) { - interfaceList, err := net.Interfaces() - if err != nil { - return nil, nil - } - - // iterate through all interfaces - for _, ifaceSingle := range interfaceList { - addresses, err := ifaceSingle.Addrs() - if err != nil { - continue - } - - // iterate through all IPs of the interfaces - for _, address := range addresses { - addressIP := address.(*net.IPNet).IP - - if addressIP.Equal(ip) { - return &ifaceSingle, address.(*net.IPNet) - } - } - } - - return nil, nil -} - -// NetworkListIPs returns a list of all IPs -func NetworkListIPs() (IPs []net.IP, err error) { - - interfaceList, err := net.Interfaces() - if err != nil { - return nil, err - } - - // iterate through all interfaces - for _, ifaceSingle := range interfaceList { - addresses, err := ifaceSingle.Addrs() - if err != nil { - continue - } - - // iterate through all IPs of the interfaces - for _, address := range addresses { - addressIP := address.(*net.IPNet).IP - IPs = append(IPs, addressIP) - } - } - - return IPs, nil -} - -// IsIPv4 checks if an IP address is IPv4 -func IsIPv4(IP net.IP) bool { - return IP.To4() != nil -} - -// IsIPv6 checks if an IP address is IPv6 -func IsIPv6(IP net.IP) bool { - return IP.To4() == nil && IP.To16() != nil -} - -// IsNetworkErrorFatal checks if a network error indicates a broken connection. -// Not every network error indicates a broken connection. This function prevents from over-dropping connections. -func IsNetworkErrorFatal(err error) bool { - if err == nil { - return false - } - - // Windows: A common error when the network adapter is disabled is "wsasendto: The requested address is not valid in its context". - if strings.Contains(err.Error(), "requested address is not valid in its context") { - return true - } - - return false -} - -// changeMonitorFrequency is the frequency in seconds to check for a network change -const changeMonitorFrequency = 10 - -// networkChangeMonitor() monitors for network changes to act accordingly -func (nets *Networks) networkChangeMonitor() { - // If manual IPs are entered, no need for monitoring for any network changes. - if len(nets.backend.Config.Listen) > 0 { - return - } - - for { - time.Sleep(time.Second * changeMonitorFrequency) - - interfaceList, err := net.Interfaces() - if err != nil { - nets.backend.LogError("networkChangeMonitor", "enumerating network adapters failed: %s\n", err.Error()) - continue - } - - ifacesNew := make(map[string][]net.Addr) - - for _, iface := range interfaceList { - addressesNew, err := iface.Addrs() - if err != nil { - nets.backend.LogError("networkChangeMonitor", "enumerating IPs for network adapter '%s': %s\n", iface.Name, err.Error()) - continue - } - ifacesNew[iface.Name] = addressesNew - - // was the interface added? - addressesExist, ok := nets.ipListen.ifacesExist[iface.Name] - if !ok { - nets.networkChangeInterfaceNew(iface, addressesNew) - } else { - // new IPs added for this interface? - for _, addr := range addressesNew { - exists := false - for _, exist := range addressesExist { - if exist.String() == addr.String() { - exists = true - break - } - } - - if !exists { - nets.networkChangeIPNew(iface, addr) - } - } - - // were IPs removed from this interface - for _, exist := range addressesExist { - removed := true - for _, addr := range addressesNew { - if exist.String() == addr.String() { - removed = false - break - } - } - - if removed { - nets.networkChangeIPRemove(iface, exist) - } - } - } - } - - // was an existing interface removed? - for ifaceExist, addressesExist := range nets.ipListen.ifacesExist { - if _, ok := ifacesNew[ifaceExist]; !ok { - nets.networkChangeInterfaceRemove(ifaceExist, addressesExist) - } - } - - nets.ipListen.ifacesExist = ifacesNew - } -} - -// networkChangeInterfaceNew is called when a new interface is detected -func (nets *Networks) networkChangeInterfaceNew(iface net.Interface, addresses []net.Addr) { - nets.backend.LogError("networkChangeInterfaceNew", "new interface '%s' (%d IPs)\n", iface.Name, len(addresses)) - - networksNew := nets.InterfaceStart(iface, addresses) - - for _, network := range networksNew { - go network.upnpAuto() - } - - go nets.backend.nodesDHT.RefreshBuckets(0) -} - -// networkChangeInterfaceRemove is called when an existing interface is removed -func (nets *Networks) networkChangeInterfaceRemove(iface string, addresses []net.Addr) { - nets.RLock() - defer nets.RUnlock() - - nets.backend.LogError("networkChangeInterfaceRemove", "removing interface '%s' (%d IPs)\n", iface, len(addresses)) - - for n, network := range nets.networks6 { - if network.iface != nil && network.iface.Name == iface { - network.Terminate() - - // remove from list - networksNew := nets.networks6[:n] - if n < len(nets.networks6)-1 { - networksNew = append(networksNew, nets.networks6[n+1:]...) - } - nets.networks6 = networksNew - } - } - - for n, network := range nets.networks4 { - if network.iface != nil && network.iface.Name == iface { - network.Terminate() - - // remove from list - networksNew := nets.networks4[:n] - if n < len(nets.networks4)-1 { - networksNew = append(networksNew, nets.networks4[n+1:]...) - } - nets.networks4 = networksNew - } - } -} - -// networkChangeIPNew is called when an existing interface lists a new IP -func (nets *Networks) networkChangeIPNew(iface net.Interface, address net.Addr) { - nets.backend.LogError("networkChangeIPNew", "new interface '%s' IP %s\n", iface.Name, address.String()) - - networksNew := nets.InterfaceStart(iface, []net.Addr{address}) - - for _, network := range networksNew { - go network.upnpAuto() - } - - go nets.backend.nodesDHT.RefreshBuckets(0) -} - -// networkChangeIPRemove is called when an existing interface removes an IP -func (nets *Networks) networkChangeIPRemove(iface net.Interface, address net.Addr) { - nets.RLock() - defer nets.RUnlock() - - nets.backend.LogError("networkChangeIPRemove", "remove interface '%s' IP %s\n", iface.Name, address.String()) - - for n, network := range nets.networks6 { - if network.address.IP.Equal(address.(*net.IPNet).IP) { - network.Terminate() - - // remove from list - networksNew := nets.networks6[:n] - if n < len(nets.networks6)-1 { - networksNew = append(networksNew, nets.networks6[n+1:]...) - } - nets.networks6 = networksNew - } - } - - for n, network := range nets.networks4 { - if network.address.IP.Equal(address.(*net.IPNet).IP) { - network.Terminate() - - // remove from list - networksNew := nets.networks4[:n] - if n < len(nets.networks4)-1 { - networksNew = append(networksNew, nets.networks4[n+1:]...) - } - nets.networks4 = networksNew - } - } -} - -// IsIPLocal reports whether ip is a private (local) address. -// The identification of private, or local, unicast addresses uses address type -// indentification as defined in RFC 1918 for ip4 and RFC 4193 (fc00::/7) for ip6 with the exception of ip4 directed broadcast addresses. -// Unassigned, reserved, multicast and limited-broadcast addresses are not handled and will return false. -// IPv6 link-local addresses (fe80::/10) are included in this check. -func IsIPLocal(ip net.IP) bool { - if ip4 := ip.To4(); ip4 != nil { - return ip4[0] == 10 || (ip4[0] == 172 && ip4[1]&0xf0 == 16) || (ip4[0] == 192 && ip4[1] == 168) - } - return len(ip) == net.IPv6len && - (ip[0]&0xfe == 0xfc || // fc00::/7 - (ip[0] == 0xfe && ip[1]&0xC0 == 0x80)) // fe80::/10 -} +/* +File Name: Network Detection.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "net" + "strings" + "time" +) + +// FindInterfaceByIP finds an interface based on the IP. The IP must be available at the interface. +func FindInterfaceByIP(ip net.IP) (iface *net.Interface, ipnet *net.IPNet) { + interfaceList, err := net.Interfaces() + if err != nil { + return nil, nil + } + + // iterate through all interfaces + for _, ifaceSingle := range interfaceList { + addresses, err := ifaceSingle.Addrs() + if err != nil { + continue + } + + // iterate through all IPs of the interfaces + for _, address := range addresses { + addressIP := address.(*net.IPNet).IP + + if addressIP.Equal(ip) { + return &ifaceSingle, address.(*net.IPNet) + } + } + } + + return nil, nil +} + +// NetworkListIPs returns a list of all IPs +func NetworkListIPs() (IPs []net.IP, err error) { + + interfaceList, err := net.Interfaces() + if err != nil { + return nil, err + } + + // iterate through all interfaces + for _, ifaceSingle := range interfaceList { + addresses, err := ifaceSingle.Addrs() + if err != nil { + continue + } + + // iterate through all IPs of the interfaces + for _, address := range addresses { + addressIP := address.(*net.IPNet).IP + IPs = append(IPs, addressIP) + } + } + + return IPs, nil +} + +// IsIPv4 checks if an IP address is IPv4 +func IsIPv4(IP net.IP) bool { + return IP.To4() != nil +} + +// IsIPv6 checks if an IP address is IPv6 +func IsIPv6(IP net.IP) bool { + return IP.To4() == nil && IP.To16() != nil +} + +// IsNetworkErrorFatal checks if a network error indicates a broken connection. +// Not every network error indicates a broken connection. This function prevents from over-dropping connections. +func IsNetworkErrorFatal(err error) bool { + if err == nil { + return false + } + + // Windows: A common error when the network adapter is disabled is "wsasendto: The requested address is not valid in its context". + if strings.Contains(err.Error(), "requested address is not valid in its context") { + return true + } + + return false +} + +// changeMonitorFrequency is the frequency in seconds to check for a network change +const changeMonitorFrequency = 10 + +// networkChangeMonitor() monitors for network changes to act accordingly +func (nets *Networks) networkChangeMonitor() { + // If manual IPs are entered, no need for monitoring for any network changes. + if len(nets.backend.Config.Listen) > 0 { + return + } + + for { + time.Sleep(time.Second * changeMonitorFrequency) + + interfaceList, err := net.Interfaces() + if err != nil { + nets.backend.LogError("networkChangeMonitor", "enumerating network adapters failed: %s\n", err.Error()) + continue + } + + ifacesNew := make(map[string][]net.Addr) + + for _, iface := range interfaceList { + addressesNew, err := iface.Addrs() + if err != nil { + nets.backend.LogError("networkChangeMonitor", "enumerating IPs for network adapter '%s': %s\n", iface.Name, err.Error()) + continue + } + ifacesNew[iface.Name] = addressesNew + + // was the interface added? + addressesExist, ok := nets.ipListen.ifacesExist[iface.Name] + if !ok { + nets.networkChangeInterfaceNew(iface, addressesNew) + } else { + // new IPs added for this interface? + for _, addr := range addressesNew { + exists := false + for _, exist := range addressesExist { + if exist.String() == addr.String() { + exists = true + break + } + } + + if !exists { + nets.networkChangeIPNew(iface, addr) + } + } + + // were IPs removed from this interface + for _, exist := range addressesExist { + removed := true + for _, addr := range addressesNew { + if exist.String() == addr.String() { + removed = false + break + } + } + + if removed { + nets.networkChangeIPRemove(iface, exist) + } + } + } + } + + // was an existing interface removed? + for ifaceExist, addressesExist := range nets.ipListen.ifacesExist { + if _, ok := ifacesNew[ifaceExist]; !ok { + nets.networkChangeInterfaceRemove(ifaceExist, addressesExist) + } + } + + nets.ipListen.ifacesExist = ifacesNew + } +} + +// networkChangeInterfaceNew is called when a new interface is detected +func (nets *Networks) networkChangeInterfaceNew(iface net.Interface, addresses []net.Addr) { + nets.backend.LogError("networkChangeInterfaceNew", "new interface '%s' (%d IPs)\n", iface.Name, len(addresses)) + + networksNew := nets.InterfaceStart(iface, addresses) + + for _, network := range networksNew { + go network.upnpAuto() + } + + go nets.backend.nodesDHT.RefreshBuckets(0) +} + +// networkChangeInterfaceRemove is called when an existing interface is removed +func (nets *Networks) networkChangeInterfaceRemove(iface string, addresses []net.Addr) { + nets.RLock() + defer nets.RUnlock() + + nets.backend.LogError("networkChangeInterfaceRemove", "removing interface '%s' (%d IPs)\n", iface, len(addresses)) + + for n, network := range nets.networks6 { + if network.iface != nil && network.iface.Name == iface { + network.Terminate() + + // remove from list + networksNew := nets.networks6[:n] + if n < len(nets.networks6)-1 { + networksNew = append(networksNew, nets.networks6[n+1:]...) + } + nets.networks6 = networksNew + } + } + + for n, network := range nets.networks4 { + if network.iface != nil && network.iface.Name == iface { + network.Terminate() + + // remove from list + networksNew := nets.networks4[:n] + if n < len(nets.networks4)-1 { + networksNew = append(networksNew, nets.networks4[n+1:]...) + } + nets.networks4 = networksNew + } + } +} + +// networkChangeIPNew is called when an existing interface lists a new IP +func (nets *Networks) networkChangeIPNew(iface net.Interface, address net.Addr) { + nets.backend.LogError("networkChangeIPNew", "new interface '%s' IP %s\n", iface.Name, address.String()) + + networksNew := nets.InterfaceStart(iface, []net.Addr{address}) + + for _, network := range networksNew { + go network.upnpAuto() + } + + go nets.backend.nodesDHT.RefreshBuckets(0) +} + +// networkChangeIPRemove is called when an existing interface removes an IP +func (nets *Networks) networkChangeIPRemove(iface net.Interface, address net.Addr) { + nets.RLock() + defer nets.RUnlock() + + nets.backend.LogError("networkChangeIPRemove", "remove interface '%s' IP %s\n", iface.Name, address.String()) + + for n, network := range nets.networks6 { + if network.address.IP.Equal(address.(*net.IPNet).IP) { + network.Terminate() + + // remove from list + networksNew := nets.networks6[:n] + if n < len(nets.networks6)-1 { + networksNew = append(networksNew, nets.networks6[n+1:]...) + } + nets.networks6 = networksNew + } + } + + for n, network := range nets.networks4 { + if network.address.IP.Equal(address.(*net.IPNet).IP) { + network.Terminate() + + // remove from list + networksNew := nets.networks4[:n] + if n < len(nets.networks4)-1 { + networksNew = append(networksNew, nets.networks4[n+1:]...) + } + nets.networks4 = networksNew + } + } +} + +// IsIPLocal reports whether ip is a private (local) address. +// The identification of private, or local, unicast addresses uses address type +// indentification as defined in RFC 1918 for ip4 and RFC 4193 (fc00::/7) for ip6 with the exception of ip4 directed broadcast addresses. +// Unassigned, reserved, multicast and limited-broadcast addresses are not handled and will return false. +// IPv6 link-local addresses (fe80::/10) are included in this check. +func IsIPLocal(ip net.IP) bool { + if ip4 := ip.To4(); ip4 != nil { + return ip4[0] == 10 || (ip4[0] == 172 && ip4[1]&0xf0 == 16) || (ip4[0] == 192 && ip4[1] == 168) + } + return len(ip) == net.IPv6len && + (ip[0]&0xfe == 0xfc || // fc00::/7 + (ip[0] == 0xfe && ip[1]&0xC0 == 0x80)) // fe80::/10 +} diff --git a/Network IPv4 Broadcast.go b/Network IPv4 Broadcast.go index 93cb546..cbda142 100644 --- a/Network IPv4 Broadcast.go +++ b/Network IPv4 Broadcast.go @@ -1,181 +1,181 @@ -/* -File Name: Network IPv4 Broadcast.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -IPv4 Multicast just sucks (can't use socket bound to 0.0.0.0:PortMain and send to 224.0.0.1:PortMulticast), so we rely on Broadcast instead. -*/ - -package core - -import ( - "encoding/hex" - "errors" - "net" - "strconv" - "time" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/reuseport" -) - -const ipv4BroadcastPort = 12912 - -// special Public-Private Key pair for local discovery -var ipv4BroadcastPrivateKey *btcec.PrivateKey -var ipv4BroadcastPublicKey *btcec.PublicKey - -const ipv4BroadcastPrivateKeyH = "5e27ecc8e54a24e71dca9ba84a9bf465400e27b8c46a977d34962d3d88558c8e" - -func initBroadcastIPv4() { - if configPK, err := hex.DecodeString(ipv4BroadcastPrivateKeyH); err == nil { - ipv4BroadcastPrivateKey, ipv4BroadcastPublicKey = btcec.PrivKeyFromBytes(btcec.S256(), configPK) - } -} - -// BroadcastIPv4 prepares sending Broadcasts -func (network *Network) BroadcastIPv4() (err error) { - if ipv4BroadcastPrivateKey == nil || ipv4BroadcastPublicKey == nil { - return - } - - // listen on a special socket - network.broadcastSocket, err = reuseport.ListenPacket("udp4", net.JoinHostPort(network.address.IP.String(), strconv.Itoa(ipv4BroadcastPort))) - if err != nil { - network.backend.LogError("BroadcastIPv4", "broadcast socket listen on IP '%s' port '%d': %v\n", network.address.IP.String(), ipv4BroadcastPort, err) - return err - } - - network.broadcastIPv4 = networkToIPv4BroadcastIPs(network.ipnet) - - go network.BroadcastIPv4Listen() - - return nil -} - -// BroadcastIPv4Listen listens for incoming broadcast packets -// Fork from network.Listen! Keep any changes synced. -func (network *Network) BroadcastIPv4Listen() { - for { - // Buffer: Must be created for each packet as it is passed as pointer. - // If the buffer is too small, ReadFromUDP only reads until its length and returns this error: "wsarecvfrom: A message sent on a datagram socket was larger than the internal message buffer or some other network limit, or the buffer used to receive a datagram into was smaller than the datagram itself." - buffer := make([]byte, maxPacketSize) - length, sender, err := network.broadcastSocket.ReadFrom(buffer) - - if err != nil { - network.backend.LogError("BroadcastIPv4Listen", "receiving UDP message: %v\n", err) // Only log for debug purposes. - time.Sleep(time.Millisecond * 50) // In case of endless errors, prevent ddos of CPU. - continue - } - - if network.networkGroup.ipListen.IsAddressSelf(sender.(*net.UDPAddr)) { - continue - } - - // For good network practice (and reducing amount of parallel connections), do not allow link-local to talk to non-link-local addresses. - if sender.(*net.UDPAddr).IP.IsLinkLocalUnicast() != network.address.IP.IsLinkLocalUnicast() { - continue - } - - //fmt.Printf("BroadcastIPv4Listen from %s at network %s\n", sender.String(), network.address.String()) - - if length < protocol.PacketLengthMin { - // Discard packets that do not meet the minimum length. - continue - } - - // send the packet to a channel which is processed by multiple workers. - network.networkGroup.rawPacketsIncoming <- networkWire{network: network, sender: sender.(*net.UDPAddr), raw: buffer[:length], receiverPublicKey: ipv4BroadcastPublicKey, unicast: false} - } -} - -// BroadcastIPv4Send sends out a single broadcast messages to discover peers -func (network *Network) BroadcastIPv4Send() (err error) { - _, blockchainHeight, blockchainVersion := network.backend.UserBlockchain.Header() - packets := protocol.EncodeAnnouncement(true, true, nil, nil, nil, network.backend.FeatureSupport(), blockchainHeight, blockchainVersion, network.backend.userAgent) - if len(packets) == 0 { - return errors.New("error encoding broadcast announcement") - } - - raw, err := protocol.PacketEncrypt(network.backend.PeerPrivateKey, ipv4BroadcastPublicKey, &protocol.PacketRaw{Protocol: protocol.ProtocolVersion, Command: protocol.CommandLocalDiscovery, Payload: packets[0]}) - if err != nil { - return err - } - - // send out the wire - for _, ip := range network.broadcastIPv4 { - err = network.send(ip, ipv4BroadcastPort, raw) - if err != nil { - network.backend.LogError("BroadcastIPv4Send", "sending UDP packet: %v\n", err) - } - } - - return nil -} - -// networkToIPv4BroadcastIPs generates the IPv4 addresses to send out the broadcast to -func networkToIPv4BroadcastIPs(ipnet *net.IPNet) (broadcastIPs []net.IP) { - broadcastIPs = append(broadcastIPs, net.IPv4bcast) - - if ipnet != nil { - if ip2 := ipv4DirectedBroadcast(ipnet); ip2 != nil { - broadcastIPs = append(broadcastIPs, ip2) - } - } else { - interfaceList, err := net.Interfaces() - if err != nil { - return - } - - for _, iface := range interfaceList { - addresses, err := iface.Addrs() - if err != nil { - continue - } - - for _, address := range addresses { - net1 := address.(*net.IPNet) - - // TODO: Does the rfc3927Net make sense? - if !IsIPv4(net1.IP) || rfc3927Net.Contains(net1.IP) { - continue - } - - if ip2 := ipv4DirectedBroadcast(net1); ip2 != nil { - broadcastIPs = append(broadcastIPs, ip2) - } - } - } - } - - // TODO: Result could contain duplicates, filter them out - - return broadcastIPs -} - -func ipv4DirectedBroadcast(n *net.IPNet) net.IP { - ip4 := n.IP.To4() - if ip4 == nil { - return nil - } - last := make(net.IP, len(ip4)) - copy(last, ip4) - for i := range ip4 { - last[i] |= ^n.Mask[i] - } - return last -} - -var ( - // rfc3927Net specifies the IPv4 auto configuration address block as - // defined by RFC3927 (169.254.0.0/16). - rfc3927Net = ipNet("169.254.0.0", 16, 32) -) - -// ipNet returns a net.IPNet struct given the passed IP address string, number -// of one bits to include at the start of the mask, and the total number of bits -// for the mask. -func ipNet(ip string, ones, bits int) net.IPNet { - return net.IPNet{IP: net.ParseIP(ip), Mask: net.CIDRMask(ones, bits)} -} +/* +File Name: Network IPv4 Broadcast.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +IPv4 Multicast just sucks (can't use socket bound to 0.0.0.0:PortMain and send to 224.0.0.1:PortMulticast), so we rely on Broadcast instead. +*/ + +package core + +import ( + "encoding/hex" + "errors" + "net" + "strconv" + "time" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/reuseport" +) + +const ipv4BroadcastPort = 12912 + +// special Public-Private Key pair for local discovery +var ipv4BroadcastPrivateKey *btcec.PrivateKey +var ipv4BroadcastPublicKey *btcec.PublicKey + +const ipv4BroadcastPrivateKeyH = "5e27ecc8e54a24e71dca9ba84a9bf465400e27b8c46a977d34962d3d88558c8e" + +func initBroadcastIPv4() { + if configPK, err := hex.DecodeString(ipv4BroadcastPrivateKeyH); err == nil { + ipv4BroadcastPrivateKey, ipv4BroadcastPublicKey = btcec.PrivKeyFromBytes(btcec.S256(), configPK) + } +} + +// BroadcastIPv4 prepares sending Broadcasts +func (network *Network) BroadcastIPv4() (err error) { + if ipv4BroadcastPrivateKey == nil || ipv4BroadcastPublicKey == nil { + return + } + + // listen on a special socket + network.broadcastSocket, err = reuseport.ListenPacket("udp4", net.JoinHostPort(network.address.IP.String(), strconv.Itoa(ipv4BroadcastPort))) + if err != nil { + network.backend.LogError("BroadcastIPv4", "broadcast socket listen on IP '%s' port '%d': %v\n", network.address.IP.String(), ipv4BroadcastPort, err) + return err + } + + network.broadcastIPv4 = networkToIPv4BroadcastIPs(network.ipnet) + + go network.BroadcastIPv4Listen() + + return nil +} + +// BroadcastIPv4Listen listens for incoming broadcast packets +// Fork from network.Listen! Keep any changes synced. +func (network *Network) BroadcastIPv4Listen() { + for { + // Buffer: Must be created for each packet as it is passed as pointer. + // If the buffer is too small, ReadFromUDP only reads until its length and returns this error: "wsarecvfrom: A message sent on a datagram socket was larger than the internal message buffer or some other network limit, or the buffer used to receive a datagram into was smaller than the datagram itself." + buffer := make([]byte, maxPacketSize) + length, sender, err := network.broadcastSocket.ReadFrom(buffer) + + if err != nil { + network.backend.LogError("BroadcastIPv4Listen", "receiving UDP message: %v\n", err) // Only log for debug purposes. + time.Sleep(time.Millisecond * 50) // In case of endless errors, prevent ddos of CPU. + continue + } + + if network.networkGroup.ipListen.IsAddressSelf(sender.(*net.UDPAddr)) { + continue + } + + // For good network practice (and reducing amount of parallel connections), do not allow link-local to talk to non-link-local addresses. + if sender.(*net.UDPAddr).IP.IsLinkLocalUnicast() != network.address.IP.IsLinkLocalUnicast() { + continue + } + + //fmt.Printf("BroadcastIPv4Listen from %s at network %s\n", sender.String(), network.address.String()) + + if length < protocol.PacketLengthMin { + // Discard packets that do not meet the minimum length. + continue + } + + // send the packet to a channel which is processed by multiple workers. + network.networkGroup.rawPacketsIncoming <- networkWire{network: network, sender: sender.(*net.UDPAddr), raw: buffer[:length], receiverPublicKey: ipv4BroadcastPublicKey, unicast: false} + } +} + +// BroadcastIPv4Send sends out a single broadcast messages to discover peers +func (network *Network) BroadcastIPv4Send() (err error) { + _, blockchainHeight, blockchainVersion := network.backend.UserBlockchain.Header() + packets := protocol.EncodeAnnouncement(true, true, nil, nil, nil, network.backend.FeatureSupport(), blockchainHeight, blockchainVersion, network.backend.userAgent) + if len(packets) == 0 { + return errors.New("error encoding broadcast announcement") + } + + raw, err := protocol.PacketEncrypt(network.backend.PeerPrivateKey, ipv4BroadcastPublicKey, &protocol.PacketRaw{Protocol: protocol.ProtocolVersion, Command: protocol.CommandLocalDiscovery, Payload: packets[0]}) + if err != nil { + return err + } + + // send out the wire + for _, ip := range network.broadcastIPv4 { + err = network.send(ip, ipv4BroadcastPort, raw) + if err != nil { + network.backend.LogError("BroadcastIPv4Send", "sending UDP packet: %v\n", err) + } + } + + return nil +} + +// networkToIPv4BroadcastIPs generates the IPv4 addresses to send out the broadcast to +func networkToIPv4BroadcastIPs(ipnet *net.IPNet) (broadcastIPs []net.IP) { + broadcastIPs = append(broadcastIPs, net.IPv4bcast) + + if ipnet != nil { + if ip2 := ipv4DirectedBroadcast(ipnet); ip2 != nil { + broadcastIPs = append(broadcastIPs, ip2) + } + } else { + interfaceList, err := net.Interfaces() + if err != nil { + return + } + + for _, iface := range interfaceList { + addresses, err := iface.Addrs() + if err != nil { + continue + } + + for _, address := range addresses { + net1 := address.(*net.IPNet) + + // TODO: Does the rfc3927Net make sense? + if !IsIPv4(net1.IP) || rfc3927Net.Contains(net1.IP) { + continue + } + + if ip2 := ipv4DirectedBroadcast(net1); ip2 != nil { + broadcastIPs = append(broadcastIPs, ip2) + } + } + } + } + + // TODO: Result could contain duplicates, filter them out + + return broadcastIPs +} + +func ipv4DirectedBroadcast(n *net.IPNet) net.IP { + ip4 := n.IP.To4() + if ip4 == nil { + return nil + } + last := make(net.IP, len(ip4)) + copy(last, ip4) + for i := range ip4 { + last[i] |= ^n.Mask[i] + } + return last +} + +var ( + // rfc3927Net specifies the IPv4 auto configuration address block as + // defined by RFC3927 (169.254.0.0/16). + rfc3927Net = ipNet("169.254.0.0", 16, 32) +) + +// ipNet returns a net.IPNet struct given the passed IP address string, number +// of one bits to include at the start of the mask, and the total number of bits +// for the mask. +func ipNet(ip string, ones, bits int) net.IPNet { + return net.IPNet{IP: net.ParseIP(ip), Mask: net.CIDRMask(ones, bits)} +} diff --git a/Network IPv6 Multicast.go b/Network IPv6 Multicast.go index 19bff38..f8f4594 100644 --- a/Network IPv6 Multicast.go +++ b/Network IPv6 Multicast.go @@ -1,153 +1,153 @@ -/* -File Name: Network IPv6 Multicast.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -IPv6 Multicast implementation to support discovery of peers within the same network (Site-local). -Loopback is enabled, which means that Multicast packets sent will be looped back and received by any local listeners. This allows to connect local processes with each other. - -Using the separate Multicast port, it allows sending unsolicited announcements without knowing the target's public key. Instead, a hard-coded key is used. - -The Multicast listener opens port 12912 with SO_REUSEADDR to allow multiple processes receive the incoming Multicast packets. -[1] mentions "If two sockets are bound to the same interface and port and are members of the same multicast group, data will be delivered to both sockets, rather than an arbitrarily chosen one." - -[1] https://docs.microsoft.com/en-us/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse -*/ - -package core - -import ( - "encoding/hex" - "errors" - "net" - "strconv" - "time" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/reuseport" - "golang.org/x/net/ipv6" -) - -// Multicast group is site-local. Group ID is 112. -const ipv6MulticastGroup = "ff05::112" -const ipv6MulticastPort = 12912 - -// special Public-Private Key pair for local discovery -var ipv6MulticastPrivateKey *btcec.PrivateKey -var ipv6MulticastPublicKey *btcec.PublicKey - -const ipv6MulticastPrivateKeyH = "016ad30bfb369926523bf18d136298b6c31d0817e9fb6c21feed89ae22cad788" - -func initMulticastIPv6() { - if configPK, err := hex.DecodeString(ipv6MulticastPrivateKeyH); err == nil { - ipv6MulticastPrivateKey, ipv6MulticastPublicKey = btcec.PrivKeyFromBytes(btcec.S256(), configPK) - } -} - -// MulticastIPv6Join joins the Multicast group -func (network *Network) MulticastIPv6Join() (err error) { - if ipv6MulticastPrivateKey == nil || ipv6MulticastPublicKey == nil { - return - } - - network.multicastIP = net.ParseIP(ipv6MulticastGroup) - - // listen on a special socket - network.multicastSocket, err = reuseport.ListenPacket("udp6", net.JoinHostPort(network.address.IP.String(), strconv.Itoa(ipv6MulticastPort))) - if err != nil { - network.backend.LogError("MulticastIPv6Join", "multicast socket listen on IP '%s' port '%d': %v\n", network.address.IP.String(), ipv6MulticastPort, err) - return err - } - - joinMulticastGroup := func(iface *net.Interface) (err error) { - pc := ipv6.NewPacketConn(network.multicastSocket) - if err := pc.JoinGroup(iface, &net.UDPAddr{IP: network.multicastIP}); err != nil { - //LogError("MulticastIPv6Join", "join multicast group iface '%s' multicast IP '%s' listen on IP '%s' port '%d': %v\n", iface.Name, network.multicastIP.String(), network.address.IP.String(), ipv6MulticastPort, err) - return err - } - - // receive messages from self or other processes running on the same computer - if loop, err := pc.MulticastLoopback(); err == nil && !loop { - if err := pc.SetMulticastLoopback(true); err != nil { - network.backend.LogError("MulticastIPv6Join", "setting multicast loopback status: %v\n", err) - } - } - - return nil - } - - // specific interface or join all? - if network.iface != nil { - if err = joinMulticastGroup(network.iface); err != nil { - return err - } - } else { - interfaceList, err := net.Interfaces() - if err != nil { - return err - } - - for _, ifaceSingle := range interfaceList { - joinMulticastGroup(&ifaceSingle) - } - } - - go network.MulticastIPv6Listen() - - return nil -} - -// MulticastIPv6Listen listens for incoming multicast packets -// Fork from network.Listen! Keep any changes synced. -func (network *Network) MulticastIPv6Listen() { - for { - // Buffer: Must be created for each packet as it is passed as pointer. - // If the buffer is too small, ReadFromUDP only reads until its length and returns this error: "wsarecvfrom: A message sent on a datagram socket was larger than the internal message buffer or some other network limit, or the buffer used to receive a datagram into was smaller than the datagram itself." - buffer := make([]byte, maxPacketSize) - length, sender, err := network.multicastSocket.ReadFrom(buffer) - - if err != nil { - network.backend.LogError("MulticastIPv6Listen", "receiving UDP message: %v\n", err) // Only log for debug purposes. - time.Sleep(time.Millisecond * 50) // In case of endless errors, prevent ddos of CPU. - continue - } - - // skip incoming packets that were looped back - if network.networkGroup.ipListen.IsAddressSelf(sender.(*net.UDPAddr)) { - continue - } - - // For good network practice (and reducing amount of parallel connections), do not allow link-local to talk to non-link-local addresses. - if sender.(*net.UDPAddr).IP.IsLinkLocalUnicast() != network.address.IP.IsLinkLocalUnicast() { - continue - } - - //fmt.Printf("MulticastIPv6Listen from %s at network %s\n", sender.String(), network.address.String()) - - if length < protocol.PacketLengthMin { - // Discard packets that do not meet the minimum length. - continue - } - - // send the packet to a channel which is processed by multiple workers. - network.networkGroup.rawPacketsIncoming <- networkWire{network: network, sender: sender.(*net.UDPAddr), raw: buffer[:length], receiverPublicKey: ipv6MulticastPublicKey, unicast: false} - } -} - -// MulticastIPv6Send sends out a single multicast messages to discover peers at the same site -func (network *Network) MulticastIPv6Send() (err error) { - _, blockchainHeight, blockchainVersion := network.backend.UserBlockchain.Header() - packets := protocol.EncodeAnnouncement(true, true, nil, nil, nil, network.backend.FeatureSupport(), blockchainHeight, blockchainVersion, network.backend.userAgent) - if len(packets) == 0 { - return errors.New("error encoding multicast announcement") - } - - raw, err := protocol.PacketEncrypt(network.backend.PeerPrivateKey, ipv6MulticastPublicKey, &protocol.PacketRaw{Protocol: protocol.ProtocolVersion, Command: protocol.CommandLocalDiscovery, Payload: packets[0]}) - if err != nil { - return err - } - - // send out the wire - return network.send(network.multicastIP, ipv6MulticastPort, raw) -} +/* +File Name: Network IPv6 Multicast.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +IPv6 Multicast implementation to support discovery of peers within the same network (Site-local). +Loopback is enabled, which means that Multicast packets sent will be looped back and received by any local listeners. This allows to connect local processes with each other. + +Using the separate Multicast port, it allows sending unsolicited announcements without knowing the target's public key. Instead, a hard-coded key is used. + +The Multicast listener opens port 12912 with SO_REUSEADDR to allow multiple processes receive the incoming Multicast packets. +[1] mentions "If two sockets are bound to the same interface and port and are members of the same multicast group, data will be delivered to both sockets, rather than an arbitrarily chosen one." + +[1] https://docs.microsoft.com/en-us/windows/win32/winsock/using-so-reuseaddr-and-so-exclusiveaddruse +*/ + +package core + +import ( + "encoding/hex" + "errors" + "net" + "strconv" + "time" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/reuseport" + "golang.org/x/net/ipv6" +) + +// Multicast group is site-local. Group ID is 112. +const ipv6MulticastGroup = "ff05::112" +const ipv6MulticastPort = 12912 + +// special Public-Private Key pair for local discovery +var ipv6MulticastPrivateKey *btcec.PrivateKey +var ipv6MulticastPublicKey *btcec.PublicKey + +const ipv6MulticastPrivateKeyH = "016ad30bfb369926523bf18d136298b6c31d0817e9fb6c21feed89ae22cad788" + +func initMulticastIPv6() { + if configPK, err := hex.DecodeString(ipv6MulticastPrivateKeyH); err == nil { + ipv6MulticastPrivateKey, ipv6MulticastPublicKey = btcec.PrivKeyFromBytes(btcec.S256(), configPK) + } +} + +// MulticastIPv6Join joins the Multicast group +func (network *Network) MulticastIPv6Join() (err error) { + if ipv6MulticastPrivateKey == nil || ipv6MulticastPublicKey == nil { + return + } + + network.multicastIP = net.ParseIP(ipv6MulticastGroup) + + // listen on a special socket + network.multicastSocket, err = reuseport.ListenPacket("udp6", net.JoinHostPort(network.address.IP.String(), strconv.Itoa(ipv6MulticastPort))) + if err != nil { + network.backend.LogError("MulticastIPv6Join", "multicast socket listen on IP '%s' port '%d': %v\n", network.address.IP.String(), ipv6MulticastPort, err) + return err + } + + joinMulticastGroup := func(iface *net.Interface) (err error) { + pc := ipv6.NewPacketConn(network.multicastSocket) + if err := pc.JoinGroup(iface, &net.UDPAddr{IP: network.multicastIP}); err != nil { + //LogError("MulticastIPv6Join", "join multicast group iface '%s' multicast IP '%s' listen on IP '%s' port '%d': %v\n", iface.Name, network.multicastIP.String(), network.address.IP.String(), ipv6MulticastPort, err) + return err + } + + // receive messages from self or other processes running on the same computer + if loop, err := pc.MulticastLoopback(); err == nil && !loop { + if err := pc.SetMulticastLoopback(true); err != nil { + network.backend.LogError("MulticastIPv6Join", "setting multicast loopback status: %v\n", err) + } + } + + return nil + } + + // specific interface or join all? + if network.iface != nil { + if err = joinMulticastGroup(network.iface); err != nil { + return err + } + } else { + interfaceList, err := net.Interfaces() + if err != nil { + return err + } + + for _, ifaceSingle := range interfaceList { + joinMulticastGroup(&ifaceSingle) + } + } + + go network.MulticastIPv6Listen() + + return nil +} + +// MulticastIPv6Listen listens for incoming multicast packets +// Fork from network.Listen! Keep any changes synced. +func (network *Network) MulticastIPv6Listen() { + for { + // Buffer: Must be created for each packet as it is passed as pointer. + // If the buffer is too small, ReadFromUDP only reads until its length and returns this error: "wsarecvfrom: A message sent on a datagram socket was larger than the internal message buffer or some other network limit, or the buffer used to receive a datagram into was smaller than the datagram itself." + buffer := make([]byte, maxPacketSize) + length, sender, err := network.multicastSocket.ReadFrom(buffer) + + if err != nil { + network.backend.LogError("MulticastIPv6Listen", "receiving UDP message: %v\n", err) // Only log for debug purposes. + time.Sleep(time.Millisecond * 50) // In case of endless errors, prevent ddos of CPU. + continue + } + + // skip incoming packets that were looped back + if network.networkGroup.ipListen.IsAddressSelf(sender.(*net.UDPAddr)) { + continue + } + + // For good network practice (and reducing amount of parallel connections), do not allow link-local to talk to non-link-local addresses. + if sender.(*net.UDPAddr).IP.IsLinkLocalUnicast() != network.address.IP.IsLinkLocalUnicast() { + continue + } + + //fmt.Printf("MulticastIPv6Listen from %s at network %s\n", sender.String(), network.address.String()) + + if length < protocol.PacketLengthMin { + // Discard packets that do not meet the minimum length. + continue + } + + // send the packet to a channel which is processed by multiple workers. + network.networkGroup.rawPacketsIncoming <- networkWire{network: network, sender: sender.(*net.UDPAddr), raw: buffer[:length], receiverPublicKey: ipv6MulticastPublicKey, unicast: false} + } +} + +// MulticastIPv6Send sends out a single multicast messages to discover peers at the same site +func (network *Network) MulticastIPv6Send() (err error) { + _, blockchainHeight, blockchainVersion := network.backend.UserBlockchain.Header() + packets := protocol.EncodeAnnouncement(true, true, nil, nil, nil, network.backend.FeatureSupport(), blockchainHeight, blockchainVersion, network.backend.userAgent) + if len(packets) == 0 { + return errors.New("error encoding multicast announcement") + } + + raw, err := protocol.PacketEncrypt(network.backend.PeerPrivateKey, ipv6MulticastPublicKey, &protocol.PacketRaw{Protocol: protocol.ProtocolVersion, Command: protocol.CommandLocalDiscovery, Payload: packets[0]}) + if err != nil { + return err + } + + // send out the wire + return network.send(network.multicastIP, ipv6MulticastPort, raw) +} diff --git a/Network Init.go b/Network Init.go index 85b251d..739fb42 100644 --- a/Network Init.go +++ b/Network Init.go @@ -1,221 +1,221 @@ -/* -File Name: Network Init.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Magic 🪄 to start the network configuration with 0 manual input. Users may specify the list of IPs (and optional ports) to listen; otherwise it listens on all. -IPv6 is always preferred. -*/ - -package core - -import ( - "errors" - "math/rand" - "net" - "strconv" - "strings" - "sync" - "time" - - "github.com/PeernetOfficial/core/btcec" -) - -// networkWire is an incoming packet -type networkWire struct { - network *Network // network which received the packet - sender *net.UDPAddr // sender of the packet - receiverPublicKey *btcec.PublicKey // public key associated with the receiver - raw []byte // buffer - unicast bool // True if the message was sent via unicast. False if sent via IPv4 broadcast or IPv6 multicast. -} - -// initNetwork sets up the network configuration and starts listening. -func (backend *Backend) initNetwork() { - rand.Seed(time.Now().UnixNano()) // we are not using "crypto/rand" for speed tradeoff - - // start listen workers - if backend.Config.ListenWorkers == 0 { - backend.Config.ListenWorkers = 2 - } - if backend.Config.ListenWorkersLite == 0 { - backend.Config.ListenWorkersLite = 2 - } - for n := 0; n < backend.Config.ListenWorkers; n++ { - go backend.networks.packetWorker() - } - for n := 0; n < backend.Config.ListenWorkersLite; n++ { - go backend.networks.packetWorkerLite() - } - - // check if user specified where to listen - if len(backend.Config.Listen) > 0 { - for _, listenA := range backend.Config.Listen { - host, portA, err := net.SplitHostPort(listenA) - if err != nil && strings.Contains(err.Error(), "missing port in address") { // port is optional - host = listenA - portA = "0" - } else if err != nil { - backend.LogError("initNetwork", "invalid input listen address '%s': %s\n", listenA, err.Error()) - continue - } - - portI, _ := strconv.Atoi(portA) - - if _, err := backend.networks.PrepareListen(host, portI); err != nil { - backend.LogError("initNetwork", "listen on '%s': %s\n", listenA, err.Error()) - continue - } - } - - return - } - - // Listen on all IPv4 and IPv6 addresses - //if _, err := networks.PrepareListen("0.0.0.0", 0); err != nil { - // LogError("initNetwork", "listen on all IPv4 addresses (0.0.0.0): %s\n", err.Error()) - //} - //if _, err := networks.PrepareListen("::", 0); err != nil { - // LogError("initNetwork", "listen on all IPv6 addresses (::): %s\n", err.Error()) - //} - - // Listen on each network adapter on each IP. This guarantees the highest deliverability, even though it brings on additional challenges such as: - // * Packet duplicates on IPv6 Multicast (listening on multiple IPs and joining the group on the same adapter) and IPv4 Broadcast (listening on multiple IPs on the same adapter). - // * Local peers are more likely to connect on the same adapter via multiple IPs (i.e. link-local and others, including public IPv6 and temporary public IPv6). - // * Network adapters and IPs might change. Simplest case is if someone changes Wifi network. - interfaceList, err := net.Interfaces() - if err != nil { - backend.LogError("initNetwork", "enumerating network adapters failed: %s\n", err.Error()) - return - } - - for _, iface := range interfaceList { - addresses, err := iface.Addrs() - if err != nil { - backend.LogError("initNetwork", "enumerating IPs for network adapter '%s': %s\n", iface.Name, err.Error()) - continue - } - - backend.networks.ipListen.ifacesExist[iface.Name] = addresses - - backend.networks.InterfaceStart(iface, addresses) - } -} - -// InterfaceStart will start the listeners on all the IP addresses for the network -func (nets *Networks) InterfaceStart(iface net.Interface, addresses []net.Addr) (networksNew []*Network) { - for _, address := range addresses { - net1 := address.(*net.IPNet) - - // Do not listen on lookpback IPs. They are not even needed for discovery of machine-local peers (they will be discovered via regular multicast/broadcast). - if net1.IP.IsLoopback() { - continue - } - - networkNew, err := nets.PrepareListen(net1.IP.String(), 0) - - if err != nil { - // Do not log common errors: - // * "listen udp4 169.254.X.X:X: bind: The requested address is not valid in its context." - // Windows reports link-local addresses for inactive network adapters. - if net1.IP.IsLinkLocalUnicast() { - continue - } - - nets.backend.LogError("networks.InterfaceStart", "listening on network adapter '%s' IPv4 '%s': %s\n", iface.Name, net1.IP.String(), err.Error()) - continue - } - - nets.ipListen.Add(networkNew.address) - - nets.backend.LogError("networks.InterfaceStart", "listen on network '%s' UDP %s\n", iface.Name, networkNew.address.String()) - - networksNew = append(networksNew, networkNew) - } - - return -} - -// PrepareListen creates a new network and prepares to listen on the given IP address. If port is 0, one is chosen automatically. -func (nets *Networks) PrepareListen(ipA string, port int) (network *Network, err error) { - ip := net.ParseIP(ipA) - if ip == nil { - return nil, errors.New("invalid input IP") - } - - network = &Network{backend: nets.backend, networkGroup: nets} - network.terminateSignal = make(chan interface{}) - - // get the network interface that belongs to the IP - if !ip.IsUnspecified() { // checks for IPv4 "0.0.0.0" and IPv6 "::" - network.iface, network.ipnet = FindInterfaceByIP(ip) - if network.iface == nil { - return nil, errors.New("error finding the network interface belonging to IP") - } - } - - // open up the port - if err = network.AutoAssignPort(ip, port); err != nil { - return nil, err - } - - nets.Lock() - - // Success - port is open. Add to the list and start accepting incoming messages. - if IsIPv4(ip) { - nets.networks4 = append(nets.networks4, network) - nets.Unlock() - network.BroadcastIPv4() - } else { - nets.networks6 = append(nets.networks6, network) - nets.Unlock() - network.MulticastIPv6Join() - } - - go network.Listen() - - return network, nil -} - -// ipList keeps track of listened IP addresses and observed interfaces -type ipList struct { - ipListen map[string]struct{} // list of IPs currently listening on - sync.RWMutex // Mutex for list - ifacesExist map[string][]net.Addr // list of currently known interfaces with list of IP addresses -} - -// NewIPList creates a new list -func NewIPList() (list *ipList) { - return &ipList{ - ipListen: make(map[string]struct{}), - ifacesExist: make(map[string][]net.Addr), - } -} - -// Add adds a listening IP:Port to the list. -func (list *ipList) Add(addr *net.UDPAddr) { - list.Lock() - list.ipListen[net.JoinHostPort(addr.IP.String(), strconv.Itoa(addr.Port))] = struct{}{} - list.Unlock() -} - -// Remove removes a listening address from the list -func (list *ipList) Remove(addr *net.UDPAddr) { - list.Lock() - delete(list.ipListen, net.JoinHostPort(addr.IP.String(), strconv.Itoa(addr.Port))) - list.Unlock() -} - -// IsAddressSelf checks if the senders address is actually listening address. This prevents loopback packets from being considered. -// Note: This does not work when listening on 0.0.0.0 or ::1 and binding the sending socket to that. -func (list *ipList) IsAddressSelf(addr *net.UDPAddr) bool { - if addr == nil { - return false - } - - // do not use addr.String() since it addds the Zone for IPv6 which may be ambiguous (can be adapter name or address literal). - list.RLock() - _, ok := list.ipListen[net.JoinHostPort(addr.IP.String(), strconv.Itoa(addr.Port))] - list.RUnlock() - return ok -} +/* +File Name: Network Init.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Magic 🪄 to start the network configuration with 0 manual input. Users may specify the list of IPs (and optional ports) to listen; otherwise it listens on all. +IPv6 is always preferred. +*/ + +package core + +import ( + "errors" + "math/rand" + "net" + "strconv" + "strings" + "sync" + "time" + + "github.com/PeernetOfficial/core/btcec" +) + +// networkWire is an incoming packet +type networkWire struct { + network *Network // network which received the packet + sender *net.UDPAddr // sender of the packet + receiverPublicKey *btcec.PublicKey // public key associated with the receiver + raw []byte // buffer + unicast bool // True if the message was sent via unicast. False if sent via IPv4 broadcast or IPv6 multicast. +} + +// initNetwork sets up the network configuration and starts listening. +func (backend *Backend) initNetwork() { + rand.Seed(time.Now().UnixNano()) // we are not using "crypto/rand" for speed tradeoff + + // start listen workers + if backend.Config.ListenWorkers == 0 { + backend.Config.ListenWorkers = 2 + } + if backend.Config.ListenWorkersLite == 0 { + backend.Config.ListenWorkersLite = 2 + } + for n := 0; n < backend.Config.ListenWorkers; n++ { + go backend.networks.packetWorker() + } + for n := 0; n < backend.Config.ListenWorkersLite; n++ { + go backend.networks.packetWorkerLite() + } + + // check if user specified where to listen + if len(backend.Config.Listen) > 0 { + for _, listenA := range backend.Config.Listen { + host, portA, err := net.SplitHostPort(listenA) + if err != nil && strings.Contains(err.Error(), "missing port in address") { // port is optional + host = listenA + portA = "0" + } else if err != nil { + backend.LogError("initNetwork", "invalid input listen address '%s': %s\n", listenA, err.Error()) + continue + } + + portI, _ := strconv.Atoi(portA) + + if _, err := backend.networks.PrepareListen(host, portI); err != nil { + backend.LogError("initNetwork", "listen on '%s': %s\n", listenA, err.Error()) + continue + } + } + + return + } + + // Listen on all IPv4 and IPv6 addresses + //if _, err := networks.PrepareListen("0.0.0.0", 0); err != nil { + // LogError("initNetwork", "listen on all IPv4 addresses (0.0.0.0): %s\n", err.Error()) + //} + //if _, err := networks.PrepareListen("::", 0); err != nil { + // LogError("initNetwork", "listen on all IPv6 addresses (::): %s\n", err.Error()) + //} + + // Listen on each network adapter on each IP. This guarantees the highest deliverability, even though it brings on additional challenges such as: + // * Packet duplicates on IPv6 Multicast (listening on multiple IPs and joining the group on the same adapter) and IPv4 Broadcast (listening on multiple IPs on the same adapter). + // * Local peers are more likely to connect on the same adapter via multiple IPs (i.e. link-local and others, including public IPv6 and temporary public IPv6). + // * Network adapters and IPs might change. Simplest case is if someone changes Wifi network. + interfaceList, err := net.Interfaces() + if err != nil { + backend.LogError("initNetwork", "enumerating network adapters failed: %s\n", err.Error()) + return + } + + for _, iface := range interfaceList { + addresses, err := iface.Addrs() + if err != nil { + backend.LogError("initNetwork", "enumerating IPs for network adapter '%s': %s\n", iface.Name, err.Error()) + continue + } + + backend.networks.ipListen.ifacesExist[iface.Name] = addresses + + backend.networks.InterfaceStart(iface, addresses) + } +} + +// InterfaceStart will start the listeners on all the IP addresses for the network +func (nets *Networks) InterfaceStart(iface net.Interface, addresses []net.Addr) (networksNew []*Network) { + for _, address := range addresses { + net1 := address.(*net.IPNet) + + // Do not listen on lookpback IPs. They are not even needed for discovery of machine-local peers (they will be discovered via regular multicast/broadcast). + if net1.IP.IsLoopback() { + continue + } + + networkNew, err := nets.PrepareListen(net1.IP.String(), 0) + + if err != nil { + // Do not log common errors: + // * "listen udp4 169.254.X.X:X: bind: The requested address is not valid in its context." + // Windows reports link-local addresses for inactive network adapters. + if net1.IP.IsLinkLocalUnicast() { + continue + } + + nets.backend.LogError("networks.InterfaceStart", "listening on network adapter '%s' IPv4 '%s': %s\n", iface.Name, net1.IP.String(), err.Error()) + continue + } + + nets.ipListen.Add(networkNew.address) + + nets.backend.LogError("networks.InterfaceStart", "listen on network '%s' UDP %s\n", iface.Name, networkNew.address.String()) + + networksNew = append(networksNew, networkNew) + } + + return +} + +// PrepareListen creates a new network and prepares to listen on the given IP address. If port is 0, one is chosen automatically. +func (nets *Networks) PrepareListen(ipA string, port int) (network *Network, err error) { + ip := net.ParseIP(ipA) + if ip == nil { + return nil, errors.New("invalid input IP") + } + + network = &Network{backend: nets.backend, networkGroup: nets} + network.terminateSignal = make(chan interface{}) + + // get the network interface that belongs to the IP + if !ip.IsUnspecified() { // checks for IPv4 "0.0.0.0" and IPv6 "::" + network.iface, network.ipnet = FindInterfaceByIP(ip) + if network.iface == nil { + return nil, errors.New("error finding the network interface belonging to IP") + } + } + + // open up the port + if err = network.AutoAssignPort(ip, port); err != nil { + return nil, err + } + + nets.Lock() + + // Success - port is open. Add to the list and start accepting incoming messages. + if IsIPv4(ip) { + nets.networks4 = append(nets.networks4, network) + nets.Unlock() + network.BroadcastIPv4() + } else { + nets.networks6 = append(nets.networks6, network) + nets.Unlock() + network.MulticastIPv6Join() + } + + go network.Listen() + + return network, nil +} + +// ipList keeps track of listened IP addresses and observed interfaces +type ipList struct { + ipListen map[string]struct{} // list of IPs currently listening on + sync.RWMutex // Mutex for list + ifacesExist map[string][]net.Addr // list of currently known interfaces with list of IP addresses +} + +// NewIPList creates a new list +func NewIPList() (list *ipList) { + return &ipList{ + ipListen: make(map[string]struct{}), + ifacesExist: make(map[string][]net.Addr), + } +} + +// Add adds a listening IP:Port to the list. +func (list *ipList) Add(addr *net.UDPAddr) { + list.Lock() + list.ipListen[net.JoinHostPort(addr.IP.String(), strconv.Itoa(addr.Port))] = struct{}{} + list.Unlock() +} + +// Remove removes a listening address from the list +func (list *ipList) Remove(addr *net.UDPAddr) { + list.Lock() + delete(list.ipListen, net.JoinHostPort(addr.IP.String(), strconv.Itoa(addr.Port))) + list.Unlock() +} + +// IsAddressSelf checks if the senders address is actually listening address. This prevents loopback packets from being considered. +// Note: This does not work when listening on 0.0.0.0 or ::1 and binding the sending socket to that. +func (list *ipList) IsAddressSelf(addr *net.UDPAddr) bool { + if addr == nil { + return false + } + + // do not use addr.String() since it addds the Zone for IPv6 which may be ambiguous (can be adapter name or address literal). + list.RLock() + _, ok := list.ipListen[net.JoinHostPort(addr.IP.String(), strconv.Itoa(addr.Port))] + list.RUnlock() + return ok +} diff --git a/Network UPnP.go b/Network UPnP.go index 9ed331e..5106f41 100644 --- a/Network UPnP.go +++ b/Network UPnP.go @@ -1,191 +1,191 @@ -/* -File Name: Network UPnP.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Currently only supports IPv4 networks. -*/ - -package core - -import ( - "math/rand" - "net" - "time" - - "github.com/PeernetOfficial/core/upnp" -) - -func (nets *Networks) startUPnP() { - nets.upnpListInterfaces = make(map[string]struct{}) - - for _, cidr := range []string{ - "10.0.0.0/8", // RFC1918 - "172.16.0.0/12", // RFC1918 - "192.168.0.0/16", // RFC1918 - } { - if _, block, err := net.ParseCIDR(cidr); err == nil { - privateIPv4Blocks = append(privateIPv4Blocks, block) - } - } - - if nets.backend.Config.PortForward > 0 { - nets.backend.Config.EnableUPnP = false - } - if !nets.backend.Config.EnableUPnP { - return - } - - for _, network := range nets.networks4 { - go network.upnpAuto() - } -} - -// upnpIsEligible checks if the network is eligible for UPnP -func (network *Network) upnpIsEligible() bool { - // IPv4 only for now. - if !IsIPv4(network.address.IP) { - return false - } - - // The network interface must be known which indicates that the IP address is NOT a wildcard. - // Port forwarding requires to specify a local IP. In case of listening on a wildcard that would not be known and guessing (looking at you btcd) is not a solution. - if network.iface == nil || network.address.IP.IsUnspecified() { - return false - } - - // IPv4/IPv6: No link-local addresses, no loopback. Multicast would be invalid anyway. - if network.address.IP.IsLinkLocalUnicast() || network.address.IP.IsLoopback() || network.address.IP.IsMulticast() { - return false - } - - // IPv4: Must be private IP. - if IsIPv4(network.address.IP) && !isPrivateIP(network.address.IP) { - return false - } - - return true -} - -var privateIPv4Blocks []*net.IPNet - -func isPrivateIP(ip net.IP) bool { - for _, block := range privateIPv4Blocks { - if block.Contains(ip) { - return true - } - } - return false -} - -// upnpAuto runs a UPnP daemon to forward the port, refresh the forwarding and continuously monitor if the forwarding remains valid. -func (network *Network) upnpAuto() { - if !network.backend.Config.EnableUPnP || !network.upnpIsEligible() { - return - } - - nat, err := upnp.Discover(network.address.IP) - if err != nil { - return - } - - network.nat = nat - - externalIP, err := nat.GetExternalAddress() - if err != nil { - return - } - - network.ipExternal = externalIP - - // Only allow 1 UPnP worker at a time for registering the adapter. - network.networkGroup.upnpMutex.Lock() - defer network.networkGroup.upnpMutex.Unlock() - - // If there is already a running UPnP on the adapter, skip. - if _, ok := network.networkGroup.upnpListInterfaces[network.GetAdapterName()]; ok { - return - } - - if err := network.upnpTryPortForward(); err != nil { - return - } - - network.networkGroup.upnpListInterfaces[network.GetAdapterName()] = struct{}{} - - go network.upnpMonitorPortForward() -} - -// upnpMonitorPortForward monitors the port forwarding status -func (network *Network) upnpMonitorPortForward() { - ticker := time.NewTicker(time.Second * 10) - -monitorLoop: - for { - select { - case <-ticker.C: - case <-network.terminateSignal: - // Remove port mapping. Note that in case the network is unavailable this is likely to fail. - network.nat.DeletePortMapping("UDP", network.portExternal) - - network.portExternal = 0 - network.ipExternal = net.IP{} - - break monitorLoop - } - - // 3 tries - var err error - for n := 0; n < 3; n++ { - if err = network.upnpValidate(); err == nil { - continue monitorLoop - } - } - - // invalid :( - network.backend.LogError("upnpMonitorPortForward", "port forwarding invalidated for local IP %s (adapter %s) external IP %s port %d\n", network.address.String(), network.iface.Name, network.ipExternal.String(), network.portExternal) - - network.portExternal = 0 - network.ipExternal = net.IP{} - - break - } - - ticker.Stop() - - network.networkGroup.upnpMutex.Lock() - delete(network.networkGroup.upnpListInterfaces, network.GetAdapterName()) - network.networkGroup.upnpMutex.Unlock() -} - -func (network *Network) upnpTryPortForward() (err error) { - // Try forwarding the port. First to the same one listening, otherwise random. - mappedExternalPort, err := network.nat.AddPortMapping("UDP", network.address.IP, uint16(network.address.Port), uint16(network.address.Port), "Peernet", 0) - if err != nil { - mappedExternalPort, err = network.nat.AddPortMapping("UDP", network.address.IP, uint16(network.address.Port), uint16(randInt(1024, 65535)), "Peernet", 0) - } - if err != nil { - return err - } - - // validate - if err := network.upnpValidate(); err != nil { - return err - } - - // valid! - network.portExternal = mappedExternalPort - - return nil -} - -func randInt(min int, max int) int { - return min + rand.Intn(max-min) -} - -func (network *Network) upnpValidate() (err error) { - // TODO: Send special message which validates the UPnP mapping - return nil -} - -// TODO: Function to check if there is an existing port forward +/* +File Name: Network UPnP.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Currently only supports IPv4 networks. +*/ + +package core + +import ( + "math/rand" + "net" + "time" + + "github.com/PeernetOfficial/core/upnp" +) + +func (nets *Networks) startUPnP() { + nets.upnpListInterfaces = make(map[string]struct{}) + + for _, cidr := range []string{ + "10.0.0.0/8", // RFC1918 + "172.16.0.0/12", // RFC1918 + "192.168.0.0/16", // RFC1918 + } { + if _, block, err := net.ParseCIDR(cidr); err == nil { + privateIPv4Blocks = append(privateIPv4Blocks, block) + } + } + + if nets.backend.Config.PortForward > 0 { + nets.backend.Config.EnableUPnP = false + } + if !nets.backend.Config.EnableUPnP { + return + } + + for _, network := range nets.networks4 { + go network.upnpAuto() + } +} + +// upnpIsEligible checks if the network is eligible for UPnP +func (network *Network) upnpIsEligible() bool { + // IPv4 only for now. + if !IsIPv4(network.address.IP) { + return false + } + + // The network interface must be known which indicates that the IP address is NOT a wildcard. + // Port forwarding requires to specify a local IP. In case of listening on a wildcard that would not be known and guessing (looking at you btcd) is not a solution. + if network.iface == nil || network.address.IP.IsUnspecified() { + return false + } + + // IPv4/IPv6: No link-local addresses, no loopback. Multicast would be invalid anyway. + if network.address.IP.IsLinkLocalUnicast() || network.address.IP.IsLoopback() || network.address.IP.IsMulticast() { + return false + } + + // IPv4: Must be private IP. + if IsIPv4(network.address.IP) && !isPrivateIP(network.address.IP) { + return false + } + + return true +} + +var privateIPv4Blocks []*net.IPNet + +func isPrivateIP(ip net.IP) bool { + for _, block := range privateIPv4Blocks { + if block.Contains(ip) { + return true + } + } + return false +} + +// upnpAuto runs a UPnP daemon to forward the port, refresh the forwarding and continuously monitor if the forwarding remains valid. +func (network *Network) upnpAuto() { + if !network.backend.Config.EnableUPnP || !network.upnpIsEligible() { + return + } + + nat, err := upnp.Discover(network.address.IP) + if err != nil { + return + } + + network.nat = nat + + externalIP, err := nat.GetExternalAddress() + if err != nil { + return + } + + network.ipExternal = externalIP + + // Only allow 1 UPnP worker at a time for registering the adapter. + network.networkGroup.upnpMutex.Lock() + defer network.networkGroup.upnpMutex.Unlock() + + // If there is already a running UPnP on the adapter, skip. + if _, ok := network.networkGroup.upnpListInterfaces[network.GetAdapterName()]; ok { + return + } + + if err := network.upnpTryPortForward(); err != nil { + return + } + + network.networkGroup.upnpListInterfaces[network.GetAdapterName()] = struct{}{} + + go network.upnpMonitorPortForward() +} + +// upnpMonitorPortForward monitors the port forwarding status +func (network *Network) upnpMonitorPortForward() { + ticker := time.NewTicker(time.Second * 10) + +monitorLoop: + for { + select { + case <-ticker.C: + case <-network.terminateSignal: + // Remove port mapping. Note that in case the network is unavailable this is likely to fail. + network.nat.DeletePortMapping("UDP", network.portExternal) + + network.portExternal = 0 + network.ipExternal = net.IP{} + + break monitorLoop + } + + // 3 tries + var err error + for n := 0; n < 3; n++ { + if err = network.upnpValidate(); err == nil { + continue monitorLoop + } + } + + // invalid :( + network.backend.LogError("upnpMonitorPortForward", "port forwarding invalidated for local IP %s (adapter %s) external IP %s port %d\n", network.address.String(), network.iface.Name, network.ipExternal.String(), network.portExternal) + + network.portExternal = 0 + network.ipExternal = net.IP{} + + break + } + + ticker.Stop() + + network.networkGroup.upnpMutex.Lock() + delete(network.networkGroup.upnpListInterfaces, network.GetAdapterName()) + network.networkGroup.upnpMutex.Unlock() +} + +func (network *Network) upnpTryPortForward() (err error) { + // Try forwarding the port. First to the same one listening, otherwise random. + mappedExternalPort, err := network.nat.AddPortMapping("UDP", network.address.IP, uint16(network.address.Port), uint16(network.address.Port), "Peernet", 0) + if err != nil { + mappedExternalPort, err = network.nat.AddPortMapping("UDP", network.address.IP, uint16(network.address.Port), uint16(randInt(1024, 65535)), "Peernet", 0) + } + if err != nil { + return err + } + + // validate + if err := network.upnpValidate(); err != nil { + return err + } + + // valid! + network.portExternal = mappedExternalPort + + return nil +} + +func randInt(min int, max int) int { + return min + rand.Intn(max-min) +} + +func (network *Network) upnpValidate() (err error) { + // TODO: Send special message which validates the UPnP mapping + return nil +} + +// TODO: Function to check if there is an existing port forward diff --git a/Network.go b/Network.go index d00f213..a7f078c 100644 --- a/Network.go +++ b/Network.go @@ -1,434 +1,434 @@ -/* -File Name: Network.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "errors" - "net" - "sync" - "sync/atomic" - "time" - - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/upnp" -) - -// Network is a connection adapter through one network interface (adapter). -// Note that for each IP on the same adapter separate network entries are created. -type Network struct { - iface *net.Interface // Network interface belonging to the IP. May not be set. - ipnet *net.IPNet // IP network the listening address belongs to. May not be set. - address *net.UDPAddr // IP:Port where the server listens - socket *net.UDPConn // active socket for send/receive - multicastIP net.IP // Multicast IP, IPv6 only. - multicastSocket net.PacketConn // Multicast socket, IPv6 only. - broadcastSocket net.PacketConn // Broadcast socket, IPv4 only. - broadcastIPv4 []net.IP // Broadcast IPs, IPv4 only. - portExternal uint16 // External port. 0 if not known. - ipExternal net.IP // External IP of the network. Usually not known. - nat upnp.NAT // UPnP: NAT information - isTerminated bool // If true, the network was signaled for termination - terminateSignal chan interface{} // gets closed on termination signal, can be used in select via "case _ = <- network.terminateSignal:" - sync.RWMutex // for sychronized closing - networkGroup *Networks // Pointer to the pool of networks that this is part of - backend *Backend -} - -// Default ports to use. This may be randomized in the future to prevent fingerprinting (and subsequent blocking) by corporate and ISP firewalls. -const defaultPort = 'p' // 112 - -// AutoAssignPort assigns a port for the given IP. Use port 0 for zero configuration. -func (network *Network) AutoAssignPort(ip net.IP, port int) (err error) { - networkA := "udp6" - if IsIPv4(ip) { - networkA = "udp4" - } - - // A common error return is "bind: The requested address is not valid in its context.". - // This error was observed when the network interface might not be ready after boot but also when listening on a link-local IPv4 (169.254.) for an inactive adapter. - // Previously the algorithm retried up to n times, but this would unnecessarily delay startup in case the IP is actual unlistenable. - connectPortTry := func(port int) (address *net.UDPAddr, socket *net.UDPConn, err error) { - address = &net.UDPAddr{IP: ip, Port: port} - if socket, err = net.ListenUDP(networkA, address); err != nil { - return nil, nil, err - } - - if port == 0 { - localAddr := socket.LocalAddr() - if localAddr == nil { - return nil, nil, errors.New("invalid port assignment") - } - address.Port = localAddr.(*net.UDPAddr).Port - } - - return address, socket, nil - } - - if port != 0 { - network.address, network.socket, err = connectPortTry(port) - return err - } - - // try default main port, then random - if network.address, network.socket, err = connectPortTry(defaultPort); err == nil { - return nil - } - - if network.address, network.socket, err = connectPortTry(0); err == nil { - return nil - } - - return err -} - -// send sends a message -func (network *Network) send(IP net.IP, port int, raw []byte) (err error) { - _, err = network.socket.WriteTo(raw, &net.UDPAddr{IP: IP, Port: port}) - return err -} - -// Max packet size is 64 KB. -const maxPacketSize = 65536 - -// Listen starts listening for incoming packets on the given UDP connection -func (network *Network) Listen() { - if !network.address.IP.IsLinkLocalUnicast() { - if IsIPv4(network.address.IP) { - atomic.AddInt64(&network.networkGroup.countListen4, 1) - } else { - atomic.AddInt64(&network.networkGroup.countListen6, 1) - } - } - - for !network.isTerminated { - // Buffer: Must be created for each packet as it is passed as pointer. - // If the buffer is too small, ReadFromUDP only reads until its length and returns this error: "wsarecvfrom: A message sent on a datagram socket was larger than the internal message buffer or some other network limit, or the buffer used to receive a datagram into was smaller than the datagram itself." - buffer := make([]byte, maxPacketSize) - length, sender, err := network.socket.ReadFromUDP(buffer) - - if err != nil { - // Exit on closed socket. Error will be "use of closed network connection". - if network.isTerminated { - return - } - - network.backend.LogError("Listen", "receiving UDP message: %v\n", err) // Only log for debug purposes. - time.Sleep(time.Millisecond * 50) // In case of endless errors, prevent ddos of CPU. - continue - } - - // handle lite packets before regular ones - if isLite, err := network.networkGroup.LiteRouter.IsPacketLite(buffer[:length]); isLite && err != nil { - continue - } else if isLite { - network.networkGroup.litePacketsIncoming <- networkWire{network: network, sender: sender, raw: buffer[:length], receiverPublicKey: network.backend.PeerPublicKey, unicast: true} - continue - } - - if length < protocol.PacketLengthMin { - // Discard packets that do not meet the minimum length. - continue - } - - // send the packet to a channel which is processed by multiple workers. - network.networkGroup.rawPacketsIncoming <- networkWire{network: network, sender: sender, raw: buffer[:length], receiverPublicKey: network.backend.PeerPublicKey, unicast: true} - } -} - -// packetWorker handles incoming packets. -func (nets *Networks) packetWorker() { - for packet := range nets.rawPacketsIncoming { - decoded, senderPublicKey, err := protocol.PacketDecrypt(packet.raw, packet.receiverPublicKey) - if err != nil { - //LogError("packetWorker", "decrypting packet from '%s': %s\n", packet.sender.String(), err.Error()) // Only log for debug purposes. - continue - } - - // immediately discard message if sender = self - if senderPublicKey.IsEqual(nets.backend.PeerPublicKey) { - continue - } - - // supported protocol version - if decoded.Protocol != 0 { - continue - } - - connection := &Connection{backend: nets.backend, Network: packet.network, Address: packet.sender, Status: ConnectionActive} - - nets.backend.Filters.PacketIn(decoded, senderPublicKey, connection) - - // A peer structure will always be returned, even if the peer won't be added to the peer list. - peer, added := nets.backend.PeerlistAdd(senderPublicKey, connection) - if !added { - connection = peer.registerConnection(connection) - } - - atomic.AddUint64(&peer.StatsPacketReceived, 1) - connection.LastPacketIn = time.Now() - - // process the packet - raw := &protocol.MessageRaw{SenderPublicKey: senderPublicKey, PacketRaw: *decoded} - - switch decoded.Command { - case protocol.CommandAnnouncement: // Announce - if announce, _ := protocol.DecodeAnnouncement(raw); announce != nil { - // Update known internal/external port and User Agent - connection.PortInternal = announce.PortInternal - connection.PortExternal = announce.PortExternal - connection.Firewall = announce.Features&(1< 0 - if len(announce.UserAgent) > 0 { - peer.UserAgent = announce.UserAgent - } - peer.Features = announce.Features - - isBlockchainUpdate := peer.BlockchainHeight != announce.BlockchainHeight || peer.BlockchainVersion != announce.BlockchainVersion - peer.BlockchainHeight = announce.BlockchainHeight - peer.BlockchainVersion = announce.BlockchainVersion - peer.blockchainLastRefresh = time.Now() - - nets.backend.Filters.MessageIn(peer, raw, announce) - - peer.cmdAnouncement(announce, connection) - - if isBlockchainUpdate { - peer.remoteBlockchainUpdate() - } - } - - case protocol.CommandResponse: // Response - if response, _ := protocol.DecodeResponse(raw); response != nil { - // Validate sequence number which prevents unsolicited responses. - isLast := response.IsLast() - sequenceInfo, valid, rtt := nets.Sequences.ValidateSequence(raw.SenderPublicKey, raw.Sequence, isLast, !isLast) - if !valid { - //LogError("packetWorker", "message with invalid sequence %d command %d from %s\n", raw.Sequence, raw.Command, raw.connection.Address.String()) // Only log for debug purposes. - continue - } else if rtt > 0 { - connection.RoundTripTime = rtt - } - raw.SequenceInfo = sequenceInfo - - // Update known internal/external port and User Agent - connection.PortInternal = response.PortInternal - connection.PortExternal = response.PortExternal - connection.Firewall = response.Features&(1< 0 - if len(response.UserAgent) > 0 { - peer.UserAgent = response.UserAgent - } - peer.Features = response.Features - - isBlockchainUpdate := peer.BlockchainHeight != response.BlockchainHeight || peer.BlockchainVersion != response.BlockchainVersion - peer.BlockchainHeight = response.BlockchainHeight - peer.BlockchainVersion = response.BlockchainVersion - peer.blockchainLastRefresh = time.Now() - - nets.backend.Filters.MessageIn(peer, raw, response) - - peer.cmdResponse(response, connection) - - if isBlockchainUpdate { - peer.remoteBlockchainUpdate() - } - } - - case protocol.CommandLocalDiscovery: // Local discovery, sent via IPv4 broadcast and IPv6 multicast - if announce, _ := protocol.DecodeAnnouncement(raw); announce != nil { - if len(announce.UserAgent) > 0 { - peer.UserAgent = announce.UserAgent - } - peer.Features = announce.Features - - isBlockchainUpdate := peer.BlockchainHeight != announce.BlockchainHeight || peer.BlockchainVersion != announce.BlockchainVersion - peer.BlockchainHeight = announce.BlockchainHeight - peer.BlockchainVersion = announce.BlockchainVersion - peer.blockchainLastRefresh = time.Now() - - nets.backend.Filters.MessageIn(peer, raw, announce) - - peer.cmdLocalDiscovery(announce, connection) - - if isBlockchainUpdate { - peer.remoteBlockchainUpdate() - } - } - - case protocol.CommandPing: // Ping - nets.backend.Filters.MessageIn(peer, raw, nil) - peer.cmdPing(raw, connection) - - case protocol.CommandPong: // Ping - // Validate sequence number which prevents unsolicited responses. - sequenceInfo, valid, rtt := nets.Sequences.ValidateSequence(raw.SenderPublicKey, raw.Sequence, true, false) - if !valid { - //LogError("packetWorker", "message with invalid sequence %d command %d from %s\n", raw.Sequence, raw.Command, raw.connection.Address.String()) // Only log for debug purposes. - continue - } else if rtt > 0 { - connection.RoundTripTime = rtt - } - raw.SequenceInfo = sequenceInfo - - nets.backend.Filters.MessageIn(peer, raw, nil) - - peer.cmdPong(raw, connection) - - case protocol.CommandChat: // Chat [debug] - nets.backend.Filters.MessageIn(peer, raw, nil) - peer.cmdChat(raw, connection) - - case protocol.CommandTraverse: - if traverse, _ := protocol.DecodeTraverse(raw); traverse != nil { - nets.backend.Filters.MessageIn(peer, raw, traverse) - if traverse.TargetPeer.IsEqual(nets.backend.PeerPublicKey) && traverse.AuthorizedRelayPeer.IsEqual(peer.PublicKey) { - peer.cmdTraverseReceive(traverse) - } else if traverse.AuthorizedRelayPeer.IsEqual(nets.backend.PeerPublicKey) { - peer.cmdTraverseForward(traverse) - } - } - - case protocol.CommandTransfer: - if msg, _ := protocol.DecodeTransfer(raw); msg != nil { - // Validate sequence number which prevents unsolicited responses. - isLast := msg.IsLast() - sequenceInfo, valid, rtt := nets.Sequences.ValidateSequenceBi(raw.SenderPublicKey, raw.Sequence, isLast) - if msg.Control != protocol.TransferControlRequestStart && !valid { - //LogError("packetWorker", "message with invalid sequence %d command %d from %s\n", raw.Sequence, raw.Command, raw.connection.Address.String()) // Only log for debug purposes. - continue - } else if rtt > 0 { - connection.RoundTripTime = rtt - } - raw.SequenceInfo = sequenceInfo - - peer.cmdTransfer(msg, connection) - } - - case protocol.CommandGetBlock: - if msg, _ := protocol.DecodeGetBlock(raw); msg != nil { - // Validate sequence number which prevents unsolicited responses. - isLast := msg.IsLast() - sequenceInfo, valid, rtt := nets.Sequences.ValidateSequenceBi(raw.SenderPublicKey, raw.Sequence, isLast) - if msg.Control != protocol.GetBlockControlRequestStart && !valid { - //LogError("packetWorker", "message with invalid sequence %d command %d from %s\n", raw.Sequence, raw.Command, raw.connection.Address.String()) // Only log for debug purposes. - continue - } else if rtt > 0 { - connection.RoundTripTime = rtt - } - raw.SequenceInfo = sequenceInfo - - peer.cmdGetBlock(msg, connection) - } - - default: // Unknown command - nets.backend.Filters.MessageIn(peer, raw, nil) - - } - - } -} - -// GetNetworks returns the list of connected networks -func (backend *Backend) GetNetworks(networkType int) (networksConnected []*Network) { - switch networkType { - case 4: - return backend.networks.networks4 - case 6: - return backend.networks.networks6 - } - return nil -} - -// GetListen returns connectivity information -func (network *Network) GetListen() (listen *net.UDPAddr, multicastIPv6 net.IP, broadcastIPv4 []net.IP, ipExternal net.IP, portExternal uint16) { - return network.address, network.multicastIP, network.broadcastIPv4, network.ipExternal, network.portExternal -} - -// GetAdapterName returns the adapter name, if available -func (network *Network) GetAdapterName() string { - if network.iface != nil { - return network.iface.Name - } - return "[unknown adapter]" -} - -// Terminate sends the termination signal to all workers. It is safe to call Terminate multiple times. -func (network *Network) Terminate() { - network.Lock() - defer network.Unlock() - - if network.isTerminated { - return - } - - if !network.address.IP.IsLinkLocalUnicast() { - if IsIPv4(network.address.IP) { - atomic.AddInt64(&network.networkGroup.countListen4, -1) - } else { - atomic.AddInt64(&network.networkGroup.countListen6, -1) - } - } - - // set the termination signal - network.isTerminated = true - close(network.terminateSignal) // safety guaranteed via lock - network.socket.Close() // Will stop the listener from blocking on network.socket.ReadFromUDP - - network.networkGroup.ipListen.Remove(network.address) -} - -// SelfReportedPorts returns the internal and external ports as self-reported by the peer to others. -func (network *Network) SelfReportedPorts() (portI, portE uint16) { - // The internal port is set to where the network listens on. - // Datacenter: This should usually be the same as the outgoing port. - // NAT: The internal port will be different than the outgoing one. - portI = uint16(network.address.Port) - - // External port: This is usually unknown, except in these 2 cases: - // UPnP: The port is forwarded automatically. - // Manual override in config: The user can specify a (global) incoming port that must be open on all listening IPs. - // This external port will be then passed onto other peers who will use it to connect. - portE = network.portExternal - - if network.backend.Config.PortForward > 0 { - portE = network.backend.Config.PortForward - } - - return portI, portE -} - -// FeatureSupport returns supported features by this peer -func (backend *Backend) FeatureSupport() (feature byte) { - if backend.networks.countListen4 > 0 { - feature |= 1 << protocol.FeatureIPv4Listen - } - if backend.networks.countListen6 > 0 { - feature |= 1 << protocol.FeatureIPv6Listen - } - if backend.networks.localFirewall { - feature |= 1 << protocol.FeatureFirewall - } - return feature -} - -// Handles incoming lite packets. It will decrypt them as needed. -func (nets *Networks) packetWorkerLite() { - for wire := range nets.litePacketsIncoming { - packet, err := nets.LiteRouter.PacketLiteDecode(wire.raw) - if err != nil { - continue - } - - // Handle the received data. Note this is called in the same Go routine. - // The underlying data receiver must not stall. - if v, ok := packet.Session.Data.(*VirtualPacketConn); ok { - // update stats TODO - //atomic.AddUint64(&packet.Session.Data.(*VirtualPacketConn).peer.StatsPacketReceived, 1) - //connection.LastPacketIn = time.Now() - - v.receiveData(packet.Payload) - } - } -} +/* +File Name: Network.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "errors" + "net" + "sync" + "sync/atomic" + "time" + + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/upnp" +) + +// Network is a connection adapter through one network interface (adapter). +// Note that for each IP on the same adapter separate network entries are created. +type Network struct { + iface *net.Interface // Network interface belonging to the IP. May not be set. + ipnet *net.IPNet // IP network the listening address belongs to. May not be set. + address *net.UDPAddr // IP:Port where the server listens + socket *net.UDPConn // active socket for send/receive + multicastIP net.IP // Multicast IP, IPv6 only. + multicastSocket net.PacketConn // Multicast socket, IPv6 only. + broadcastSocket net.PacketConn // Broadcast socket, IPv4 only. + broadcastIPv4 []net.IP // Broadcast IPs, IPv4 only. + portExternal uint16 // External port. 0 if not known. + ipExternal net.IP // External IP of the network. Usually not known. + nat upnp.NAT // UPnP: NAT information + isTerminated bool // If true, the network was signaled for termination + terminateSignal chan interface{} // gets closed on termination signal, can be used in select via "case _ = <- network.terminateSignal:" + sync.RWMutex // for sychronized closing + networkGroup *Networks // Pointer to the pool of networks that this is part of + backend *Backend +} + +// Default ports to use. This may be randomized in the future to prevent fingerprinting (and subsequent blocking) by corporate and ISP firewalls. +const defaultPort = 'p' // 112 + +// AutoAssignPort assigns a port for the given IP. Use port 0 for zero configuration. +func (network *Network) AutoAssignPort(ip net.IP, port int) (err error) { + networkA := "udp6" + if IsIPv4(ip) { + networkA = "udp4" + } + + // A common error return is "bind: The requested address is not valid in its context.". + // This error was observed when the network interface might not be ready after boot but also when listening on a link-local IPv4 (169.254.) for an inactive adapter. + // Previously the algorithm retried up to n times, but this would unnecessarily delay startup in case the IP is actual unlistenable. + connectPortTry := func(port int) (address *net.UDPAddr, socket *net.UDPConn, err error) { + address = &net.UDPAddr{IP: ip, Port: port} + if socket, err = net.ListenUDP(networkA, address); err != nil { + return nil, nil, err + } + + if port == 0 { + localAddr := socket.LocalAddr() + if localAddr == nil { + return nil, nil, errors.New("invalid port assignment") + } + address.Port = localAddr.(*net.UDPAddr).Port + } + + return address, socket, nil + } + + if port != 0 { + network.address, network.socket, err = connectPortTry(port) + return err + } + + // try default main port, then random + if network.address, network.socket, err = connectPortTry(defaultPort); err == nil { + return nil + } + + if network.address, network.socket, err = connectPortTry(0); err == nil { + return nil + } + + return err +} + +// send sends a message +func (network *Network) send(IP net.IP, port int, raw []byte) (err error) { + _, err = network.socket.WriteTo(raw, &net.UDPAddr{IP: IP, Port: port}) + return err +} + +// Max packet size is 64 KB. +const maxPacketSize = 65536 + +// Listen starts listening for incoming packets on the given UDP connection +func (network *Network) Listen() { + if !network.address.IP.IsLinkLocalUnicast() { + if IsIPv4(network.address.IP) { + atomic.AddInt64(&network.networkGroup.countListen4, 1) + } else { + atomic.AddInt64(&network.networkGroup.countListen6, 1) + } + } + + for !network.isTerminated { + // Buffer: Must be created for each packet as it is passed as pointer. + // If the buffer is too small, ReadFromUDP only reads until its length and returns this error: "wsarecvfrom: A message sent on a datagram socket was larger than the internal message buffer or some other network limit, or the buffer used to receive a datagram into was smaller than the datagram itself." + buffer := make([]byte, maxPacketSize) + length, sender, err := network.socket.ReadFromUDP(buffer) + + if err != nil { + // Exit on closed socket. Error will be "use of closed network connection". + if network.isTerminated { + return + } + + network.backend.LogError("Listen", "receiving UDP message: %v\n", err) // Only log for debug purposes. + time.Sleep(time.Millisecond * 50) // In case of endless errors, prevent ddos of CPU. + continue + } + + // handle lite packets before regular ones + if isLite, err := network.networkGroup.LiteRouter.IsPacketLite(buffer[:length]); isLite && err != nil { + continue + } else if isLite { + network.networkGroup.litePacketsIncoming <- networkWire{network: network, sender: sender, raw: buffer[:length], receiverPublicKey: network.backend.PeerPublicKey, unicast: true} + continue + } + + if length < protocol.PacketLengthMin { + // Discard packets that do not meet the minimum length. + continue + } + + // send the packet to a channel which is processed by multiple workers. + network.networkGroup.rawPacketsIncoming <- networkWire{network: network, sender: sender, raw: buffer[:length], receiverPublicKey: network.backend.PeerPublicKey, unicast: true} + } +} + +// packetWorker handles incoming packets. +func (nets *Networks) packetWorker() { + for packet := range nets.rawPacketsIncoming { + decoded, senderPublicKey, err := protocol.PacketDecrypt(packet.raw, packet.receiverPublicKey) + if err != nil { + //LogError("packetWorker", "decrypting packet from '%s': %s\n", packet.sender.String(), err.Error()) // Only log for debug purposes. + continue + } + + // immediately discard message if sender = self + if senderPublicKey.IsEqual(nets.backend.PeerPublicKey) { + continue + } + + // supported protocol version + if decoded.Protocol != 0 { + continue + } + + connection := &Connection{backend: nets.backend, Network: packet.network, Address: packet.sender, Status: ConnectionActive} + + nets.backend.Filters.PacketIn(decoded, senderPublicKey, connection) + + // A peer structure will always be returned, even if the peer won't be added to the peer list. + peer, added := nets.backend.PeerlistAdd(senderPublicKey, connection) + if !added { + connection = peer.registerConnection(connection) + } + + atomic.AddUint64(&peer.StatsPacketReceived, 1) + connection.LastPacketIn = time.Now() + + // process the packet + raw := &protocol.MessageRaw{SenderPublicKey: senderPublicKey, PacketRaw: *decoded} + + switch decoded.Command { + case protocol.CommandAnnouncement: // Announce + if announce, _ := protocol.DecodeAnnouncement(raw); announce != nil { + // Update known internal/external port and User Agent + connection.PortInternal = announce.PortInternal + connection.PortExternal = announce.PortExternal + connection.Firewall = announce.Features&(1< 0 + if len(announce.UserAgent) > 0 { + peer.UserAgent = announce.UserAgent + } + peer.Features = announce.Features + + isBlockchainUpdate := peer.BlockchainHeight != announce.BlockchainHeight || peer.BlockchainVersion != announce.BlockchainVersion + peer.BlockchainHeight = announce.BlockchainHeight + peer.BlockchainVersion = announce.BlockchainVersion + peer.blockchainLastRefresh = time.Now() + + nets.backend.Filters.MessageIn(peer, raw, announce) + + peer.cmdAnouncement(announce, connection) + + if isBlockchainUpdate { + peer.remoteBlockchainUpdate() + } + } + + case protocol.CommandResponse: // Response + if response, _ := protocol.DecodeResponse(raw); response != nil { + // Validate sequence number which prevents unsolicited responses. + isLast := response.IsLast() + sequenceInfo, valid, rtt := nets.Sequences.ValidateSequence(raw.SenderPublicKey, raw.Sequence, isLast, !isLast) + if !valid { + //LogError("packetWorker", "message with invalid sequence %d command %d from %s\n", raw.Sequence, raw.Command, raw.connection.Address.String()) // Only log for debug purposes. + continue + } else if rtt > 0 { + connection.RoundTripTime = rtt + } + raw.SequenceInfo = sequenceInfo + + // Update known internal/external port and User Agent + connection.PortInternal = response.PortInternal + connection.PortExternal = response.PortExternal + connection.Firewall = response.Features&(1< 0 + if len(response.UserAgent) > 0 { + peer.UserAgent = response.UserAgent + } + peer.Features = response.Features + + isBlockchainUpdate := peer.BlockchainHeight != response.BlockchainHeight || peer.BlockchainVersion != response.BlockchainVersion + peer.BlockchainHeight = response.BlockchainHeight + peer.BlockchainVersion = response.BlockchainVersion + peer.blockchainLastRefresh = time.Now() + + nets.backend.Filters.MessageIn(peer, raw, response) + + peer.cmdResponse(response, connection) + + if isBlockchainUpdate { + peer.remoteBlockchainUpdate() + } + } + + case protocol.CommandLocalDiscovery: // Local discovery, sent via IPv4 broadcast and IPv6 multicast + if announce, _ := protocol.DecodeAnnouncement(raw); announce != nil { + if len(announce.UserAgent) > 0 { + peer.UserAgent = announce.UserAgent + } + peer.Features = announce.Features + + isBlockchainUpdate := peer.BlockchainHeight != announce.BlockchainHeight || peer.BlockchainVersion != announce.BlockchainVersion + peer.BlockchainHeight = announce.BlockchainHeight + peer.BlockchainVersion = announce.BlockchainVersion + peer.blockchainLastRefresh = time.Now() + + nets.backend.Filters.MessageIn(peer, raw, announce) + + peer.cmdLocalDiscovery(announce, connection) + + if isBlockchainUpdate { + peer.remoteBlockchainUpdate() + } + } + + case protocol.CommandPing: // Ping + nets.backend.Filters.MessageIn(peer, raw, nil) + peer.cmdPing(raw, connection) + + case protocol.CommandPong: // Ping + // Validate sequence number which prevents unsolicited responses. + sequenceInfo, valid, rtt := nets.Sequences.ValidateSequence(raw.SenderPublicKey, raw.Sequence, true, false) + if !valid { + //LogError("packetWorker", "message with invalid sequence %d command %d from %s\n", raw.Sequence, raw.Command, raw.connection.Address.String()) // Only log for debug purposes. + continue + } else if rtt > 0 { + connection.RoundTripTime = rtt + } + raw.SequenceInfo = sequenceInfo + + nets.backend.Filters.MessageIn(peer, raw, nil) + + peer.cmdPong(raw, connection) + + case protocol.CommandChat: // Chat [debug] + nets.backend.Filters.MessageIn(peer, raw, nil) + peer.cmdChat(raw, connection) + + case protocol.CommandTraverse: + if traverse, _ := protocol.DecodeTraverse(raw); traverse != nil { + nets.backend.Filters.MessageIn(peer, raw, traverse) + if traverse.TargetPeer.IsEqual(nets.backend.PeerPublicKey) && traverse.AuthorizedRelayPeer.IsEqual(peer.PublicKey) { + peer.cmdTraverseReceive(traverse) + } else if traverse.AuthorizedRelayPeer.IsEqual(nets.backend.PeerPublicKey) { + peer.cmdTraverseForward(traverse) + } + } + + case protocol.CommandTransfer: + if msg, _ := protocol.DecodeTransfer(raw); msg != nil { + // Validate sequence number which prevents unsolicited responses. + isLast := msg.IsLast() + sequenceInfo, valid, rtt := nets.Sequences.ValidateSequenceBi(raw.SenderPublicKey, raw.Sequence, isLast) + if msg.Control != protocol.TransferControlRequestStart && !valid { + //LogError("packetWorker", "message with invalid sequence %d command %d from %s\n", raw.Sequence, raw.Command, raw.connection.Address.String()) // Only log for debug purposes. + continue + } else if rtt > 0 { + connection.RoundTripTime = rtt + } + raw.SequenceInfo = sequenceInfo + + peer.cmdTransfer(msg, connection) + } + + case protocol.CommandGetBlock: + if msg, _ := protocol.DecodeGetBlock(raw); msg != nil { + // Validate sequence number which prevents unsolicited responses. + isLast := msg.IsLast() + sequenceInfo, valid, rtt := nets.Sequences.ValidateSequenceBi(raw.SenderPublicKey, raw.Sequence, isLast) + if msg.Control != protocol.GetBlockControlRequestStart && !valid { + //LogError("packetWorker", "message with invalid sequence %d command %d from %s\n", raw.Sequence, raw.Command, raw.connection.Address.String()) // Only log for debug purposes. + continue + } else if rtt > 0 { + connection.RoundTripTime = rtt + } + raw.SequenceInfo = sequenceInfo + + peer.cmdGetBlock(msg, connection) + } + + default: // Unknown command + nets.backend.Filters.MessageIn(peer, raw, nil) + + } + + } +} + +// GetNetworks returns the list of connected networks +func (backend *Backend) GetNetworks(networkType int) (networksConnected []*Network) { + switch networkType { + case 4: + return backend.networks.networks4 + case 6: + return backend.networks.networks6 + } + return nil +} + +// GetListen returns connectivity information +func (network *Network) GetListen() (listen *net.UDPAddr, multicastIPv6 net.IP, broadcastIPv4 []net.IP, ipExternal net.IP, portExternal uint16) { + return network.address, network.multicastIP, network.broadcastIPv4, network.ipExternal, network.portExternal +} + +// GetAdapterName returns the adapter name, if available +func (network *Network) GetAdapterName() string { + if network.iface != nil { + return network.iface.Name + } + return "[unknown adapter]" +} + +// Terminate sends the termination signal to all workers. It is safe to call Terminate multiple times. +func (network *Network) Terminate() { + network.Lock() + defer network.Unlock() + + if network.isTerminated { + return + } + + if !network.address.IP.IsLinkLocalUnicast() { + if IsIPv4(network.address.IP) { + atomic.AddInt64(&network.networkGroup.countListen4, -1) + } else { + atomic.AddInt64(&network.networkGroup.countListen6, -1) + } + } + + // set the termination signal + network.isTerminated = true + close(network.terminateSignal) // safety guaranteed via lock + network.socket.Close() // Will stop the listener from blocking on network.socket.ReadFromUDP + + network.networkGroup.ipListen.Remove(network.address) +} + +// SelfReportedPorts returns the internal and external ports as self-reported by the peer to others. +func (network *Network) SelfReportedPorts() (portI, portE uint16) { + // The internal port is set to where the network listens on. + // Datacenter: This should usually be the same as the outgoing port. + // NAT: The internal port will be different than the outgoing one. + portI = uint16(network.address.Port) + + // External port: This is usually unknown, except in these 2 cases: + // UPnP: The port is forwarded automatically. + // Manual override in config: The user can specify a (global) incoming port that must be open on all listening IPs. + // This external port will be then passed onto other peers who will use it to connect. + portE = network.portExternal + + if network.backend.Config.PortForward > 0 { + portE = network.backend.Config.PortForward + } + + return portI, portE +} + +// FeatureSupport returns supported features by this peer +func (backend *Backend) FeatureSupport() (feature byte) { + if backend.networks.countListen4 > 0 { + feature |= 1 << protocol.FeatureIPv4Listen + } + if backend.networks.countListen6 > 0 { + feature |= 1 << protocol.FeatureIPv6Listen + } + if backend.networks.localFirewall { + feature |= 1 << protocol.FeatureFirewall + } + return feature +} + +// Handles incoming lite packets. It will decrypt them as needed. +func (nets *Networks) packetWorkerLite() { + for wire := range nets.litePacketsIncoming { + packet, err := nets.LiteRouter.PacketLiteDecode(wire.raw) + if err != nil { + continue + } + + // Handle the received data. Note this is called in the same Go routine. + // The underlying data receiver must not stall. + if v, ok := packet.Session.Data.(*VirtualPacketConn); ok { + // update stats TODO + //atomic.AddUint64(&packet.Session.Data.(*VirtualPacketConn).peer.StatsPacketReceived, 1) + //connection.LastPacketIn = time.Now() + + v.receiveData(packet.Payload) + } + } +} diff --git a/Networks.go b/Networks.go index de8929c..991aeee 100644 --- a/Networks.go +++ b/Networks.go @@ -1,94 +1,94 @@ -/* -File Name: Networks.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "os" - "sync" - - "github.com/PeernetOfficial/core/protocol" -) - -// Networks is the collection of all connected networks -type Networks struct { - // networks is a list of all connected networks - networks4, networks6 []*Network - - // Mutex for both network lists. Higher granularity currently not needed. - sync.RWMutex - - // countListenX is the number of networks listened to, excluding link-local only listeners. This number might be different than len(networksN). - // This is useful to determine if there are any IPv4 or IPv6 listeners for potential external connections. This can be used to determine IPv4_LISTEN and IPv6_LISTEN. - countListen4, countListen6 int64 - - // channel for processing incoming decoded packets by workers, across all networks - rawPacketsIncoming chan networkWire - litePacketsIncoming chan networkWire - - // Sequences keeps track of all message sequence number, regardless of the network connection. - Sequences *protocol.SequenceManager - - // Keep track of valid IDs for lite packets. - LiteRouter *protocol.LiteRouter - - // ipListen keeps a simple list of IPs listened to. This allows quickly identifying if an IP matches with a listened one. - ipListen *ipList - - // localFirewall indicates if a local firewall may drop unsolicited incoming packets - localFirewall bool - - // UPnP data - upnpListInterfaces map[string]struct{} - upnpMutex sync.RWMutex - - // backend - backend *Backend -} - -// ReplyTimeout is the round-trip timeout for message sequences. -const ReplyTimeout = 20 - -func (backend *Backend) initMessageSequence() { - backend.networks = &Networks{backend: backend} - - backend.networks.rawPacketsIncoming = make(chan networkWire, 1000) // buffer up to 1000 UDP packets before they get buffered by the OS network stack and eventually dropped - backend.networks.litePacketsIncoming = make(chan networkWire, 1000) // buffer up to 1000 UDP packets before they get buffered by the OS network stack and eventually dropped - - backend.networks.Sequences = protocol.NewSequenceManager(ReplyTimeout) - backend.networks.LiteRouter = protocol.NewLiteRouter() - - backend.networks.ipListen = NewIPList() - - // There is currently no suitable live firewall detection code. Instead, there is the config flag. - // Windows: If the user runs as non-admin, it can be assumed that the Windows Firewall creates a rule to drop unsolicited incoming packets. - // Changing the Windows Firewall (via netsh or otherwise) requires elevated admin rights. - // This flag will be passed on to other peers to indicate that uncontacted peers shall use the Traverse message for establishing connections. - backend.firewallDetectIndicatorFile() - backend.networks.localFirewall = backend.Config.LocalFirewall -} - -const firewallIndicatorFile = "firewallnotset" - -// Detects the file "firewallnotset". It may be set by an Peernet client installer to indicate the presence of a local firewall. -func (backend *Backend) firewallDetectIndicatorFile() { - if _, err := os.Stat(firewallIndicatorFile); err == nil { - // If the config firewall flag isn't set, set it. - if !backend.Config.LocalFirewall { - backend.Config.LocalFirewall = true - - backend.SaveConfig() - } - - // the indicator is one-time only, remove - os.Remove(firewallIndicatorFile) - } -} - -// List of all lite sessions -func (backend *Backend) LiteSessions() (sessions []*protocol.LiteID) { - return backend.networks.LiteRouter.All() -} +/* +File Name: Networks.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "os" + "sync" + + "github.com/PeernetOfficial/core/protocol" +) + +// Networks is the collection of all connected networks +type Networks struct { + // networks is a list of all connected networks + networks4, networks6 []*Network + + // Mutex for both network lists. Higher granularity currently not needed. + sync.RWMutex + + // countListenX is the number of networks listened to, excluding link-local only listeners. This number might be different than len(networksN). + // This is useful to determine if there are any IPv4 or IPv6 listeners for potential external connections. This can be used to determine IPv4_LISTEN and IPv6_LISTEN. + countListen4, countListen6 int64 + + // channel for processing incoming decoded packets by workers, across all networks + rawPacketsIncoming chan networkWire + litePacketsIncoming chan networkWire + + // Sequences keeps track of all message sequence number, regardless of the network connection. + Sequences *protocol.SequenceManager + + // Keep track of valid IDs for lite packets. + LiteRouter *protocol.LiteRouter + + // ipListen keeps a simple list of IPs listened to. This allows quickly identifying if an IP matches with a listened one. + ipListen *ipList + + // localFirewall indicates if a local firewall may drop unsolicited incoming packets + localFirewall bool + + // UPnP data + upnpListInterfaces map[string]struct{} + upnpMutex sync.RWMutex + + // backend + backend *Backend +} + +// ReplyTimeout is the round-trip timeout for message sequences. +const ReplyTimeout = 20 + +func (backend *Backend) initMessageSequence() { + backend.networks = &Networks{backend: backend} + + backend.networks.rawPacketsIncoming = make(chan networkWire, 1000) // buffer up to 1000 UDP packets before they get buffered by the OS network stack and eventually dropped + backend.networks.litePacketsIncoming = make(chan networkWire, 1000) // buffer up to 1000 UDP packets before they get buffered by the OS network stack and eventually dropped + + backend.networks.Sequences = protocol.NewSequenceManager(ReplyTimeout) + backend.networks.LiteRouter = protocol.NewLiteRouter() + + backend.networks.ipListen = NewIPList() + + // There is currently no suitable live firewall detection code. Instead, there is the config flag. + // Windows: If the user runs as non-admin, it can be assumed that the Windows Firewall creates a rule to drop unsolicited incoming packets. + // Changing the Windows Firewall (via netsh or otherwise) requires elevated admin rights. + // This flag will be passed on to other peers to indicate that uncontacted peers shall use the Traverse message for establishing connections. + backend.firewallDetectIndicatorFile() + backend.networks.localFirewall = backend.Config.LocalFirewall +} + +const firewallIndicatorFile = "firewallnotset" + +// Detects the file "firewallnotset". It may be set by an Peernet client installer to indicate the presence of a local firewall. +func (backend *Backend) firewallDetectIndicatorFile() { + if _, err := os.Stat(firewallIndicatorFile); err == nil { + // If the config firewall flag isn't set, set it. + if !backend.Config.LocalFirewall { + backend.Config.LocalFirewall = true + + backend.SaveConfig() + } + + // the indicator is one-time only, remove + os.Remove(firewallIndicatorFile) + } +} + +// List of all lite sessions +func (backend *Backend) LiteSessions() (sessions []*protocol.LiteID) { + return backend.networks.LiteRouter.All() +} diff --git a/Peer ID.go b/Peer ID.go index 2736a14..b736dd2 100644 --- a/Peer ID.go +++ b/Peer ID.go @@ -1,308 +1,308 @@ -/* -File Name: Peer ID.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "encoding/hex" - "errors" - "math/rand" - "net" - "os" - "sync" - "time" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/dht" - "github.com/PeernetOfficial/core/protocol" -) - -func (backend *Backend) initPeerID() { - backend.peerList = make(map[[btcec.PubKeyBytesLenCompressed]byte]*PeerInfo) - backend.nodeList = make(map[[protocol.HashSize]byte]*PeerInfo) - - // load existing key from config, if available - if len(backend.Config.PrivateKey) > 0 { - configPK, err := hex.DecodeString(backend.Config.PrivateKey) - if err == nil { - backend.PeerPrivateKey, backend.PeerPublicKey = btcec.PrivKeyFromBytes(btcec.S256(), configPK) - backend.nodeID = protocol.PublicKey2NodeID(backend.PeerPublicKey) - - if backend.Config.AutoUpdateSeedList { - backend.configUpdateSeedList() - } - return - } - - backend.LogError("initPeerID", "private key in config is corrupted! Error: %s\n", err.Error()) - os.Exit(ExitPrivateKeyCorrupt) - } - - // if the peer ID is empty, create a new user public-private key pair - var err error - backend.PeerPrivateKey, backend.PeerPublicKey, err = Secp256k1NewPrivateKey() - if err != nil { - backend.LogError("initPeerID", "generating public-private key pairs: %s\n", err.Error()) - os.Exit(ExitPrivateKeyCreate) - } - backend.nodeID = protocol.PublicKey2NodeID(backend.PeerPublicKey) - - // save the newly generated private key into the config - backend.Config.PrivateKey = hex.EncodeToString(backend.PeerPrivateKey.Serialize()) - - backend.SaveConfig() -} - -// Secp256k1NewPrivateKey creates a new public-private key pair -func Secp256k1NewPrivateKey() (privateKey *btcec.PrivateKey, publicKey *btcec.PublicKey, err error) { - key, err := btcec.NewPrivateKey(btcec.S256()) - if err != nil { - return nil, nil, err - } - - return key, (*btcec.PublicKey)(&key.PublicKey), nil -} - -// ExportPrivateKey returns the peers public and private key -func (backend *Backend) ExportPrivateKey() (privateKey *btcec.PrivateKey, publicKey *btcec.PublicKey) { - return backend.PeerPrivateKey, backend.PeerPublicKey -} - -// SelfNodeID returns the node ID used for DHT -func (backend *Backend) SelfNodeID() []byte { - return backend.nodeID -} - -// SelfUserAgent returns the User Agent -func (backend *Backend) SelfUserAgent() string { - return backend.userAgent -} - -// PeerInfo stores information about a single remote peer -type PeerInfo struct { - PublicKey *btcec.PublicKey // Public key - NodeID []byte // Node ID in Kademlia network = blake3(Public Key). - connectionActive []*Connection // List of active established connections to the peer. - connectionInactive []*Connection // List of former connections that are no longer valid. They may be removed after a while. - connectionLatest *Connection // Latest valid connection. - sync.RWMutex // Mutex for access to list of connections. - messageSequence uint32 // Sequence number. Increased with every message. - IsRootPeer bool // Whether the peer is a trusted root peer. - UserAgent string // User Agent reported by remote peer. Empty if no Announcement/Response message was yet received. - Features uint8 // Feature bit array. 0 = IPv4_LISTEN, 1 = IPv6_LISTEN, 1 = FIREWALL - isVirtual bool // Whether it is a virtual peer for establishing a connection. - targetAddresses []*peerAddress // Virtual peer: Addresses to send any replies. - traversePeer *PeerInfo // Virtual peer: Same field as in connection. - BlockchainHeight uint64 // Blockchain height - BlockchainVersion uint64 // Blockchain version - blockchainLastRefresh time.Time // Last refresh of the blockchain info. - - // statistics - StatsPacketSent uint64 // Count of packets sent - StatsPacketReceived uint64 // Count of packets received - - Backend *Backend -} - -type peerAddress struct { - IP net.IP - Port uint16 - PortInternal uint16 -} - -// PeerlistAdd adds a new peer to the peer list. It does not validate the peer info. If the peer is already added, it does nothing. Connections must be live. -func (backend *Backend) PeerlistAdd(PublicKey *btcec.PublicKey, connections ...*Connection) (peer *PeerInfo, added bool) { - if len(connections) == 0 { - return nil, false - } - publicKeyCompressed := publicKey2Compressed(PublicKey) - - backend.peerlistMutex.Lock() - defer backend.peerlistMutex.Unlock() - - peer, ok := backend.peerList[publicKeyCompressed] - if ok { - return peer, false - } - - peer = &PeerInfo{Backend: backend, PublicKey: PublicKey, connectionActive: connections, connectionLatest: connections[0], NodeID: protocol.PublicKey2NodeID(PublicKey), messageSequence: rand.Uint32()} - _, peer.IsRootPeer = rootPeers[publicKeyCompressed] - - backend.peerList[publicKeyCompressed] = peer - - // also add to mirrored nodeList - var nodeID [protocol.HashSize]byte - copy(nodeID[:], peer.NodeID) - backend.nodeList[nodeID] = peer - - // add to Kademlia - backend.nodesDHT.AddNode(&dht.Node{ID: peer.NodeID, Info: peer}) - - // TODO: If the node isn't added to Kademlia, it should be either added temporarily to the peerList with an expiration, or to a temp list, or not at all. - - // send to all channels non-blocking - for _, monitor := range backend.peerMonitor { - select { - case monitor <- peer: - default: - } - } - - backend.Filters.NewPeer(peer, connections[0]) - backend.Filters.NewPeerConnection(peer, connections[0]) - - return peer, true -} - -// PeerlistRemove removes a peer from the peer list. -func (backend *Backend) PeerlistRemove(peer *PeerInfo) { - backend.peerlistMutex.Lock() - defer backend.peerlistMutex.Unlock() - - // remove from Kademlia - backend.nodesDHT.RemoveNode(peer.NodeID) - - delete(backend.peerList, publicKey2Compressed(peer.PublicKey)) - - var nodeID [protocol.HashSize]byte - copy(nodeID[:], peer.NodeID) - - delete(backend.nodeList, nodeID) -} - -// PeerlistGet returns the full peer list -func (backend *Backend) PeerlistGet() (peers []*PeerInfo) { - backend.peerlistMutex.RLock() - defer backend.peerlistMutex.RUnlock() - - for _, peer := range backend.peerList { - peers = append(peers, peer) - } - - return peers -} - -// PeerlistLookup returns the peer from the list with the public key -func (backend *Backend) PeerlistLookup(publicKey *btcec.PublicKey) (peer *PeerInfo) { - backend.peerlistMutex.RLock() - defer backend.peerlistMutex.RUnlock() - - return backend.peerList[publicKey2Compressed(publicKey)] -} - -// NodelistLookup returns the peer from the list with the node ID -func (backend *Backend) NodelistLookup(nodeID []byte) (peer *PeerInfo) { - backend.peerlistMutex.RLock() - defer backend.peerlistMutex.RUnlock() - - var nodeID2 [protocol.HashSize]byte - copy(nodeID2[:], nodeID) - - return backend.nodeList[nodeID2] -} - -// PeerlistCount returns the current count of peers in the peer list -func (backend *Backend) PeerlistCount() (count int) { - backend.peerlistMutex.RLock() - defer backend.peerlistMutex.RUnlock() - - return len(backend.peerList) -} - -func publicKey2Compressed(publicKey *btcec.PublicKey) [btcec.PubKeyBytesLenCompressed]byte { - var key [btcec.PubKeyBytesLenCompressed]byte - copy(key[:], publicKey.SerializeCompressed()) - return key -} - -// records2Nodes translates infoPeer structures to nodes reported by the peer. If the reported nodes are not in the peer table, it will create temporary PeerInfo structures. -// LastContact is passed on in the Node.LastSeen field. -func (peerSource *PeerInfo) records2Nodes(records []protocol.PeerRecord) (nodes []*dht.Node) { - for _, record := range records { - if peerSource.Backend.isReturnedPeerBadQuality(&record) { - continue - } - - var peer *PeerInfo - if record.PublicKey.IsEqual(peerSource.PublicKey) { - // Special case if peer that stores info = sender. In that case IP:Port in the record would be empty anyway. - peer = peerSource - } else if peer = peerSource.Backend.PeerlistLookup(record.PublicKey); peer == nil { - // Create temporary peer which is not added to the global list and not added to Kademlia. - // traversePeer is set to the peer who provided the node information. - addresses := peerRecordToAddresses(&record) - if len(addresses) == 0 { - continue - } - - peer = &PeerInfo{Backend: peerSource.Backend, PublicKey: record.PublicKey, connectionActive: nil, connectionLatest: nil, NodeID: protocol.PublicKey2NodeID(record.PublicKey), messageSequence: rand.Uint32(), isVirtual: true, targetAddresses: addresses, traversePeer: peerSource, Features: record.Features} - } - - nodes = append(nodes, &dht.Node{ID: peer.NodeID, LastSeen: record.LastContactT, Info: peer}) - } - - return -} - -// selfPeerRecord returns self as peer record -func (backend *Backend) selfPeerRecord() (result protocol.PeerRecord) { - return protocol.PeerRecord{ - PublicKey: backend.PeerPublicKey, - NodeID: backend.nodeID, - //IP: network.address.IP, - //Port: uint16(network.address.Port), - LastContact: 0, - Features: backend.FeatureSupport(), - } -} - -// registerPeerMonitor registers a channel to receive all new peers -func (backend *Backend) registerPeerMonitor(channel chan<- *PeerInfo) { - backend.peerlistMutex.Lock() - defer backend.peerlistMutex.Unlock() - - backend.peerMonitor = append(backend.peerMonitor, channel) -} - -// unregisterPeerMonitor unregisters a channel -func (backend *Backend) unregisterPeerMonitor(channel chan<- *PeerInfo) { - backend.peerlistMutex.Lock() - defer backend.peerlistMutex.Unlock() - - for n, channel2 := range backend.peerMonitor { - if channel == channel2 { - peerMonitorNew := backend.peerMonitor[:n] - if n < len(backend.peerMonitor)-1 { - peerMonitorNew = append(peerMonitorNew, backend.peerMonitor[n+1:]...) - } - backend.peerMonitor = peerMonitorNew - break - } - } -} - -// DeleteAccount deletes the account -func (backend *Backend) DeleteAccount() { - // delete the blockchain - backend.UserBlockchain.DeleteBlockchain() - - // delete the warehouse - backend.UserWarehouse.DeleteWarehouse() - - // delete the private key - backend.Config.PrivateKey = "" - backend.SaveConfig() -} - -// PublicKeyFromPeerID decodes the peer ID (hex encoded) into a public key. -func PublicKeyFromPeerID(peerID string) (publicKey *btcec.PublicKey, err error) { - hash, err := hex.DecodeString(peerID) - if err != nil || len(hash) != 33 { - return nil, errors.New("invalid peer ID length") - } - - return btcec.ParsePubKey(hash, btcec.S256()) -} +/* +File Name: Peer ID.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "encoding/hex" + "errors" + "math/rand" + "net" + "os" + "sync" + "time" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/dht" + "github.com/PeernetOfficial/core/protocol" +) + +func (backend *Backend) initPeerID() { + backend.peerList = make(map[[btcec.PubKeyBytesLenCompressed]byte]*PeerInfo) + backend.nodeList = make(map[[protocol.HashSize]byte]*PeerInfo) + + // load existing key from config, if available + if len(backend.Config.PrivateKey) > 0 { + configPK, err := hex.DecodeString(backend.Config.PrivateKey) + if err == nil { + backend.PeerPrivateKey, backend.PeerPublicKey = btcec.PrivKeyFromBytes(btcec.S256(), configPK) + backend.nodeID = protocol.PublicKey2NodeID(backend.PeerPublicKey) + + if backend.Config.AutoUpdateSeedList { + backend.configUpdateSeedList() + } + return + } + + backend.LogError("initPeerID", "private key in config is corrupted! Error: %s\n", err.Error()) + os.Exit(ExitPrivateKeyCorrupt) + } + + // if the peer ID is empty, create a new user public-private key pair + var err error + backend.PeerPrivateKey, backend.PeerPublicKey, err = Secp256k1NewPrivateKey() + if err != nil { + backend.LogError("initPeerID", "generating public-private key pairs: %s\n", err.Error()) + os.Exit(ExitPrivateKeyCreate) + } + backend.nodeID = protocol.PublicKey2NodeID(backend.PeerPublicKey) + + // save the newly generated private key into the config + backend.Config.PrivateKey = hex.EncodeToString(backend.PeerPrivateKey.Serialize()) + + backend.SaveConfig() +} + +// Secp256k1NewPrivateKey creates a new public-private key pair +func Secp256k1NewPrivateKey() (privateKey *btcec.PrivateKey, publicKey *btcec.PublicKey, err error) { + key, err := btcec.NewPrivateKey(btcec.S256()) + if err != nil { + return nil, nil, err + } + + return key, (*btcec.PublicKey)(&key.PublicKey), nil +} + +// ExportPrivateKey returns the peers public and private key +func (backend *Backend) ExportPrivateKey() (privateKey *btcec.PrivateKey, publicKey *btcec.PublicKey) { + return backend.PeerPrivateKey, backend.PeerPublicKey +} + +// SelfNodeID returns the node ID used for DHT +func (backend *Backend) SelfNodeID() []byte { + return backend.nodeID +} + +// SelfUserAgent returns the User Agent +func (backend *Backend) SelfUserAgent() string { + return backend.userAgent +} + +// PeerInfo stores information about a single remote peer +type PeerInfo struct { + PublicKey *btcec.PublicKey // Public key + NodeID []byte // Node ID in Kademlia network = blake3(Public Key). + connectionActive []*Connection // List of active established connections to the peer. + connectionInactive []*Connection // List of former connections that are no longer valid. They may be removed after a while. + connectionLatest *Connection // Latest valid connection. + sync.RWMutex // Mutex for access to list of connections. + messageSequence uint32 // Sequence number. Increased with every message. + IsRootPeer bool // Whether the peer is a trusted root peer. + UserAgent string // User Agent reported by remote peer. Empty if no Announcement/Response message was yet received. + Features uint8 // Feature bit array. 0 = IPv4_LISTEN, 1 = IPv6_LISTEN, 1 = FIREWALL + isVirtual bool // Whether it is a virtual peer for establishing a connection. + targetAddresses []*peerAddress // Virtual peer: Addresses to send any replies. + traversePeer *PeerInfo // Virtual peer: Same field as in connection. + BlockchainHeight uint64 // Blockchain height + BlockchainVersion uint64 // Blockchain version + blockchainLastRefresh time.Time // Last refresh of the blockchain info. + + // statistics + StatsPacketSent uint64 // Count of packets sent + StatsPacketReceived uint64 // Count of packets received + + Backend *Backend +} + +type peerAddress struct { + IP net.IP + Port uint16 + PortInternal uint16 +} + +// PeerlistAdd adds a new peer to the peer list. It does not validate the peer info. If the peer is already added, it does nothing. Connections must be live. +func (backend *Backend) PeerlistAdd(PublicKey *btcec.PublicKey, connections ...*Connection) (peer *PeerInfo, added bool) { + if len(connections) == 0 { + return nil, false + } + publicKeyCompressed := publicKey2Compressed(PublicKey) + + backend.peerlistMutex.Lock() + defer backend.peerlistMutex.Unlock() + + peer, ok := backend.peerList[publicKeyCompressed] + if ok { + return peer, false + } + + peer = &PeerInfo{Backend: backend, PublicKey: PublicKey, connectionActive: connections, connectionLatest: connections[0], NodeID: protocol.PublicKey2NodeID(PublicKey), messageSequence: rand.Uint32()} + _, peer.IsRootPeer = rootPeers[publicKeyCompressed] + + backend.peerList[publicKeyCompressed] = peer + + // also add to mirrored nodeList + var nodeID [protocol.HashSize]byte + copy(nodeID[:], peer.NodeID) + backend.nodeList[nodeID] = peer + + // add to Kademlia + backend.nodesDHT.AddNode(&dht.Node{ID: peer.NodeID, Info: peer}) + + // TODO: If the node isn't added to Kademlia, it should be either added temporarily to the peerList with an expiration, or to a temp list, or not at all. + + // send to all channels non-blocking + for _, monitor := range backend.peerMonitor { + select { + case monitor <- peer: + default: + } + } + + backend.Filters.NewPeer(peer, connections[0]) + backend.Filters.NewPeerConnection(peer, connections[0]) + + return peer, true +} + +// PeerlistRemove removes a peer from the peer list. +func (backend *Backend) PeerlistRemove(peer *PeerInfo) { + backend.peerlistMutex.Lock() + defer backend.peerlistMutex.Unlock() + + // remove from Kademlia + backend.nodesDHT.RemoveNode(peer.NodeID) + + delete(backend.peerList, publicKey2Compressed(peer.PublicKey)) + + var nodeID [protocol.HashSize]byte + copy(nodeID[:], peer.NodeID) + + delete(backend.nodeList, nodeID) +} + +// PeerlistGet returns the full peer list +func (backend *Backend) PeerlistGet() (peers []*PeerInfo) { + backend.peerlistMutex.RLock() + defer backend.peerlistMutex.RUnlock() + + for _, peer := range backend.peerList { + peers = append(peers, peer) + } + + return peers +} + +// PeerlistLookup returns the peer from the list with the public key +func (backend *Backend) PeerlistLookup(publicKey *btcec.PublicKey) (peer *PeerInfo) { + backend.peerlistMutex.RLock() + defer backend.peerlistMutex.RUnlock() + + return backend.peerList[publicKey2Compressed(publicKey)] +} + +// NodelistLookup returns the peer from the list with the node ID +func (backend *Backend) NodelistLookup(nodeID []byte) (peer *PeerInfo) { + backend.peerlistMutex.RLock() + defer backend.peerlistMutex.RUnlock() + + var nodeID2 [protocol.HashSize]byte + copy(nodeID2[:], nodeID) + + return backend.nodeList[nodeID2] +} + +// PeerlistCount returns the current count of peers in the peer list +func (backend *Backend) PeerlistCount() (count int) { + backend.peerlistMutex.RLock() + defer backend.peerlistMutex.RUnlock() + + return len(backend.peerList) +} + +func publicKey2Compressed(publicKey *btcec.PublicKey) [btcec.PubKeyBytesLenCompressed]byte { + var key [btcec.PubKeyBytesLenCompressed]byte + copy(key[:], publicKey.SerializeCompressed()) + return key +} + +// records2Nodes translates infoPeer structures to nodes reported by the peer. If the reported nodes are not in the peer table, it will create temporary PeerInfo structures. +// LastContact is passed on in the Node.LastSeen field. +func (peerSource *PeerInfo) records2Nodes(records []protocol.PeerRecord) (nodes []*dht.Node) { + for _, record := range records { + if peerSource.Backend.isReturnedPeerBadQuality(&record) { + continue + } + + var peer *PeerInfo + if record.PublicKey.IsEqual(peerSource.PublicKey) { + // Special case if peer that stores info = sender. In that case IP:Port in the record would be empty anyway. + peer = peerSource + } else if peer = peerSource.Backend.PeerlistLookup(record.PublicKey); peer == nil { + // Create temporary peer which is not added to the global list and not added to Kademlia. + // traversePeer is set to the peer who provided the node information. + addresses := peerRecordToAddresses(&record) + if len(addresses) == 0 { + continue + } + + peer = &PeerInfo{Backend: peerSource.Backend, PublicKey: record.PublicKey, connectionActive: nil, connectionLatest: nil, NodeID: protocol.PublicKey2NodeID(record.PublicKey), messageSequence: rand.Uint32(), isVirtual: true, targetAddresses: addresses, traversePeer: peerSource, Features: record.Features} + } + + nodes = append(nodes, &dht.Node{ID: peer.NodeID, LastSeen: record.LastContactT, Info: peer}) + } + + return +} + +// selfPeerRecord returns self as peer record +func (backend *Backend) selfPeerRecord() (result protocol.PeerRecord) { + return protocol.PeerRecord{ + PublicKey: backend.PeerPublicKey, + NodeID: backend.nodeID, + //IP: network.address.IP, + //Port: uint16(network.address.Port), + LastContact: 0, + Features: backend.FeatureSupport(), + } +} + +// registerPeerMonitor registers a channel to receive all new peers +func (backend *Backend) registerPeerMonitor(channel chan<- *PeerInfo) { + backend.peerlistMutex.Lock() + defer backend.peerlistMutex.Unlock() + + backend.peerMonitor = append(backend.peerMonitor, channel) +} + +// unregisterPeerMonitor unregisters a channel +func (backend *Backend) unregisterPeerMonitor(channel chan<- *PeerInfo) { + backend.peerlistMutex.Lock() + defer backend.peerlistMutex.Unlock() + + for n, channel2 := range backend.peerMonitor { + if channel == channel2 { + peerMonitorNew := backend.peerMonitor[:n] + if n < len(backend.peerMonitor)-1 { + peerMonitorNew = append(peerMonitorNew, backend.peerMonitor[n+1:]...) + } + backend.peerMonitor = peerMonitorNew + break + } + } +} + +// DeleteAccount deletes the account +func (backend *Backend) DeleteAccount() { + // delete the blockchain + backend.UserBlockchain.DeleteBlockchain() + + // delete the warehouse + backend.UserWarehouse.DeleteWarehouse() + + // delete the private key + backend.Config.PrivateKey = "" + backend.SaveConfig() +} + +// PublicKeyFromPeerID decodes the peer ID (hex encoded) into a public key. +func PublicKeyFromPeerID(peerID string) (publicKey *btcec.PublicKey, err error) { + hash, err := hex.DecodeString(peerID) + if err != nil || len(hash) != 33 { + return nil, errors.New("invalid peer ID length") + } + + return btcec.ParsePubKey(hash, btcec.S256()) +} diff --git a/Peernet.go b/Peernet.go index b9ea5f9..d2902e6 100644 --- a/Peernet.go +++ b/Peernet.go @@ -1,122 +1,122 @@ -/* -File Name: Peernet.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "sync" - - "github.com/PeernetOfficial/core/blockchain" - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/dht" - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/search" - "github.com/PeernetOfficial/core/store" - "github.com/PeernetOfficial/core/warehouse" -) - -// Init initializes the client. If the config file does not exist or is empty, a default one will be created. -// The User Agent must be provided in the form "Application Name/1.0". -// The returned status is of type ExitX. Anything other than ExitSuccess indicates a fatal failure. -func Init(UserAgent string, ConfigFilename string, Filters *Filters, ConfigOut interface{}) (backend *Backend, status int, err error) { - if UserAgent == "" { - return - } - - backend = &Backend{ - ConfigFilename: ConfigFilename, - userAgent: UserAgent, - Stdout: newMultiWriter(), - } - - if Filters != nil { - backend.Filters = *Filters - } - - // The configuration and log init are fatal events if they fail. - if status, err = LoadConfig(ConfigFilename, &backend.Config); status != ExitSuccess { - return nil, status, err - } - if ConfigOut != nil { - if status, err = LoadConfig(ConfigFilename, ConfigOut); status != ExitSuccess { - return nil, status, err - } - backend.ConfigClient = ConfigOut - } - - if err = backend.initLog(); err != nil { - return nil, ExitErrorLogInit, err - } - - backend.initFilters() - backend.initPeerID() - backend.initUserBlockchain() - backend.initUserWarehouse() - backend.initKademlia() - backend.initMessageSequence() - backend.initSeedList() - initMulticastIPv6() - initBroadcastIPv4() - backend.initStore() - backend.initNetwork() - backend.initBlockchainCache() - - if backend.SearchIndex, err = search.InitSearchIndexStore(backend.Config.SearchIndex); err != nil { - backend.LogError("Init", "search index '%s' init: %s", backend.Config.SearchIndex, err.Error()) - } else { - backend.userBlockchainUpdateSearchIndex() - } - - return backend, ExitSuccess, nil -} - -// Connect starts bootstrapping and local peer discovery. -func (backend *Backend) Connect() { - go backend.bootstrapKademlia() - go backend.bootstrap() - go backend.networks.autoMulticastBroadcast() - go backend.autoPingAll() - go backend.networks.networkChangeMonitor() - go backend.networks.startUPnP() - go backend.autoBucketRefresh() -} - -// The Backend represents an instance of a Peernet client to be used by a frontend. -// Global variables and init functions are to be merged. -type Backend struct { - ConfigFilename string // Filename of the configuration file. - Config *Config // Core configuration - ConfigClient interface{} // Custom configuration from the client - Filters Filters // Filters allow to install hooks. - userAgent string // User Agent - GlobalBlockchainCache *BlockchainCache // Caches blockchains of other peers. - SearchIndex *search.SearchIndexStore // Search index of blockchain records. - networks *Networks // All connected networks. - dhtStore store.Store // dhtStore contains all key-value data served via DHT - UserBlockchain *blockchain.Blockchain // UserBlockchain is the user's blockchain and exports functions to directly read and write it - UserWarehouse *warehouse.Warehouse // UserWarehouse is the user's warehouse for storing files that are shared - nodesDHT *dht.DHT // Nodes connected in the DHT. - - // peerID is the current peer's ID. It is a ECDSA (secp256k1) 257-bit public key. - PeerPrivateKey *btcec.PrivateKey - PeerPublicKey *btcec.PublicKey - - // The node ID is the blake3 hash of the public key compressed form. - nodeID []byte - - // peerList keeps track of all peers - peerList map[[btcec.PubKeyBytesLenCompressed]byte]*PeerInfo - peerlistMutex sync.RWMutex - - // nodeList is a mirror of peerList but using the node ID - nodeList map[[protocol.HashSize]byte]*PeerInfo - - // peerMonitor is a list of channels receiving information about new peers - peerMonitor []chan<- *PeerInfo - - // Stdout bundles any output for the end-user. Writers may subscribe/unsubscribe. - Stdout *multiWriter -} +/* +File Name: Peernet.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "sync" + + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/dht" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/search" + "github.com/PeernetOfficial/core/store" + "github.com/PeernetOfficial/core/warehouse" +) + +// Init initializes the client. If the config file does not exist or is empty, a default one will be created. +// The User Agent must be provided in the form "Application Name/1.0". +// The returned status is of type ExitX. Anything other than ExitSuccess indicates a fatal failure. +func Init(UserAgent string, ConfigFilename string, Filters *Filters, ConfigOut interface{}) (backend *Backend, status int, err error) { + if UserAgent == "" { + return + } + + backend = &Backend{ + ConfigFilename: ConfigFilename, + userAgent: UserAgent, + Stdout: newMultiWriter(), + } + + if Filters != nil { + backend.Filters = *Filters + } + + // The configuration and log init are fatal events if they fail. + if status, err = LoadConfig(ConfigFilename, &backend.Config); status != ExitSuccess { + return nil, status, err + } + if ConfigOut != nil { + if status, err = LoadConfig(ConfigFilename, ConfigOut); status != ExitSuccess { + return nil, status, err + } + backend.ConfigClient = ConfigOut + } + + if err = backend.initLog(); err != nil { + return nil, ExitErrorLogInit, err + } + + backend.initFilters() + backend.initPeerID() + backend.initUserBlockchain() + backend.initUserWarehouse() + backend.initKademlia() + backend.initMessageSequence() + backend.initSeedList() + initMulticastIPv6() + initBroadcastIPv4() + backend.initStore() + backend.initNetwork() + backend.initBlockchainCache() + + if backend.SearchIndex, err = search.InitSearchIndexStore(backend.Config.SearchIndex); err != nil { + backend.LogError("Init", "search index '%s' init: %s", backend.Config.SearchIndex, err.Error()) + } else { + backend.userBlockchainUpdateSearchIndex() + } + + return backend, ExitSuccess, nil +} + +// Connect starts bootstrapping and local peer discovery. +func (backend *Backend) Connect() { + go backend.bootstrapKademlia() + go backend.bootstrap() + go backend.networks.autoMulticastBroadcast() + go backend.autoPingAll() + go backend.networks.networkChangeMonitor() + go backend.networks.startUPnP() + go backend.autoBucketRefresh() +} + +// The Backend represents an instance of a Peernet client to be used by a frontend. +// Global variables and init functions are to be merged. +type Backend struct { + ConfigFilename string // Filename of the configuration file. + Config *Config // Core configuration + ConfigClient interface{} // Custom configuration from the client + Filters Filters // Filters allow to install hooks. + userAgent string // User Agent + GlobalBlockchainCache *BlockchainCache // Caches blockchains of other peers. + SearchIndex *search.SearchIndexStore // Search index of blockchain records. + networks *Networks // All connected networks. + dhtStore store.Store // dhtStore contains all key-value data served via DHT + UserBlockchain *blockchain.Blockchain // UserBlockchain is the user's blockchain and exports functions to directly read and write it + UserWarehouse *warehouse.Warehouse // UserWarehouse is the user's warehouse for storing files that are shared + nodesDHT *dht.DHT // Nodes connected in the DHT. + + // peerID is the current peer's ID. It is a ECDSA (secp256k1) 257-bit public key. + PeerPrivateKey *btcec.PrivateKey + PeerPublicKey *btcec.PublicKey + + // The node ID is the blake3 hash of the public key compressed form. + nodeID []byte + + // peerList keeps track of all peers + peerList map[[btcec.PubKeyBytesLenCompressed]byte]*PeerInfo + peerlistMutex sync.RWMutex + + // nodeList is a mirror of peerList but using the node ID + nodeList map[[protocol.HashSize]byte]*PeerInfo + + // peerMonitor is a list of channels receiving information about new peers + peerMonitor []chan<- *PeerInfo + + // Stdout bundles any output for the end-user. Writers may subscribe/unsubscribe. + Stdout *multiWriter +} diff --git a/Ping.go b/Ping.go index 58c7ef7..eb04ded 100644 --- a/Ping.go +++ b/Ping.go @@ -1,78 +1,78 @@ -/* -File Name: Ping.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "time" -) - -// pingTime is the time in seconds to send out ping messages -const pingTime = 10 - -// thresholdBlockchainRefresh is the threshold to refresh the blockchain information by sending an Announcement (and expecting the Response message). -// This helps for keeping the global blockchain cache up to date. -const thresholdBlockchainRefresh = 60 * time.Second - -// connectionInvalidate is the threshold in seconds to invalidate formerly active connections that no longer receive incoming packets. -const connectionInvalidate = 22 - -// connectionRemove is the threshold in seconds to remove inactive connections in case there is at least one active connection known. -const connectionRemove = 2 * 60 - -// autoPingAll sends out regular ping messages to all connections of all peers. This allows to detect invalid connections and eventually drop them. -func (backend *Backend) autoPingAll() { - for { - time.Sleep(time.Second) - thresholdInvalidate1 := time.Now().Add(-connectionInvalidate * time.Second) - thresholdInvalidate2 := time.Now().Add(-connectionInvalidate * time.Second * 4) - thresholdPingOut1 := time.Now().Add(-pingTime * time.Second) - thresholdPingOut2 := time.Now().Add(-pingTime * time.Second * 4) - thresholdBlockchainRefresh := time.Now().Add(-thresholdBlockchainRefresh) - - for _, peer := range backend.PeerlistGet() { - // first handle active connections - for _, connection := range peer.GetConnections(true) { - thresholdPing := thresholdPingOut1 - thresholdInv := thresholdInvalidate1 - - if connection.Status == ConnectionRedundant { - thresholdPing = thresholdPingOut2 - thresholdInv = thresholdInvalidate2 - } - - if connection.LastPacketIn.Before(thresholdInv) { - peer.invalidateActiveConnection(connection) - continue - } - - if connection.LastPacketIn.Before(thresholdPing) && connection.LastPingOut.Before(thresholdPing) { - if connection.Status == ConnectionActive && peer.blockchainLastRefresh.Before(thresholdBlockchainRefresh) { - peer.pingConnectionAnnouncement(connection) - } else { - // just a regular ping otherwise - peer.pingConnection(connection) - } - continue - } - } - - // handle inactive connections - for _, connection := range peer.GetConnections(false) { - // If the inactive connection is expired, remove it; although only if there is at least one active connection, or two other inactive ones. - if (len(peer.connectionActive) >= 1 || len(peer.connectionInactive) > 2) && connection.Expires.Before(time.Now()) { - peer.removeInactiveConnection(connection) - continue - } - - // if no ping was sent recently, send one now - if connection.LastPingOut.Before(thresholdPingOut1) { - peer.pingConnection(connection) - } - } - } - } -} +/* +File Name: Ping.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "time" +) + +// pingTime is the time in seconds to send out ping messages +const pingTime = 10 + +// thresholdBlockchainRefresh is the threshold to refresh the blockchain information by sending an Announcement (and expecting the Response message). +// This helps for keeping the global blockchain cache up to date. +const thresholdBlockchainRefresh = 60 * time.Second + +// connectionInvalidate is the threshold in seconds to invalidate formerly active connections that no longer receive incoming packets. +const connectionInvalidate = 22 + +// connectionRemove is the threshold in seconds to remove inactive connections in case there is at least one active connection known. +const connectionRemove = 2 * 60 + +// autoPingAll sends out regular ping messages to all connections of all peers. This allows to detect invalid connections and eventually drop them. +func (backend *Backend) autoPingAll() { + for { + time.Sleep(time.Second) + thresholdInvalidate1 := time.Now().Add(-connectionInvalidate * time.Second) + thresholdInvalidate2 := time.Now().Add(-connectionInvalidate * time.Second * 4) + thresholdPingOut1 := time.Now().Add(-pingTime * time.Second) + thresholdPingOut2 := time.Now().Add(-pingTime * time.Second * 4) + thresholdBlockchainRefresh := time.Now().Add(-thresholdBlockchainRefresh) + + for _, peer := range backend.PeerlistGet() { + // first handle active connections + for _, connection := range peer.GetConnections(true) { + thresholdPing := thresholdPingOut1 + thresholdInv := thresholdInvalidate1 + + if connection.Status == ConnectionRedundant { + thresholdPing = thresholdPingOut2 + thresholdInv = thresholdInvalidate2 + } + + if connection.LastPacketIn.Before(thresholdInv) { + peer.invalidateActiveConnection(connection) + continue + } + + if connection.LastPacketIn.Before(thresholdPing) && connection.LastPingOut.Before(thresholdPing) { + if connection.Status == ConnectionActive && peer.blockchainLastRefresh.Before(thresholdBlockchainRefresh) { + peer.pingConnectionAnnouncement(connection) + } else { + // just a regular ping otherwise + peer.pingConnection(connection) + } + continue + } + } + + // handle inactive connections + for _, connection := range peer.GetConnections(false) { + // If the inactive connection is expired, remove it; although only if there is at least one active connection, or two other inactive ones. + if (len(peer.connectionActive) >= 1 || len(peer.connectionInactive) > 2) && connection.Expires.Before(time.Now()) { + peer.removeInactiveConnection(connection) + continue + } + + // if no ping was sent recently, send one now + if connection.LastPingOut.Before(thresholdPingOut1) { + peer.pingConnection(connection) + } + } + } + } +} diff --git a/Transfer Block.go b/Transfer Block.go index a56485b..2c05a60 100644 --- a/Transfer Block.go +++ b/Transfer Block.go @@ -1,183 +1,183 @@ -/* -File Name: Transfer Block.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "errors" - "time" - - "github.com/PeernetOfficial/core/blockchain" - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/udt" - "github.com/google/uuid" -) - -// blockSequenceTimeout is the timeout for a follow-up message to appear, otherwise the transfer will be terminated. -var blockSequenceTimeout = time.Second * 10 - -// Whether to use the lite protocol for transfer of data. -const blockTransferLite = true - -// startBlockTransfer starts the transfer of blocks. Currently it only serves the user's blockchain. -func (peer *PeerInfo) startBlockTransfer(BlockchainPublicKey *btcec.PublicKey, LimitBlockCount uint64, MaxBlockSize uint64, TargetBlocks []protocol.BlockRange, sequenceNumber uint32, transferID uuid.UUID) (err error) { - virtualConn := newVirtualPacketConn(peer, func(data []byte, sequenceNumber uint32, transferID uuid.UUID) { - peer.sendGetBlock(data, protocol.GetBlockControlActive, BlockchainPublicKey, 0, 0, nil, sequenceNumber, transferID, blockTransferLite) - }) - virtualConn.Stats = &BlockTransferStats{BlockchainPublicKey: BlockchainPublicKey, Direction: DirectionOut, LimitBlockCount: LimitBlockCount, MaxBlockSize: MaxBlockSize, TargetBlocks: TargetBlocks} - - // use the transfer ID indicated by the remote peer - // 17.01.2021: Due to using lite IDs, the sequence termination function in RegisterSequenceBi is no longer used, as data packets are only sent via lite packets. - virtualConn.transferID = transferID - peer.Backend.networks.LiteRouter.RegisterLiteID(transferID, virtualConn, blockSequenceTimeout, virtualConn.sequenceTerminate) - - // register the sequence since packets are sent bi-directional - virtualConn.sequenceNumber = sequenceNumber - peer.Backend.networks.Sequences.RegisterSequenceBi(peer.PublicKey, sequenceNumber, virtualConn, blockSequenceTimeout, nil) - - udtConfig := udt.DefaultConfig() - udtConfig.MaxPacketSize = protocol.TransferMaxEmbedSizeLite - udtConfig.MaxFlowWinSize = maxFlowWinSize - - // start UDT sender - // Set streaming to true, otherwise udtSocket.Read returns the error "Message truncated" in case the reader has a smaller buffer. - udtConn, err := udt.DialUDT(udtConfig, virtualConn, virtualConn.incomingData, virtualConn.outgoingData, virtualConn.terminationSignal, true) - if err != nil { - return err - } - - defer udtConn.Close() - virtualConn.Stats.(*BlockTransferStats).UDTConn = udtConn - - // loop through the requested TargetBlocks range. - sentBlocks := uint64(0) - - for _, target := range TargetBlocks { - for blockN := target.Offset; blockN < target.Offset+target.Limit; blockN++ { - blockData, status, err := peer.Backend.UserBlockchain.GetBlockRaw(blockN) - if err != nil { - protocol.BlockTransferWriteHeader(udtConn, protocol.GetBlockStatusNotAvailable, protocol.BlockRange{Offset: blockN, Limit: 1}, 0) - continue - } - blockSize := uint64(len(blockData)) - - if status != blockchain.StatusOK { - protocol.BlockTransferWriteHeader(udtConn, protocol.GetBlockStatusNotAvailable, protocol.BlockRange{Offset: blockN, Limit: 1}, 0) - continue - } else if blockSize > MaxBlockSize { - protocol.BlockTransferWriteHeader(udtConn, protocol.GetBlockStatusSizeExceed, protocol.BlockRange{Offset: blockN, Limit: 1}, blockSize) - continue - } - - protocol.BlockTransferWriteHeader(udtConn, protocol.GetBlockStatusAvailable, protocol.BlockRange{Offset: blockN, Limit: 1}, blockSize) - udtConn.Write(blockData) - - sentBlocks++ - if sentBlocks >= LimitBlockCount { - break - } - } - } - - return err -} - -// BlockTransferRequest requests blocks from the peer. -// The caller must call udtConn.Close() when done. Do not use any of the closing functions of virtualConn. -func (peer *PeerInfo) BlockTransferRequest(BlockchainPublicKey *btcec.PublicKey, LimitBlockCount uint64, MaxBlockSize uint64, TargetBlocks []protocol.BlockRange) (udtConn *udt.UDTSocket, virtualConn *VirtualPacketConn, err error) { - virtualConn = newVirtualPacketConn(peer, func(data []byte, sequenceNumber uint32, transferID uuid.UUID) { - peer.sendGetBlock(data, protocol.GetBlockControlActive, BlockchainPublicKey, 0, 0, nil, sequenceNumber, transferID, blockTransferLite) - }) - virtualConn.Stats = &BlockTransferStats{BlockchainPublicKey: BlockchainPublicKey, Direction: DirectionIn, LimitBlockCount: LimitBlockCount, MaxBlockSize: MaxBlockSize, TargetBlocks: TargetBlocks} - - // new lite ID - liteID := peer.Backend.networks.LiteRouter.NewLiteID(virtualConn, blockSequenceTimeout, virtualConn.sequenceTerminate) - virtualConn.transferID = liteID.ID - - // new sequence - sequence := peer.Backend.networks.Sequences.NewSequenceBi(peer.PublicKey, &peer.messageSequence, virtualConn, blockSequenceTimeout, nil) - if sequence == nil { - return nil, nil, errors.New("cannot acquire sequence") - } - virtualConn.sequenceNumber = sequence.SequenceNumber - - udtConfig := udt.DefaultConfig() - udtConfig.MaxPacketSize = protocol.TransferMaxEmbedSizeLite - udtConfig.MaxFlowWinSize = maxFlowWinSize - - // start UDT receiver - udtListener := udt.ListenUDT(udtConfig, virtualConn, virtualConn.incomingData, virtualConn.outgoingData, virtualConn.terminationSignal) - - // request block transfer - err = peer.sendGetBlock(nil, protocol.GetBlockControlRequestStart, BlockchainPublicKey, LimitBlockCount, MaxBlockSize, TargetBlocks, virtualConn.sequenceNumber, virtualConn.transferID, false) - if err != nil { - udtListener.Close() - return nil, nil, err - } - - // accept the connection - udtConn, err = udtListener.Accept() // TODO: Add timeout! - if err != nil { - udtListener.Close() - return nil, nil, err - } - virtualConn.Stats.(*BlockTransferStats).UDTConn = udtConn - - // We do not close the UDT listener here. It should automatically close after udtConn is closed. - - return udtConn, virtualConn, nil -} - -// Downloads the requested blocks for the selected blockchain from the remote peer. The callback is called for each result. -func (peer *PeerInfo) BlockDownload(BlockchainPublicKey *btcec.PublicKey, LimitBlockCount, MaxBlockSize uint64, TargetBlocks []protocol.BlockRange, callback func(data []byte, targetBlock protocol.BlockRange, blockSize uint64, availability uint8)) (err error) { - conn, _, err := peer.BlockTransferRequest(BlockchainPublicKey, LimitBlockCount, MaxBlockSize, TargetBlocks) - if err != nil { - return err - } - defer conn.Close() - - var limit uint64 - for _, target := range TargetBlocks { - limit += target.Limit - } - - for n := uint64(0); n < limit; { - data, targetBlock, blockSize, availability, err := protocol.BlockTransferReadBlock(conn, MaxBlockSize) - if err != nil { - return err - } else if !isTargetInRange(TargetBlocks, targetBlock.Offset, targetBlock.Limit) { - return errors.New("invalid returned block range") - } - - // TODO: Check if the block was already returned in case the block is available. This can be done via simple map. - - callback(data, targetBlock, blockSize, availability) - - n += targetBlock.Limit - } - - return nil -} - -func isTargetInRange(targets []protocol.BlockRange, offset, limit uint64) (valid bool) { - for _, target := range targets { - if offset >= target.Offset && offset+limit <= target.Offset+target.Limit { - return true - } - } - - return false -} - -type BlockTransferStats struct { - BlockchainPublicKey *btcec.PublicKey // Target blockchain - Direction int // Direction of the data transfer - LimitBlockCount uint64 // Max count of blocks to be transferred - MaxBlockSize uint64 // Max single block size to transfer - TargetBlocks []protocol.BlockRange // List of blocks to transfer - UDTConn *udt.UDTSocket // Underlying UDT connection -} +/* +File Name: Transfer Block.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "errors" + "time" + + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/udt" + "github.com/google/uuid" +) + +// blockSequenceTimeout is the timeout for a follow-up message to appear, otherwise the transfer will be terminated. +var blockSequenceTimeout = time.Second * 10 + +// Whether to use the lite protocol for transfer of data. +const blockTransferLite = true + +// startBlockTransfer starts the transfer of blocks. Currently it only serves the user's blockchain. +func (peer *PeerInfo) startBlockTransfer(BlockchainPublicKey *btcec.PublicKey, LimitBlockCount uint64, MaxBlockSize uint64, TargetBlocks []protocol.BlockRange, sequenceNumber uint32, transferID uuid.UUID) (err error) { + virtualConn := newVirtualPacketConn(peer, func(data []byte, sequenceNumber uint32, transferID uuid.UUID) { + peer.sendGetBlock(data, protocol.GetBlockControlActive, BlockchainPublicKey, 0, 0, nil, sequenceNumber, transferID, blockTransferLite) + }) + virtualConn.Stats = &BlockTransferStats{BlockchainPublicKey: BlockchainPublicKey, Direction: DirectionOut, LimitBlockCount: LimitBlockCount, MaxBlockSize: MaxBlockSize, TargetBlocks: TargetBlocks} + + // use the transfer ID indicated by the remote peer + // 17.01.2021: Due to using lite IDs, the sequence termination function in RegisterSequenceBi is no longer used, as data packets are only sent via lite packets. + virtualConn.transferID = transferID + peer.Backend.networks.LiteRouter.RegisterLiteID(transferID, virtualConn, blockSequenceTimeout, virtualConn.sequenceTerminate) + + // register the sequence since packets are sent bi-directional + virtualConn.sequenceNumber = sequenceNumber + peer.Backend.networks.Sequences.RegisterSequenceBi(peer.PublicKey, sequenceNumber, virtualConn, blockSequenceTimeout, nil) + + udtConfig := udt.DefaultConfig() + udtConfig.MaxPacketSize = protocol.TransferMaxEmbedSizeLite + udtConfig.MaxFlowWinSize = maxFlowWinSize + + // start UDT sender + // Set streaming to true, otherwise udtSocket.Read returns the error "Message truncated" in case the reader has a smaller buffer. + udtConn, err := udt.DialUDT(udtConfig, virtualConn, virtualConn.incomingData, virtualConn.outgoingData, virtualConn.terminationSignal, true) + if err != nil { + return err + } + + defer udtConn.Close() + virtualConn.Stats.(*BlockTransferStats).UDTConn = udtConn + + // loop through the requested TargetBlocks range. + sentBlocks := uint64(0) + + for _, target := range TargetBlocks { + for blockN := target.Offset; blockN < target.Offset+target.Limit; blockN++ { + blockData, status, err := peer.Backend.UserBlockchain.GetBlockRaw(blockN) + if err != nil { + protocol.BlockTransferWriteHeader(udtConn, protocol.GetBlockStatusNotAvailable, protocol.BlockRange{Offset: blockN, Limit: 1}, 0) + continue + } + blockSize := uint64(len(blockData)) + + if status != blockchain.StatusOK { + protocol.BlockTransferWriteHeader(udtConn, protocol.GetBlockStatusNotAvailable, protocol.BlockRange{Offset: blockN, Limit: 1}, 0) + continue + } else if blockSize > MaxBlockSize { + protocol.BlockTransferWriteHeader(udtConn, protocol.GetBlockStatusSizeExceed, protocol.BlockRange{Offset: blockN, Limit: 1}, blockSize) + continue + } + + protocol.BlockTransferWriteHeader(udtConn, protocol.GetBlockStatusAvailable, protocol.BlockRange{Offset: blockN, Limit: 1}, blockSize) + udtConn.Write(blockData) + + sentBlocks++ + if sentBlocks >= LimitBlockCount { + break + } + } + } + + return err +} + +// BlockTransferRequest requests blocks from the peer. +// The caller must call udtConn.Close() when done. Do not use any of the closing functions of virtualConn. +func (peer *PeerInfo) BlockTransferRequest(BlockchainPublicKey *btcec.PublicKey, LimitBlockCount uint64, MaxBlockSize uint64, TargetBlocks []protocol.BlockRange) (udtConn *udt.UDTSocket, virtualConn *VirtualPacketConn, err error) { + virtualConn = newVirtualPacketConn(peer, func(data []byte, sequenceNumber uint32, transferID uuid.UUID) { + peer.sendGetBlock(data, protocol.GetBlockControlActive, BlockchainPublicKey, 0, 0, nil, sequenceNumber, transferID, blockTransferLite) + }) + virtualConn.Stats = &BlockTransferStats{BlockchainPublicKey: BlockchainPublicKey, Direction: DirectionIn, LimitBlockCount: LimitBlockCount, MaxBlockSize: MaxBlockSize, TargetBlocks: TargetBlocks} + + // new lite ID + liteID := peer.Backend.networks.LiteRouter.NewLiteID(virtualConn, blockSequenceTimeout, virtualConn.sequenceTerminate) + virtualConn.transferID = liteID.ID + + // new sequence + sequence := peer.Backend.networks.Sequences.NewSequenceBi(peer.PublicKey, &peer.messageSequence, virtualConn, blockSequenceTimeout, nil) + if sequence == nil { + return nil, nil, errors.New("cannot acquire sequence") + } + virtualConn.sequenceNumber = sequence.SequenceNumber + + udtConfig := udt.DefaultConfig() + udtConfig.MaxPacketSize = protocol.TransferMaxEmbedSizeLite + udtConfig.MaxFlowWinSize = maxFlowWinSize + + // start UDT receiver + udtListener := udt.ListenUDT(udtConfig, virtualConn, virtualConn.incomingData, virtualConn.outgoingData, virtualConn.terminationSignal) + + // request block transfer + err = peer.sendGetBlock(nil, protocol.GetBlockControlRequestStart, BlockchainPublicKey, LimitBlockCount, MaxBlockSize, TargetBlocks, virtualConn.sequenceNumber, virtualConn.transferID, false) + if err != nil { + udtListener.Close() + return nil, nil, err + } + + // accept the connection + udtConn, err = udtListener.Accept() // TODO: Add timeout! + if err != nil { + udtListener.Close() + return nil, nil, err + } + virtualConn.Stats.(*BlockTransferStats).UDTConn = udtConn + + // We do not close the UDT listener here. It should automatically close after udtConn is closed. + + return udtConn, virtualConn, nil +} + +// Downloads the requested blocks for the selected blockchain from the remote peer. The callback is called for each result. +func (peer *PeerInfo) BlockDownload(BlockchainPublicKey *btcec.PublicKey, LimitBlockCount, MaxBlockSize uint64, TargetBlocks []protocol.BlockRange, callback func(data []byte, targetBlock protocol.BlockRange, blockSize uint64, availability uint8)) (err error) { + conn, _, err := peer.BlockTransferRequest(BlockchainPublicKey, LimitBlockCount, MaxBlockSize, TargetBlocks) + if err != nil { + return err + } + defer conn.Close() + + var limit uint64 + for _, target := range TargetBlocks { + limit += target.Limit + } + + for n := uint64(0); n < limit; { + data, targetBlock, blockSize, availability, err := protocol.BlockTransferReadBlock(conn, MaxBlockSize) + if err != nil { + return err + } else if !isTargetInRange(TargetBlocks, targetBlock.Offset, targetBlock.Limit) { + return errors.New("invalid returned block range") + } + + // TODO: Check if the block was already returned in case the block is available. This can be done via simple map. + + callback(data, targetBlock, blockSize, availability) + + n += targetBlock.Limit + } + + return nil +} + +func isTargetInRange(targets []protocol.BlockRange, offset, limit uint64) (valid bool) { + for _, target := range targets { + if offset >= target.Offset && offset+limit <= target.Offset+target.Limit { + return true + } + } + + return false +} + +type BlockTransferStats struct { + BlockchainPublicKey *btcec.PublicKey // Target blockchain + Direction int // Direction of the data transfer + LimitBlockCount uint64 // Max count of blocks to be transferred + MaxBlockSize uint64 // Max single block size to transfer + TargetBlocks []protocol.BlockRange // List of blocks to transfer + UDTConn *udt.UDTSocket // Underlying UDT connection +} diff --git a/Transfer UDT.go b/Transfer UDT.go index 3802fcf..1d8635d 100644 --- a/Transfer UDT.go +++ b/Transfer UDT.go @@ -1,135 +1,135 @@ -/* -File Name: Transfer UDT.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -TODO: Add timeouts for listening and sending. -*/ - -package core - -import ( - "errors" - "time" - - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/udt" - "github.com/google/uuid" -) - -// transferSequenceTimeout is the timeout for a follow-up message to appear, otherwise the transfer will be terminated. -var transferSequenceTimeout = time.Minute * 1 - -// maxFlowWinSize is the maximum number of unacknowledged packets to permit. A higher number means using more memory, but reduces potential overhead since it does not stop so quickly for missing packets. -// Each unacknowledged packet may store protocol.TransferMaxEmbedSize (currently 1121 bytes) payload data in memory. A too high number may impact the speed of real-time streaming in case of lost packets. -// The actual used number will be negotiated through the UDT handshake and must be a minimum of 32. -const maxFlowWinSize = 64 - -// Whether to use the lite protocol for transfer of data. -const transferLite = true - -// startFileTransferUDT starts a file transfer from the local warehouse to the remote peer. -// It creates a virtual UDT client to transfer data to a remote peer. Counterintuitively, this will be the "file server" peer. -func (peer *PeerInfo) startFileTransferUDT(hash []byte, fileSize uint64, offset, limit uint64, sequenceNumber uint32, transferID uuid.UUID, transferProtocol uint8) (err error) { - if limit > 0 && offset+limit > fileSize { - return errors.New("invalid limit") - } else if offset > fileSize { - return errors.New("invalid offset") - } else if limit == 0 { - limit = fileSize - offset - } - - virtualConn := newVirtualPacketConn(peer, func(data []byte, sequenceNumber uint32, transferID uuid.UUID) { - peer.sendTransfer(data, protocol.TransferControlActive, 0, hash, offset, limit, sequenceNumber, transferID, transferLite) - }) - virtualConn.Stats = &FileTransferStats{Hash: hash, Direction: DirectionOut, FileSize: fileSize, Offset: offset, Limit: limit} - - // use the transfer ID indicated by the remote peer - // 17.01.2021: Due to using lite IDs, the sequence termination function in RegisterSequenceBi is no longer used, as data packets are only sent via lite packets. - virtualConn.transferID = transferID - peer.Backend.networks.LiteRouter.RegisterLiteID(transferID, virtualConn, transferSequenceTimeout, virtualConn.sequenceTerminate) - - // register the sequence since packets are sent bi-directional - virtualConn.sequenceNumber = sequenceNumber - peer.Backend.networks.Sequences.RegisterSequenceBi(peer.PublicKey, sequenceNumber, virtualConn, transferSequenceTimeout, nil) - - udtConfig := udt.DefaultConfig() - udtConfig.MaxPacketSize = protocol.TransferMaxEmbedSizeLite - udtConfig.MaxFlowWinSize = maxFlowWinSize - - // start UDT sender - // Set streaming to true, otherwise udtSocket.Read returns the error "Message truncated" in case the reader has a smaller buffer. - udtConn, err := udt.DialUDT(udtConfig, virtualConn, virtualConn.incomingData, virtualConn.outgoingData, virtualConn.terminationSignal, true) - if err != nil { - return err - } - - defer udtConn.Close() - virtualConn.Stats.(*FileTransferStats).UDTConn = udtConn - - // First send the header (Total File Size, Transfer Size) and then the file data. - protocol.FileTransferWriteHeader(udtConn, fileSize, limit) - - _, _, err = peer.Backend.UserWarehouse.ReadFile(hash, int64(offset), int64(limit), udtConn) - - return err -} - -// FileTransferRequestUDT creates a UDT server listening for incoming data transfer via the lite protocol and requests a file transfer from a remote peer. -// The caller must call udtConn.Close() when done. Do not use any of the closing functions of virtualConn. -// Limit is optional. 0 means the entire file. -func (peer *PeerInfo) FileTransferRequestUDT(hash []byte, offset, limit uint64) (udtConn *udt.UDTSocket, virtualConn *VirtualPacketConn, err error) { - virtualConn = newVirtualPacketConn(peer, func(data []byte, sequenceNumber uint32, transferID uuid.UUID) { - peer.sendTransfer(data, protocol.TransferControlActive, protocol.TransferProtocolUDT, hash, offset, limit, sequenceNumber, transferID, transferLite) - }) - - // new lite ID - liteID := peer.Backend.networks.LiteRouter.NewLiteID(virtualConn, transferSequenceTimeout, virtualConn.sequenceTerminate) - virtualConn.transferID = liteID.ID - virtualConn.Stats = &FileTransferStats{Hash: hash, Direction: DirectionIn, Offset: offset, Limit: limit} - - // new sequence - sequence := peer.Backend.networks.Sequences.NewSequenceBi(peer.PublicKey, &peer.messageSequence, virtualConn, transferSequenceTimeout, nil) - if sequence == nil { - return nil, nil, errors.New("cannot acquire sequence") - } - virtualConn.sequenceNumber = sequence.SequenceNumber - - udtConfig := udt.DefaultConfig() - udtConfig.MaxPacketSize = protocol.TransferMaxEmbedSizeLite - udtConfig.MaxFlowWinSize = maxFlowWinSize - - // start UDT receiver - udtListener := udt.ListenUDT(udtConfig, virtualConn, virtualConn.incomingData, virtualConn.outgoingData, virtualConn.terminationSignal) - - // request file transfer - peer.sendTransfer(nil, protocol.TransferControlRequestStart, protocol.TransferProtocolUDT, hash, offset, limit, virtualConn.sequenceNumber, virtualConn.transferID, false) - - // accept the connection - udtConn, err = udtListener.Accept() - if err != nil { - udtListener.Close() - return nil, nil, err - } - virtualConn.Stats.(*FileTransferStats).UDTConn = udtConn - - // We do not close the UDT listener here. It should automatically close after udtConn is closed. - - return udtConn, virtualConn, nil -} - -type FileTransferStats struct { - Hash []byte // Hash of the file to transfer - Direction int // Direction of the data transfer - FileSize uint64 // File size if known - Offset uint64 // Offset to start the transfer - Limit uint64 // Limit in bytes to transfer - UDTConn *udt.UDTSocket // Underlying UDT connection -} - -// Transfer directions -const ( - DirectionIn = 0 - DirectionOut = 1 - DirectionBi = 2 -) +/* +File Name: Transfer UDT.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +TODO: Add timeouts for listening and sending. +*/ + +package core + +import ( + "errors" + "time" + + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/udt" + "github.com/google/uuid" +) + +// transferSequenceTimeout is the timeout for a follow-up message to appear, otherwise the transfer will be terminated. +var transferSequenceTimeout = time.Minute * 1 + +// maxFlowWinSize is the maximum number of unacknowledged packets to permit. A higher number means using more memory, but reduces potential overhead since it does not stop so quickly for missing packets. +// Each unacknowledged packet may store protocol.TransferMaxEmbedSize (currently 1121 bytes) payload data in memory. A too high number may impact the speed of real-time streaming in case of lost packets. +// The actual used number will be negotiated through the UDT handshake and must be a minimum of 32. +const maxFlowWinSize = 64 + +// Whether to use the lite protocol for transfer of data. +const transferLite = true + +// startFileTransferUDT starts a file transfer from the local warehouse to the remote peer. +// It creates a virtual UDT client to transfer data to a remote peer. Counterintuitively, this will be the "file server" peer. +func (peer *PeerInfo) startFileTransferUDT(hash []byte, fileSize uint64, offset, limit uint64, sequenceNumber uint32, transferID uuid.UUID, transferProtocol uint8) (err error) { + if limit > 0 && offset+limit > fileSize { + return errors.New("invalid limit") + } else if offset > fileSize { + return errors.New("invalid offset") + } else if limit == 0 { + limit = fileSize - offset + } + + virtualConn := newVirtualPacketConn(peer, func(data []byte, sequenceNumber uint32, transferID uuid.UUID) { + peer.sendTransfer(data, protocol.TransferControlActive, 0, hash, offset, limit, sequenceNumber, transferID, transferLite) + }) + virtualConn.Stats = &FileTransferStats{Hash: hash, Direction: DirectionOut, FileSize: fileSize, Offset: offset, Limit: limit} + + // use the transfer ID indicated by the remote peer + // 17.01.2021: Due to using lite IDs, the sequence termination function in RegisterSequenceBi is no longer used, as data packets are only sent via lite packets. + virtualConn.transferID = transferID + peer.Backend.networks.LiteRouter.RegisterLiteID(transferID, virtualConn, transferSequenceTimeout, virtualConn.sequenceTerminate) + + // register the sequence since packets are sent bi-directional + virtualConn.sequenceNumber = sequenceNumber + peer.Backend.networks.Sequences.RegisterSequenceBi(peer.PublicKey, sequenceNumber, virtualConn, transferSequenceTimeout, nil) + + udtConfig := udt.DefaultConfig() + udtConfig.MaxPacketSize = protocol.TransferMaxEmbedSizeLite + udtConfig.MaxFlowWinSize = maxFlowWinSize + + // start UDT sender + // Set streaming to true, otherwise udtSocket.Read returns the error "Message truncated" in case the reader has a smaller buffer. + udtConn, err := udt.DialUDT(udtConfig, virtualConn, virtualConn.incomingData, virtualConn.outgoingData, virtualConn.terminationSignal, true) + if err != nil { + return err + } + + defer udtConn.Close() + virtualConn.Stats.(*FileTransferStats).UDTConn = udtConn + + // First send the header (Total File Size, Transfer Size) and then the file data. + protocol.FileTransferWriteHeader(udtConn, fileSize, limit) + + _, _, err = peer.Backend.UserWarehouse.ReadFile(hash, int64(offset), int64(limit), udtConn) + + return err +} + +// FileTransferRequestUDT creates a UDT server listening for incoming data transfer via the lite protocol and requests a file transfer from a remote peer. +// The caller must call udtConn.Close() when done. Do not use any of the closing functions of virtualConn. +// Limit is optional. 0 means the entire file. +func (peer *PeerInfo) FileTransferRequestUDT(hash []byte, offset, limit uint64) (udtConn *udt.UDTSocket, virtualConn *VirtualPacketConn, err error) { + virtualConn = newVirtualPacketConn(peer, func(data []byte, sequenceNumber uint32, transferID uuid.UUID) { + peer.sendTransfer(data, protocol.TransferControlActive, protocol.TransferProtocolUDT, hash, offset, limit, sequenceNumber, transferID, transferLite) + }) + + // new lite ID + liteID := peer.Backend.networks.LiteRouter.NewLiteID(virtualConn, transferSequenceTimeout, virtualConn.sequenceTerminate) + virtualConn.transferID = liteID.ID + virtualConn.Stats = &FileTransferStats{Hash: hash, Direction: DirectionIn, Offset: offset, Limit: limit} + + // new sequence + sequence := peer.Backend.networks.Sequences.NewSequenceBi(peer.PublicKey, &peer.messageSequence, virtualConn, transferSequenceTimeout, nil) + if sequence == nil { + return nil, nil, errors.New("cannot acquire sequence") + } + virtualConn.sequenceNumber = sequence.SequenceNumber + + udtConfig := udt.DefaultConfig() + udtConfig.MaxPacketSize = protocol.TransferMaxEmbedSizeLite + udtConfig.MaxFlowWinSize = maxFlowWinSize + + // start UDT receiver + udtListener := udt.ListenUDT(udtConfig, virtualConn, virtualConn.incomingData, virtualConn.outgoingData, virtualConn.terminationSignal) + + // request file transfer + peer.sendTransfer(nil, protocol.TransferControlRequestStart, protocol.TransferProtocolUDT, hash, offset, limit, virtualConn.sequenceNumber, virtualConn.transferID, false) + + // accept the connection + udtConn, err = udtListener.Accept() + if err != nil { + udtListener.Close() + return nil, nil, err + } + virtualConn.Stats.(*FileTransferStats).UDTConn = udtConn + + // We do not close the UDT listener here. It should automatically close after udtConn is closed. + + return udtConn, virtualConn, nil +} + +type FileTransferStats struct { + Hash []byte // Hash of the file to transfer + Direction int // Direction of the data transfer + FileSize uint64 // File size if known + Offset uint64 // Offset to start the transfer + Limit uint64 // Limit in bytes to transfer + UDTConn *udt.UDTSocket // Underlying UDT connection +} + +// Transfer directions +const ( + DirectionIn = 0 + DirectionOut = 1 + DirectionBi = 2 +) diff --git a/Transfer Virtual Connection.go b/Transfer Virtual Connection.go index 45a497d..766989d 100644 --- a/Transfer Virtual Connection.go +++ b/Transfer Virtual Connection.go @@ -1,133 +1,133 @@ -/* -File Name: Transfer Virtual Connection.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -This file defines a virtual connection between a transfer protocol and Peernet messages. -If either the downstream transfer protocol or upstream Peernet messages indicate termination, the virtual connection ceases to exist. -*/ - -package core - -import ( - "sync" - - "github.com/google/uuid" -) - -// VirtualPacketConn is a virtual connection. -type VirtualPacketConn struct { - Peer *PeerInfo - - // Stats are maintained by the caller - Stats interface{} - - // function to send data to the remote peer - sendData func(data []byte, sequenceNumber uint32, transferID uuid.UUID) - - // Sequence number from the first outgoing or incoming packet. - sequenceNumber uint32 - - // Transfer ID represents a session ID valid only for the duration of the transfer. - transferID uuid.UUID - - // data channel - incomingData chan []byte - outgoingData chan []byte - - // internal data - closed bool - terminationSignal chan struct{} // The termination signal shall be used by the underlying protocol to detect upstream termination. - reason int // Reason why it was closed - sync.Mutex -} - -// newVirtualPacketConn creates a new virtual connection (both incoming and outgoing). -func newVirtualPacketConn(peer *PeerInfo, sendData func(data []byte, sequenceNumber uint32, transferID uuid.UUID)) (v *VirtualPacketConn) { - v = &VirtualPacketConn{ - Peer: peer, - sendData: sendData, - incomingData: make(chan []byte, 512), - outgoingData: make(chan []byte), - terminationSignal: make(chan struct{}), - } - - go v.writeForward() - - return -} - -// writeForward forwards outgoing messages -func (v *VirtualPacketConn) writeForward() { - for { - select { - case data := <-v.outgoingData: - v.sendData(data, v.sequenceNumber, v.transferID) - - case <-v.terminationSignal: - return - } - } -} - -// receiveData receives incoming data via an external message. Non-blocking. -func (v *VirtualPacketConn) receiveData(data []byte) { - if v.IsTerminated() { - return - } - - // pass the data on - select { - case v.incomingData <- data: - case <-v.terminationSignal: - default: - // packet lost - } -} - -// Terminate closes the connection. Do not call this function manually. Use the underlying protocol's function to close the connection. -// Reason: 404 = Remote peer does not store file (upstream), 2 = Remote termination signal (upstream), 3 = Sequence invalidation or expiration (upstream), 1000+ = Transfer protocol indicated closing (downstream) -func (v *VirtualPacketConn) Terminate(reason int) (err error) { - v.Lock() - defer v.Unlock() - - if v.closed { // if already closed, take no action - return - } - - v.closed = true - v.reason = reason - close(v.terminationSignal) - - return -} - -// IsTerminated checks if the connection is terminated -func (v *VirtualPacketConn) IsTerminated() bool { - return v.closed -} - -// sequenceTerminate is a wrapper for sequenece termination (invalidation or expiration) -func (v *VirtualPacketConn) sequenceTerminate() { - v.Terminate(3) -} - -// Close provides a Close function to be called by the underlying transfer protocol. -// Do not call the function manually; otherwise the underlying transfer protocol may not have time to send a termination message (and the remote peer would subsequently try to reconnect). -// Rather, use the underlying transfer protocol's close function. -func (v *VirtualPacketConn) Close(reason int) (err error) { - v.Peer.Backend.networks.Sequences.InvalidateSequence(v.Peer.PublicKey, v.sequenceNumber, true) - return v.Terminate(reason) -} - -// CloseLinger is to be called by the underlying transfer protocol when it will close the socket soon after lingering around. -// Lingering happens to resend packets at the end of transfer, when it is not immediately known whether the remote peer received all packets. -func (v *VirtualPacketConn) CloseLinger(reason int) (err error) { - v.reason = reason - return nil -} - -// GetTerminateReason returns the termination reason. 0 = Not yet terminated. -func (v *VirtualPacketConn) GetTerminateReason() int { - return v.reason -} +/* +File Name: Transfer Virtual Connection.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +This file defines a virtual connection between a transfer protocol and Peernet messages. +If either the downstream transfer protocol or upstream Peernet messages indicate termination, the virtual connection ceases to exist. +*/ + +package core + +import ( + "sync" + + "github.com/google/uuid" +) + +// VirtualPacketConn is a virtual connection. +type VirtualPacketConn struct { + Peer *PeerInfo + + // Stats are maintained by the caller + Stats interface{} + + // function to send data to the remote peer + sendData func(data []byte, sequenceNumber uint32, transferID uuid.UUID) + + // Sequence number from the first outgoing or incoming packet. + sequenceNumber uint32 + + // Transfer ID represents a session ID valid only for the duration of the transfer. + transferID uuid.UUID + + // data channel + incomingData chan []byte + outgoingData chan []byte + + // internal data + closed bool + terminationSignal chan struct{} // The termination signal shall be used by the underlying protocol to detect upstream termination. + reason int // Reason why it was closed + sync.Mutex +} + +// newVirtualPacketConn creates a new virtual connection (both incoming and outgoing). +func newVirtualPacketConn(peer *PeerInfo, sendData func(data []byte, sequenceNumber uint32, transferID uuid.UUID)) (v *VirtualPacketConn) { + v = &VirtualPacketConn{ + Peer: peer, + sendData: sendData, + incomingData: make(chan []byte, 512), + outgoingData: make(chan []byte), + terminationSignal: make(chan struct{}), + } + + go v.writeForward() + + return +} + +// writeForward forwards outgoing messages +func (v *VirtualPacketConn) writeForward() { + for { + select { + case data := <-v.outgoingData: + v.sendData(data, v.sequenceNumber, v.transferID) + + case <-v.terminationSignal: + return + } + } +} + +// receiveData receives incoming data via an external message. Non-blocking. +func (v *VirtualPacketConn) receiveData(data []byte) { + if v.IsTerminated() { + return + } + + // pass the data on + select { + case v.incomingData <- data: + case <-v.terminationSignal: + default: + // packet lost + } +} + +// Terminate closes the connection. Do not call this function manually. Use the underlying protocol's function to close the connection. +// Reason: 404 = Remote peer does not store file (upstream), 2 = Remote termination signal (upstream), 3 = Sequence invalidation or expiration (upstream), 1000+ = Transfer protocol indicated closing (downstream) +func (v *VirtualPacketConn) Terminate(reason int) (err error) { + v.Lock() + defer v.Unlock() + + if v.closed { // if already closed, take no action + return + } + + v.closed = true + v.reason = reason + close(v.terminationSignal) + + return +} + +// IsTerminated checks if the connection is terminated +func (v *VirtualPacketConn) IsTerminated() bool { + return v.closed +} + +// sequenceTerminate is a wrapper for sequenece termination (invalidation or expiration) +func (v *VirtualPacketConn) sequenceTerminate() { + v.Terminate(3) +} + +// Close provides a Close function to be called by the underlying transfer protocol. +// Do not call the function manually; otherwise the underlying transfer protocol may not have time to send a termination message (and the remote peer would subsequently try to reconnect). +// Rather, use the underlying transfer protocol's close function. +func (v *VirtualPacketConn) Close(reason int) (err error) { + v.Peer.Backend.networks.Sequences.InvalidateSequence(v.Peer.PublicKey, v.sequenceNumber, true) + return v.Terminate(reason) +} + +// CloseLinger is to be called by the underlying transfer protocol when it will close the socket soon after lingering around. +// Lingering happens to resend packets at the end of transfer, when it is not immediately known whether the remote peer received all packets. +func (v *VirtualPacketConn) CloseLinger(reason int) (err error) { + v.reason = reason + return nil +} + +// GetTerminateReason returns the termination reason. 0 = Not yet terminated. +func (v *VirtualPacketConn) GetTerminateReason() int { + return v.reason +} diff --git a/Warehouse.go b/Warehouse.go index f637c80..b6b176e 100644 --- a/Warehouse.go +++ b/Warehouse.go @@ -1,20 +1,20 @@ -/* -File Name: Warehouse.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package core - -import ( - "github.com/PeernetOfficial/core/warehouse" -) - -func (backend *Backend) initUserWarehouse() { - var err error - backend.UserWarehouse, err = warehouse.Init(backend.Config.WarehouseMain) - - if err != nil { - backend.LogError("initUserWarehouse", "error: %s\n", err.Error()) - } -} +/* +File Name: Warehouse.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package core + +import ( + "github.com/PeernetOfficial/core/warehouse" +) + +func (backend *Backend) initUserWarehouse() { + var err error + backend.UserWarehouse, err = warehouse.Init(backend.Config.WarehouseMain) + + if err != nil { + backend.LogError("initUserWarehouse", "error: %s\n", err.Error()) + } +} diff --git a/blockchain/Block Record File.go b/blockchain/Block Record File.go index a019fa3..69de041 100644 --- a/blockchain/Block Record File.go +++ b/blockchain/Block Record File.go @@ -1,246 +1,246 @@ -/* -File Name: Block Record File.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -File records: -Offset Size Info -0 32 Hash blake3 of the file content -32 16 File ID -48 32 Merkle Root Hash -80 8 Fragment Size -88 1 File Type -89 2 File Format -91 8 File Size -99 2 Count of Tags -101 ? Tags - -Each file tag provides additional optional information: -Offset Size Info -0 2 Type -2 4 Size of data that follows -6 ? Data according to the tag type - -Tag data record contains only raw data and may be referenced by Tags in File records. -This is a basic embedded way of compression when tags are repetitive in multiple files within the same block. - -*/ - -package blockchain - -import ( - "encoding/binary" - "errors" - "math" - - "github.com/PeernetOfficial/core/protocol" - "github.com/google/uuid" -) - -// BlockRecordFile is the metadata of a file published on the blockchain -type BlockRecordFile struct { - Hash []byte // Hash of the file data - ID uuid.UUID // ID of the file - MerkleRootHash []byte // Merkle Root Hash - FragmentSize uint64 // Fragment Size - Type uint8 // File Type - Format uint16 // File Format - Size uint64 // Size of the file data - NodeID []byte // Node ID, owner of the file - Tags []BlockRecordFileTag // Tags provide additional metadata -} - -// BlockRecordFileTag provides metadata about the file. -type BlockRecordFileTag struct { - Type uint16 // See TagX constants. - Data []byte // Data - - // If top bit of Type is set, then Data must be 2, 4, or 8 bytes representing the distance number (positive or negative) of raw record in the block that will be used as data. - // This is an embedded basic compression algorithm for repetitive tag. For example directory tags or album tags might be heavily repetitive among files. -} - -const blockRecordFileMinSize = 101 - -// decodeBlockRecordFiles decodes only file records. Other records are ignored. -func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files []BlockRecordFile, err error) { - for i, record := range recordsRaw { - switch record.Type { - case RecordTypeFile: - if len(record.Data) < blockRecordFileMinSize { - return nil, errors.New("file record invalid size") - } - - file := BlockRecordFile{NodeID: nodeID} - file.Hash = make([]byte, protocol.HashSize) - copy(file.Hash, record.Data[0:0+protocol.HashSize]) - copy(file.ID[:], record.Data[32:32+16]) - - file.MerkleRootHash = make([]byte, protocol.HashSize) - copy(file.MerkleRootHash, record.Data[48:48+protocol.HashSize]) - file.FragmentSize = binary.LittleEndian.Uint64(record.Data[80 : 80+8]) - - file.Type = record.Data[88] - file.Format = binary.LittleEndian.Uint16(record.Data[89 : 89+2]) - file.Size = binary.LittleEndian.Uint64(record.Data[91 : 91+8]) - - countTags := binary.LittleEndian.Uint16(record.Data[99 : 99+2]) - - index := blockRecordFileMinSize - - for n := uint16(0); n < countTags; n++ { - if index+6 > len(record.Data) { - return nil, errors.New("file record tags invalid size") - } - - tag := BlockRecordFileTag{} - tag.Type = binary.LittleEndian.Uint16(record.Data[index:index+2]) & 0x7FFF - tagSize := binary.LittleEndian.Uint32(record.Data[index+2 : index+2+4]) - isDataReference := record.Data[index+1]&0x80 != 0 - - if index+6+int(tagSize) > len(record.Data) { - return nil, errors.New("file record tag data invalid size") - } - - if isDataReference { // reference to RecordTypeTagData record? - var refRecordNumber int - if tagSize == 2 { - refRecordNumber = i + int(int16(binary.LittleEndian.Uint16(record.Data[index+6:index+6+2]))) - } else if tagSize == 4 { - refRecordNumber = i + int(int32(binary.LittleEndian.Uint32(record.Data[index+6:index+6+4]))) - } else if tagSize == 8 { - refRecordNumber = i + int(int64(binary.LittleEndian.Uint64(record.Data[index+6:index+6+8]))) - } else { - return nil, errors.New("file record tag reference invalid size") - } - - if refRecordNumber < 0 || refRecordNumber >= len(recordsRaw) { - return nil, errors.New("file record tag reference not available") - } else if recordsRaw[refRecordNumber].Type != RecordTypeTagData { - return nil, errors.New("file record tag reference invalid") - } - - tag.Data = recordsRaw[refRecordNumber].Data - - } else { - tag.Data = record.Data[index+6 : index+6+int(tagSize)] - } - - file.Tags = append(file.Tags, tag) - - index += 6 + int(tagSize) - } - - file.Tags = append(file.Tags, TagFromDate(TagDateShared, record.Date)) - - files = append(files, file) - } - } - - return files, err -} - -// encodeBlockRecordFiles encodes files into the block record data -// This function should be called grouped with all files in the same folder. The folder name is deduplicated; only unique folder records will be returned. -// Note that this function only stores the folder names as tags; it does not create separate TypeFolder file records. -func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRaw, err error) { - uniqueTagDataMap := make(map[string]struct{}) - duplicateTagDataMap := make(map[string]int) // list of tag data that appeared twice. Number in recordsRaw. - - // loop through all tags to encode them and create list of duplicates that will be replaced by references - for n := range files { - for _, tag := range files[n].Tags { - if len(tag.Data) > 4 { - if _, ok := uniqueTagDataMap[string(tag.Data)]; !ok { - uniqueTagDataMap[string(tag.Data)] = struct{}{} - } else if _, ok := duplicateTagDataMap[string(tag.Data)]; !ok { - recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeTagData, Data: tag.Data}) - duplicateTagDataMap[string(tag.Data)] = len(recordsRaw) - 1 - } - } - } - } - - // then encode all files as records - for n := range files { - data := make([]byte, blockRecordFileMinSize) - - if len(files[n].Hash) != protocol.HashSize { - return nil, errors.New("encodeBlockRecords invalid file hash") - } else if len(files[n].MerkleRootHash) != protocol.HashSize { - return nil, errors.New("encodeBlockRecords invalid merkle root hash") - } - - copy(data[0:32], files[n].Hash[0:32]) - copy(data[32:32+16], files[n].ID[:]) - copy(data[48:48+32], files[n].MerkleRootHash[0:32]) - binary.LittleEndian.PutUint64(data[80:80+8], files[n].FragmentSize) - - data[88] = files[n].Type - binary.LittleEndian.PutUint16(data[89:89+2], files[n].Format) - binary.LittleEndian.PutUint64(data[91:91+8], files[n].Size) - - var tagCount uint16 - - for _, tag := range files[n].Tags { - // Some tags are virtual and never stored on the blockchain. If attempted to write, ignore. - if tag.IsVirtual() { - continue - } - tagCount++ - - if len(tag.Data) > 4 { - if refNumber, ok := duplicateTagDataMap[string(tag.Data)]; ok { - // In case the data is duplicated, use reference to the RecordTypeTagData instead - tag.Type |= 0x8000 - tag.Data = intToBytes(-(len(recordsRaw) - refNumber)) - } - } - - var tempTag [6]byte - - binary.LittleEndian.PutUint16(tempTag[0:2], tag.Type) - binary.LittleEndian.PutUint32(tempTag[2:2+4], uint32(len(tag.Data))) - - data = append(data, tempTag[:]...) - data = append(data, tag.Data...) - } - - binary.LittleEndian.PutUint16(data[99:99+2], tagCount) - - recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeFile, Data: data}) - } - - return recordsRaw, nil -} - -// intToBytes encodes int to little endian byte array as it fits to 16, 32 or 64 bit. -func intToBytes(number int) (buffer []byte) { - buffer = make([]byte, 4) - - if number <= math.MaxInt16 && number >= math.MinInt16 { - binary.LittleEndian.PutUint16(buffer[0:2], uint16(number)) - return buffer[0:2] - } else if number <= math.MaxInt32 && number >= math.MinInt32 { - binary.LittleEndian.PutUint32(buffer[0:4], uint32(number)) - return buffer[0:4] - } - - binary.LittleEndian.PutUint64(buffer[0:8], uint64(number)) - return buffer[0:8] -} - -// SizeInBlock returns the full size this file takes up in a single block. (i.e., the record size) -// If paired with other files in a single block, compression (via tag references) may reduce the actual size. -func (file *BlockRecordFile) SizeInBlock() (size uint64) { - size = blockRecordHeaderSize + blockRecordFileMinSize - - for _, tag := range file.Tags { - if tag.IsVirtual() { - continue - } - - size += 6 + uint64(len(tag.Data)) - } - - return size -} +/* +File Name: Block Record File.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +File records: +Offset Size Info +0 32 Hash blake3 of the file content +32 16 File ID +48 32 Merkle Root Hash +80 8 Fragment Size +88 1 File Type +89 2 File Format +91 8 File Size +99 2 Count of Tags +101 ? Tags + +Each file tag provides additional optional information: +Offset Size Info +0 2 Type +2 4 Size of data that follows +6 ? Data according to the tag type + +Tag data record contains only raw data and may be referenced by Tags in File records. +This is a basic embedded way of compression when tags are repetitive in multiple files within the same block. + +*/ + +package blockchain + +import ( + "encoding/binary" + "errors" + "math" + + "github.com/PeernetOfficial/core/protocol" + "github.com/google/uuid" +) + +// BlockRecordFile is the metadata of a file published on the blockchain +type BlockRecordFile struct { + Hash []byte // Hash of the file data + ID uuid.UUID // ID of the file + MerkleRootHash []byte // Merkle Root Hash + FragmentSize uint64 // Fragment Size + Type uint8 // File Type + Format uint16 // File Format + Size uint64 // Size of the file data + NodeID []byte // Node ID, owner of the file + Tags []BlockRecordFileTag // Tags provide additional metadata +} + +// BlockRecordFileTag provides metadata about the file. +type BlockRecordFileTag struct { + Type uint16 // See TagX constants. + Data []byte // Data + + // If top bit of Type is set, then Data must be 2, 4, or 8 bytes representing the distance number (positive or negative) of raw record in the block that will be used as data. + // This is an embedded basic compression algorithm for repetitive tag. For example directory tags or album tags might be heavily repetitive among files. +} + +const blockRecordFileMinSize = 101 + +// decodeBlockRecordFiles decodes only file records. Other records are ignored. +func decodeBlockRecordFiles(recordsRaw []BlockRecordRaw, nodeID []byte) (files []BlockRecordFile, err error) { + for i, record := range recordsRaw { + switch record.Type { + case RecordTypeFile: + if len(record.Data) < blockRecordFileMinSize { + return nil, errors.New("file record invalid size") + } + + file := BlockRecordFile{NodeID: nodeID} + file.Hash = make([]byte, protocol.HashSize) + copy(file.Hash, record.Data[0:0+protocol.HashSize]) + copy(file.ID[:], record.Data[32:32+16]) + + file.MerkleRootHash = make([]byte, protocol.HashSize) + copy(file.MerkleRootHash, record.Data[48:48+protocol.HashSize]) + file.FragmentSize = binary.LittleEndian.Uint64(record.Data[80 : 80+8]) + + file.Type = record.Data[88] + file.Format = binary.LittleEndian.Uint16(record.Data[89 : 89+2]) + file.Size = binary.LittleEndian.Uint64(record.Data[91 : 91+8]) + + countTags := binary.LittleEndian.Uint16(record.Data[99 : 99+2]) + + index := blockRecordFileMinSize + + for n := uint16(0); n < countTags; n++ { + if index+6 > len(record.Data) { + return nil, errors.New("file record tags invalid size") + } + + tag := BlockRecordFileTag{} + tag.Type = binary.LittleEndian.Uint16(record.Data[index:index+2]) & 0x7FFF + tagSize := binary.LittleEndian.Uint32(record.Data[index+2 : index+2+4]) + isDataReference := record.Data[index+1]&0x80 != 0 + + if index+6+int(tagSize) > len(record.Data) { + return nil, errors.New("file record tag data invalid size") + } + + if isDataReference { // reference to RecordTypeTagData record? + var refRecordNumber int + if tagSize == 2 { + refRecordNumber = i + int(int16(binary.LittleEndian.Uint16(record.Data[index+6:index+6+2]))) + } else if tagSize == 4 { + refRecordNumber = i + int(int32(binary.LittleEndian.Uint32(record.Data[index+6:index+6+4]))) + } else if tagSize == 8 { + refRecordNumber = i + int(int64(binary.LittleEndian.Uint64(record.Data[index+6:index+6+8]))) + } else { + return nil, errors.New("file record tag reference invalid size") + } + + if refRecordNumber < 0 || refRecordNumber >= len(recordsRaw) { + return nil, errors.New("file record tag reference not available") + } else if recordsRaw[refRecordNumber].Type != RecordTypeTagData { + return nil, errors.New("file record tag reference invalid") + } + + tag.Data = recordsRaw[refRecordNumber].Data + + } else { + tag.Data = record.Data[index+6 : index+6+int(tagSize)] + } + + file.Tags = append(file.Tags, tag) + + index += 6 + int(tagSize) + } + + file.Tags = append(file.Tags, TagFromDate(TagDateShared, record.Date)) + + files = append(files, file) + } + } + + return files, err +} + +// encodeBlockRecordFiles encodes files into the block record data +// This function should be called grouped with all files in the same folder. The folder name is deduplicated; only unique folder records will be returned. +// Note that this function only stores the folder names as tags; it does not create separate TypeFolder file records. +func encodeBlockRecordFiles(files []BlockRecordFile) (recordsRaw []BlockRecordRaw, err error) { + uniqueTagDataMap := make(map[string]struct{}) + duplicateTagDataMap := make(map[string]int) // list of tag data that appeared twice. Number in recordsRaw. + + // loop through all tags to encode them and create list of duplicates that will be replaced by references + for n := range files { + for _, tag := range files[n].Tags { + if len(tag.Data) > 4 { + if _, ok := uniqueTagDataMap[string(tag.Data)]; !ok { + uniqueTagDataMap[string(tag.Data)] = struct{}{} + } else if _, ok := duplicateTagDataMap[string(tag.Data)]; !ok { + recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeTagData, Data: tag.Data}) + duplicateTagDataMap[string(tag.Data)] = len(recordsRaw) - 1 + } + } + } + } + + // then encode all files as records + for n := range files { + data := make([]byte, blockRecordFileMinSize) + + if len(files[n].Hash) != protocol.HashSize { + return nil, errors.New("encodeBlockRecords invalid file hash") + } else if len(files[n].MerkleRootHash) != protocol.HashSize { + return nil, errors.New("encodeBlockRecords invalid merkle root hash") + } + + copy(data[0:32], files[n].Hash[0:32]) + copy(data[32:32+16], files[n].ID[:]) + copy(data[48:48+32], files[n].MerkleRootHash[0:32]) + binary.LittleEndian.PutUint64(data[80:80+8], files[n].FragmentSize) + + data[88] = files[n].Type + binary.LittleEndian.PutUint16(data[89:89+2], files[n].Format) + binary.LittleEndian.PutUint64(data[91:91+8], files[n].Size) + + var tagCount uint16 + + for _, tag := range files[n].Tags { + // Some tags are virtual and never stored on the blockchain. If attempted to write, ignore. + if tag.IsVirtual() { + continue + } + tagCount++ + + if len(tag.Data) > 4 { + if refNumber, ok := duplicateTagDataMap[string(tag.Data)]; ok { + // In case the data is duplicated, use reference to the RecordTypeTagData instead + tag.Type |= 0x8000 + tag.Data = intToBytes(-(len(recordsRaw) - refNumber)) + } + } + + var tempTag [6]byte + + binary.LittleEndian.PutUint16(tempTag[0:2], tag.Type) + binary.LittleEndian.PutUint32(tempTag[2:2+4], uint32(len(tag.Data))) + + data = append(data, tempTag[:]...) + data = append(data, tag.Data...) + } + + binary.LittleEndian.PutUint16(data[99:99+2], tagCount) + + recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeFile, Data: data}) + } + + return recordsRaw, nil +} + +// intToBytes encodes int to little endian byte array as it fits to 16, 32 or 64 bit. +func intToBytes(number int) (buffer []byte) { + buffer = make([]byte, 4) + + if number <= math.MaxInt16 && number >= math.MinInt16 { + binary.LittleEndian.PutUint16(buffer[0:2], uint16(number)) + return buffer[0:2] + } else if number <= math.MaxInt32 && number >= math.MinInt32 { + binary.LittleEndian.PutUint32(buffer[0:4], uint32(number)) + return buffer[0:4] + } + + binary.LittleEndian.PutUint64(buffer[0:8], uint64(number)) + return buffer[0:8] +} + +// SizeInBlock returns the full size this file takes up in a single block. (i.e., the record size) +// If paired with other files in a single block, compression (via tag references) may reduce the actual size. +func (file *BlockRecordFile) SizeInBlock() (size uint64) { + size = blockRecordHeaderSize + blockRecordFileMinSize + + for _, tag := range file.Tags { + if tag.IsVirtual() { + continue + } + + size += 6 + uint64(len(tag.Data)) + } + + return size +} diff --git a/blockchain/Block Record Profile.go b/blockchain/Block Record Profile.go index ecac278..535bcc8 100644 --- a/blockchain/Block Record Profile.go +++ b/blockchain/Block Record Profile.go @@ -1,75 +1,75 @@ -/* -File Name: Block Record Profile.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Profile records: -Offset Size Info -0 2 Type -2 ? Data according to the type - -*/ - -package blockchain - -import ( - "encoding/binary" - "errors" - "math" -) - -// BlockRecordProfile provides information about the end user. -type BlockRecordProfile struct { - Type uint16 // See ProfileX constants. - Data []byte // Data -} - -// decodeBlockRecordProfile decodes only profile records. Other records are ignored. -func decodeBlockRecordProfile(recordsRaw []BlockRecordRaw) (fields []BlockRecordProfile, err error) { - fieldMap := make(map[uint16][]byte) - - for _, record := range recordsRaw { - if record.Type != RecordTypeProfile { - continue - } - - if len(record.Data) < 2 { - return nil, errors.New("profile record invalid size") - } - - fieldType := binary.LittleEndian.Uint16(record.Data[0:2]) - fieldMap[fieldType] = record.Data[2:] - } - - for fieldType, fieldData := range fieldMap { - fields = append(fields, BlockRecordProfile{Type: fieldType, Data: fieldData}) - } - - return fields, nil -} - -// encodeBlockRecordProfile encodes the profile record. -func encodeBlockRecordProfile(fields []BlockRecordProfile) (recordsRaw []BlockRecordRaw, err error) { - if len(fields) > math.MaxUint16 { - return nil, errors.New("exceeding max count of fields") - } - - for n := range fields { - if uint64(len(fields[n].Data)) > math.MaxUint32 { - return nil, errors.New("exceeding max field size") - } - - data := make([]byte, 2) - binary.LittleEndian.PutUint16(data[0:2], fields[n].Type) - data = append(data, fields[n].Data...) - - recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeProfile, Data: data}) - } - - return recordsRaw, nil -} - -// SizeInBlock returns the full size this file takes up in a single block. (i.e., the record size) -func (field *BlockRecordProfile) SizeInBlock() (size uint64) { - return blockRecordHeaderSize + 2 + uint64(len(field.Data)) -} +/* +File Name: Block Record Profile.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Profile records: +Offset Size Info +0 2 Type +2 ? Data according to the type + +*/ + +package blockchain + +import ( + "encoding/binary" + "errors" + "math" +) + +// BlockRecordProfile provides information about the end user. +type BlockRecordProfile struct { + Type uint16 // See ProfileX constants. + Data []byte // Data +} + +// decodeBlockRecordProfile decodes only profile records. Other records are ignored. +func decodeBlockRecordProfile(recordsRaw []BlockRecordRaw) (fields []BlockRecordProfile, err error) { + fieldMap := make(map[uint16][]byte) + + for _, record := range recordsRaw { + if record.Type != RecordTypeProfile { + continue + } + + if len(record.Data) < 2 { + return nil, errors.New("profile record invalid size") + } + + fieldType := binary.LittleEndian.Uint16(record.Data[0:2]) + fieldMap[fieldType] = record.Data[2:] + } + + for fieldType, fieldData := range fieldMap { + fields = append(fields, BlockRecordProfile{Type: fieldType, Data: fieldData}) + } + + return fields, nil +} + +// encodeBlockRecordProfile encodes the profile record. +func encodeBlockRecordProfile(fields []BlockRecordProfile) (recordsRaw []BlockRecordRaw, err error) { + if len(fields) > math.MaxUint16 { + return nil, errors.New("exceeding max count of fields") + } + + for n := range fields { + if uint64(len(fields[n].Data)) > math.MaxUint32 { + return nil, errors.New("exceeding max field size") + } + + data := make([]byte, 2) + binary.LittleEndian.PutUint16(data[0:2], fields[n].Type) + data = append(data, fields[n].Data...) + + recordsRaw = append(recordsRaw, BlockRecordRaw{Type: RecordTypeProfile, Data: data}) + } + + return recordsRaw, nil +} + +// SizeInBlock returns the full size this file takes up in a single block. (i.e., the record size) +func (field *BlockRecordProfile) SizeInBlock() (size uint64) { + return blockRecordHeaderSize + 2 + uint64(len(field.Data)) +} diff --git a/blockchain/Block Record.go b/blockchain/Block Record.go index 6e4020b..5a411e6 100644 --- a/blockchain/Block Record.go +++ b/blockchain/Block Record.go @@ -1,55 +1,55 @@ -/* -File Name: Block Record.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Each record inside the block has this basic structure: -Offset Size Info -0 1 Record type -1 8 Date created. This remains the same in case of block refactoring. -9 4 Size of data -13 ? Data (encoding depends on record type) - -*/ - -package blockchain - -// RecordTypeX defines the type of the record -const ( - RecordTypeProfile = 0 // Profile data about the end user. - RecordTypeTagData = 1 // Tag data record to be referenced by one or multiple tags. Only valid in the context of the current block. - RecordTypeFile = 2 // File - RecordTypeInvalid1 = 3 // Do not use. - RecordTypeCertificate = 4 // Certificate to certify provided information in the blockchain issued by a trusted 3rd party. - RecordTypeContentRating = 5 // Content rating (positive). - RecordTypeContentReport = 6 // Content report (negative). -) - -// BlockDecoded contains the decoded records from a block -type BlockDecoded struct { - Block - RecordsDecoded []interface{} // Decoded records. See BlockRecordX structures. -} - -// decodeBlockRecords decodes all raw records in the block and returns a high-level decoded structure -// Use decodeBlockRecordX instead for specific record decoding. -func decodeBlockRecords(block *Block) (decoded *BlockDecoded, err error) { - decoded = &BlockDecoded{Block: *block} - - files, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) - if err != nil { - return nil, err - } - - for _, file := range files { - decoded.RecordsDecoded = append(decoded.RecordsDecoded, file) - } - - if profileFields, err := decodeBlockRecordProfile(block.RecordsRaw); err != nil { - return nil, err - } else if len(profileFields) > 0 { - decoded.RecordsDecoded = append(decoded.RecordsDecoded, profileFields) - } - - return decoded, nil -} +/* +File Name: Block Record.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Each record inside the block has this basic structure: +Offset Size Info +0 1 Record type +1 8 Date created. This remains the same in case of block refactoring. +9 4 Size of data +13 ? Data (encoding depends on record type) + +*/ + +package blockchain + +// RecordTypeX defines the type of the record +const ( + RecordTypeProfile = 0 // Profile data about the end user. + RecordTypeTagData = 1 // Tag data record to be referenced by one or multiple tags. Only valid in the context of the current block. + RecordTypeFile = 2 // File + RecordTypeInvalid1 = 3 // Do not use. + RecordTypeCertificate = 4 // Certificate to certify provided information in the blockchain issued by a trusted 3rd party. + RecordTypeContentRating = 5 // Content rating (positive). + RecordTypeContentReport = 6 // Content report (negative). +) + +// BlockDecoded contains the decoded records from a block +type BlockDecoded struct { + Block + RecordsDecoded []interface{} // Decoded records. See BlockRecordX structures. +} + +// decodeBlockRecords decodes all raw records in the block and returns a high-level decoded structure +// Use decodeBlockRecordX instead for specific record decoding. +func decodeBlockRecords(block *Block) (decoded *BlockDecoded, err error) { + decoded = &BlockDecoded{Block: *block} + + files, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) + if err != nil { + return nil, err + } + + for _, file := range files { + decoded.RecordsDecoded = append(decoded.RecordsDecoded, file) + } + + if profileFields, err := decodeBlockRecordProfile(block.RecordsRaw); err != nil { + return nil, err + } else if len(profileFields) > 0 { + decoded.RecordsDecoded = append(decoded.RecordsDecoded, profileFields) + } + + return decoded, nil +} diff --git a/blockchain/Block.go b/blockchain/Block.go index e08d5e6..9b546f6 100644 --- a/blockchain/Block.go +++ b/blockchain/Block.go @@ -1,162 +1,162 @@ -/* -File Name: Block.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Encoding of a block (it is the same stored in the database and shared in a message): -Offset Size Info -0 65 Signature of entire block -65 32 Hash (blake3) of last block. 0 for first one. -97 8 Blockchain version number -105 8 Block number -113 4 Size of entire block including this header -117 2 Count of records that follow - -*/ - -package blockchain - -import ( - "bytes" - "encoding/binary" - "errors" - "time" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/protocol" -) - -// Block is a single block containing a set of records (metadata). -// It has no upper size limit, although a soft limit of 64 KB - overhead is encouraged for efficiency. -type Block struct { - OwnerPublicKey *btcec.PublicKey // Owner Public Key, ECDSA (secp256k1) 257-bit - NodeID []byte // Node ID of the owner (derived from the public key) - LastBlockHash []byte // Hash of the last block. Blake3. - BlockchainVersion uint64 // Blockchain version - Number uint64 // Block number - RecordsRaw []BlockRecordRaw // Block records raw -} - -// BlockRecordRaw is a single block record (not decoded) -type BlockRecordRaw struct { - Type uint8 // Record Type. See RecordTypeX. - Date time.Time // Date created. This remains the same in case of block refactoring. - Data []byte // Data according to the type -} - -const blockHeaderSize = 119 -const blockRecordHeaderSize = 13 - -// decodeBlock decodes a single block -func decodeBlock(raw []byte) (block *Block, err error) { - if len(raw) < blockHeaderSize { - return nil, errors.New("decodeBlock invalid block size") - } - - block = &Block{} - - signature := raw[0 : 0+65] - - block.OwnerPublicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, protocol.HashData(raw[65:])) - if err != nil { - return nil, err - } - - block.NodeID = protocol.PublicKey2NodeID(block.OwnerPublicKey) - - block.LastBlockHash = make([]byte, protocol.HashSize) - copy(block.LastBlockHash, raw[65:65+protocol.HashSize]) - - block.BlockchainVersion = binary.LittleEndian.Uint64(raw[97 : 97+8]) - block.Number = binary.LittleEndian.Uint64(raw[105 : 105+8]) - - blockSize := binary.LittleEndian.Uint32(raw[113 : 113+4]) - if blockSize != uint32(len(raw)) { - return nil, errors.New("decodeBlock invalid block size") - } - - // decode on a low-level all block records - countRecords := binary.LittleEndian.Uint16(raw[117 : 117+2]) - index := blockHeaderSize - - for n := uint16(0); n < countRecords; n++ { - if index+blockRecordHeaderSize > len(raw) { - return nil, errors.New("decodeBlock record exceeds block size") - } - - recordType := raw[index] - recordDate := int64(binary.LittleEndian.Uint64(raw[index+1 : index+9])) // Unix time int64, the number of seconds elapsed since January 1, 1970 UTC - recordSize := binary.LittleEndian.Uint32(raw[index+9 : index+9+4]) - index += blockRecordHeaderSize - - if index+int(recordSize) > len(raw) { - return nil, errors.New("decodeBlock record exceeds block size") - } - - block.RecordsRaw = append(block.RecordsRaw, BlockRecordRaw{Type: recordType, Data: raw[index : index+int(recordSize)], Date: time.Unix(recordDate, 0)}) - - index += int(recordSize) - } - - return block, nil -} - -func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, err error) { - var buffer bytes.Buffer - buffer.Write(make([]byte, 65)) // Signature, filled at the end - - if block.Number > 0 && len(block.LastBlockHash) != protocol.HashSize { - return nil, errors.New("encodeBlock invalid last block hash") - } else if block.Number == 0 { // Block 0: Empty last hash - block.LastBlockHash = make([]byte, 32) - } - buffer.Write(block.LastBlockHash) - - var temp [8]byte - binary.LittleEndian.PutUint64(temp[0:8], block.BlockchainVersion) - buffer.Write(temp[:8]) - - binary.LittleEndian.PutUint64(temp[0:8], block.Number) - buffer.Write(temp[:8]) - - buffer.Write(make([]byte, 4)) // Size of block, filled later - buffer.Write(make([]byte, 2)) // Count of records, filled later - - // write all records - countRecords := uint16(0) - - for _, record := range block.RecordsRaw { - if record.Date == (time.Time{}) { // Always set date if not already set - record.Date = time.Now() - } - - var tempSize, tempDate [8]byte - binary.LittleEndian.PutUint32(tempSize[0:4], uint32(len(record.Data))) - binary.LittleEndian.PutUint64(tempDate[0:8], uint64(record.Date.UTC().Unix())) - - buffer.Write([]byte{record.Type}) // Record Type - buffer.Write(tempDate[:8]) // Date created - buffer.Write(tempSize[:4]) // Size of data - buffer.Write(record.Data) // Data - - countRecords++ - } - - // finalize the block - raw = buffer.Bytes() - if len(raw) < blockHeaderSize { - return nil, errors.New("encodeBlock invalid block size") - } - - binary.LittleEndian.PutUint32(raw[113:113+4], uint32(len(raw))) // Size of block - binary.LittleEndian.PutUint16(raw[117:117+2], countRecords) // Count of records - - // signature is last - signature, err := btcec.SignCompact(btcec.S256(), ownerPrivateKey, protocol.HashData(raw[65:]), true) - if err != nil { - return nil, err - } - copy(raw[0:0+65], signature) - - return raw, nil -} +/* +File Name: Block.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Encoding of a block (it is the same stored in the database and shared in a message): +Offset Size Info +0 65 Signature of entire block +65 32 Hash (blake3) of last block. 0 for first one. +97 8 Blockchain version number +105 8 Block number +113 4 Size of entire block including this header +117 2 Count of records that follow + +*/ + +package blockchain + +import ( + "bytes" + "encoding/binary" + "errors" + "time" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" +) + +// Block is a single block containing a set of records (metadata). +// It has no upper size limit, although a soft limit of 64 KB - overhead is encouraged for efficiency. +type Block struct { + OwnerPublicKey *btcec.PublicKey // Owner Public Key, ECDSA (secp256k1) 257-bit + NodeID []byte // Node ID of the owner (derived from the public key) + LastBlockHash []byte // Hash of the last block. Blake3. + BlockchainVersion uint64 // Blockchain version + Number uint64 // Block number + RecordsRaw []BlockRecordRaw // Block records raw +} + +// BlockRecordRaw is a single block record (not decoded) +type BlockRecordRaw struct { + Type uint8 // Record Type. See RecordTypeX. + Date time.Time // Date created. This remains the same in case of block refactoring. + Data []byte // Data according to the type +} + +const blockHeaderSize = 119 +const blockRecordHeaderSize = 13 + +// decodeBlock decodes a single block +func decodeBlock(raw []byte) (block *Block, err error) { + if len(raw) < blockHeaderSize { + return nil, errors.New("decodeBlock invalid block size") + } + + block = &Block{} + + signature := raw[0 : 0+65] + + block.OwnerPublicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, protocol.HashData(raw[65:])) + if err != nil { + return nil, err + } + + block.NodeID = protocol.PublicKey2NodeID(block.OwnerPublicKey) + + block.LastBlockHash = make([]byte, protocol.HashSize) + copy(block.LastBlockHash, raw[65:65+protocol.HashSize]) + + block.BlockchainVersion = binary.LittleEndian.Uint64(raw[97 : 97+8]) + block.Number = binary.LittleEndian.Uint64(raw[105 : 105+8]) + + blockSize := binary.LittleEndian.Uint32(raw[113 : 113+4]) + if blockSize != uint32(len(raw)) { + return nil, errors.New("decodeBlock invalid block size") + } + + // decode on a low-level all block records + countRecords := binary.LittleEndian.Uint16(raw[117 : 117+2]) + index := blockHeaderSize + + for n := uint16(0); n < countRecords; n++ { + if index+blockRecordHeaderSize > len(raw) { + return nil, errors.New("decodeBlock record exceeds block size") + } + + recordType := raw[index] + recordDate := int64(binary.LittleEndian.Uint64(raw[index+1 : index+9])) // Unix time int64, the number of seconds elapsed since January 1, 1970 UTC + recordSize := binary.LittleEndian.Uint32(raw[index+9 : index+9+4]) + index += blockRecordHeaderSize + + if index+int(recordSize) > len(raw) { + return nil, errors.New("decodeBlock record exceeds block size") + } + + block.RecordsRaw = append(block.RecordsRaw, BlockRecordRaw{Type: recordType, Data: raw[index : index+int(recordSize)], Date: time.Unix(recordDate, 0)}) + + index += int(recordSize) + } + + return block, nil +} + +func encodeBlock(block *Block, ownerPrivateKey *btcec.PrivateKey) (raw []byte, err error) { + var buffer bytes.Buffer + buffer.Write(make([]byte, 65)) // Signature, filled at the end + + if block.Number > 0 && len(block.LastBlockHash) != protocol.HashSize { + return nil, errors.New("encodeBlock invalid last block hash") + } else if block.Number == 0 { // Block 0: Empty last hash + block.LastBlockHash = make([]byte, 32) + } + buffer.Write(block.LastBlockHash) + + var temp [8]byte + binary.LittleEndian.PutUint64(temp[0:8], block.BlockchainVersion) + buffer.Write(temp[:8]) + + binary.LittleEndian.PutUint64(temp[0:8], block.Number) + buffer.Write(temp[:8]) + + buffer.Write(make([]byte, 4)) // Size of block, filled later + buffer.Write(make([]byte, 2)) // Count of records, filled later + + // write all records + countRecords := uint16(0) + + for _, record := range block.RecordsRaw { + if record.Date == (time.Time{}) { // Always set date if not already set + record.Date = time.Now() + } + + var tempSize, tempDate [8]byte + binary.LittleEndian.PutUint32(tempSize[0:4], uint32(len(record.Data))) + binary.LittleEndian.PutUint64(tempDate[0:8], uint64(record.Date.UTC().Unix())) + + buffer.Write([]byte{record.Type}) // Record Type + buffer.Write(tempDate[:8]) // Date created + buffer.Write(tempSize[:4]) // Size of data + buffer.Write(record.Data) // Data + + countRecords++ + } + + // finalize the block + raw = buffer.Bytes() + if len(raw) < blockHeaderSize { + return nil, errors.New("encodeBlock invalid block size") + } + + binary.LittleEndian.PutUint32(raw[113:113+4], uint32(len(raw))) // Size of block + binary.LittleEndian.PutUint16(raw[117:117+2], countRecords) // Count of records + + // signature is last + signature, err := btcec.SignCompact(btcec.S256(), ownerPrivateKey, protocol.HashData(raw[65:]), true) + if err != nil { + return nil, err + } + copy(raw[0:0+65], signature) + + return raw, nil +} diff --git a/blockchain/Blockchain.go b/blockchain/Blockchain.go index bc47ecf..f2a7e99 100644 --- a/blockchain/Blockchain.go +++ b/blockchain/Blockchain.go @@ -1,443 +1,443 @@ -/* -File Name: Blockchain.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -All blocks and the blockchain header are stored in a key-value database. -The key for the blockchain header is keyHeader and for each block is the block number as 64-bit unsigned integer little endian. - -Encoding of the blockchain header: -Offset Size Info -0 8 Height of the blockchain -8 8 Version of the blockchain -16 2 Format of the blockchain. This provides backward compatibility. -18 65 Signature - -*/ - -package blockchain - -import ( - "encoding/binary" - "errors" - "sync" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/store" -) - -// TargetBlockSize is the target size that a generated block shall not exceed. This ensures the block will be transferred via blockchain exchange and cached in DHT. -// Large blocks may be ignored by clients for size and spam reasons, resulting in decreased discoverability. -var TargetBlockSize = uint64(4096) - -// MinAcceptableBlockSize is the minimum block size peers must accept. -const MinAcceptableBlockSize = uint64(1024) - -// Blockchain stores the blockchain's header in memory. Any changes must be synced to disk! -type Blockchain struct { - // header - height uint64 // Height is exchanged as uint32 in the protocol, but stored as uint64. - version uint64 // Version is always uint64. - format uint16 // Format is only locally used. - - // internals - publicKey *btcec.PublicKey // Public Key of the owner. This must match the ones used on disk. - privateKey *btcec.PrivateKey // Private Key of the owner. This must match the ones used on disk. - path string // Path of the blockchain on disk. Depends on key-value store whether a filename or folder. - database store.Store // The database storing the blockchain. - sync.Mutex // synchronized access to the header - - // callback - BlockchainUpdate func(blockchain *Blockchain, oldHeight, oldVersion, newHeight, newVersion uint64) -} - -// Init initializes the given blockchain. It creates the blockchain file if it does not exist already. -func Init(privateKey *btcec.PrivateKey, path string) (blockchain *Blockchain, err error) { - blockchain = &Blockchain{privateKey: privateKey, path: path} - publicKey := privateKey.PubKey() - - // open existing blockchain file or create new one - if blockchain.database, err = store.NewPogrebStore(path); err != nil { - return nil, err - } - - // verify header - var found bool - - found, err = blockchain.headerRead() - if err != nil { - return blockchain, err // likely corrupt blockchain database - } else if !found { - // First run: create header signature! - blockchain.publicKey = publicKey - - if err := blockchain.headerWrite(0, 0); err != nil { - return blockchain, err - } - } else if !blockchain.publicKey.IsEqual(publicKey) { - return blockchain, errors.New("corrupt user blockchain database. Public key mismatch") - } - - return blockchain, nil -} - -// the key names in the key-value database are constant and must not collide with block numbers (i.e. they must be >64 bit) -const keyHeader = "header blockchain" - -// headerRead reads the header from the blockchain and decodes it. -func (blockchain *Blockchain) headerRead() (found bool, err error) { - buffer, found := blockchain.database.Get([]byte(keyHeader)) - if !found { - return false, nil - } - - if len(buffer) != 83 { - return true, errors.New("blockchain header size mismatch") - } - - blockchain.height = binary.LittleEndian.Uint64(buffer[0:8]) - blockchain.version = binary.LittleEndian.Uint64(buffer[8:16]) - blockchain.format = binary.LittleEndian.Uint16(buffer[16:18]) - signature := buffer[18 : 18+65] - - if blockchain.format != 0 { - return true, errors.New("future blockchain format not supported. You must go back to the future!") - } - - blockchain.publicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, protocol.HashData(buffer[0:18])) - - return -} - -// headerWrite writes the header to the blockchain and signs it. -func (blockchain *Blockchain) headerWrite(height, version uint64) (err error) { - oldHeight := blockchain.height - oldVersion := blockchain.version - - blockchain.height = height - blockchain.version = version - - var buffer [83]byte - binary.LittleEndian.PutUint64(buffer[0:8], height) - binary.LittleEndian.PutUint64(buffer[8:16], version) - binary.LittleEndian.PutUint16(buffer[16:18], 0) // Current format is 0 - - signature, err := btcec.SignCompact(btcec.S256(), blockchain.privateKey, protocol.HashData(buffer[0:18]), true) - - if err != nil { - return err - } else if len(signature) != 65 { - return errors.New("signature length invalid") - } - - copy(buffer[18:18+65], signature) - - err = blockchain.database.Set([]byte(keyHeader), buffer[:]) - - // call the callback, if any - if blockchain.BlockchainUpdate != nil { - blockchain.BlockchainUpdate(blockchain, oldHeight, oldVersion, blockchain.height, blockchain.version) - } - - return err -} - -// StatusX provides information about the blockchain status. Some errors codes indicate a corruption. -const ( - StatusOK = 0 // No problems in the blockchain detected. - StatusBlockNotFound = 1 // Missing block in the blockchain. - StatusCorruptBlock = 2 // Error block encoding - StatusCorruptBlockRecord = 3 // Error block record encoding - StatusDataNotFound = 4 // Requested data not available in the blockchain - StatusNotInWarehouse = 5 // File to be added to blockchain does not exist in the Warehouse -) - -// blockNumberToKey returns the database key for the given block number -func blockNumberToKey(number uint64) (key []byte) { - var target [8]byte - binary.LittleEndian.PutUint64(target[:], number) - - return target[:] -} - -// Iterate iterates over the blockchain. Status is StatusX. -// If the callback returns non-zero, the function aborts and returns the inner status code. -func (blockchain *Blockchain) Iterate(callback func(block *Block) int) (status int) { - // read all blocks until height is reached - height := blockchain.height - - for blockN := uint64(0); blockN < height; blockN++ { - blockRaw, found := blockchain.database.Get(blockNumberToKey(blockN)) - if !found || len(blockRaw) == 0 { - return StatusBlockNotFound - } - - block, err := decodeBlock(blockRaw) - if err != nil { - return StatusCorruptBlock - } - - if statusI := callback(block); statusI != StatusOK { - return statusI - } - } - - return StatusOK -} - -// IterateDeleteRecord iterates over the blockchain to find records to delete. Status is StatusX. -// deleteAction is 0 = no action on record, 1 = delete record, 2 = replace record, 3 = error blockchain corrupt -// If the callback returns true, the record will be deleted. The blockchain will be automatically refactored and height and version updated. -func (blockchain *Blockchain) IterateDeleteRecord(callbackFile func(file *BlockRecordFile) (deleteAction int), callbackOther func(record *BlockRecordRaw) (deleteAction int)) (newHeight, newVersion uint64, status int) { - blockchain.Lock() - defer blockchain.Unlock() - - // New blockchain keeps track of the new blocks. If anything changes in the blockchain, it must be recalculated and the version number increased. - var blockchainNew []Block - refactorBlockchain := false - refactorVersion := blockchain.version + 1 - - // Read all blocks until height is reached. At the end the height and version might be different if blocks are deleted. - height := blockchain.height - - for blockN := uint64(0); blockN < height; blockN++ { - blockRaw, found := blockchain.database.Get(blockNumberToKey(blockN)) - if !found || len(blockRaw) == 0 { - return 0, 0, StatusBlockNotFound - } - - block, err := decodeBlock(blockRaw) - if err != nil { - return 0, 0, StatusCorruptBlock - } - - refactorBlock := false - - // Decode all file records at once. This is needed due to potential referenced tags. - // If a file is deleted or referenced tag data changed, it would corrupt the blockchain if the other records were not updated. - filesD, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) - if err != nil { - return 0, 0, StatusCorruptBlock - } - - // loop through all file records in this block - var newFileRecords []BlockRecordFile - - if callbackFile == nil { - newFileRecords = filesD - } else { - for n := range filesD { - switch callbackFile(&filesD[n]) { - case 0: // no action on record - newFileRecords = append(newFileRecords, filesD[n]) - - case 1: // delete record - refactorBlock = true - refactorBlockchain = true - - case 2: // replace record - newFileRecords = append(newFileRecords, filesD[n]) - refactorBlock = true - refactorBlockchain = true - - case 3: // error blockchain corrupt - return 0, 0, StatusCorruptBlockRecord - } - } - } - - // loop through all other (non-file) records in this block - var newRecordsRaw []BlockRecordRaw - - for n := range block.RecordsRaw { - // File and Tag records were already handled in above loop. - if block.RecordsRaw[n].Type == RecordTypeFile || block.RecordsRaw[n].Type == RecordTypeTagData { - continue - } - - if callbackOther == nil { - newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) - } else { - switch callbackOther(&block.RecordsRaw[n]) { - case 0: // no action on record - newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) - - case 1: // delete record - refactorBlock = true - refactorBlockchain = true - - case 2: // replace record - newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) - refactorBlock = true - refactorBlockchain = true - - case 3: // error blockchain corrupt - return 0, 0, StatusCorruptBlockRecord - } - } - } - - // If refactor, re-calculate the block. All later blocks need to be re-encoded due to change of previous block hash. The version number needs to change. - // Note: Deleting records may leave referenced records orphaned, such as RecordTypeTagData for deleted file records. - if refactorBlock { - // re-encode the block - filesRecords, err := encodeBlockRecordFiles(newFileRecords) - if err != nil { - return 0, 0, StatusCorruptBlock - } - - newRecordsRaw = append(newRecordsRaw, filesRecords...) - - if len(newRecordsRaw) > 0 { - blockchainNew = append(blockchainNew, Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: newRecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))}) - } - } else { - blockchainNew = append(blockchainNew, Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: block.RecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))}) - } - } - - if refactorBlockchain { - var lastBlockHash []byte - - for _, block := range blockchainNew { - block.LastBlockHash = lastBlockHash - - raw, err := encodeBlock(&block, blockchain.privateKey) - if err != nil { - return 0, 0, StatusCorruptBlock - } - - // store the block - blockchain.database.Set(blockNumberToKey(block.Number), raw) - - lastBlockHash = protocol.HashData(raw) - } - - // update the blockchain header in the database - blockchain.headerWrite(uint64(len(blockchainNew)), refactorVersion) - - // delete orphaned blocks - for n := blockchain.height; n < height; n++ { - blockchain.database.Delete(blockNumberToKey(n)) - } - } - - return blockchain.height, blockchain.version, StatusOK -} - -// ---- blockchain manipulation functions ---- - -// Header returns the users blockchain header which stores the height and version number. -func (blockchain *Blockchain) Header() (publicKey *btcec.PublicKey, height uint64, version uint64) { - blockchain.Lock() - defer blockchain.Unlock() - - return blockchain.publicKey, blockchain.height, blockchain.version -} - -// Append appends a new block to the blockchain based on the provided raw records. Status is StatusX. -func (blockchain *Blockchain) Append(RecordsRaw []BlockRecordRaw) (newHeight, newVersion uint64, status int) { - blockchain.Lock() - defer blockchain.Unlock() - - if len(RecordsRaw) == 0 { - return blockchain.height, blockchain.version, StatusOK - } - - block := &Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: RecordsRaw} - - // set the last block hash first - if blockchain.height > 0 { - previousBlockRaw, found := blockchain.database.Get(blockNumberToKey(blockchain.height - 1)) - if !found || len(previousBlockRaw) == 0 { - return 0, 0, StatusBlockNotFound - } - - block.LastBlockHash = protocol.HashData(previousBlockRaw) - } - - block.Number = blockchain.height - block.BlockchainVersion = blockchain.version - - raw, err := encodeBlock(block, blockchain.privateKey) - if err != nil { - return 0, 0, StatusCorruptBlock - } - - // store the block - blockchain.database.Set(blockNumberToKey(block.Number), raw) - - // update the blockchain header in the database, increase blockchain height - blockchain.headerWrite(blockchain.height+1, blockchain.version) - - return blockchain.height, blockchain.version, StatusOK -} - -// Read reads the block number from the blockchain. Status is StatusX. -func (blockchain *Blockchain) Read(number uint64) (decoded *BlockDecoded, status int, err error) { - if number >= blockchain.height { - return nil, StatusBlockNotFound, errors.New("block number exceeds blockchain height") - } - - blockRaw, found := blockchain.database.Get(blockNumberToKey(number)) - if !found || len(blockRaw) == 0 { - return nil, StatusBlockNotFound, errors.New("block not found") - } - - block, err := decodeBlock(blockRaw) - if err != nil { - return nil, StatusCorruptBlock, err - } - - decoded, err = decodeBlockRecords(block) - if err != nil { - return nil, StatusCorruptBlock, err - } - - return decoded, StatusOK, nil -} - -// DeleteBlockchain deletes the entire blockchain -func (blockchain *Blockchain) DeleteBlockchain() (status int, err error) { - blockchain.Lock() - defer blockchain.Unlock() - - for n := uint64(0); n < blockchain.height; n++ { - blockchain.database.Delete(blockNumberToKey(n)) - } - - // update the blockchain header in the database, reset height, increase version - blockchain.headerWrite(0, blockchain.version+1) - - return StatusOK, nil -} - -// GetBlockRaw returns the encoded block from the blockchain. Status is StatusX. -func (blockchain *Blockchain) GetBlockRaw(number uint64) (data []byte, status int, err error) { - if number >= blockchain.height { - return nil, StatusBlockNotFound, errors.New("block number exceeds blockchain height") - } - - blockRaw, found := blockchain.database.Get(blockNumberToKey(number)) - if !found || len(blockRaw) == 0 { - return nil, StatusBlockNotFound, errors.New("block not found") - } - - return blockRaw, StatusOK, nil -} - -// DecodeBlockRaw decodes the raw block. Status is StatusX. -func DecodeBlockRaw(blockRaw []byte) (decoded *BlockDecoded, status int, err error) { - block, err := decodeBlock(blockRaw) - if err != nil { - return nil, StatusCorruptBlock, err - } - - decoded, err = decodeBlockRecords(block) - if err != nil { - return nil, StatusCorruptBlock, err - } - - return decoded, StatusOK, nil -} +/* +File Name: Blockchain.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +All blocks and the blockchain header are stored in a key-value database. +The key for the blockchain header is keyHeader and for each block is the block number as 64-bit unsigned integer little endian. + +Encoding of the blockchain header: +Offset Size Info +0 8 Height of the blockchain +8 8 Version of the blockchain +16 2 Format of the blockchain. This provides backward compatibility. +18 65 Signature + +*/ + +package blockchain + +import ( + "encoding/binary" + "errors" + "sync" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/store" +) + +// TargetBlockSize is the target size that a generated block shall not exceed. This ensures the block will be transferred via blockchain exchange and cached in DHT. +// Large blocks may be ignored by clients for size and spam reasons, resulting in decreased discoverability. +var TargetBlockSize = uint64(4096) + +// MinAcceptableBlockSize is the minimum block size peers must accept. +const MinAcceptableBlockSize = uint64(1024) + +// Blockchain stores the blockchain's header in memory. Any changes must be synced to disk! +type Blockchain struct { + // header + height uint64 // Height is exchanged as uint32 in the protocol, but stored as uint64. + version uint64 // Version is always uint64. + format uint16 // Format is only locally used. + + // internals + publicKey *btcec.PublicKey // Public Key of the owner. This must match the ones used on disk. + privateKey *btcec.PrivateKey // Private Key of the owner. This must match the ones used on disk. + path string // Path of the blockchain on disk. Depends on key-value store whether a filename or folder. + database store.Store // The database storing the blockchain. + sync.Mutex // synchronized access to the header + + // callback + BlockchainUpdate func(blockchain *Blockchain, oldHeight, oldVersion, newHeight, newVersion uint64) +} + +// Init initializes the given blockchain. It creates the blockchain file if it does not exist already. +func Init(privateKey *btcec.PrivateKey, path string) (blockchain *Blockchain, err error) { + blockchain = &Blockchain{privateKey: privateKey, path: path} + publicKey := privateKey.PubKey() + + // open existing blockchain file or create new one + if blockchain.database, err = store.NewPogrebStore(path); err != nil { + return nil, err + } + + // verify header + var found bool + + found, err = blockchain.headerRead() + if err != nil { + return blockchain, err // likely corrupt blockchain database + } else if !found { + // First run: create header signature! + blockchain.publicKey = publicKey + + if err := blockchain.headerWrite(0, 0); err != nil { + return blockchain, err + } + } else if !blockchain.publicKey.IsEqual(publicKey) { + return blockchain, errors.New("corrupt user blockchain database. Public key mismatch") + } + + return blockchain, nil +} + +// the key names in the key-value database are constant and must not collide with block numbers (i.e. they must be >64 bit) +const keyHeader = "header blockchain" + +// headerRead reads the header from the blockchain and decodes it. +func (blockchain *Blockchain) headerRead() (found bool, err error) { + buffer, found := blockchain.database.Get([]byte(keyHeader)) + if !found { + return false, nil + } + + if len(buffer) != 83 { + return true, errors.New("blockchain header size mismatch") + } + + blockchain.height = binary.LittleEndian.Uint64(buffer[0:8]) + blockchain.version = binary.LittleEndian.Uint64(buffer[8:16]) + blockchain.format = binary.LittleEndian.Uint16(buffer[16:18]) + signature := buffer[18 : 18+65] + + if blockchain.format != 0 { + return true, errors.New("future blockchain format not supported. You must go back to the future!") + } + + blockchain.publicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, protocol.HashData(buffer[0:18])) + + return +} + +// headerWrite writes the header to the blockchain and signs it. +func (blockchain *Blockchain) headerWrite(height, version uint64) (err error) { + oldHeight := blockchain.height + oldVersion := blockchain.version + + blockchain.height = height + blockchain.version = version + + var buffer [83]byte + binary.LittleEndian.PutUint64(buffer[0:8], height) + binary.LittleEndian.PutUint64(buffer[8:16], version) + binary.LittleEndian.PutUint16(buffer[16:18], 0) // Current format is 0 + + signature, err := btcec.SignCompact(btcec.S256(), blockchain.privateKey, protocol.HashData(buffer[0:18]), true) + + if err != nil { + return err + } else if len(signature) != 65 { + return errors.New("signature length invalid") + } + + copy(buffer[18:18+65], signature) + + err = blockchain.database.Set([]byte(keyHeader), buffer[:]) + + // call the callback, if any + if blockchain.BlockchainUpdate != nil { + blockchain.BlockchainUpdate(blockchain, oldHeight, oldVersion, blockchain.height, blockchain.version) + } + + return err +} + +// StatusX provides information about the blockchain status. Some errors codes indicate a corruption. +const ( + StatusOK = 0 // No problems in the blockchain detected. + StatusBlockNotFound = 1 // Missing block in the blockchain. + StatusCorruptBlock = 2 // Error block encoding + StatusCorruptBlockRecord = 3 // Error block record encoding + StatusDataNotFound = 4 // Requested data not available in the blockchain + StatusNotInWarehouse = 5 // File to be added to blockchain does not exist in the Warehouse +) + +// blockNumberToKey returns the database key for the given block number +func blockNumberToKey(number uint64) (key []byte) { + var target [8]byte + binary.LittleEndian.PutUint64(target[:], number) + + return target[:] +} + +// Iterate iterates over the blockchain. Status is StatusX. +// If the callback returns non-zero, the function aborts and returns the inner status code. +func (blockchain *Blockchain) Iterate(callback func(block *Block) int) (status int) { + // read all blocks until height is reached + height := blockchain.height + + for blockN := uint64(0); blockN < height; blockN++ { + blockRaw, found := blockchain.database.Get(blockNumberToKey(blockN)) + if !found || len(blockRaw) == 0 { + return StatusBlockNotFound + } + + block, err := decodeBlock(blockRaw) + if err != nil { + return StatusCorruptBlock + } + + if statusI := callback(block); statusI != StatusOK { + return statusI + } + } + + return StatusOK +} + +// IterateDeleteRecord iterates over the blockchain to find records to delete. Status is StatusX. +// deleteAction is 0 = no action on record, 1 = delete record, 2 = replace record, 3 = error blockchain corrupt +// If the callback returns true, the record will be deleted. The blockchain will be automatically refactored and height and version updated. +func (blockchain *Blockchain) IterateDeleteRecord(callbackFile func(file *BlockRecordFile) (deleteAction int), callbackOther func(record *BlockRecordRaw) (deleteAction int)) (newHeight, newVersion uint64, status int) { + blockchain.Lock() + defer blockchain.Unlock() + + // New blockchain keeps track of the new blocks. If anything changes in the blockchain, it must be recalculated and the version number increased. + var blockchainNew []Block + refactorBlockchain := false + refactorVersion := blockchain.version + 1 + + // Read all blocks until height is reached. At the end the height and version might be different if blocks are deleted. + height := blockchain.height + + for blockN := uint64(0); blockN < height; blockN++ { + blockRaw, found := blockchain.database.Get(blockNumberToKey(blockN)) + if !found || len(blockRaw) == 0 { + return 0, 0, StatusBlockNotFound + } + + block, err := decodeBlock(blockRaw) + if err != nil { + return 0, 0, StatusCorruptBlock + } + + refactorBlock := false + + // Decode all file records at once. This is needed due to potential referenced tags. + // If a file is deleted or referenced tag data changed, it would corrupt the blockchain if the other records were not updated. + filesD, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) + if err != nil { + return 0, 0, StatusCorruptBlock + } + + // loop through all file records in this block + var newFileRecords []BlockRecordFile + + if callbackFile == nil { + newFileRecords = filesD + } else { + for n := range filesD { + switch callbackFile(&filesD[n]) { + case 0: // no action on record + newFileRecords = append(newFileRecords, filesD[n]) + + case 1: // delete record + refactorBlock = true + refactorBlockchain = true + + case 2: // replace record + newFileRecords = append(newFileRecords, filesD[n]) + refactorBlock = true + refactorBlockchain = true + + case 3: // error blockchain corrupt + return 0, 0, StatusCorruptBlockRecord + } + } + } + + // loop through all other (non-file) records in this block + var newRecordsRaw []BlockRecordRaw + + for n := range block.RecordsRaw { + // File and Tag records were already handled in above loop. + if block.RecordsRaw[n].Type == RecordTypeFile || block.RecordsRaw[n].Type == RecordTypeTagData { + continue + } + + if callbackOther == nil { + newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) + } else { + switch callbackOther(&block.RecordsRaw[n]) { + case 0: // no action on record + newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) + + case 1: // delete record + refactorBlock = true + refactorBlockchain = true + + case 2: // replace record + newRecordsRaw = append(newRecordsRaw, block.RecordsRaw[n]) + refactorBlock = true + refactorBlockchain = true + + case 3: // error blockchain corrupt + return 0, 0, StatusCorruptBlockRecord + } + } + } + + // If refactor, re-calculate the block. All later blocks need to be re-encoded due to change of previous block hash. The version number needs to change. + // Note: Deleting records may leave referenced records orphaned, such as RecordTypeTagData for deleted file records. + if refactorBlock { + // re-encode the block + filesRecords, err := encodeBlockRecordFiles(newFileRecords) + if err != nil { + return 0, 0, StatusCorruptBlock + } + + newRecordsRaw = append(newRecordsRaw, filesRecords...) + + if len(newRecordsRaw) > 0 { + blockchainNew = append(blockchainNew, Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: newRecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))}) + } + } else { + blockchainNew = append(blockchainNew, Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: block.RecordsRaw, BlockchainVersion: refactorVersion, Number: uint64(len(blockchainNew))}) + } + } + + if refactorBlockchain { + var lastBlockHash []byte + + for _, block := range blockchainNew { + block.LastBlockHash = lastBlockHash + + raw, err := encodeBlock(&block, blockchain.privateKey) + if err != nil { + return 0, 0, StatusCorruptBlock + } + + // store the block + blockchain.database.Set(blockNumberToKey(block.Number), raw) + + lastBlockHash = protocol.HashData(raw) + } + + // update the blockchain header in the database + blockchain.headerWrite(uint64(len(blockchainNew)), refactorVersion) + + // delete orphaned blocks + for n := blockchain.height; n < height; n++ { + blockchain.database.Delete(blockNumberToKey(n)) + } + } + + return blockchain.height, blockchain.version, StatusOK +} + +// ---- blockchain manipulation functions ---- + +// Header returns the users blockchain header which stores the height and version number. +func (blockchain *Blockchain) Header() (publicKey *btcec.PublicKey, height uint64, version uint64) { + blockchain.Lock() + defer blockchain.Unlock() + + return blockchain.publicKey, blockchain.height, blockchain.version +} + +// Append appends a new block to the blockchain based on the provided raw records. Status is StatusX. +func (blockchain *Blockchain) Append(RecordsRaw []BlockRecordRaw) (newHeight, newVersion uint64, status int) { + blockchain.Lock() + defer blockchain.Unlock() + + if len(RecordsRaw) == 0 { + return blockchain.height, blockchain.version, StatusOK + } + + block := &Block{OwnerPublicKey: blockchain.publicKey, RecordsRaw: RecordsRaw} + + // set the last block hash first + if blockchain.height > 0 { + previousBlockRaw, found := blockchain.database.Get(blockNumberToKey(blockchain.height - 1)) + if !found || len(previousBlockRaw) == 0 { + return 0, 0, StatusBlockNotFound + } + + block.LastBlockHash = protocol.HashData(previousBlockRaw) + } + + block.Number = blockchain.height + block.BlockchainVersion = blockchain.version + + raw, err := encodeBlock(block, blockchain.privateKey) + if err != nil { + return 0, 0, StatusCorruptBlock + } + + // store the block + blockchain.database.Set(blockNumberToKey(block.Number), raw) + + // update the blockchain header in the database, increase blockchain height + blockchain.headerWrite(blockchain.height+1, blockchain.version) + + return blockchain.height, blockchain.version, StatusOK +} + +// Read reads the block number from the blockchain. Status is StatusX. +func (blockchain *Blockchain) Read(number uint64) (decoded *BlockDecoded, status int, err error) { + if number >= blockchain.height { + return nil, StatusBlockNotFound, errors.New("block number exceeds blockchain height") + } + + blockRaw, found := blockchain.database.Get(blockNumberToKey(number)) + if !found || len(blockRaw) == 0 { + return nil, StatusBlockNotFound, errors.New("block not found") + } + + block, err := decodeBlock(blockRaw) + if err != nil { + return nil, StatusCorruptBlock, err + } + + decoded, err = decodeBlockRecords(block) + if err != nil { + return nil, StatusCorruptBlock, err + } + + return decoded, StatusOK, nil +} + +// DeleteBlockchain deletes the entire blockchain +func (blockchain *Blockchain) DeleteBlockchain() (status int, err error) { + blockchain.Lock() + defer blockchain.Unlock() + + for n := uint64(0); n < blockchain.height; n++ { + blockchain.database.Delete(blockNumberToKey(n)) + } + + // update the blockchain header in the database, reset height, increase version + blockchain.headerWrite(0, blockchain.version+1) + + return StatusOK, nil +} + +// GetBlockRaw returns the encoded block from the blockchain. Status is StatusX. +func (blockchain *Blockchain) GetBlockRaw(number uint64) (data []byte, status int, err error) { + if number >= blockchain.height { + return nil, StatusBlockNotFound, errors.New("block number exceeds blockchain height") + } + + blockRaw, found := blockchain.database.Get(blockNumberToKey(number)) + if !found || len(blockRaw) == 0 { + return nil, StatusBlockNotFound, errors.New("block not found") + } + + return blockRaw, StatusOK, nil +} + +// DecodeBlockRaw decodes the raw block. Status is StatusX. +func DecodeBlockRaw(blockRaw []byte) (decoded *BlockDecoded, status int, err error) { + block, err := decodeBlock(blockRaw) + if err != nil { + return nil, StatusCorruptBlock, err + } + + decoded, err = decodeBlockRecords(block) + if err != nil { + return nil, StatusCorruptBlock, err + } + + return decoded, StatusOK, nil +} diff --git a/blockchain/File Tag.go b/blockchain/File Tag.go index 61493c5..8be13c3 100644 --- a/blockchain/File Tag.go +++ b/blockchain/File Tag.go @@ -1,104 +1,104 @@ -/* -File Name: File Tag.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Metadata tags provide meta information about files. -*/ - -package blockchain - -import ( - "encoding/binary" - "errors" - "time" -) - -// List of defined file tags. Virtual tags are generated at runtime and are read-only. They cannot be stored on the blockchain. -const ( - TagName = 0 // Name of file. - TagFolder = 1 // Folder name. - TagDescription = 2 // Arbitrary description of the file. May contain hashtags. - TagDateShared = 3 // When the file was published on the blockchain. Virtual. - TagDateCreated = 4 // Date when the file was originally created. This may differ from the date in the block record, which indicates when the file was shared. - TagSharedByCount = 5 // Count of peers that share the file. Virtual. - TagSharedByGeoIP = 6 // GeoIP data of peers that are sharing the file. CSV encoded with header "latitude,longitude". Virtual. -) - -// Future tags to be defined for audio/video: Artist, Album, Title, Length, Bitrate, Codec -// Windows list: https://docs.microsoft.com/en-us/windows/win32/wmdm/metadata-constants - -// ---- encoding ---- - -// Date returns the tags data as date encoded -func (tag *BlockRecordFileTag) Date() (time.Time, error) { - if tag == nil { - return time.Time{}, errors.New("tag not available") - } else if len(tag.Data) != 8 { - return time.Time{}, errors.New("file tag date invalid size") - } - - timeB := int64(binary.LittleEndian.Uint64(tag.Data[0:8])) - return time.Unix(timeB, 0).UTC(), nil -} - -// Text returns the tags data as text encoded -func (tag *BlockRecordFileTag) Text() string { - return string(tag.Data) -} - -// Number returns the tags data as uint64. It returns 0 if the data cannot be decoded. -func (tag *BlockRecordFileTag) Number() uint64 { - if len(tag.Data) != 8 { - return 0 - } - - return binary.LittleEndian.Uint64(tag.Data[0:8]) -} - -// IsVirtual checks if the tag is virtual. -func (tag *BlockRecordFileTag) IsVirtual() bool { - return IsTagVirtual(tag.Type) -} - -// TagFromDate returns a tag from date -func TagFromDate(Type uint16, Date time.Time) BlockRecordFileTag { - var tempDate [8]byte - binary.LittleEndian.PutUint64(tempDate[0:8], uint64(Date.UTC().Unix())) - - return BlockRecordFileTag{Type: Type, Data: tempDate[:]} -} - -// TagFromText returns a tag from text -func TagFromText(Type uint16, Text string) BlockRecordFileTag { - return BlockRecordFileTag{Type: Type, Data: []byte(Text)} -} - -// TagFromNumber returns a tag from a number -func TagFromNumber(Type uint16, Number uint64) BlockRecordFileTag { - var tempDate [8]byte - binary.LittleEndian.PutUint64(tempDate[0:8], Number) - - return BlockRecordFileTag{Type: Type, Data: tempDate[:]} -} - -// IsTagVirtual checks if the tag is a virtual one. -func IsTagVirtual(Type uint16) bool { - switch Type { - case TagDateShared, TagSharedByCount, TagSharedByGeoIP: - return true - default: - return false - } -} - -// GetTag returns the tag with the type or nil if not available. -func (file *BlockRecordFile) GetTag(Type uint16) (tag *BlockRecordFileTag) { - for n := range file.Tags { - if file.Tags[n].Type == Type { - return &file.Tags[n] - } - } - - return nil -} +/* +File Name: File Tag.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Metadata tags provide meta information about files. +*/ + +package blockchain + +import ( + "encoding/binary" + "errors" + "time" +) + +// List of defined file tags. Virtual tags are generated at runtime and are read-only. They cannot be stored on the blockchain. +const ( + TagName = 0 // Name of file. + TagFolder = 1 // Folder name. + TagDescription = 2 // Arbitrary description of the file. May contain hashtags. + TagDateShared = 3 // When the file was published on the blockchain. Virtual. + TagDateCreated = 4 // Date when the file was originally created. This may differ from the date in the block record, which indicates when the file was shared. + TagSharedByCount = 5 // Count of peers that share the file. Virtual. + TagSharedByGeoIP = 6 // GeoIP data of peers that are sharing the file. CSV encoded with header "latitude,longitude". Virtual. +) + +// Future tags to be defined for audio/video: Artist, Album, Title, Length, Bitrate, Codec +// Windows list: https://docs.microsoft.com/en-us/windows/win32/wmdm/metadata-constants + +// ---- encoding ---- + +// Date returns the tags data as date encoded +func (tag *BlockRecordFileTag) Date() (time.Time, error) { + if tag == nil { + return time.Time{}, errors.New("tag not available") + } else if len(tag.Data) != 8 { + return time.Time{}, errors.New("file tag date invalid size") + } + + timeB := int64(binary.LittleEndian.Uint64(tag.Data[0:8])) + return time.Unix(timeB, 0).UTC(), nil +} + +// Text returns the tags data as text encoded +func (tag *BlockRecordFileTag) Text() string { + return string(tag.Data) +} + +// Number returns the tags data as uint64. It returns 0 if the data cannot be decoded. +func (tag *BlockRecordFileTag) Number() uint64 { + if len(tag.Data) != 8 { + return 0 + } + + return binary.LittleEndian.Uint64(tag.Data[0:8]) +} + +// IsVirtual checks if the tag is virtual. +func (tag *BlockRecordFileTag) IsVirtual() bool { + return IsTagVirtual(tag.Type) +} + +// TagFromDate returns a tag from date +func TagFromDate(Type uint16, Date time.Time) BlockRecordFileTag { + var tempDate [8]byte + binary.LittleEndian.PutUint64(tempDate[0:8], uint64(Date.UTC().Unix())) + + return BlockRecordFileTag{Type: Type, Data: tempDate[:]} +} + +// TagFromText returns a tag from text +func TagFromText(Type uint16, Text string) BlockRecordFileTag { + return BlockRecordFileTag{Type: Type, Data: []byte(Text)} +} + +// TagFromNumber returns a tag from a number +func TagFromNumber(Type uint16, Number uint64) BlockRecordFileTag { + var tempDate [8]byte + binary.LittleEndian.PutUint64(tempDate[0:8], Number) + + return BlockRecordFileTag{Type: Type, Data: tempDate[:]} +} + +// IsTagVirtual checks if the tag is a virtual one. +func IsTagVirtual(Type uint16) bool { + switch Type { + case TagDateShared, TagSharedByCount, TagSharedByGeoIP: + return true + default: + return false + } +} + +// GetTag returns the tag with the type or nil if not available. +func (file *BlockRecordFile) GetTag(Type uint16) (tag *BlockRecordFileTag) { + for n := range file.Tags { + if file.Tags[n].Type == Type { + return &file.Tags[n] + } + } + + return nil +} diff --git a/blockchain/File.go b/blockchain/File.go index f7eb30e..7e82908 100644 --- a/blockchain/File.go +++ b/blockchain/File.go @@ -1,117 +1,117 @@ -/* -File Name: File.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Note that files include virtual folders as well for any operation. -*/ - -package blockchain - -import ( - "bytes" - - "github.com/google/uuid" -) - -// AddFiles adds files to the blockchain. Status is StatusX. -// It makes sense to group all files in the same directory into one call, since only one directory record will be created per unique directory per block. -func (blockchain *Blockchain) AddFiles(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { - encodeFilesAppend := func(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { - encoded, err := encodeBlockRecordFiles(files) - if err != nil { - return 0, 0, StatusCorruptBlockRecord - } - - return blockchain.Append(encoded) - } - - blockSize := uint64(blockHeaderSize) - var recordFiles []BlockRecordFile - - for _, file := range files { - recordSize := file.SizeInBlock() - - // need to create a new block due to target block size? - if len(recordFiles) > 0 && blockSize+recordSize > TargetBlockSize { - if newHeight, newVersion, status = encodeFilesAppend(recordFiles); status != StatusOK { - return newHeight, newVersion, status - } - - blockSize = blockHeaderSize - recordFiles = nil - } - - blockSize += recordSize - recordFiles = append(recordFiles, file) - } - - return encodeFilesAppend(recordFiles) -} - -// ListFiles returns a list of all files. Status is StatusX. -// If there is a corruption in the blockchain it will stop reading but return the files parsed so far. -func (blockchain *Blockchain) ListFiles() (files []BlockRecordFile, status int) { - status = blockchain.Iterate(func(block *Block) (statusI int) { - filesMore, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) - if err != nil { - return StatusCorruptBlockRecord - } - files = append(files, filesMore...) - - return StatusOK - }) - - return files, status -} - -// FileExists checks if the file (identified via its hash) exists. -// If there is a corruption in the blockchain it will stop reading but return the files found so far. -func (blockchain *Blockchain) FileExists(hash []byte) (files []BlockRecordFile, status int) { - status = blockchain.Iterate(func(block *Block) (statusI int) { - filesD, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) - if err != nil { - return StatusCorruptBlockRecord - } - for _, file := range filesD { - if bytes.Equal(file.Hash, hash) { - files = append(files, file) - } - } - - return StatusOK - }) - - return files, status -} - -// DeleteFiles deletes files from the blockchain. Status is StatusX. -func (blockchain *Blockchain) DeleteFiles(IDs []uuid.UUID) (newHeight, newVersion uint64, deletedFiles []*BlockRecordFile, status int) { - newHeight, newVersion, status = blockchain.IterateDeleteRecord(func(file *BlockRecordFile) (deleteAction int) { - for _, id := range IDs { - if file.ID == id { // found a file ID to delete? - deletedFiles = append(deletedFiles, file) - return 1 // delete record - } - } - - return 0 // no action on record - }, nil) - - return -} - -// ReplaceFiles is a convenience wrapper to replace files in the blockchain identified via their IDs. Status is StatusX. -// If a file does not exist on the blockchain, it acts as add. -func (blockchain *Blockchain) ReplaceFiles(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { - var deleteIDs []uuid.UUID - for n := range files { - deleteIDs = append(deleteIDs, files[n].ID) - } - - if newHeight, newVersion, _, status = blockchain.DeleteFiles(deleteIDs); status != StatusOK { - return - } - - return blockchain.AddFiles(files) -} +/* +File Name: File.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Note that files include virtual folders as well for any operation. +*/ + +package blockchain + +import ( + "bytes" + + "github.com/google/uuid" +) + +// AddFiles adds files to the blockchain. Status is StatusX. +// It makes sense to group all files in the same directory into one call, since only one directory record will be created per unique directory per block. +func (blockchain *Blockchain) AddFiles(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { + encodeFilesAppend := func(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { + encoded, err := encodeBlockRecordFiles(files) + if err != nil { + return 0, 0, StatusCorruptBlockRecord + } + + return blockchain.Append(encoded) + } + + blockSize := uint64(blockHeaderSize) + var recordFiles []BlockRecordFile + + for _, file := range files { + recordSize := file.SizeInBlock() + + // need to create a new block due to target block size? + if len(recordFiles) > 0 && blockSize+recordSize > TargetBlockSize { + if newHeight, newVersion, status = encodeFilesAppend(recordFiles); status != StatusOK { + return newHeight, newVersion, status + } + + blockSize = blockHeaderSize + recordFiles = nil + } + + blockSize += recordSize + recordFiles = append(recordFiles, file) + } + + return encodeFilesAppend(recordFiles) +} + +// ListFiles returns a list of all files. Status is StatusX. +// If there is a corruption in the blockchain it will stop reading but return the files parsed so far. +func (blockchain *Blockchain) ListFiles() (files []BlockRecordFile, status int) { + status = blockchain.Iterate(func(block *Block) (statusI int) { + filesMore, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) + if err != nil { + return StatusCorruptBlockRecord + } + files = append(files, filesMore...) + + return StatusOK + }) + + return files, status +} + +// FileExists checks if the file (identified via its hash) exists. +// If there is a corruption in the blockchain it will stop reading but return the files found so far. +func (blockchain *Blockchain) FileExists(hash []byte) (files []BlockRecordFile, status int) { + status = blockchain.Iterate(func(block *Block) (statusI int) { + filesD, err := decodeBlockRecordFiles(block.RecordsRaw, block.NodeID) + if err != nil { + return StatusCorruptBlockRecord + } + for _, file := range filesD { + if bytes.Equal(file.Hash, hash) { + files = append(files, file) + } + } + + return StatusOK + }) + + return files, status +} + +// DeleteFiles deletes files from the blockchain. Status is StatusX. +func (blockchain *Blockchain) DeleteFiles(IDs []uuid.UUID) (newHeight, newVersion uint64, deletedFiles []*BlockRecordFile, status int) { + newHeight, newVersion, status = blockchain.IterateDeleteRecord(func(file *BlockRecordFile) (deleteAction int) { + for _, id := range IDs { + if file.ID == id { // found a file ID to delete? + deletedFiles = append(deletedFiles, file) + return 1 // delete record + } + } + + return 0 // no action on record + }, nil) + + return +} + +// ReplaceFiles is a convenience wrapper to replace files in the blockchain identified via their IDs. Status is StatusX. +// If a file does not exist on the blockchain, it acts as add. +func (blockchain *Blockchain) ReplaceFiles(files []BlockRecordFile) (newHeight, newVersion uint64, status int) { + var deleteIDs []uuid.UUID + for n := range files { + deleteIDs = append(deleteIDs, files[n].ID) + } + + if newHeight, newVersion, _, status = blockchain.DeleteFiles(deleteIDs); status != StatusOK { + return + } + + return blockchain.AddFiles(files) +} diff --git a/blockchain/Multi.go b/blockchain/Multi.go index 0954cb7..032b3e6 100644 --- a/blockchain/Multi.go +++ b/blockchain/Multi.go @@ -1,292 +1,292 @@ -/* -File Name: Multi.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Multi-blockchain store implementation. - -Keys used in the key-value store: -1. Key: Public key compressed, Value: Header -2. Key: Public key compressed + version + block number, Value: Block - -*/ - -package blockchain - -import ( - "encoding/binary" - "errors" - "time" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/store" -) - -const ( - MultiStatusOK = 0 // Success. - MultiStatusErrorReadHeader = 1 // Error reading header for blockchain. - MultiStatusHeaderNA = 2 // Header not available. This indicates nothing is stored for the blockchain. - MultiStatusInvalidRemote = 3 // Invalid remote reported blockchain version and height. (local cache is "newer" which should not happen) - MultiStatusEqual = 4 // Remote blockchain and local cache are equal. - MultiStatusNewVersion = 5 // Local cache is out of date. A new version of the blockchain is available. - MultiStatusNewBlocks = 6 // Local cache is out of date. Additional blocks are available. -) - -// MultiStore stores multiple blockchains. -type MultiStore struct { - path string // Path of the blockchain on disk. Depends on key-value store whether a filename or folder. - Database store.Store // The database storing the blockchain. - - // callbacks - FilterStatisticUpdate func(multi *MultiStore, header *MultiBlockchainHeader, statsOld BlockchainStats) - FilterBlockchainDelete func(multi *MultiStore, header *MultiBlockchainHeader) -} - -func InitMultiStore(path string) (multi *MultiStore, err error) { - multi = &MultiStore{path: path} - - // open existing blockchain file or create new one - if multi.Database, err = store.NewPogrebStore(path); err != nil { - return nil, err - } - - return multi, nil -} - -/* -Header for blockchains: - -Offset Size Info -0 8 Version of the blockchain -8 8 Height of the blockchain -16 8 Count of blocks -24 8 Date first block added -32 8 Date last block added -40 8 Stats: Count of file records (from available blocks) -48 8 Stats: Size of all files combined (from available blocks) -56 8 * n List of block numbers that are stored - -Note: The statistics fields only count available stored blocks. - -*/ - -const multiBlockchainHeaderSize = 56 - -// This is a header for a single blockchain stored in a multi store. -type MultiBlockchainHeader struct { - PublicKey *btcec.PublicKey // Public Key of the blockchain - Height uint64 // Height is exchanged as uint32 in the protocol, but stored as uint64. - Version uint64 // Version is always uint64. - DateFirstBlockAdded time.Time // Date the first block was added - DateLastBlockAdded time.Time // Date the last block was added - ListBlocks []uint64 // List of block numbers that are stored - Stats BlockchainStats // Statistics about the blockchain (only about stored blocks) -} - -// Keep statistics about blocks stored for a blockchain -type BlockchainStats struct { - CountFileRecords uint64 // Count of file records in stored blocks - SizeAllFiles uint64 // Size of all files combined in stored blocks -} - -func decodeBlockchainHeader(publicKey *btcec.PublicKey, buffer []byte) (header *MultiBlockchainHeader, err error) { - if len(buffer) < multiBlockchainHeaderSize { - return nil, errors.New("header length too small") - } - - header = &MultiBlockchainHeader{PublicKey: publicKey} - header.Version = binary.LittleEndian.Uint64(buffer[0:8]) - header.Height = binary.LittleEndian.Uint64(buffer[8:16]) - countBlocks := binary.LittleEndian.Uint64(buffer[16:24]) - header.DateFirstBlockAdded = time.Unix(int64(binary.LittleEndian.Uint64(buffer[24:32])), 0) - header.DateLastBlockAdded = time.Unix(int64(binary.LittleEndian.Uint64(buffer[32:40])), 0) - header.Stats.CountFileRecords = binary.LittleEndian.Uint64(buffer[40:48]) - header.Stats.SizeAllFiles = binary.LittleEndian.Uint64(buffer[48:56]) - - if uint64(len(buffer)) < multiBlockchainHeaderSize+8*countBlocks { - return nil, errors.New("header length too small") - } - - index := multiBlockchainHeaderSize - - for n := uint64(0); n < countBlocks; n++ { - blockN := binary.LittleEndian.Uint64(buffer[index : index+8]) - header.ListBlocks = append(header.ListBlocks, blockN) - index += 8 - } - - return header, nil -} - -// Reads a blockchains header if available. -func (multi *MultiStore) ReadBlockchainHeader(publicKey *btcec.PublicKey) (header *MultiBlockchainHeader, found bool, err error) { - buffer, found := multi.Database.Get(publicKey.SerializeCompressed()) - if !found { - return nil, false, nil - } - - header, err = decodeBlockchainHeader(publicKey, buffer) - return header, err == nil, err -} - -// WriteBlockchainHeader writes a blockchain header. If one exists, it will be overwritten. -func (multi *MultiStore) WriteBlockchainHeader(header *MultiBlockchainHeader) (err error) { - raw := make([]byte, multiBlockchainHeaderSize+8*len(header.ListBlocks)) - - binary.LittleEndian.PutUint64(raw[0:8], header.Version) - binary.LittleEndian.PutUint64(raw[8:16], header.Height) - binary.LittleEndian.PutUint64(raw[16:24], uint64(len(header.ListBlocks))) - binary.LittleEndian.PutUint64(raw[24:32], uint64(header.DateFirstBlockAdded.UTC().Unix())) - binary.LittleEndian.PutUint64(raw[32:40], uint64(header.DateLastBlockAdded.UTC().Unix())) - binary.LittleEndian.PutUint64(raw[40:48], header.Stats.CountFileRecords) - binary.LittleEndian.PutUint64(raw[48:56], header.Stats.SizeAllFiles) - - index := multiBlockchainHeaderSize - - for _, blockN := range header.ListBlocks { - binary.LittleEndian.PutUint64(raw[index:index+8], blockN) - index += 8 - } - - return multi.Database.Set(header.PublicKey.SerializeCompressed(), raw) -} - -func lookupKeyForBlock(publicKey *btcec.PublicKey, version, blockNumber uint64) (key []byte) { - var buffer [16]byte - binary.LittleEndian.PutUint64(buffer[0:8], version) - binary.LittleEndian.PutUint64(buffer[8:16], blockNumber) - - key = append(key, publicKey.SerializeCompressed()...) - key = append(key, buffer[:]...) - - return key -} - -// ReadBlock reads a raw block -func (multi *MultiStore) ReadBlock(publicKey *btcec.PublicKey, version, blockNumber uint64) (raw []byte, found bool) { - return multi.Database.Get(lookupKeyForBlock(publicKey, version, blockNumber)) -} - -// WriteBlock writes a raw block. It does not update the blockchain header. -func (multi *MultiStore) WriteBlock(publicKey *btcec.PublicKey, version, blockNumber uint64, raw []byte) (err error) { - return multi.Database.Set(lookupKeyForBlock(publicKey, version, blockNumber), raw) -} - -// AssessBlockchainHeader reads the blockchain header, if available, and assesses the status. -func (multi *MultiStore) AssessBlockchainHeader(publicKey *btcec.PublicKey, version, height uint64) (header *MultiBlockchainHeader, status int, err error) { - // check if there is an existing header for the blockchain - header, found, err := multi.ReadBlockchainHeader(publicKey) - if err != nil { - return nil, MultiStatusErrorReadHeader, err - } else if !found { - return nil, MultiStatusHeaderNA, nil - } - - if header.Version == version && header.Height == height { - return header, MultiStatusEqual, nil - } - - // Check if existing version is newer than reported - indicates illegal behavior by the remote peer. - // Improper refactoring could happen if the local blockchain folder is deleted and automatically recreated. - if header.Version > version || (header.Version == version && header.Height > height) { - return header, MultiStatusInvalidRemote, nil - } - - if version > header.Version { - return header, MultiStatusNewVersion, nil - } - - return header, MultiStatusNewBlocks, nil -} - -// Deletes an entire blockchain from the store. It will delete each block individually and then the header. -func (multi *MultiStore) DeleteBlockchain(header *MultiBlockchainHeader) { - // first delete all blocks - for _, blockN := range header.ListBlocks { - multi.Database.Delete(lookupKeyForBlock(header.PublicKey, header.Version, blockN)) - } - - // delete the header - multi.Database.Delete(header.PublicKey.SerializeCompressed()) - - if multi.FilterBlockchainDelete != nil { - multi.FilterBlockchainDelete(multi, header) - } -} - -func (multi *MultiStore) NewBlockchainHeader(publicKey *btcec.PublicKey, version, height uint64) (header *MultiBlockchainHeader, err error) { - timeN := time.Now().UTC() - header = &MultiBlockchainHeader{ - PublicKey: publicKey, - Height: height, - Version: version, - DateFirstBlockAdded: timeN, - DateLastBlockAdded: timeN, - } - - return header, multi.WriteBlockchainHeader(header) -} - -// Updates the statistics fields of the blockchain header based on the new decoded block records. -// It does not write the blockchain header; multi.WriteBlockchainHeader must called to store the changes. -// The caller must make sure not to call this function on records already processed. -func (multi *MultiStore) UpdateBlockchainStatistics(header *MultiBlockchainHeader, recordsDecoded []interface{}) { - updatedStats := false - statsOld := header.Stats - - for _, decodedR := range recordsDecoded { - if file, ok := decodedR.(BlockRecordFile); ok { - header.Stats.SizeAllFiles += file.Size - header.Stats.CountFileRecords++ - - updatedStats = true - } - } - - if updatedStats && multi.FilterStatisticUpdate != nil { - multi.FilterStatisticUpdate(multi, header, statsOld) - } -} - -// Iterates over all blockchains stored in the cache -func (multi *MultiStore) IterateBlockchains(callback func(header *MultiBlockchainHeader)) { - multi.Database.Iterate(func(key, value []byte) { - if len(key) == 33 { - // Length 33 indicates key = Public key compressed, value = Header - if blockchainPublicKey, err := btcec.ParsePubKey(key, btcec.S256()); err == nil { - if header, err := decodeBlockchainHeader(blockchainPublicKey, value); err == nil { - callback(header) - } - } - } - }) -} - -// IngestBlock ingests a new block into the store. It fails if a block is already stored for the given blockchain and block number. -// It will update the blockchain header including the statistics. -func (multi *MultiStore) IngestBlock(header *MultiBlockchainHeader, blockNumber uint64, raw []byte, failIfInvalid bool) (decoded *BlockDecoded, err error) { - // check if already exists - if _, found := multi.ReadBlock(header.PublicKey, header.Version, blockNumber); found { - return nil, errors.New("already exists") - } - - // decode it - decoded, status, err := DecodeBlockRaw(raw) - if failIfInvalid && err != nil { - return nil, err - } - - // store the transferred block in the cache - multi.WriteBlock(header.PublicKey, header.Version, blockNumber, raw) - header.ListBlocks = append(header.ListBlocks, blockNumber) - - // update blockchain header stats if records were decoded - if status == StatusOK { - multi.UpdateBlockchainStatistics(header, decoded.RecordsDecoded) - } - - // update the blockchain header - multi.WriteBlockchainHeader(header) - - return decoded, nil -} +/* +File Name: Multi.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Multi-blockchain store implementation. + +Keys used in the key-value store: +1. Key: Public key compressed, Value: Header +2. Key: Public key compressed + version + block number, Value: Block + +*/ + +package blockchain + +import ( + "encoding/binary" + "errors" + "time" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/store" +) + +const ( + MultiStatusOK = 0 // Success. + MultiStatusErrorReadHeader = 1 // Error reading header for blockchain. + MultiStatusHeaderNA = 2 // Header not available. This indicates nothing is stored for the blockchain. + MultiStatusInvalidRemote = 3 // Invalid remote reported blockchain version and height. (local cache is "newer" which should not happen) + MultiStatusEqual = 4 // Remote blockchain and local cache are equal. + MultiStatusNewVersion = 5 // Local cache is out of date. A new version of the blockchain is available. + MultiStatusNewBlocks = 6 // Local cache is out of date. Additional blocks are available. +) + +// MultiStore stores multiple blockchains. +type MultiStore struct { + path string // Path of the blockchain on disk. Depends on key-value store whether a filename or folder. + Database store.Store // The database storing the blockchain. + + // callbacks + FilterStatisticUpdate func(multi *MultiStore, header *MultiBlockchainHeader, statsOld BlockchainStats) + FilterBlockchainDelete func(multi *MultiStore, header *MultiBlockchainHeader) +} + +func InitMultiStore(path string) (multi *MultiStore, err error) { + multi = &MultiStore{path: path} + + // open existing blockchain file or create new one + if multi.Database, err = store.NewPogrebStore(path); err != nil { + return nil, err + } + + return multi, nil +} + +/* +Header for blockchains: + +Offset Size Info +0 8 Version of the blockchain +8 8 Height of the blockchain +16 8 Count of blocks +24 8 Date first block added +32 8 Date last block added +40 8 Stats: Count of file records (from available blocks) +48 8 Stats: Size of all files combined (from available blocks) +56 8 * n List of block numbers that are stored + +Note: The statistics fields only count available stored blocks. + +*/ + +const multiBlockchainHeaderSize = 56 + +// This is a header for a single blockchain stored in a multi store. +type MultiBlockchainHeader struct { + PublicKey *btcec.PublicKey // Public Key of the blockchain + Height uint64 // Height is exchanged as uint32 in the protocol, but stored as uint64. + Version uint64 // Version is always uint64. + DateFirstBlockAdded time.Time // Date the first block was added + DateLastBlockAdded time.Time // Date the last block was added + ListBlocks []uint64 // List of block numbers that are stored + Stats BlockchainStats // Statistics about the blockchain (only about stored blocks) +} + +// Keep statistics about blocks stored for a blockchain +type BlockchainStats struct { + CountFileRecords uint64 // Count of file records in stored blocks + SizeAllFiles uint64 // Size of all files combined in stored blocks +} + +func decodeBlockchainHeader(publicKey *btcec.PublicKey, buffer []byte) (header *MultiBlockchainHeader, err error) { + if len(buffer) < multiBlockchainHeaderSize { + return nil, errors.New("header length too small") + } + + header = &MultiBlockchainHeader{PublicKey: publicKey} + header.Version = binary.LittleEndian.Uint64(buffer[0:8]) + header.Height = binary.LittleEndian.Uint64(buffer[8:16]) + countBlocks := binary.LittleEndian.Uint64(buffer[16:24]) + header.DateFirstBlockAdded = time.Unix(int64(binary.LittleEndian.Uint64(buffer[24:32])), 0) + header.DateLastBlockAdded = time.Unix(int64(binary.LittleEndian.Uint64(buffer[32:40])), 0) + header.Stats.CountFileRecords = binary.LittleEndian.Uint64(buffer[40:48]) + header.Stats.SizeAllFiles = binary.LittleEndian.Uint64(buffer[48:56]) + + if uint64(len(buffer)) < multiBlockchainHeaderSize+8*countBlocks { + return nil, errors.New("header length too small") + } + + index := multiBlockchainHeaderSize + + for n := uint64(0); n < countBlocks; n++ { + blockN := binary.LittleEndian.Uint64(buffer[index : index+8]) + header.ListBlocks = append(header.ListBlocks, blockN) + index += 8 + } + + return header, nil +} + +// Reads a blockchains header if available. +func (multi *MultiStore) ReadBlockchainHeader(publicKey *btcec.PublicKey) (header *MultiBlockchainHeader, found bool, err error) { + buffer, found := multi.Database.Get(publicKey.SerializeCompressed()) + if !found { + return nil, false, nil + } + + header, err = decodeBlockchainHeader(publicKey, buffer) + return header, err == nil, err +} + +// WriteBlockchainHeader writes a blockchain header. If one exists, it will be overwritten. +func (multi *MultiStore) WriteBlockchainHeader(header *MultiBlockchainHeader) (err error) { + raw := make([]byte, multiBlockchainHeaderSize+8*len(header.ListBlocks)) + + binary.LittleEndian.PutUint64(raw[0:8], header.Version) + binary.LittleEndian.PutUint64(raw[8:16], header.Height) + binary.LittleEndian.PutUint64(raw[16:24], uint64(len(header.ListBlocks))) + binary.LittleEndian.PutUint64(raw[24:32], uint64(header.DateFirstBlockAdded.UTC().Unix())) + binary.LittleEndian.PutUint64(raw[32:40], uint64(header.DateLastBlockAdded.UTC().Unix())) + binary.LittleEndian.PutUint64(raw[40:48], header.Stats.CountFileRecords) + binary.LittleEndian.PutUint64(raw[48:56], header.Stats.SizeAllFiles) + + index := multiBlockchainHeaderSize + + for _, blockN := range header.ListBlocks { + binary.LittleEndian.PutUint64(raw[index:index+8], blockN) + index += 8 + } + + return multi.Database.Set(header.PublicKey.SerializeCompressed(), raw) +} + +func lookupKeyForBlock(publicKey *btcec.PublicKey, version, blockNumber uint64) (key []byte) { + var buffer [16]byte + binary.LittleEndian.PutUint64(buffer[0:8], version) + binary.LittleEndian.PutUint64(buffer[8:16], blockNumber) + + key = append(key, publicKey.SerializeCompressed()...) + key = append(key, buffer[:]...) + + return key +} + +// ReadBlock reads a raw block +func (multi *MultiStore) ReadBlock(publicKey *btcec.PublicKey, version, blockNumber uint64) (raw []byte, found bool) { + return multi.Database.Get(lookupKeyForBlock(publicKey, version, blockNumber)) +} + +// WriteBlock writes a raw block. It does not update the blockchain header. +func (multi *MultiStore) WriteBlock(publicKey *btcec.PublicKey, version, blockNumber uint64, raw []byte) (err error) { + return multi.Database.Set(lookupKeyForBlock(publicKey, version, blockNumber), raw) +} + +// AssessBlockchainHeader reads the blockchain header, if available, and assesses the status. +func (multi *MultiStore) AssessBlockchainHeader(publicKey *btcec.PublicKey, version, height uint64) (header *MultiBlockchainHeader, status int, err error) { + // check if there is an existing header for the blockchain + header, found, err := multi.ReadBlockchainHeader(publicKey) + if err != nil { + return nil, MultiStatusErrorReadHeader, err + } else if !found { + return nil, MultiStatusHeaderNA, nil + } + + if header.Version == version && header.Height == height { + return header, MultiStatusEqual, nil + } + + // Check if existing version is newer than reported - indicates illegal behavior by the remote peer. + // Improper refactoring could happen if the local blockchain folder is deleted and automatically recreated. + if header.Version > version || (header.Version == version && header.Height > height) { + return header, MultiStatusInvalidRemote, nil + } + + if version > header.Version { + return header, MultiStatusNewVersion, nil + } + + return header, MultiStatusNewBlocks, nil +} + +// Deletes an entire blockchain from the store. It will delete each block individually and then the header. +func (multi *MultiStore) DeleteBlockchain(header *MultiBlockchainHeader) { + // first delete all blocks + for _, blockN := range header.ListBlocks { + multi.Database.Delete(lookupKeyForBlock(header.PublicKey, header.Version, blockN)) + } + + // delete the header + multi.Database.Delete(header.PublicKey.SerializeCompressed()) + + if multi.FilterBlockchainDelete != nil { + multi.FilterBlockchainDelete(multi, header) + } +} + +func (multi *MultiStore) NewBlockchainHeader(publicKey *btcec.PublicKey, version, height uint64) (header *MultiBlockchainHeader, err error) { + timeN := time.Now().UTC() + header = &MultiBlockchainHeader{ + PublicKey: publicKey, + Height: height, + Version: version, + DateFirstBlockAdded: timeN, + DateLastBlockAdded: timeN, + } + + return header, multi.WriteBlockchainHeader(header) +} + +// Updates the statistics fields of the blockchain header based on the new decoded block records. +// It does not write the blockchain header; multi.WriteBlockchainHeader must called to store the changes. +// The caller must make sure not to call this function on records already processed. +func (multi *MultiStore) UpdateBlockchainStatistics(header *MultiBlockchainHeader, recordsDecoded []interface{}) { + updatedStats := false + statsOld := header.Stats + + for _, decodedR := range recordsDecoded { + if file, ok := decodedR.(BlockRecordFile); ok { + header.Stats.SizeAllFiles += file.Size + header.Stats.CountFileRecords++ + + updatedStats = true + } + } + + if updatedStats && multi.FilterStatisticUpdate != nil { + multi.FilterStatisticUpdate(multi, header, statsOld) + } +} + +// Iterates over all blockchains stored in the cache +func (multi *MultiStore) IterateBlockchains(callback func(header *MultiBlockchainHeader)) { + multi.Database.Iterate(func(key, value []byte) { + if len(key) == 33 { + // Length 33 indicates key = Public key compressed, value = Header + if blockchainPublicKey, err := btcec.ParsePubKey(key, btcec.S256()); err == nil { + if header, err := decodeBlockchainHeader(blockchainPublicKey, value); err == nil { + callback(header) + } + } + } + }) +} + +// IngestBlock ingests a new block into the store. It fails if a block is already stored for the given blockchain and block number. +// It will update the blockchain header including the statistics. +func (multi *MultiStore) IngestBlock(header *MultiBlockchainHeader, blockNumber uint64, raw []byte, failIfInvalid bool) (decoded *BlockDecoded, err error) { + // check if already exists + if _, found := multi.ReadBlock(header.PublicKey, header.Version, blockNumber); found { + return nil, errors.New("already exists") + } + + // decode it + decoded, status, err := DecodeBlockRaw(raw) + if failIfInvalid && err != nil { + return nil, err + } + + // store the transferred block in the cache + multi.WriteBlock(header.PublicKey, header.Version, blockNumber, raw) + header.ListBlocks = append(header.ListBlocks, blockNumber) + + // update blockchain header stats if records were decoded + if status == StatusOK { + multi.UpdateBlockchainStatistics(header, decoded.RecordsDecoded) + } + + // update the blockchain header + multi.WriteBlockchainHeader(header) + + return decoded, nil +} diff --git a/blockchain/Profile Data.go b/blockchain/Profile Data.go index d4d85ba..0999824 100644 --- a/blockchain/Profile Data.go +++ b/blockchain/Profile Data.go @@ -1,32 +1,32 @@ -/* -File Name: Profile Data.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package blockchain - -// List of recognized profile fields. -const ( - ProfileName = 0 // Arbitrary username - ProfileEmail = 1 // Email address - ProfileWebsite = 2 // Website address - ProfileTwitter = 3 // Twitter account without the @ - ProfileYouTube = 4 // YouTube channel URL - ProfileAddress = 5 // Physical address - ProfilePicture = 6 // Profile picture, blob -) - -// The encoding of profile fields depends on the field. Text data is always UTF-8 text encoded. -// Note that all profile data is arbitrary and shall be considered untrusted and unverified. -// To establish trust, the user must load Certificates into the blockchain that validate certain data. - -// Text returns the profile field as text encoded -func (info *BlockRecordProfile) Text() string { - return string(info.Data) -} - -// ProfileFieldFromText returns a profile field from text -func ProfileFieldFromText(Type uint16, Text string) BlockRecordProfile { - return BlockRecordProfile{Type: Type, Data: []byte(Text)} -} +/* +File Name: Profile Data.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package blockchain + +// List of recognized profile fields. +const ( + ProfileName = 0 // Arbitrary username + ProfileEmail = 1 // Email address + ProfileWebsite = 2 // Website address + ProfileTwitter = 3 // Twitter account without the @ + ProfileYouTube = 4 // YouTube channel URL + ProfileAddress = 5 // Physical address + ProfilePicture = 6 // Profile picture, blob +) + +// The encoding of profile fields depends on the field. Text data is always UTF-8 text encoded. +// Note that all profile data is arbitrary and shall be considered untrusted and unverified. +// To establish trust, the user must load Certificates into the blockchain that validate certain data. + +// Text returns the profile field as text encoded +func (info *BlockRecordProfile) Text() string { + return string(info.Data) +} + +// ProfileFieldFromText returns a profile field from text +func ProfileFieldFromText(Type uint16, Text string) BlockRecordProfile { + return BlockRecordProfile{Type: Type, Data: []byte(Text)} +} diff --git a/blockchain/Profile.go b/blockchain/Profile.go index 1ce8c19..2929434 100644 --- a/blockchain/Profile.go +++ b/blockchain/Profile.go @@ -1,119 +1,119 @@ -/* -File Name: Profile.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package blockchain - -// ProfileReadField reads the specified profile field. See ProfileX for the list of recognized fields. The encoding depends on the field type. Status is StatusX. -func (blockchain *Blockchain) ProfileReadField(index uint16) (data []byte, status int) { - found := false - - status = blockchain.Iterate(func(block *Block) (statusI int) { - fields, err := decodeBlockRecordProfile(block.RecordsRaw) - if err != nil { - return StatusCorruptBlockRecord - } else if len(fields) == 0 { - return StatusOK - } - - // Check if the field is available in the profile record. If there are multiple records, only return the latest one. - for n := range fields { - if fields[n].Type == index { - data = fields[n].Data - found = true - } - } - - return StatusOK - }) - - if status != StatusOK { - return nil, status - } else if !found { - return nil, StatusDataNotFound - } - - return data, StatusOK -} - -// ProfileList lists all profile fields. Status is StatusX. -func (blockchain *Blockchain) ProfileList() (fields []BlockRecordProfile, status int) { - uniqueFields := make(map[uint16][]byte) - - status = blockchain.Iterate(func(block *Block) (statusI int) { - fields, err := decodeBlockRecordProfile(block.RecordsRaw) - if err != nil { - return StatusCorruptBlockRecord - } - - for n := range fields { - uniqueFields[fields[n].Type] = fields[n].Data - } - - return StatusOK - }) - - for key, value := range uniqueFields { - fields = append(fields, BlockRecordProfile{Type: key, Data: value}) - } - - return fields, status -} - -// ProfileWrite writes profile fields and blobs to the blockchain. Status is StatusX. -func (blockchain *Blockchain) ProfileWrite(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) { - encodeProfileAppend := func(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) { - encoded, err := encodeBlockRecordProfile(fields) - if err != nil { - return 0, 0, StatusCorruptBlockRecord - } - - return blockchain.Append(encoded) - } - - blockSize := uint64(blockHeaderSize) - var recordFields []BlockRecordProfile - - for _, field := range fields { - recordSize := field.SizeInBlock() - - // need to create a new block due to target block size? - if len(recordFields) > 0 && blockSize+recordSize > TargetBlockSize { - if newHeight, newVersion, status = encodeProfileAppend(recordFields); status != StatusOK { - return newHeight, newVersion, status - } - - blockSize = blockHeaderSize - recordFields = nil - } - - blockSize += recordSize - recordFields = append(recordFields, field) - } - - return encodeProfileAppend(recordFields) -} - -// ProfileDelete deletes fields and blobs from the blockchain. Status is StatusX. -func (blockchain *Blockchain) ProfileDelete(fields []uint16) (newHeight, newVersion uint64, status int) { - return blockchain.IterateDeleteRecord(nil, func(record *BlockRecordRaw) (deleteAction int) { - if record.Type != RecordTypeProfile { - return 0 // no action - } - - existingFields, err := decodeBlockRecordProfile([]BlockRecordRaw{*record}) - if err != nil || len(existingFields) != 1 { - return 3 // error blockchain corrupt - } - - for _, i := range fields { - if i == existingFields[0].Type { // found a file ID to delete? - return 1 // delete record - } - } - - return 0 // no action on record - }) -} +/* +File Name: Profile.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package blockchain + +// ProfileReadField reads the specified profile field. See ProfileX for the list of recognized fields. The encoding depends on the field type. Status is StatusX. +func (blockchain *Blockchain) ProfileReadField(index uint16) (data []byte, status int) { + found := false + + status = blockchain.Iterate(func(block *Block) (statusI int) { + fields, err := decodeBlockRecordProfile(block.RecordsRaw) + if err != nil { + return StatusCorruptBlockRecord + } else if len(fields) == 0 { + return StatusOK + } + + // Check if the field is available in the profile record. If there are multiple records, only return the latest one. + for n := range fields { + if fields[n].Type == index { + data = fields[n].Data + found = true + } + } + + return StatusOK + }) + + if status != StatusOK { + return nil, status + } else if !found { + return nil, StatusDataNotFound + } + + return data, StatusOK +} + +// ProfileList lists all profile fields. Status is StatusX. +func (blockchain *Blockchain) ProfileList() (fields []BlockRecordProfile, status int) { + uniqueFields := make(map[uint16][]byte) + + status = blockchain.Iterate(func(block *Block) (statusI int) { + fields, err := decodeBlockRecordProfile(block.RecordsRaw) + if err != nil { + return StatusCorruptBlockRecord + } + + for n := range fields { + uniqueFields[fields[n].Type] = fields[n].Data + } + + return StatusOK + }) + + for key, value := range uniqueFields { + fields = append(fields, BlockRecordProfile{Type: key, Data: value}) + } + + return fields, status +} + +// ProfileWrite writes profile fields and blobs to the blockchain. Status is StatusX. +func (blockchain *Blockchain) ProfileWrite(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) { + encodeProfileAppend := func(fields []BlockRecordProfile) (newHeight, newVersion uint64, status int) { + encoded, err := encodeBlockRecordProfile(fields) + if err != nil { + return 0, 0, StatusCorruptBlockRecord + } + + return blockchain.Append(encoded) + } + + blockSize := uint64(blockHeaderSize) + var recordFields []BlockRecordProfile + + for _, field := range fields { + recordSize := field.SizeInBlock() + + // need to create a new block due to target block size? + if len(recordFields) > 0 && blockSize+recordSize > TargetBlockSize { + if newHeight, newVersion, status = encodeProfileAppend(recordFields); status != StatusOK { + return newHeight, newVersion, status + } + + blockSize = blockHeaderSize + recordFields = nil + } + + blockSize += recordSize + recordFields = append(recordFields, field) + } + + return encodeProfileAppend(recordFields) +} + +// ProfileDelete deletes fields and blobs from the blockchain. Status is StatusX. +func (blockchain *Blockchain) ProfileDelete(fields []uint16) (newHeight, newVersion uint64, status int) { + return blockchain.IterateDeleteRecord(nil, func(record *BlockRecordRaw) (deleteAction int) { + if record.Type != RecordTypeProfile { + return 0 // no action + } + + existingFields, err := decodeBlockRecordProfile([]BlockRecordRaw{*record}) + if err != nil || len(existingFields) != 1 { + return 3 // error blockchain corrupt + } + + for _, i := range fields { + if i == existingFields[0].Type { // found a file ID to delete? + return 1 // delete record + } + } + + return 0 // no action on record + }) +} diff --git a/blockchain/Test_test.go b/blockchain/Test_test.go index 3672a28..4798c58 100644 --- a/blockchain/Test_test.go +++ b/blockchain/Test_test.go @@ -1,264 +1,264 @@ -package blockchain - -import ( - "bytes" - "encoding/hex" - "fmt" - "testing" - - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/merkle" - "github.com/PeernetOfficial/core/protocol" - "github.com/google/uuid" -) - -func TestBlockEncoding(t *testing.T) { - privateKey, err := btcec.NewPrivateKey(btcec.S256()) - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - - encoded1, _ := encodeBlockRecordProfile([]BlockRecordProfile{ProfileFieldFromText(ProfileName, "Test User 1")}) - - file1, _ := createBlockRecordFile([]byte("Test data"), "Filename 1.txt", "documents\\sub folder") - file2, _ := createBlockRecordFile([]byte("Test data 2!"), "Filename 2.txt", "documents\\sub folder") - - encodedFiles, err := encodeBlockRecordFiles([]BlockRecordFile{file1, file2}) - if err != nil { - fmt.Printf("Error encoding files: %s\n", err.Error()) - return - } - - blockE := &Block{BlockchainVersion: 42, Number: 0} - blockE.RecordsRaw = append(blockE.RecordsRaw, encoded1...) - blockE.RecordsRaw = append(blockE.RecordsRaw, encodedFiles...) - - raw, err := encodeBlock(blockE, privateKey) - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - - block, err := decodeBlock(raw) - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - - decoded, err := decodeBlockRecords(block) - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - - // output the block details - fmt.Printf("Block details:\n----------------\nNumber: %d\nVersion: %d\nLast Hash: %s\nPublic Key: %s\n", block.Number, block.BlockchainVersion, hex.EncodeToString(block.LastBlockHash), hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())) - - for _, decodedR := range decoded.RecordsDecoded { - switch record := decodedR.(type) { - case BlockRecordFile: - printFile(record) - - case []BlockRecordProfile: - for _, profileR := range record { - printProfileField(profileR) - } - } - } -} - -func createBlockRecordFile(data []byte, name, folder string) (file BlockRecordFile, err error) { - file = BlockRecordFile{Hash: protocol.HashData(data), Type: testTypeText, Format: testFormatText, Size: uint64(len(data)), ID: uuid.New()} - file.Tags = append(file.Tags, TagFromText(TagName, name)) - file.Tags = append(file.Tags, TagFromText(TagFolder, folder)) - - file.FragmentSize = merkle.CalculateFragmentSize(file.Size) - tree, err := merkle.NewMerkleTree(file.Size, file.FragmentSize, bytes.NewBuffer(data)) - if err != nil { - return file, err - } - file.MerkleRootHash = tree.RootHash - - return file, nil -} - -func initTestPrivateKey() (blockchain *Blockchain, err error) { - // use static test key, otherwise tests will be inconsistent (would otherwise fail to open blockchain database) - privateKeyTestA := "d65da474861d826edd29c1307f1250d79e9dbf84e3a2449022658445c8d8ed63" - privateKeyB, _ := hex.DecodeString(privateKeyTestA) - peerPrivateKey, peerPublicKey := btcec.PrivKeyFromBytes(btcec.S256(), privateKeyB) - - fmt.Printf("Loaded public key: %s\n", hex.EncodeToString(peerPublicKey.SerializeCompressed())) - - return Init(peerPrivateKey, "test.blockchain") -} - -func TestBlockchainAdd(t *testing.T) { - blockchain, err := initTestPrivateKey() - if err != nil { - return - } - - file1, _ := createBlockRecordFile([]byte("Test data"), "Filename 1.txt", "documents\\sub folder") - - newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1}) - - switch status { - case 0: - case 1: // Error previous block not found - fmt.Printf("Error adding file to blockchain: Previous block not found.\n") - case 2: // Error block encoding - fmt.Printf("Error adding file to blockchain: Error block encoding.\n") - case 3: // Error block record encoding - fmt.Printf("Error adding file to blockchain: Error block record encoding.\n") - default: - fmt.Printf("Error adding file to blockchain: Unknown status %d\n", status) - } - - if status != 0 { - return - } - - fmt.Printf("Success adding files to blockchain. New blockchain height %d version %d\n", newHeight, newVersion) -} - -func TestBlockchainRead(t *testing.T) { - blockchain, err := initTestPrivateKey() - if err != nil { - return - } - - blockNumber := uint64(0) - - decoded, status, err := blockchain.Read(blockNumber) - switch status { - case 0: - case 1: // Error block not found - fmt.Printf("Error reading block %d: Block not found.\n", blockNumber) - case 2: // Error block encoding - fmt.Printf("Error reading block %d: Block encoding corrupt: %s\n", blockNumber, err.Error()) - case 3: // Error block record encoding - fmt.Printf("Error reading block %d: Block record encoding corrupt.\n", blockNumber) - default: - fmt.Printf("Error reading block %d: Unknown status %d\n", blockNumber, status) - } - - if status != 0 { - return - } - - for _, decodedR := range decoded.RecordsDecoded { - if file, ok := decodedR.(BlockRecordFile); ok { - printFile(file) - } - } -} - -func printFile(file BlockRecordFile) { - fmt.Printf("* File %s\n", file.ID.String()) - fmt.Printf(" Size %d\n", file.Size) - fmt.Printf(" Type %d\n", file.Type) - fmt.Printf(" Format %d\n", file.Format) - fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) - fmt.Printf(" Merkle Root Hash %s\n", hex.EncodeToString(file.MerkleRootHash)) - fmt.Printf(" Fragment Size %d\n", file.FragmentSize) - - for _, tag := range file.Tags { - switch tag.Type { - case TagName: - fmt.Printf(" Name %s\n", tag.Text()) - case TagFolder: - fmt.Printf(" Folder %s\n", tag.Text()) - case TagDescription: - fmt.Printf(" Description %s\n", tag.Text()) - } - } -} - -func TestBlockchainDelete(t *testing.T) { - blockchain, err := initTestPrivateKey() - if err != nil { - return - } - - // test add file - file1, _ := createBlockRecordFile([]byte("Test data"), "Test file to be deleted.txt", "documents\\sub folder") - - newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1}) - fmt.Printf("Added file: Status %d height %d version %d\n", status, newHeight, newVersion) - - // list files - files, _ := blockchain.ListFiles() - for _, file := range files { - printFile(file) - } - - fmt.Printf("----------------\n") - - // delete the file - newHeight, newVersion, _, status = blockchain.DeleteFiles([]uuid.UUID{file1.ID}) - fmt.Printf("Deleted file: Status %d height %d version %d\n", status, newHeight, newVersion) - - // list all files - files, _ = blockchain.ListFiles() - for _, file := range files { - printFile(file) - } -} - -func TestBlockchainProfile(t *testing.T) { - blockchain, err := initTestPrivateKey() - if err != nil { - return - } - - // write some test profile data - newHeight, newVersion, status := blockchain.ProfileWrite([]BlockRecordProfile{ - ProfileFieldFromText(ProfileName, "Test User 1"), - ProfileFieldFromText(ProfileEmail, "test@test.com"), - {Type: 100, Data: []byte{0, 1, 2, 3}}}) - - fmt.Printf("Write profile data: Status %d height %d version %d\n", status, newHeight, newVersion) - - // list all profile info - printProfileData(blockchain) - - fmt.Printf("----------------\n") - - // delete profile info - newHeight, newVersion, status = blockchain.ProfileDelete([]uint16{ProfileEmail}) - fmt.Printf("Deleted profile email: Status %d height %d version %d\n", status, newHeight, newVersion) - - printProfileData(blockchain) -} - -func printProfileData(blockchain *Blockchain) { - fields, status := blockchain.ProfileList() - if status != StatusOK { - fmt.Printf("Reading profile data error status: %d\n", status) - return - } - - if len(fields) == 0 { - fmt.Printf("No profile data to visualize.\n") - return - } - - for _, field := range fields { - printProfileField(field) - } -} - -func printProfileField(field BlockRecordProfile) { - switch field.Type { - case ProfileName, ProfileEmail, ProfileWebsite, ProfileTwitter, ProfileYouTube, ProfileAddress: - fmt.Printf("* Field %d = %s\n", field.Type, string(field.Data)) - - default: - fmt.Printf("* Field %d = %s\n", field.Type, hex.EncodeToString(field.Data)) - } -} - -const testTypeText = 1 -const testFormatText = 10 +package blockchain + +import ( + "bytes" + "encoding/hex" + "fmt" + "testing" + + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/merkle" + "github.com/PeernetOfficial/core/protocol" + "github.com/google/uuid" +) + +func TestBlockEncoding(t *testing.T) { + privateKey, err := btcec.NewPrivateKey(btcec.S256()) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + encoded1, _ := encodeBlockRecordProfile([]BlockRecordProfile{ProfileFieldFromText(ProfileName, "Test User 1")}) + + file1, _ := createBlockRecordFile([]byte("Test data"), "Filename 1.txt", "documents\\sub folder") + file2, _ := createBlockRecordFile([]byte("Test data 2!"), "Filename 2.txt", "documents\\sub folder") + + encodedFiles, err := encodeBlockRecordFiles([]BlockRecordFile{file1, file2}) + if err != nil { + fmt.Printf("Error encoding files: %s\n", err.Error()) + return + } + + blockE := &Block{BlockchainVersion: 42, Number: 0} + blockE.RecordsRaw = append(blockE.RecordsRaw, encoded1...) + blockE.RecordsRaw = append(blockE.RecordsRaw, encodedFiles...) + + raw, err := encodeBlock(blockE, privateKey) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + block, err := decodeBlock(raw) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + decoded, err := decodeBlockRecords(block) + if err != nil { + fmt.Printf("Error: %s\n", err.Error()) + return + } + + // output the block details + fmt.Printf("Block details:\n----------------\nNumber: %d\nVersion: %d\nLast Hash: %s\nPublic Key: %s\n", block.Number, block.BlockchainVersion, hex.EncodeToString(block.LastBlockHash), hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())) + + for _, decodedR := range decoded.RecordsDecoded { + switch record := decodedR.(type) { + case BlockRecordFile: + printFile(record) + + case []BlockRecordProfile: + for _, profileR := range record { + printProfileField(profileR) + } + } + } +} + +func createBlockRecordFile(data []byte, name, folder string) (file BlockRecordFile, err error) { + file = BlockRecordFile{Hash: protocol.HashData(data), Type: testTypeText, Format: testFormatText, Size: uint64(len(data)), ID: uuid.New()} + file.Tags = append(file.Tags, TagFromText(TagName, name)) + file.Tags = append(file.Tags, TagFromText(TagFolder, folder)) + + file.FragmentSize = merkle.CalculateFragmentSize(file.Size) + tree, err := merkle.NewMerkleTree(file.Size, file.FragmentSize, bytes.NewBuffer(data)) + if err != nil { + return file, err + } + file.MerkleRootHash = tree.RootHash + + return file, nil +} + +func initTestPrivateKey() (blockchain *Blockchain, err error) { + // use static test key, otherwise tests will be inconsistent (would otherwise fail to open blockchain database) + privateKeyTestA := "d65da474861d826edd29c1307f1250d79e9dbf84e3a2449022658445c8d8ed63" + privateKeyB, _ := hex.DecodeString(privateKeyTestA) + peerPrivateKey, peerPublicKey := btcec.PrivKeyFromBytes(btcec.S256(), privateKeyB) + + fmt.Printf("Loaded public key: %s\n", hex.EncodeToString(peerPublicKey.SerializeCompressed())) + + return Init(peerPrivateKey, "test.blockchain") +} + +func TestBlockchainAdd(t *testing.T) { + blockchain, err := initTestPrivateKey() + if err != nil { + return + } + + file1, _ := createBlockRecordFile([]byte("Test data"), "Filename 1.txt", "documents\\sub folder") + + newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1}) + + switch status { + case 0: + case 1: // Error previous block not found + fmt.Printf("Error adding file to blockchain: Previous block not found.\n") + case 2: // Error block encoding + fmt.Printf("Error adding file to blockchain: Error block encoding.\n") + case 3: // Error block record encoding + fmt.Printf("Error adding file to blockchain: Error block record encoding.\n") + default: + fmt.Printf("Error adding file to blockchain: Unknown status %d\n", status) + } + + if status != 0 { + return + } + + fmt.Printf("Success adding files to blockchain. New blockchain height %d version %d\n", newHeight, newVersion) +} + +func TestBlockchainRead(t *testing.T) { + blockchain, err := initTestPrivateKey() + if err != nil { + return + } + + blockNumber := uint64(0) + + decoded, status, err := blockchain.Read(blockNumber) + switch status { + case 0: + case 1: // Error block not found + fmt.Printf("Error reading block %d: Block not found.\n", blockNumber) + case 2: // Error block encoding + fmt.Printf("Error reading block %d: Block encoding corrupt: %s\n", blockNumber, err.Error()) + case 3: // Error block record encoding + fmt.Printf("Error reading block %d: Block record encoding corrupt.\n", blockNumber) + default: + fmt.Printf("Error reading block %d: Unknown status %d\n", blockNumber, status) + } + + if status != 0 { + return + } + + for _, decodedR := range decoded.RecordsDecoded { + if file, ok := decodedR.(BlockRecordFile); ok { + printFile(file) + } + } +} + +func printFile(file BlockRecordFile) { + fmt.Printf("* File %s\n", file.ID.String()) + fmt.Printf(" Size %d\n", file.Size) + fmt.Printf(" Type %d\n", file.Type) + fmt.Printf(" Format %d\n", file.Format) + fmt.Printf(" Hash %s\n", hex.EncodeToString(file.Hash)) + fmt.Printf(" Merkle Root Hash %s\n", hex.EncodeToString(file.MerkleRootHash)) + fmt.Printf(" Fragment Size %d\n", file.FragmentSize) + + for _, tag := range file.Tags { + switch tag.Type { + case TagName: + fmt.Printf(" Name %s\n", tag.Text()) + case TagFolder: + fmt.Printf(" Folder %s\n", tag.Text()) + case TagDescription: + fmt.Printf(" Description %s\n", tag.Text()) + } + } +} + +func TestBlockchainDelete(t *testing.T) { + blockchain, err := initTestPrivateKey() + if err != nil { + return + } + + // test add file + file1, _ := createBlockRecordFile([]byte("Test data"), "Test file to be deleted.txt", "documents\\sub folder") + + newHeight, newVersion, status := blockchain.AddFiles([]BlockRecordFile{file1}) + fmt.Printf("Added file: Status %d height %d version %d\n", status, newHeight, newVersion) + + // list files + files, _ := blockchain.ListFiles() + for _, file := range files { + printFile(file) + } + + fmt.Printf("----------------\n") + + // delete the file + newHeight, newVersion, _, status = blockchain.DeleteFiles([]uuid.UUID{file1.ID}) + fmt.Printf("Deleted file: Status %d height %d version %d\n", status, newHeight, newVersion) + + // list all files + files, _ = blockchain.ListFiles() + for _, file := range files { + printFile(file) + } +} + +func TestBlockchainProfile(t *testing.T) { + blockchain, err := initTestPrivateKey() + if err != nil { + return + } + + // write some test profile data + newHeight, newVersion, status := blockchain.ProfileWrite([]BlockRecordProfile{ + ProfileFieldFromText(ProfileName, "Test User 1"), + ProfileFieldFromText(ProfileEmail, "test@test.com"), + {Type: 100, Data: []byte{0, 1, 2, 3}}}) + + fmt.Printf("Write profile data: Status %d height %d version %d\n", status, newHeight, newVersion) + + // list all profile info + printProfileData(blockchain) + + fmt.Printf("----------------\n") + + // delete profile info + newHeight, newVersion, status = blockchain.ProfileDelete([]uint16{ProfileEmail}) + fmt.Printf("Deleted profile email: Status %d height %d version %d\n", status, newHeight, newVersion) + + printProfileData(blockchain) +} + +func printProfileData(blockchain *Blockchain) { + fields, status := blockchain.ProfileList() + if status != StatusOK { + fmt.Printf("Reading profile data error status: %d\n", status) + return + } + + if len(fields) == 0 { + fmt.Printf("No profile data to visualize.\n") + return + } + + for _, field := range fields { + printProfileField(field) + } +} + +func printProfileField(field BlockRecordProfile) { + switch field.Type { + case ProfileName, ProfileEmail, ProfileWebsite, ProfileTwitter, ProfileYouTube, ProfileAddress: + fmt.Printf("* Field %d = %s\n", field.Type, string(field.Data)) + + default: + fmt.Printf("* Field %d = %s\n", field.Type, hex.EncodeToString(field.Data)) + } +} + +const testTypeText = 1 +const testFormatText = 10 diff --git a/btcec/field.go b/btcec/field.go index 98105ed..d13d899 100644 --- a/btcec/field.go +++ b/btcec/field.go @@ -125,27 +125,30 @@ var ( // the arithmetic needed for elliptic curve operations. // // The following depicts the internal representation: -// ----------------------------------------------------------------- -// | n[9] | n[8] | ... | n[0] | -// | 32 bits available | 32 bits available | ... | 32 bits available | -// | 22 bits for value | 26 bits for value | ... | 26 bits for value | -// | 10 bits overflow | 6 bits overflow | ... | 6 bits overflow | -// | Mult: 2^(26*9) | Mult: 2^(26*8) | ... | Mult: 2^(26*0) | -// ----------------------------------------------------------------- +// +// ----------------------------------------------------------------- +// | n[9] | n[8] | ... | n[0] | +// | 32 bits available | 32 bits available | ... | 32 bits available | +// | 22 bits for value | 26 bits for value | ... | 26 bits for value | +// | 10 bits overflow | 6 bits overflow | ... | 6 bits overflow | +// | Mult: 2^(26*9) | Mult: 2^(26*8) | ... | Mult: 2^(26*0) | +// ----------------------------------------------------------------- // // For example, consider the number 2^49 + 1. It would be represented as: -// n[0] = 1 -// n[1] = 2^23 -// n[2..9] = 0 +// +// n[0] = 1 +// n[1] = 2^23 +// n[2..9] = 0 // // The full 256-bit value is then calculated by looping i from 9..0 and // doing sum(n[i] * 2^(26i)) like so: -// n[9] * 2^(26*9) = 0 * 2^234 = 0 -// n[8] * 2^(26*8) = 0 * 2^208 = 0 -// ... -// n[1] * 2^(26*1) = 2^23 * 2^26 = 2^49 -// n[0] * 2^(26*0) = 1 * 2^0 = 1 -// Sum: 0 + 0 + ... + 2^49 + 1 = 2^49 + 1 +// +// n[9] * 2^(26*9) = 0 * 2^234 = 0 +// n[8] * 2^(26*8) = 0 * 2^208 = 0 +// ... +// n[1] * 2^(26*1) = 2^23 * 2^26 = 2^49 +// n[0] * 2^(26*0) = 1 * 2^0 = 1 +// Sum: 0 + 0 + ... + 2^49 + 1 = 2^49 + 1 type fieldVal struct { n [10]uint32 } diff --git a/btcec/gensecp256k1.go b/btcec/gensecp256k1.go index 1928702..e079f55 100644 --- a/btcec/gensecp256k1.go +++ b/btcec/gensecp256k1.go @@ -4,6 +4,7 @@ // This file is ignored during the regular build due to the following build tag. // This build tag is set during go generate. +//go:build gensecp256k1 // +build gensecp256k1 package btcec diff --git a/dht/DHT Lite.go b/dht/DHT Lite.go index ebd183d..55b0b32 100644 --- a/dht/DHT Lite.go +++ b/dht/DHT Lite.go @@ -1,211 +1,211 @@ -/* -File Name: DHT Lite.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -A "lite" DHT implementation without any direct network and store code. There is really no reason for any of the heavy network implementation to be part of this. -*/ - -package dht - -import ( - "encoding/hex" - "errors" - "time" -) - -// DHT represents the state of the local node in the distributed hash table -type DHT struct { - ht *hashTable - - // A small number representing the degree of parallelism in network calls. - // The alpha amount of nodes will be contacted in parallel for finding the target. - alpha int - - // Functions below must be set and provided by the caller. - - // ShouldEvict determines whether node 1 shall be evicted in favor of node 2 - ShouldEvict func(node1, node2 *Node) bool - - // SendRequestStore sends an announcement-store message to the remote node. It informs the remote node that the local one stores the given key-value. - SendRequestStore func(node *Node, key []byte, dataSize uint64) - - // SendRequestFindNode sends an information request to find a particular node. nodes are the nodes to send the request to. - SendRequestFindNode func(request *InformationRequest) - - // SendRequestFindValue sends an information request to find data. nodes are the nodes to send the request to. - SendRequestFindValue func(request *InformationRequest) - - // FilterSearchStatus is called with updates of searches in the DHT - FilterSearchStatus func(client *SearchClient, function, format string, v ...interface{}) - - // TimeoutSearch is the maximum time a search may take. - TimeoutSearch time.Duration - - // TimeoutIR is the maximum an information request to a node may take. - TimeoutIR time.Duration -} - -// NewDHT initializes a new DHT node with default values. -func NewDHT(self *Node, bits, bucketSize, alpha int) *DHT { - return &DHT{ - ht: newHashTable(self, bits, bucketSize), - alpha: alpha, - FilterSearchStatus: func(client *SearchClient, function, format string, v ...interface{}) {}, - TimeoutSearch: 10 * time.Second, - TimeoutIR: 6 * time.Second, - } -} - -// NumNodes returns the total number of nodes stored in the local routing table -func (dht *DHT) NumNodes() int { - return dht.ht.totalNodes() -} - -// Nodes returns the nodes themselves sotred in the routing table. -func (dht *DHT) Nodes() []*Node { - return dht.ht.Nodes() -} - -// GetSelfID returns the identifier of the local node -func (dht *DHT) GetSelfID() []byte { - return dht.ht.Self.ID -} - -// AddNode adds a node into the appropriate k bucket. These buckets are stored in big-endian order so we look at the bits from right to left in order to find the appropriate bucket. -func (dht *DHT) AddNode(node *Node) { - // The previous code made an immediate ping to the oldest node to "ping the oldest node to find out if it responds back in a reasonable amount of time. If not - remove it." - // In DHT Lite, however, it will be up to the caller to determine nodes to remove. - dht.ht.insertNode(node, dht.ShouldEvict) -} - -// RemoveNode removes a node -func (dht *DHT) RemoveNode(ID []byte) { - dht.ht.removeNode(ID) -} - -// GetClosestContacts returns the closes contacts in the hash table -func (dht *DHT) GetClosestContacts(count int, target []byte, filterFunc NodeFilterFunc, ignoredNodes ...[]byte) []*Node { - closest := dht.ht.getClosestContacts(count, target, filterFunc, ignoredNodes...) - return closest.Nodes -} - -// MarkNodeAsSeen marks a node as seen, which pushes it to the top in the bucket list. -func (dht *DHT) MarkNodeAsSeen(ID []byte) { - dht.ht.markNodeAsSeen(dht.ht.getBucketIndexFromDifferingBit(ID), ID) -} - -// IsNodeCloser compares 2 nodes to self. If true, the first node is closer (= smaller distance) to self than the second. -func (dht *DHT) IsNodeCloser(node1, node2 []byte) bool { - iDist := getDistance(node1, dht.ht.Self.ID) - jDist := getDistance(node2, dht.ht.Self.ID) - - return iDist.Cmp(jDist) == -1 -} - -// IsNodeContact checks if the given node is in the local routing table -func (dht *DHT) IsNodeContact(ID []byte) (node *Node) { - return dht.ht.doesNodeExist(ID) -} - -// ---- Synchronous network query functions below ---- - -// Store informs the network about data stored locally. -// Data size informs how big the data is without sending the actual data. closestCount is the number of closest nodes to contact. -func (dht *DHT) Store(key []byte, dataSize uint64, closestCount int) (err error) { - if len(key)*8 != dht.ht.bBits { - return errors.New("invalid key size") - } - - // TODO: Introduce ActionFindClosestNodes? - - search := dht.NewSearch(ActionFindNode, key, dht.TimeoutSearch, dht.TimeoutIR, dht.alpha) - search.LogStatus = func(function, format string, v ...interface{}) { - dht.FilterSearchStatus(search, function, format, v...) - } - search.LogStatus("dht.Store", "Search for closest nodes to key %s. Full timeout %s, per node %s. Alpha = %d.\n", hex.EncodeToString(key), dht.TimeoutSearch.String(), dht.TimeoutIR.String(), dht.alpha) - search.SearchAway() - - // search.Results channel is ignored here. Only the closest nodes to the key are of interest. It is not expected to find a match of key and node ID. - <-search.TerminateSignal - - // Contact the closes nodes found. - for n := 0; n < closestCount && n < len(search.list.Nodes); n++ { - node := search.list.Nodes[n] - search.LogStatus("dht.Store", "Send info-store message to node %s\n", hex.EncodeToString(node.ID)) - dht.SendRequestStore(node, key, dataSize) - } - - return nil -} - -// Get retrieves data from the network using key -func (dht *DHT) Get(key []byte) (value []byte, senderID []byte, found bool, err error) { - if len(key)*8 != dht.ht.bBits { - return nil, nil, false, errors.New("invalid key size") - } - - search := dht.NewSearch(ActionFindValue, key, dht.TimeoutSearch, dht.TimeoutIR, dht.alpha) - search.LogStatus = func(function, format string, v ...interface{}) { - dht.FilterSearchStatus(search, function, format, v...) - } - search.LogStatus("dht.Get", "Search for node %s. Full timeout %s, per node %s. Alpha = %d.\n", hex.EncodeToString(key), dht.TimeoutSearch.String(), dht.TimeoutIR.String(), dht.alpha) - search.SearchAway() - - select { - case <-search.TerminateSignal: - return nil, nil, false, nil - case result := <-search.Results: - return result.Data, result.SenderID, true, nil - } -} - -// FindNode finds the target node in the network. Blocking! -// The caller may use dht.NewSearch directly and take advantage of the asynchronous response and custom timeouts. -func (dht *DHT) FindNode(key []byte) (node *Node, err error) { - if len(key)*8 != dht.ht.bBits { - return nil, errors.New("invalid key size") - } - - search := dht.NewSearch(ActionFindNode, key, dht.TimeoutSearch, dht.TimeoutIR, dht.alpha) - search.LogStatus = func(function, format string, v ...interface{}) { - dht.FilterSearchStatus(search, function, format, v...) - } - search.LogStatus("dht.FindNode", "Search for node %s. Full timeout %s, per node %s. Alpha = %d.\n", hex.EncodeToString(key), dht.TimeoutSearch.String(), dht.TimeoutIR.String(), dht.alpha) - search.SearchAway() - - result, ok := <-search.Results - if !ok { // Check if closed channel. Redundant with checking <-search.TerminateSignal. - return nil, nil - } - return result.TargetNode, nil -} - -// ---- DHT Health ---- - -// DisableBucketRefresh is an option for debug purposes to reduce noise. It can be useful to disable bucket refresh when debugging outgoing DHT searches. -var DisableBucketRefresh = false - -// RefreshBuckets refreshes all buckets not meeting the target node number. 0 to refresh all. -func (dht *DHT) RefreshBuckets(target int) { - if DisableBucketRefresh { - return - } - - for bucket, total := range dht.ht.getTotalNodesPerBucket() { - if target == 0 || total < target { - nodeR := dht.ht.getRandomIDFromBucket(bucket) - - // Refreshing closest bucket? Use self ID instead of random one. - if bucket == 0 { - nodeR = dht.ht.Self.ID - } - - dht.FindNode(nodeR) - } - - if DisableBucketRefresh { // may be disabled while in full refresh which may take some time - return - } - } -} +/* +File Name: DHT Lite.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +A "lite" DHT implementation without any direct network and store code. There is really no reason for any of the heavy network implementation to be part of this. +*/ + +package dht + +import ( + "encoding/hex" + "errors" + "time" +) + +// DHT represents the state of the local node in the distributed hash table +type DHT struct { + ht *hashTable + + // A small number representing the degree of parallelism in network calls. + // The alpha amount of nodes will be contacted in parallel for finding the target. + alpha int + + // Functions below must be set and provided by the caller. + + // ShouldEvict determines whether node 1 shall be evicted in favor of node 2 + ShouldEvict func(node1, node2 *Node) bool + + // SendRequestStore sends an announcement-store message to the remote node. It informs the remote node that the local one stores the given key-value. + SendRequestStore func(node *Node, key []byte, dataSize uint64) + + // SendRequestFindNode sends an information request to find a particular node. nodes are the nodes to send the request to. + SendRequestFindNode func(request *InformationRequest) + + // SendRequestFindValue sends an information request to find data. nodes are the nodes to send the request to. + SendRequestFindValue func(request *InformationRequest) + + // FilterSearchStatus is called with updates of searches in the DHT + FilterSearchStatus func(client *SearchClient, function, format string, v ...interface{}) + + // TimeoutSearch is the maximum time a search may take. + TimeoutSearch time.Duration + + // TimeoutIR is the maximum an information request to a node may take. + TimeoutIR time.Duration +} + +// NewDHT initializes a new DHT node with default values. +func NewDHT(self *Node, bits, bucketSize, alpha int) *DHT { + return &DHT{ + ht: newHashTable(self, bits, bucketSize), + alpha: alpha, + FilterSearchStatus: func(client *SearchClient, function, format string, v ...interface{}) {}, + TimeoutSearch: 10 * time.Second, + TimeoutIR: 6 * time.Second, + } +} + +// NumNodes returns the total number of nodes stored in the local routing table +func (dht *DHT) NumNodes() int { + return dht.ht.totalNodes() +} + +// Nodes returns the nodes themselves sotred in the routing table. +func (dht *DHT) Nodes() []*Node { + return dht.ht.Nodes() +} + +// GetSelfID returns the identifier of the local node +func (dht *DHT) GetSelfID() []byte { + return dht.ht.Self.ID +} + +// AddNode adds a node into the appropriate k bucket. These buckets are stored in big-endian order so we look at the bits from right to left in order to find the appropriate bucket. +func (dht *DHT) AddNode(node *Node) { + // The previous code made an immediate ping to the oldest node to "ping the oldest node to find out if it responds back in a reasonable amount of time. If not - remove it." + // In DHT Lite, however, it will be up to the caller to determine nodes to remove. + dht.ht.insertNode(node, dht.ShouldEvict) +} + +// RemoveNode removes a node +func (dht *DHT) RemoveNode(ID []byte) { + dht.ht.removeNode(ID) +} + +// GetClosestContacts returns the closes contacts in the hash table +func (dht *DHT) GetClosestContacts(count int, target []byte, filterFunc NodeFilterFunc, ignoredNodes ...[]byte) []*Node { + closest := dht.ht.getClosestContacts(count, target, filterFunc, ignoredNodes...) + return closest.Nodes +} + +// MarkNodeAsSeen marks a node as seen, which pushes it to the top in the bucket list. +func (dht *DHT) MarkNodeAsSeen(ID []byte) { + dht.ht.markNodeAsSeen(dht.ht.getBucketIndexFromDifferingBit(ID), ID) +} + +// IsNodeCloser compares 2 nodes to self. If true, the first node is closer (= smaller distance) to self than the second. +func (dht *DHT) IsNodeCloser(node1, node2 []byte) bool { + iDist := getDistance(node1, dht.ht.Self.ID) + jDist := getDistance(node2, dht.ht.Self.ID) + + return iDist.Cmp(jDist) == -1 +} + +// IsNodeContact checks if the given node is in the local routing table +func (dht *DHT) IsNodeContact(ID []byte) (node *Node) { + return dht.ht.doesNodeExist(ID) +} + +// ---- Synchronous network query functions below ---- + +// Store informs the network about data stored locally. +// Data size informs how big the data is without sending the actual data. closestCount is the number of closest nodes to contact. +func (dht *DHT) Store(key []byte, dataSize uint64, closestCount int) (err error) { + if len(key)*8 != dht.ht.bBits { + return errors.New("invalid key size") + } + + // TODO: Introduce ActionFindClosestNodes? + + search := dht.NewSearch(ActionFindNode, key, dht.TimeoutSearch, dht.TimeoutIR, dht.alpha) + search.LogStatus = func(function, format string, v ...interface{}) { + dht.FilterSearchStatus(search, function, format, v...) + } + search.LogStatus("dht.Store", "Search for closest nodes to key %s. Full timeout %s, per node %s. Alpha = %d.\n", hex.EncodeToString(key), dht.TimeoutSearch.String(), dht.TimeoutIR.String(), dht.alpha) + search.SearchAway() + + // search.Results channel is ignored here. Only the closest nodes to the key are of interest. It is not expected to find a match of key and node ID. + <-search.TerminateSignal + + // Contact the closes nodes found. + for n := 0; n < closestCount && n < len(search.list.Nodes); n++ { + node := search.list.Nodes[n] + search.LogStatus("dht.Store", "Send info-store message to node %s\n", hex.EncodeToString(node.ID)) + dht.SendRequestStore(node, key, dataSize) + } + + return nil +} + +// Get retrieves data from the network using key +func (dht *DHT) Get(key []byte) (value []byte, senderID []byte, found bool, err error) { + if len(key)*8 != dht.ht.bBits { + return nil, nil, false, errors.New("invalid key size") + } + + search := dht.NewSearch(ActionFindValue, key, dht.TimeoutSearch, dht.TimeoutIR, dht.alpha) + search.LogStatus = func(function, format string, v ...interface{}) { + dht.FilterSearchStatus(search, function, format, v...) + } + search.LogStatus("dht.Get", "Search for node %s. Full timeout %s, per node %s. Alpha = %d.\n", hex.EncodeToString(key), dht.TimeoutSearch.String(), dht.TimeoutIR.String(), dht.alpha) + search.SearchAway() + + select { + case <-search.TerminateSignal: + return nil, nil, false, nil + case result := <-search.Results: + return result.Data, result.SenderID, true, nil + } +} + +// FindNode finds the target node in the network. Blocking! +// The caller may use dht.NewSearch directly and take advantage of the asynchronous response and custom timeouts. +func (dht *DHT) FindNode(key []byte) (node *Node, err error) { + if len(key)*8 != dht.ht.bBits { + return nil, errors.New("invalid key size") + } + + search := dht.NewSearch(ActionFindNode, key, dht.TimeoutSearch, dht.TimeoutIR, dht.alpha) + search.LogStatus = func(function, format string, v ...interface{}) { + dht.FilterSearchStatus(search, function, format, v...) + } + search.LogStatus("dht.FindNode", "Search for node %s. Full timeout %s, per node %s. Alpha = %d.\n", hex.EncodeToString(key), dht.TimeoutSearch.String(), dht.TimeoutIR.String(), dht.alpha) + search.SearchAway() + + result, ok := <-search.Results + if !ok { // Check if closed channel. Redundant with checking <-search.TerminateSignal. + return nil, nil + } + return result.TargetNode, nil +} + +// ---- DHT Health ---- + +// DisableBucketRefresh is an option for debug purposes to reduce noise. It can be useful to disable bucket refresh when debugging outgoing DHT searches. +var DisableBucketRefresh = false + +// RefreshBuckets refreshes all buckets not meeting the target node number. 0 to refresh all. +func (dht *DHT) RefreshBuckets(target int) { + if DisableBucketRefresh { + return + } + + for bucket, total := range dht.ht.getTotalNodesPerBucket() { + if target == 0 || total < target { + nodeR := dht.ht.getRandomIDFromBucket(bucket) + + // Refreshing closest bucket? Use self ID instead of random one. + if bucket == 0 { + nodeR = dht.ht.Self.ID + } + + dht.FindNode(nodeR) + } + + if DisableBucketRefresh { // may be disabled while in full refresh which may take some time + return + } + } +} diff --git a/dht/Hash Table.go b/dht/Hash Table.go index ae022fa..67e2564 100644 --- a/dht/Hash Table.go +++ b/dht/Hash Table.go @@ -306,7 +306,7 @@ func (ht *hashTable) getTotalNodesPerBucket() (total []int) { ht.mutex.RLock() defer ht.mutex.RUnlock() - for n, _ := range ht.RoutingTable { + for n := range ht.RoutingTable { total = append(total, len(ht.RoutingTable[n])) } diff --git a/dht/Information Request.go b/dht/Information Request.go index fa5e0f8..0e88036 100644 --- a/dht/Information Request.go +++ b/dht/Information Request.go @@ -1,113 +1,113 @@ -/* -File Name: Information Request.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Information requests are asynchronous queries sent to nodes. -*/ - -package dht - -import ( - "sync" - "sync/atomic" - "time" -) - -// InformationRequest is an asynchronous request sent to nodes. It tracks any asynchronous replies and handles timeouts. -type InformationRequest struct { - Action int // ActionX - Key []byte // Key that is being queried - ResultChan chan *NodeMessage // Result channel - ResultChanExt chan *NodeMessage // External result channel to use instead - ActiveNodes uint64 // Number of nodes actively handling the request. - Nodes []*Node // Nodes that are receiving the request. - IsTerminated bool // If true, it was signaled for termination - TerminateSignal chan struct{} // gets closed on termination signal, can be used in select via "case _ = <- network.terminateSignal:" - sync.Mutex // for sychronized closing -} - -// Actions for performing the information request -const ( - ActionFindNode = iota // Find a node - ActionFindValue // Find a value -) - -const messageChannelSize = 100 - -// NewInformationRequest creates a new information request and adds it to the list. -// It marks the count of nodes as active, meaning the caller should later decrease it via ActiveNodesSub. -func (dht *DHT) NewInformationRequest(Action int, Key []byte, Nodes []*Node) (ir *InformationRequest) { - ir = &InformationRequest{ - ResultChan: make(chan *NodeMessage, messageChannelSize), - TerminateSignal: make(chan struct{}), - Action: Action, - Key: Key, - Nodes: Nodes, - ActiveNodes: uint64(len(Nodes)), - } - - return -} - -// CollectResults collects all information request responses within the given timeout. -func (ir *InformationRequest) CollectResults(timeout time.Duration) (results []*NodeMessage) { - for { - select { - case result, ok := <-ir.ResultChan: - if !ok { // channel closed? - return - } - - results = append(results, result) - - case <-time.After(timeout): - ir.Terminate() - return - - case <-ir.TerminateSignal: - return - } - } -} - -// Terminate sends the termination signal to all workers. It is safe to call Terminate multiple times. -func (ir *InformationRequest) Terminate() { - ir.Lock() - defer ir.Unlock() - - if ir.IsTerminated { - return - } - - // set the termination signal - ir.IsTerminated = true - close(ir.TerminateSignal) // safety guaranteed via lock - - close(ir.ResultChan) -} - -// Done is called when a remote node is done. -func (ir *InformationRequest) Done() { - if atomic.AddUint64(&ir.ActiveNodes, ^uint64(0)) <= 0 { - // If the counter reaches 0, it means no nodes are handling this request anymore -> terminate it. - ir.Terminate() - } -} - -// QueueResult accepts incoming results and queues them to the result channel. Non-blocking. -func (ir *InformationRequest) QueueResult(message *NodeMessage) { - ir.Lock() - if !ir.IsTerminated { - targetChan := ir.ResultChan - if ir.ResultChanExt != nil { - targetChan = ir.ResultChanExt - } - - select { - case targetChan <- message: - default: - } - } - ir.Unlock() -} +/* +File Name: Information Request.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Information requests are asynchronous queries sent to nodes. +*/ + +package dht + +import ( + "sync" + "sync/atomic" + "time" +) + +// InformationRequest is an asynchronous request sent to nodes. It tracks any asynchronous replies and handles timeouts. +type InformationRequest struct { + Action int // ActionX + Key []byte // Key that is being queried + ResultChan chan *NodeMessage // Result channel + ResultChanExt chan *NodeMessage // External result channel to use instead + ActiveNodes uint64 // Number of nodes actively handling the request. + Nodes []*Node // Nodes that are receiving the request. + IsTerminated bool // If true, it was signaled for termination + TerminateSignal chan struct{} // gets closed on termination signal, can be used in select via "case _ = <- network.terminateSignal:" + sync.Mutex // for sychronized closing +} + +// Actions for performing the information request +const ( + ActionFindNode = iota // Find a node + ActionFindValue // Find a value +) + +const messageChannelSize = 100 + +// NewInformationRequest creates a new information request and adds it to the list. +// It marks the count of nodes as active, meaning the caller should later decrease it via ActiveNodesSub. +func (dht *DHT) NewInformationRequest(Action int, Key []byte, Nodes []*Node) (ir *InformationRequest) { + ir = &InformationRequest{ + ResultChan: make(chan *NodeMessage, messageChannelSize), + TerminateSignal: make(chan struct{}), + Action: Action, + Key: Key, + Nodes: Nodes, + ActiveNodes: uint64(len(Nodes)), + } + + return +} + +// CollectResults collects all information request responses within the given timeout. +func (ir *InformationRequest) CollectResults(timeout time.Duration) (results []*NodeMessage) { + for { + select { + case result, ok := <-ir.ResultChan: + if !ok { // channel closed? + return + } + + results = append(results, result) + + case <-time.After(timeout): + ir.Terminate() + return + + case <-ir.TerminateSignal: + return + } + } +} + +// Terminate sends the termination signal to all workers. It is safe to call Terminate multiple times. +func (ir *InformationRequest) Terminate() { + ir.Lock() + defer ir.Unlock() + + if ir.IsTerminated { + return + } + + // set the termination signal + ir.IsTerminated = true + close(ir.TerminateSignal) // safety guaranteed via lock + + close(ir.ResultChan) +} + +// Done is called when a remote node is done. +func (ir *InformationRequest) Done() { + if atomic.AddUint64(&ir.ActiveNodes, ^uint64(0)) <= 0 { + // If the counter reaches 0, it means no nodes are handling this request anymore -> terminate it. + ir.Terminate() + } +} + +// QueueResult accepts incoming results and queues them to the result channel. Non-blocking. +func (ir *InformationRequest) QueueResult(message *NodeMessage) { + ir.Lock() + if !ir.IsTerminated { + targetChan := ir.ResultChan + if ir.ResultChanExt != nil { + targetChan = ir.ResultChanExt + } + + select { + case targetChan <- message: + default: + } + } + ir.Unlock() +} diff --git a/dht/Search Client.go b/dht/Search Client.go index 41b2051..8ebf588 100644 --- a/dht/Search Client.go +++ b/dht/Search Client.go @@ -1,325 +1,325 @@ -/* -File Name: Search Client.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -A search client runs concurrent information requests for a single query. It solves the query efficiently by using levels. -Any result that is closer to the target gets pushed down into a new lower level, which contacts nodes closer to the result. -Level are running concurrently. -*/ - -package dht - -import ( - "bytes" - "encoding/hex" - "sync" - "sync/atomic" - "time" -) - -// MaxAcceptKnownStore is maximum count accepted of known peers that store the value -const MaxAcceptKnownStore = 10 - -// MaxClosest is maximum number of closest peers accepted -const MaxClosest = 3 - -// MaxLevel defines the max level. -const MaxLevel = 10 - -// SearchClient defines a search in the distributed hash table involving multiple information requests. -// The search can be created for a node or for a value, both identified by the hash (known as the key). -type SearchClient struct { - Action int // ActionX - Key []byte // Key that is being queried - IsTerminated bool // If true, it was signaled for termination - TerminateSignal chan struct{} // gets closed on termination signal, can be used in select via "case _ = <- TerminateSignal:" - sync.Mutex // for sychronized closing - timeStart time.Time // When the search started. - timeEnd time.Time // When the search ended. - dht *DHT // DHT used - timeoutTotal time.Duration // Timeout after the entire search will be terminated client-side. - timeoutIR time.Duration // Timeout for information requests (entire roundtrip). - alpha int // Count of concurrent information requests per level. - Results chan *SearchResult // Result channel - list *shortList // List of nodes to contact - contactedNodesMap map[string]struct{} // List of nodes already contacted - contactedNodesMutex sync.RWMutex // Sync map access - storing chan []*Node // Internal channel to signal nodes that indicate storing the searched value. - activeLevels uint64 // demo - LogStatus func(function, format string, v ...interface{}) // Filter function for status output -} - -// SearchResult is a single result to the search. Depending on the search type and parameters, multiple results may be sent. -type SearchResult struct { - Key []byte // Original key that was searched for - Action int // Original action - SenderID []byte // Sender node ID of the result - - // data for ActionFindNode - TargetNode *Node // The node that was requested. - - // data for ActionFindValue - Data []byte // Actual data -} - -// NewSearch creates a new search client. -// Action indicates the action to take (from ActionX constants), to either find a node, or a value. -// Timeout is the total time the search may take, covering all information requests. TimeoutIR is the time an information request may take. -// Alpha is the number of concurrent requests that will be performed. -func (dht *DHT) NewSearch(Action int, Key []byte, Timeout, TimeoutIR time.Duration, Alpha int) (client *SearchClient) { - client = &SearchClient{ - Action: Action, - Key: Key, - dht: dht, - timeoutTotal: Timeout, - timeoutIR: TimeoutIR, - alpha: Alpha, - contactedNodesMap: make(map[string]struct{}), - storing: make(chan []*Node, Alpha*2), - TerminateSignal: make(chan struct{}), - Results: make(chan *SearchResult), - LogStatus: func(function, format string, v ...interface{}) {}, - } - - return -} - -// Terminate sends the termination signal to all workers. It is safe to call Terminate multiple times. -func (client *SearchClient) Terminate() { - client.Lock() - defer client.Unlock() - - if client.IsTerminated { - return - } - - // set the termination signal - client.IsTerminated = true - close(client.TerminateSignal) // safety guaranteed via lock - - client.timeEnd = time.Now() - - close(client.Results) - close(client.storing) -} - -// isContactedNode checks if a node was contacted -func (client *SearchClient) isContactedNode(ID []byte, Set bool) (contacted bool) { - client.contactedNodesMutex.Lock() - _, contacted = client.contactedNodesMap[string(ID)] - if Set { - client.contactedNodesMap[string(ID)] = struct{}{} - } - client.contactedNodesMutex.Unlock() - return contacted -} - -// filterUncontactedNodes returns only nodes that were not contacted so far. All nodes will be set to contacted. Limit is optional (0 for no limit). -func (client *SearchClient) filterUncontactedNodes(input []*Node, limit int) (output []*Node) { - client.contactedNodesMutex.Lock() - - for _, node := range input { - if _, ok := client.contactedNodesMap[string(node.ID)]; !ok { - output = append(output, node) - client.contactedNodesMap[string(node.ID)] = struct{}{} - - if limit > 0 { - limit-- - if limit == 0 { - break - } - } - } - } - - client.contactedNodesMutex.Unlock() - return output -} - -// SearchAway starts the search. Non-blocking! -func (client *SearchClient) SearchAway() { - client.timeStart = time.Now() - - // create the first search level and start it - client.list = client.dht.ht.getClosestContacts(client.alpha, client.Key, nil) - if len(client.list.Nodes) == 0 { - client.Terminate() - return - } - - go client.queryNodesKnownStore() - - // start the first information request - go client.startSearch(0) - - // start an automated termination function for the timeout - go func(client *SearchClient) { - // sleep + watch for closing - select { - case <-client.TerminateSignal: // exit the function on other signal - return - case <-time.After(client.timeoutTotal): - client.Terminate() - } - }(client) -} - -// sendInfoRequest sends out a new info request to the nodes -func (client *SearchClient) sendInfoRequest(nodes []*Node, resultChan chan *NodeMessage) (info *InformationRequest) { - if client.IsTerminated { - return nil - } - - for _, node := range nodes { - client.LogStatus("search.sendInfoRequest", "contact node %s\n", hex.EncodeToString(node.ID)) - } - - info = client.dht.NewInformationRequest(client.Action, client.Key, nodes) - info.ResultChanExt = resultChan - - switch client.Action { - case ActionFindNode: - client.dht.SendRequestFindNode(info) - case ActionFindValue: - client.dht.SendRequestFindValue(info) - } - - go func() { - select { - case <-client.TerminateSignal: - case <-time.After(client.timeoutIR): - } - info.Terminate() - }() - - return info -} - -// queryNodesKnownStore queries nodes that are known to store the value. Only for ActionFindValue. -// Returned 'closest nodes' are ignored, as the queried nodes are expected to store the value. This might be adjusted in the future. -func (client *SearchClient) queryNodesKnownStore() { - // all results are redirected to a single channel - resultChan := make(chan *NodeMessage, client.alpha) - - for { - select { - case <-client.TerminateSignal: - return - - case nodes := <-client.storing: - client.sendInfoRequest(nodes, resultChan) - - case result := <-resultChan: - if len(result.Data) > 0 { - client.Results <- &SearchResult{Key: client.Key, Action: client.Action, SenderID: result.SenderID, Data: result.Data} - client.Terminate() - return - } - } - } -} - -func (client *SearchClient) startSearch(level int) { - atomic.AddUint64(&client.activeLevels, 1) - defer atomic.AddUint64(&client.activeLevels, ^uint64(0)) - nestedStarted := false - - results := make(chan *NodeMessage, client.alpha*2) - - closestNode := client.list.Nodes[0] - - // start an info request - startInfoRequest := func() (info *InformationRequest) { - nodes := client.list.GetUncontacted(client.alpha, true) - if len(nodes) == 0 { - client.LogStatus("search.startSearch", "search in level %d aborted, no new nodes to contact\n", level) - return nil - } - client.LogStatus("search.startSearch", "start search in level %d contacting %d nodes\n", level, len(nodes)) - return client.sendInfoRequest(nodes, results) - } - - info := startInfoRequest() - if info == nil { - return - } - - for { - select { - case <-client.TerminateSignal: - client.LogStatus("search.startSearch", "search in level %d aborted, search client termination signal\n", level) - return - - case result := <-results: - - switch client.Action { - case ActionFindValue: - // search for value and it was found? - if len(result.Data) > 0 { - client.LogStatus("search.startSearch", "result: sender %s: data found (%d bytes)\n", hex.EncodeToString(result.SenderID), len(result.Data)) - client.Results <- &SearchResult{Key: client.Key, Action: client.Action, SenderID: result.SenderID, Data: result.Data} - client.Terminate() - return - } - - result.Storing = client.filterUncontactedNodes(result.Storing, MaxAcceptKnownStore) - result.Closest = client.filterUncontactedNodes(result.Closest, MaxClosest) - - client.LogStatus("search.startSearch", "result: sender %s: %d uncontacted nodes store and %d nodes are close to value\n", hex.EncodeToString(result.SenderID), len(result.Storing), len(result.Closest)) - - // Find value: Nodes known to store the value are queried in a separate function. - if len(result.Storing) > 0 { - client.storing <- result.Storing - } - - case ActionFindNode: - // search for node and it was found? - for _, closePeer := range result.Closest { - if bytes.Equal(closePeer.ID, client.Key) { - client.LogStatus("search.startSearch", "result: sender %s: node found!\n", hex.EncodeToString(result.SenderID)) - client.Results <- &SearchResult{Key: client.Key, Action: client.Action, SenderID: result.SenderID, TargetNode: closePeer} - client.Terminate() - return - } - } - - result.Closest = client.filterUncontactedNodes(result.Closest, MaxClosest) - - client.LogStatus("search.startSearch", "find node: sender %s: %d nodes are close to value\n", hex.EncodeToString(result.SenderID), len(result.Closest)) - - } - - // Add closest to list - client.list.AppendUniqueNodes(result.Closest...) - - // If no subsequent level, and there's closer nodes, start one! - if !nestedStarted && !bytes.Equal(client.list.Nodes[0].ID, closestNode.ID) && level < MaxLevel { - nestedStarted = true - go client.startSearch(level + 1) - } - - case <-info.TerminateSignal: - // If highest level (= not nested), and there was no conclusive result, try one more round. - // This helps against result poisoning. - - if !nestedStarted { - client.LogStatus("search.startSearch", "search in level %d aborted, info request termination signal. Final try.\n", level) - if info = startInfoRequest(); info != nil { - continue - } - } - - if client.activeLevels == 1 { // if this was the last level, no more results will appear - client.LogStatus("search.startSearch", "level %d last active level, not found, terminate search\n", level) - client.Terminate() - } else { - client.LogStatus("search.startSearch", "level %d end, info request termination signal\n", level) - } - return - - //case <-time.After(time.Second): - // Future todo: Launch another routine with the with uncontacted nodes if any, to speed up the query - } - } -} +/* +File Name: Search Client.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +A search client runs concurrent information requests for a single query. It solves the query efficiently by using levels. +Any result that is closer to the target gets pushed down into a new lower level, which contacts nodes closer to the result. +Level are running concurrently. +*/ + +package dht + +import ( + "bytes" + "encoding/hex" + "sync" + "sync/atomic" + "time" +) + +// MaxAcceptKnownStore is maximum count accepted of known peers that store the value +const MaxAcceptKnownStore = 10 + +// MaxClosest is maximum number of closest peers accepted +const MaxClosest = 3 + +// MaxLevel defines the max level. +const MaxLevel = 10 + +// SearchClient defines a search in the distributed hash table involving multiple information requests. +// The search can be created for a node or for a value, both identified by the hash (known as the key). +type SearchClient struct { + Action int // ActionX + Key []byte // Key that is being queried + IsTerminated bool // If true, it was signaled for termination + TerminateSignal chan struct{} // gets closed on termination signal, can be used in select via "case _ = <- TerminateSignal:" + sync.Mutex // for sychronized closing + timeStart time.Time // When the search started. + timeEnd time.Time // When the search ended. + dht *DHT // DHT used + timeoutTotal time.Duration // Timeout after the entire search will be terminated client-side. + timeoutIR time.Duration // Timeout for information requests (entire roundtrip). + alpha int // Count of concurrent information requests per level. + Results chan *SearchResult // Result channel + list *shortList // List of nodes to contact + contactedNodesMap map[string]struct{} // List of nodes already contacted + contactedNodesMutex sync.RWMutex // Sync map access + storing chan []*Node // Internal channel to signal nodes that indicate storing the searched value. + activeLevels uint64 // demo + LogStatus func(function, format string, v ...interface{}) // Filter function for status output +} + +// SearchResult is a single result to the search. Depending on the search type and parameters, multiple results may be sent. +type SearchResult struct { + Key []byte // Original key that was searched for + Action int // Original action + SenderID []byte // Sender node ID of the result + + // data for ActionFindNode + TargetNode *Node // The node that was requested. + + // data for ActionFindValue + Data []byte // Actual data +} + +// NewSearch creates a new search client. +// Action indicates the action to take (from ActionX constants), to either find a node, or a value. +// Timeout is the total time the search may take, covering all information requests. TimeoutIR is the time an information request may take. +// Alpha is the number of concurrent requests that will be performed. +func (dht *DHT) NewSearch(Action int, Key []byte, Timeout, TimeoutIR time.Duration, Alpha int) (client *SearchClient) { + client = &SearchClient{ + Action: Action, + Key: Key, + dht: dht, + timeoutTotal: Timeout, + timeoutIR: TimeoutIR, + alpha: Alpha, + contactedNodesMap: make(map[string]struct{}), + storing: make(chan []*Node, Alpha*2), + TerminateSignal: make(chan struct{}), + Results: make(chan *SearchResult), + LogStatus: func(function, format string, v ...interface{}) {}, + } + + return +} + +// Terminate sends the termination signal to all workers. It is safe to call Terminate multiple times. +func (client *SearchClient) Terminate() { + client.Lock() + defer client.Unlock() + + if client.IsTerminated { + return + } + + // set the termination signal + client.IsTerminated = true + close(client.TerminateSignal) // safety guaranteed via lock + + client.timeEnd = time.Now() + + close(client.Results) + close(client.storing) +} + +// isContactedNode checks if a node was contacted +func (client *SearchClient) isContactedNode(ID []byte, Set bool) (contacted bool) { + client.contactedNodesMutex.Lock() + _, contacted = client.contactedNodesMap[string(ID)] + if Set { + client.contactedNodesMap[string(ID)] = struct{}{} + } + client.contactedNodesMutex.Unlock() + return contacted +} + +// filterUncontactedNodes returns only nodes that were not contacted so far. All nodes will be set to contacted. Limit is optional (0 for no limit). +func (client *SearchClient) filterUncontactedNodes(input []*Node, limit int) (output []*Node) { + client.contactedNodesMutex.Lock() + + for _, node := range input { + if _, ok := client.contactedNodesMap[string(node.ID)]; !ok { + output = append(output, node) + client.contactedNodesMap[string(node.ID)] = struct{}{} + + if limit > 0 { + limit-- + if limit == 0 { + break + } + } + } + } + + client.contactedNodesMutex.Unlock() + return output +} + +// SearchAway starts the search. Non-blocking! +func (client *SearchClient) SearchAway() { + client.timeStart = time.Now() + + // create the first search level and start it + client.list = client.dht.ht.getClosestContacts(client.alpha, client.Key, nil) + if len(client.list.Nodes) == 0 { + client.Terminate() + return + } + + go client.queryNodesKnownStore() + + // start the first information request + go client.startSearch(0) + + // start an automated termination function for the timeout + go func(client *SearchClient) { + // sleep + watch for closing + select { + case <-client.TerminateSignal: // exit the function on other signal + return + case <-time.After(client.timeoutTotal): + client.Terminate() + } + }(client) +} + +// sendInfoRequest sends out a new info request to the nodes +func (client *SearchClient) sendInfoRequest(nodes []*Node, resultChan chan *NodeMessage) (info *InformationRequest) { + if client.IsTerminated { + return nil + } + + for _, node := range nodes { + client.LogStatus("search.sendInfoRequest", "contact node %s\n", hex.EncodeToString(node.ID)) + } + + info = client.dht.NewInformationRequest(client.Action, client.Key, nodes) + info.ResultChanExt = resultChan + + switch client.Action { + case ActionFindNode: + client.dht.SendRequestFindNode(info) + case ActionFindValue: + client.dht.SendRequestFindValue(info) + } + + go func() { + select { + case <-client.TerminateSignal: + case <-time.After(client.timeoutIR): + } + info.Terminate() + }() + + return info +} + +// queryNodesKnownStore queries nodes that are known to store the value. Only for ActionFindValue. +// Returned 'closest nodes' are ignored, as the queried nodes are expected to store the value. This might be adjusted in the future. +func (client *SearchClient) queryNodesKnownStore() { + // all results are redirected to a single channel + resultChan := make(chan *NodeMessage, client.alpha) + + for { + select { + case <-client.TerminateSignal: + return + + case nodes := <-client.storing: + client.sendInfoRequest(nodes, resultChan) + + case result := <-resultChan: + if len(result.Data) > 0 { + client.Results <- &SearchResult{Key: client.Key, Action: client.Action, SenderID: result.SenderID, Data: result.Data} + client.Terminate() + return + } + } + } +} + +func (client *SearchClient) startSearch(level int) { + atomic.AddUint64(&client.activeLevels, 1) + defer atomic.AddUint64(&client.activeLevels, ^uint64(0)) + nestedStarted := false + + results := make(chan *NodeMessage, client.alpha*2) + + closestNode := client.list.Nodes[0] + + // start an info request + startInfoRequest := func() (info *InformationRequest) { + nodes := client.list.GetUncontacted(client.alpha, true) + if len(nodes) == 0 { + client.LogStatus("search.startSearch", "search in level %d aborted, no new nodes to contact\n", level) + return nil + } + client.LogStatus("search.startSearch", "start search in level %d contacting %d nodes\n", level, len(nodes)) + return client.sendInfoRequest(nodes, results) + } + + info := startInfoRequest() + if info == nil { + return + } + + for { + select { + case <-client.TerminateSignal: + client.LogStatus("search.startSearch", "search in level %d aborted, search client termination signal\n", level) + return + + case result := <-results: + + switch client.Action { + case ActionFindValue: + // search for value and it was found? + if len(result.Data) > 0 { + client.LogStatus("search.startSearch", "result: sender %s: data found (%d bytes)\n", hex.EncodeToString(result.SenderID), len(result.Data)) + client.Results <- &SearchResult{Key: client.Key, Action: client.Action, SenderID: result.SenderID, Data: result.Data} + client.Terminate() + return + } + + result.Storing = client.filterUncontactedNodes(result.Storing, MaxAcceptKnownStore) + result.Closest = client.filterUncontactedNodes(result.Closest, MaxClosest) + + client.LogStatus("search.startSearch", "result: sender %s: %d uncontacted nodes store and %d nodes are close to value\n", hex.EncodeToString(result.SenderID), len(result.Storing), len(result.Closest)) + + // Find value: Nodes known to store the value are queried in a separate function. + if len(result.Storing) > 0 { + client.storing <- result.Storing + } + + case ActionFindNode: + // search for node and it was found? + for _, closePeer := range result.Closest { + if bytes.Equal(closePeer.ID, client.Key) { + client.LogStatus("search.startSearch", "result: sender %s: node found!\n", hex.EncodeToString(result.SenderID)) + client.Results <- &SearchResult{Key: client.Key, Action: client.Action, SenderID: result.SenderID, TargetNode: closePeer} + client.Terminate() + return + } + } + + result.Closest = client.filterUncontactedNodes(result.Closest, MaxClosest) + + client.LogStatus("search.startSearch", "find node: sender %s: %d nodes are close to value\n", hex.EncodeToString(result.SenderID), len(result.Closest)) + + } + + // Add closest to list + client.list.AppendUniqueNodes(result.Closest...) + + // If no subsequent level, and there's closer nodes, start one! + if !nestedStarted && !bytes.Equal(client.list.Nodes[0].ID, closestNode.ID) && level < MaxLevel { + nestedStarted = true + go client.startSearch(level + 1) + } + + case <-info.TerminateSignal: + // If highest level (= not nested), and there was no conclusive result, try one more round. + // This helps against result poisoning. + + if !nestedStarted { + client.LogStatus("search.startSearch", "search in level %d aborted, info request termination signal. Final try.\n", level) + if info = startInfoRequest(); info != nil { + continue + } + } + + if client.activeLevels == 1 { // if this was the last level, no more results will appear + client.LogStatus("search.startSearch", "level %d last active level, not found, terminate search\n", level) + client.Terminate() + } else { + client.LogStatus("search.startSearch", "level %d end, info request termination signal\n", level) + } + return + + //case <-time.After(time.Second): + // Future todo: Launch another routine with the with uncontacted nodes if any, to speed up the query + } + } +} diff --git a/merkle/Fragment.go b/merkle/Fragment.go index b87d96c..ca57b96 100644 --- a/merkle/Fragment.go +++ b/merkle/Fragment.go @@ -1,57 +1,57 @@ -/* -File Name: Fragment.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package merkle - -// MinimumFragmentSize is the minimum size a fragment must be. Merkle trees are not used equal or below that size. -const MinimumFragmentSize = 256 * KB - -const KB = 1024 -const MB = 1024 * KB -const GB = 1024 * MB -const TB = 1024 * GB -const PB = 1024 * TB - -// CalculateFragmentSize calculates the fragment size based on the file size. -func CalculateFragmentSize(fileSize uint64) (fragmentSize uint64) { - switch { - case fileSize <= 256*MB: - return 256 * KB - - case fileSize <= 1*GB: - return 512 * KB - - case fileSize <= 2*GB: - return 1 * MB - - case fileSize <= 4*GB: - return 2 * MB - - case fileSize <= 8*GB: - return 8 * MB - - case fileSize <= 16*GB: - return 16 * MB - - case fileSize <= 32*GB: - return 32 * MB - - case fileSize <= 64*GB: - return 64 * MB - - case fileSize <= 1*TB: - return 64 * MB - - case fileSize <= 2*TB: - return 128 * MB - - case fileSize <= 1*PB: - return 512 * MB - - default: // 1 PB+ - return 1 * GB - } -} +/* +File Name: Fragment.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package merkle + +// MinimumFragmentSize is the minimum size a fragment must be. Merkle trees are not used equal or below that size. +const MinimumFragmentSize = 256 * KB + +const KB = 1024 +const MB = 1024 * KB +const GB = 1024 * MB +const TB = 1024 * GB +const PB = 1024 * TB + +// CalculateFragmentSize calculates the fragment size based on the file size. +func CalculateFragmentSize(fileSize uint64) (fragmentSize uint64) { + switch { + case fileSize <= 256*MB: + return 256 * KB + + case fileSize <= 1*GB: + return 512 * KB + + case fileSize <= 2*GB: + return 1 * MB + + case fileSize <= 4*GB: + return 2 * MB + + case fileSize <= 8*GB: + return 8 * MB + + case fileSize <= 16*GB: + return 16 * MB + + case fileSize <= 32*GB: + return 32 * MB + + case fileSize <= 64*GB: + return 64 * MB + + case fileSize <= 1*TB: + return 64 * MB + + case fileSize <= 2*TB: + return 128 * MB + + case fileSize <= 1*PB: + return 512 * MB + + default: // 1 PB+ + return 1 * GB + } +} diff --git a/merkle/Merkle Tree.go b/merkle/Merkle Tree.go index c5dd74b..b657e9c 100644 --- a/merkle/Merkle Tree.go +++ b/merkle/Merkle Tree.go @@ -1,311 +1,311 @@ -/* -File Name: Merkle Tree.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Generates the merkle tree based on input data. -In case of uneven number of fragments, the last uneven fragment is moved up a level. -*/ - -package merkle - -import ( - "bytes" - "encoding/binary" - "errors" - "io" - - "lukechampine.com/blake3" -) - -// MerkleTree represents an entire merkle tree -type MerkleTree struct { - // information about the original file - FileSize uint64 - FragmentSize uint64 - FragmentCount uint64 - - // list of hashes - FragmentHashes [][]byte // List of hashes for each fragment - MiddleHashes [][][]byte // All hashes in the middle, bottom up. - RootHash []byte // Root hash. -} - -// NewMerkleTree creates a new merkle tree from the input -func NewMerkleTree(fileSize, fragmentSize uint64, reader io.Reader) (tree *MerkleTree, err error) { - if fragmentSize == 0 { - return nil, errors.New("invalid fragment size") - } - - tree = &MerkleTree{ - FileSize: fileSize, - FragmentSize: fragmentSize, - FragmentCount: fileSizeToFragmentCount(fileSize, fragmentSize), - } - - // Special case: No fragments, in case of empty data. - if tree.FragmentCount == 0 { - hash := blake3.Sum256(nil) - tree.RootHash = hash[:] - - return tree, nil - } else if tree.FragmentCount == 1 { - // Special case: Single fragment. - data := make([]byte, fileSize) - if _, err := io.ReadAtLeast(reader, data, int(fileSize)); err != nil { - return nil, err - } - - hash := blake3.Sum256(data) - tree.RootHash = hash[:] - - return tree, nil - } - - // calculate the hash per fragment - data := make([]byte, fragmentSize) - remaining := fileSize - - for n := uint64(0); n < tree.FragmentCount; n++ { - if fragmentSize > remaining { - fragmentSize = remaining - } - - if _, err := io.ReadAtLeast(reader, data, int(fragmentSize)); err != nil { - return nil, err - } - - // hash the fragment - hash := blake3.Sum256(data[:fragmentSize]) - - tree.FragmentHashes = append(tree.FragmentHashes, hash[:]) - - remaining -= fragmentSize - } - - // calculate the intermediate hashes - tree.calculateMiddleHashes(0) - - return tree, nil -} - -func fileSizeToFragmentCount(fileSize, fragmentSize uint64) (count uint64) { - return (fileSize + fragmentSize - 1) / fragmentSize -} - -func (tree *MerkleTree) calculateMiddleHashes(level uint64) { - if len(tree.FragmentHashes) == 0 { - return - } - - var newHashes, inputHashes [][]byte - - if level == 0 { - inputHashes = tree.FragmentHashes - } else { - inputHashes = tree.MiddleHashes[level-1] - } - - for n := 0; n+1 <= len(inputHashes)-1; n += 2 { - newHashes = append(newHashes, calculateMiddleHash(inputHashes[n], inputHashes[n+1])) - } - - // Uneven leafs? in this case the new hash is just a copy of the uneven one. No point in artifically recalcualting it with itself like Bitcoin does. - // For other possible implementations see https://medium.com/coinmonks/merkle-trees-concepts-and-use-cases-5da873702318. - if len(inputHashes)%2 != 0 { - newHashes = append(newHashes, inputHashes[len(inputHashes)-1]) - } - - if len(newHashes) == 1 { - // Only one hash generated. - tree.RootHash = newHashes[0] - } else if len(newHashes) > 1 { - tree.MiddleHashes = append(tree.MiddleHashes, newHashes) - - tree.calculateMiddleHashes(level + 1) - } -} - -func calculateMiddleHash(hash1 []byte, hash2 []byte) (newHash []byte) { - var data []byte - data = append(data, hash1...) - data = append(data, hash2...) - - hash := blake3.Sum256(data) - - return hash[:] -} - -// CreateVerification returns the verification hashes for the given fragment number. The root hash itself is not included. -// The result might be empty if there is no or a single fragment. -// Each verification hash has a preceding left (= 0)/right (= 1) indicator that indicates where the verification is positioned. -// This makes the algorithm future proof, in case uneven leafs will be handled differently. -func (tree *MerkleTree) CreateVerification(fragment uint64) (verificationHashes [][]byte) { - // 0 fragments: Empty data. - // 1 fragment: The hash of the fragment is the root hash. - if tree.FragmentCount <= 1 { - return nil - } else if fragment >= tree.FragmentCount { - // invalid fragment index - return nil - } - - // first hash it he neighbor fragment hash, if available - if fragment == tree.FragmentCount-1 && fragment%2 == 0 { - } else if fragment%2 == 0 { - verificationHashes = append(verificationHashes, append([]byte{1}, tree.FragmentHashes[fragment+1]...)) - } else { - verificationHashes = append(verificationHashes, append([]byte{0}, tree.FragmentHashes[fragment-1]...)) - } - - // go through all middle hash levels - for n := 0; n < len(tree.MiddleHashes); n++ { - fragment = fragment / 2 - - if fragment == uint64(len(tree.MiddleHashes[n])-1) && fragment%2 == 0 { - } else if fragment%2 == 0 { - verificationHashes = append(verificationHashes, append([]byte{1}, tree.MiddleHashes[n][fragment+1]...)) - } else { - verificationHashes = append(verificationHashes, append([]byte{0}, tree.MiddleHashes[n][fragment-1]...)) - } - } - - return -} - -// MerkleVerify validates the hashed data against the verification hashes and the known root hash. -func MerkleVerify(rootHash []byte, dataHash []byte, verificationHashes [][]byte) (valid bool) { - for _, verifyHash := range verificationHashes { - if verifyHash[0] == 0 { - dataHash = calculateMiddleHash(verifyHash[1:], dataHash) - } else { - dataHash = calculateMiddleHash(dataHash, verifyHash[1:]) - } - } - - return bytes.Equal(rootHash, dataHash) -} - -/* -Export/Import of the merkle tree structure: - -Offset Size Info -0 8 File Size -8 8 Fragment Size -16 32 Merkle Root Hash -48 32 * n Fragment Hashes -? 32 * n Middle Hashes - -*/ - -const MerkleTreeFileHeaderSize = 8 + 8 + 32 - -// calculateTotalHashCount returns the total number of fragment and middle hashes needed for the given count of fragments -func calculateTotalHashCount(fragmentCount uint64) (count uint64) { - // Special case no or 1 fragment: None needed, since the fragment hash is directly stored as root hash. - if fragmentCount <= 1 { - return 0 - } - - // Equal count of fragment hashes needed - count = fragmentCount - - // Calculate middle hashes number - for countHashesLast := fragmentCount; ; { - countMiddleNew := (countHashesLast + 1) / 2 // round up - if countMiddleNew <= 1 { - break - } - - count += countMiddleNew - countHashesLast = countMiddleNew - } - - return count -} - -// Export stores the tree as blob -func (tree *MerkleTree) Export() (data []byte) { - data = make([]byte, MerkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32) - - // header - binary.LittleEndian.PutUint64(data[0:8], tree.FileSize) - binary.LittleEndian.PutUint64(data[8:16], tree.FragmentSize) - copy(data[16:16+32], tree.RootHash) - - // fragment hashes - offset := 48 - for _, hash := range tree.FragmentHashes { - copy(data[offset:offset+32], hash) - offset += 32 - } - - // middle hashes - for n := 0; n < len(tree.MiddleHashes); n++ { - for _, hash := range tree.MiddleHashes[n] { - copy(data[offset:offset+32], hash) - offset += 32 - } - } - - return data[:offset] -} - -// Import reads the tree from the input data -func ImportMerkleTree(data []byte) (tree *MerkleTree) { - if tree = ReadMerkleTreeHeader(data); tree == nil || tree.FragmentCount <= 1 { - return tree - } - - // verify size - if uint64(len(data)) < MerkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32 { - return nil - } - - // fragment hashes - offset := 48 - for n := 0; n < int(tree.FragmentCount); n++ { - hash := data[offset : offset+32] - tree.FragmentHashes = append(tree.FragmentHashes, hash) - offset += 32 - } - - // middle hashes - n := tree.FragmentCount / 2 - if tree.FragmentCount > 2 && tree.FragmentCount%2 != 0 { - n++ - } - - for ; n > 1; n = n / 2 { - var hashList [][]byte - for m := uint64(0); m < n; m++ { - hash := data[offset : offset+32] - hashList = append(hashList, hash) - offset += 32 - } - - tree.MiddleHashes = append(tree.MiddleHashes, hashList) - if len(hashList)%2 != 0 { - n++ - } - } - - return -} - -// ReadMerkleTreeHeader reads the merkle tree header. Fragment and middle hashes are not loaded. -func ReadMerkleTreeHeader(data []byte) (tree *MerkleTree) { - // Read the header. Enforce the minimum size. - if len(data) < MerkleTreeFileHeaderSize { - return nil - } - - tree = &MerkleTree{ - FileSize: binary.LittleEndian.Uint64(data[0:8]), - FragmentSize: binary.LittleEndian.Uint64(data[8:16]), - } - tree.FragmentCount = fileSizeToFragmentCount(tree.FileSize, tree.FragmentSize) - tree.RootHash = data[16 : 16+32] - - return tree -} +/* +File Name: Merkle Tree.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Generates the merkle tree based on input data. +In case of uneven number of fragments, the last uneven fragment is moved up a level. +*/ + +package merkle + +import ( + "bytes" + "encoding/binary" + "errors" + "io" + + "lukechampine.com/blake3" +) + +// MerkleTree represents an entire merkle tree +type MerkleTree struct { + // information about the original file + FileSize uint64 + FragmentSize uint64 + FragmentCount uint64 + + // list of hashes + FragmentHashes [][]byte // List of hashes for each fragment + MiddleHashes [][][]byte // All hashes in the middle, bottom up. + RootHash []byte // Root hash. +} + +// NewMerkleTree creates a new merkle tree from the input +func NewMerkleTree(fileSize, fragmentSize uint64, reader io.Reader) (tree *MerkleTree, err error) { + if fragmentSize == 0 { + return nil, errors.New("invalid fragment size") + } + + tree = &MerkleTree{ + FileSize: fileSize, + FragmentSize: fragmentSize, + FragmentCount: fileSizeToFragmentCount(fileSize, fragmentSize), + } + + // Special case: No fragments, in case of empty data. + if tree.FragmentCount == 0 { + hash := blake3.Sum256(nil) + tree.RootHash = hash[:] + + return tree, nil + } else if tree.FragmentCount == 1 { + // Special case: Single fragment. + data := make([]byte, fileSize) + if _, err := io.ReadAtLeast(reader, data, int(fileSize)); err != nil { + return nil, err + } + + hash := blake3.Sum256(data) + tree.RootHash = hash[:] + + return tree, nil + } + + // calculate the hash per fragment + data := make([]byte, fragmentSize) + remaining := fileSize + + for n := uint64(0); n < tree.FragmentCount; n++ { + if fragmentSize > remaining { + fragmentSize = remaining + } + + if _, err := io.ReadAtLeast(reader, data, int(fragmentSize)); err != nil { + return nil, err + } + + // hash the fragment + hash := blake3.Sum256(data[:fragmentSize]) + + tree.FragmentHashes = append(tree.FragmentHashes, hash[:]) + + remaining -= fragmentSize + } + + // calculate the intermediate hashes + tree.calculateMiddleHashes(0) + + return tree, nil +} + +func fileSizeToFragmentCount(fileSize, fragmentSize uint64) (count uint64) { + return (fileSize + fragmentSize - 1) / fragmentSize +} + +func (tree *MerkleTree) calculateMiddleHashes(level uint64) { + if len(tree.FragmentHashes) == 0 { + return + } + + var newHashes, inputHashes [][]byte + + if level == 0 { + inputHashes = tree.FragmentHashes + } else { + inputHashes = tree.MiddleHashes[level-1] + } + + for n := 0; n+1 <= len(inputHashes)-1; n += 2 { + newHashes = append(newHashes, calculateMiddleHash(inputHashes[n], inputHashes[n+1])) + } + + // Uneven leafs? in this case the new hash is just a copy of the uneven one. No point in artifically recalcualting it with itself like Bitcoin does. + // For other possible implementations see https://medium.com/coinmonks/merkle-trees-concepts-and-use-cases-5da873702318. + if len(inputHashes)%2 != 0 { + newHashes = append(newHashes, inputHashes[len(inputHashes)-1]) + } + + if len(newHashes) == 1 { + // Only one hash generated. + tree.RootHash = newHashes[0] + } else if len(newHashes) > 1 { + tree.MiddleHashes = append(tree.MiddleHashes, newHashes) + + tree.calculateMiddleHashes(level + 1) + } +} + +func calculateMiddleHash(hash1 []byte, hash2 []byte) (newHash []byte) { + var data []byte + data = append(data, hash1...) + data = append(data, hash2...) + + hash := blake3.Sum256(data) + + return hash[:] +} + +// CreateVerification returns the verification hashes for the given fragment number. The root hash itself is not included. +// The result might be empty if there is no or a single fragment. +// Each verification hash has a preceding left (= 0)/right (= 1) indicator that indicates where the verification is positioned. +// This makes the algorithm future proof, in case uneven leafs will be handled differently. +func (tree *MerkleTree) CreateVerification(fragment uint64) (verificationHashes [][]byte) { + // 0 fragments: Empty data. + // 1 fragment: The hash of the fragment is the root hash. + if tree.FragmentCount <= 1 { + return nil + } else if fragment >= tree.FragmentCount { + // invalid fragment index + return nil + } + + // first hash it he neighbor fragment hash, if available + if fragment == tree.FragmentCount-1 && fragment%2 == 0 { + } else if fragment%2 == 0 { + verificationHashes = append(verificationHashes, append([]byte{1}, tree.FragmentHashes[fragment+1]...)) + } else { + verificationHashes = append(verificationHashes, append([]byte{0}, tree.FragmentHashes[fragment-1]...)) + } + + // go through all middle hash levels + for n := 0; n < len(tree.MiddleHashes); n++ { + fragment = fragment / 2 + + if fragment == uint64(len(tree.MiddleHashes[n])-1) && fragment%2 == 0 { + } else if fragment%2 == 0 { + verificationHashes = append(verificationHashes, append([]byte{1}, tree.MiddleHashes[n][fragment+1]...)) + } else { + verificationHashes = append(verificationHashes, append([]byte{0}, tree.MiddleHashes[n][fragment-1]...)) + } + } + + return +} + +// MerkleVerify validates the hashed data against the verification hashes and the known root hash. +func MerkleVerify(rootHash []byte, dataHash []byte, verificationHashes [][]byte) (valid bool) { + for _, verifyHash := range verificationHashes { + if verifyHash[0] == 0 { + dataHash = calculateMiddleHash(verifyHash[1:], dataHash) + } else { + dataHash = calculateMiddleHash(dataHash, verifyHash[1:]) + } + } + + return bytes.Equal(rootHash, dataHash) +} + +/* +Export/Import of the merkle tree structure: + +Offset Size Info +0 8 File Size +8 8 Fragment Size +16 32 Merkle Root Hash +48 32 * n Fragment Hashes +? 32 * n Middle Hashes + +*/ + +const MerkleTreeFileHeaderSize = 8 + 8 + 32 + +// calculateTotalHashCount returns the total number of fragment and middle hashes needed for the given count of fragments +func calculateTotalHashCount(fragmentCount uint64) (count uint64) { + // Special case no or 1 fragment: None needed, since the fragment hash is directly stored as root hash. + if fragmentCount <= 1 { + return 0 + } + + // Equal count of fragment hashes needed + count = fragmentCount + + // Calculate middle hashes number + for countHashesLast := fragmentCount; ; { + countMiddleNew := (countHashesLast + 1) / 2 // round up + if countMiddleNew <= 1 { + break + } + + count += countMiddleNew + countHashesLast = countMiddleNew + } + + return count +} + +// Export stores the tree as blob +func (tree *MerkleTree) Export() (data []byte) { + data = make([]byte, MerkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32) + + // header + binary.LittleEndian.PutUint64(data[0:8], tree.FileSize) + binary.LittleEndian.PutUint64(data[8:16], tree.FragmentSize) + copy(data[16:16+32], tree.RootHash) + + // fragment hashes + offset := 48 + for _, hash := range tree.FragmentHashes { + copy(data[offset:offset+32], hash) + offset += 32 + } + + // middle hashes + for n := 0; n < len(tree.MiddleHashes); n++ { + for _, hash := range tree.MiddleHashes[n] { + copy(data[offset:offset+32], hash) + offset += 32 + } + } + + return data[:offset] +} + +// Import reads the tree from the input data +func ImportMerkleTree(data []byte) (tree *MerkleTree) { + if tree = ReadMerkleTreeHeader(data); tree == nil || tree.FragmentCount <= 1 { + return tree + } + + // verify size + if uint64(len(data)) < MerkleTreeFileHeaderSize+calculateTotalHashCount(tree.FragmentCount)*32 { + return nil + } + + // fragment hashes + offset := 48 + for n := 0; n < int(tree.FragmentCount); n++ { + hash := data[offset : offset+32] + tree.FragmentHashes = append(tree.FragmentHashes, hash) + offset += 32 + } + + // middle hashes + n := tree.FragmentCount / 2 + if tree.FragmentCount > 2 && tree.FragmentCount%2 != 0 { + n++ + } + + for ; n > 1; n = n / 2 { + var hashList [][]byte + for m := uint64(0); m < n; m++ { + hash := data[offset : offset+32] + hashList = append(hashList, hash) + offset += 32 + } + + tree.MiddleHashes = append(tree.MiddleHashes, hashList) + if len(hashList)%2 != 0 { + n++ + } + } + + return +} + +// ReadMerkleTreeHeader reads the merkle tree header. Fragment and middle hashes are not loaded. +func ReadMerkleTreeHeader(data []byte) (tree *MerkleTree) { + // Read the header. Enforce the minimum size. + if len(data) < MerkleTreeFileHeaderSize { + return nil + } + + tree = &MerkleTree{ + FileSize: binary.LittleEndian.Uint64(data[0:8]), + FragmentSize: binary.LittleEndian.Uint64(data[8:16]), + } + tree.FragmentCount = fileSizeToFragmentCount(tree.FileSize, tree.FragmentSize) + tree.RootHash = data[16 : 16+32] + + return tree +} diff --git a/merkle/Test_test.go b/merkle/Test_test.go index 171d37c..4b8a471 100644 --- a/merkle/Test_test.go +++ b/merkle/Test_test.go @@ -1,139 +1,139 @@ -package merkle - -import ( - "bytes" - "crypto/rand" - "encoding/hex" - "fmt" - "io" - "testing" - - "lukechampine.com/blake3" -) - -func TestFragment0(t *testing.T) { - dataSize := uint64(11*1024*1024 + 100) - - data := make([]byte, dataSize) - - if _, err := io.ReadFull(rand.Reader, data); err != nil { - return - } - - fragmentSize := CalculateFragmentSize(dataSize) - - tree, err := NewMerkleTree(dataSize, fragmentSize, bytes.NewBuffer(data)) - - if err != nil { - fmt.Printf("Error creating merkle tree: %v\n", err) - return - } - - printMerkleTree(tree) - - // Validate all hashes. - for n := uint64(0); n < tree.FragmentCount; n++ { - verificationHashes := tree.CreateVerification(n) - - dataSize := tree.FragmentSize - if n == tree.FragmentCount-1 { - dataSize = tree.FileSize - n*tree.FragmentSize - } - dataHash := blake3.Sum256(data[n*tree.FragmentSize : n*tree.FragmentSize+dataSize]) - - valid := MerkleVerify(tree.RootHash, dataHash[:], verificationHashes) - - fmt.Printf("Validate fragment %d: %t\n", n, valid) - if !valid { - for m := 0; m < len(verificationHashes); m++ { - fmt.Printf("-> Middle hash [level %d]: %s\n", m-1, hex.EncodeToString(verificationHashes[m])) - } - } - } -} - -func printMerkleTree(tree *MerkleTree) { - fmt.Printf("File size: %d\n", tree.FileSize) - fmt.Printf("Fragment size: %d\n", tree.FragmentSize) - fmt.Printf("Fragment count: %d\n", tree.FragmentCount) - - fmt.Printf("Merkle root hash: %s\n", hex.EncodeToString(tree.RootHash)) - - for n := 0; n < len(tree.FragmentHashes); n++ { - fmt.Printf("Fragment %d: %s\n", n, hex.EncodeToString(tree.FragmentHashes[n])) - } - for n := 0; n < len(tree.MiddleHashes); n++ { - for m := 0; m < len(tree.MiddleHashes[n]); m++ { - fmt.Printf("Middle hash [level %d] %d: %s\n", n, m, hex.EncodeToString(tree.MiddleHashes[n][m])) - } - } -} - -func TestMerkleFileExport(t *testing.T) { - dataSize := uint64(11*1024*1024 + 100) - data := make([]byte, dataSize) - - if _, err := io.ReadFull(rand.Reader, data); err != nil { - return - } - - fragmentSize := CalculateFragmentSize(dataSize) - - tree, err := NewMerkleTree(dataSize, fragmentSize, bytes.NewBuffer(data)) - - if err != nil { - fmt.Printf("Error creating merkle tree: %v\n", err) - return - } - - printMerkleTree(tree) - - treeData := tree.Export() - - tree2 := ImportMerkleTree(treeData) - if tree2 == nil { - fmt.Printf("Error importing tree\n") - return - } - - printMerkleTree(tree2) - - // verify both trees - if tree.FileSize != tree2.FileSize || tree.FragmentSize != tree2.FragmentSize || tree.FragmentCount != tree2.FragmentCount { - fmt.Printf("Error: Header of trees mismatch\n") - return - } else if !bytes.Equal(tree.RootHash, tree2.RootHash) { - fmt.Printf("Error: Merkle root hash mismatch\n") - return - } else if len(tree.FragmentHashes) != len(tree2.FragmentHashes) { - fmt.Printf("Error: Fragment hashes mismatch count\n") - return - } else if len(tree.MiddleHashes) != len(tree2.MiddleHashes) { - fmt.Printf("Error: Middle hashes level mismatch\n") - return - } - - // fragment hashes and middle hashes - for n, hash := range tree.FragmentHashes { - if !bytes.Equal(hash, tree2.FragmentHashes[n]) { - fmt.Printf("Error: Fragment hash %d mismatch\n", n) - return - } - } - - for n := range tree.MiddleHashes { - if len(tree.MiddleHashes[n]) != len(tree2.MiddleHashes[n]) { - fmt.Printf("Error: Middle hashes level %d mismatch\n", n) - return - } - - for m, hash := range tree.MiddleHashes[n] { - if !bytes.Equal(hash, tree2.MiddleHashes[n][m]) { - fmt.Printf("Error: Middle hash %d %d mismatch\n", n, m) - return - } - } - } - - fmt.Printf("Success. Import/export match.\n") -} +package merkle + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "fmt" + "io" + "testing" + + "lukechampine.com/blake3" +) + +func TestFragment0(t *testing.T) { + dataSize := uint64(11*1024*1024 + 100) + + data := make([]byte, dataSize) + + if _, err := io.ReadFull(rand.Reader, data); err != nil { + return + } + + fragmentSize := CalculateFragmentSize(dataSize) + + tree, err := NewMerkleTree(dataSize, fragmentSize, bytes.NewBuffer(data)) + + if err != nil { + fmt.Printf("Error creating merkle tree: %v\n", err) + return + } + + printMerkleTree(tree) + + // Validate all hashes. + for n := uint64(0); n < tree.FragmentCount; n++ { + verificationHashes := tree.CreateVerification(n) + + dataSize := tree.FragmentSize + if n == tree.FragmentCount-1 { + dataSize = tree.FileSize - n*tree.FragmentSize + } + dataHash := blake3.Sum256(data[n*tree.FragmentSize : n*tree.FragmentSize+dataSize]) + + valid := MerkleVerify(tree.RootHash, dataHash[:], verificationHashes) + + fmt.Printf("Validate fragment %d: %t\n", n, valid) + if !valid { + for m := 0; m < len(verificationHashes); m++ { + fmt.Printf("-> Middle hash [level %d]: %s\n", m-1, hex.EncodeToString(verificationHashes[m])) + } + } + } +} + +func printMerkleTree(tree *MerkleTree) { + fmt.Printf("File size: %d\n", tree.FileSize) + fmt.Printf("Fragment size: %d\n", tree.FragmentSize) + fmt.Printf("Fragment count: %d\n", tree.FragmentCount) + + fmt.Printf("Merkle root hash: %s\n", hex.EncodeToString(tree.RootHash)) + + for n := 0; n < len(tree.FragmentHashes); n++ { + fmt.Printf("Fragment %d: %s\n", n, hex.EncodeToString(tree.FragmentHashes[n])) + } + for n := 0; n < len(tree.MiddleHashes); n++ { + for m := 0; m < len(tree.MiddleHashes[n]); m++ { + fmt.Printf("Middle hash [level %d] %d: %s\n", n, m, hex.EncodeToString(tree.MiddleHashes[n][m])) + } + } +} + +func TestMerkleFileExport(t *testing.T) { + dataSize := uint64(11*1024*1024 + 100) + data := make([]byte, dataSize) + + if _, err := io.ReadFull(rand.Reader, data); err != nil { + return + } + + fragmentSize := CalculateFragmentSize(dataSize) + + tree, err := NewMerkleTree(dataSize, fragmentSize, bytes.NewBuffer(data)) + + if err != nil { + fmt.Printf("Error creating merkle tree: %v\n", err) + return + } + + printMerkleTree(tree) + + treeData := tree.Export() + + tree2 := ImportMerkleTree(treeData) + if tree2 == nil { + fmt.Printf("Error importing tree\n") + return + } + + printMerkleTree(tree2) + + // verify both trees + if tree.FileSize != tree2.FileSize || tree.FragmentSize != tree2.FragmentSize || tree.FragmentCount != tree2.FragmentCount { + fmt.Printf("Error: Header of trees mismatch\n") + return + } else if !bytes.Equal(tree.RootHash, tree2.RootHash) { + fmt.Printf("Error: Merkle root hash mismatch\n") + return + } else if len(tree.FragmentHashes) != len(tree2.FragmentHashes) { + fmt.Printf("Error: Fragment hashes mismatch count\n") + return + } else if len(tree.MiddleHashes) != len(tree2.MiddleHashes) { + fmt.Printf("Error: Middle hashes level mismatch\n") + return + } + + // fragment hashes and middle hashes + for n, hash := range tree.FragmentHashes { + if !bytes.Equal(hash, tree2.FragmentHashes[n]) { + fmt.Printf("Error: Fragment hash %d mismatch\n", n) + return + } + } + + for n := range tree.MiddleHashes { + if len(tree.MiddleHashes[n]) != len(tree2.MiddleHashes[n]) { + fmt.Printf("Error: Middle hashes level %d mismatch\n", n) + return + } + + for m, hash := range tree.MiddleHashes[n] { + if !bytes.Equal(hash, tree2.MiddleHashes[n][m]) { + fmt.Printf("Error: Middle hash %d %d mismatch\n", n, m) + return + } + } + } + + fmt.Printf("Success. Import/export match.\n") +} diff --git a/protocol/Command.go b/protocol/Command.go index 91b3234..a721386 100644 --- a/protocol/Command.go +++ b/protocol/Command.go @@ -1,27 +1,27 @@ -/* -File Name: Command.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package protocol - -// Commands between peers -const ( - // Peer List Management - CommandAnnouncement = 0 // Announcement - CommandResponse = 1 // Response - CommandPing = 2 // Keep-alive message (no payload). - CommandPong = 3 // Response to ping (no payload). - CommandLocalDiscovery = 4 // Local discovery - CommandTraverse = 5 // Help establish a connection between 2 remote peers - - // Blockchain - CommandGetBlock = 6 // Request blocks for specified peer. - - // File Discovery - CommandTransfer = 8 // File transfer. - - // Debug - CommandChat = 10 // Chat message [debug] -) +/* +File Name: Command.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package protocol + +// Commands between peers +const ( + // Peer List Management + CommandAnnouncement = 0 // Announcement + CommandResponse = 1 // Response + CommandPing = 2 // Keep-alive message (no payload). + CommandPong = 3 // Response to ping (no payload). + CommandLocalDiscovery = 4 // Local discovery + CommandTraverse = 5 // Help establish a connection between 2 remote peers + + // Blockchain + CommandGetBlock = 6 // Request blocks for specified peer. + + // File Discovery + CommandTransfer = 8 // File transfer. + + // Debug + CommandChat = 10 // Chat message [debug] +) diff --git a/protocol/File Transfer.go b/protocol/File Transfer.go index 50c8456..6e9aed7 100644 --- a/protocol/File Transfer.go +++ b/protocol/File Transfer.go @@ -1,49 +1,49 @@ -/* -File Name: File Transfer.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Encoding of file transfer protocol. Each transfer starts with a header: -Offset Size Info -0 8 Total File Size -8 8 Transfer Size -*/ - -package protocol - -import ( - "encoding/binary" - "errors" - "io" -) - -// FileTransferWriteHeader starts writing the header for a file transfer. -func FileTransferWriteHeader(writer io.Writer, fileSize, transferSize uint64) (err error) { - // Send the header: Total File Size and Transfer Size. - header := make([]byte, 16) - binary.LittleEndian.PutUint64(header[0:8], fileSize) - binary.LittleEndian.PutUint64(header[8:16], transferSize) - if n, err := writer.Write(header); err != nil { - return err - } else if n != len(header) { - return errors.New("error sending header") - } - - return nil -} - -// FileTransferReadHeader starts reading the header for a file transfer. It will only read the header and keeps the connection open. -func FileTransferReadHeader(reader io.Reader) (fileSize, transferSize uint64, err error) { - // read the header - header := make([]byte, 16) - if n, err := reader.Read(header); err != nil { - return 0, 0, err - } else if n != len(header) { - return 0, 0, errors.New("error reading header") - } - - fileSize = binary.LittleEndian.Uint64(header[0:8]) - transferSize = binary.LittleEndian.Uint64(header[8:16]) - - return fileSize, transferSize, nil -} +/* +File Name: File Transfer.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Encoding of file transfer protocol. Each transfer starts with a header: +Offset Size Info +0 8 Total File Size +8 8 Transfer Size +*/ + +package protocol + +import ( + "encoding/binary" + "errors" + "io" +) + +// FileTransferWriteHeader starts writing the header for a file transfer. +func FileTransferWriteHeader(writer io.Writer, fileSize, transferSize uint64) (err error) { + // Send the header: Total File Size and Transfer Size. + header := make([]byte, 16) + binary.LittleEndian.PutUint64(header[0:8], fileSize) + binary.LittleEndian.PutUint64(header[8:16], transferSize) + if n, err := writer.Write(header); err != nil { + return err + } else if n != len(header) { + return errors.New("error sending header") + } + + return nil +} + +// FileTransferReadHeader starts reading the header for a file transfer. It will only read the header and keeps the connection open. +func FileTransferReadHeader(reader io.Reader) (fileSize, transferSize uint64, err error) { + // read the header + header := make([]byte, 16) + if n, err := reader.Read(header); err != nil { + return 0, 0, err + } else if n != len(header) { + return 0, 0, errors.New("error reading header") + } + + fileSize = binary.LittleEndian.Uint64(header[0:8]) + transferSize = binary.LittleEndian.Uint64(header[8:16]) + + return fileSize, transferSize, nil +} diff --git a/protocol/Hash.go b/protocol/Hash.go index ed1a5c6..4360c81 100644 --- a/protocol/Hash.go +++ b/protocol/Hash.go @@ -1,27 +1,27 @@ -/* -File Name: Hash.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package protocol - -import ( - "github.com/PeernetOfficial/core/btcec" - "lukechampine.com/blake3" -) - -// HashData abstracts the hash function. -func HashData(data []byte) (hash []byte) { - hash32 := blake3.Sum256(data) - return hash32[:] -} - -// HashSize is blake3 hash digest size = 256 bits -const HashSize = 32 - -// PublicKey2NodeID translates the Public Key into the node ID used in the Kademlia network. -// It is also referenced in various other places including blockchain data at runtime. The node ID identifies the owner. -func PublicKey2NodeID(publicKey *btcec.PublicKey) (nodeID []byte) { - return HashData(publicKey.SerializeCompressed()) -} +/* +File Name: Hash.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package protocol + +import ( + "github.com/PeernetOfficial/core/btcec" + "lukechampine.com/blake3" +) + +// HashData abstracts the hash function. +func HashData(data []byte) (hash []byte) { + hash32 := blake3.Sum256(data) + return hash32[:] +} + +// HashSize is blake3 hash digest size = 256 bits +const HashSize = 32 + +// PublicKey2NodeID translates the Public Key into the node ID used in the Kademlia network. +// It is also referenced in various other places including blockchain data at runtime. The node ID identifies the owner. +func PublicKey2NodeID(publicKey *btcec.PublicKey) (nodeID []byte) { + return HashData(publicKey.SerializeCompressed()) +} diff --git a/protocol/Message Encoding Announcement.go b/protocol/Message Encoding Announcement.go index 950d762..70a3daf 100644 --- a/protocol/Message Encoding Announcement.go +++ b/protocol/Message Encoding Announcement.go @@ -1,310 +1,310 @@ -/* -File Name: Message Encoding Announcement.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package protocol - -import ( - "encoding/binary" - "errors" - "unicode/utf8" -) - -// MessageAnnouncement is the decoded announcement message. -type MessageAnnouncement struct { - *MessageRaw // Underlying raw message - Protocol uint8 // Protocol version supported (low 4 bits). - Features uint8 // Feature support - Actions uint8 // Action bit array. See ActionX - BlockchainHeight uint64 // Blockchain height - BlockchainVersion uint64 // Blockchain version - PortInternal uint16 // Internal port. Can be used to detect NATs. - PortExternal uint16 // External port if known. 0 if not. Can be used for UPnP support. - UserAgent string // User Agent. Format "Software/Version". Required in the initial announcement/bootstrap. UTF-8 encoded. Max length is 255 bytes. - FindPeerKeys []KeyHash // FIND_PEER data - FindDataKeys []KeyHash // FIND_VALUE data - InfoStoreFiles []InfoStore // INFO_STORE data -} - -// KeyHash is a single blake3 key hash -type KeyHash struct { - Hash []byte -} - -// InfoStore informs about files stored -type InfoStore struct { - ID KeyHash // Hash of the file - Size uint64 // Size of the file - Type uint8 // Type of the file: 0 = File, 1 = Header file containing list of parts -} - -// Features are sent as bit array in the Announcement message. -const ( - FeatureIPv4Listen = 0 // Sender listens on IPv4 - FeatureIPv6Listen = 1 // Sender listens on IPv6 - FeatureFirewall = 2 // Sender indicates a potential firewall. This informs uncontacted peers that a Traverse message might be required to establish a connection. -) - -// Actions between peers, sent via Announcement message. They correspond to the bit array index. -const ( - ActionFindSelf = 0 // FIND_SELF Request closest neighbors to self - ActionFindPeer = 1 // FIND_PEER Request closest neighbors to target peer - ActionFindValue = 2 // FIND_VALUE Request data or closest peers - ActionInfoStore = 3 // INFO_STORE Sender indicates storing provided data -) - -// Minimum length of Announcement payload header without User Agent -const announcementPayloadHeaderSize = 24 - -// DecodeAnnouncement decodes the incoming announcement message. Returns nil if invalid. -func DecodeAnnouncement(msg *MessageRaw) (result *MessageAnnouncement, err error) { - result = &MessageAnnouncement{ - MessageRaw: msg, - } - - if len(msg.Payload) < announcementPayloadHeaderSize { - return nil, errors.New("announcement: invalid minimum length") - } - - result.Protocol = msg.Payload[0] & 0x0F // Protocol version support is stored in the first 4 bits - result.Features = msg.Payload[1] // Feature support - result.Actions = msg.Payload[2] - result.BlockchainHeight = binary.LittleEndian.Uint64(msg.Payload[3 : 3+8]) - result.BlockchainVersion = binary.LittleEndian.Uint64(msg.Payload[11 : 11+8]) - result.PortInternal = binary.LittleEndian.Uint16(msg.Payload[19 : 19+2]) - result.PortExternal = binary.LittleEndian.Uint16(msg.Payload[21 : 21+2]) - - userAgentLength := int(msg.Payload[23]) - if userAgentLength > 0 { - if userAgentLength > len(msg.Payload)-announcementPayloadHeaderSize { - return nil, errors.New("announcement: user agent overflow") - } - - userAgentB := msg.Payload[announcementPayloadHeaderSize : announcementPayloadHeaderSize+userAgentLength] - if !utf8.Valid(userAgentB) { - return nil, errors.New("announcement: user agent invalid encoding") - } - - result.UserAgent = string(userAgentB) - } - - data := msg.Payload[announcementPayloadHeaderSize+userAgentLength:] - - // FIND_PEER - if result.Actions&(1< 0 { - keys, read, valid := decodeKeys(data) - if !valid { - return nil, errors.New("announcement: FIND_PEER invalid data") - } - - data = data[read:] - result.FindPeerKeys = keys - } - - // FIND_VALUE - if result.Actions&(1< 0 { - keys, read, valid := decodeKeys(data) - if !valid { - return nil, errors.New("announcement: FIND_VALUE invalid data") - } - - data = data[read:] - result.FindDataKeys = keys - } - - // INFO_STORE - if result.Actions&(1< 0 { - files, _, valid := decodeInfoStore(data) - if !valid { - return nil, errors.New("announcement: INFO_STORE invalid data") - } - - // commented out because never used - //data = data[read:] - result.InfoStoreFiles = files - } - - // Accept extra data in case future features append additional data - //if len(data) > 0 { - // return nil, errors.New("announcement: Unexpected extra data") - //} - - return -} - -// decodeKeys decodes keys. Header is 2 bytes (count) followed by the actual keys (each 32 bytes blake3 hash). -func decodeKeys(data []byte) (keys []KeyHash, read int, valid bool) { - if len(data) < 2+HashSize { // minimum length - return nil, 0, false - } - - count := binary.LittleEndian.Uint16(data[0:2]) - - if read = 2 + int(count)*HashSize; len(data) < read { - return nil, 0, false - } - - for n := 0; n < int(count); n++ { - key := make([]byte, HashSize) - copy(key, data[2+n*HashSize:2+n*HashSize+HashSize]) - keys = append(keys, KeyHash{Hash: key}) - } - - return keys, read, true -} - -func decodeInfoStore(data []byte) (files []InfoStore, read int, valid bool) { - if len(data) < 2+41 { // minimum length - return nil, 0, false - } - - count := binary.LittleEndian.Uint16(data[0:2]) - - if read = 2 + int(count)*41; len(data) < read { - return nil, 0, false - } - - for n := 0; n < int(count); n++ { - file := InfoStore{} - file.ID.Hash = make([]byte, HashSize) - copy(file.ID.Hash, data[2+n*41:2+n*41+HashSize]) - file.Size = binary.LittleEndian.Uint64(data[2+n*41+32 : 2+n*41+32+8]) - file.Type = data[2+n*41+40] - - files = append(files, file) - } - - return files, read, true -} - -// EncodeAnnouncement encodes an announcement message. It may return multiple messages if the input does not fit into one. -// findPeer is a list of node IDs (blake3 hash of peer ID compressed form) -// findValue is a list of hashes -// files is a list of files stored to inform about -func EncodeAnnouncement(sendUA, findSelf bool, findPeer []KeyHash, findValue []KeyHash, files []InfoStore, features byte, blockchainHeight, blockchainVersion uint64, userAgent string) (packetsRaw [][]byte) { -createPacketLoop: - for { - raw := make([]byte, 64*1024) // max UDP packet size - packetSize := announcementPayloadHeaderSize - - raw[0] = byte(ProtocolVersion) // Protocol - raw[1] = features // Feature support - //raw[2] = Actions // Action bit array - - binary.LittleEndian.PutUint64(raw[3:3+8], blockchainHeight) - binary.LittleEndian.PutUint64(raw[11:11+8], blockchainVersion) - - // only on initial announcement the User Agent must be provided according to the protocol spec - if sendUA { - userAgentB := []byte(userAgent) - if len(userAgentB) > 255 { - userAgentB = userAgentB[:255] - } - - raw[23] = byte(len(userAgentB)) - copy(raw[announcementPayloadHeaderSize:announcementPayloadHeaderSize+len(userAgentB)], userAgentB) - packetSize += len(userAgentB) - } - - // FIND_SELF - if findSelf { - raw[2] |= 1 << ActionFindSelf - } - - // FIND_PEER - if len(findPeer) > 0 { - // check if there is enough space for at least the header and 1 record - if isPacketSizeExceed(packetSize, 2+32) { - packetsRaw = append(packetsRaw, raw[:packetSize]) - continue createPacketLoop - } - - raw[2] |= 1 << ActionFindPeer - index := packetSize - packetSize += 2 - - for n, find := range findPeer { - // check if minimum length is available in packet - if isPacketSizeExceed(packetSize, 32) { - packetsRaw = append(packetsRaw, raw[:packetSize]) - findPeer = findPeer[n:] - continue createPacketLoop - } - - binary.LittleEndian.PutUint16(raw[index:index+2], uint16(n+1)) - copy(raw[index+2+32*n:index+2+32*n+32], find.Hash) - packetSize += 32 - } - - findPeer = nil - } - - // FIND_VALUE - if len(findValue) > 0 { - // check if there is enough space for at least the header and 1 record - if isPacketSizeExceed(packetSize, 2+32) { - packetsRaw = append(packetsRaw, raw[:packetSize]) - continue createPacketLoop - } - - raw[2] |= 1 << ActionFindValue - index := packetSize - packetSize += 2 - - for n, find := range findValue { - // check if minimum length is available in packet - if isPacketSizeExceed(packetSize, 32) { - packetsRaw = append(packetsRaw, raw[:packetSize]) - findValue = findValue[n:] - continue createPacketLoop - } - - binary.LittleEndian.PutUint16(raw[index:index+2], uint16(n+1)) - copy(raw[index+2+32*n:index+2+32*n+32], find.Hash) - packetSize += 32 - } - - findValue = nil - } - - // INFO_STORE - if len(files) > 0 { - // check if there is enough space for at least the header and 1 record - if isPacketSizeExceed(packetSize, 2+41) { - packetsRaw = append(packetsRaw, raw[:packetSize]) - continue createPacketLoop - } - - raw[2] |= 1 << ActionInfoStore - index := packetSize - packetSize += 2 - - for n, file := range files { - // check if minimum length is available in packet - if isPacketSizeExceed(packetSize, 41) { - packetsRaw = append(packetsRaw, raw[:packetSize]) - files = files[n:] - continue createPacketLoop - } - - binary.LittleEndian.PutUint16(raw[index:index+2], uint16(n+1)) - copy(raw[index+2+41*n:index+2+41*n+32], file.ID.Hash) - - binary.LittleEndian.PutUint64(raw[index+2+41*n+32:index+2+41*n+32+8], file.Size) - raw[index+2+41*n+40] = file.Type - - packetSize += 41 - } - - files = nil - } - - packetsRaw = append(packetsRaw, raw[:packetSize]) - - if len(findPeer) == 0 && len(findValue) == 0 && len(files) == 0 { - return - } - } -} +/* +File Name: Message Encoding Announcement.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package protocol + +import ( + "encoding/binary" + "errors" + "unicode/utf8" +) + +// MessageAnnouncement is the decoded announcement message. +type MessageAnnouncement struct { + *MessageRaw // Underlying raw message + Protocol uint8 // Protocol version supported (low 4 bits). + Features uint8 // Feature support + Actions uint8 // Action bit array. See ActionX + BlockchainHeight uint64 // Blockchain height + BlockchainVersion uint64 // Blockchain version + PortInternal uint16 // Internal port. Can be used to detect NATs. + PortExternal uint16 // External port if known. 0 if not. Can be used for UPnP support. + UserAgent string // User Agent. Format "Software/Version". Required in the initial announcement/bootstrap. UTF-8 encoded. Max length is 255 bytes. + FindPeerKeys []KeyHash // FIND_PEER data + FindDataKeys []KeyHash // FIND_VALUE data + InfoStoreFiles []InfoStore // INFO_STORE data +} + +// KeyHash is a single blake3 key hash +type KeyHash struct { + Hash []byte +} + +// InfoStore informs about files stored +type InfoStore struct { + ID KeyHash // Hash of the file + Size uint64 // Size of the file + Type uint8 // Type of the file: 0 = File, 1 = Header file containing list of parts +} + +// Features are sent as bit array in the Announcement message. +const ( + FeatureIPv4Listen = 0 // Sender listens on IPv4 + FeatureIPv6Listen = 1 // Sender listens on IPv6 + FeatureFirewall = 2 // Sender indicates a potential firewall. This informs uncontacted peers that a Traverse message might be required to establish a connection. +) + +// Actions between peers, sent via Announcement message. They correspond to the bit array index. +const ( + ActionFindSelf = 0 // FIND_SELF Request closest neighbors to self + ActionFindPeer = 1 // FIND_PEER Request closest neighbors to target peer + ActionFindValue = 2 // FIND_VALUE Request data or closest peers + ActionInfoStore = 3 // INFO_STORE Sender indicates storing provided data +) + +// Minimum length of Announcement payload header without User Agent +const announcementPayloadHeaderSize = 24 + +// DecodeAnnouncement decodes the incoming announcement message. Returns nil if invalid. +func DecodeAnnouncement(msg *MessageRaw) (result *MessageAnnouncement, err error) { + result = &MessageAnnouncement{ + MessageRaw: msg, + } + + if len(msg.Payload) < announcementPayloadHeaderSize { + return nil, errors.New("announcement: invalid minimum length") + } + + result.Protocol = msg.Payload[0] & 0x0F // Protocol version support is stored in the first 4 bits + result.Features = msg.Payload[1] // Feature support + result.Actions = msg.Payload[2] + result.BlockchainHeight = binary.LittleEndian.Uint64(msg.Payload[3 : 3+8]) + result.BlockchainVersion = binary.LittleEndian.Uint64(msg.Payload[11 : 11+8]) + result.PortInternal = binary.LittleEndian.Uint16(msg.Payload[19 : 19+2]) + result.PortExternal = binary.LittleEndian.Uint16(msg.Payload[21 : 21+2]) + + userAgentLength := int(msg.Payload[23]) + if userAgentLength > 0 { + if userAgentLength > len(msg.Payload)-announcementPayloadHeaderSize { + return nil, errors.New("announcement: user agent overflow") + } + + userAgentB := msg.Payload[announcementPayloadHeaderSize : announcementPayloadHeaderSize+userAgentLength] + if !utf8.Valid(userAgentB) { + return nil, errors.New("announcement: user agent invalid encoding") + } + + result.UserAgent = string(userAgentB) + } + + data := msg.Payload[announcementPayloadHeaderSize+userAgentLength:] + + // FIND_PEER + if result.Actions&(1< 0 { + keys, read, valid := decodeKeys(data) + if !valid { + return nil, errors.New("announcement: FIND_PEER invalid data") + } + + data = data[read:] + result.FindPeerKeys = keys + } + + // FIND_VALUE + if result.Actions&(1< 0 { + keys, read, valid := decodeKeys(data) + if !valid { + return nil, errors.New("announcement: FIND_VALUE invalid data") + } + + data = data[read:] + result.FindDataKeys = keys + } + + // INFO_STORE + if result.Actions&(1< 0 { + files, _, valid := decodeInfoStore(data) + if !valid { + return nil, errors.New("announcement: INFO_STORE invalid data") + } + + // commented out because never used + //data = data[read:] + result.InfoStoreFiles = files + } + + // Accept extra data in case future features append additional data + //if len(data) > 0 { + // return nil, errors.New("announcement: Unexpected extra data") + //} + + return +} + +// decodeKeys decodes keys. Header is 2 bytes (count) followed by the actual keys (each 32 bytes blake3 hash). +func decodeKeys(data []byte) (keys []KeyHash, read int, valid bool) { + if len(data) < 2+HashSize { // minimum length + return nil, 0, false + } + + count := binary.LittleEndian.Uint16(data[0:2]) + + if read = 2 + int(count)*HashSize; len(data) < read { + return nil, 0, false + } + + for n := 0; n < int(count); n++ { + key := make([]byte, HashSize) + copy(key, data[2+n*HashSize:2+n*HashSize+HashSize]) + keys = append(keys, KeyHash{Hash: key}) + } + + return keys, read, true +} + +func decodeInfoStore(data []byte) (files []InfoStore, read int, valid bool) { + if len(data) < 2+41 { // minimum length + return nil, 0, false + } + + count := binary.LittleEndian.Uint16(data[0:2]) + + if read = 2 + int(count)*41; len(data) < read { + return nil, 0, false + } + + for n := 0; n < int(count); n++ { + file := InfoStore{} + file.ID.Hash = make([]byte, HashSize) + copy(file.ID.Hash, data[2+n*41:2+n*41+HashSize]) + file.Size = binary.LittleEndian.Uint64(data[2+n*41+32 : 2+n*41+32+8]) + file.Type = data[2+n*41+40] + + files = append(files, file) + } + + return files, read, true +} + +// EncodeAnnouncement encodes an announcement message. It may return multiple messages if the input does not fit into one. +// findPeer is a list of node IDs (blake3 hash of peer ID compressed form) +// findValue is a list of hashes +// files is a list of files stored to inform about +func EncodeAnnouncement(sendUA, findSelf bool, findPeer []KeyHash, findValue []KeyHash, files []InfoStore, features byte, blockchainHeight, blockchainVersion uint64, userAgent string) (packetsRaw [][]byte) { +createPacketLoop: + for { + raw := make([]byte, 64*1024) // max UDP packet size + packetSize := announcementPayloadHeaderSize + + raw[0] = byte(ProtocolVersion) // Protocol + raw[1] = features // Feature support + //raw[2] = Actions // Action bit array + + binary.LittleEndian.PutUint64(raw[3:3+8], blockchainHeight) + binary.LittleEndian.PutUint64(raw[11:11+8], blockchainVersion) + + // only on initial announcement the User Agent must be provided according to the protocol spec + if sendUA { + userAgentB := []byte(userAgent) + if len(userAgentB) > 255 { + userAgentB = userAgentB[:255] + } + + raw[23] = byte(len(userAgentB)) + copy(raw[announcementPayloadHeaderSize:announcementPayloadHeaderSize+len(userAgentB)], userAgentB) + packetSize += len(userAgentB) + } + + // FIND_SELF + if findSelf { + raw[2] |= 1 << ActionFindSelf + } + + // FIND_PEER + if len(findPeer) > 0 { + // check if there is enough space for at least the header and 1 record + if isPacketSizeExceed(packetSize, 2+32) { + packetsRaw = append(packetsRaw, raw[:packetSize]) + continue createPacketLoop + } + + raw[2] |= 1 << ActionFindPeer + index := packetSize + packetSize += 2 + + for n, find := range findPeer { + // check if minimum length is available in packet + if isPacketSizeExceed(packetSize, 32) { + packetsRaw = append(packetsRaw, raw[:packetSize]) + findPeer = findPeer[n:] + continue createPacketLoop + } + + binary.LittleEndian.PutUint16(raw[index:index+2], uint16(n+1)) + copy(raw[index+2+32*n:index+2+32*n+32], find.Hash) + packetSize += 32 + } + + findPeer = nil + } + + // FIND_VALUE + if len(findValue) > 0 { + // check if there is enough space for at least the header and 1 record + if isPacketSizeExceed(packetSize, 2+32) { + packetsRaw = append(packetsRaw, raw[:packetSize]) + continue createPacketLoop + } + + raw[2] |= 1 << ActionFindValue + index := packetSize + packetSize += 2 + + for n, find := range findValue { + // check if minimum length is available in packet + if isPacketSizeExceed(packetSize, 32) { + packetsRaw = append(packetsRaw, raw[:packetSize]) + findValue = findValue[n:] + continue createPacketLoop + } + + binary.LittleEndian.PutUint16(raw[index:index+2], uint16(n+1)) + copy(raw[index+2+32*n:index+2+32*n+32], find.Hash) + packetSize += 32 + } + + findValue = nil + } + + // INFO_STORE + if len(files) > 0 { + // check if there is enough space for at least the header and 1 record + if isPacketSizeExceed(packetSize, 2+41) { + packetsRaw = append(packetsRaw, raw[:packetSize]) + continue createPacketLoop + } + + raw[2] |= 1 << ActionInfoStore + index := packetSize + packetSize += 2 + + for n, file := range files { + // check if minimum length is available in packet + if isPacketSizeExceed(packetSize, 41) { + packetsRaw = append(packetsRaw, raw[:packetSize]) + files = files[n:] + continue createPacketLoop + } + + binary.LittleEndian.PutUint16(raw[index:index+2], uint16(n+1)) + copy(raw[index+2+41*n:index+2+41*n+32], file.ID.Hash) + + binary.LittleEndian.PutUint64(raw[index+2+41*n+32:index+2+41*n+32+8], file.Size) + raw[index+2+41*n+40] = file.Type + + packetSize += 41 + } + + files = nil + } + + packetsRaw = append(packetsRaw, raw[:packetSize]) + + if len(findPeer) == 0 && len(findValue) == 0 && len(files) == 0 { + return + } + } +} diff --git a/protocol/Message Encoding Get Block.go b/protocol/Message Encoding Get Block.go index e98a901..0d91553 100644 --- a/protocol/Message Encoding Get Block.go +++ b/protocol/Message Encoding Get Block.go @@ -1,230 +1,230 @@ -/* -File Name: Message Encoding Get Block.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Get Block message encoding: -Offset Size Info -0 1 Control -1 33 Peer ID compressed form identifying which blockchain to transfer - -Control = 0: Request Blocks -34 8 Limit total count of blocks to transfer. The transfer will be terminated if the limit is reached. -42 8 Limit of bytes per block to transfer max. Blocks exceeding this limit will not be transferred. -50 16 Transfer ID. This will identify lite packets. -66 2 Count of block ranges -68 16 * ? List of block ranges - -Block range: -0 8 Block number -8 8 Count of blocks - -Control = 3: Active -34 ? Embedded block data as stream. - -For the block stream there is a header preceding each block: -Offset Size Info -0 1 Availability - 0 = Block range is available. - 1 = Block range not available. - 2 = Block range exceeds size limit. -1 16 Block range -17 8 Block size - -The limit in block range must be 1 if a block is returned. -*/ - -package protocol - -import ( - "encoding/binary" - "errors" - "io" - - "github.com/PeernetOfficial/core/btcec" - "github.com/google/uuid" -) - -const ( - GetBlockControlRequestStart = 0 // Request start transfer of blocks - GetBlockControlNotAvailable = 1 // Requested blockchain not available (not found) - GetBlockControlActive = 2 // Active block transfer - GetBlockControlTerminate = 3 // Terminate - GetBlockControlEmpty = 4 // Requested blockchain has 0 blocks -) - -const ( - GetBlockStatusAvailable = 0 - GetBlockStatusNotAvailable = 1 - GetBlockStatusSizeExceed = 2 -) - -// Min size of header for Get Block control 0 message. -const getBlockRequestHeaderSize = 68 - -// MessageGetBlock is the decoded Get Block message. -type MessageGetBlock struct { - *MessageRaw // Underlying raw message. - Control uint8 // Control. See TransferControlX. - BlockchainPublicKey *btcec.PublicKey // Peer ID of blockchain to transfer. - - // fields valid only for GetBlockControlRequestStart - TransferID uuid.UUID // Transfer ID to identify lite packets. - LimitBlockCount uint64 // Limit total count of blocks to transfer - MaxBlockSize uint64 // Limit of bytes per block to transfer max. Blocks exceeding this limit will not be transferred. - TargetBlocks []BlockRange // Target list of block ranges to transfer. - - // fields valid only for GetBlockControlActive - Data []byte // Embedded protocol data. -} - -// BlockRange is a single start-count range. -type BlockRange struct { - Offset uint64 // Block number start - Limit uint64 // Count of blocks -} - -// DecodeGetBlock decodes a Get Block message -func DecodeGetBlock(msg *MessageRaw) (result *MessageGetBlock, err error) { - if len(msg.Payload) < 34 { - return nil, errors.New("get block: invalid minimum length") - } - - result = &MessageGetBlock{ - MessageRaw: msg, - } - - result.Control = msg.Payload[0] - - peerIDcompressed := msg.Payload[1:34] - if result.BlockchainPublicKey, err = btcec.ParsePubKey(peerIDcompressed, btcec.S256()); err != nil { - return nil, err - } - - if result.Control == GetBlockControlRequestStart { - if len(msg.Payload) < getBlockRequestHeaderSize { - return nil, errors.New("get block: invalid minimum length") - } - - result.LimitBlockCount = binary.LittleEndian.Uint64(msg.Payload[34 : 34+8]) - result.MaxBlockSize = binary.LittleEndian.Uint64(msg.Payload[42 : 42+8]) - copy(result.TransferID[:], msg.Payload[50:50+16]) - - countBlockRanges := int(binary.LittleEndian.Uint16(msg.Payload[66 : 66+2])) - if countBlockRanges == 0 { - return nil, errors.New("get block: empty block range") - } else if len(msg.Payload) < getBlockRequestHeaderSize+16*countBlockRanges { - return nil, errors.New("get block: cound block ranges exceeds length") - } - - index := getBlockRequestHeaderSize - - for n := 0; n < countBlockRanges; n++ { - var target BlockRange - target.Offset = binary.LittleEndian.Uint64(msg.Payload[index : index+8]) - target.Limit = binary.LittleEndian.Uint64(msg.Payload[index+8 : index+16]) - result.TargetBlocks = append(result.TargetBlocks, target) - - index += 16 - } - } else if result.Control == GetBlockControlActive { - result.Data = msg.Payload[34:] - } - - return result, nil -} - -// EncodeGetBlock encodes a Get Block message. The embedded packet size must be smaller than TransferMaxEmbedSize. -func EncodeGetBlock(senderPrivateKey *btcec.PrivateKey, data []byte, control uint8, blockchainPublicKey *btcec.PublicKey, limitBlockCount, maxBlockSize uint64, targetBlocks []BlockRange, transferID uuid.UUID) (packetRaw []byte, err error) { - if control == GetBlockControlRequestStart && len(data) != 0 { - return nil, errors.New("get block encode: payload not allowed in start") - } else if isPacketSizeExceed(transferPayloadHeaderSize, len(data)) { - return nil, errors.New("get block encode: embedded packet too big") - } else if control == GetBlockControlRequestStart && isPacketSizeExceed(getBlockRequestHeaderSize, len(targetBlocks)*16) { - return nil, errors.New("get block encode: too many target block ranges") - } - - packetSize := transferPayloadHeaderSize - if control == GetBlockControlRequestStart { - packetSize = getBlockRequestHeaderSize + len(targetBlocks)*16 - } else if control == GetBlockControlActive { - packetSize += len(data) - } - - raw := make([]byte, packetSize) - - raw[0] = control - targetPeerID := blockchainPublicKey.SerializeCompressed() - copy(raw[1:34], targetPeerID) - - if control == GetBlockControlRequestStart { - binary.LittleEndian.PutUint64(raw[34:34+8], limitBlockCount) - binary.LittleEndian.PutUint64(raw[42:42+8], maxBlockSize) - copy(raw[50:50+16], transferID[:]) - binary.LittleEndian.PutUint16(raw[66:66+2], uint16(len(targetBlocks))) - - index := getBlockRequestHeaderSize - for _, target := range targetBlocks { - binary.LittleEndian.PutUint64(raw[index:index+8], target.Offset) - binary.LittleEndian.PutUint64(raw[index+8:index+16], target.Limit) - - index += 16 - } - } else if control == GetBlockControlActive { - copy(raw[34:34+len(data)], data) - } - - return raw, nil -} - -// IsLast checks if the incoming message is the last one in this transfer. -func (msg *MessageGetBlock) IsLast() bool { - return msg.Control == GetBlockControlTerminate || msg.Control == GetBlockControlNotAvailable || msg.Control == GetBlockControlEmpty -} - -// BlockTransferWriteHeader starts writing the header for a block transfer. -func BlockTransferWriteHeader(writer io.Writer, availability uint8, targetBlock BlockRange, blockSize uint64) (err error) { - header := make([]byte, 25) - header[0] = availability - binary.LittleEndian.PutUint64(header[1:9], targetBlock.Offset) - binary.LittleEndian.PutUint64(header[9:17], targetBlock.Limit) - binary.LittleEndian.PutUint64(header[17:25], blockSize) - - _, err = writer.Write(header) - return err -} - -// BlockTransferReadBlock reads the header and the block from the reader -func BlockTransferReadBlock(reader io.Reader, maxBlockSize uint64) (data []byte, targetBlock BlockRange, blockSize uint64, availability uint8, err error) { - header := make([]byte, 25) - - if _, err := io.ReadAtLeast(reader, header, len(header)); err != nil { - return nil, targetBlock, 0, 0, err - } - - availability = header[0] - targetBlock.Offset = binary.LittleEndian.Uint64(header[1:9]) - targetBlock.Limit = binary.LittleEndian.Uint64(header[9:17]) - blockSize = binary.LittleEndian.Uint64(header[17:25]) - - if targetBlock.Limit == 0 { - return nil, targetBlock, blockSize, availability, errors.New("empty target block limit") - } else if availability != GetBlockStatusAvailable { // return if status indicates the block is not available - return nil, targetBlock, blockSize, availability, nil - } - - if blockSize > maxBlockSize { - return nil, targetBlock, blockSize, availability, errors.New("remote block size exceeds limit") - } else if targetBlock.Limit != 1 { - return nil, targetBlock, blockSize, availability, errors.New("invalid target block limit") - } - - // read the block - block := make([]byte, blockSize) - - if _, err := io.ReadAtLeast(reader, block, len(block)); err != nil { - return nil, targetBlock, blockSize, availability, err - } - - return block, targetBlock, blockSize, availability, nil -} +/* +File Name: Message Encoding Get Block.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Get Block message encoding: +Offset Size Info +0 1 Control +1 33 Peer ID compressed form identifying which blockchain to transfer + +Control = 0: Request Blocks +34 8 Limit total count of blocks to transfer. The transfer will be terminated if the limit is reached. +42 8 Limit of bytes per block to transfer max. Blocks exceeding this limit will not be transferred. +50 16 Transfer ID. This will identify lite packets. +66 2 Count of block ranges +68 16 * ? List of block ranges + +Block range: +0 8 Block number +8 8 Count of blocks + +Control = 3: Active +34 ? Embedded block data as stream. + +For the block stream there is a header preceding each block: +Offset Size Info +0 1 Availability + 0 = Block range is available. + 1 = Block range not available. + 2 = Block range exceeds size limit. +1 16 Block range +17 8 Block size + +The limit in block range must be 1 if a block is returned. +*/ + +package protocol + +import ( + "encoding/binary" + "errors" + "io" + + "github.com/PeernetOfficial/core/btcec" + "github.com/google/uuid" +) + +const ( + GetBlockControlRequestStart = 0 // Request start transfer of blocks + GetBlockControlNotAvailable = 1 // Requested blockchain not available (not found) + GetBlockControlActive = 2 // Active block transfer + GetBlockControlTerminate = 3 // Terminate + GetBlockControlEmpty = 4 // Requested blockchain has 0 blocks +) + +const ( + GetBlockStatusAvailable = 0 + GetBlockStatusNotAvailable = 1 + GetBlockStatusSizeExceed = 2 +) + +// Min size of header for Get Block control 0 message. +const getBlockRequestHeaderSize = 68 + +// MessageGetBlock is the decoded Get Block message. +type MessageGetBlock struct { + *MessageRaw // Underlying raw message. + Control uint8 // Control. See TransferControlX. + BlockchainPublicKey *btcec.PublicKey // Peer ID of blockchain to transfer. + + // fields valid only for GetBlockControlRequestStart + TransferID uuid.UUID // Transfer ID to identify lite packets. + LimitBlockCount uint64 // Limit total count of blocks to transfer + MaxBlockSize uint64 // Limit of bytes per block to transfer max. Blocks exceeding this limit will not be transferred. + TargetBlocks []BlockRange // Target list of block ranges to transfer. + + // fields valid only for GetBlockControlActive + Data []byte // Embedded protocol data. +} + +// BlockRange is a single start-count range. +type BlockRange struct { + Offset uint64 // Block number start + Limit uint64 // Count of blocks +} + +// DecodeGetBlock decodes a Get Block message +func DecodeGetBlock(msg *MessageRaw) (result *MessageGetBlock, err error) { + if len(msg.Payload) < 34 { + return nil, errors.New("get block: invalid minimum length") + } + + result = &MessageGetBlock{ + MessageRaw: msg, + } + + result.Control = msg.Payload[0] + + peerIDcompressed := msg.Payload[1:34] + if result.BlockchainPublicKey, err = btcec.ParsePubKey(peerIDcompressed, btcec.S256()); err != nil { + return nil, err + } + + if result.Control == GetBlockControlRequestStart { + if len(msg.Payload) < getBlockRequestHeaderSize { + return nil, errors.New("get block: invalid minimum length") + } + + result.LimitBlockCount = binary.LittleEndian.Uint64(msg.Payload[34 : 34+8]) + result.MaxBlockSize = binary.LittleEndian.Uint64(msg.Payload[42 : 42+8]) + copy(result.TransferID[:], msg.Payload[50:50+16]) + + countBlockRanges := int(binary.LittleEndian.Uint16(msg.Payload[66 : 66+2])) + if countBlockRanges == 0 { + return nil, errors.New("get block: empty block range") + } else if len(msg.Payload) < getBlockRequestHeaderSize+16*countBlockRanges { + return nil, errors.New("get block: cound block ranges exceeds length") + } + + index := getBlockRequestHeaderSize + + for n := 0; n < countBlockRanges; n++ { + var target BlockRange + target.Offset = binary.LittleEndian.Uint64(msg.Payload[index : index+8]) + target.Limit = binary.LittleEndian.Uint64(msg.Payload[index+8 : index+16]) + result.TargetBlocks = append(result.TargetBlocks, target) + + index += 16 + } + } else if result.Control == GetBlockControlActive { + result.Data = msg.Payload[34:] + } + + return result, nil +} + +// EncodeGetBlock encodes a Get Block message. The embedded packet size must be smaller than TransferMaxEmbedSize. +func EncodeGetBlock(senderPrivateKey *btcec.PrivateKey, data []byte, control uint8, blockchainPublicKey *btcec.PublicKey, limitBlockCount, maxBlockSize uint64, targetBlocks []BlockRange, transferID uuid.UUID) (packetRaw []byte, err error) { + if control == GetBlockControlRequestStart && len(data) != 0 { + return nil, errors.New("get block encode: payload not allowed in start") + } else if isPacketSizeExceed(transferPayloadHeaderSize, len(data)) { + return nil, errors.New("get block encode: embedded packet too big") + } else if control == GetBlockControlRequestStart && isPacketSizeExceed(getBlockRequestHeaderSize, len(targetBlocks)*16) { + return nil, errors.New("get block encode: too many target block ranges") + } + + packetSize := transferPayloadHeaderSize + if control == GetBlockControlRequestStart { + packetSize = getBlockRequestHeaderSize + len(targetBlocks)*16 + } else if control == GetBlockControlActive { + packetSize += len(data) + } + + raw := make([]byte, packetSize) + + raw[0] = control + targetPeerID := blockchainPublicKey.SerializeCompressed() + copy(raw[1:34], targetPeerID) + + if control == GetBlockControlRequestStart { + binary.LittleEndian.PutUint64(raw[34:34+8], limitBlockCount) + binary.LittleEndian.PutUint64(raw[42:42+8], maxBlockSize) + copy(raw[50:50+16], transferID[:]) + binary.LittleEndian.PutUint16(raw[66:66+2], uint16(len(targetBlocks))) + + index := getBlockRequestHeaderSize + for _, target := range targetBlocks { + binary.LittleEndian.PutUint64(raw[index:index+8], target.Offset) + binary.LittleEndian.PutUint64(raw[index+8:index+16], target.Limit) + + index += 16 + } + } else if control == GetBlockControlActive { + copy(raw[34:34+len(data)], data) + } + + return raw, nil +} + +// IsLast checks if the incoming message is the last one in this transfer. +func (msg *MessageGetBlock) IsLast() bool { + return msg.Control == GetBlockControlTerminate || msg.Control == GetBlockControlNotAvailable || msg.Control == GetBlockControlEmpty +} + +// BlockTransferWriteHeader starts writing the header for a block transfer. +func BlockTransferWriteHeader(writer io.Writer, availability uint8, targetBlock BlockRange, blockSize uint64) (err error) { + header := make([]byte, 25) + header[0] = availability + binary.LittleEndian.PutUint64(header[1:9], targetBlock.Offset) + binary.LittleEndian.PutUint64(header[9:17], targetBlock.Limit) + binary.LittleEndian.PutUint64(header[17:25], blockSize) + + _, err = writer.Write(header) + return err +} + +// BlockTransferReadBlock reads the header and the block from the reader +func BlockTransferReadBlock(reader io.Reader, maxBlockSize uint64) (data []byte, targetBlock BlockRange, blockSize uint64, availability uint8, err error) { + header := make([]byte, 25) + + if _, err := io.ReadAtLeast(reader, header, len(header)); err != nil { + return nil, targetBlock, 0, 0, err + } + + availability = header[0] + targetBlock.Offset = binary.LittleEndian.Uint64(header[1:9]) + targetBlock.Limit = binary.LittleEndian.Uint64(header[9:17]) + blockSize = binary.LittleEndian.Uint64(header[17:25]) + + if targetBlock.Limit == 0 { + return nil, targetBlock, blockSize, availability, errors.New("empty target block limit") + } else if availability != GetBlockStatusAvailable { // return if status indicates the block is not available + return nil, targetBlock, blockSize, availability, nil + } + + if blockSize > maxBlockSize { + return nil, targetBlock, blockSize, availability, errors.New("remote block size exceeds limit") + } else if targetBlock.Limit != 1 { + return nil, targetBlock, blockSize, availability, errors.New("invalid target block limit") + } + + // read the block + block := make([]byte, blockSize) + + if _, err := io.ReadAtLeast(reader, block, len(block)); err != nil { + return nil, targetBlock, blockSize, availability, err + } + + return block, targetBlock, blockSize, availability, nil +} diff --git a/protocol/Message Encoding Response.go b/protocol/Message Encoding Response.go index f132fd2..b4211dc 100644 --- a/protocol/Message Encoding Response.go +++ b/protocol/Message Encoding Response.go @@ -1,445 +1,445 @@ -/* -File Name: Message Encoding Response.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package protocol - -import ( - "bytes" - "encoding/binary" - "errors" - "net" - "time" - "unicode/utf8" - - "github.com/PeernetOfficial/core/btcec" -) - -// MessageResponse is the decoded response message. -type MessageResponse struct { - *MessageRaw // Underlying raw message - Protocol uint8 // Protocol version supported (low 4 bits). - Features uint8 // Feature support (high 4 bits). Future use. - Actions uint8 // Action bit array. See ActionX - BlockchainHeight uint64 // Blockchain height - BlockchainVersion uint64 // Blockchain version - PortInternal uint16 // Internal port. Can be used to detect NATs. - PortExternal uint16 // External port if known. 0 if not. Can be used for UPnP support. - UserAgent string // User Agent. Format "Software/Version". Required in the initial announcement/bootstrap. UTF-8 encoded. Max length is 255 bytes. - Hash2Peers []Hash2Peer // List of peers that know the requested hashes or at least are close to it - FilesEmbed []EmbeddedFileData // Files that were embedded in the response - HashesNotFound [][]byte // Hashes that were reported back as not found -} - -// PeerRecord informs about a peer -type PeerRecord struct { - PublicKey *btcec.PublicKey // Public Key - NodeID []byte // Kademlia Node ID - IPv4 net.IP // IPv4 address. 0 if not set. - IPv4Port uint16 // Port (actual one used for connection) - IPv4PortReportedInternal uint16 // Internal port as reported by that peer. This can be used to identify whether the peer is potentially behind a NAT. - IPv4PortReportedExternal uint16 // External port as reported by that peer. This is used in case of port forwarding (manual or automated). - IPv6 net.IP // IPv6 address. 0 if not set. - IPv6Port uint16 // Port (actual one used for connection) - IPv6PortReportedInternal uint16 // Internal port as reported by that peer. This can be used to identify whether the peer is potentially behind a NAT. - IPv6PortReportedExternal uint16 // External port as reported by that peer. This is used in case of port forwarding (manual or automated). - LastContact uint32 // Last contact in seconds - LastContactT time.Time // Last contact time translated from seconds - Features uint8 // Feature support. Same as in Announcement/Response message. -} - -// Hash2Peer links a hash to peers who are known to store the data and to peers who are considered close to the hash -type Hash2Peer struct { - ID KeyHash // Hash that was queried - Closest []PeerRecord // Closest peers - Storing []PeerRecord // Peers known to store the data identified by the hash - IsLast bool // Whether it is the last records returned for the requested hash and no more results will follow -} - -// EmbeddedFileData contains embedded data sent within a response -type EmbeddedFileData struct { - ID KeyHash // Hash of the file - Data []byte // Data -} - -// Actions in Response message -const ( - ActionSequenceLast = 0 // SEQUENCE_LAST Last response to the announcement in the sequence -) - -// DecodeResponse decodes the incoming response message. Returns nil if invalid. -func DecodeResponse(msg *MessageRaw) (result *MessageResponse, err error) { - result = &MessageResponse{ - MessageRaw: msg, - } - - if len(msg.Payload) < announcementPayloadHeaderSize+6 { - return nil, errors.New("response: invalid minimum length") - } - - result.Protocol = msg.Payload[0] & 0x0F // Protocol version support is stored in the first 4 bits - result.Features = msg.Payload[1] // Feature support - result.Actions = msg.Payload[2] - result.BlockchainHeight = binary.LittleEndian.Uint64(msg.Payload[3 : 3+8]) - result.BlockchainVersion = binary.LittleEndian.Uint64(msg.Payload[11 : 11+8]) - result.PortInternal = binary.LittleEndian.Uint16(msg.Payload[19 : 19+2]) - result.PortExternal = binary.LittleEndian.Uint16(msg.Payload[21 : 21+2]) - - userAgentLength := int(msg.Payload[23]) - read := announcementPayloadHeaderSize - - if userAgentLength > 0 { - if userAgentLength > len(msg.Payload)-announcementPayloadHeaderSize { - return nil, errors.New("response: user agent overflow") - } - - userAgentB := msg.Payload[announcementPayloadHeaderSize : announcementPayloadHeaderSize+userAgentLength] - if !utf8.Valid(userAgentB) { - return nil, errors.New("response: user agent invalid encoding") - } - - result.UserAgent = string(userAgentB) - read += userAgentLength - } - - countPeerResponses := binary.LittleEndian.Uint16(msg.Payload[read+0 : read+0+2]) - countEmbeddedFiles := binary.LittleEndian.Uint16(msg.Payload[read+2 : read+2+2]) - countHashesNotFound := binary.LittleEndian.Uint16(msg.Payload[read+4 : read+4+2]) - read += 6 - - if countPeerResponses == 0 && countEmbeddedFiles == 0 && countHashesNotFound == 0 { - // Empty responses are allowed. They can be useful as quasi-pings to get the latest blockchain info of the peer. - return - } - - data := msg.Payload[read:] - - // Peer response data - if countPeerResponses > 0 { - hash2Peers, read, valid := decodePeerRecord(data, int(countPeerResponses)) - if !valid { - return nil, errors.New("response: peer info invalid data") - } - data = data[read:] - - result.Hash2Peers = append(result.Hash2Peers, hash2Peers...) - } - - // Embedded files - if countEmbeddedFiles > 0 { - filesEmbed, read, valid := decodeEmbeddedFile(data, int(countEmbeddedFiles)) - if !valid { - return nil, errors.New("response: embedded file invalid data") - } - data = data[read:] - - result.FilesEmbed = append(result.FilesEmbed, filesEmbed...) - } - - // Hashes not found - if countHashesNotFound > 0 { - if len(data) < int(countHashesNotFound)*32 { - return nil, errors.New("response: hash list invalid data") - } - - for n := 0; n < int(countHashesNotFound); n++ { - hash := make([]byte, HashSize) - copy(hash, data[n*32:n*32+32]) - - result.HashesNotFound = append(result.HashesNotFound, hash) - } - } - - return -} - -// Length of peer record in bytes -const peerRecordSize = 70 - -// decodePeerRecord decodes the response data for FIND_SELF, FIND_PEER and FIND_VALUE messages -func decodePeerRecord(data []byte, count int) (hash2Peers []Hash2Peer, read int, valid bool) { - index := 0 - - for n := 0; n < count; n++ { - if read += 34; len(data) < read { - return nil, 0, false - } - - hash := make([]byte, HashSize) - copy(hash, data[index:index+32]) - countField := binary.LittleEndian.Uint16(data[index+32:index+32+2]) & 0x7FFF - isLast := binary.LittleEndian.Uint16(data[index+32:index+32+2])&0x8000 > 0 - index += 34 - - hash2Peer := Hash2Peer{ID: KeyHash{hash}, IsLast: isLast} - - // Response contains peer records - for m := 0; m < int(countField); m++ { - if read += peerRecordSize; len(data) < read { - return nil, 0, false - } - - peer := PeerRecord{} - - peerIDcompressed := make([]byte, 33) - copy(peerIDcompressed[:], data[index:index+33]) - - // IPv4 - ipv4B := make([]byte, 4) - copy(ipv4B[:], data[index+33:index+33+4]) - - peer.IPv4 = ipv4B - peer.IPv4Port = binary.LittleEndian.Uint16(data[index+37 : index+37+2]) - peer.IPv4PortReportedInternal = binary.LittleEndian.Uint16(data[index+39 : index+39+2]) - peer.IPv4PortReportedExternal = binary.LittleEndian.Uint16(data[index+41 : index+41+2]) - - // IPv6 - ipv6B := make([]byte, 16) - copy(ipv6B[:], data[index+43:index+43+16]) - - peer.IPv6 = ipv6B - peer.IPv6Port = binary.LittleEndian.Uint16(data[index+59 : index+59+2]) - peer.IPv6PortReportedInternal = binary.LittleEndian.Uint16(data[index+61 : index+61+2]) - peer.IPv6PortReportedExternal = binary.LittleEndian.Uint16(data[index+63 : index+63+2]) - - if peer.IPv6.To4() != nil { // IPv6 address mismatch - return nil, 0, false - } - - peer.LastContact = binary.LittleEndian.Uint32(data[index+65 : index+65+4]) - peer.LastContactT = time.Now().Add(-time.Second * time.Duration(peer.LastContact)) - peer.Features = data[index+69] & 0x7F - reason := data[index+69] >> 7 - - var err error - if peer.PublicKey, err = btcec.ParsePubKey(peerIDcompressed, btcec.S256()); err != nil { - return nil, 0, false - } - - peer.NodeID = PublicKey2NodeID(peer.PublicKey) - - if reason == 0 { // Peer was returned because it is close to the requested hash - hash2Peer.Closest = append(hash2Peer.Closest, peer) - } else if reason == 1 { // Peer stores the data - hash2Peer.Storing = append(hash2Peer.Storing, peer) - } - - index += peerRecordSize - } - - hash2Peers = append(hash2Peers, hash2Peer) - } - - return hash2Peers, read, true -} - -// decodeEmbeddedFile decodes the embedded file response data for FIND_VALUE -func decodeEmbeddedFile(data []byte, count int) (filesEmbed []EmbeddedFileData, read int, valid bool) { - index := 0 - - for n := 0; n < count; n++ { - if read += 34; len(data) < read { - return nil, 0, false - } - - hash := make([]byte, HashSize) - copy(hash, data[index:index+32]) - sizeField := int(binary.LittleEndian.Uint16(data[index+32 : index+32+2])) - index += 34 - - if read += sizeField; len(data) < read { - return nil, 0, false - } - - fileData := make([]byte, sizeField) - copy(fileData[:], data[index:index+sizeField]) - - index += sizeField - - // validate the hash - if !bytes.Equal(hash, HashData(fileData)) { - return nil, read, false - } - - filesEmbed = append(filesEmbed, EmbeddedFileData{ID: KeyHash{Hash: hash}, Data: fileData}) - } - - return filesEmbed, read, true -} - -// EmbeddedFileSizeMax is the maximum size of embedded files in response messages. Any file exceeding that must be shared via regular file transfer. -const EmbeddedFileSizeMax = udpMaxPacketSize - PacketLengthMin - announcementPayloadHeaderSize - 2 - 35 - -// EncodeResponse encodes a response message -// hash2Peers will be modified. -func EncodeResponse(sendUA bool, hash2Peers []Hash2Peer, filesEmbed []EmbeddedFileData, hashesNotFound [][]byte, features byte, blockchainHeight, blockchainVersion uint64, userAgent string) (packetsRaw [][]byte, err error) { - for n := range filesEmbed { - if len(filesEmbed[n].Data) > EmbeddedFileSizeMax { - return nil, errors.New("embedded file too big") - } - } - -createPacketLoop: - for { - raw := make([]byte, 64*1024) // max UDP packet size - packetSize := announcementPayloadHeaderSize - - raw[0] = byte(ProtocolVersion) // Protocol - raw[1] = features // Feature support - //raw[2] = Actions // Action bit array - - binary.LittleEndian.PutUint64(raw[3:3+8], blockchainHeight) - binary.LittleEndian.PutUint64(raw[11:11+8], blockchainVersion) - - // only on initial response the User Agent must be provided according to the protocol spec - if sendUA { - userAgentB := []byte(userAgent) - if len(userAgentB) > 255 { - userAgentB = userAgentB[:255] - } - - raw[23] = byte(len(userAgentB)) - copy(raw[announcementPayloadHeaderSize:announcementPayloadHeaderSize+len(userAgentB)], userAgentB) - packetSize += len(userAgentB) - } - - // 3 count field at raw[index]: count of peer responses, embedded files, and hashes not found - countIndex := packetSize - packetSize += 6 - - // Encode the peer response data for FIND_SELF, FIND_PEER and FIND_VALUE requests. - if len(hash2Peers) > 0 { - for n, hash2Peer := range hash2Peers { - if isPacketSizeExceed(packetSize, 34+peerRecordSize) { // check if minimum length is available in packet - packetsRaw = append(packetsRaw, raw[:packetSize]) - hash2Peers = hash2Peers[n:] - continue createPacketLoop - } - - index := packetSize - copy(raw[index:index+32], hash2Peer.ID.Hash) - count2Index := index + 32 - - packetSize += 34 - count2 := uint16(0) - - for m := range hash2Peer.Storing { - if isPacketSizeExceed(packetSize, peerRecordSize) { // check if minimum length is available in packet - packetsRaw = append(packetsRaw, raw[:packetSize]) - hash2Peers = hash2Peers[n:] - hash2Peer.Storing = hash2Peer.Storing[m:] - continue createPacketLoop - } - - index := packetSize - encodePeerRecord(raw[index:index+peerRecordSize], &hash2Peer.Storing[m], 1) - - packetSize += peerRecordSize - binary.LittleEndian.PutUint16(raw[count2Index+0:count2Index+2], uint16(m+1)) - count2++ - } - - hash2Peer.Storing = nil - - for m := range hash2Peer.Closest { - if isPacketSizeExceed(packetSize, peerRecordSize) { // check if minimum length is available in packet - packetsRaw = append(packetsRaw, raw[:packetSize]) - hash2Peers = hash2Peers[n:] - hash2Peer.Closest = hash2Peer.Closest[m:] - continue createPacketLoop - } - - index := packetSize - encodePeerRecord(raw[index:index+peerRecordSize], &hash2Peer.Closest[m], 0) - - packetSize += peerRecordSize - count2++ - binary.LittleEndian.PutUint16(raw[count2Index+0:count2Index+2], count2) - } - - binary.LittleEndian.PutUint16(raw[count2Index+0:count2Index+2], count2|0x8000) // signal the last result for the key with bit 15 - binary.LittleEndian.PutUint16(raw[countIndex+0:countIndex+0+2], uint16(n+1)) // count of peer responses - } - - hash2Peers = nil - } - - // FIND_VALUE response embedded data - if len(filesEmbed) > 0 { - if isPacketSizeExceed(packetSize, 34+len(filesEmbed[0].Data)) { // check if there is enough space for at least the header and 1 record - packetsRaw = append(packetsRaw, raw[:packetSize]) - continue createPacketLoop - } - - for n, file := range filesEmbed { - if isPacketSizeExceed(packetSize, 34+len(file.Data)) { // check if minimum length is available in packet - packetsRaw = append(packetsRaw, raw[:packetSize]) - filesEmbed = filesEmbed[n:] - continue createPacketLoop - } - - index := packetSize - copy(raw[index:index+32], file.ID.Hash) - binary.LittleEndian.PutUint16(raw[index+32:index+32+2], uint16(len(file.Data))) - copy(raw[index+34:index+34+len(file.Data)], file.Data) - - binary.LittleEndian.PutUint16(raw[countIndex+2:countIndex+2+2], uint16(n+1)) // count of embedded files - packetSize += 34 + len(file.Data) - } - - filesEmbed = nil - } - - // Hashes not found - if len(hashesNotFound) > 0 { - index := packetSize - - for n, hash := range hashesNotFound { - if isPacketSizeExceed(packetSize, 32) { // check if there is enough space for at least the header and 1 record - packetsRaw = append(packetsRaw, raw[:packetSize]) - continue createPacketLoop - } - - copy(raw[index+n*32:index+n*32+32], hash) - - binary.LittleEndian.PutUint16(raw[countIndex+4:countIndex+4+2], uint16(n+1)) // count of hashes not found - packetSize += 32 - } - - hashesNotFound = nil - } - - raw[2] |= 1 << ActionSequenceLast // Indicate that no more responses will be sent in this sequence - packetsRaw = append(packetsRaw, raw[:packetSize]) - - if len(hash2Peers) == 0 && len(filesEmbed) == 0 && len(hashesNotFound) == 0 { // this should always be the case here - return - } - } -} - -// encodePeerRecord encodes a single peer record and stores it into raw -func encodePeerRecord(raw []byte, peer *PeerRecord, reason uint8) { - copy(raw[0:0+33], peer.PublicKey.SerializeCompressed()) - binary.LittleEndian.PutUint32(raw[65:65+4], peer.LastContact) - raw[69] = peer.Features | reason<<7 - - // IPv4 - copy(raw[33:33+4], peer.IPv4.To4()) - binary.LittleEndian.PutUint16(raw[37:37+2], peer.IPv4Port) - binary.LittleEndian.PutUint16(raw[39:39+2], peer.IPv4PortReportedInternal) - binary.LittleEndian.PutUint16(raw[41:41+2], peer.IPv4PortReportedExternal) - - // IPv6 - copy(raw[43:43+16], peer.IPv6.To16()) - binary.LittleEndian.PutUint16(raw[59:59+2], peer.IPv6Port) - binary.LittleEndian.PutUint16(raw[61:61+2], peer.IPv6PortReportedInternal) - binary.LittleEndian.PutUint16(raw[63:63+2], peer.IPv6PortReportedExternal) -} - -// IsLast checks if the incoming message is the last expected response in this sequence. -func (msg *MessageResponse) IsLast() bool { - return msg.Actions&(1< 0 -} +/* +File Name: Message Encoding Response.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package protocol + +import ( + "bytes" + "encoding/binary" + "errors" + "net" + "time" + "unicode/utf8" + + "github.com/PeernetOfficial/core/btcec" +) + +// MessageResponse is the decoded response message. +type MessageResponse struct { + *MessageRaw // Underlying raw message + Protocol uint8 // Protocol version supported (low 4 bits). + Features uint8 // Feature support (high 4 bits). Future use. + Actions uint8 // Action bit array. See ActionX + BlockchainHeight uint64 // Blockchain height + BlockchainVersion uint64 // Blockchain version + PortInternal uint16 // Internal port. Can be used to detect NATs. + PortExternal uint16 // External port if known. 0 if not. Can be used for UPnP support. + UserAgent string // User Agent. Format "Software/Version". Required in the initial announcement/bootstrap. UTF-8 encoded. Max length is 255 bytes. + Hash2Peers []Hash2Peer // List of peers that know the requested hashes or at least are close to it + FilesEmbed []EmbeddedFileData // Files that were embedded in the response + HashesNotFound [][]byte // Hashes that were reported back as not found +} + +// PeerRecord informs about a peer +type PeerRecord struct { + PublicKey *btcec.PublicKey // Public Key + NodeID []byte // Kademlia Node ID + IPv4 net.IP // IPv4 address. 0 if not set. + IPv4Port uint16 // Port (actual one used for connection) + IPv4PortReportedInternal uint16 // Internal port as reported by that peer. This can be used to identify whether the peer is potentially behind a NAT. + IPv4PortReportedExternal uint16 // External port as reported by that peer. This is used in case of port forwarding (manual or automated). + IPv6 net.IP // IPv6 address. 0 if not set. + IPv6Port uint16 // Port (actual one used for connection) + IPv6PortReportedInternal uint16 // Internal port as reported by that peer. This can be used to identify whether the peer is potentially behind a NAT. + IPv6PortReportedExternal uint16 // External port as reported by that peer. This is used in case of port forwarding (manual or automated). + LastContact uint32 // Last contact in seconds + LastContactT time.Time // Last contact time translated from seconds + Features uint8 // Feature support. Same as in Announcement/Response message. +} + +// Hash2Peer links a hash to peers who are known to store the data and to peers who are considered close to the hash +type Hash2Peer struct { + ID KeyHash // Hash that was queried + Closest []PeerRecord // Closest peers + Storing []PeerRecord // Peers known to store the data identified by the hash + IsLast bool // Whether it is the last records returned for the requested hash and no more results will follow +} + +// EmbeddedFileData contains embedded data sent within a response +type EmbeddedFileData struct { + ID KeyHash // Hash of the file + Data []byte // Data +} + +// Actions in Response message +const ( + ActionSequenceLast = 0 // SEQUENCE_LAST Last response to the announcement in the sequence +) + +// DecodeResponse decodes the incoming response message. Returns nil if invalid. +func DecodeResponse(msg *MessageRaw) (result *MessageResponse, err error) { + result = &MessageResponse{ + MessageRaw: msg, + } + + if len(msg.Payload) < announcementPayloadHeaderSize+6 { + return nil, errors.New("response: invalid minimum length") + } + + result.Protocol = msg.Payload[0] & 0x0F // Protocol version support is stored in the first 4 bits + result.Features = msg.Payload[1] // Feature support + result.Actions = msg.Payload[2] + result.BlockchainHeight = binary.LittleEndian.Uint64(msg.Payload[3 : 3+8]) + result.BlockchainVersion = binary.LittleEndian.Uint64(msg.Payload[11 : 11+8]) + result.PortInternal = binary.LittleEndian.Uint16(msg.Payload[19 : 19+2]) + result.PortExternal = binary.LittleEndian.Uint16(msg.Payload[21 : 21+2]) + + userAgentLength := int(msg.Payload[23]) + read := announcementPayloadHeaderSize + + if userAgentLength > 0 { + if userAgentLength > len(msg.Payload)-announcementPayloadHeaderSize { + return nil, errors.New("response: user agent overflow") + } + + userAgentB := msg.Payload[announcementPayloadHeaderSize : announcementPayloadHeaderSize+userAgentLength] + if !utf8.Valid(userAgentB) { + return nil, errors.New("response: user agent invalid encoding") + } + + result.UserAgent = string(userAgentB) + read += userAgentLength + } + + countPeerResponses := binary.LittleEndian.Uint16(msg.Payload[read+0 : read+0+2]) + countEmbeddedFiles := binary.LittleEndian.Uint16(msg.Payload[read+2 : read+2+2]) + countHashesNotFound := binary.LittleEndian.Uint16(msg.Payload[read+4 : read+4+2]) + read += 6 + + if countPeerResponses == 0 && countEmbeddedFiles == 0 && countHashesNotFound == 0 { + // Empty responses are allowed. They can be useful as quasi-pings to get the latest blockchain info of the peer. + return + } + + data := msg.Payload[read:] + + // Peer response data + if countPeerResponses > 0 { + hash2Peers, read, valid := decodePeerRecord(data, int(countPeerResponses)) + if !valid { + return nil, errors.New("response: peer info invalid data") + } + data = data[read:] + + result.Hash2Peers = append(result.Hash2Peers, hash2Peers...) + } + + // Embedded files + if countEmbeddedFiles > 0 { + filesEmbed, read, valid := decodeEmbeddedFile(data, int(countEmbeddedFiles)) + if !valid { + return nil, errors.New("response: embedded file invalid data") + } + data = data[read:] + + result.FilesEmbed = append(result.FilesEmbed, filesEmbed...) + } + + // Hashes not found + if countHashesNotFound > 0 { + if len(data) < int(countHashesNotFound)*32 { + return nil, errors.New("response: hash list invalid data") + } + + for n := 0; n < int(countHashesNotFound); n++ { + hash := make([]byte, HashSize) + copy(hash, data[n*32:n*32+32]) + + result.HashesNotFound = append(result.HashesNotFound, hash) + } + } + + return +} + +// Length of peer record in bytes +const peerRecordSize = 70 + +// decodePeerRecord decodes the response data for FIND_SELF, FIND_PEER and FIND_VALUE messages +func decodePeerRecord(data []byte, count int) (hash2Peers []Hash2Peer, read int, valid bool) { + index := 0 + + for n := 0; n < count; n++ { + if read += 34; len(data) < read { + return nil, 0, false + } + + hash := make([]byte, HashSize) + copy(hash, data[index:index+32]) + countField := binary.LittleEndian.Uint16(data[index+32:index+32+2]) & 0x7FFF + isLast := binary.LittleEndian.Uint16(data[index+32:index+32+2])&0x8000 > 0 + index += 34 + + hash2Peer := Hash2Peer{ID: KeyHash{hash}, IsLast: isLast} + + // Response contains peer records + for m := 0; m < int(countField); m++ { + if read += peerRecordSize; len(data) < read { + return nil, 0, false + } + + peer := PeerRecord{} + + peerIDcompressed := make([]byte, 33) + copy(peerIDcompressed[:], data[index:index+33]) + + // IPv4 + ipv4B := make([]byte, 4) + copy(ipv4B[:], data[index+33:index+33+4]) + + peer.IPv4 = ipv4B + peer.IPv4Port = binary.LittleEndian.Uint16(data[index+37 : index+37+2]) + peer.IPv4PortReportedInternal = binary.LittleEndian.Uint16(data[index+39 : index+39+2]) + peer.IPv4PortReportedExternal = binary.LittleEndian.Uint16(data[index+41 : index+41+2]) + + // IPv6 + ipv6B := make([]byte, 16) + copy(ipv6B[:], data[index+43:index+43+16]) + + peer.IPv6 = ipv6B + peer.IPv6Port = binary.LittleEndian.Uint16(data[index+59 : index+59+2]) + peer.IPv6PortReportedInternal = binary.LittleEndian.Uint16(data[index+61 : index+61+2]) + peer.IPv6PortReportedExternal = binary.LittleEndian.Uint16(data[index+63 : index+63+2]) + + if peer.IPv6.To4() != nil { // IPv6 address mismatch + return nil, 0, false + } + + peer.LastContact = binary.LittleEndian.Uint32(data[index+65 : index+65+4]) + peer.LastContactT = time.Now().Add(-time.Second * time.Duration(peer.LastContact)) + peer.Features = data[index+69] & 0x7F + reason := data[index+69] >> 7 + + var err error + if peer.PublicKey, err = btcec.ParsePubKey(peerIDcompressed, btcec.S256()); err != nil { + return nil, 0, false + } + + peer.NodeID = PublicKey2NodeID(peer.PublicKey) + + if reason == 0 { // Peer was returned because it is close to the requested hash + hash2Peer.Closest = append(hash2Peer.Closest, peer) + } else if reason == 1 { // Peer stores the data + hash2Peer.Storing = append(hash2Peer.Storing, peer) + } + + index += peerRecordSize + } + + hash2Peers = append(hash2Peers, hash2Peer) + } + + return hash2Peers, read, true +} + +// decodeEmbeddedFile decodes the embedded file response data for FIND_VALUE +func decodeEmbeddedFile(data []byte, count int) (filesEmbed []EmbeddedFileData, read int, valid bool) { + index := 0 + + for n := 0; n < count; n++ { + if read += 34; len(data) < read { + return nil, 0, false + } + + hash := make([]byte, HashSize) + copy(hash, data[index:index+32]) + sizeField := int(binary.LittleEndian.Uint16(data[index+32 : index+32+2])) + index += 34 + + if read += sizeField; len(data) < read { + return nil, 0, false + } + + fileData := make([]byte, sizeField) + copy(fileData[:], data[index:index+sizeField]) + + index += sizeField + + // validate the hash + if !bytes.Equal(hash, HashData(fileData)) { + return nil, read, false + } + + filesEmbed = append(filesEmbed, EmbeddedFileData{ID: KeyHash{Hash: hash}, Data: fileData}) + } + + return filesEmbed, read, true +} + +// EmbeddedFileSizeMax is the maximum size of embedded files in response messages. Any file exceeding that must be shared via regular file transfer. +const EmbeddedFileSizeMax = udpMaxPacketSize - PacketLengthMin - announcementPayloadHeaderSize - 2 - 35 + +// EncodeResponse encodes a response message +// hash2Peers will be modified. +func EncodeResponse(sendUA bool, hash2Peers []Hash2Peer, filesEmbed []EmbeddedFileData, hashesNotFound [][]byte, features byte, blockchainHeight, blockchainVersion uint64, userAgent string) (packetsRaw [][]byte, err error) { + for n := range filesEmbed { + if len(filesEmbed[n].Data) > EmbeddedFileSizeMax { + return nil, errors.New("embedded file too big") + } + } + +createPacketLoop: + for { + raw := make([]byte, 64*1024) // max UDP packet size + packetSize := announcementPayloadHeaderSize + + raw[0] = byte(ProtocolVersion) // Protocol + raw[1] = features // Feature support + //raw[2] = Actions // Action bit array + + binary.LittleEndian.PutUint64(raw[3:3+8], blockchainHeight) + binary.LittleEndian.PutUint64(raw[11:11+8], blockchainVersion) + + // only on initial response the User Agent must be provided according to the protocol spec + if sendUA { + userAgentB := []byte(userAgent) + if len(userAgentB) > 255 { + userAgentB = userAgentB[:255] + } + + raw[23] = byte(len(userAgentB)) + copy(raw[announcementPayloadHeaderSize:announcementPayloadHeaderSize+len(userAgentB)], userAgentB) + packetSize += len(userAgentB) + } + + // 3 count field at raw[index]: count of peer responses, embedded files, and hashes not found + countIndex := packetSize + packetSize += 6 + + // Encode the peer response data for FIND_SELF, FIND_PEER and FIND_VALUE requests. + if len(hash2Peers) > 0 { + for n, hash2Peer := range hash2Peers { + if isPacketSizeExceed(packetSize, 34+peerRecordSize) { // check if minimum length is available in packet + packetsRaw = append(packetsRaw, raw[:packetSize]) + hash2Peers = hash2Peers[n:] + continue createPacketLoop + } + + index := packetSize + copy(raw[index:index+32], hash2Peer.ID.Hash) + count2Index := index + 32 + + packetSize += 34 + count2 := uint16(0) + + for m := range hash2Peer.Storing { + if isPacketSizeExceed(packetSize, peerRecordSize) { // check if minimum length is available in packet + packetsRaw = append(packetsRaw, raw[:packetSize]) + hash2Peers = hash2Peers[n:] + hash2Peer.Storing = hash2Peer.Storing[m:] + continue createPacketLoop + } + + index := packetSize + encodePeerRecord(raw[index:index+peerRecordSize], &hash2Peer.Storing[m], 1) + + packetSize += peerRecordSize + binary.LittleEndian.PutUint16(raw[count2Index+0:count2Index+2], uint16(m+1)) + count2++ + } + + hash2Peer.Storing = nil + + for m := range hash2Peer.Closest { + if isPacketSizeExceed(packetSize, peerRecordSize) { // check if minimum length is available in packet + packetsRaw = append(packetsRaw, raw[:packetSize]) + hash2Peers = hash2Peers[n:] + hash2Peer.Closest = hash2Peer.Closest[m:] + continue createPacketLoop + } + + index := packetSize + encodePeerRecord(raw[index:index+peerRecordSize], &hash2Peer.Closest[m], 0) + + packetSize += peerRecordSize + count2++ + binary.LittleEndian.PutUint16(raw[count2Index+0:count2Index+2], count2) + } + + binary.LittleEndian.PutUint16(raw[count2Index+0:count2Index+2], count2|0x8000) // signal the last result for the key with bit 15 + binary.LittleEndian.PutUint16(raw[countIndex+0:countIndex+0+2], uint16(n+1)) // count of peer responses + } + + hash2Peers = nil + } + + // FIND_VALUE response embedded data + if len(filesEmbed) > 0 { + if isPacketSizeExceed(packetSize, 34+len(filesEmbed[0].Data)) { // check if there is enough space for at least the header and 1 record + packetsRaw = append(packetsRaw, raw[:packetSize]) + continue createPacketLoop + } + + for n, file := range filesEmbed { + if isPacketSizeExceed(packetSize, 34+len(file.Data)) { // check if minimum length is available in packet + packetsRaw = append(packetsRaw, raw[:packetSize]) + filesEmbed = filesEmbed[n:] + continue createPacketLoop + } + + index := packetSize + copy(raw[index:index+32], file.ID.Hash) + binary.LittleEndian.PutUint16(raw[index+32:index+32+2], uint16(len(file.Data))) + copy(raw[index+34:index+34+len(file.Data)], file.Data) + + binary.LittleEndian.PutUint16(raw[countIndex+2:countIndex+2+2], uint16(n+1)) // count of embedded files + packetSize += 34 + len(file.Data) + } + + filesEmbed = nil + } + + // Hashes not found + if len(hashesNotFound) > 0 { + index := packetSize + + for n, hash := range hashesNotFound { + if isPacketSizeExceed(packetSize, 32) { // check if there is enough space for at least the header and 1 record + packetsRaw = append(packetsRaw, raw[:packetSize]) + continue createPacketLoop + } + + copy(raw[index+n*32:index+n*32+32], hash) + + binary.LittleEndian.PutUint16(raw[countIndex+4:countIndex+4+2], uint16(n+1)) // count of hashes not found + packetSize += 32 + } + + hashesNotFound = nil + } + + raw[2] |= 1 << ActionSequenceLast // Indicate that no more responses will be sent in this sequence + packetsRaw = append(packetsRaw, raw[:packetSize]) + + if len(hash2Peers) == 0 && len(filesEmbed) == 0 && len(hashesNotFound) == 0 { // this should always be the case here + return + } + } +} + +// encodePeerRecord encodes a single peer record and stores it into raw +func encodePeerRecord(raw []byte, peer *PeerRecord, reason uint8) { + copy(raw[0:0+33], peer.PublicKey.SerializeCompressed()) + binary.LittleEndian.PutUint32(raw[65:65+4], peer.LastContact) + raw[69] = peer.Features | reason<<7 + + // IPv4 + copy(raw[33:33+4], peer.IPv4.To4()) + binary.LittleEndian.PutUint16(raw[37:37+2], peer.IPv4Port) + binary.LittleEndian.PutUint16(raw[39:39+2], peer.IPv4PortReportedInternal) + binary.LittleEndian.PutUint16(raw[41:41+2], peer.IPv4PortReportedExternal) + + // IPv6 + copy(raw[43:43+16], peer.IPv6.To16()) + binary.LittleEndian.PutUint16(raw[59:59+2], peer.IPv6Port) + binary.LittleEndian.PutUint16(raw[61:61+2], peer.IPv6PortReportedInternal) + binary.LittleEndian.PutUint16(raw[63:63+2], peer.IPv6PortReportedExternal) +} + +// IsLast checks if the incoming message is the last expected response in this sequence. +func (msg *MessageResponse) IsLast() bool { + return msg.Actions&(1< 0 +} diff --git a/protocol/Message Encoding Transfer.go b/protocol/Message Encoding Transfer.go index bffca6d..4e6f8c0 100644 --- a/protocol/Message Encoding Transfer.go +++ b/protocol/Message Encoding Transfer.go @@ -1,136 +1,136 @@ -/* -File Name: Message Encoding Transfer.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Transfer message encoding: -Offset Size Info -0 1 Control -1 1 Transfer Protocol -2 32 File Hash - -Control = 0: Request Start -34 8 Offset to start reading in the file -42 8 Limit of bytes to read at the offset -50 16 Transfer ID. This will identify lite packets. - -Offset + limit must not exceed the file size. Actual data transfer should be sent via lite packets. -The regular Peernet packets would be too CPU expensive and slow due to public key signing. - -*/ - -package protocol - -import ( - "encoding/binary" - "errors" - - "github.com/PeernetOfficial/core/btcec" - "github.com/google/uuid" -) - -// MessageTransfer is the decoded transfer message. -// It is sent to initiate a file transfer, and to send data as part of a file transfer. The actual file data is encapsulated via UDT. -type MessageTransfer struct { - *MessageRaw // Underlying raw message. - Control uint8 // Control. See TransferControlX. - TransferProtocol uint8 // Embedded transfer protocol: 0 = UDT - Hash []byte // Hash of the file to transfer. - Offset uint64 // Offset to start reading at. Only TransferControlRequestStart. - Limit uint64 // Limit (count of bytes) to read starting at the offset. Only TransferControlRequestStart. - TransferID uuid.UUID // Transfer ID to identify lite packets. - Data []byte // Embedded protocol data. Only TransferControlActive. -} - -const ( - TransferControlRequestStart = 0 // Request start transfer of file. Data at byte 34 is offset and limit to read, each 8 bytes. Limit may be 0 to indicate entire file. - TransferControlNotAvailable = 1 // Requested file not available - TransferControlActive = 2 // Active file transfer - TransferControlTerminate = 3 // Terminate -) - -const ( - TransferProtocolUDT = 0 // UDT via lite packets. No encryption. -) - -const transferPayloadHeaderSize = 34 - -// DecodeTransfer decodes a transfer message -func DecodeTransfer(msg *MessageRaw) (result *MessageTransfer, err error) { - if len(msg.Payload) < transferPayloadHeaderSize { - return nil, errors.New("transfer: invalid minimum length") - } - - result = &MessageTransfer{ - MessageRaw: msg, - Hash: make([]byte, HashSize), - } - - result.Control = msg.Payload[0] - result.TransferProtocol = msg.Payload[1] - copy(result.Hash, msg.Payload[2:2+HashSize]) - - switch result.Control { - case TransferControlRequestStart: - // Offset and Limit must be provided after the header. - if len(msg.Payload) < transferPayloadHeaderSize+16 { - return nil, errors.New("transfer: invalid minimum length") - } - - result.Offset = binary.LittleEndian.Uint64(msg.Payload[34 : 34+8]) - result.Limit = binary.LittleEndian.Uint64(msg.Payload[42 : 42+8]) - copy(result.TransferID[:], msg.Payload[50:50+16]) - - case TransferControlActive: - // Data should be transferred via lite packets for performance reasons, but it is allowed to be encapsulated in Peernet packets. - result.Data = msg.Payload[transferPayloadHeaderSize:] - - } - - return result, nil -} - -// TransferMaxEmbedSize is a recommended default upper size of embedded data inside the Transfer message, to be used as MaxPacketSize limit in the embedded protocol. -// This value is chosen as the lowest denominator of different environments (IPv4, IPv6, Ethernet, Internet) for safe transfer, not for highest performance. -// The caller may send bigger payloads but may risk that data packets are simply dropped and never arrive. A MTU negotiation or detection could pimp that. -const TransferMaxEmbedSize = internetSafeMTU - PacketLengthMin - transferPayloadHeaderSize - -// Same as TransferMaxEmbedSize but for encoding via lite packets. -const TransferMaxEmbedSizeLite = internetSafeMTU - PacketLiteSizeMin - -// EncodeTransfer encodes a transfer message. The embedded packet size must be smaller than TransferMaxEmbedSize. -func EncodeTransfer(senderPrivateKey *btcec.PrivateKey, data []byte, control, transferProtocol uint8, hash []byte, offset, limit uint64, transferID uuid.UUID) (packetRaw []byte, err error) { - if control == TransferControlRequestStart && len(data) != 0 { - return nil, errors.New("transfer encode: payload not allowed in start") - } else if isPacketSizeExceed(transferPayloadHeaderSize, len(data)) { - return nil, errors.New("transfer encode: embedded packet too big") - } - - packetSize := transferPayloadHeaderSize - if control == TransferControlRequestStart { - packetSize += 32 - } else if control == TransferControlActive { - packetSize += len(data) - } - - raw := make([]byte, packetSize) - - raw[0] = control - raw[1] = transferProtocol - copy(raw[2:2+HashSize], hash) - - if control == TransferControlRequestStart { - binary.LittleEndian.PutUint64(raw[34:34+8], offset) - binary.LittleEndian.PutUint64(raw[42:42+8], limit) - copy(raw[50:50+16], transferID[:]) - } else if control == TransferControlActive { - copy(raw[34:34+len(data)], data) - } - - return raw, nil -} - -// IsLast checks if the incoming message is the last one in this transfer. -func (msg *MessageTransfer) IsLast() bool { - return msg.Control == TransferControlTerminate || msg.Control == TransferControlNotAvailable -} +/* +File Name: Message Encoding Transfer.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Transfer message encoding: +Offset Size Info +0 1 Control +1 1 Transfer Protocol +2 32 File Hash + +Control = 0: Request Start +34 8 Offset to start reading in the file +42 8 Limit of bytes to read at the offset +50 16 Transfer ID. This will identify lite packets. + +Offset + limit must not exceed the file size. Actual data transfer should be sent via lite packets. +The regular Peernet packets would be too CPU expensive and slow due to public key signing. + +*/ + +package protocol + +import ( + "encoding/binary" + "errors" + + "github.com/PeernetOfficial/core/btcec" + "github.com/google/uuid" +) + +// MessageTransfer is the decoded transfer message. +// It is sent to initiate a file transfer, and to send data as part of a file transfer. The actual file data is encapsulated via UDT. +type MessageTransfer struct { + *MessageRaw // Underlying raw message. + Control uint8 // Control. See TransferControlX. + TransferProtocol uint8 // Embedded transfer protocol: 0 = UDT + Hash []byte // Hash of the file to transfer. + Offset uint64 // Offset to start reading at. Only TransferControlRequestStart. + Limit uint64 // Limit (count of bytes) to read starting at the offset. Only TransferControlRequestStart. + TransferID uuid.UUID // Transfer ID to identify lite packets. + Data []byte // Embedded protocol data. Only TransferControlActive. +} + +const ( + TransferControlRequestStart = 0 // Request start transfer of file. Data at byte 34 is offset and limit to read, each 8 bytes. Limit may be 0 to indicate entire file. + TransferControlNotAvailable = 1 // Requested file not available + TransferControlActive = 2 // Active file transfer + TransferControlTerminate = 3 // Terminate +) + +const ( + TransferProtocolUDT = 0 // UDT via lite packets. No encryption. +) + +const transferPayloadHeaderSize = 34 + +// DecodeTransfer decodes a transfer message +func DecodeTransfer(msg *MessageRaw) (result *MessageTransfer, err error) { + if len(msg.Payload) < transferPayloadHeaderSize { + return nil, errors.New("transfer: invalid minimum length") + } + + result = &MessageTransfer{ + MessageRaw: msg, + Hash: make([]byte, HashSize), + } + + result.Control = msg.Payload[0] + result.TransferProtocol = msg.Payload[1] + copy(result.Hash, msg.Payload[2:2+HashSize]) + + switch result.Control { + case TransferControlRequestStart: + // Offset and Limit must be provided after the header. + if len(msg.Payload) < transferPayloadHeaderSize+16 { + return nil, errors.New("transfer: invalid minimum length") + } + + result.Offset = binary.LittleEndian.Uint64(msg.Payload[34 : 34+8]) + result.Limit = binary.LittleEndian.Uint64(msg.Payload[42 : 42+8]) + copy(result.TransferID[:], msg.Payload[50:50+16]) + + case TransferControlActive: + // Data should be transferred via lite packets for performance reasons, but it is allowed to be encapsulated in Peernet packets. + result.Data = msg.Payload[transferPayloadHeaderSize:] + + } + + return result, nil +} + +// TransferMaxEmbedSize is a recommended default upper size of embedded data inside the Transfer message, to be used as MaxPacketSize limit in the embedded protocol. +// This value is chosen as the lowest denominator of different environments (IPv4, IPv6, Ethernet, Internet) for safe transfer, not for highest performance. +// The caller may send bigger payloads but may risk that data packets are simply dropped and never arrive. A MTU negotiation or detection could pimp that. +const TransferMaxEmbedSize = internetSafeMTU - PacketLengthMin - transferPayloadHeaderSize + +// Same as TransferMaxEmbedSize but for encoding via lite packets. +const TransferMaxEmbedSizeLite = internetSafeMTU - PacketLiteSizeMin + +// EncodeTransfer encodes a transfer message. The embedded packet size must be smaller than TransferMaxEmbedSize. +func EncodeTransfer(senderPrivateKey *btcec.PrivateKey, data []byte, control, transferProtocol uint8, hash []byte, offset, limit uint64, transferID uuid.UUID) (packetRaw []byte, err error) { + if control == TransferControlRequestStart && len(data) != 0 { + return nil, errors.New("transfer encode: payload not allowed in start") + } else if isPacketSizeExceed(transferPayloadHeaderSize, len(data)) { + return nil, errors.New("transfer encode: embedded packet too big") + } + + packetSize := transferPayloadHeaderSize + if control == TransferControlRequestStart { + packetSize += 32 + } else if control == TransferControlActive { + packetSize += len(data) + } + + raw := make([]byte, packetSize) + + raw[0] = control + raw[1] = transferProtocol + copy(raw[2:2+HashSize], hash) + + if control == TransferControlRequestStart { + binary.LittleEndian.PutUint64(raw[34:34+8], offset) + binary.LittleEndian.PutUint64(raw[42:42+8], limit) + copy(raw[50:50+16], transferID[:]) + } else if control == TransferControlActive { + copy(raw[34:34+len(data)], data) + } + + return raw, nil +} + +// IsLast checks if the incoming message is the last one in this transfer. +func (msg *MessageTransfer) IsLast() bool { + return msg.Control == TransferControlTerminate || msg.Control == TransferControlNotAvailable +} diff --git a/protocol/Message Encoding Traverse.go b/protocol/Message Encoding Traverse.go index 76a6900..513c917 100644 --- a/protocol/Message Encoding Traverse.go +++ b/protocol/Message Encoding Traverse.go @@ -1,165 +1,165 @@ -/* -File Name: Message Encoding Traverse.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package protocol - -import ( - "encoding/binary" - "errors" - "net" - "time" - - "github.com/PeernetOfficial/core/btcec" -) - -// MessageTraverse is the decoded traverse message. -// It is sent by an original sender to a relay, to a final receiver (targert peer). -type MessageTraverse struct { - *MessageRaw // Underlying raw message. - TargetPeer *btcec.PublicKey // End receiver peer ID. - AuthorizedRelayPeer *btcec.PublicKey // Peer ID that is authorized to relay this message to the end receiver. - Expires time.Time // Expiration time when this forwarded message becomes invalid. - EmbeddedPacketRaw []byte // Embedded packet. - SignerPublicKey *btcec.PublicKey // Public key that signed this message, ECDSA (secp256k1) 257-bit - IPv4 net.IP // IPv4 address of the original sender. Set by authorized relay. 0 if not set. - PortIPv4 uint16 // Port (actual one used for connection) of the original sender. Set by authorized relay. - PortIPv4ReportedExternal uint16 // External port as reported by the original sender. This is used in case of port forwarding (manual or automated). - IPv6 net.IP // IPv6 address of the original sender. Set by authorized relay. 0 if not set. - PortIPv6 uint16 // Port (actual one used for connection) of the original sender. Set by authorized relay. - PortIPv6ReportedExternal uint16 // External port as reported by the original sender. This is used in case of port forwarding (manual or automated). -} - -const traversePayloadHeaderSize = 76 + 65 + 28 - -// DecodeTraverse decodes a traverse message. -// It does not verify if the receiver is authorized to read or forward this message. -// It validates the signature, but does not validate the signer. -func DecodeTraverse(msg *MessageRaw) (result *MessageTraverse, err error) { - result = &MessageTraverse{ - MessageRaw: msg, - } - - if len(msg.Payload) < traversePayloadHeaderSize { - return nil, errors.New("traverse: invalid minimum length") - } - - targetPeerIDcompressed := msg.Payload[0:33] - authorizedRelayPeerIDcompressed := msg.Payload[33:66] - - if result.TargetPeer, err = btcec.ParsePubKey(targetPeerIDcompressed, btcec.S256()); err != nil { - return nil, err - } - if result.AuthorizedRelayPeer, err = btcec.ParsePubKey(authorizedRelayPeerIDcompressed, btcec.S256()); err != nil { - return nil, err - } - - // receiver and target must not be the same - if result.TargetPeer.IsEqual(result.AuthorizedRelayPeer) { - return nil, errors.New("traverse: target and relay invalid") - } - - expires64 := binary.LittleEndian.Uint64(msg.Payload[66 : 66+8]) - result.Expires = time.Unix(int64(expires64), 0) - - sizePacketEmbed := binary.LittleEndian.Uint16(msg.Payload[74 : 74+2]) - if int(sizePacketEmbed) != len(msg.Payload)-traversePayloadHeaderSize { - return nil, errors.New("traverse: size embedded packet mismatch") - } - - result.EmbeddedPacketRaw = msg.Payload[76 : 76+sizePacketEmbed] - - signature := msg.Payload[76+sizePacketEmbed : 76+sizePacketEmbed+65] - - result.SignerPublicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, HashData(msg.Payload[:76+sizePacketEmbed])) - if err != nil { - return nil, err - } - - // IPv4 - ipv4B := make([]byte, 4) - copy(ipv4B[:], msg.Payload[76+sizePacketEmbed+65:76+sizePacketEmbed+65+4]) - - result.IPv4 = ipv4B - result.PortIPv4 = binary.LittleEndian.Uint16(msg.Payload[76+sizePacketEmbed+65+4 : 76+sizePacketEmbed+65+4+2]) - result.PortIPv4ReportedExternal = binary.LittleEndian.Uint16(msg.Payload[76+sizePacketEmbed+65+6 : 76+sizePacketEmbed+65+6+2]) - - // IPv6 - ipv6B := make([]byte, 16) - copy(ipv6B[:], msg.Payload[76+sizePacketEmbed+65+8:76+sizePacketEmbed+65+8+16]) - - result.IPv6 = ipv6B - result.PortIPv6 = binary.LittleEndian.Uint16(msg.Payload[76+sizePacketEmbed+65+24 : 76+sizePacketEmbed+65+24+2]) - result.PortIPv6ReportedExternal = binary.LittleEndian.Uint16(msg.Payload[76+sizePacketEmbed+65+26 : 76+sizePacketEmbed+65+26+2]) - - // TODO: Validate IPv4 and IPv6. Only external ones allowed. - if result.IPv6.To4() != nil { - return nil, errors.New("traverse: ipv6 address mismatch") - } - - return result, nil -} - -// EncodeTraverse encodes a traverse message -func EncodeTraverse(senderPrivateKey *btcec.PrivateKey, embeddedPacketRaw []byte, receiverEnd *btcec.PublicKey, relayPeer *btcec.PublicKey) (packetRaw []byte, err error) { - sizePacketEmbed := len(embeddedPacketRaw) - if isPacketSizeExceed(traversePayloadHeaderSize, sizePacketEmbed) { - return nil, errors.New("traverse encode: embedded packet too big") - } - - raw := make([]byte, traversePayloadHeaderSize+sizePacketEmbed) - - targetPeerID := receiverEnd.SerializeCompressed() - copy(raw[0:33], targetPeerID) - authorizedRelayPeerID := relayPeer.SerializeCompressed() - copy(raw[33:66], authorizedRelayPeerID) - - expires64 := time.Now().Add(time.Hour).UTC().Unix() - binary.LittleEndian.PutUint64(raw[66:66+8], uint64(expires64)) - - binary.LittleEndian.PutUint16(raw[74:74+2], uint16(sizePacketEmbed)) - copy(raw[76:76+sizePacketEmbed], embeddedPacketRaw) - - // add signature - signature, err := btcec.SignCompact(btcec.S256(), senderPrivateKey, HashData(raw[:76+sizePacketEmbed]), true) - if err != nil { - return nil, err - } - copy(raw[76+sizePacketEmbed:76+sizePacketEmbed+65], signature) - - // IP and ports are to be filled by authorized relay peer - - return raw, nil -} - -// EncodeTraverseSetAddress sets the IP and Port in a traverse message that shall be forwarded to another peer -func EncodeTraverseSetAddress(raw []byte, IPv4 net.IP, PortIPv4, PortIPv4ReportedExternal uint16, IPv6 net.IP, PortIPv6, PortIPv6ReportedExternal uint16) (err error) { - if isPacketSizeExceed(len(raw), 0) { - return errors.New("traverse encode 2: embedded packet too big") - } else if len(raw) < traversePayloadHeaderSize { - return errors.New("traverse encode 2: invalid packet") - } - - sizePacketEmbed := binary.LittleEndian.Uint16(raw[74 : 74+2]) - if int(sizePacketEmbed) != len(raw)-traversePayloadHeaderSize { - return errors.New("traverse encode 2: size embedded packet mismatch") - } - - // IPv4 - if IPv4 != nil && len(IPv4) == net.IPv4len { - copy(raw[76+sizePacketEmbed+65:76+sizePacketEmbed+65+4], IPv4.To4()) - binary.LittleEndian.PutUint16(raw[76+sizePacketEmbed+65+4:76+sizePacketEmbed+65+4+2], PortIPv4) - binary.LittleEndian.PutUint16(raw[76+sizePacketEmbed+65+6:76+sizePacketEmbed+65+6+2], PortIPv4ReportedExternal) - } - - // IPv6 - if IPv6 != nil && len(IPv6) == net.IPv6len { - copy(raw[76+sizePacketEmbed+65+8:76+sizePacketEmbed+65+8+16], IPv6.To16()) - binary.LittleEndian.PutUint16(raw[76+sizePacketEmbed+65+24:76+sizePacketEmbed+65+24+2], PortIPv6) - binary.LittleEndian.PutUint16(raw[76+sizePacketEmbed+65+26:76+sizePacketEmbed+65+26+2], PortIPv6ReportedExternal) - } - - return nil -} +/* +File Name: Message Encoding Traverse.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package protocol + +import ( + "encoding/binary" + "errors" + "net" + "time" + + "github.com/PeernetOfficial/core/btcec" +) + +// MessageTraverse is the decoded traverse message. +// It is sent by an original sender to a relay, to a final receiver (targert peer). +type MessageTraverse struct { + *MessageRaw // Underlying raw message. + TargetPeer *btcec.PublicKey // End receiver peer ID. + AuthorizedRelayPeer *btcec.PublicKey // Peer ID that is authorized to relay this message to the end receiver. + Expires time.Time // Expiration time when this forwarded message becomes invalid. + EmbeddedPacketRaw []byte // Embedded packet. + SignerPublicKey *btcec.PublicKey // Public key that signed this message, ECDSA (secp256k1) 257-bit + IPv4 net.IP // IPv4 address of the original sender. Set by authorized relay. 0 if not set. + PortIPv4 uint16 // Port (actual one used for connection) of the original sender. Set by authorized relay. + PortIPv4ReportedExternal uint16 // External port as reported by the original sender. This is used in case of port forwarding (manual or automated). + IPv6 net.IP // IPv6 address of the original sender. Set by authorized relay. 0 if not set. + PortIPv6 uint16 // Port (actual one used for connection) of the original sender. Set by authorized relay. + PortIPv6ReportedExternal uint16 // External port as reported by the original sender. This is used in case of port forwarding (manual or automated). +} + +const traversePayloadHeaderSize = 76 + 65 + 28 + +// DecodeTraverse decodes a traverse message. +// It does not verify if the receiver is authorized to read or forward this message. +// It validates the signature, but does not validate the signer. +func DecodeTraverse(msg *MessageRaw) (result *MessageTraverse, err error) { + result = &MessageTraverse{ + MessageRaw: msg, + } + + if len(msg.Payload) < traversePayloadHeaderSize { + return nil, errors.New("traverse: invalid minimum length") + } + + targetPeerIDcompressed := msg.Payload[0:33] + authorizedRelayPeerIDcompressed := msg.Payload[33:66] + + if result.TargetPeer, err = btcec.ParsePubKey(targetPeerIDcompressed, btcec.S256()); err != nil { + return nil, err + } + if result.AuthorizedRelayPeer, err = btcec.ParsePubKey(authorizedRelayPeerIDcompressed, btcec.S256()); err != nil { + return nil, err + } + + // receiver and target must not be the same + if result.TargetPeer.IsEqual(result.AuthorizedRelayPeer) { + return nil, errors.New("traverse: target and relay invalid") + } + + expires64 := binary.LittleEndian.Uint64(msg.Payload[66 : 66+8]) + result.Expires = time.Unix(int64(expires64), 0) + + sizePacketEmbed := binary.LittleEndian.Uint16(msg.Payload[74 : 74+2]) + if int(sizePacketEmbed) != len(msg.Payload)-traversePayloadHeaderSize { + return nil, errors.New("traverse: size embedded packet mismatch") + } + + result.EmbeddedPacketRaw = msg.Payload[76 : 76+sizePacketEmbed] + + signature := msg.Payload[76+sizePacketEmbed : 76+sizePacketEmbed+65] + + result.SignerPublicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature, HashData(msg.Payload[:76+sizePacketEmbed])) + if err != nil { + return nil, err + } + + // IPv4 + ipv4B := make([]byte, 4) + copy(ipv4B[:], msg.Payload[76+sizePacketEmbed+65:76+sizePacketEmbed+65+4]) + + result.IPv4 = ipv4B + result.PortIPv4 = binary.LittleEndian.Uint16(msg.Payload[76+sizePacketEmbed+65+4 : 76+sizePacketEmbed+65+4+2]) + result.PortIPv4ReportedExternal = binary.LittleEndian.Uint16(msg.Payload[76+sizePacketEmbed+65+6 : 76+sizePacketEmbed+65+6+2]) + + // IPv6 + ipv6B := make([]byte, 16) + copy(ipv6B[:], msg.Payload[76+sizePacketEmbed+65+8:76+sizePacketEmbed+65+8+16]) + + result.IPv6 = ipv6B + result.PortIPv6 = binary.LittleEndian.Uint16(msg.Payload[76+sizePacketEmbed+65+24 : 76+sizePacketEmbed+65+24+2]) + result.PortIPv6ReportedExternal = binary.LittleEndian.Uint16(msg.Payload[76+sizePacketEmbed+65+26 : 76+sizePacketEmbed+65+26+2]) + + // TODO: Validate IPv4 and IPv6. Only external ones allowed. + if result.IPv6.To4() != nil { + return nil, errors.New("traverse: ipv6 address mismatch") + } + + return result, nil +} + +// EncodeTraverse encodes a traverse message +func EncodeTraverse(senderPrivateKey *btcec.PrivateKey, embeddedPacketRaw []byte, receiverEnd *btcec.PublicKey, relayPeer *btcec.PublicKey) (packetRaw []byte, err error) { + sizePacketEmbed := len(embeddedPacketRaw) + if isPacketSizeExceed(traversePayloadHeaderSize, sizePacketEmbed) { + return nil, errors.New("traverse encode: embedded packet too big") + } + + raw := make([]byte, traversePayloadHeaderSize+sizePacketEmbed) + + targetPeerID := receiverEnd.SerializeCompressed() + copy(raw[0:33], targetPeerID) + authorizedRelayPeerID := relayPeer.SerializeCompressed() + copy(raw[33:66], authorizedRelayPeerID) + + expires64 := time.Now().Add(time.Hour).UTC().Unix() + binary.LittleEndian.PutUint64(raw[66:66+8], uint64(expires64)) + + binary.LittleEndian.PutUint16(raw[74:74+2], uint16(sizePacketEmbed)) + copy(raw[76:76+sizePacketEmbed], embeddedPacketRaw) + + // add signature + signature, err := btcec.SignCompact(btcec.S256(), senderPrivateKey, HashData(raw[:76+sizePacketEmbed]), true) + if err != nil { + return nil, err + } + copy(raw[76+sizePacketEmbed:76+sizePacketEmbed+65], signature) + + // IP and ports are to be filled by authorized relay peer + + return raw, nil +} + +// EncodeTraverseSetAddress sets the IP and Port in a traverse message that shall be forwarded to another peer +func EncodeTraverseSetAddress(raw []byte, IPv4 net.IP, PortIPv4, PortIPv4ReportedExternal uint16, IPv6 net.IP, PortIPv6, PortIPv6ReportedExternal uint16) (err error) { + if isPacketSizeExceed(len(raw), 0) { + return errors.New("traverse encode 2: embedded packet too big") + } else if len(raw) < traversePayloadHeaderSize { + return errors.New("traverse encode 2: invalid packet") + } + + sizePacketEmbed := binary.LittleEndian.Uint16(raw[74 : 74+2]) + if int(sizePacketEmbed) != len(raw)-traversePayloadHeaderSize { + return errors.New("traverse encode 2: size embedded packet mismatch") + } + + // IPv4 + if IPv4 != nil && len(IPv4) == net.IPv4len { + copy(raw[76+sizePacketEmbed+65:76+sizePacketEmbed+65+4], IPv4.To4()) + binary.LittleEndian.PutUint16(raw[76+sizePacketEmbed+65+4:76+sizePacketEmbed+65+4+2], PortIPv4) + binary.LittleEndian.PutUint16(raw[76+sizePacketEmbed+65+6:76+sizePacketEmbed+65+6+2], PortIPv4ReportedExternal) + } + + // IPv6 + if IPv6 != nil && len(IPv6) == net.IPv6len { + copy(raw[76+sizePacketEmbed+65+8:76+sizePacketEmbed+65+8+16], IPv6.To16()) + binary.LittleEndian.PutUint16(raw[76+sizePacketEmbed+65+24:76+sizePacketEmbed+65+24+2], PortIPv6) + binary.LittleEndian.PutUint16(raw[76+sizePacketEmbed+65+26:76+sizePacketEmbed+65+26+2], PortIPv6ReportedExternal) + } + + return nil +} diff --git a/protocol/Message Encoding.go b/protocol/Message Encoding.go index 2803ab4..8e306ee 100644 --- a/protocol/Message Encoding.go +++ b/protocol/Message Encoding.go @@ -1,40 +1,40 @@ -/* -File Name: Message Encoding.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Intermediary between low-level packets and high-level interpretation. -*/ - -package protocol - -import ( - "github.com/PeernetOfficial/core/btcec" -) - -// ProtocolVersion is the current protocol version -const ProtocolVersion = 0 - -// MessageRaw is a high-level message between peers that has not been decoded -type MessageRaw struct { - PacketRaw - SenderPublicKey *btcec.PublicKey // Sender Public Key, ECDSA (secp256k1) 257-bit - SequenceInfo *SequenceExpiry // Sequence -} - -// The maximum packet size is = 65535 - 8 UDP byte header - 40 byte IPv6 header (IPv4 header is only 20 bytes). -// However, due to the MTU soft limit and fragmentation, packets should be as small as possible. -const udpMaxPacketSize = 65535 - 8 - 40 - -// internetSafeMTU is a value relatively safe to use for transmitting over the internet -// Theory: The value is different for IPv4 (min 576 bytes, Ethernet 1500 bytes) and IPv6 (min 1280 bytes). 8 byte UDP header must be subtracted, as well as the IP header (20 bytes for IPv4, 40 for IPv6). -// One simple test during development showed that 1500 - 8 - 40 - 8 worked for file transfer over IPv6 in Prague. -// For IPv6 the internet recommends the minimal possible value: 1280 bytes. -// This will be good enough for now. MTU negotiation that deviates from this value can be implemented separately (for example as part of file transfer). -// Since packets may be sent at anytime via IPv4/IPv6 connections (even concurrently on multiple), there is a single MTU value here. -const internetSafeMTU = 1280 - 8 - 40 - -// isPacketSizeExceed checks if the max packet size would be exceeded with the payload -func isPacketSizeExceed(currentSize int, testSize int) bool { - return currentSize+testSize > udpMaxPacketSize-PacketLengthMin -} +/* +File Name: Message Encoding.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Intermediary between low-level packets and high-level interpretation. +*/ + +package protocol + +import ( + "github.com/PeernetOfficial/core/btcec" +) + +// ProtocolVersion is the current protocol version +const ProtocolVersion = 0 + +// MessageRaw is a high-level message between peers that has not been decoded +type MessageRaw struct { + PacketRaw + SenderPublicKey *btcec.PublicKey // Sender Public Key, ECDSA (secp256k1) 257-bit + SequenceInfo *SequenceExpiry // Sequence +} + +// The maximum packet size is = 65535 - 8 UDP byte header - 40 byte IPv6 header (IPv4 header is only 20 bytes). +// However, due to the MTU soft limit and fragmentation, packets should be as small as possible. +const udpMaxPacketSize = 65535 - 8 - 40 + +// internetSafeMTU is a value relatively safe to use for transmitting over the internet +// Theory: The value is different for IPv4 (min 576 bytes, Ethernet 1500 bytes) and IPv6 (min 1280 bytes). 8 byte UDP header must be subtracted, as well as the IP header (20 bytes for IPv4, 40 for IPv6). +// One simple test during development showed that 1500 - 8 - 40 - 8 worked for file transfer over IPv6 in Prague. +// For IPv6 the internet recommends the minimal possible value: 1280 bytes. +// This will be good enough for now. MTU negotiation that deviates from this value can be implemented separately (for example as part of file transfer). +// Since packets may be sent at anytime via IPv4/IPv6 connections (even concurrently on multiple), there is a single MTU value here. +const internetSafeMTU = 1280 - 8 - 40 + +// isPacketSizeExceed checks if the max packet size would be exceeded with the payload +func isPacketSizeExceed(currentSize int, testSize int) bool { + return currentSize+testSize > udpMaxPacketSize-PacketLengthMin +} diff --git a/protocol/Packet Encoding.go b/protocol/Packet Encoding.go index 271c08e..5e3214c 100644 --- a/protocol/Packet Encoding.go +++ b/protocol/Packet Encoding.go @@ -1,157 +1,157 @@ -/* -File Name: Packet Encoding.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Basic packet structure of ALL packets: -Offset Size Info -0 4 Nonce -4 1 Protocol version = 0 -5 1 Command -6 4 Sequence -10 2 Size of payload data -12 ? Payload - ? Randomized garbage -? 65 Signature, ECDSA secp256k1 512-bit + 1 header byte - -The peer ID of the sender, which is a ECDSA (secp256k1) 257-bit public key, can be extracted from the ECDSA signature. -The signature is applied on the entire packet, which guarantees that the signature becomes invalid should someone try to forge the receiver (i.e. forward the packet). -Because the signature could be a possible fingerpint, it is encrypted itself. -*/ - -package protocol - -import ( - "encoding/binary" - "errors" - "math/rand" - - "github.com/PeernetOfficial/core/btcec" - "golang.org/x/crypto/salsa20" -) - -// PacketRaw is a decrypted P2P message -type PacketRaw struct { - Protocol uint8 // Protocol version = 0 - Command uint8 // 0 = Announcement - Sequence uint32 // Sequence number - Payload []byte // Payload -} - -// The minimum packet size is 12 bytes (minimum header size) + 65 bytes (signature) -const PacketLengthMin = 12 + signatureSize -const signatureSize = 65 -const maxRandomGarbage = 20 - -// PacketDecrypt decrypts the packet, verifies its signature and returns a high-level version of the packet. -func PacketDecrypt(raw []byte, receiverPublicKey *btcec.PublicKey) (packet *PacketRaw, senderPublicKey *btcec.PublicKey, err error) { - // Packet is assumed to be already checked for minimum length. - - // Prepare Salsa20 nonce and key. Nonce = 2x first 4 bytes. For size reasons, only 4 bytes (instead of 8 bytes) is supplied in the packet. - // This could be a risk, but considering we only use the PUBLIC key as decryption key, it is negligible. - nonce := make([]byte, 8) - copy(nonce[0:4], raw[0:4]) - copy(nonce[4:8], raw[0:4]) - - // Verify the signature and extract the public key from it. - var signature [signatureSize]byte - copy(signature[:], raw[len(raw)-signatureSize:]) - keySalsa := publicKeyToSalsa20Key(receiverPublicKey) - salsa20.XORKeyStream(signature[:], signature[:], nonce, keySalsa) - - senderPublicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature[:], HashData(raw[:len(raw)-signatureSize])) - if err != nil { - return nil, nil, err - } - - // Decrypt the packet using Salsa20. - bufferDecrypted := make([]byte, len(raw)-signatureSize-4) // full length -signature -nonce - salsa20.XORKeyStream(bufferDecrypted[:], raw[4:len(raw)-signatureSize], nonce, keySalsa) - - // copy all fields - packet = &PacketRaw{Protocol: bufferDecrypted[0], Command: bufferDecrypted[1]} - packet.Sequence = binary.LittleEndian.Uint32(bufferDecrypted[2:6]) - - sizePayload := binary.LittleEndian.Uint16(bufferDecrypted[6:8]) - if int(sizePayload) > len(bufferDecrypted)-8 { // invalid length? - return nil, nil, errors.New("invalid length field") - } - if sizePayload > 0 { - packet.Payload = make([]byte, int(sizePayload)) - copy(packet.Payload, bufferDecrypted[8:8+int(sizePayload)]) - } - - return packet, senderPublicKey, nil -} - -// PacketEncrypt encrypts a packet using the provided senders private key and receivers compressed public key. -func PacketEncrypt(senderPrivateKey *btcec.PrivateKey, receiverPublicKey *btcec.PublicKey, packet *PacketRaw) (raw []byte, err error) { - garbage := packetGarbage(PacketLengthMin + len(packet.Payload)) - raw = make([]byte, PacketLengthMin+len(packet.Payload)+len(garbage)) - - nonceC := rand.Uint32() - nonce := make([]byte, 8) - binary.LittleEndian.PutUint32(nonce[0:4], nonceC) - binary.LittleEndian.PutUint32(nonce[4:8], nonceC) - copy(raw[0:4], nonce[0:4]) - - raw[4] = packet.Protocol - raw[5] = packet.Command - - binary.LittleEndian.PutUint32(raw[6:10], uint32(packet.Sequence)) - binary.LittleEndian.PutUint16(raw[10:12], uint16(len(packet.Payload))) - copy(raw[12:], packet.Payload) - copy(raw[12+len(packet.Payload):12+len(packet.Payload)+len(garbage)], garbage) - - // encrypt it using Salsa20 - keySalsa := publicKeyToSalsa20Key(receiverPublicKey) - salsa20.XORKeyStream(raw[4:12+len(packet.Payload)+len(garbage)], raw[4:12+len(packet.Payload)+len(garbage)], nonce, keySalsa) - - // add signature - signature, err := btcec.SignCompact(btcec.S256(), senderPrivateKey, HashData(raw[:len(raw)-signatureSize]), true) - if err != nil { - return nil, err - } - - salsa20.XORKeyStream(signature[:], signature[:], nonce, keySalsa) - copy(raw[len(raw)-signatureSize:], signature) - - return raw, nil -} - -func packetGarbage(packetLength int) (random []byte) { - // Align maximum length at 508 bytes (UDP minimum no fragmentation) and at a relatively safe MTU. - maxLength := maxRandomGarbage - switch { - case packetLength == 508, packetLength == internetSafeMTU: - return nil - case packetLength < 508 && (508-packetLength) < maxRandomGarbage: - maxLength = 508 - packetLength - case packetLength < internetSafeMTU && (internetSafeMTU-packetLength) < maxRandomGarbage: - maxLength = internetSafeMTU - packetLength - } - - b := make([]byte, rand.Intn(maxLength)) - if _, err := rand.Read(b); err != nil { - return nil - } - return b -} - -func publicKeyToSalsa20Key(publicKey *btcec.PublicKey) (key *[32]byte) { - // bit 0 from PublicKey.Y is ignored here, but is negligible for this purpose - key = new([32]byte) - copy(key[:], publicKey.SerializeCompressed()[1:]) - return key -} - -// SetSelfReportedPorts sets the fields Internal Port and External Port according to the connection details. -// This is important for the remote peer to make smart decisions whether this peer is behind a NAT/firewall and supports port forwarding/UPnP. -func (packet *PacketRaw) SetSelfReportedPorts(portI, portE uint16) { - if packet.Command != CommandAnnouncement && packet.Command != CommandResponse { // only for Announcement and Response messages - return - } - - binary.LittleEndian.PutUint16(packet.Payload[19:19+2], portI) - binary.LittleEndian.PutUint16(packet.Payload[21:21+2], portE) -} +/* +File Name: Packet Encoding.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Basic packet structure of ALL packets: +Offset Size Info +0 4 Nonce +4 1 Protocol version = 0 +5 1 Command +6 4 Sequence +10 2 Size of payload data +12 ? Payload + ? Randomized garbage +? 65 Signature, ECDSA secp256k1 512-bit + 1 header byte + +The peer ID of the sender, which is a ECDSA (secp256k1) 257-bit public key, can be extracted from the ECDSA signature. +The signature is applied on the entire packet, which guarantees that the signature becomes invalid should someone try to forge the receiver (i.e. forward the packet). +Because the signature could be a possible fingerpint, it is encrypted itself. +*/ + +package protocol + +import ( + "encoding/binary" + "errors" + "math/rand" + + "github.com/PeernetOfficial/core/btcec" + "golang.org/x/crypto/salsa20" +) + +// PacketRaw is a decrypted P2P message +type PacketRaw struct { + Protocol uint8 // Protocol version = 0 + Command uint8 // 0 = Announcement + Sequence uint32 // Sequence number + Payload []byte // Payload +} + +// The minimum packet size is 12 bytes (minimum header size) + 65 bytes (signature) +const PacketLengthMin = 12 + signatureSize +const signatureSize = 65 +const maxRandomGarbage = 20 + +// PacketDecrypt decrypts the packet, verifies its signature and returns a high-level version of the packet. +func PacketDecrypt(raw []byte, receiverPublicKey *btcec.PublicKey) (packet *PacketRaw, senderPublicKey *btcec.PublicKey, err error) { + // Packet is assumed to be already checked for minimum length. + + // Prepare Salsa20 nonce and key. Nonce = 2x first 4 bytes. For size reasons, only 4 bytes (instead of 8 bytes) is supplied in the packet. + // This could be a risk, but considering we only use the PUBLIC key as decryption key, it is negligible. + nonce := make([]byte, 8) + copy(nonce[0:4], raw[0:4]) + copy(nonce[4:8], raw[0:4]) + + // Verify the signature and extract the public key from it. + var signature [signatureSize]byte + copy(signature[:], raw[len(raw)-signatureSize:]) + keySalsa := publicKeyToSalsa20Key(receiverPublicKey) + salsa20.XORKeyStream(signature[:], signature[:], nonce, keySalsa) + + senderPublicKey, _, err = btcec.RecoverCompact(btcec.S256(), signature[:], HashData(raw[:len(raw)-signatureSize])) + if err != nil { + return nil, nil, err + } + + // Decrypt the packet using Salsa20. + bufferDecrypted := make([]byte, len(raw)-signatureSize-4) // full length -signature -nonce + salsa20.XORKeyStream(bufferDecrypted[:], raw[4:len(raw)-signatureSize], nonce, keySalsa) + + // copy all fields + packet = &PacketRaw{Protocol: bufferDecrypted[0], Command: bufferDecrypted[1]} + packet.Sequence = binary.LittleEndian.Uint32(bufferDecrypted[2:6]) + + sizePayload := binary.LittleEndian.Uint16(bufferDecrypted[6:8]) + if int(sizePayload) > len(bufferDecrypted)-8 { // invalid length? + return nil, nil, errors.New("invalid length field") + } + if sizePayload > 0 { + packet.Payload = make([]byte, int(sizePayload)) + copy(packet.Payload, bufferDecrypted[8:8+int(sizePayload)]) + } + + return packet, senderPublicKey, nil +} + +// PacketEncrypt encrypts a packet using the provided senders private key and receivers compressed public key. +func PacketEncrypt(senderPrivateKey *btcec.PrivateKey, receiverPublicKey *btcec.PublicKey, packet *PacketRaw) (raw []byte, err error) { + garbage := packetGarbage(PacketLengthMin + len(packet.Payload)) + raw = make([]byte, PacketLengthMin+len(packet.Payload)+len(garbage)) + + nonceC := rand.Uint32() + nonce := make([]byte, 8) + binary.LittleEndian.PutUint32(nonce[0:4], nonceC) + binary.LittleEndian.PutUint32(nonce[4:8], nonceC) + copy(raw[0:4], nonce[0:4]) + + raw[4] = packet.Protocol + raw[5] = packet.Command + + binary.LittleEndian.PutUint32(raw[6:10], uint32(packet.Sequence)) + binary.LittleEndian.PutUint16(raw[10:12], uint16(len(packet.Payload))) + copy(raw[12:], packet.Payload) + copy(raw[12+len(packet.Payload):12+len(packet.Payload)+len(garbage)], garbage) + + // encrypt it using Salsa20 + keySalsa := publicKeyToSalsa20Key(receiverPublicKey) + salsa20.XORKeyStream(raw[4:12+len(packet.Payload)+len(garbage)], raw[4:12+len(packet.Payload)+len(garbage)], nonce, keySalsa) + + // add signature + signature, err := btcec.SignCompact(btcec.S256(), senderPrivateKey, HashData(raw[:len(raw)-signatureSize]), true) + if err != nil { + return nil, err + } + + salsa20.XORKeyStream(signature[:], signature[:], nonce, keySalsa) + copy(raw[len(raw)-signatureSize:], signature) + + return raw, nil +} + +func packetGarbage(packetLength int) (random []byte) { + // Align maximum length at 508 bytes (UDP minimum no fragmentation) and at a relatively safe MTU. + maxLength := maxRandomGarbage + switch { + case packetLength == 508, packetLength == internetSafeMTU: + return nil + case packetLength < 508 && (508-packetLength) < maxRandomGarbage: + maxLength = 508 - packetLength + case packetLength < internetSafeMTU && (internetSafeMTU-packetLength) < maxRandomGarbage: + maxLength = internetSafeMTU - packetLength + } + + b := make([]byte, rand.Intn(maxLength)) + if _, err := rand.Read(b); err != nil { + return nil + } + return b +} + +func publicKeyToSalsa20Key(publicKey *btcec.PublicKey) (key *[32]byte) { + // bit 0 from PublicKey.Y is ignored here, but is negligible for this purpose + key = new([32]byte) + copy(key[:], publicKey.SerializeCompressed()[1:]) + return key +} + +// SetSelfReportedPorts sets the fields Internal Port and External Port according to the connection details. +// This is important for the remote peer to make smart decisions whether this peer is behind a NAT/firewall and supports port forwarding/UPnP. +func (packet *PacketRaw) SetSelfReportedPorts(portI, portE uint16) { + if packet.Command != CommandAnnouncement && packet.Command != CommandResponse { // only for Announcement and Response messages + return + } + + binary.LittleEndian.PutUint16(packet.Payload[19:19+2], portI) + binary.LittleEndian.PutUint16(packet.Payload[21:21+2], portE) +} diff --git a/protocol/Packet Lite.go b/protocol/Packet Lite.go index 5996d69..31eb1a5 100644 --- a/protocol/Packet Lite.go +++ b/protocol/Packet Lite.go @@ -1,199 +1,199 @@ -/* -File Name: Packet Lite.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -The lite packet header is used for encoding data transfer packets. The regular header is too expensive in terms of CPU consumption due to public key signing. -Instead, a simple session ID will identify lite packets. The ID is randomized and only valid during the session. -Unsolicited lite packets are therefore impossible; the receiver must have the ID already whitelisted for the packet to be recognized. - -Offset Size Info -0 16 ID -16 2 Size of data to follow - -*/ - -package protocol - -import ( - "encoding/binary" - "errors" - "sync" - "time" - - "github.com/google/uuid" -) - -// PacketLiteRaw is a decrypted P2P lite packet -type PacketLiteRaw struct { - ID uuid.UUID // ID - Payload []byte // Payload - Session *LiteID // Session info -} - -// Minimum packet size of lite packets. -const PacketLiteSizeMin = 16 + 2 - -// IsPacketLite identifies a lite packet based on its ID. If the ID is not recognized, it fails. -func (router *LiteRouter) IsPacketLite(raw []byte) (isLite bool, err error) { - if len(raw) < PacketLiteSizeMin { - return false, errors.New("invalid packet size") - } - - // Parse the ID and then look it up. - var id uuid.UUID - copy(id[:], raw[0:16]) - - return router.LookupLiteID(id) != nil, nil -} - -// PacketLiteDecode a lite packet. It will identify the lite packet based on its ID. If the ID is not recognized (which is the case for regular Peernet packets), the function fails. -// It does not perform any decryption. -func (router *LiteRouter) PacketLiteDecode(raw []byte) (packet *PacketLiteRaw, err error) { - if len(raw) < PacketLiteSizeMin { - return nil, errors.New("invalid packet size") - } - - // Parse the ID and look it up. It will contain information about the decryption algorithm to use. - var id uuid.UUID - copy(id[:], raw[0:16]) - - session := router.LookupLiteID(id) - if session == nil { - return nil, errors.New("packet ID not found") - } - - // TODO: Decrypt the data if indicated by the session. - - sizePayload := binary.LittleEndian.Uint16(raw[16 : 16+2]) - if int(sizePayload) > len(raw)-PacketLiteSizeMin { // invalid size field? - return nil, errors.New("invalid packet size field") - } - - // Valid packet received, extend expiration. - session.expires = time.Now().Add(session.timeout) - - return &PacketLiteRaw{Payload: raw[PacketLiteSizeMin:], ID: id, Session: session}, nil -} - -// Encodes a lite packet. -func PacketLiteEncode(id uuid.UUID, data []byte) (raw []byte, err error) { - raw = make([]byte, PacketLiteSizeMin+len(data)) - - copy(raw[0:16], id[:]) - binary.LittleEndian.PutUint16(raw[16:16+2], uint16(len(data))) - copy(raw[PacketLiteSizeMin:], data) - - return raw, nil -} - -// ---- Lite packet ID management. This is similar to packet sequences. ---- - -// LiteRouter keeps track of accepted (expected) packet IDs. -type LiteRouter struct { - // list of recognized IDs - ids map[uuid.UUID]*LiteID - - sync.Mutex // synchronized access to the IDs -} - -// LiteID contains session information for a bidirectional transfer of data -type LiteID struct { - ID uuid.UUID // ID - created time.Time // When the ID was created. - expires time.Time // When the ID expires. This can be extended on the fly! - Data interface{} // Optional high-level data associated with the ID - timeout time.Duration // Timeout for receiving the next message - invalidateFunc func() // Called on expiration. -} - -// Creates a new manager to keep track of accepted IDs. -func NewLiteRouter() (router *LiteRouter) { - router = &LiteRouter{ - ids: make(map[uuid.UUID]*LiteID), - } - - go router.autoDeleteExpired() - - return -} - -// autoDeleteExpired deletes all IDs that are expired. -func (router *LiteRouter) autoDeleteExpired() { - for { - time.Sleep(4 * time.Second) - now := time.Now() - - router.Lock() - for id, info := range router.ids { - if info.expires.Before(now) { - delete(router.ids, id) - - if info.invalidateFunc != nil { - go info.invalidateFunc() - } - } - } - router.Unlock() - } -} - -func (router *LiteRouter) LookupLiteID(id uuid.UUID) (info *LiteID) { - router.Lock() - info = router.ids[id] - router.Unlock() - - return info -} - -// Returns a new lite ID to be used. -func (router *LiteRouter) NewLiteID(data interface{}, timeout time.Duration, invalidateFunc func()) (info *LiteID) { - info = &LiteID{ - created: time.Now(), - expires: time.Now().Add(timeout), - timeout: timeout, - invalidateFunc: invalidateFunc, - Data: data, - ID: uuid.New(), - } - - router.Lock() - router.ids[info.ID] = info - router.Unlock() - - return -} - -func (router *LiteRouter) RegisterLiteID(id uuid.UUID, data interface{}, timeout time.Duration, invalidateFunc func()) (info *LiteID) { - info = &LiteID{ - ID: id, - created: time.Now(), - expires: time.Now().Add(timeout), - timeout: timeout, - invalidateFunc: invalidateFunc, - Data: data, - } - - router.Lock() - existingInfo := router.ids[info.ID] - router.ids[info.ID] = info - router.Unlock() - - // Call the invalidate function if there is a collision. This should never happen. - if existingInfo != nil && existingInfo.invalidateFunc != nil { - go existingInfo.invalidateFunc() - } - - return -} - -// Returns all lite sessions -func (router *LiteRouter) All() (sessions []*LiteID) { - router.Lock() - for _, info := range router.ids { - sessions = append(sessions, info) - } - router.Unlock() - - return sessions -} +/* +File Name: Packet Lite.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +The lite packet header is used for encoding data transfer packets. The regular header is too expensive in terms of CPU consumption due to public key signing. +Instead, a simple session ID will identify lite packets. The ID is randomized and only valid during the session. +Unsolicited lite packets are therefore impossible; the receiver must have the ID already whitelisted for the packet to be recognized. + +Offset Size Info +0 16 ID +16 2 Size of data to follow + +*/ + +package protocol + +import ( + "encoding/binary" + "errors" + "sync" + "time" + + "github.com/google/uuid" +) + +// PacketLiteRaw is a decrypted P2P lite packet +type PacketLiteRaw struct { + ID uuid.UUID // ID + Payload []byte // Payload + Session *LiteID // Session info +} + +// Minimum packet size of lite packets. +const PacketLiteSizeMin = 16 + 2 + +// IsPacketLite identifies a lite packet based on its ID. If the ID is not recognized, it fails. +func (router *LiteRouter) IsPacketLite(raw []byte) (isLite bool, err error) { + if len(raw) < PacketLiteSizeMin { + return false, errors.New("invalid packet size") + } + + // Parse the ID and then look it up. + var id uuid.UUID + copy(id[:], raw[0:16]) + + return router.LookupLiteID(id) != nil, nil +} + +// PacketLiteDecode a lite packet. It will identify the lite packet based on its ID. If the ID is not recognized (which is the case for regular Peernet packets), the function fails. +// It does not perform any decryption. +func (router *LiteRouter) PacketLiteDecode(raw []byte) (packet *PacketLiteRaw, err error) { + if len(raw) < PacketLiteSizeMin { + return nil, errors.New("invalid packet size") + } + + // Parse the ID and look it up. It will contain information about the decryption algorithm to use. + var id uuid.UUID + copy(id[:], raw[0:16]) + + session := router.LookupLiteID(id) + if session == nil { + return nil, errors.New("packet ID not found") + } + + // TODO: Decrypt the data if indicated by the session. + + sizePayload := binary.LittleEndian.Uint16(raw[16 : 16+2]) + if int(sizePayload) > len(raw)-PacketLiteSizeMin { // invalid size field? + return nil, errors.New("invalid packet size field") + } + + // Valid packet received, extend expiration. + session.expires = time.Now().Add(session.timeout) + + return &PacketLiteRaw{Payload: raw[PacketLiteSizeMin:], ID: id, Session: session}, nil +} + +// Encodes a lite packet. +func PacketLiteEncode(id uuid.UUID, data []byte) (raw []byte, err error) { + raw = make([]byte, PacketLiteSizeMin+len(data)) + + copy(raw[0:16], id[:]) + binary.LittleEndian.PutUint16(raw[16:16+2], uint16(len(data))) + copy(raw[PacketLiteSizeMin:], data) + + return raw, nil +} + +// ---- Lite packet ID management. This is similar to packet sequences. ---- + +// LiteRouter keeps track of accepted (expected) packet IDs. +type LiteRouter struct { + // list of recognized IDs + ids map[uuid.UUID]*LiteID + + sync.Mutex // synchronized access to the IDs +} + +// LiteID contains session information for a bidirectional transfer of data +type LiteID struct { + ID uuid.UUID // ID + created time.Time // When the ID was created. + expires time.Time // When the ID expires. This can be extended on the fly! + Data interface{} // Optional high-level data associated with the ID + timeout time.Duration // Timeout for receiving the next message + invalidateFunc func() // Called on expiration. +} + +// Creates a new manager to keep track of accepted IDs. +func NewLiteRouter() (router *LiteRouter) { + router = &LiteRouter{ + ids: make(map[uuid.UUID]*LiteID), + } + + go router.autoDeleteExpired() + + return +} + +// autoDeleteExpired deletes all IDs that are expired. +func (router *LiteRouter) autoDeleteExpired() { + for { + time.Sleep(4 * time.Second) + now := time.Now() + + router.Lock() + for id, info := range router.ids { + if info.expires.Before(now) { + delete(router.ids, id) + + if info.invalidateFunc != nil { + go info.invalidateFunc() + } + } + } + router.Unlock() + } +} + +func (router *LiteRouter) LookupLiteID(id uuid.UUID) (info *LiteID) { + router.Lock() + info = router.ids[id] + router.Unlock() + + return info +} + +// Returns a new lite ID to be used. +func (router *LiteRouter) NewLiteID(data interface{}, timeout time.Duration, invalidateFunc func()) (info *LiteID) { + info = &LiteID{ + created: time.Now(), + expires: time.Now().Add(timeout), + timeout: timeout, + invalidateFunc: invalidateFunc, + Data: data, + ID: uuid.New(), + } + + router.Lock() + router.ids[info.ID] = info + router.Unlock() + + return +} + +func (router *LiteRouter) RegisterLiteID(id uuid.UUID, data interface{}, timeout time.Duration, invalidateFunc func()) (info *LiteID) { + info = &LiteID{ + ID: id, + created: time.Now(), + expires: time.Now().Add(timeout), + timeout: timeout, + invalidateFunc: invalidateFunc, + Data: data, + } + + router.Lock() + existingInfo := router.ids[info.ID] + router.ids[info.ID] = info + router.Unlock() + + // Call the invalidate function if there is a collision. This should never happen. + if existingInfo != nil && existingInfo.invalidateFunc != nil { + go existingInfo.invalidateFunc() + } + + return +} + +// Returns all lite sessions +func (router *LiteRouter) All() (sessions []*LiteID) { + router.Lock() + for _, info := range router.ids { + sessions = append(sessions, info) + } + router.Unlock() + + return sessions +} diff --git a/protocol/Sequence.go b/protocol/Sequence.go index 4017536..fe6c557 100644 --- a/protocol/Sequence.go +++ b/protocol/Sequence.go @@ -1,262 +1,262 @@ -/* -File Name: Sequence.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -This code caches and verifies message sequences. Sequence numbers are valid on a peer level, independent of which network connection was used. -They can be used to map incoming response messages to previous outgoing requests. The remote peer ID is used together with a consecutive sequence number as unique key. - -Sequences are created for: -* Unidirectional messages (responses are one-way) -* Bidirectional messages (responses may be sent back and forth). They use a different key namespace since remote sequence numbers could collide with locally created ones. - -Advantages: -* This secures against replay and poisoning attacks. -* If used correctly it can also deduplicate messages (which occurs when 2 peers have multiple registered connections to each other but none are active and subsequent fallback to broadcast). -* The round-trip time can be measured and used to determine the connection quality. -* (future) It can be used to detect missed and lost replies. - -*/ - -package protocol - -import ( - "math/rand" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/PeernetOfficial/core/btcec" -) - -// SequenceManager stores all message sequence numbers that are valid at the moment -type SequenceManager struct { - ReplyTimeout int // The round-trip timeout for message sequences. - - // sequences is the list of sequence numbers that are valid at the moment. The value represents the time the sequence number. - // Key = Peer ID + Sequence Number - sequences map[string]*SequenceExpiry - - sync.Mutex // synchronized access to the sequences -} - -// SequenceExpiry contains the decoded sequence information of a message. -type SequenceExpiry struct { - SequenceNumber uint32 // Sequence number - created time.Time // When the sequence was created. - expires time.Time // When the sequence expires. This can be extended on the fly! - counter int // How many replies used the sequence. Multiple Response messages may be returned for a single Announcement one. - Data interface{} // Optional high-level data associated with the sequence - // bidirectional sequences only - bidirectional bool // Whether this sequence is used in a bidirectional way - timeout time.Duration // Timeout for receiving the next message - invalidateFunc func() // The invalidation callback is in case a sequence collision or expiration invalidates the sequence. -} - -// NewSequenceManager creates a new sequence manager. The ReplyTimeout is in seconds. The expiration function is started immediately. -func NewSequenceManager(ReplyTimeout int) (manager *SequenceManager) { - manager = &SequenceManager{ - ReplyTimeout: ReplyTimeout, - sequences: make(map[string]*SequenceExpiry), - } - - go manager.autoDeleteExpired() - - return -} - -// autoDeleteExpired deletes all sequences that are expired. -func (manager *SequenceManager) autoDeleteExpired() { - for { - time.Sleep(time.Duration(manager.ReplyTimeout) * time.Second) - now := time.Now() - - manager.Lock() - for key, sequence := range manager.sequences { - if sequence.expires.Before(now) { - delete(manager.sequences, key) - - if sequence.invalidateFunc != nil { - go sequence.invalidateFunc() - } - } - } - manager.Unlock() - } -} - -// sequence2Key creates the lookup key of a sequence for a peer. -// Since bidirectional sequence numbers may be created from either side (remote or local peer), it does not share a namespace with unidirectional sequence numbers. -func sequence2Key(bidirectional bool, publicKey *btcec.PublicKey, sequenceNumber uint32) (key string) { - if !bidirectional { - return "u" + string(publicKey.SerializeCompressed()) + strconv.FormatUint(uint64(sequenceNumber), 10) - } else { - return "b" + string(publicKey.SerializeCompressed()) + strconv.FormatUint(uint64(sequenceNumber), 10) - } -} - -// NewSequence returns a new sequence and registers it. messageSequence must point to the variable holding the continuous next sequence number. -// Use only for Announcement and Ping messages. -func (manager *SequenceManager) NewSequence(publicKey *btcec.PublicKey, messageSequence *uint32, data interface{}) (info *SequenceExpiry) { - info = &SequenceExpiry{ - SequenceNumber: atomic.AddUint32(messageSequence, 1), - created: time.Now(), - expires: time.Now().Add(time.Duration(manager.ReplyTimeout) * time.Second), - Data: data, - } - - // Add the sequence to the list. Sequences are unique enough that collisions are unlikely and negligible. - key := sequence2Key(false, publicKey, info.SequenceNumber) - manager.Lock() - manager.sequences[key] = info - manager.Unlock() - - return -} - -// ArbitrarySequence returns an arbitrary sequence to be used for uncontacted peers -func (manager *SequenceManager) ArbitrarySequence(publicKey *btcec.PublicKey, data interface{}) (info *SequenceExpiry) { - info = &SequenceExpiry{ - SequenceNumber: rand.Uint32(), - created: time.Now(), - expires: time.Now().Add(time.Duration(manager.ReplyTimeout) * time.Second), - Data: data, - } - - // Add the sequence to the list. Sequences are unique enough that collisions are unlikely and negligible. - key := sequence2Key(false, publicKey, info.SequenceNumber) - manager.Lock() - manager.sequences[key] = info - manager.Unlock() - - return -} - -// ValidateSequence validates the sequence number of an incoming message. It will set raw.sequence if valid. -func (manager *SequenceManager) ValidateSequence(publicKey *btcec.PublicKey, sequenceNumber uint32, invalidate, extendValidity bool) (sequenceInfo *SequenceExpiry, valid bool, rtt time.Duration) { - key := sequence2Key(false, publicKey, sequenceNumber) - - manager.Lock() - defer manager.Unlock() - - // lookup the sequence - sequence, ok := manager.sequences[key] - if !ok { - return nil, false, rtt - } - - // Initial reply: Store latest roundtrip time. That value might be distorted on Response vs Pong since Response messages might send data - // up to 64 KB which obviously would be transmitted slower than an empty Pong reply. However, for the real world this is good enough. - if sequence.counter == 0 { - rtt = time.Since(sequence.created) - } - - sequence.counter++ - - // invalidate the sequence immediately? - if invalidate { - delete(manager.sequences, key) - } else if extendValidity { - // Special case CommandResponse: Extend validity in case there are follow-up responses, by half of the round-trip time since they will be sent one-way. - sequence.expires = time.Now().Add(time.Duration(manager.ReplyTimeout) * time.Second / 2) - } - - return sequence, sequence.expires.After(time.Now()), rtt -} - -// InvalidateSequence invalidates the sequence number. It does not call invalidateFunc. -func (manager *SequenceManager) InvalidateSequence(publicKey *btcec.PublicKey, sequenceNumber uint32, bidirectional bool) { - key := sequence2Key(bidirectional, publicKey, sequenceNumber) - - manager.Lock() - delete(manager.sequences, key) - manager.Unlock() -} - -// ---- bidirectional sequences ---- - -// RegisterSequenceBi registers a bidirectional sequence initiated by a remote peer. The caller must specify the timeout (which will be reset every time a new message appears in this sequence). -// This is needed for bidirectional responses to accept subsequent incoming messages from the remote peer. -func (manager *SequenceManager) RegisterSequenceBi(publicKey *btcec.PublicKey, sequenceNumber uint32, data interface{}, timeout time.Duration, invalidateFunc func()) (info *SequenceExpiry) { - info = &SequenceExpiry{ - SequenceNumber: sequenceNumber, - created: time.Now(), - expires: time.Now().Add(timeout), - timeout: timeout, - invalidateFunc: invalidateFunc, - Data: data, - } - - // Before registering the sequence, check if there is a collision. If yes, invalidate the original one. - key := sequence2Key(true, publicKey, info.SequenceNumber) - manager.Lock() - existingSequence := manager.sequences[key] - manager.sequences[key] = info - manager.Unlock() - - // Call the invalidate function if there is a collision. - if existingSequence != nil && existingSequence.invalidateFunc != nil { - go existingSequence.invalidateFunc() - } - - return -} - -// NewSequenceBi returns a new bidirectional sequence and registers it. messageSequence must point to the variable holding the continuous next sequence number. -func (manager *SequenceManager) NewSequenceBi(publicKey *btcec.PublicKey, messageSequence *uint32, data interface{}, timeout time.Duration, invalidateFunc func()) (info *SequenceExpiry) { - info = &SequenceExpiry{ - created: time.Now(), - expires: time.Now().Add(timeout), - bidirectional: true, - timeout: timeout, - invalidateFunc: invalidateFunc, - Data: data, - } - - manager.Lock() - defer manager.Unlock() - - // The likelihood of a collision is low but not impossible. - for n := 0; n < 10000; n++ { - info.SequenceNumber = atomic.AddUint32(messageSequence, 1) - - key := sequence2Key(true, publicKey, info.SequenceNumber) - if infoE := manager.sequences[key]; infoE == nil { - manager.sequences[key] = info - return info - } - } - - return nil -} - -// ValidateSequenceBi validates the sequence number of an incoming message. It will set raw.sequence if valid. -func (manager *SequenceManager) ValidateSequenceBi(publicKey *btcec.PublicKey, sequenceNumber uint32, isLast bool) (sequenceInfo *SequenceExpiry, valid bool, rtt time.Duration) { - key := sequence2Key(true, publicKey, sequenceNumber) - - manager.Lock() - defer manager.Unlock() - - // lookup the sequence - sequence, ok := manager.sequences[key] - if !ok { - return nil, false, rtt - } - - // Initial reply: Store latest roundtrip time. - if sequence.counter == 0 { - rtt = time.Since(sequence.created) - } - - sequence.counter++ - - // invalidate the sequence immediately? - if isLast { - delete(manager.sequences, key) - } else { - sequence.expires = time.Now().Add(sequence.timeout) - } - - return sequence, sequence.expires.After(time.Now()), rtt -} +/* +File Name: Sequence.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +This code caches and verifies message sequences. Sequence numbers are valid on a peer level, independent of which network connection was used. +They can be used to map incoming response messages to previous outgoing requests. The remote peer ID is used together with a consecutive sequence number as unique key. + +Sequences are created for: +* Unidirectional messages (responses are one-way) +* Bidirectional messages (responses may be sent back and forth). They use a different key namespace since remote sequence numbers could collide with locally created ones. + +Advantages: +* This secures against replay and poisoning attacks. +* If used correctly it can also deduplicate messages (which occurs when 2 peers have multiple registered connections to each other but none are active and subsequent fallback to broadcast). +* The round-trip time can be measured and used to determine the connection quality. +* (future) It can be used to detect missed and lost replies. + +*/ + +package protocol + +import ( + "math/rand" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/PeernetOfficial/core/btcec" +) + +// SequenceManager stores all message sequence numbers that are valid at the moment +type SequenceManager struct { + ReplyTimeout int // The round-trip timeout for message sequences. + + // sequences is the list of sequence numbers that are valid at the moment. The value represents the time the sequence number. + // Key = Peer ID + Sequence Number + sequences map[string]*SequenceExpiry + + sync.Mutex // synchronized access to the sequences +} + +// SequenceExpiry contains the decoded sequence information of a message. +type SequenceExpiry struct { + SequenceNumber uint32 // Sequence number + created time.Time // When the sequence was created. + expires time.Time // When the sequence expires. This can be extended on the fly! + counter int // How many replies used the sequence. Multiple Response messages may be returned for a single Announcement one. + Data interface{} // Optional high-level data associated with the sequence + // bidirectional sequences only + bidirectional bool // Whether this sequence is used in a bidirectional way + timeout time.Duration // Timeout for receiving the next message + invalidateFunc func() // The invalidation callback is in case a sequence collision or expiration invalidates the sequence. +} + +// NewSequenceManager creates a new sequence manager. The ReplyTimeout is in seconds. The expiration function is started immediately. +func NewSequenceManager(ReplyTimeout int) (manager *SequenceManager) { + manager = &SequenceManager{ + ReplyTimeout: ReplyTimeout, + sequences: make(map[string]*SequenceExpiry), + } + + go manager.autoDeleteExpired() + + return +} + +// autoDeleteExpired deletes all sequences that are expired. +func (manager *SequenceManager) autoDeleteExpired() { + for { + time.Sleep(time.Duration(manager.ReplyTimeout) * time.Second) + now := time.Now() + + manager.Lock() + for key, sequence := range manager.sequences { + if sequence.expires.Before(now) { + delete(manager.sequences, key) + + if sequence.invalidateFunc != nil { + go sequence.invalidateFunc() + } + } + } + manager.Unlock() + } +} + +// sequence2Key creates the lookup key of a sequence for a peer. +// Since bidirectional sequence numbers may be created from either side (remote or local peer), it does not share a namespace with unidirectional sequence numbers. +func sequence2Key(bidirectional bool, publicKey *btcec.PublicKey, sequenceNumber uint32) (key string) { + if !bidirectional { + return "u" + string(publicKey.SerializeCompressed()) + strconv.FormatUint(uint64(sequenceNumber), 10) + } else { + return "b" + string(publicKey.SerializeCompressed()) + strconv.FormatUint(uint64(sequenceNumber), 10) + } +} + +// NewSequence returns a new sequence and registers it. messageSequence must point to the variable holding the continuous next sequence number. +// Use only for Announcement and Ping messages. +func (manager *SequenceManager) NewSequence(publicKey *btcec.PublicKey, messageSequence *uint32, data interface{}) (info *SequenceExpiry) { + info = &SequenceExpiry{ + SequenceNumber: atomic.AddUint32(messageSequence, 1), + created: time.Now(), + expires: time.Now().Add(time.Duration(manager.ReplyTimeout) * time.Second), + Data: data, + } + + // Add the sequence to the list. Sequences are unique enough that collisions are unlikely and negligible. + key := sequence2Key(false, publicKey, info.SequenceNumber) + manager.Lock() + manager.sequences[key] = info + manager.Unlock() + + return +} + +// ArbitrarySequence returns an arbitrary sequence to be used for uncontacted peers +func (manager *SequenceManager) ArbitrarySequence(publicKey *btcec.PublicKey, data interface{}) (info *SequenceExpiry) { + info = &SequenceExpiry{ + SequenceNumber: rand.Uint32(), + created: time.Now(), + expires: time.Now().Add(time.Duration(manager.ReplyTimeout) * time.Second), + Data: data, + } + + // Add the sequence to the list. Sequences are unique enough that collisions are unlikely and negligible. + key := sequence2Key(false, publicKey, info.SequenceNumber) + manager.Lock() + manager.sequences[key] = info + manager.Unlock() + + return +} + +// ValidateSequence validates the sequence number of an incoming message. It will set raw.sequence if valid. +func (manager *SequenceManager) ValidateSequence(publicKey *btcec.PublicKey, sequenceNumber uint32, invalidate, extendValidity bool) (sequenceInfo *SequenceExpiry, valid bool, rtt time.Duration) { + key := sequence2Key(false, publicKey, sequenceNumber) + + manager.Lock() + defer manager.Unlock() + + // lookup the sequence + sequence, ok := manager.sequences[key] + if !ok { + return nil, false, rtt + } + + // Initial reply: Store latest roundtrip time. That value might be distorted on Response vs Pong since Response messages might send data + // up to 64 KB which obviously would be transmitted slower than an empty Pong reply. However, for the real world this is good enough. + if sequence.counter == 0 { + rtt = time.Since(sequence.created) + } + + sequence.counter++ + + // invalidate the sequence immediately? + if invalidate { + delete(manager.sequences, key) + } else if extendValidity { + // Special case CommandResponse: Extend validity in case there are follow-up responses, by half of the round-trip time since they will be sent one-way. + sequence.expires = time.Now().Add(time.Duration(manager.ReplyTimeout) * time.Second / 2) + } + + return sequence, sequence.expires.After(time.Now()), rtt +} + +// InvalidateSequence invalidates the sequence number. It does not call invalidateFunc. +func (manager *SequenceManager) InvalidateSequence(publicKey *btcec.PublicKey, sequenceNumber uint32, bidirectional bool) { + key := sequence2Key(bidirectional, publicKey, sequenceNumber) + + manager.Lock() + delete(manager.sequences, key) + manager.Unlock() +} + +// ---- bidirectional sequences ---- + +// RegisterSequenceBi registers a bidirectional sequence initiated by a remote peer. The caller must specify the timeout (which will be reset every time a new message appears in this sequence). +// This is needed for bidirectional responses to accept subsequent incoming messages from the remote peer. +func (manager *SequenceManager) RegisterSequenceBi(publicKey *btcec.PublicKey, sequenceNumber uint32, data interface{}, timeout time.Duration, invalidateFunc func()) (info *SequenceExpiry) { + info = &SequenceExpiry{ + SequenceNumber: sequenceNumber, + created: time.Now(), + expires: time.Now().Add(timeout), + timeout: timeout, + invalidateFunc: invalidateFunc, + Data: data, + } + + // Before registering the sequence, check if there is a collision. If yes, invalidate the original one. + key := sequence2Key(true, publicKey, info.SequenceNumber) + manager.Lock() + existingSequence := manager.sequences[key] + manager.sequences[key] = info + manager.Unlock() + + // Call the invalidate function if there is a collision. + if existingSequence != nil && existingSequence.invalidateFunc != nil { + go existingSequence.invalidateFunc() + } + + return +} + +// NewSequenceBi returns a new bidirectional sequence and registers it. messageSequence must point to the variable holding the continuous next sequence number. +func (manager *SequenceManager) NewSequenceBi(publicKey *btcec.PublicKey, messageSequence *uint32, data interface{}, timeout time.Duration, invalidateFunc func()) (info *SequenceExpiry) { + info = &SequenceExpiry{ + created: time.Now(), + expires: time.Now().Add(timeout), + bidirectional: true, + timeout: timeout, + invalidateFunc: invalidateFunc, + Data: data, + } + + manager.Lock() + defer manager.Unlock() + + // The likelihood of a collision is low but not impossible. + for n := 0; n < 10000; n++ { + info.SequenceNumber = atomic.AddUint32(messageSequence, 1) + + key := sequence2Key(true, publicKey, info.SequenceNumber) + if infoE := manager.sequences[key]; infoE == nil { + manager.sequences[key] = info + return info + } + } + + return nil +} + +// ValidateSequenceBi validates the sequence number of an incoming message. It will set raw.sequence if valid. +func (manager *SequenceManager) ValidateSequenceBi(publicKey *btcec.PublicKey, sequenceNumber uint32, isLast bool) (sequenceInfo *SequenceExpiry, valid bool, rtt time.Duration) { + key := sequence2Key(true, publicKey, sequenceNumber) + + manager.Lock() + defer manager.Unlock() + + // lookup the sequence + sequence, ok := manager.sequences[key] + if !ok { + return nil, false, rtt + } + + // Initial reply: Store latest roundtrip time. + if sequence.counter == 0 { + rtt = time.Since(sequence.created) + } + + sequence.counter++ + + // invalidate the sequence immediately? + if isLast { + delete(manager.sequences, key) + } else { + sequence.expires = time.Now().Add(sequence.timeout) + } + + return sequence, sequence.expires.After(time.Now()), rtt +} diff --git a/protocol/Test_test.go b/protocol/Test_test.go index 6312bf9..ac71a30 100644 --- a/protocol/Test_test.go +++ b/protocol/Test_test.go @@ -1,85 +1,85 @@ -// Functions to manually debug encoding/decoding. No actual automated unit tests. -package protocol - -import ( - "fmt" - "testing" - - "github.com/PeernetOfficial/core/btcec" -) - -func TestMessageEncodingAnnouncement(t *testing.T) { - privateKey, err := btcec.NewPrivateKey(btcec.S256()) - if err != nil { - fmt.Printf("Error: %s\n", err.Error()) - return - } - publicKey := (*btcec.PublicKey)(&privateKey.PublicKey) - - // encode and decode announcement - packetR := PacketRaw{Protocol: 0, Command: CommandAnnouncement, Sequence: 123} - - var findPeer []KeyHash - var findValue []KeyHash - var files []InfoStore - - hash1 := HashData([]byte("test")) - hash2 := HashData([]byte("test3")) - findPeer = append(findPeer, KeyHash{Hash: hash1}) - findValue = append(findValue, KeyHash{Hash: hash2}) - - packets := EncodeAnnouncement(true, true, findPeer, findValue, files, 1< PATH_MAX_LENGTH { - directory = directory[:PATH_MAX_LENGTH] - } - - return directory -} - -// PathFile sanitizes the filename. -func PathFile(filename string) string { - // Enforce max filename length. - if len(filename) > PATH_MAX_LENGTH { - filename = filename[:PATH_MAX_LENGTH] - } - - return filename -} - -// Username sanitizes the username. -func Username(input string) string { - if !utf8.ValidString(input) { - return "" - } - - input = strings.TrimSpace(input) - input = strings.ReplaceAll(input, "\n", " ") - input = strings.ReplaceAll(input, "\r", "") - - // Max length for sanitized version is 36, resembling the limit from StackOverflow. - if len(input) > 36 { - input = input[:36] - } - - return input -} +/* +File Name: Sanitize.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package sanitize + +import ( + "path" + "strings" + "unicode/utf8" +) + +const PATH_MAX_LENGTH = 32767 // Windows Maximum Path Length for UNC paths + +// PathDirectory sanitizes a directory path (without filename) +func PathDirectory(directory string) string { + // Enforced forward slashes as directory separator and clean the path. + directory = strings.ReplaceAll(directory, "\\", "/") + directory = path.Clean(directory) + + // No slash at the beginning and end to save space. + directory = strings.Trim(directory, "/") + + // Enforce max length. + if len(directory) > PATH_MAX_LENGTH { + directory = directory[:PATH_MAX_LENGTH] + } + + return directory +} + +// PathFile sanitizes the filename. +func PathFile(filename string) string { + // Enforce max filename length. + if len(filename) > PATH_MAX_LENGTH { + filename = filename[:PATH_MAX_LENGTH] + } + + return filename +} + +// Username sanitizes the username. +func Username(input string) string { + if !utf8.ValidString(input) { + return "" + } + + input = strings.TrimSpace(input) + input = strings.ReplaceAll(input, "\n", " ") + input = strings.ReplaceAll(input, "\r", "") + + // Max length for sanitized version is 36, resembling the limit from StackOverflow. + if len(input) > 36 { + input = input[:36] + } + + return input +} diff --git a/search/Normalize.go b/search/Normalize.go index 1d8f510..3fb1cd2 100644 --- a/search/Normalize.go +++ b/search/Normalize.go @@ -1,42 +1,42 @@ -/* -File Name: Normalizing.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Normalizing text so that it can be hashed. -*/ - -package search - -import ( - "path" - "strings" -) - -// sanitizeGeneric sanitizes the text. It intentionally does not lowercase the text so CamelCase can be detcted later. -func sanitizeGeneric(filename string) string { - filename = strings.ToValidUTF8(filename, "") - filename = strings.TrimSpace(filename) - - return filename -} - -// sanitizeInputTerm sanitizes a search term provided by the end user. It includes the generic rules that are done on indexing. -func sanitizeInputTerm(inputTerm string) (outputTerm string, isExact, isWildcard bool) { - inputTerm = sanitizeGeneric(inputTerm) - - // detect and remove quotes at the beginning and end - isExact = false - if len(inputTerm) >= 2 && ((strings.HasPrefix(inputTerm, "\"") && strings.HasSuffix(inputTerm, "\"")) || (strings.HasPrefix(inputTerm, "'") && strings.HasSuffix(inputTerm, "'"))) { - inputTerm = inputTerm[1 : len(inputTerm)-1] - isExact = true - } - - isWildcard = strings.Contains(inputTerm, "*") - - return inputTerm, isExact, isWildcard -} - -func filenameRemoveExtension(filename string) string { - return strings.TrimSuffix(filename, path.Ext(filename)) -} +/* +File Name: Normalizing.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Normalizing text so that it can be hashed. +*/ + +package search + +import ( + "path" + "strings" +) + +// sanitizeGeneric sanitizes the text. It intentionally does not lowercase the text so CamelCase can be detcted later. +func sanitizeGeneric(filename string) string { + filename = strings.ToValidUTF8(filename, "") + filename = strings.TrimSpace(filename) + + return filename +} + +// sanitizeInputTerm sanitizes a search term provided by the end user. It includes the generic rules that are done on indexing. +func sanitizeInputTerm(inputTerm string) (outputTerm string, isExact, isWildcard bool) { + inputTerm = sanitizeGeneric(inputTerm) + + // detect and remove quotes at the beginning and end + isExact = false + if len(inputTerm) >= 2 && ((strings.HasPrefix(inputTerm, "\"") && strings.HasSuffix(inputTerm, "\"")) || (strings.HasPrefix(inputTerm, "'") && strings.HasSuffix(inputTerm, "'"))) { + inputTerm = inputTerm[1 : len(inputTerm)-1] + isExact = true + } + + isWildcard = strings.Contains(inputTerm, "*") + + return inputTerm, isExact, isWildcard +} + +func filenameRemoveExtension(filename string) string { + return strings.TrimSuffix(filename, path.Ext(filename)) +} diff --git a/search/Search Index.go b/search/Search Index.go index b60f662..107e4fa 100644 --- a/search/Search Index.go +++ b/search/Search Index.go @@ -1,290 +1,290 @@ -/* -File Name: Search Index.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package search - -import ( - "encoding/binary" - "errors" - "sync" - - "github.com/PeernetOfficial/core/blockchain" - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/store" - "github.com/google/uuid" -) - -// A search selector is a term that discovers a file. -type SearchSelector struct { - Word string // Normalized version of the word - Hash []byte // Hash of the word - ExactSearch bool // Indicates this is an exact search term, for example a full filename. -} - -// SearchIndexRecord identifies a hash to a given file -type SearchIndexRecord struct { - // List of selectors that found the result. Multiple keywords may find the same file. - Selectors []SearchSelector - - // result data - FileID uuid.UUID - PublicKey *btcec.PublicKey - BlockchainVersion uint64 - BlockNumber uint64 -} - -// This database stores hashes of keywords for file search. -type SearchIndexStore struct { - Database store.Store // The database storing the blockchain. - sync.RWMutex -} - -func InitSearchIndexStore(DatabaseDirectory string) (searchIndex *SearchIndexStore, err error) { - if DatabaseDirectory == "" { - return - } - - searchIndex = &SearchIndexStore{} - - if searchIndex.Database, err = store.NewPogrebStore(DatabaseDirectory); err != nil { - return nil, err - } - - return searchIndex, nil -} - -func (index *SearchIndexStore) IndexNewBlock(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, raw []byte) { - if index == nil { - return - } - - // decode all files from the block - decoded, status, err := blockchain.DecodeBlockRaw(raw) - if err != nil || status != blockchain.StatusOK { - return - } - - index.IndexNewBlockDecoded(publicKey, blockchainVersion, blockNumber, decoded.RecordsDecoded) -} - -// Indexes a new decoded block. Currently it only indexes file records. -func (index *SearchIndexStore) IndexNewBlockDecoded(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, recordsDecoded []interface{}) { - if index == nil { - return - } - - for _, decodedR := range recordsDecoded { - if file, ok := decodedR.(blockchain.BlockRecordFile); ok { - var filename, folder, description string - for _, tag := range file.Tags { - switch tag.Type { - case blockchain.TagName: - filename = sanitizeGeneric(tag.Text()) - case blockchain.TagFolder: - folder = sanitizeGeneric(tag.Text()) - case blockchain.TagDescription: - description = sanitizeGeneric(tag.Text()) - } - } - - hashes := make(map[[32]byte]string) - filename2Hashes(filename, folder, hashes) - text2Hashes(description, hashes) - - for hash := range hashes { - index.IndexHash(publicKey, blockchainVersion, blockNumber, file.ID, hash[:]) - } - } - } -} - -// UnindexBlockchain deletes all index for a given blockchain. This is intentionally not done on a version/block level, because it could easily lead to orphans. -func (index *SearchIndexStore) UnindexBlockchain(publicKey *btcec.PublicKey) { - if index == nil { - return - } - - // get the reverse record - key := publicKey.SerializeCompressed() - raw, found := index.Database.Get(key) - - if !found || len(raw)%reverseIndexRecordSize != 0 { // corrupt record - return - } - - for offset := 0; offset < len(raw); offset += reverseIndexRecordSize { - var hash []byte - var fileID uuid.UUID - - hash = raw[offset : offset+32] - copy(fileID[:], raw[offset+32:offset+32+16]) - - // delete the index record - index.UnindexHash(fileID, hash) - } - - // delete the reverse record - index.Database.Delete(key) -} - -// IndexHash indexes a new hash -func (index *SearchIndexStore) IndexHash(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { - if index == nil { - return - } - - index.Lock() - defer index.Unlock() - - // parse existing records, check if already stored - raw, found := index.Database.Get(hash) - if found && len(raw)%indexRecordSize == 0 { // check if record is corrupt - for offset := 0; offset < len(raw); offset += indexRecordSize { - if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil { - if fileID == record.FileID { - return errors.New("already indexed") - } - } - } - } - - raw = append(raw, encodeIndexRecord(publicKey, blockchainVersion, blockNumber, fileID)...) - - // create the reverse record - index.createReverseIndexRecord(publicKey, blockchainVersion, blockNumber, fileID, hash) - - return index.Database.Set(hash, raw) -} - -// UnindexHash deletes a index record. If there are no more files associated with the hash, the entire hash record is deleted. -func (index *SearchIndexStore) UnindexHash(fileID uuid.UUID, hash []byte) (err error) { - if index == nil { - return - } - - index.Lock() - defer index.Unlock() - - var newRaw []byte - - raw, found := index.Database.Get(hash) - if !found { - return errors.New("index record not found") - } - - if len(raw)%indexRecordSize == 0 { // check if record is corrupt - for offset := 0; offset < len(raw); offset += indexRecordSize { - if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil { - if fileID != record.FileID { - newRaw = append(newRaw, raw[offset:offset+indexRecordSize]...) - } - } - } - } - - if len(newRaw) == 0 { - // delete the entire hash key - index.Database.Delete(hash) - return - } - - return index.Database.Set(hash, newRaw) -} - -// LookupHash returns all index records stored for the hash. -func (index *SearchIndexStore) LookupHash(selector SearchSelector, resultMap map[uuid.UUID]*SearchIndexRecord) (err error) { - if index == nil { - return - } - - index.RLock() - defer index.RUnlock() - - raw, found := index.Database.Get(selector.Hash) - if !found { - return nil - } else if len(raw)%indexRecordSize != 0 { // check if record is corrupt - return errors.New("corrupt index record") - } - - for offset := 0; offset < len(raw); offset += indexRecordSize { - if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil { - if existingRecord, ok := resultMap[record.FileID]; ok { - existingRecord.Selectors = append(existingRecord.Selectors, selector) - } else { - record.Selectors = []SearchSelector{selector} - resultMap[record.FileID] = record - } - } - } - - return nil -} - -// ---- index and reverse index code ---- - -/* -Index records are records looked up based on a hash (the search term) to uniquely identify a single file. -Structure for each index record: - -Offset Size Info -0 16 File ID -16 33 Public Key compressed -49 8 Blockchain Version -57 8 Block Number -*/ - -const indexRecordSize = 65 - -// decodeIndexRecord decodes the index record and sets the fields File ID, Public Key, and Block Number. -func decodeIndexRecord(raw []byte) (record *SearchIndexRecord) { - if len(raw) < indexRecordSize { - return - } - - record = &SearchIndexRecord{} - copy(record.FileID[:], raw[0:16]) - - var err error - if record.PublicKey, err = btcec.ParsePubKey(raw[16:16+33], btcec.S256()); err != nil { - return nil - } - - record.BlockchainVersion = binary.LittleEndian.Uint64(raw[49 : 49+8]) - record.BlockNumber = binary.LittleEndian.Uint64(raw[57 : 57+8]) - - return record -} - -func encodeIndexRecord(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID) (raw []byte) { - raw = make([]byte, indexRecordSize) - - copy(raw[0:16], fileID[:]) - copy(raw[16:16+33], publicKey.SerializeCompressed()) - binary.LittleEndian.PutUint64(raw[49:49+8], blockchainVersion) - binary.LittleEndian.PutUint64(raw[57:57+8], blockNumber) - - return raw -} - -// Reverse index records keep track of all hashes and file IDs searchable for a given blockchain. -const reverseIndexRecordSize = 32 + 16 - -// This creates a reverse index record. It uses the blockchain and block number as key, and provides the hash and file ID as value. -// This function must be called in a RW locked database state. The caller must ensure that this does not result in a duplicate. -func (index *SearchIndexStore) createReverseIndexRecord(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { - key := publicKey.SerializeCompressed() - raw, _ := index.Database.Get(key) - - // each record is only hash + file ID - reverseRecord := make([]byte, reverseIndexRecordSize) - copy(reverseRecord[0:32], hash) - copy(reverseRecord[32:32+16], fileID[:]) - - raw = append(raw, reverseRecord...) - - return index.Database.Set(key, raw) -} +/* +File Name: Search Index.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package search + +import ( + "encoding/binary" + "errors" + "sync" + + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/store" + "github.com/google/uuid" +) + +// A search selector is a term that discovers a file. +type SearchSelector struct { + Word string // Normalized version of the word + Hash []byte // Hash of the word + ExactSearch bool // Indicates this is an exact search term, for example a full filename. +} + +// SearchIndexRecord identifies a hash to a given file +type SearchIndexRecord struct { + // List of selectors that found the result. Multiple keywords may find the same file. + Selectors []SearchSelector + + // result data + FileID uuid.UUID + PublicKey *btcec.PublicKey + BlockchainVersion uint64 + BlockNumber uint64 +} + +// This database stores hashes of keywords for file search. +type SearchIndexStore struct { + Database store.Store // The database storing the blockchain. + sync.RWMutex +} + +func InitSearchIndexStore(DatabaseDirectory string) (searchIndex *SearchIndexStore, err error) { + if DatabaseDirectory == "" { + return + } + + searchIndex = &SearchIndexStore{} + + if searchIndex.Database, err = store.NewPogrebStore(DatabaseDirectory); err != nil { + return nil, err + } + + return searchIndex, nil +} + +func (index *SearchIndexStore) IndexNewBlock(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, raw []byte) { + if index == nil { + return + } + + // decode all files from the block + decoded, status, err := blockchain.DecodeBlockRaw(raw) + if err != nil || status != blockchain.StatusOK { + return + } + + index.IndexNewBlockDecoded(publicKey, blockchainVersion, blockNumber, decoded.RecordsDecoded) +} + +// Indexes a new decoded block. Currently it only indexes file records. +func (index *SearchIndexStore) IndexNewBlockDecoded(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, recordsDecoded []interface{}) { + if index == nil { + return + } + + for _, decodedR := range recordsDecoded { + if file, ok := decodedR.(blockchain.BlockRecordFile); ok { + var filename, folder, description string + for _, tag := range file.Tags { + switch tag.Type { + case blockchain.TagName: + filename = sanitizeGeneric(tag.Text()) + case blockchain.TagFolder: + folder = sanitizeGeneric(tag.Text()) + case blockchain.TagDescription: + description = sanitizeGeneric(tag.Text()) + } + } + + hashes := make(map[[32]byte]string) + filename2Hashes(filename, folder, hashes) + text2Hashes(description, hashes) + + for hash := range hashes { + index.IndexHash(publicKey, blockchainVersion, blockNumber, file.ID, hash[:]) + } + } + } +} + +// UnindexBlockchain deletes all index for a given blockchain. This is intentionally not done on a version/block level, because it could easily lead to orphans. +func (index *SearchIndexStore) UnindexBlockchain(publicKey *btcec.PublicKey) { + if index == nil { + return + } + + // get the reverse record + key := publicKey.SerializeCompressed() + raw, found := index.Database.Get(key) + + if !found || len(raw)%reverseIndexRecordSize != 0 { // corrupt record + return + } + + for offset := 0; offset < len(raw); offset += reverseIndexRecordSize { + var hash []byte + var fileID uuid.UUID + + hash = raw[offset : offset+32] + copy(fileID[:], raw[offset+32:offset+32+16]) + + // delete the index record + index.UnindexHash(fileID, hash) + } + + // delete the reverse record + index.Database.Delete(key) +} + +// IndexHash indexes a new hash +func (index *SearchIndexStore) IndexHash(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { + if index == nil { + return + } + + index.Lock() + defer index.Unlock() + + // parse existing records, check if already stored + raw, found := index.Database.Get(hash) + if found && len(raw)%indexRecordSize == 0 { // check if record is corrupt + for offset := 0; offset < len(raw); offset += indexRecordSize { + if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil { + if fileID == record.FileID { + return errors.New("already indexed") + } + } + } + } + + raw = append(raw, encodeIndexRecord(publicKey, blockchainVersion, blockNumber, fileID)...) + + // create the reverse record + index.createReverseIndexRecord(publicKey, blockchainVersion, blockNumber, fileID, hash) + + return index.Database.Set(hash, raw) +} + +// UnindexHash deletes a index record. If there are no more files associated with the hash, the entire hash record is deleted. +func (index *SearchIndexStore) UnindexHash(fileID uuid.UUID, hash []byte) (err error) { + if index == nil { + return + } + + index.Lock() + defer index.Unlock() + + var newRaw []byte + + raw, found := index.Database.Get(hash) + if !found { + return errors.New("index record not found") + } + + if len(raw)%indexRecordSize == 0 { // check if record is corrupt + for offset := 0; offset < len(raw); offset += indexRecordSize { + if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil { + if fileID != record.FileID { + newRaw = append(newRaw, raw[offset:offset+indexRecordSize]...) + } + } + } + } + + if len(newRaw) == 0 { + // delete the entire hash key + index.Database.Delete(hash) + return + } + + return index.Database.Set(hash, newRaw) +} + +// LookupHash returns all index records stored for the hash. +func (index *SearchIndexStore) LookupHash(selector SearchSelector, resultMap map[uuid.UUID]*SearchIndexRecord) (err error) { + if index == nil { + return + } + + index.RLock() + defer index.RUnlock() + + raw, found := index.Database.Get(selector.Hash) + if !found { + return nil + } else if len(raw)%indexRecordSize != 0 { // check if record is corrupt + return errors.New("corrupt index record") + } + + for offset := 0; offset < len(raw); offset += indexRecordSize { + if record := decodeIndexRecord(raw[offset : offset+indexRecordSize]); record != nil { + if existingRecord, ok := resultMap[record.FileID]; ok { + existingRecord.Selectors = append(existingRecord.Selectors, selector) + } else { + record.Selectors = []SearchSelector{selector} + resultMap[record.FileID] = record + } + } + } + + return nil +} + +// ---- index and reverse index code ---- + +/* +Index records are records looked up based on a hash (the search term) to uniquely identify a single file. +Structure for each index record: + +Offset Size Info +0 16 File ID +16 33 Public Key compressed +49 8 Blockchain Version +57 8 Block Number +*/ + +const indexRecordSize = 65 + +// decodeIndexRecord decodes the index record and sets the fields File ID, Public Key, and Block Number. +func decodeIndexRecord(raw []byte) (record *SearchIndexRecord) { + if len(raw) < indexRecordSize { + return + } + + record = &SearchIndexRecord{} + copy(record.FileID[:], raw[0:16]) + + var err error + if record.PublicKey, err = btcec.ParsePubKey(raw[16:16+33], btcec.S256()); err != nil { + return nil + } + + record.BlockchainVersion = binary.LittleEndian.Uint64(raw[49 : 49+8]) + record.BlockNumber = binary.LittleEndian.Uint64(raw[57 : 57+8]) + + return record +} + +func encodeIndexRecord(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID) (raw []byte) { + raw = make([]byte, indexRecordSize) + + copy(raw[0:16], fileID[:]) + copy(raw[16:16+33], publicKey.SerializeCompressed()) + binary.LittleEndian.PutUint64(raw[49:49+8], blockchainVersion) + binary.LittleEndian.PutUint64(raw[57:57+8], blockNumber) + + return raw +} + +// Reverse index records keep track of all hashes and file IDs searchable for a given blockchain. +const reverseIndexRecordSize = 32 + 16 + +// This creates a reverse index record. It uses the blockchain and block number as key, and provides the hash and file ID as value. +// This function must be called in a RW locked database state. The caller must ensure that this does not result in a duplicate. +func (index *SearchIndexStore) createReverseIndexRecord(publicKey *btcec.PublicKey, blockchainVersion, blockNumber uint64, fileID uuid.UUID, hash []byte) (err error) { + key := publicKey.SerializeCompressed() + raw, _ := index.Database.Get(key) + + // each record is only hash + file ID + reverseRecord := make([]byte, reverseIndexRecordSize) + copy(reverseRecord[0:32], hash) + copy(reverseRecord[32:32+16], fileID[:]) + + raw = append(raw, reverseRecord...) + + return index.Database.Set(key, raw) +} diff --git a/search/Search Term.go b/search/Search Term.go index 96265da..a0f6885 100644 --- a/search/Search Term.go +++ b/search/Search Term.go @@ -1,57 +1,57 @@ -/* -File Name: Search Term.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package search - -import ( - "github.com/google/uuid" -) - -func (index *SearchIndexStore) Search(term string) (results []SearchIndexRecord) { - if index == nil { // Search index may not be available. - return nil - } - - termS, isExact, _ := sanitizeInputTerm(term) - - if len(termS) < wordMinLength { - return - } - - resultMap := make(map[uuid.UUID]*SearchIndexRecord) - resultMapToSlice := func() (results []SearchIndexRecord) { - for _, result := range resultMap { - results = append(results, *result) - } - return results - } - - // start with exact search - hashExact, wordH := hashWord(termS) - if hashExact != nil { - index.LookupHash(SearchSelector{Hash: hashExact, Word: wordH, ExactSearch: true}, resultMap) - } - - // exact search only? - if isExact { - return resultMapToSlice() - } - - // break up the term into hashes - hashes := make(map[[32]byte]string) - - text2Hashes(termS, hashes) - - // The exact search was already performed, exclude it. - hashMapDelete(hashExact, hashes) - - // look up the hashes! - for hash, keyword := range hashes { - index.LookupHash(SearchSelector{Hash: hash[:], Word: keyword}, resultMap) - } - - return resultMapToSlice() -} +/* +File Name: Search Term.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package search + +import ( + "github.com/google/uuid" +) + +func (index *SearchIndexStore) Search(term string) (results []SearchIndexRecord) { + if index == nil { // Search index may not be available. + return nil + } + + termS, isExact, _ := sanitizeInputTerm(term) + + if len(termS) < wordMinLength { + return + } + + resultMap := make(map[uuid.UUID]*SearchIndexRecord) + resultMapToSlice := func() (results []SearchIndexRecord) { + for _, result := range resultMap { + results = append(results, *result) + } + return results + } + + // start with exact search + hashExact, wordH := hashWord(termS) + if hashExact != nil { + index.LookupHash(SearchSelector{Hash: hashExact, Word: wordH, ExactSearch: true}, resultMap) + } + + // exact search only? + if isExact { + return resultMapToSlice() + } + + // break up the term into hashes + hashes := make(map[[32]byte]string) + + text2Hashes(termS, hashes) + + // The exact search was already performed, exclude it. + hashMapDelete(hashExact, hashes) + + // look up the hashes! + for hash, keyword := range hashes { + index.LookupHash(SearchSelector{Hash: hash[:], Word: keyword}, resultMap) + } + + return resultMapToSlice() +} diff --git a/search/Text 2 Hash.go b/search/Text 2 Hash.go index 80491cb..41edb1e 100644 --- a/search/Text 2 Hash.go +++ b/search/Text 2 Hash.go @@ -1,176 +1,176 @@ -/* -File Name: Text 2 Hash.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package search - -import ( - "strings" - "unicode" - - "lukechampine.com/blake3" -) - -// Words must be the minimum length to be indexed. -const wordMinLength = 3 - -// text2Hashes creates hashes from words in the text. Text may be CamelCased. -// Text must be already validated for valid UTF8 when calling this function. -func text2Hashes(text string, hashes map[[32]byte]string) { - words := strings.FieldsFunc(text, func(char rune) bool { - if unicode.IsSpace(char) { - return true - } - return strings.ContainsAny(string(char), "+-._()[],–") - }) - - for _, word := range words { - // remove hash tag prefix - word = strings.TrimPrefix(word, "#") - - hashWordMap(word, hashes) - - // CamelCase word detection - for _, word2 := range CamelCaseSplit(word) { - if word2 != word { - hashWordMap(word2, hashes) - } - } - } -} - -// filename2Hashes creates hashes based on the filename and folder. -func filename2Hashes(filename, folder string, hashes map[[32]byte]string) { - if len(filename) < wordMinLength { - return - } - - // First hash is created on the entire filename including extension. This is done for perfect matches. - hashWordMap(filename, hashes) - - // Hash the filename without extension (and proceed without extension) - filename = filenameRemoveExtension(filename) - hashWordMap(filename, hashes) - - // Hash each individual word of the filename and directory - text2Hashes(filename, hashes) - - folder = strings.ReplaceAll(folder, "\\", " ") - folder = strings.ReplaceAll(folder, "/", " ") - text2Hashes(folder, hashes) -} - -// hashWordMap hashes a word and stores it on the map. This immediately deduplicated hashes. It always lowercases the word. -func hashWordMap(word string, hashes map[[32]byte]string) { - word = strings.TrimSpace(strings.ToLower(word)) - if len(word) < wordMinLength { - return - } - - hashes[blake3.Sum256([]byte(word))] = word -} - -// hashMapDelete deletes a hash from the list if the hash is valid -func hashMapDelete(hash []byte, hashes map[[32]byte]string) { - if len(hash) == 0 { - return - } - - var hashB [32]byte - copy(hashB[:], hash) - - delete(hashes, hashB) -} - -// hashWord hashes a single word. It returns nil if not suitable. It always lowercases the word. -func hashWord(word string) (hash []byte, wordHashed string) { - word = strings.TrimSpace(strings.ToLower(word)) - if len(word) < wordMinLength { - return - } - - hashB := blake3.Sum256([]byte(word)) - return hashB[:], word -} - -// fork from https://github.com/fatih/camelcase/blob/master/camelcase.go - -// Split splits the camelcase word and returns a list of words. It also -// supports digits. Both lower camel case and upper camel case are supported. -// For more info please check: http://en.wikipedia.org/wiki/CamelCase -// -// Examples -// -// "" => [""] -// "lowercase" => ["lowercase"] -// "Class" => ["Class"] -// "MyClass" => ["My", "Class"] -// "MyC" => ["My", "C"] -// "HTML" => ["HTML"] -// "PDFLoader" => ["PDF", "Loader"] -// "AString" => ["A", "String"] -// "SimpleXMLParser" => ["Simple", "XML", "Parser"] -// "vimRPCPlugin" => ["vim", "RPC", "Plugin"] -// "GL11Version" => ["GL", "11", "Version"] -// "99Bottles" => ["99", "Bottles"] -// "May5" => ["May", "5"] -// "BFG9000" => ["BFG", "9000"] -// "BöseÜberraschung" => ["Böse", "Überraschung"] -// "Two spaces" => ["Two", " ", "spaces"] -// "BadUTF8\xe2\xe2\xa1" => ["BadUTF8\xe2\xe2\xa1"] -// -// Splitting rules -// -// 1) If string is not valid UTF-8, return it without splitting as -> removed in this fork. -// single item array. -// 2) Assign all unicode characters into one of 4 sets: lower case -// letters, upper case letters, numbers, and all other characters. -// 3) Iterate through characters of string, introducing splits -// between adjacent characters that belong to different sets. -// 4) Iterate through array of split strings, and if a given string -// is upper case: -// if subsequent string is lower case: -// move last character of upper case string to beginning of -// lower case string -func CamelCaseSplit(src string) (entries []string) { - entries = []string{} - var runes [][]rune - lastClass := 0 - class := 0 - // split into fields based on class of unicode character - for _, r := range src { - switch true { - case unicode.IsLower(r): - class = 1 - case unicode.IsUpper(r): - class = 2 - case unicode.IsDigit(r): - class = 3 - default: - class = 4 - } - if class == lastClass { - runes[len(runes)-1] = append(runes[len(runes)-1], r) - } else { - runes = append(runes, []rune{r}) - } - lastClass = class - } - // handle upper case -> lower case sequences, e.g. - // "PDFL", "oader" -> "PDF", "Loader" - for i := 0; i < len(runes)-1; i++ { - if unicode.IsUpper(runes[i][0]) && unicode.IsLower(runes[i+1][0]) { - runes[i+1] = append([]rune{runes[i][len(runes[i])-1]}, runes[i+1]...) - runes[i] = runes[i][:len(runes[i])-1] - } - } - // construct []string from results - for _, s := range runes { - if len(s) > 0 { - entries = append(entries, string(s)) - } - } - return -} +/* +File Name: Text 2 Hash.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package search + +import ( + "strings" + "unicode" + + "lukechampine.com/blake3" +) + +// Words must be the minimum length to be indexed. +const wordMinLength = 3 + +// text2Hashes creates hashes from words in the text. Text may be CamelCased. +// Text must be already validated for valid UTF8 when calling this function. +func text2Hashes(text string, hashes map[[32]byte]string) { + words := strings.FieldsFunc(text, func(char rune) bool { + if unicode.IsSpace(char) { + return true + } + return strings.ContainsAny(string(char), "+-._()[],–") + }) + + for _, word := range words { + // remove hash tag prefix + word = strings.TrimPrefix(word, "#") + + hashWordMap(word, hashes) + + // CamelCase word detection + for _, word2 := range CamelCaseSplit(word) { + if word2 != word { + hashWordMap(word2, hashes) + } + } + } +} + +// filename2Hashes creates hashes based on the filename and folder. +func filename2Hashes(filename, folder string, hashes map[[32]byte]string) { + if len(filename) < wordMinLength { + return + } + + // First hash is created on the entire filename including extension. This is done for perfect matches. + hashWordMap(filename, hashes) + + // Hash the filename without extension (and proceed without extension) + filename = filenameRemoveExtension(filename) + hashWordMap(filename, hashes) + + // Hash each individual word of the filename and directory + text2Hashes(filename, hashes) + + folder = strings.ReplaceAll(folder, "\\", " ") + folder = strings.ReplaceAll(folder, "/", " ") + text2Hashes(folder, hashes) +} + +// hashWordMap hashes a word and stores it on the map. This immediately deduplicated hashes. It always lowercases the word. +func hashWordMap(word string, hashes map[[32]byte]string) { + word = strings.TrimSpace(strings.ToLower(word)) + if len(word) < wordMinLength { + return + } + + hashes[blake3.Sum256([]byte(word))] = word +} + +// hashMapDelete deletes a hash from the list if the hash is valid +func hashMapDelete(hash []byte, hashes map[[32]byte]string) { + if len(hash) == 0 { + return + } + + var hashB [32]byte + copy(hashB[:], hash) + + delete(hashes, hashB) +} + +// hashWord hashes a single word. It returns nil if not suitable. It always lowercases the word. +func hashWord(word string) (hash []byte, wordHashed string) { + word = strings.TrimSpace(strings.ToLower(word)) + if len(word) < wordMinLength { + return + } + + hashB := blake3.Sum256([]byte(word)) + return hashB[:], word +} + +// fork from https://github.com/fatih/camelcase/blob/master/camelcase.go + +// Split splits the camelcase word and returns a list of words. It also +// supports digits. Both lower camel case and upper camel case are supported. +// For more info please check: http://en.wikipedia.org/wiki/CamelCase +// +// Examples +// +// "" => [""] +// "lowercase" => ["lowercase"] +// "Class" => ["Class"] +// "MyClass" => ["My", "Class"] +// "MyC" => ["My", "C"] +// "HTML" => ["HTML"] +// "PDFLoader" => ["PDF", "Loader"] +// "AString" => ["A", "String"] +// "SimpleXMLParser" => ["Simple", "XML", "Parser"] +// "vimRPCPlugin" => ["vim", "RPC", "Plugin"] +// "GL11Version" => ["GL", "11", "Version"] +// "99Bottles" => ["99", "Bottles"] +// "May5" => ["May", "5"] +// "BFG9000" => ["BFG", "9000"] +// "BöseÜberraschung" => ["Böse", "Überraschung"] +// "Two spaces" => ["Two", " ", "spaces"] +// "BadUTF8\xe2\xe2\xa1" => ["BadUTF8\xe2\xe2\xa1"] +// +// Splitting rules +// +// 1) If string is not valid UTF-8, return it without splitting as -> removed in this fork. +// single item array. +// 2) Assign all unicode characters into one of 4 sets: lower case +// letters, upper case letters, numbers, and all other characters. +// 3) Iterate through characters of string, introducing splits +// between adjacent characters that belong to different sets. +// 4) Iterate through array of split strings, and if a given string +// is upper case: +// if subsequent string is lower case: +// move last character of upper case string to beginning of +// lower case string +func CamelCaseSplit(src string) (entries []string) { + entries = []string{} + var runes [][]rune + lastClass := 0 + class := 0 + // split into fields based on class of unicode character + for _, r := range src { + switch true { + case unicode.IsLower(r): + class = 1 + case unicode.IsUpper(r): + class = 2 + case unicode.IsDigit(r): + class = 3 + default: + class = 4 + } + if class == lastClass { + runes[len(runes)-1] = append(runes[len(runes)-1], r) + } else { + runes = append(runes, []rune{r}) + } + lastClass = class + } + // handle upper case -> lower case sequences, e.g. + // "PDFL", "oader" -> "PDF", "Loader" + for i := 0; i < len(runes)-1; i++ { + if unicode.IsUpper(runes[i][0]) && unicode.IsLower(runes[i+1][0]) { + runes[i+1] = append([]rune{runes[i][len(runes[i])-1]}, runes[i+1]...) + runes[i] = runes[i][:len(runes[i])-1] + } + } + // construct []string from results + for _, s := range runes { + if len(s) > 0 { + entries = append(entries, string(s)) + } + } + return +} diff --git a/store/Memory.go b/store/Memory.go index 65ab666..5e987f9 100644 --- a/store/Memory.go +++ b/store/Memory.go @@ -1,94 +1,94 @@ -/* -File Name: Memory.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package store - -import ( - "sync" - "time" -) - -// MemoryStore is a simple in-memory key-value store for testing purposes. -type MemoryStore struct { - mutex *sync.Mutex - data map[string][]byte - expireMap map[string]time.Time -} - -// NewMemoryStore create a properly initialized memory store. -func NewMemoryStore() *MemoryStore { - return &MemoryStore{ - data: make(map[string][]byte), - mutex: &sync.Mutex{}, - expireMap: make(map[string]time.Time), - } -} - -// ExpireKeys is called to delete all keys that are marked for expiration. -func (ms *MemoryStore) ExpireKeys() { - ms.mutex.Lock() - defer ms.mutex.Unlock() - for k, v := range ms.expireMap { - if time.Now().After(v) { - delete(ms.expireMap, k) - delete(ms.data, k) - } - } -} - -// Set stores the key-value pair. -func (ms *MemoryStore) Set(key []byte, data []byte) error { - ms.mutex.Lock() - ms.data[string(key)] = data - ms.mutex.Unlock() - return nil -} - -// StoreExpire stores the key-value pair and deletes it after the expiration time. -func (ms *MemoryStore) StoreExpire(key []byte, data []byte, expiration time.Time) error { - ms.mutex.Lock() - ms.expireMap[string(key)] = expiration - ms.data[string(key)] = data - ms.mutex.Unlock() - return nil -} - -// Get returns the value for the key if present. -func (ms *MemoryStore) Get(key []byte) (data []byte, found bool) { - ms.mutex.Lock() - data, found = ms.data[string(key)] - ms.mutex.Unlock() - return data, found -} - -// Delete deletes a key-value pair. -func (ms *MemoryStore) Delete(key []byte) { - ms.mutex.Lock() - delete(ms.expireMap, string(key)) - delete(ms.data, string(key)) - ms.mutex.Unlock() -} - -// Count returns the count of records stored. -func (ms *MemoryStore) Count() uint64 { - ms.mutex.Lock() - defer ms.mutex.Unlock() - return uint64(len(ms.data)) -} - -// Iterate iterates over all records. -func (ms *MemoryStore) Iterate(callback func(key, value []byte)) { - ms.mutex.Lock() - defer ms.mutex.Unlock() - - for key, value := range ms.data { - ms.mutex.Unlock() // allow access to the map while calling the callback - - callback([]byte(key), value) - - ms.mutex.Lock() - } -} +/* +File Name: Memory.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package store + +import ( + "sync" + "time" +) + +// MemoryStore is a simple in-memory key-value store for testing purposes. +type MemoryStore struct { + mutex *sync.Mutex + data map[string][]byte + expireMap map[string]time.Time +} + +// NewMemoryStore create a properly initialized memory store. +func NewMemoryStore() *MemoryStore { + return &MemoryStore{ + data: make(map[string][]byte), + mutex: &sync.Mutex{}, + expireMap: make(map[string]time.Time), + } +} + +// ExpireKeys is called to delete all keys that are marked for expiration. +func (ms *MemoryStore) ExpireKeys() { + ms.mutex.Lock() + defer ms.mutex.Unlock() + for k, v := range ms.expireMap { + if time.Now().After(v) { + delete(ms.expireMap, k) + delete(ms.data, k) + } + } +} + +// Set stores the key-value pair. +func (ms *MemoryStore) Set(key []byte, data []byte) error { + ms.mutex.Lock() + ms.data[string(key)] = data + ms.mutex.Unlock() + return nil +} + +// StoreExpire stores the key-value pair and deletes it after the expiration time. +func (ms *MemoryStore) StoreExpire(key []byte, data []byte, expiration time.Time) error { + ms.mutex.Lock() + ms.expireMap[string(key)] = expiration + ms.data[string(key)] = data + ms.mutex.Unlock() + return nil +} + +// Get returns the value for the key if present. +func (ms *MemoryStore) Get(key []byte) (data []byte, found bool) { + ms.mutex.Lock() + data, found = ms.data[string(key)] + ms.mutex.Unlock() + return data, found +} + +// Delete deletes a key-value pair. +func (ms *MemoryStore) Delete(key []byte) { + ms.mutex.Lock() + delete(ms.expireMap, string(key)) + delete(ms.data, string(key)) + ms.mutex.Unlock() +} + +// Count returns the count of records stored. +func (ms *MemoryStore) Count() uint64 { + ms.mutex.Lock() + defer ms.mutex.Unlock() + return uint64(len(ms.data)) +} + +// Iterate iterates over all records. +func (ms *MemoryStore) Iterate(callback func(key, value []byte)) { + ms.mutex.Lock() + defer ms.mutex.Unlock() + + for key, value := range ms.data { + ms.mutex.Unlock() // allow access to the map while calling the callback + + callback([]byte(key), value) + + ms.mutex.Lock() + } +} diff --git a/store/Pebble.go b/store/Pebble.go index 8065b30..dd2c088 100644 --- a/store/Pebble.go +++ b/store/Pebble.go @@ -1,72 +1,72 @@ -/* -File Name: Pebble.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Note: It turned out that pebble has many dependencies and increases the binary file size by ~6 MB. -*/ - -package store - -/* -import ( - "errors" - "sync" - "time" - - "github.com/cockroachdb/pebble" -) - -// PebbleStore is a key/value store using Pebble from CockroachDB. -// Expiration is currently not supported. -type PebbleStore struct { - mutex *sync.Mutex - filename string - db *pebble.DB -} - -// NewPebbleStore create a properly initialized pebble store. -func NewPebbleStore(filename string) (store *PebbleStore, err error) { - // if the database does not exist, it will be created - db, err := pebble.Open(filename, &pebble.Options{}) - if err != nil { - return nil, err - } - - return &PebbleStore{ - mutex: &sync.Mutex{}, - filename: filename, - db: db, - }, nil -} - -func (store *PebbleStore) ExpireKeys() { - // Not yet implemented -} - -// Store stores the key/value pair. -func (store *PebbleStore) Set(key []byte, data []byte) error { - return store.db.Set(key, data, pebble.Sync) -} - -// StoreExpire stores the key/value pair and deletes it after the expiration time. -func (store *PebbleStore) StoreExpire(key []byte, data []byte, expiration time.Time) error { - // Not yet implemented - return errors.New("not yet implemented") -} - -// Get returns the value for the key if present. -func (store *PebbleStore) Get(key []byte) (data []byte, found bool) { - value, closer, err := store.db.Get(key) - if err != nil { - return nil, false - } - closer.Close() - return value, true -} - -// Delete deletes a key/value pair. -func (store *PebbleStore) Delete(key []byte) { - store.db.Delete(key, pebble.Sync) -} -*/ +/* +File Name: Pebble.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Note: It turned out that pebble has many dependencies and increases the binary file size by ~6 MB. +*/ + +package store + +/* +import ( + "errors" + "sync" + "time" + + "github.com/cockroachdb/pebble" +) + +// PebbleStore is a key/value store using Pebble from CockroachDB. +// Expiration is currently not supported. +type PebbleStore struct { + mutex *sync.Mutex + filename string + db *pebble.DB +} + +// NewPebbleStore create a properly initialized pebble store. +func NewPebbleStore(filename string) (store *PebbleStore, err error) { + // if the database does not exist, it will be created + db, err := pebble.Open(filename, &pebble.Options{}) + if err != nil { + return nil, err + } + + return &PebbleStore{ + mutex: &sync.Mutex{}, + filename: filename, + db: db, + }, nil +} + +func (store *PebbleStore) ExpireKeys() { + // Not yet implemented +} + +// Store stores the key/value pair. +func (store *PebbleStore) Set(key []byte, data []byte) error { + return store.db.Set(key, data, pebble.Sync) +} + +// StoreExpire stores the key/value pair and deletes it after the expiration time. +func (store *PebbleStore) StoreExpire(key []byte, data []byte, expiration time.Time) error { + // Not yet implemented + return errors.New("not yet implemented") +} + +// Get returns the value for the key if present. +func (store *PebbleStore) Get(key []byte) (data []byte, found bool) { + value, closer, err := store.db.Get(key) + if err != nil { + return nil, false + } + closer.Close() + return value, true +} + +// Delete deletes a key/value pair. +func (store *PebbleStore) Delete(key []byte) { + store.db.Delete(key, pebble.Sync) +} +*/ diff --git a/store/Pogreb.go b/store/Pogreb.go index e1636e1..5d10ea1 100644 --- a/store/Pogreb.go +++ b/store/Pogreb.go @@ -1,89 +1,89 @@ -/* -File Name: Pogreb.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package store - -import ( - "errors" - "io" - "log" - "sync" - "time" - - "github.com/akrylysov/pogreb" -) - -// PogrebStore is a key-value store using Pogreb. -// Expiration is currently not supported. -type PogrebStore struct { - mutex *sync.Mutex - filename string - db *pogreb.DB -} - -// NewPogrebStore create a properly initialized Pogreb store. -func NewPogrebStore(filename string) (store *PogrebStore, err error) { - pogreb.SetLogger(log.New(io.Discard, "", 0)) - - // if the database does not exist, it will be created - db, err := pogreb.Open(filename, nil) - if err != nil { - return nil, err - } - - return &PogrebStore{ - mutex: &sync.Mutex{}, - filename: filename, - db: db, - }, nil -} - -func (store *PogrebStore) ExpireKeys() { - // Not yet implemented -} - -// Store stores the key-value pair. -func (store *PogrebStore) Set(key []byte, data []byte) error { - return store.db.Put(key, data) -} - -// StoreExpire stores the key-value pair and deletes it after the expiration time. -func (store *PogrebStore) StoreExpire(key []byte, data []byte, expiration time.Time) error { - // Not yet implemented - return errors.New("not yet implemented") -} - -// Get returns the value for the key if present. -func (store *PogrebStore) Get(key []byte) (data []byte, found bool) { - value, err := store.db.Get(key) - if err != nil || value == nil { - return nil, false - } - return value, true -} - -// Delete deletes a key-value pair. -func (store *PogrebStore) Delete(key []byte) { - store.db.Delete(key) -} - -// Count returns the count of records stored. -func (store *PogrebStore) Count() uint64 { - return uint64(store.db.Count()) -} - -// Iterate iterates over all records. -func (store *PogrebStore) Iterate(callback func(key, value []byte)) { - iterator := store.db.Items() - for { - key, value, err := iterator.Next() - if err != nil { - break - } - - callback(key, value) - } -} +/* +File Name: Pogreb.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package store + +import ( + "errors" + "io" + "log" + "sync" + "time" + + "github.com/akrylysov/pogreb" +) + +// PogrebStore is a key-value store using Pogreb. +// Expiration is currently not supported. +type PogrebStore struct { + mutex *sync.Mutex + filename string + db *pogreb.DB +} + +// NewPogrebStore create a properly initialized Pogreb store. +func NewPogrebStore(filename string) (store *PogrebStore, err error) { + pogreb.SetLogger(log.New(io.Discard, "", 0)) + + // if the database does not exist, it will be created + db, err := pogreb.Open(filename, nil) + if err != nil { + return nil, err + } + + return &PogrebStore{ + mutex: &sync.Mutex{}, + filename: filename, + db: db, + }, nil +} + +func (store *PogrebStore) ExpireKeys() { + // Not yet implemented +} + +// Store stores the key-value pair. +func (store *PogrebStore) Set(key []byte, data []byte) error { + return store.db.Put(key, data) +} + +// StoreExpire stores the key-value pair and deletes it after the expiration time. +func (store *PogrebStore) StoreExpire(key []byte, data []byte, expiration time.Time) error { + // Not yet implemented + return errors.New("not yet implemented") +} + +// Get returns the value for the key if present. +func (store *PogrebStore) Get(key []byte) (data []byte, found bool) { + value, err := store.db.Get(key) + if err != nil || value == nil { + return nil, false + } + return value, true +} + +// Delete deletes a key-value pair. +func (store *PogrebStore) Delete(key []byte) { + store.db.Delete(key) +} + +// Count returns the count of records stored. +func (store *PogrebStore) Count() uint64 { + return uint64(store.db.Count()) +} + +// Iterate iterates over all records. +func (store *PogrebStore) Iterate(callback func(key, value []byte)) { + iterator := store.db.Items() + for { + key, value, err := iterator.Next() + if err != nil { + break + } + + callback(key, value) + } +} diff --git a/store/Store.go b/store/Store.go index 3c95bf5..4e9a47d 100644 --- a/store/Store.go +++ b/store/Store.go @@ -1,38 +1,38 @@ -/* -File Name: Store.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner - -Simple key-value store interface. -*/ - -package store - -import ( - "time" -) - -// Store is the interface for implementing the storage mechanism for the DHT. -type Store interface { - // Set stores the key-value pair. - Set(key []byte, data []byte) error - - // StoreExpire stores the key-value pair and deletes it after the expiration time. - // If key-value already exists, it will be overwritten and the new expiration time applies. - StoreExpire(key []byte, data []byte, expiration time.Time) error - - // Get returns the value for the key if present. - Get(key []byte) (data []byte, found bool) - - // Delete deletes a key-value pair. - Delete(key []byte) - - // ExpireKeys is called to delete all keys that are marked for expiration. - ExpireKeys() - - // Count returns the total number of records stored. - Count() uint64 - - // Iterate iterates over all records. - Iterate(callback func(key, value []byte)) -} +/* +File Name: Store.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner + +Simple key-value store interface. +*/ + +package store + +import ( + "time" +) + +// Store is the interface for implementing the storage mechanism for the DHT. +type Store interface { + // Set stores the key-value pair. + Set(key []byte, data []byte) error + + // StoreExpire stores the key-value pair and deletes it after the expiration time. + // If key-value already exists, it will be overwritten and the new expiration time applies. + StoreExpire(key []byte, data []byte, expiration time.Time) error + + // Get returns the value for the key if present. + Get(key []byte) (data []byte, found bool) + + // Delete deletes a key-value pair. + Delete(key []byte) + + // ExpireKeys is called to delete all keys that are marked for expiration. + ExpireKeys() + + // Count returns the total number of records stored. + Count() uint64 + + // Iterate iterates over all records. + Iterate(callback func(key, value []byte)) +} diff --git a/system/Execute.go b/system/Execute.go index 272af34..03c4f8a 100644 --- a/system/Execute.go +++ b/system/Execute.go @@ -1,91 +1,91 @@ -/* -File Name: Execute.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package system - -import ( - "archive/zip" - "errors" - "io" - "os" - "path/filepath" - "strings" -) - -// Execute the actions described in the info header -func (update *UpdatePackage) Execute(DataFolder, PluginFolder string) (err error) { - for _, action := range update.Header.Actions { - switch action.Action { - case "extract": - destination := resolveFolders(action.Target, DataFolder, PluginFolder) - - for _, f := range update.Reader.File { - // filter the filename based on source - if !strings.HasPrefix(f.Name, action.Source) { - continue - } - - err := unzipFile(f, destination) - if err != nil { - return err - } - } - } - } - - return nil -} - -// Deletes the update package file -func (update *UpdatePackage) Delete() (err error) { - return os.Remove(update.Filename) -} - -func unzipFile(f *zip.File, destination string) error { - // 4. Check if file paths are not vulnerable to Zip Slip - filePath := filepath.Join(destination, f.Name) - if !strings.HasPrefix(filePath, filepath.Clean(destination)+string(os.PathSeparator)) { - return errors.New("invalid file path: " + filePath) - } - - // 5. Create directory tree - if f.FileInfo().IsDir() { - if err := os.MkdirAll(filePath, os.ModePerm); err != nil { - return err - } - return nil - } - - if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { - return err - } - - // 6. Create a destination file for unzipped content - destinationFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } - defer destinationFile.Close() - - // 7. Unzip the content of a file and copy it to the destination file - zippedFile, err := f.Open() - if err != nil { - return err - } - defer zippedFile.Close() - - if _, err := io.Copy(destinationFile, zippedFile); err != nil { - return err - } - return nil -} - -func resolveFolders(folder, dataFolder, pluginFolder string) (resolved string) { - resolved = strings.Replace(folder, "%plugin%", pluginFolder, 1) - resolved = strings.Replace(resolved, "%data%", dataFolder, 1) - - return resolved -} +/* +File Name: Execute.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package system + +import ( + "archive/zip" + "errors" + "io" + "os" + "path/filepath" + "strings" +) + +// Execute the actions described in the info header +func (update *UpdatePackage) Execute(DataFolder, PluginFolder string) (err error) { + for _, action := range update.Header.Actions { + switch action.Action { + case "extract": + destination := resolveFolders(action.Target, DataFolder, PluginFolder) + + for _, f := range update.Reader.File { + // filter the filename based on source + if !strings.HasPrefix(f.Name, action.Source) { + continue + } + + err := unzipFile(f, destination) + if err != nil { + return err + } + } + } + } + + return nil +} + +// Deletes the update package file +func (update *UpdatePackage) Delete() (err error) { + return os.Remove(update.Filename) +} + +func unzipFile(f *zip.File, destination string) error { + // 4. Check if file paths are not vulnerable to Zip Slip + filePath := filepath.Join(destination, f.Name) + if !strings.HasPrefix(filePath, filepath.Clean(destination)+string(os.PathSeparator)) { + return errors.New("invalid file path: " + filePath) + } + + // 5. Create directory tree + if f.FileInfo().IsDir() { + if err := os.MkdirAll(filePath, os.ModePerm); err != nil { + return err + } + return nil + } + + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + return err + } + + // 6. Create a destination file for unzipped content + destinationFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer destinationFile.Close() + + // 7. Unzip the content of a file and copy it to the destination file + zippedFile, err := f.Open() + if err != nil { + return err + } + defer zippedFile.Close() + + if _, err := io.Copy(destinationFile, zippedFile); err != nil { + return err + } + return nil +} + +func resolveFolders(folder, dataFolder, pluginFolder string) (resolved string) { + resolved = strings.Replace(folder, "%plugin%", pluginFolder, 1) + resolved = strings.Replace(resolved, "%data%", dataFolder, 1) + + return resolved +} diff --git a/system/Read Package.go b/system/Read Package.go index fac5b1c..f430adb 100644 --- a/system/Read Package.go +++ b/system/Read Package.go @@ -1,137 +1,137 @@ -/* -File Name: Read Package.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package system - -import ( - "archive/zip" - "io/ioutil" - "path" - "strings" - - "gopkg.in/ini.v1" -) - -type UpdatePackage struct { - Filename string // Filename of the update package - Err error // Parsing error if any - Header *IniFile // Header info - Reader *zip.ReadCloser // Access to files in the ZIP file -} - -// IniFile contains the parsed data from the info.ini file -type IniFile struct { - Name string // Name of the package - Organization string // Organization that created this package - Architecture string // Target architecture. For example "windows/amd64". - Actions []IniAction // Actions to take -} - -type IniAction struct { - Action string `ini:"action"` // Action: extract, delete - Source string `ini:"source"` // Folder or file in the ZIP file - Target string `ini:"target"` // Target folder or file. Certain virtual folders such as "%plugins%" are supported. -} - -const IniFilename = "info.ini" - -// ParseUpdateFiles returns a list of parsed update packages. -// It will check each file in the directory if a ZIP file containing a valid info.ini file. -// The caller must close all returned readers. -func ParseUpdateFiles(Directory string) (files []UpdatePackage, err error) { - // check all files in the directory - filesDir, err := ioutil.ReadDir(Directory) - if err != nil { - return nil, err - } - - for _, file := range filesDir { - if file.IsDir() { - continue - } - - filename := file.Name() - - // ZIP file? - if strings.ToLower(path.Ext(filename)) == ".zip" { - filenamePath := path.Join(Directory, filename) - - // check if the ZIP archive contains a info.ini file - reader, err := zip.OpenReader(filenamePath) - if err != nil { // invalid ZIP file - continue - } - - // read info.ini file - file, err := reader.Open(IniFilename) - if err != nil { - continue - } - - data, err := ioutil.ReadAll(file) - if err != nil { - continue - } - - header, err := ParseIniFile(data) - - files = append(files, UpdatePackage{Filename: filenamePath, Err: err, Header: header, Reader: reader}) - } - } - - return files, nil -} - -func ParseIniFile(data []byte) (header *IniFile, err error) { - inidata, err := ini.Load(data) - if err != nil { - return nil, err - } - - // parse the main section first - section, err := inidata.GetSection("main") - if err != nil { - return nil, err - } - - name, err := section.GetKey("name") - if err != nil { - return nil, err - } - - organization, err := section.GetKey("organization") - if err != nil { - return nil, err - } - - architecture, err := section.GetKey("architecture") - if err != nil { - return nil, err - } - - header = &IniFile{ - Name: name.String(), - Organization: organization.String(), - Architecture: architecture.String(), - } - - // parse any other section - for _, section := range inidata.Sections() { - if section.Name() == "main" || section.Name() == "DEFAULT" { - continue - } - - var action IniAction - - if section.MapTo(&action) != nil { - continue - } - - header.Actions = append(header.Actions, action) - } - - return header, nil -} +/* +File Name: Read Package.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package system + +import ( + "archive/zip" + "io/ioutil" + "path" + "strings" + + "gopkg.in/ini.v1" +) + +type UpdatePackage struct { + Filename string // Filename of the update package + Err error // Parsing error if any + Header *IniFile // Header info + Reader *zip.ReadCloser // Access to files in the ZIP file +} + +// IniFile contains the parsed data from the info.ini file +type IniFile struct { + Name string // Name of the package + Organization string // Organization that created this package + Architecture string // Target architecture. For example "windows/amd64". + Actions []IniAction // Actions to take +} + +type IniAction struct { + Action string `ini:"action"` // Action: extract, delete + Source string `ini:"source"` // Folder or file in the ZIP file + Target string `ini:"target"` // Target folder or file. Certain virtual folders such as "%plugins%" are supported. +} + +const IniFilename = "info.ini" + +// ParseUpdateFiles returns a list of parsed update packages. +// It will check each file in the directory if a ZIP file containing a valid info.ini file. +// The caller must close all returned readers. +func ParseUpdateFiles(Directory string) (files []UpdatePackage, err error) { + // check all files in the directory + filesDir, err := ioutil.ReadDir(Directory) + if err != nil { + return nil, err + } + + for _, file := range filesDir { + if file.IsDir() { + continue + } + + filename := file.Name() + + // ZIP file? + if strings.ToLower(path.Ext(filename)) == ".zip" { + filenamePath := path.Join(Directory, filename) + + // check if the ZIP archive contains a info.ini file + reader, err := zip.OpenReader(filenamePath) + if err != nil { // invalid ZIP file + continue + } + + // read info.ini file + file, err := reader.Open(IniFilename) + if err != nil { + continue + } + + data, err := ioutil.ReadAll(file) + if err != nil { + continue + } + + header, err := ParseIniFile(data) + + files = append(files, UpdatePackage{Filename: filenamePath, Err: err, Header: header, Reader: reader}) + } + } + + return files, nil +} + +func ParseIniFile(data []byte) (header *IniFile, err error) { + inidata, err := ini.Load(data) + if err != nil { + return nil, err + } + + // parse the main section first + section, err := inidata.GetSection("main") + if err != nil { + return nil, err + } + + name, err := section.GetKey("name") + if err != nil { + return nil, err + } + + organization, err := section.GetKey("organization") + if err != nil { + return nil, err + } + + architecture, err := section.GetKey("architecture") + if err != nil { + return nil, err + } + + header = &IniFile{ + Name: name.String(), + Organization: organization.String(), + Architecture: architecture.String(), + } + + // parse any other section + for _, section := range inidata.Sections() { + if section.Name() == "main" || section.Name() == "DEFAULT" { + continue + } + + var action IniAction + + if section.MapTo(&action) != nil { + continue + } + + header.Actions = append(header.Actions, action) + } + + return header, nil +} diff --git a/udt/FloydRivest.go b/udt/FloydRivest.go index 76a824e..68db78d 100644 --- a/udt/FloydRivest.go +++ b/udt/FloydRivest.go @@ -1,123 +1,123 @@ -package udt - -import ( - "math" - "sort" -) - -// fork from github.com/furstenheim/nth_element/FloydRivest - -// FloydRivestBuckets. Sort a slice into buckets of given size. All elements from one bucket are smaller than any element from the next one. -// elements at position i * bucketSize are guaranteed to be the (i * bucketSize) th smallest elements -// s := // some slice -// FloydRivest.Buckets(sort.Interface(s), 5) -// s is now sorted into buckets of size 5 -// max(s[0:5]) < min(s[5:10]) -// max(s[10: 15]) < min(s[15:20]) -// ... -func FloydRivestBuckets(slice sort.Interface, bucketSize int) { - left := 0 - right := slice.Len() - 1 - s := floydRivestStack([]int{left, right}) - var mid int - for len(s) > 0 { - s, right = s.Pop() - s, left = s.Pop() - if right-left <= bucketSize { - continue - } - // + bucketSize - 1 is to do math ceil - mid = left + ((right-left+bucketSize-1)/bucketSize/2)*bucketSize - FloydRivestSelect(slice, mid, left, right) - s = s.Push(left) - s = s.Push(mid) - s = s.Push(mid) - s = s.Push(right) - } -} - -// left is the left index for the interval -// right is the right index for the interval -// k is the desired index value, where array[k] is the k+1 smallest element -// when left = 0 -func FloydRivestSelect(array sort.Interface, k, left, right int) { - length := array.Len() - for right > left { - if right-left > 600 { - var n = float64(right - left + 1) - var kf = float64(k) - var m = float64(k - left + 1) - var z = math.Log(n) - var s = 0.5 * math.Exp(2*z/3) - sign := float64(1) - if m-n/2 < 0 { - sign = -1 - } - var sd = 0.5 * math.Sqrt(z*s*(n-s)/n) * sign - var newLeft = intMax(left, int(math.Floor(kf-m*s/n+sd))) - var newRight = intMin(right, int(math.Floor(kf+(n-m)*s/n+sd))) - FloydRivestSelect(array, k, newLeft, newRight) - } - - var i = left - var j = right - array.Swap(left, k) - // in the original algorithm array[k] is stored to a value. To use golangs sort interface we need to keep track of the changes for the index - // we define it as right because in the first iteration of for i= 0 && array.Less(pointIndex, j) { - j-- - } - } - // All equal points - if !array.Less(left, pointIndex) && !array.Less(pointIndex, left) { - array.Swap(left, j) - } else { - j++ - array.Swap(j, right) - } - if j <= k { - left = j + 1 - } - if k <= j { - right = j - 1 - } - } -} - -func intMin(a, b int) int { - if a < b { - return a - } - return b -} - -func intMax(a, b int) int { - if a > b { - return a - } - return b -} - -type floydRivestStack []int - -func (s floydRivestStack) Push(v int) floydRivestStack { - return append(s, v) -} -func (s floydRivestStack) Pop() (floydRivestStack, int) { - l := len(s) - return s[:l-1], s[l-1] -} +package udt + +import ( + "math" + "sort" +) + +// fork from github.com/furstenheim/nth_element/FloydRivest + +// FloydRivestBuckets. Sort a slice into buckets of given size. All elements from one bucket are smaller than any element from the next one. +// elements at position i * bucketSize are guaranteed to be the (i * bucketSize) th smallest elements +// s := // some slice +// FloydRivest.Buckets(sort.Interface(s), 5) +// s is now sorted into buckets of size 5 +// max(s[0:5]) < min(s[5:10]) +// max(s[10: 15]) < min(s[15:20]) +// ... +func FloydRivestBuckets(slice sort.Interface, bucketSize int) { + left := 0 + right := slice.Len() - 1 + s := floydRivestStack([]int{left, right}) + var mid int + for len(s) > 0 { + s, right = s.Pop() + s, left = s.Pop() + if right-left <= bucketSize { + continue + } + // + bucketSize - 1 is to do math ceil + mid = left + ((right-left+bucketSize-1)/bucketSize/2)*bucketSize + FloydRivestSelect(slice, mid, left, right) + s = s.Push(left) + s = s.Push(mid) + s = s.Push(mid) + s = s.Push(right) + } +} + +// left is the left index for the interval +// right is the right index for the interval +// k is the desired index value, where array[k] is the k+1 smallest element +// when left = 0 +func FloydRivestSelect(array sort.Interface, k, left, right int) { + length := array.Len() + for right > left { + if right-left > 600 { + var n = float64(right - left + 1) + var kf = float64(k) + var m = float64(k - left + 1) + var z = math.Log(n) + var s = 0.5 * math.Exp(2*z/3) + sign := float64(1) + if m-n/2 < 0 { + sign = -1 + } + var sd = 0.5 * math.Sqrt(z*s*(n-s)/n) * sign + var newLeft = intMax(left, int(math.Floor(kf-m*s/n+sd))) + var newRight = intMin(right, int(math.Floor(kf+(n-m)*s/n+sd))) + FloydRivestSelect(array, k, newLeft, newRight) + } + + var i = left + var j = right + array.Swap(left, k) + // in the original algorithm array[k] is stored to a value. To use golangs sort interface we need to keep track of the changes for the index + // we define it as right because in the first iteration of for i= 0 && array.Less(pointIndex, j) { + j-- + } + } + // All equal points + if !array.Less(left, pointIndex) && !array.Less(pointIndex, left) { + array.Swap(left, j) + } else { + j++ + array.Swap(j, right) + } + if j <= k { + left = j + 1 + } + if k <= j { + right = j - 1 + } + } +} + +func intMin(a, b int) int { + if a < b { + return a + } + return b +} + +func intMax(a, b int) int { + if a > b { + return a + } + return b +} + +type floydRivestStack []int + +func (s floydRivestStack) Push(v int) floydRivestStack { + return append(s, v) +} +func (s floydRivestStack) Pop() (floydRivestStack, int) { + l := len(s) + return s[:l-1], s[l-1] +} diff --git a/upnp/UPnP.go b/upnp/UPnP.go index f1bfd67..dbcd0d5 100644 --- a/upnp/UPnP.go +++ b/upnp/UPnP.go @@ -1,408 +1,408 @@ -/* -File Name: UPnP.go -Copyright: 2021 Peernet s.r.o. -Author: Peter Kleissner -*/ - -package upnp - -import ( - "bytes" - "encoding/xml" - "errors" - "net" - "net/http" - "strconv" - "strings" - "time" -) - -// NAT is an interface representing a NAT traversal options for example UPNP or NAT-PMP. -// It provides methods to query and manipulate this traversal to allow access to services. -type NAT interface { - // Get the external address from outside the NAT. - GetExternalAddress() (addr net.IP, err error) - // Add a port mapping for protocol ("udp" or "tcp") from external port to internal port with description lasting for timeout. - AddPortMapping(protocol string, internalIP net.IP, internalPort, externalPort uint16, description string, timeout int) (mappedExternalPort uint16, err error) - // Remove a previously added port mapping from external port to internal port. - DeletePortMapping(protocol string, externalPort uint16) (err error) -} - -type upnpNAT struct { - serviceURL string - urnDomain string - localIP net.IP -} - -// Discover searches the local network for a UPnP router returning a NAT for the network if so, nil if not. -// Socket must be an active local socket. -func Discover(localIP net.IP) (nat NAT, err error) { - ssdp, err := net.ResolveUDPAddr("udp4", "239.255.255.250:1900") - if err != nil { - return - } - conn, err := net.ListenPacket("udp4", net.JoinHostPort(localIP.String(), "0")) // use a random port - if err != nil { - return - } - socket := conn.(*net.UDPConn) - defer socket.Close() - - err = socket.SetDeadline(time.Now().Add(3 * time.Second)) - if err != nil { - return - } - - st := "InternetGatewayDevice:1" - - buf := bytes.NewBufferString( - "M-SEARCH * HTTP/1.1\r\n" + - "HOST: 239.255.255.250:1900\r\n" + - "ST: ssdp:all\r\n" + - "MAN: \"ssdp:discover\"\r\n" + - "MX: 2\r\n\r\n") - message := buf.Bytes() - answerBytes := make([]byte, 1024) - for i := 0; i < 3; i++ { - _, err = socket.WriteToUDP(message, ssdp) - if err != nil { - return - } - var n int - _, _, err = socket.ReadFromUDP(answerBytes) - if err != nil { - return - } - for { - n, _, err = socket.ReadFromUDP(answerBytes) - if err != nil { - break - } - answer := string(answerBytes[0:n]) - if !strings.Contains(answer, st) { - continue - } - // HTTP header field names are case-insensitive. - // http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 - locString := "\r\nlocation:" - answer = strings.ToLower(answer) - locIndex := strings.Index(answer, locString) - if locIndex < 0 { - continue - } - loc := answer[locIndex+len(locString):] - endIndex := strings.Index(loc, "\r\n") - if endIndex < 0 { - continue - } - locURL := strings.TrimSpace(loc[0:endIndex]) - var serviceURL, urnDomain string - serviceURL, urnDomain, err = getServiceURL(localIP, locURL) - if err != nil { - return - } - nat = &upnpNAT{serviceURL: serviceURL, urnDomain: urnDomain, localIP: localIP} - return - } - } - err = errors.New("UPnP port discovery failed") - return -} - -// service represents the Service type in an UPnP xml description. -// Only the parts we care about are present and thus the xml may have more -// fields than present in the structure. -type service struct { - ServiceType string `xml:"serviceType"` - ControlURL string `xml:"controlURL"` -} - -// deviceList represents the deviceList type in an UPnP xml description. -// Only the parts we care about are present and thus the xml may have more -// fields than present in the structure. -type deviceList struct { - XMLName xml.Name `xml:"deviceList"` - Device []device `xml:"device"` -} - -// serviceList represents the serviceList type in an UPnP xml description. -// Only the parts we care about are present and thus the xml may have more -// fields than present in the structure. -type serviceList struct { - XMLName xml.Name `xml:"serviceList"` - Service []service `xml:"service"` -} - -// device represents the device type in an UPnP xml description. -// Only the parts we care about are present and thus the xml may have more -// fields than present in the structure. -type device struct { - XMLName xml.Name `xml:"device"` - DeviceType string `xml:"deviceType"` - DeviceList deviceList `xml:"deviceList"` - ServiceList serviceList `xml:"serviceList"` -} - -// specVersion represents the specVersion in a UPnP xml description. -// Only the parts we care about are present and thus the xml may have more -// fields than present in the structure. -type specVersion struct { - XMLName xml.Name `xml:"specVersion"` - Major int `xml:"major"` - Minor int `xml:"minor"` -} - -// root represents the Root document for a UPnP xml description. -// Only the parts we care about are present and thus the xml may have more -// fields than present in the structure. -type root struct { - XMLName xml.Name `xml:"root"` - SpecVersion specVersion - Device device -} - -// getChildDevice searches the children of device for a device with the given -// type. -func getChildDevice(d *device, deviceType string) *device { - for i := range d.DeviceList.Device { - if strings.Contains(d.DeviceList.Device[i].DeviceType, deviceType) { - return &d.DeviceList.Device[i] - } - } - return nil -} - -// getChildDevice searches the service list of device for a service with the -// given type. -func getChildService(d *device, serviceType string) *service { - for i := range d.ServiceList.Service { - if strings.Contains(d.ServiceList.Service[i].ServiceType, serviceType) { - return &d.ServiceList.Service[i] - } - } - return nil -} - -// getServiceURL parses the xml description at the given root url to find the -// url for the WANIPConnection service to be used for port forwarding. -func getServiceURL(localIP net.IP, rootURL string) (url, urnDomain string, err error) { - - webclient := &http.Client{ - Transport: &http.Transport{ - Proxy: nil, - DialContext: (&net.Dialer{ - LocalAddr: &net.TCPAddr{ - IP: localIP, - }, - Timeout: 3 * time.Second, - DualStack: true, - }).DialContext, - TLSHandshakeTimeout: 3 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - Timeout: 3 * time.Second, - } - - r, err := webclient.Get(rootURL) - if err != nil { - return - } - defer r.Body.Close() - - if r.StatusCode >= 400 { - err = errors.New("Unexpected status code " + strconv.Itoa(r.StatusCode)) - return - } - var root root - err = xml.NewDecoder(r.Body).Decode(&root) - if err != nil { - return - } - a := &root.Device - if !strings.Contains(a.DeviceType, "InternetGatewayDevice:1") { - err = errors.New("no InternetGatewayDevice") - return - } - b := getChildDevice(a, "WANDevice:1") - if b == nil { - err = errors.New("no WANDevice") - return - } - c := getChildDevice(b, "WANConnectionDevice:1") - if c == nil { - err = errors.New("no WANConnectionDevice") - return - } - d := getChildService(c, "WANIPConnection:1") - if d == nil { - // Some routers don't follow the UPnP spec, and put WanIPConnection under WanDevice, - // instead of under WanConnectionDevice - d = getChildService(b, "WANIPConnection:1") - - if d == nil { - err = errors.New("no WANIPConnection") - return - } - } - // Extract the domain name, which isn't always 'schemas-upnp-org' - urnDomain = strings.Split(d.ServiceType, ":")[1] - url = combineURL(rootURL, d.ControlURL) - return url, urnDomain, err -} - -// combineURL appends subURL onto rootURL. -func combineURL(rootURL, subURL string) string { - protocolEnd := "://" - protoEndIndex := strings.Index(rootURL, protocolEnd) - a := rootURL[protoEndIndex+len(protocolEnd):] - rootIndex := strings.Index(a, "/") - return rootURL[0:protoEndIndex+len(protocolEnd)+rootIndex] + subURL -} - -// soapBody represents the element in a SOAP reply. -// fields we don't care about are elided. -type soapBody struct { - XMLName xml.Name `xml:"Body"` - Data []byte `xml:",innerxml"` -} - -// soapEnvelope represents the element in a SOAP reply. -// fields we don't care about are elided. -type soapEnvelope struct { - XMLName xml.Name `xml:"Envelope"` - Body soapBody `xml:"Body"` -} - -// soapRequests performs a soap request with the given parameters and returns -// the xml replied stripped of the soap headers. in the case that the request is -// unsuccessful the an error is returned. -func (n *upnpNAT) soapRequest(url, function, message, domain string) (replyXML []byte, err error) { - fullMessage := "" + - "\r\n" + - "" + message + "" - - req, err := http.NewRequest("POST", url, strings.NewReader(fullMessage)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "text/xml ; charset=\"utf-8\"") - req.Header.Set("User-Agent", "Darwin/10.0.0, UPnP/1.0, MiniUPnPc/1.3") - req.Header.Set("SOAPAction", "\"urn:"+domain+":service:WANIPConnection:1#"+function+"\"") - req.Header.Set("Connection", "Close") - req.Header.Set("Cache-Control", "no-cache") - req.Header.Set("Pragma", "no-cache") - - webclient := &http.Client{ - Transport: &http.Transport{ - Proxy: nil, - DialContext: (&net.Dialer{ - LocalAddr: &net.TCPAddr{ - IP: n.localIP, - }, - Timeout: 3 * time.Second, - DualStack: true, - }).DialContext, - TLSHandshakeTimeout: 3 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, - Timeout: 3 * time.Second, - } - - r, err := webclient.Do(req) - if err != nil { - return nil, err - } - if r.Body != nil { - defer r.Body.Close() - } - - if r.StatusCode >= 400 { - err = errors.New("Error " + strconv.Itoa(r.StatusCode) + " for " + function) - r = nil - return - } - var reply soapEnvelope - err = xml.NewDecoder(r.Body).Decode(&reply) - if err != nil { - return nil, err - } - return reply.Body.Data, nil -} - -// getExternalIPAddressResponse represents the XML response to a -// GetExternalIPAddress SOAP request. -type getExternalIPAddressResponse struct { - XMLName xml.Name `xml:"GetExternalIPAddressResponse"` - ExternalIPAddress string `xml:"NewExternalIPAddress"` -} - -// GetExternalAddress implements the NAT interface by fetching the external IP -// from the UPnP router. -func (n *upnpNAT) GetExternalAddress() (addr net.IP, err error) { - message := "\r\n" - response, err := n.soapRequest(n.serviceURL, "GetExternalIPAddress", message, n.urnDomain) - if err != nil { - return nil, err - } - - var reply getExternalIPAddressResponse - err = xml.Unmarshal(response, &reply) - if err != nil { - return nil, err - } - - addr = net.ParseIP(reply.ExternalIPAddress) - if addr == nil { - return nil, errors.New("unable to parse ip address") - } - return addr, nil -} - -// AddPortMapping forwards a port at the UPnP router to the specified IP address and port. Lease duration is in seconds. -// FritzBox routers: Forwarding an already forwarded port results in no error. If the internal port is already forwarded under a different external port, error code 718 is returned in XML. -func (n *upnpNAT) AddPortMapping(protocol string, internalIP net.IP, internalPort, externalPort uint16, description string, leaseDuration int) (mappedExternalPort uint16, err error) { - // A single concatenation would break ARM compilation. - message := "\r\n" + - "" + strconv.Itoa(int(externalPort)) - message += "" + strings.ToUpper(protocol) + "" - message += "" + strconv.Itoa(int(internalPort)) + "" + - "" + internalIP.String() + "" + - "1" - message += description + - "" + strconv.Itoa(leaseDuration) + - "" - - response, err := n.soapRequest(n.serviceURL, "AddPortMapping", message, n.urnDomain) - if err != nil { - // If UPnP is not allowed for the host (with FritzBox routers the user must manually enable it), the router returns "606" - // in the XML response with HTTP code 500 Internal Server Error. - // If the internal port is already forwarded under a different external port, error code 718 is returned in XML. - return - } - - // TODO: check response to see if the port was forwarded - // If the port was not wildcard we don't get an reply with the port in it. Not sure about wildcard yet. miniupnpc just checks for error codes here. - mappedExternalPort = externalPort - _ = response - - return mappedExternalPort, err -} - -// DeletePortMapping deletes a port mapping. -func (n *upnpNAT) DeletePortMapping(protocol string, externalPort uint16) (err error) { - - message := "\r\n" + - "" + strconv.Itoa(int(externalPort)) + - "" + strings.ToUpper(protocol) + "" + - "" - - response, err := n.soapRequest(n.serviceURL, "DeletePortMapping", message, n.urnDomain) - if err != nil { - return - } - - // TODO: check response to see if the port was deleted - // log.Println(message, response) - _ = response - return -} +/* +File Name: UPnP.go +Copyright: 2021 Peernet s.r.o. +Author: Peter Kleissner +*/ + +package upnp + +import ( + "bytes" + "encoding/xml" + "errors" + "net" + "net/http" + "strconv" + "strings" + "time" +) + +// NAT is an interface representing a NAT traversal options for example UPNP or NAT-PMP. +// It provides methods to query and manipulate this traversal to allow access to services. +type NAT interface { + // Get the external address from outside the NAT. + GetExternalAddress() (addr net.IP, err error) + // Add a port mapping for protocol ("udp" or "tcp") from external port to internal port with description lasting for timeout. + AddPortMapping(protocol string, internalIP net.IP, internalPort, externalPort uint16, description string, timeout int) (mappedExternalPort uint16, err error) + // Remove a previously added port mapping from external port to internal port. + DeletePortMapping(protocol string, externalPort uint16) (err error) +} + +type upnpNAT struct { + serviceURL string + urnDomain string + localIP net.IP +} + +// Discover searches the local network for a UPnP router returning a NAT for the network if so, nil if not. +// Socket must be an active local socket. +func Discover(localIP net.IP) (nat NAT, err error) { + ssdp, err := net.ResolveUDPAddr("udp4", "239.255.255.250:1900") + if err != nil { + return + } + conn, err := net.ListenPacket("udp4", net.JoinHostPort(localIP.String(), "0")) // use a random port + if err != nil { + return + } + socket := conn.(*net.UDPConn) + defer socket.Close() + + err = socket.SetDeadline(time.Now().Add(3 * time.Second)) + if err != nil { + return + } + + st := "InternetGatewayDevice:1" + + buf := bytes.NewBufferString( + "M-SEARCH * HTTP/1.1\r\n" + + "HOST: 239.255.255.250:1900\r\n" + + "ST: ssdp:all\r\n" + + "MAN: \"ssdp:discover\"\r\n" + + "MX: 2\r\n\r\n") + message := buf.Bytes() + answerBytes := make([]byte, 1024) + for i := 0; i < 3; i++ { + _, err = socket.WriteToUDP(message, ssdp) + if err != nil { + return + } + var n int + _, _, err = socket.ReadFromUDP(answerBytes) + if err != nil { + return + } + for { + n, _, err = socket.ReadFromUDP(answerBytes) + if err != nil { + break + } + answer := string(answerBytes[0:n]) + if !strings.Contains(answer, st) { + continue + } + // HTTP header field names are case-insensitive. + // http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 + locString := "\r\nlocation:" + answer = strings.ToLower(answer) + locIndex := strings.Index(answer, locString) + if locIndex < 0 { + continue + } + loc := answer[locIndex+len(locString):] + endIndex := strings.Index(loc, "\r\n") + if endIndex < 0 { + continue + } + locURL := strings.TrimSpace(loc[0:endIndex]) + var serviceURL, urnDomain string + serviceURL, urnDomain, err = getServiceURL(localIP, locURL) + if err != nil { + return + } + nat = &upnpNAT{serviceURL: serviceURL, urnDomain: urnDomain, localIP: localIP} + return + } + } + err = errors.New("UPnP port discovery failed") + return +} + +// service represents the Service type in an UPnP xml description. +// Only the parts we care about are present and thus the xml may have more +// fields than present in the structure. +type service struct { + ServiceType string `xml:"serviceType"` + ControlURL string `xml:"controlURL"` +} + +// deviceList represents the deviceList type in an UPnP xml description. +// Only the parts we care about are present and thus the xml may have more +// fields than present in the structure. +type deviceList struct { + XMLName xml.Name `xml:"deviceList"` + Device []device `xml:"device"` +} + +// serviceList represents the serviceList type in an UPnP xml description. +// Only the parts we care about are present and thus the xml may have more +// fields than present in the structure. +type serviceList struct { + XMLName xml.Name `xml:"serviceList"` + Service []service `xml:"service"` +} + +// device represents the device type in an UPnP xml description. +// Only the parts we care about are present and thus the xml may have more +// fields than present in the structure. +type device struct { + XMLName xml.Name `xml:"device"` + DeviceType string `xml:"deviceType"` + DeviceList deviceList `xml:"deviceList"` + ServiceList serviceList `xml:"serviceList"` +} + +// specVersion represents the specVersion in a UPnP xml description. +// Only the parts we care about are present and thus the xml may have more +// fields than present in the structure. +type specVersion struct { + XMLName xml.Name `xml:"specVersion"` + Major int `xml:"major"` + Minor int `xml:"minor"` +} + +// root represents the Root document for a UPnP xml description. +// Only the parts we care about are present and thus the xml may have more +// fields than present in the structure. +type root struct { + XMLName xml.Name `xml:"root"` + SpecVersion specVersion + Device device +} + +// getChildDevice searches the children of device for a device with the given +// type. +func getChildDevice(d *device, deviceType string) *device { + for i := range d.DeviceList.Device { + if strings.Contains(d.DeviceList.Device[i].DeviceType, deviceType) { + return &d.DeviceList.Device[i] + } + } + return nil +} + +// getChildDevice searches the service list of device for a service with the +// given type. +func getChildService(d *device, serviceType string) *service { + for i := range d.ServiceList.Service { + if strings.Contains(d.ServiceList.Service[i].ServiceType, serviceType) { + return &d.ServiceList.Service[i] + } + } + return nil +} + +// getServiceURL parses the xml description at the given root url to find the +// url for the WANIPConnection service to be used for port forwarding. +func getServiceURL(localIP net.IP, rootURL string) (url, urnDomain string, err error) { + + webclient := &http.Client{ + Transport: &http.Transport{ + Proxy: nil, + DialContext: (&net.Dialer{ + LocalAddr: &net.TCPAddr{ + IP: localIP, + }, + Timeout: 3 * time.Second, + DualStack: true, + }).DialContext, + TLSHandshakeTimeout: 3 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + Timeout: 3 * time.Second, + } + + r, err := webclient.Get(rootURL) + if err != nil { + return + } + defer r.Body.Close() + + if r.StatusCode >= 400 { + err = errors.New("Unexpected status code " + strconv.Itoa(r.StatusCode)) + return + } + var root root + err = xml.NewDecoder(r.Body).Decode(&root) + if err != nil { + return + } + a := &root.Device + if !strings.Contains(a.DeviceType, "InternetGatewayDevice:1") { + err = errors.New("no InternetGatewayDevice") + return + } + b := getChildDevice(a, "WANDevice:1") + if b == nil { + err = errors.New("no WANDevice") + return + } + c := getChildDevice(b, "WANConnectionDevice:1") + if c == nil { + err = errors.New("no WANConnectionDevice") + return + } + d := getChildService(c, "WANIPConnection:1") + if d == nil { + // Some routers don't follow the UPnP spec, and put WanIPConnection under WanDevice, + // instead of under WanConnectionDevice + d = getChildService(b, "WANIPConnection:1") + + if d == nil { + err = errors.New("no WANIPConnection") + return + } + } + // Extract the domain name, which isn't always 'schemas-upnp-org' + urnDomain = strings.Split(d.ServiceType, ":")[1] + url = combineURL(rootURL, d.ControlURL) + return url, urnDomain, err +} + +// combineURL appends subURL onto rootURL. +func combineURL(rootURL, subURL string) string { + protocolEnd := "://" + protoEndIndex := strings.Index(rootURL, protocolEnd) + a := rootURL[protoEndIndex+len(protocolEnd):] + rootIndex := strings.Index(a, "/") + return rootURL[0:protoEndIndex+len(protocolEnd)+rootIndex] + subURL +} + +// soapBody represents the element in a SOAP reply. +// fields we don't care about are elided. +type soapBody struct { + XMLName xml.Name `xml:"Body"` + Data []byte `xml:",innerxml"` +} + +// soapEnvelope represents the element in a SOAP reply. +// fields we don't care about are elided. +type soapEnvelope struct { + XMLName xml.Name `xml:"Envelope"` + Body soapBody `xml:"Body"` +} + +// soapRequests performs a soap request with the given parameters and returns +// the xml replied stripped of the soap headers. in the case that the request is +// unsuccessful the an error is returned. +func (n *upnpNAT) soapRequest(url, function, message, domain string) (replyXML []byte, err error) { + fullMessage := "" + + "\r\n" + + "" + message + "" + + req, err := http.NewRequest("POST", url, strings.NewReader(fullMessage)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "text/xml ; charset=\"utf-8\"") + req.Header.Set("User-Agent", "Darwin/10.0.0, UPnP/1.0, MiniUPnPc/1.3") + req.Header.Set("SOAPAction", "\"urn:"+domain+":service:WANIPConnection:1#"+function+"\"") + req.Header.Set("Connection", "Close") + req.Header.Set("Cache-Control", "no-cache") + req.Header.Set("Pragma", "no-cache") + + webclient := &http.Client{ + Transport: &http.Transport{ + Proxy: nil, + DialContext: (&net.Dialer{ + LocalAddr: &net.TCPAddr{ + IP: n.localIP, + }, + Timeout: 3 * time.Second, + DualStack: true, + }).DialContext, + TLSHandshakeTimeout: 3 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + Timeout: 3 * time.Second, + } + + r, err := webclient.Do(req) + if err != nil { + return nil, err + } + if r.Body != nil { + defer r.Body.Close() + } + + if r.StatusCode >= 400 { + err = errors.New("Error " + strconv.Itoa(r.StatusCode) + " for " + function) + r = nil + return + } + var reply soapEnvelope + err = xml.NewDecoder(r.Body).Decode(&reply) + if err != nil { + return nil, err + } + return reply.Body.Data, nil +} + +// getExternalIPAddressResponse represents the XML response to a +// GetExternalIPAddress SOAP request. +type getExternalIPAddressResponse struct { + XMLName xml.Name `xml:"GetExternalIPAddressResponse"` + ExternalIPAddress string `xml:"NewExternalIPAddress"` +} + +// GetExternalAddress implements the NAT interface by fetching the external IP +// from the UPnP router. +func (n *upnpNAT) GetExternalAddress() (addr net.IP, err error) { + message := "\r\n" + response, err := n.soapRequest(n.serviceURL, "GetExternalIPAddress", message, n.urnDomain) + if err != nil { + return nil, err + } + + var reply getExternalIPAddressResponse + err = xml.Unmarshal(response, &reply) + if err != nil { + return nil, err + } + + addr = net.ParseIP(reply.ExternalIPAddress) + if addr == nil { + return nil, errors.New("unable to parse ip address") + } + return addr, nil +} + +// AddPortMapping forwards a port at the UPnP router to the specified IP address and port. Lease duration is in seconds. +// FritzBox routers: Forwarding an already forwarded port results in no error. If the internal port is already forwarded under a different external port, error code 718 is returned in XML. +func (n *upnpNAT) AddPortMapping(protocol string, internalIP net.IP, internalPort, externalPort uint16, description string, leaseDuration int) (mappedExternalPort uint16, err error) { + // A single concatenation would break ARM compilation. + message := "\r\n" + + "" + strconv.Itoa(int(externalPort)) + message += "" + strings.ToUpper(protocol) + "" + message += "" + strconv.Itoa(int(internalPort)) + "" + + "" + internalIP.String() + "" + + "1" + message += description + + "" + strconv.Itoa(leaseDuration) + + "" + + response, err := n.soapRequest(n.serviceURL, "AddPortMapping", message, n.urnDomain) + if err != nil { + // If UPnP is not allowed for the host (with FritzBox routers the user must manually enable it), the router returns "606" + // in the XML response with HTTP code 500 Internal Server Error. + // If the internal port is already forwarded under a different external port, error code 718 is returned in XML. + return + } + + // TODO: check response to see if the port was forwarded + // If the port was not wildcard we don't get an reply with the port in it. Not sure about wildcard yet. miniupnpc just checks for error codes here. + mappedExternalPort = externalPort + _ = response + + return mappedExternalPort, err +} + +// DeletePortMapping deletes a port mapping. +func (n *upnpNAT) DeletePortMapping(protocol string, externalPort uint16) (err error) { + + message := "\r\n" + + "" + strconv.Itoa(int(externalPort)) + + "" + strings.ToUpper(protocol) + "" + + "" + + response, err := n.soapRequest(n.serviceURL, "DeletePortMapping", message, n.urnDomain) + if err != nil { + return + } + + // TODO: check response to see if the port was deleted + // log.Println(message, response) + _ = response + return +} diff --git a/upnp/UPnP_test.go b/upnp/UPnP_test.go index 60f5d05..3c2bb8c 100644 --- a/upnp/UPnP_test.go +++ b/upnp/UPnP_test.go @@ -1,27 +1,27 @@ -// Small code for manual tests/development. - -package upnp - -import ( - "fmt" - "net" - "testing" -) - -func TestUPnP(t *testing.T) { - localIP := net.ParseIP("0.0.0.0") - - nat, err := Discover(localIP) - if err != nil { - fmt.Printf("%v\n", err) - return - } - - addr, err := nat.GetExternalAddress() - if err != nil { - fmt.Printf("%v\n", err) - return - } - - fmt.Printf("%s\n", addr.String()) -} +// Small code for manual tests/development. + +package upnp + +import ( + "fmt" + "net" + "testing" +) + +func TestUPnP(t *testing.T) { + localIP := net.ParseIP("0.0.0.0") + + nat, err := Discover(localIP) + if err != nil { + fmt.Printf("%v\n", err) + return + } + + addr, err := nat.GetExternalAddress() + if err != nil { + fmt.Printf("%v\n", err) + return + } + + fmt.Printf("%s\n", addr.String()) +} diff --git a/warehouse/Merkle.go b/warehouse/Merkle.go index d5a47dd..ac552dd 100644 --- a/warehouse/Merkle.go +++ b/warehouse/Merkle.go @@ -1,120 +1,120 @@ -/* -File Name: Merkle.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package warehouse - -import ( - "errors" - "io" - "os" - "path/filepath" - - "github.com/PeernetOfficial/core/merkle" -) - -// Merkle companion files are created to store the entire merkle tree for files that are bigger than the minimum fragment size. -const merkleCompanionExt = ".merkle" - -// MerkleFileExists checks if the merkle companion file exists. It returns StatusInvalidHash, StatusFileNotFound, or StatusOK. -func (wh *Warehouse) MerkleFileExists(hash []byte) (path string, fileSize uint64, status int, err error) { - hashA, err := ValidateHash(hash) - if err != nil { - return "", 0, StatusInvalidHash, err - } - - a, b := buildPath(wh.Directory, hashA) - path = filepath.Join(a, b) + merkleCompanionExt - - if fileInfo, err := os.Stat(path); err == nil { - // file exists - return path, uint64(fileInfo.Size()), StatusOK, nil - } - - return "", 0, StatusFileNotFound, os.ErrNotExist -} - -// createMerkleCompanionFile creates a merkle companion file. If the merkle companion file already exists, it is overwritten. -// dataFilePath is the full path to the data file in the warehouse. -func (wh *Warehouse) createMerkleCompanionFile(dataFilePath string) (status int, err error) { - // open the data file - dataFile, err := os.Open(dataFilePath) - if err != nil && os.IsNotExist(err) { - return StatusFileNotFound, err - } else if err != nil { - return StatusErrorOpenFile, err - } - defer dataFile.Close() - - var fileSize uint64 - stat, err := dataFile.Stat() - if err != nil { - return StatusErrorOpenFile, err - } - fileSize = uint64(stat.Size()) - - // Merkle files are only created if merkle trees are actually used, which means the file must be bigger than the minimum fragment size. - // Otherwise the merkle root hash will be just the file hash and the merkle overhead provides no advantage whatsoever. - if fileSize <= merkle.MinimumFragmentSize { - return StatusOK, nil - } - - // Create a new merkle file. If one exists, overwrite. - merkleFile := dataFilePath + merkleCompanionExt - - fileM, err := os.OpenFile(merkleFile, os.O_WRONLY|os.O_CREATE, 0666) // 666 = All uses can read/write - if err != nil { - return StatusErrorCreateTarget, err - } - defer fileM.Close() - - // create the merkle tree and write it to the companion file - fragmentSize := merkle.CalculateFragmentSize(fileSize) - tree, err := merkle.NewMerkleTree(fileSize, fragmentSize, dataFile) - if err != nil { - return StatusErrorCreateMerkle, err - } - - fileM.Write(tree.Export()) - - return StatusOK, nil -} - -// ReadMerkleTree reads the merkle tree from the companion file associated with the hash. -// It is the callers responsibility to first check if a merkle tree file is to be expected (files smaller or equal than the minimum fragment size do not use a merkle tree). -func (wh *Warehouse) ReadMerkleTree(hash []byte, headerOnly bool) (tree *merkle.MerkleTree, status int, err error) { - path, _, status, err := wh.MerkleFileExists(hash) - if status != StatusOK { // file does not exist or invalid hash - return nil, status, err - } - - fileM, err := os.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return nil, StatusErrorOpenFile, err - } - defer fileM.Close() - - if headerOnly { - data := make([]byte, merkle.MerkleTreeFileHeaderSize) - if _, err = io.ReadFull(fileM, data); err != nil { - return nil, StatusErrorReadFile, err - } - - if tree = merkle.ReadMerkleTreeHeader(data); tree == nil { - return nil, StatusErrorMerkleTreeFile, errors.New("invalid merkle tree file header") - } - } else { - data, err := io.ReadAll(fileM) - if err != nil { - return nil, StatusErrorReadFile, err - } - - if tree = merkle.ImportMerkleTree(data); tree == nil { - return nil, StatusErrorMerkleTreeFile, errors.New("invalid merkle tree file header") - } - } - - return tree, StatusOK, nil -} +/* +File Name: Merkle.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package warehouse + +import ( + "errors" + "io" + "os" + "path/filepath" + + "github.com/PeernetOfficial/core/merkle" +) + +// Merkle companion files are created to store the entire merkle tree for files that are bigger than the minimum fragment size. +const merkleCompanionExt = ".merkle" + +// MerkleFileExists checks if the merkle companion file exists. It returns StatusInvalidHash, StatusFileNotFound, or StatusOK. +func (wh *Warehouse) MerkleFileExists(hash []byte) (path string, fileSize uint64, status int, err error) { + hashA, err := ValidateHash(hash) + if err != nil { + return "", 0, StatusInvalidHash, err + } + + a, b := buildPath(wh.Directory, hashA) + path = filepath.Join(a, b) + merkleCompanionExt + + if fileInfo, err := os.Stat(path); err == nil { + // file exists + return path, uint64(fileInfo.Size()), StatusOK, nil + } + + return "", 0, StatusFileNotFound, os.ErrNotExist +} + +// createMerkleCompanionFile creates a merkle companion file. If the merkle companion file already exists, it is overwritten. +// dataFilePath is the full path to the data file in the warehouse. +func (wh *Warehouse) createMerkleCompanionFile(dataFilePath string) (status int, err error) { + // open the data file + dataFile, err := os.Open(dataFilePath) + if err != nil && os.IsNotExist(err) { + return StatusFileNotFound, err + } else if err != nil { + return StatusErrorOpenFile, err + } + defer dataFile.Close() + + var fileSize uint64 + stat, err := dataFile.Stat() + if err != nil { + return StatusErrorOpenFile, err + } + fileSize = uint64(stat.Size()) + + // Merkle files are only created if merkle trees are actually used, which means the file must be bigger than the minimum fragment size. + // Otherwise the merkle root hash will be just the file hash and the merkle overhead provides no advantage whatsoever. + if fileSize <= merkle.MinimumFragmentSize { + return StatusOK, nil + } + + // Create a new merkle file. If one exists, overwrite. + merkleFile := dataFilePath + merkleCompanionExt + + fileM, err := os.OpenFile(merkleFile, os.O_WRONLY|os.O_CREATE, 0666) // 666 = All uses can read/write + if err != nil { + return StatusErrorCreateTarget, err + } + defer fileM.Close() + + // create the merkle tree and write it to the companion file + fragmentSize := merkle.CalculateFragmentSize(fileSize) + tree, err := merkle.NewMerkleTree(fileSize, fragmentSize, dataFile) + if err != nil { + return StatusErrorCreateMerkle, err + } + + fileM.Write(tree.Export()) + + return StatusOK, nil +} + +// ReadMerkleTree reads the merkle tree from the companion file associated with the hash. +// It is the callers responsibility to first check if a merkle tree file is to be expected (files smaller or equal than the minimum fragment size do not use a merkle tree). +func (wh *Warehouse) ReadMerkleTree(hash []byte, headerOnly bool) (tree *merkle.MerkleTree, status int, err error) { + path, _, status, err := wh.MerkleFileExists(hash) + if status != StatusOK { // file does not exist or invalid hash + return nil, status, err + } + + fileM, err := os.OpenFile(path, os.O_RDONLY, 0) + if err != nil { + return nil, StatusErrorOpenFile, err + } + defer fileM.Close() + + if headerOnly { + data := make([]byte, merkle.MerkleTreeFileHeaderSize) + if _, err = io.ReadFull(fileM, data); err != nil { + return nil, StatusErrorReadFile, err + } + + if tree = merkle.ReadMerkleTreeHeader(data); tree == nil { + return nil, StatusErrorMerkleTreeFile, errors.New("invalid merkle tree file header") + } + } else { + data, err := io.ReadAll(fileM) + if err != nil { + return nil, StatusErrorReadFile, err + } + + if tree = merkle.ImportMerkleTree(data); tree == nil { + return nil, StatusErrorMerkleTreeFile, errors.New("invalid merkle tree file header") + } + } + + return tree, StatusOK, nil +} diff --git a/warehouse/Store.go b/warehouse/Store.go index 0f79bce..73cd83e 100644 --- a/warehouse/Store.go +++ b/warehouse/Store.go @@ -1,256 +1,256 @@ -/* -File Name: Store.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package warehouse - -import ( - "io" - "os" - "path/filepath" - "strings" - "time" - - "github.com/PeernetOfficial/core/merkle" - "lukechampine.com/blake3" -) - -const ( - StatusOK = 0 // Success. - StatusErrorCreateTempFile = 1 // Error creating a temporary file. - StatusErrorWriteTempFile = 2 // Error writing temporary file. - StatusErrorCloseTempFile = 3 // Error closing temporary file. - StatusErrorRenameTempFile = 4 // Error renaming temporary file. - StatusErrorCreatePath = 5 // Error creating path for target file in warehouse. - StatusErrorOpenFile = 7 // Error opening file. - StatusInvalidHash = 8 // Invalid hash. - StatusFileNotFound = 9 // File not found. - StatusErrorDeleteFile = 10 // Error deleting file. - StatusErrorReadFile = 11 // Error reading file. - StatusErrorSeekFile = 12 // Error seeking to position in file. - StatusErrorTargetExists = 13 // Target file already exists. - StatusErrorCreateTarget = 14 // Error creating target file. - StatusErrorCreateMerkle = 15 // Error creating merkle tree. - StatusErrorMerkleTreeFile = 16 // Invalid merkle tree companion file. -) - -// CreateFile creates a new file in the warehouse -// If fileSize is provided, creating the merkle tree is significantly faster as it will be created on the fly. If the file size is unknown, set the size to 0. -func (wh *Warehouse) CreateFile(data io.Reader, fileSize uint64) (hash []byte, status int, err error) { - // create a temporary file to hold the body content - tmpFile, err := wh.tempFile() - if err != nil { - return nil, StatusErrorCreateTempFile, err - } - - tmpFileName := tmpFile.Name() - - // create merkle tree in parallel if the file size is known (which means the fragment size can be calculated) - if fileSize > 0 { - // TODO - } - - // create the hash-writer - hashWriter := blake3.New(hashSize, nil) - - // the multi-writer writes to the temp-file and the hash simultaneously - mw := io.MultiWriter(tmpFile, hashWriter) - - // copy into the multiwriter - if _, err = io.Copy(mw, data); err != nil { - tmpFile.Close() - os.Remove(tmpFileName) - return nil, StatusErrorWriteTempFile, err - } - - if err := tmpFile.Close(); err != nil { - os.Remove(tmpFileName) - return nil, StatusErrorCloseTempFile, err - } - - hash = hashWriter.Sum(nil) - - // Check if the file exists - if _, _, status, _ := wh.FileExists(hash); status == StatusOK { - // file exists already, temp file not needed - os.Remove(tmpFileName) - - // return success - return hash, StatusOK, nil - } - - // Destination - pathFull, err := wh.createFilePath(hash) - if err != nil { - os.Remove(tmpFileName) - return nil, StatusErrorCreatePath, err - } - - // first check if the file is already stored. if not rename the temp file to the final one - if _, err := os.Stat(pathFull); err == nil { - // file exists already, temp file not needed - os.Remove(tmpFileName) - } else { - // rename temp file to final one with proper path - if err := os.Rename(tmpFileName, pathFull); err != nil { - os.Remove(tmpFileName) - - // A race condition may exist where the file exists here. If it does, continue successfully. - if _, err = os.Stat(pathFull); err != nil { - return nil, StatusErrorRenameTempFile, err - } - } - - // create the merkle tree companion file - if fileSize == 0 || fileSize > merkle.MinimumFragmentSize { - if status, err = wh.createMerkleCompanionFile(pathFull); status != StatusOK { - return hash, status, err - } - } - } - - return hash, StatusOK, nil -} - -// CreateFileFromPath creates a file from an existing file path. -// Warning: An attacker could supply any local file using this function, put them into storage and read them! No input path verification or limitation is done. -func (wh *Warehouse) CreateFileFromPath(file string) (hash []byte, status int, err error) { - fileHandle, err := os.Open(file) - if err != nil && os.IsNotExist(err) { - return nil, StatusFileNotFound, err - } else if err != nil { - // cannot open file - return nil, StatusErrorOpenFile, err - } - defer fileHandle.Close() - - var fileSize uint64 - if stat, err := fileHandle.Stat(); err == nil { - fileSize = uint64(stat.Size()) - } - - // create the file using the opened file - return wh.CreateFile(fileHandle, fileSize) -} - -// ReadFile reads a file from the warehouse and outputs it to the writer -// Offset is the position in the file to start reading. Limit (0 = not used) defines how many bytes to read starting at the offset. -// Return status codes: StatusInvalidHash, StatusFileNotFound, StatusErrorOpenFile, StatusErrorSeekFile, StatusErrorReadFile, StatusOK -func (wh *Warehouse) ReadFile(hash []byte, offset, limit int64, writer io.Writer) (status int, bytesRead int64, err error) { - // validate the hash and build the path - // 17.01.2022: This code previously used wh.FileExists which is not performant when used frequently. It is faster to instead catch the file-not-exist error on os.Open. - hashA, err := ValidateHash(hash) - if err != nil { - return StatusInvalidHash, 0, err - } - - a, b := buildPath(wh.Directory, hashA) - path := filepath.Join(a, b) - - // read the file from disk - var reader io.ReadSeeker - retryCount := 0 -retryOpenFile: - - file, err := os.Open(path) - if err != nil && os.IsNotExist(err) { - // Catch the error file not exist here. - return StatusFileNotFound, 0, err - } else if err != nil { - // There may be a race condition when the file is being written: "The process cannot access the file because it is being used by another process." - // Wait up to 3 times for 400ms. - if strings.Contains(err.Error(), "cannot access the file because it is being used by another process") && retryCount < 3 { - retryCount++ - time.Sleep(time.Millisecond * 400) - goto retryOpenFile - } - - return StatusErrorOpenFile, 0, err - } - defer file.Close() - - reader = file - - // seek to offset, if provided - if offset > 0 { - if _, err = reader.Seek(offset, io.SeekStart); err != nil { - return StatusErrorSeekFile, 0, err - } - } - - // read the file and copy it into the output - if limit > 0 { - bytesRead, err = io.Copy(writer, io.LimitReader(reader, limit)) - } else { - bytesRead, err = io.Copy(writer, reader) - } - - // do not consider EOF an error if all bytes were read - if err != nil { - return StatusErrorReadFile, bytesRead, err - } - - return StatusOK, bytesRead, nil -} - -// DeleteFile deletes a file from the warehouse -func (wh *Warehouse) DeleteFile(hash []byte) (status int, err error) { - path, _, status, err := wh.FileExists(hash) - if status != StatusOK { - return status, err - } - - if err := os.Remove(path); err != nil { - return StatusErrorDeleteFile, err - } - - return StatusOK, nil -} - -// FileExists checks if the file exists. It returns StatusInvalidHash, StatusFileNotFound, or StatusOK. -func (wh *Warehouse) FileExists(hash []byte) (path string, fileSize uint64, status int, err error) { - hashA, err := ValidateHash(hash) - if err != nil { - return "", 0, StatusInvalidHash, err - } - - a, b := buildPath(wh.Directory, hashA) - path = filepath.Join(a, b) - - if fileInfo, err := os.Stat(path); err == nil { - // file exists - return path, uint64(fileInfo.Size()), StatusOK, nil - } - - return "", 0, StatusFileNotFound, os.ErrNotExist -} - -// DeleteWarehouse deletes all files in the warehouse -func (wh *Warehouse) DeleteWarehouse() (err error) { - return wh.IterateFiles(func(Hash []byte, Size int64) (Continue bool) { - wh.DeleteFile(Hash) - - return true - }) -} - -// ReadFileToDisk reads a file from the warehouse and outputs it to the target file. The function fails with StatusErrorTargetExists if the target file already exists. -// Offset is the position in the file to start reading. Limit (0 = not used) defines how many bytes to read starting at the offset. -// Return status codes: StatusInvalidHash, StatusFileNotFound, StatusErrorTargetExists, StatusErrorCreateTarget, StatusErrorOpenFile, StatusErrorSeekFile, StatusErrorReadFile, StatusOK -func (wh *Warehouse) ReadFileToDisk(hash []byte, offset, limit int64, fileTarget string) (status int, bytesRead int64, err error) { - // check if the target file already exist - if _, err := os.Stat(fileTarget); err == nil { - return StatusErrorTargetExists, 0, nil - } - - // create the target file - fileT, err := os.OpenFile(fileTarget, os.O_WRONLY|os.O_CREATE, 0666) // 666 = All uses can read/write - if err != nil { - return StatusErrorCreateTarget, 0, err - } - defer fileT.Close() - - return wh.ReadFile(hash, offset, limit, fileT) -} +/* +File Name: Store.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package warehouse + +import ( + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/PeernetOfficial/core/merkle" + "lukechampine.com/blake3" +) + +const ( + StatusOK = 0 // Success. + StatusErrorCreateTempFile = 1 // Error creating a temporary file. + StatusErrorWriteTempFile = 2 // Error writing temporary file. + StatusErrorCloseTempFile = 3 // Error closing temporary file. + StatusErrorRenameTempFile = 4 // Error renaming temporary file. + StatusErrorCreatePath = 5 // Error creating path for target file in warehouse. + StatusErrorOpenFile = 7 // Error opening file. + StatusInvalidHash = 8 // Invalid hash. + StatusFileNotFound = 9 // File not found. + StatusErrorDeleteFile = 10 // Error deleting file. + StatusErrorReadFile = 11 // Error reading file. + StatusErrorSeekFile = 12 // Error seeking to position in file. + StatusErrorTargetExists = 13 // Target file already exists. + StatusErrorCreateTarget = 14 // Error creating target file. + StatusErrorCreateMerkle = 15 // Error creating merkle tree. + StatusErrorMerkleTreeFile = 16 // Invalid merkle tree companion file. +) + +// CreateFile creates a new file in the warehouse +// If fileSize is provided, creating the merkle tree is significantly faster as it will be created on the fly. If the file size is unknown, set the size to 0. +func (wh *Warehouse) CreateFile(data io.Reader, fileSize uint64) (hash []byte, status int, err error) { + // create a temporary file to hold the body content + tmpFile, err := wh.tempFile() + if err != nil { + return nil, StatusErrorCreateTempFile, err + } + + tmpFileName := tmpFile.Name() + + // create merkle tree in parallel if the file size is known (which means the fragment size can be calculated) + if fileSize > 0 { + // TODO + } + + // create the hash-writer + hashWriter := blake3.New(hashSize, nil) + + // the multi-writer writes to the temp-file and the hash simultaneously + mw := io.MultiWriter(tmpFile, hashWriter) + + // copy into the multiwriter + if _, err = io.Copy(mw, data); err != nil { + tmpFile.Close() + os.Remove(tmpFileName) + return nil, StatusErrorWriteTempFile, err + } + + if err := tmpFile.Close(); err != nil { + os.Remove(tmpFileName) + return nil, StatusErrorCloseTempFile, err + } + + hash = hashWriter.Sum(nil) + + // Check if the file exists + if _, _, status, _ := wh.FileExists(hash); status == StatusOK { + // file exists already, temp file not needed + os.Remove(tmpFileName) + + // return success + return hash, StatusOK, nil + } + + // Destination + pathFull, err := wh.createFilePath(hash) + if err != nil { + os.Remove(tmpFileName) + return nil, StatusErrorCreatePath, err + } + + // first check if the file is already stored. if not rename the temp file to the final one + if _, err := os.Stat(pathFull); err == nil { + // file exists already, temp file not needed + os.Remove(tmpFileName) + } else { + // rename temp file to final one with proper path + if err := os.Rename(tmpFileName, pathFull); err != nil { + os.Remove(tmpFileName) + + // A race condition may exist where the file exists here. If it does, continue successfully. + if _, err = os.Stat(pathFull); err != nil { + return nil, StatusErrorRenameTempFile, err + } + } + + // create the merkle tree companion file + if fileSize == 0 || fileSize > merkle.MinimumFragmentSize { + if status, err = wh.createMerkleCompanionFile(pathFull); status != StatusOK { + return hash, status, err + } + } + } + + return hash, StatusOK, nil +} + +// CreateFileFromPath creates a file from an existing file path. +// Warning: An attacker could supply any local file using this function, put them into storage and read them! No input path verification or limitation is done. +func (wh *Warehouse) CreateFileFromPath(file string) (hash []byte, status int, err error) { + fileHandle, err := os.Open(file) + if err != nil && os.IsNotExist(err) { + return nil, StatusFileNotFound, err + } else if err != nil { + // cannot open file + return nil, StatusErrorOpenFile, err + } + defer fileHandle.Close() + + var fileSize uint64 + if stat, err := fileHandle.Stat(); err == nil { + fileSize = uint64(stat.Size()) + } + + // create the file using the opened file + return wh.CreateFile(fileHandle, fileSize) +} + +// ReadFile reads a file from the warehouse and outputs it to the writer +// Offset is the position in the file to start reading. Limit (0 = not used) defines how many bytes to read starting at the offset. +// Return status codes: StatusInvalidHash, StatusFileNotFound, StatusErrorOpenFile, StatusErrorSeekFile, StatusErrorReadFile, StatusOK +func (wh *Warehouse) ReadFile(hash []byte, offset, limit int64, writer io.Writer) (status int, bytesRead int64, err error) { + // validate the hash and build the path + // 17.01.2022: This code previously used wh.FileExists which is not performant when used frequently. It is faster to instead catch the file-not-exist error on os.Open. + hashA, err := ValidateHash(hash) + if err != nil { + return StatusInvalidHash, 0, err + } + + a, b := buildPath(wh.Directory, hashA) + path := filepath.Join(a, b) + + // read the file from disk + var reader io.ReadSeeker + retryCount := 0 +retryOpenFile: + + file, err := os.Open(path) + if err != nil && os.IsNotExist(err) { + // Catch the error file not exist here. + return StatusFileNotFound, 0, err + } else if err != nil { + // There may be a race condition when the file is being written: "The process cannot access the file because it is being used by another process." + // Wait up to 3 times for 400ms. + if strings.Contains(err.Error(), "cannot access the file because it is being used by another process") && retryCount < 3 { + retryCount++ + time.Sleep(time.Millisecond * 400) + goto retryOpenFile + } + + return StatusErrorOpenFile, 0, err + } + defer file.Close() + + reader = file + + // seek to offset, if provided + if offset > 0 { + if _, err = reader.Seek(offset, io.SeekStart); err != nil { + return StatusErrorSeekFile, 0, err + } + } + + // read the file and copy it into the output + if limit > 0 { + bytesRead, err = io.Copy(writer, io.LimitReader(reader, limit)) + } else { + bytesRead, err = io.Copy(writer, reader) + } + + // do not consider EOF an error if all bytes were read + if err != nil { + return StatusErrorReadFile, bytesRead, err + } + + return StatusOK, bytesRead, nil +} + +// DeleteFile deletes a file from the warehouse +func (wh *Warehouse) DeleteFile(hash []byte) (status int, err error) { + path, _, status, err := wh.FileExists(hash) + if status != StatusOK { + return status, err + } + + if err := os.Remove(path); err != nil { + return StatusErrorDeleteFile, err + } + + return StatusOK, nil +} + +// FileExists checks if the file exists. It returns StatusInvalidHash, StatusFileNotFound, or StatusOK. +func (wh *Warehouse) FileExists(hash []byte) (path string, fileSize uint64, status int, err error) { + hashA, err := ValidateHash(hash) + if err != nil { + return "", 0, StatusInvalidHash, err + } + + a, b := buildPath(wh.Directory, hashA) + path = filepath.Join(a, b) + + if fileInfo, err := os.Stat(path); err == nil { + // file exists + return path, uint64(fileInfo.Size()), StatusOK, nil + } + + return "", 0, StatusFileNotFound, os.ErrNotExist +} + +// DeleteWarehouse deletes all files in the warehouse +func (wh *Warehouse) DeleteWarehouse() (err error) { + return wh.IterateFiles(func(Hash []byte, Size int64) (Continue bool) { + wh.DeleteFile(Hash) + + return true + }) +} + +// ReadFileToDisk reads a file from the warehouse and outputs it to the target file. The function fails with StatusErrorTargetExists if the target file already exists. +// Offset is the position in the file to start reading. Limit (0 = not used) defines how many bytes to read starting at the offset. +// Return status codes: StatusInvalidHash, StatusFileNotFound, StatusErrorTargetExists, StatusErrorCreateTarget, StatusErrorOpenFile, StatusErrorSeekFile, StatusErrorReadFile, StatusOK +func (wh *Warehouse) ReadFileToDisk(hash []byte, offset, limit int64, fileTarget string) (status int, bytesRead int64, err error) { + // check if the target file already exist + if _, err := os.Stat(fileTarget); err == nil { + return StatusErrorTargetExists, 0, nil + } + + // create the target file + fileT, err := os.OpenFile(fileTarget, os.O_WRONLY|os.O_CREATE, 0666) // 666 = All uses can read/write + if err != nil { + return StatusErrorCreateTarget, 0, err + } + defer fileT.Close() + + return wh.ReadFile(hash, offset, limit, fileT) +} diff --git a/warehouse/Warehouse.go b/warehouse/Warehouse.go index 6020b5d..a0479a6 100644 --- a/warehouse/Warehouse.go +++ b/warehouse/Warehouse.go @@ -1,147 +1,147 @@ -/* -File Name: Warehouse.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package warehouse - -import ( - "encoding/hex" - "io/ioutil" - "os" - "path/filepath" -) - -// Blake3 hash size = 32 bytes. -const ( - hashSize = 256 / 8 -) - -// Warehouse represents a folder on disk. -type Warehouse struct { - Directory string // The main directory for the files - Temp string // Temporary folder -} - -// Init initializes the warehouse -func Init(Directory string) (wh *Warehouse, err error) { - // The temp folder will always be a sub-folder named "_Temp" - wh = &Warehouse{Directory: Directory, Temp: filepath.Join(Directory, "_Temp")} - - if err = createDirectory(wh.Directory); err != nil { - return nil, err - } - if err = createDirectory(wh.Temp); err != nil { - return nil, err - } - - return -} - -// ---- hash functions ---- - -func ValidateHash(hash []byte) (hashA string, err error) { - if len(hash) != hashSize { - return "", os.ErrInvalid - } - return hex.EncodeToString(hash), nil -} - -// ---- path ---- - -func createDirectory(path string) (err error) { - if _, err = os.Stat(path); err != nil && os.IsNotExist(err) { - err = os.MkdirAll(path, os.ModePerm) - } - return err -} - -// buildPath returns the full directory and the filename for the hash -func buildPath(storagePath, hash string) (directory string, filename string) { - part1 := hash[:4] - part2 := hash[4:8] - filename = hash[8:] - - newPath := filepath.Join(storagePath, part1, part2) - - return newPath, filename -} - -// tempFile creates a temporary file in the Warehouse. Do not forget to delete. -func (wh *Warehouse) tempFile() (file *os.File, err error) { - file, err = ioutil.TempFile(wh.Temp, "wh") - - return -} - -// createFilePath creates the file path for the specified hash and returns the full file path -func (wh *Warehouse) createFilePath(hash []byte) (pathFull string, err error) { - path, filename := buildPath(wh.Directory, hex.EncodeToString(hash)) - return filepath.Join(path, filename), createDirectory(path) -} - -// IterateFiles iterates through all the files and calls the callback -func (wh *Warehouse) IterateFiles(Callback func(Hash []byte, Size int64) (Continue bool)) (err error) { - // list all directories in the local Storage folder. We have to walk 2 levels down to see the actual files. - files, err := ioutil.ReadDir(wh.Directory) - if err != nil { - return err - } - - for _, file := range files { - name1 := file.Name() - _, err = hex.DecodeString(name1) - - // we are only looking for directories. Name has to be "XXXX" hex chars only. - if !file.IsDir() || len(name1) != 4 || err != nil { - continue - } - - // iterate through the next level - files1, err := ioutil.ReadDir(filepath.Join(wh.Directory, name1)) - if err != nil { - return err - } - - for _, file2 := range files1 { - name2 := file2.Name() - _, err = hex.DecodeString(name2) - - // we are only looking for directories. Name has to be "XXXX" hex chars only. - if !file2.IsDir() || len(name2) != 4 || err != nil { - continue - } - - // iterate through the next level - files2, err := ioutil.ReadDir(filepath.Join(wh.Directory, name1, name2)) - if err != nil { - return err - } - - for _, file3 := range files2 { - name3 := file3.Name() - _, err = hex.DecodeString(name3) - - // finally we are only looking for files - if file3.IsDir() || len(name3) != hashSize*2-8 || err != nil { - continue - } - - size := file3.Size() - - hash, err := hex.DecodeString(name1 + name2 + name3) - if err != nil { - return err - } - - // found the hash! forward - if !Callback(hash, size) { - return nil - } - } - } - } - - return nil -} +/* +File Name: Warehouse.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package warehouse + +import ( + "encoding/hex" + "io/ioutil" + "os" + "path/filepath" +) + +// Blake3 hash size = 32 bytes. +const ( + hashSize = 256 / 8 +) + +// Warehouse represents a folder on disk. +type Warehouse struct { + Directory string // The main directory for the files + Temp string // Temporary folder +} + +// Init initializes the warehouse +func Init(Directory string) (wh *Warehouse, err error) { + // The temp folder will always be a sub-folder named "_Temp" + wh = &Warehouse{Directory: Directory, Temp: filepath.Join(Directory, "_Temp")} + + if err = createDirectory(wh.Directory); err != nil { + return nil, err + } + if err = createDirectory(wh.Temp); err != nil { + return nil, err + } + + return +} + +// ---- hash functions ---- + +func ValidateHash(hash []byte) (hashA string, err error) { + if len(hash) != hashSize { + return "", os.ErrInvalid + } + return hex.EncodeToString(hash), nil +} + +// ---- path ---- + +func createDirectory(path string) (err error) { + if _, err = os.Stat(path); err != nil && os.IsNotExist(err) { + err = os.MkdirAll(path, os.ModePerm) + } + return err +} + +// buildPath returns the full directory and the filename for the hash +func buildPath(storagePath, hash string) (directory string, filename string) { + part1 := hash[:4] + part2 := hash[4:8] + filename = hash[8:] + + newPath := filepath.Join(storagePath, part1, part2) + + return newPath, filename +} + +// tempFile creates a temporary file in the Warehouse. Do not forget to delete. +func (wh *Warehouse) tempFile() (file *os.File, err error) { + file, err = ioutil.TempFile(wh.Temp, "wh") + + return +} + +// createFilePath creates the file path for the specified hash and returns the full file path +func (wh *Warehouse) createFilePath(hash []byte) (pathFull string, err error) { + path, filename := buildPath(wh.Directory, hex.EncodeToString(hash)) + return filepath.Join(path, filename), createDirectory(path) +} + +// IterateFiles iterates through all the files and calls the callback +func (wh *Warehouse) IterateFiles(Callback func(Hash []byte, Size int64) (Continue bool)) (err error) { + // list all directories in the local Storage folder. We have to walk 2 levels down to see the actual files. + files, err := ioutil.ReadDir(wh.Directory) + if err != nil { + return err + } + + for _, file := range files { + name1 := file.Name() + _, err = hex.DecodeString(name1) + + // we are only looking for directories. Name has to be "XXXX" hex chars only. + if !file.IsDir() || len(name1) != 4 || err != nil { + continue + } + + // iterate through the next level + files1, err := ioutil.ReadDir(filepath.Join(wh.Directory, name1)) + if err != nil { + return err + } + + for _, file2 := range files1 { + name2 := file2.Name() + _, err = hex.DecodeString(name2) + + // we are only looking for directories. Name has to be "XXXX" hex chars only. + if !file2.IsDir() || len(name2) != 4 || err != nil { + continue + } + + // iterate through the next level + files2, err := ioutil.ReadDir(filepath.Join(wh.Directory, name1, name2)) + if err != nil { + return err + } + + for _, file3 := range files2 { + name3 := file3.Name() + _, err = hex.DecodeString(name3) + + // finally we are only looking for files + if file3.IsDir() || len(name3) != hashSize*2-8 || err != nil { + continue + } + + size := file3.Size() + + hash, err := hex.DecodeString(name1 + name2 + name3) + if err != nil { + return err + } + + // found the hash! forward + if !Callback(hash, size) { + return nil + } + } + } + } + + return nil +} diff --git a/webapi/API.go b/webapi/API.go index fd70180..ca215fc 100644 --- a/webapi/API.go +++ b/webapi/API.go @@ -1,198 +1,198 @@ -/* -File Name: API.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "crypto/tls" - "encoding/json" - "errors" - "net/http" - "sync" - "time" - - "github.com/IncSW/geoip2" - "github.com/PeernetOfficial/core" - "github.com/google/uuid" - "github.com/gorilla/mux" - "github.com/gorilla/websocket" -) - -type WebapiInstance struct { - backend *core.Backend - geoipCityReader *geoip2.CityReader - - // Router can be used to register additional API functions - Router *mux.Router - AllowKeyInParam []string // List of paths that accept the API key as &k= parameter - - // search jobs - allJobs map[uuid.UUID]*SearchJob - allJobsMutex sync.RWMutex - - // download info - downloads map[uuid.UUID]*downloadInfo - downloadsMutex sync.RWMutex -} - -// WSUpgrader is used for websocket functionality. It allows all requests. -var WSUpgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, - CheckOrigin: func(r *http.Request) bool { - // allow all connections by default - return true - }, -} - -// Start starts the API. ListenAddresses is a list of IP:Ports. -// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. -// The API key may be uuid.Nil to disable it although this is not recommended for security reasons. -func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKey string, TimeoutRead, TimeoutWrite time.Duration, APIKey uuid.UUID) (api *WebapiInstance) { - if len(ListenAddresses) == 0 { - return nil - } - - api = &WebapiInstance{ - backend: Backend, - Router: mux.NewRouter(), - AllowKeyInParam: []string{"/file/read", "/file/view"}, - allJobs: make(map[uuid.UUID]*SearchJob), - downloads: make(map[uuid.UUID]*downloadInfo), - } - - if APIKey != uuid.Nil { - api.Router.Use(api.authenticateMiddleware(APIKey)) - } - - api.Router.HandleFunc("/test", apiTest).Methods("GET") - api.Router.HandleFunc("/status", api.apiStatus).Methods("GET") - api.Router.HandleFunc("/status/peers", api.apiStatusPeers).Methods("GET") - api.Router.HandleFunc("/account/info", api.apiAccountInfo).Methods("GET") - api.Router.HandleFunc("/account/delete", api.apiAccountDelete).Methods("GET") - api.Router.HandleFunc("/blockchain/header", api.apiBlockchainHeaderFunc).Methods("GET") - api.Router.HandleFunc("/blockchain/append", api.apiBlockchainAppend).Methods("POST") - api.Router.HandleFunc("/blockchain/read", api.apiBlockchainRead).Methods("GET") - api.Router.HandleFunc("/blockchain/file/add", api.apiBlockchainFileAdd).Methods("POST") - api.Router.HandleFunc("/blockchain/file/list", api.apiBlockchainFileList).Methods("GET") - api.Router.HandleFunc("/blockchain/file/delete", api.apiBlockchainFileDelete).Methods("POST") - api.Router.HandleFunc("/blockchain/file/update", api.apiBlockchainFileUpdate).Methods("POST") - api.Router.HandleFunc("/profile/list", api.apiProfileList).Methods("GET") - api.Router.HandleFunc("/profile/read", api.apiProfileRead).Methods("GET") - api.Router.HandleFunc("/profile/write", api.apiProfileWrite).Methods("POST") - api.Router.HandleFunc("/profile/delete", api.apiProfileDelete).Methods("POST") - api.Router.HandleFunc("/search", api.apiSearch).Methods("POST") - api.Router.HandleFunc("/search/result", api.apiSearchResult).Methods("GET") - api.Router.HandleFunc("/search/result/ws", api.apiSearchResultStream).Methods("GET") - api.Router.HandleFunc("/search/statistic", api.apiSearchStatistic).Methods("GET") - api.Router.HandleFunc("/search/terminate", api.apiSearchTerminate).Methods("GET") - api.Router.HandleFunc("/explore", api.apiExplore).Methods("GET") - api.Router.HandleFunc("/explore/node", api.apiExploreNodeID).Methods("GET") - api.Router.HandleFunc("/file/format", api.apiFileFormat).Methods("GET") - api.Router.HandleFunc("/download/start", api.apiDownloadStart).Methods("GET") - api.Router.HandleFunc("/download/status", api.apiDownloadStatus).Methods("GET") - api.Router.HandleFunc("/download/action", api.apiDownloadAction).Methods("GET") - api.Router.HandleFunc("/warehouse/create", api.apiWarehouseCreateFile).Methods("POST") - api.Router.HandleFunc("/warehouse/create/path", api.apiWarehouseCreateFilePath).Methods("GET") - api.Router.HandleFunc("/warehouse/read", api.apiWarehouseReadFile).Methods("GET") - api.Router.HandleFunc("/warehouse/read/path", api.apiWarehouseReadFilePath).Methods("GET") - api.Router.HandleFunc("/warehouse/delete", api.apiWarehouseDeleteFile).Methods("GET") - api.Router.HandleFunc("/file/read", api.apiFileRead).Methods("GET") - api.Router.HandleFunc("/file/view", api.apiFileView).Methods("GET") - - for _, listen := range ListenAddresses { - go startWebAPI(Backend, listen, UseSSL, CertificateFile, CertificateKey, api.Router, "API", TimeoutRead, TimeoutWrite) - } - - return api -} - -// startWebAPI starts a web-server with given parameters and logs the status. If may block forever and only returns if there is an error. -// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. -func startWebAPI(Backend *core.Backend, WebListen string, UseSSL bool, CertificateFile, CertificateKey string, Handler http.Handler, Info string, ReadTimeout, WriteTimeout time.Duration) { - Backend.LogError("startWebAPI", "Start API at '%s'\n", WebListen) - - tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} // for security reasons disable TLS 1.0/1.1 - - server := &http.Server{ - Addr: WebListen, - Handler: Handler, - ReadTimeout: ReadTimeout, // ReadTimeout is the maximum duration for reading the entire request, including the body. - WriteTimeout: WriteTimeout, // WriteTimeout is the maximum duration before timing out writes of the response. This includes processing time and is therefore the max time any HTTP function may take. - //IdleTimeout: IdleTimeout, // IdleTimeout is the maximum amount of time to wait for the next request when keep-alives are enabled. - TLSConfig: tlsConfig, - } - - if UseSSL { - // HTTPS - if err := server.ListenAndServeTLS(CertificateFile, CertificateKey); err != nil { - Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) - } - } else { - // HTTP - if err := server.ListenAndServe(); err != nil { - Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) - } - } -} - -// EncodeJSON encodes the data as JSON -func EncodeJSON(Backend *core.Backend, w http.ResponseWriter, r *http.Request, data interface{}) (err error) { - w.Header().Set("Content-Type", "application/json") - - err = json.NewEncoder(w).Encode(data) - if err != nil { - Backend.LogError("EncodeJSON", "Error writing data for route '%s': %v\n", r.URL.Path, err) - } - - return err -} - -// DecodeJSON decodes input JSON data server side sent either via GET or POST. It does not limit the maximum amount to read. -// In case of error it will automatically send an error to the client. -func DecodeJSON(w http.ResponseWriter, r *http.Request, data interface{}) (err error) { - if r.Body == nil { - http.Error(w, "", http.StatusBadRequest) - return errors.New("no data") - } - - err = json.NewDecoder(r.Body).Decode(data) - if err != nil { - http.Error(w, "", http.StatusBadRequest) - return err - } - - return nil -} - -// authenticateMiddleware returns a middleware function to be used with mux.Router.Use(). It handles all authentication functionality. -func (api *WebapiInstance) authenticateMiddleware(APIKey uuid.UUID) func(http.Handler) http.Handler { - return (func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - keyID, err := uuid.Parse(r.Header.Get("x-api-key")) - if err != nil { // special case for some paths - for _, exceptPath := range api.AllowKeyInParam { - if exceptPath == r.URL.Path { - r.ParseForm() - keyID, err = uuid.Parse(r.Form.Get("k")) - break - } - } - } - if err != nil { // Invalid key format - w.WriteHeader(http.StatusUnauthorized) - return - } - - if keyID != APIKey { - w.WriteHeader(http.StatusUnauthorized) - return - } - - next.ServeHTTP(w, r) - }) - }) -} +/* +File Name: API.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "crypto/tls" + "encoding/json" + "errors" + "net/http" + "sync" + "time" + + "github.com/IncSW/geoip2" + "github.com/PeernetOfficial/core" + "github.com/google/uuid" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" +) + +type WebapiInstance struct { + backend *core.Backend + geoipCityReader *geoip2.CityReader + + // Router can be used to register additional API functions + Router *mux.Router + AllowKeyInParam []string // List of paths that accept the API key as &k= parameter + + // search jobs + allJobs map[uuid.UUID]*SearchJob + allJobsMutex sync.RWMutex + + // download info + downloads map[uuid.UUID]*downloadInfo + downloadsMutex sync.RWMutex +} + +// WSUpgrader is used for websocket functionality. It allows all requests. +var WSUpgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { + // allow all connections by default + return true + }, +} + +// Start starts the API. ListenAddresses is a list of IP:Ports. +// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. +// The API key may be uuid.Nil to disable it although this is not recommended for security reasons. +func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKey string, TimeoutRead, TimeoutWrite time.Duration, APIKey uuid.UUID) (api *WebapiInstance) { + if len(ListenAddresses) == 0 { + return nil + } + + api = &WebapiInstance{ + backend: Backend, + Router: mux.NewRouter(), + AllowKeyInParam: []string{"/file/read", "/file/view"}, + allJobs: make(map[uuid.UUID]*SearchJob), + downloads: make(map[uuid.UUID]*downloadInfo), + } + + if APIKey != uuid.Nil { + api.Router.Use(api.authenticateMiddleware(APIKey)) + } + + api.Router.HandleFunc("/test", apiTest).Methods("GET") + api.Router.HandleFunc("/status", api.apiStatus).Methods("GET") + api.Router.HandleFunc("/status/peers", api.apiStatusPeers).Methods("GET") + api.Router.HandleFunc("/account/info", api.apiAccountInfo).Methods("GET") + api.Router.HandleFunc("/account/delete", api.apiAccountDelete).Methods("GET") + api.Router.HandleFunc("/blockchain/header", api.apiBlockchainHeaderFunc).Methods("GET") + api.Router.HandleFunc("/blockchain/append", api.apiBlockchainAppend).Methods("POST") + api.Router.HandleFunc("/blockchain/read", api.apiBlockchainRead).Methods("GET") + api.Router.HandleFunc("/blockchain/file/add", api.apiBlockchainFileAdd).Methods("POST") + api.Router.HandleFunc("/blockchain/file/list", api.apiBlockchainFileList).Methods("GET") + api.Router.HandleFunc("/blockchain/file/delete", api.apiBlockchainFileDelete).Methods("POST") + api.Router.HandleFunc("/blockchain/file/update", api.apiBlockchainFileUpdate).Methods("POST") + api.Router.HandleFunc("/profile/list", api.apiProfileList).Methods("GET") + api.Router.HandleFunc("/profile/read", api.apiProfileRead).Methods("GET") + api.Router.HandleFunc("/profile/write", api.apiProfileWrite).Methods("POST") + api.Router.HandleFunc("/profile/delete", api.apiProfileDelete).Methods("POST") + api.Router.HandleFunc("/search", api.apiSearch).Methods("POST") + api.Router.HandleFunc("/search/result", api.apiSearchResult).Methods("GET") + api.Router.HandleFunc("/search/result/ws", api.apiSearchResultStream).Methods("GET") + api.Router.HandleFunc("/search/statistic", api.apiSearchStatistic).Methods("GET") + api.Router.HandleFunc("/search/terminate", api.apiSearchTerminate).Methods("GET") + api.Router.HandleFunc("/explore", api.apiExplore).Methods("GET") + api.Router.HandleFunc("/explore/node", api.apiExploreNodeID).Methods("GET") + api.Router.HandleFunc("/file/format", api.apiFileFormat).Methods("GET") + api.Router.HandleFunc("/download/start", api.apiDownloadStart).Methods("GET") + api.Router.HandleFunc("/download/status", api.apiDownloadStatus).Methods("GET") + api.Router.HandleFunc("/download/action", api.apiDownloadAction).Methods("GET") + api.Router.HandleFunc("/warehouse/create", api.apiWarehouseCreateFile).Methods("POST") + api.Router.HandleFunc("/warehouse/create/path", api.apiWarehouseCreateFilePath).Methods("GET") + api.Router.HandleFunc("/warehouse/read", api.apiWarehouseReadFile).Methods("GET") + api.Router.HandleFunc("/warehouse/read/path", api.apiWarehouseReadFilePath).Methods("GET") + api.Router.HandleFunc("/warehouse/delete", api.apiWarehouseDeleteFile).Methods("GET") + api.Router.HandleFunc("/file/read", api.apiFileRead).Methods("GET") + api.Router.HandleFunc("/file/view", api.apiFileView).Methods("GET") + + for _, listen := range ListenAddresses { + go startWebAPI(Backend, listen, UseSSL, CertificateFile, CertificateKey, api.Router, "API", TimeoutRead, TimeoutWrite) + } + + return api +} + +// startWebAPI starts a web-server with given parameters and logs the status. If may block forever and only returns if there is an error. +// The certificate file and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout. +func startWebAPI(Backend *core.Backend, WebListen string, UseSSL bool, CertificateFile, CertificateKey string, Handler http.Handler, Info string, ReadTimeout, WriteTimeout time.Duration) { + Backend.LogError("startWebAPI", "Start API at '%s'\n", WebListen) + + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} // for security reasons disable TLS 1.0/1.1 + + server := &http.Server{ + Addr: WebListen, + Handler: Handler, + ReadTimeout: ReadTimeout, // ReadTimeout is the maximum duration for reading the entire request, including the body. + WriteTimeout: WriteTimeout, // WriteTimeout is the maximum duration before timing out writes of the response. This includes processing time and is therefore the max time any HTTP function may take. + //IdleTimeout: IdleTimeout, // IdleTimeout is the maximum amount of time to wait for the next request when keep-alives are enabled. + TLSConfig: tlsConfig, + } + + if UseSSL { + // HTTPS + if err := server.ListenAndServeTLS(CertificateFile, CertificateKey); err != nil { + Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) + } + } else { + // HTTP + if err := server.ListenAndServe(); err != nil { + Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err) + } + } +} + +// EncodeJSON encodes the data as JSON +func EncodeJSON(Backend *core.Backend, w http.ResponseWriter, r *http.Request, data interface{}) (err error) { + w.Header().Set("Content-Type", "application/json") + + err = json.NewEncoder(w).Encode(data) + if err != nil { + Backend.LogError("EncodeJSON", "Error writing data for route '%s': %v\n", r.URL.Path, err) + } + + return err +} + +// DecodeJSON decodes input JSON data server side sent either via GET or POST. It does not limit the maximum amount to read. +// In case of error it will automatically send an error to the client. +func DecodeJSON(w http.ResponseWriter, r *http.Request, data interface{}) (err error) { + if r.Body == nil { + http.Error(w, "", http.StatusBadRequest) + return errors.New("no data") + } + + err = json.NewDecoder(r.Body).Decode(data) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return err + } + + return nil +} + +// authenticateMiddleware returns a middleware function to be used with mux.Router.Use(). It handles all authentication functionality. +func (api *WebapiInstance) authenticateMiddleware(APIKey uuid.UUID) func(http.Handler) http.Handler { + return (func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + keyID, err := uuid.Parse(r.Header.Get("x-api-key")) + if err != nil { // special case for some paths + for _, exceptPath := range api.AllowKeyInParam { + if exceptPath == r.URL.Path { + r.ParseForm() + keyID, err = uuid.Parse(r.Form.Get("k")) + break + } + } + } + if err != nil { // Invalid key format + w.WriteHeader(http.StatusUnauthorized) + return + } + + if keyID != APIKey { + w.WriteHeader(http.StatusUnauthorized) + return + } + + next.ServeHTTP(w, r) + }) + }) +} diff --git a/webapi/Blockchain.go b/webapi/Blockchain.go index 7d7fea0..88e7960 100644 --- a/webapi/Blockchain.go +++ b/webapi/Blockchain.go @@ -1,122 +1,122 @@ -/* -File Name: Blockchain.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "encoding/hex" - "net/http" - "strconv" - - "github.com/PeernetOfficial/core/blockchain" -) - -type apiBlockchainHeader struct { - PeerID string `json:"peerid"` // Peer ID hex encoded. - Version uint64 `json:"version"` // Current version number of the blockchain. - Height uint64 `json:"height"` // Height of the blockchain (number of blocks). If 0, no data exists. -} - -/* -apiBlockchainHeaderFunc returns the current blockchain header information - -Request: GET /blockchain/header -Result: 200 with JSON structure apiResponsePeerSelf -*/ -func (api *WebapiInstance) apiBlockchainHeaderFunc(w http.ResponseWriter, r *http.Request) { - publicKey, height, version := api.Backend.UserBlockchain.Header() - - EncodeJSON(api.Backend, w, r, apiBlockchainHeader{Version: version, Height: height, PeerID: hex.EncodeToString(publicKey.SerializeCompressed())}) -} - -type apiBlockRecordRaw struct { - Type uint8 `json:"type"` // Record Type. See core.RecordTypeX. - Data []byte `json:"data"` // Data according to the type. -} - -// apiBlockchainBlockRaw contains a raw block of the blockchain via API -type apiBlockchainBlockRaw struct { - Records []apiBlockRecordRaw `json:"records"` // Block records in encoded raw format. -} - -type apiBlockchainBlockStatus struct { - Status int `json:"status"` // See blockchain.StatusX. - Height uint64 `json:"height"` // Height of the blockchain (number of blocks). - Version uint64 `json:"version"` // Version of the blockchain. -} - -/* -apiBlockchainAppend appends a block to the blockchain. This is a low-level function for already encoded blocks. -Do not use this function. Adding invalid data to the blockchain may corrupt it which might result in blacklisting by other peers. - -Request: POST /blockchain/append with JSON structure apiBlockchainBlockRaw -Response: 200 with JSON structure apiBlockchainBlockStatus -*/ -func (api *WebapiInstance) apiBlockchainAppend(w http.ResponseWriter, r *http.Request) { - var input apiBlockchainBlockRaw - if err := DecodeJSON(w, r, &input); err != nil { - return - } - - var records []blockchain.BlockRecordRaw - - for _, record := range input.Records { - records = append(records, blockchain.BlockRecordRaw{Type: record.Type, Data: record.Data}) - } - - newHeight, newVersion, status := api.Backend.UserBlockchain.Append(records) - - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) -} - -type apiBlockchainBlock struct { - Status int `json:"status"` // See blockchain.StatusX. - PeerID string `json:"peerid"` // Peer ID hex encoded. - LastBlockHash []byte `json:"lastblockhash"` // Hash of the last block. Blake3. - BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version - Number uint64 `json:"blocknumber"` // Block number - RecordsRaw []apiBlockRecordRaw `json:"recordsraw"` // Records raw. Successfully decoded records are parsed into the below fields. - RecordsDecoded []interface{} `json:"recordsdecoded"` // Records decoded. The encoding for each record depends on its type. -} - -/* -apiBlockchainRead reads a block and returns the decoded information. - -Request: GET /blockchain/read?block=[number] -Result: 200 with JSON structure apiBlockchainBlock -*/ -func (api *WebapiInstance) apiBlockchainRead(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - blockN, err := strconv.Atoi(r.Form.Get("block")) - if err != nil || blockN < 0 { - http.Error(w, "", http.StatusBadRequest) - return - } - - block, status, _ := api.Backend.UserBlockchain.Read(uint64(blockN)) - result := apiBlockchainBlock{Status: status} - - if status == 0 { - for _, record := range block.RecordsRaw { - result.RecordsRaw = append(result.RecordsRaw, apiBlockRecordRaw{Type: record.Type, Data: record.Data}) - } - - result.PeerID = hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed()) - - for _, record := range block.RecordsDecoded { - switch v := record.(type) { - case blockchain.BlockRecordFile: - result.RecordsDecoded = append(result.RecordsDecoded, blockRecordFileToAPI(v)) - - case blockchain.BlockRecordProfile: - result.RecordsDecoded = append(result.RecordsDecoded, blockRecordProfileToAPI(v)) - - } - } - } - - EncodeJSON(api.Backend, w, r, result) -} +/* +File Name: Blockchain.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "encoding/hex" + "net/http" + "strconv" + + "github.com/PeernetOfficial/core/blockchain" +) + +type apiBlockchainHeader struct { + PeerID string `json:"peerid"` // Peer ID hex encoded. + Version uint64 `json:"version"` // Current version number of the blockchain. + Height uint64 `json:"height"` // Height of the blockchain (number of blocks). If 0, no data exists. +} + +/* +apiBlockchainHeaderFunc returns the current blockchain header information + +Request: GET /blockchain/header +Result: 200 with JSON structure apiResponsePeerSelf +*/ +func (api *WebapiInstance) apiBlockchainHeaderFunc(w http.ResponseWriter, r *http.Request) { + publicKey, height, version := api.Backend.UserBlockchain.Header() + + EncodeJSON(api.Backend, w, r, apiBlockchainHeader{Version: version, Height: height, PeerID: hex.EncodeToString(publicKey.SerializeCompressed())}) +} + +type apiBlockRecordRaw struct { + Type uint8 `json:"type"` // Record Type. See core.RecordTypeX. + Data []byte `json:"data"` // Data according to the type. +} + +// apiBlockchainBlockRaw contains a raw block of the blockchain via API +type apiBlockchainBlockRaw struct { + Records []apiBlockRecordRaw `json:"records"` // Block records in encoded raw format. +} + +type apiBlockchainBlockStatus struct { + Status int `json:"status"` // See blockchain.StatusX. + Height uint64 `json:"height"` // Height of the blockchain (number of blocks). + Version uint64 `json:"version"` // Version of the blockchain. +} + +/* +apiBlockchainAppend appends a block to the blockchain. This is a low-level function for already encoded blocks. +Do not use this function. Adding invalid data to the blockchain may corrupt it which might result in blacklisting by other peers. + +Request: POST /blockchain/append with JSON structure apiBlockchainBlockRaw +Response: 200 with JSON structure apiBlockchainBlockStatus +*/ +func (api *WebapiInstance) apiBlockchainAppend(w http.ResponseWriter, r *http.Request) { + var input apiBlockchainBlockRaw + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var records []blockchain.BlockRecordRaw + + for _, record := range input.Records { + records = append(records, blockchain.BlockRecordRaw{Type: record.Type, Data: record.Data}) + } + + newHeight, newVersion, status := api.Backend.UserBlockchain.Append(records) + + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +type apiBlockchainBlock struct { + Status int `json:"status"` // See blockchain.StatusX. + PeerID string `json:"peerid"` // Peer ID hex encoded. + LastBlockHash []byte `json:"lastblockhash"` // Hash of the last block. Blake3. + BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version + Number uint64 `json:"blocknumber"` // Block number + RecordsRaw []apiBlockRecordRaw `json:"recordsraw"` // Records raw. Successfully decoded records are parsed into the below fields. + RecordsDecoded []interface{} `json:"recordsdecoded"` // Records decoded. The encoding for each record depends on its type. +} + +/* +apiBlockchainRead reads a block and returns the decoded information. + +Request: GET /blockchain/read?block=[number] +Result: 200 with JSON structure apiBlockchainBlock +*/ +func (api *WebapiInstance) apiBlockchainRead(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + blockN, err := strconv.Atoi(r.Form.Get("block")) + if err != nil || blockN < 0 { + http.Error(w, "", http.StatusBadRequest) + return + } + + block, status, _ := api.Backend.UserBlockchain.Read(uint64(blockN)) + result := apiBlockchainBlock{Status: status} + + if status == 0 { + for _, record := range block.RecordsRaw { + result.RecordsRaw = append(result.RecordsRaw, apiBlockRecordRaw{Type: record.Type, Data: record.Data}) + } + + result.PeerID = hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed()) + + for _, record := range block.RecordsDecoded { + switch v := record.(type) { + case blockchain.BlockRecordFile: + result.RecordsDecoded = append(result.RecordsDecoded, blockRecordFileToAPI(v)) + + case blockchain.BlockRecordProfile: + result.RecordsDecoded = append(result.RecordsDecoded, blockRecordProfileToAPI(v)) + + } + } + } + + EncodeJSON(api.Backend, w, r, result) +} diff --git a/webapi/Download Transfer.go b/webapi/Download Transfer.go index b3a59d2..be41124 100644 --- a/webapi/Download Transfer.go +++ b/webapi/Download Transfer.go @@ -1,202 +1,202 @@ -/* -File Name: Download Transfer.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner - -Temporary download code to provide dummy results for testing. To be replaced! -*/ - -package webapi - -import ( - "bytes" - "os" - "time" - - "github.com/PeernetOfficial/core/warehouse" -) - -// Starts the download. -func (info *downloadInfo) Start() { - // current user? - if bytes.Equal(info.nodeID, info.backend.SelfNodeID()) { - info.DownloadSelf() - return - } - - for n := 0; n < 3 && info.peer == nil; n++ { - _, info.peer, _ = info.backend.FindNode(info.nodeID, time.Second*5) - - if info.status == DownloadCanceled { - return - } - } - - if info.peer != nil { - info.Download() - } else { - info.status = DownloadCanceled - } -} - -func (info *downloadInfo) Download() { - //fmt.Printf("Download start of %s\n", hex.EncodeToString(info.hash)) - - // try to download the entire file - reader, fileSize, transferSize, err := FileStartReader(info.peer, info.hash, 0, 0, nil) - if reader != nil { - defer reader.Close() - } - if err != nil { - info.status = DownloadCanceled - return - } else if fileSize != transferSize { - info.status = DownloadCanceled - return - } - - info.file.Size = fileSize - info.status = DownloadActive - - // download in a loop - var fileOffset, totalRead uint64 - dataRemaining := fileSize - readSize := uint64(4096) - - for dataRemaining > 0 { - //fmt.Printf("data remaining: downloaded %d from total %d = %d %%\n", totalRead, fileSize, totalRead*100/fileSize) - if dataRemaining < readSize { - readSize = dataRemaining - } - - data := make([]byte, readSize) - n, err := reader.Read(data) - - totalRead += uint64(n) - dataRemaining -= uint64(n) - data = data[:n] - - if err != nil { - info.status = DownloadCanceled - return - } - - info.storeDownloadData(data[:n], fileOffset) - - fileOffset += uint64(n) - } - - //fmt.Printf("data finished: downloaded %d from total %d = %d %%\n", totalRead, fileSize, totalRead*100/fileSize) - - info.Finish() - info.DeleteDefer(time.Hour * 1) // cache the details for 1 hour before removing -} - -// Pause pauses the download. Status is DownloadResponseX. -func (info *downloadInfo) Pause() (status int) { - info.Lock() - defer info.Unlock() - - if info.status != DownloadActive { // The download must be active to be paused. - return DownloadResponseActionInvalid - } - - info.status = DownloadPause - - return DownloadResponseSuccess -} - -// Resume resumes the download. Status is DownloadResponseX. -func (info *downloadInfo) Resume() (status int) { - info.Lock() - defer info.Unlock() - - if info.status != DownloadPause { // The download must be paused to resume. - return DownloadResponseActionInvalid - } - - info.status = DownloadActive - - return DownloadResponseSuccess -} - -// Cancel cancels the download. Status is DownloadResponseX. -func (info *downloadInfo) Cancel() (status int) { - info.Lock() - defer info.Unlock() - - if info.status >= DownloadCanceled { // The download must not be already canceled or finished. - return DownloadResponseActionInvalid - } - - info.status = DownloadCanceled - info.DiskFile.Handle.Close() - - return DownloadResponseSuccess -} - -// Finish marks the download as finished. -func (info *downloadInfo) Finish() (status int) { - info.Lock() - defer info.Unlock() - - if info.status != DownloadActive { // The download must be active. - return DownloadResponseActionInvalid - } - - info.status = DownloadFinished - info.DiskFile.Handle.Close() - - return DownloadResponseSuccess -} - -// initDiskFile creates the target file -func (info *downloadInfo) initDiskFile(path string) (err error) { - info.DiskFile.Name = path - info.DiskFile.Handle, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) // 666 : All uses can read/write - - return err -} - -// storeDownloadData stores downloaded data. It does not change the download status. -func (info *downloadInfo) storeDownloadData(data []byte, offset uint64) (status int) { - info.Lock() - defer info.Unlock() - - if info.status != DownloadActive { // The download must be active. - return DownloadResponseActionInvalid - } - - if _, err := info.DiskFile.Handle.WriteAt(data, int64(offset)); err != nil { - return DownloadResponseFileWrite - } - - info.DiskFile.StoredSize += uint64(len(data)) - - return DownloadResponseSuccess -} - -func (info *downloadInfo) DownloadSelf() { - // Check if the file is available in the local warehouse. - _, fileSize, status, _ := info.backend.UserWarehouse.FileExists(info.hash) - if status != warehouse.StatusOK { - info.status = DownloadCanceled - return - } - - info.file.Size = fileSize - info.status = DownloadActive - - // read the file - status, bytesRead, _ := info.backend.UserWarehouse.ReadFile(info.hash, 0, int64(info.file.Size), info.DiskFile.Handle) - - info.DiskFile.StoredSize = uint64(bytesRead) - - if status != warehouse.StatusOK { - info.status = DownloadCanceled - return - } - - info.Finish() - info.DeleteDefer(time.Hour * 1) // cache the details for 1 hour before removing} -} +/* +File Name: Download Transfer.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner + +Temporary download code to provide dummy results for testing. To be replaced! +*/ + +package webapi + +import ( + "bytes" + "os" + "time" + + "github.com/PeernetOfficial/core/warehouse" +) + +// Starts the download. +func (info *downloadInfo) Start() { + // current user? + if bytes.Equal(info.nodeID, info.backend.SelfNodeID()) { + info.DownloadSelf() + return + } + + for n := 0; n < 3 && info.peer == nil; n++ { + _, info.peer, _ = info.backend.FindNode(info.nodeID, time.Second*5) + + if info.status == DownloadCanceled { + return + } + } + + if info.peer != nil { + info.Download() + } else { + info.status = DownloadCanceled + } +} + +func (info *downloadInfo) Download() { + //fmt.Printf("Download start of %s\n", hex.EncodeToString(info.hash)) + + // try to download the entire file + reader, fileSize, transferSize, err := FileStartReader(info.peer, info.hash, 0, 0, nil) + if reader != nil { + defer reader.Close() + } + if err != nil { + info.status = DownloadCanceled + return + } else if fileSize != transferSize { + info.status = DownloadCanceled + return + } + + info.file.Size = fileSize + info.status = DownloadActive + + // download in a loop + var fileOffset, totalRead uint64 + dataRemaining := fileSize + readSize := uint64(4096) + + for dataRemaining > 0 { + //fmt.Printf("data remaining: downloaded %d from total %d = %d %%\n", totalRead, fileSize, totalRead*100/fileSize) + if dataRemaining < readSize { + readSize = dataRemaining + } + + data := make([]byte, readSize) + n, err := reader.Read(data) + + totalRead += uint64(n) + dataRemaining -= uint64(n) + data = data[:n] + + if err != nil { + info.status = DownloadCanceled + return + } + + info.storeDownloadData(data[:n], fileOffset) + + fileOffset += uint64(n) + } + + //fmt.Printf("data finished: downloaded %d from total %d = %d %%\n", totalRead, fileSize, totalRead*100/fileSize) + + info.Finish() + info.DeleteDefer(time.Hour * 1) // cache the details for 1 hour before removing +} + +// Pause pauses the download. Status is DownloadResponseX. +func (info *downloadInfo) Pause() (status int) { + info.Lock() + defer info.Unlock() + + if info.status != DownloadActive { // The download must be active to be paused. + return DownloadResponseActionInvalid + } + + info.status = DownloadPause + + return DownloadResponseSuccess +} + +// Resume resumes the download. Status is DownloadResponseX. +func (info *downloadInfo) Resume() (status int) { + info.Lock() + defer info.Unlock() + + if info.status != DownloadPause { // The download must be paused to resume. + return DownloadResponseActionInvalid + } + + info.status = DownloadActive + + return DownloadResponseSuccess +} + +// Cancel cancels the download. Status is DownloadResponseX. +func (info *downloadInfo) Cancel() (status int) { + info.Lock() + defer info.Unlock() + + if info.status >= DownloadCanceled { // The download must not be already canceled or finished. + return DownloadResponseActionInvalid + } + + info.status = DownloadCanceled + info.DiskFile.Handle.Close() + + return DownloadResponseSuccess +} + +// Finish marks the download as finished. +func (info *downloadInfo) Finish() (status int) { + info.Lock() + defer info.Unlock() + + if info.status != DownloadActive { // The download must be active. + return DownloadResponseActionInvalid + } + + info.status = DownloadFinished + info.DiskFile.Handle.Close() + + return DownloadResponseSuccess +} + +// initDiskFile creates the target file +func (info *downloadInfo) initDiskFile(path string) (err error) { + info.DiskFile.Name = path + info.DiskFile.Handle, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) // 666 : All uses can read/write + + return err +} + +// storeDownloadData stores downloaded data. It does not change the download status. +func (info *downloadInfo) storeDownloadData(data []byte, offset uint64) (status int) { + info.Lock() + defer info.Unlock() + + if info.status != DownloadActive { // The download must be active. + return DownloadResponseActionInvalid + } + + if _, err := info.DiskFile.Handle.WriteAt(data, int64(offset)); err != nil { + return DownloadResponseFileWrite + } + + info.DiskFile.StoredSize += uint64(len(data)) + + return DownloadResponseSuccess +} + +func (info *downloadInfo) DownloadSelf() { + // Check if the file is available in the local warehouse. + _, fileSize, status, _ := info.backend.UserWarehouse.FileExists(info.hash) + if status != warehouse.StatusOK { + info.status = DownloadCanceled + return + } + + info.file.Size = fileSize + info.status = DownloadActive + + // read the file + status, bytesRead, _ := info.backend.UserWarehouse.ReadFile(info.hash, 0, int64(info.file.Size), info.DiskFile.Handle) + + info.DiskFile.StoredSize = uint64(bytesRead) + + if status != warehouse.StatusOK { + info.status = DownloadCanceled + return + } + + info.Finish() + info.DeleteDefer(time.Hour * 1) // cache the details for 1 hour before removing} +} diff --git a/webapi/Download.go b/webapi/Download.go index 5bde24f..9e90209 100644 --- a/webapi/Download.go +++ b/webapi/Download.go @@ -1,243 +1,243 @@ -/* -File Name: Download.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "encoding/hex" - "math" - "net/http" - "os" - "strconv" - "sync" - "time" - - "github.com/PeernetOfficial/core" - "github.com/google/uuid" -) - -type apiResponseDownloadStatus struct { - APIStatus int `json:"apistatus"` // Status of the API call. See DownloadResponseX. - ID uuid.UUID `json:"id"` // Download ID. This can be used to query the latest status and take actions. - DownloadStatus int `json:"downloadstatus"` // Status of the download. See DownloadX. - File apiFile `json:"file"` // File information. Only available for status >= DownloadWaitSwarm. - Progress struct { - TotalSize uint64 `json:"totalsize"` // Total size in bytes. - DownloadedSize uint64 `json:"downloadedsize"` // Count of bytes download so far. - Percentage float64 `json:"percentage"` // Percentage downloaded. Rounded to 2 decimal points. Between 0.00 and 100.00. - } `json:"progress"` // Progress of the download. Only valid for status >= DownloadWaitSwarm. - Swarm struct { - CountPeers uint64 `json:"countpeers"` // Count of peers participating in the swarm. - } `json:"swarm"` // Information about the swarm. Only valid for status >= DownloadActive. -} - -const ( - DownloadResponseSuccess = 0 // Success - DownloadResponseIDNotFound = 1 // Error: Download ID not found. - DownloadResponseFileInvalid = 2 // Error: Target file cannot be used. For example, permissions denied to create it. - DownloadResponseActionInvalid = 4 // Error: Invalid action. Pausing a non-active download, resuming a non-paused download, or canceling already canceled or finished download. - DownloadResponseFileWrite = 5 // Error writing file. -) - -// Download status list -const ( - DownloadWaitMetadata = 0 // Wait for file metadata. - DownloadWaitSwarm = 1 // Wait to join swarm. - DownloadActive = 2 // Active downloading. It could still be stuck at any percentage (including 0%) if no seeders are available. - DownloadPause = 3 // Paused by the user. - DownloadCanceled = 4 // Canceled by the user before the download finished. Once canceled, a new download has to be started if the file shall be downloaded. - DownloadFinished = 5 // Download finished 100%. -) - -/* -apiDownloadStart starts the download of a file. The path is the full path on disk to store the file. -The hash parameter identifies the file to download. The node ID identifies the blockchain (i.e., the "owner" of the file). - -Request: GET /download/start?path=[target path on disk]&hash=[file hash to download]&node=[node ID] -Result: 200 with JSON structure apiResponseDownloadStatus -*/ -func (api *WebapiInstance) apiDownloadStart(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - - // validate hashes, must be blake3 - hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) - nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) - if !valid1 || !valid2 { - http.Error(w, "", http.StatusBadRequest) - return - } - - filePath := r.Form.Get("path") - if filePath == "" { - http.Error(w, "", http.StatusBadRequest) - return - } - - info := &downloadInfo{backend: api.Backend, api: api, id: uuid.New(), created: time.Now(), hash: hash, nodeID: nodeID} - - // create the file immediately - if info.initDiskFile(filePath) != nil { - EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseFileInvalid}) - return - } - - // add the download to the list - api.downloadAdd(info) - - // start the download! - go info.Start() - - EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseSuccess, ID: info.id, DownloadStatus: DownloadWaitMetadata}) -} - -/* -apiDownloadStatus returns the status of an active download. - -Request: GET /download/status?id=[download ID] -Result: 200 with JSON structure apiResponseDownloadStatus -*/ -func (api *WebapiInstance) apiDownloadStatus(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - id, err := uuid.Parse(r.Form.Get("id")) - if err != nil { - http.Error(w, "", http.StatusBadRequest) - return - } - - info := api.downloadLookup(id) - if info == nil { - EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseIDNotFound}) - return - } - - info.RLock() - - response := apiResponseDownloadStatus{APIStatus: DownloadResponseSuccess, ID: info.id, DownloadStatus: info.status} - - if info.status >= DownloadWaitSwarm { - response.File = info.file - - response.Progress.TotalSize = info.file.Size - response.Progress.DownloadedSize = info.DiskFile.StoredSize - - response.Progress.Percentage = math.Round(float64(info.DiskFile.StoredSize)/float64(info.file.Size)*100*100) / 100 - } - - if info.status >= DownloadActive { - response.Swarm.CountPeers = info.Swarm.CountPeers - } - - info.RUnlock() - - EncodeJSON(api.Backend, w, r, response) -} - -/* -apiDownloadAction pauses, resumes, and cancels a download. Once canceled, a new download has to be started if the file shall be downloaded. -Only active downloads can be paused. While a download is in discovery phase (querying metadata, joining swarm), it can only be canceled. -Action: 0 = Pause, 1 = Resume, 2 = Cancel. - -Request: GET /download/action?id=[download ID]&action=[action] -Result: 200 with JSON structure apiResponseDownloadStatus (using APIStatus and DownloadStatus) -*/ -func (api *WebapiInstance) apiDownloadAction(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - id, err := uuid.Parse(r.Form.Get("id")) - action, err2 := strconv.Atoi(r.Form.Get("action")) - if err != nil || err2 != nil || action < 0 || action > 2 { - http.Error(w, "", http.StatusBadRequest) - return - } - - info := api.downloadLookup(id) - if info == nil { - EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseIDNotFound}) - return - } - - apiStatus := 0 - - switch action { - case 0: // Pause - apiStatus = info.Pause() - - case 1: // Resume - apiStatus = info.Resume() - - case 2: // Cancel - apiStatus = info.Cancel() - } - - EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: apiStatus, ID: info.id, DownloadStatus: info.status}) -} - -// ---- download tracking ---- - -type downloadInfo struct { - id uuid.UUID // Download ID - status int // Current status. See DownloadX. - sync.RWMutex // Mutext for changing the status - - // input - hash []byte // File hash - nodeID []byte // Node ID of the owner - - // runtime data - created time.Time // When the download was created. - ended time.Time // When the download was finished (only status = DownloadFinished). - - file apiFile // File metadata (only status >= DownloadWaitSwarm) - - DiskFile struct { // Target file on disk to store downloaded data - Name string // File name - Handle *os.File // Target file (on disk) to store downloaded data - StoredSize uint64 // Count of bytes downloaded and stored in the file - } - - Swarm struct { // Information about the swarm. Only valid for status >= DownloadActive. - CountPeers uint64 // Count of peers participating in the swarm. - } - - // live connections, to be changed - peer *core.PeerInfo - - api *WebapiInstance - backend *core.Backend -} - -func (api *WebapiInstance) downloadAdd(info *downloadInfo) { - api.downloadsMutex.Lock() - api.downloads[info.id] = info - api.downloadsMutex.Unlock() -} - -func (api *WebapiInstance) downloadDelete(id uuid.UUID) { - api.downloadsMutex.Lock() - delete(api.downloads, id) - api.downloadsMutex.Unlock() -} - -func (api *WebapiInstance) downloadLookup(id uuid.UUID) (info *downloadInfo) { - api.downloadsMutex.Lock() - info = api.downloads[id] - api.downloadsMutex.Unlock() - return info -} - -// DeleteDefer deletes the download from the downloads list after the given duration. -// It does not wait for the download to be finished. -func (info *downloadInfo) DeleteDefer(Duration time.Duration) { - go func() { - <-time.After(Duration) - info.api.downloadDelete(info.id) - }() -} - -// DecodeBlake3Hash decodes a blake3 hash that is hex encoded -func DecodeBlake3Hash(text string) (hash []byte, valid bool) { - hash, err := hex.DecodeString(text) - return hash, err == nil && len(hash) == 256/8 -} +/* +File Name: Download.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "encoding/hex" + "math" + "net/http" + "os" + "strconv" + "sync" + "time" + + "github.com/PeernetOfficial/core" + "github.com/google/uuid" +) + +type apiResponseDownloadStatus struct { + APIStatus int `json:"apistatus"` // Status of the API call. See DownloadResponseX. + ID uuid.UUID `json:"id"` // Download ID. This can be used to query the latest status and take actions. + DownloadStatus int `json:"downloadstatus"` // Status of the download. See DownloadX. + File apiFile `json:"file"` // File information. Only available for status >= DownloadWaitSwarm. + Progress struct { + TotalSize uint64 `json:"totalsize"` // Total size in bytes. + DownloadedSize uint64 `json:"downloadedsize"` // Count of bytes download so far. + Percentage float64 `json:"percentage"` // Percentage downloaded. Rounded to 2 decimal points. Between 0.00 and 100.00. + } `json:"progress"` // Progress of the download. Only valid for status >= DownloadWaitSwarm. + Swarm struct { + CountPeers uint64 `json:"countpeers"` // Count of peers participating in the swarm. + } `json:"swarm"` // Information about the swarm. Only valid for status >= DownloadActive. +} + +const ( + DownloadResponseSuccess = 0 // Success + DownloadResponseIDNotFound = 1 // Error: Download ID not found. + DownloadResponseFileInvalid = 2 // Error: Target file cannot be used. For example, permissions denied to create it. + DownloadResponseActionInvalid = 4 // Error: Invalid action. Pausing a non-active download, resuming a non-paused download, or canceling already canceled or finished download. + DownloadResponseFileWrite = 5 // Error writing file. +) + +// Download status list +const ( + DownloadWaitMetadata = 0 // Wait for file metadata. + DownloadWaitSwarm = 1 // Wait to join swarm. + DownloadActive = 2 // Active downloading. It could still be stuck at any percentage (including 0%) if no seeders are available. + DownloadPause = 3 // Paused by the user. + DownloadCanceled = 4 // Canceled by the user before the download finished. Once canceled, a new download has to be started if the file shall be downloaded. + DownloadFinished = 5 // Download finished 100%. +) + +/* +apiDownloadStart starts the download of a file. The path is the full path on disk to store the file. +The hash parameter identifies the file to download. The node ID identifies the blockchain (i.e., the "owner" of the file). + +Request: GET /download/start?path=[target path on disk]&hash=[file hash to download]&node=[node ID] +Result: 200 with JSON structure apiResponseDownloadStatus +*/ +func (api *WebapiInstance) apiDownloadStart(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + + // validate hashes, must be blake3 + hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) + if !valid1 || !valid2 { + http.Error(w, "", http.StatusBadRequest) + return + } + + filePath := r.Form.Get("path") + if filePath == "" { + http.Error(w, "", http.StatusBadRequest) + return + } + + info := &downloadInfo{backend: api.Backend, api: api, id: uuid.New(), created: time.Now(), hash: hash, nodeID: nodeID} + + // create the file immediately + if info.initDiskFile(filePath) != nil { + EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseFileInvalid}) + return + } + + // add the download to the list + api.downloadAdd(info) + + // start the download! + go info.Start() + + EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseSuccess, ID: info.id, DownloadStatus: DownloadWaitMetadata}) +} + +/* +apiDownloadStatus returns the status of an active download. + +Request: GET /download/status?id=[download ID] +Result: 200 with JSON structure apiResponseDownloadStatus +*/ +func (api *WebapiInstance) apiDownloadStatus(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + id, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + info := api.downloadLookup(id) + if info == nil { + EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseIDNotFound}) + return + } + + info.RLock() + + response := apiResponseDownloadStatus{APIStatus: DownloadResponseSuccess, ID: info.id, DownloadStatus: info.status} + + if info.status >= DownloadWaitSwarm { + response.File = info.file + + response.Progress.TotalSize = info.file.Size + response.Progress.DownloadedSize = info.DiskFile.StoredSize + + response.Progress.Percentage = math.Round(float64(info.DiskFile.StoredSize)/float64(info.file.Size)*100*100) / 100 + } + + if info.status >= DownloadActive { + response.Swarm.CountPeers = info.Swarm.CountPeers + } + + info.RUnlock() + + EncodeJSON(api.Backend, w, r, response) +} + +/* +apiDownloadAction pauses, resumes, and cancels a download. Once canceled, a new download has to be started if the file shall be downloaded. +Only active downloads can be paused. While a download is in discovery phase (querying metadata, joining swarm), it can only be canceled. +Action: 0 = Pause, 1 = Resume, 2 = Cancel. + +Request: GET /download/action?id=[download ID]&action=[action] +Result: 200 with JSON structure apiResponseDownloadStatus (using APIStatus and DownloadStatus) +*/ +func (api *WebapiInstance) apiDownloadAction(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + id, err := uuid.Parse(r.Form.Get("id")) + action, err2 := strconv.Atoi(r.Form.Get("action")) + if err != nil || err2 != nil || action < 0 || action > 2 { + http.Error(w, "", http.StatusBadRequest) + return + } + + info := api.downloadLookup(id) + if info == nil { + EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseIDNotFound}) + return + } + + apiStatus := 0 + + switch action { + case 0: // Pause + apiStatus = info.Pause() + + case 1: // Resume + apiStatus = info.Resume() + + case 2: // Cancel + apiStatus = info.Cancel() + } + + EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: apiStatus, ID: info.id, DownloadStatus: info.status}) +} + +// ---- download tracking ---- + +type downloadInfo struct { + id uuid.UUID // Download ID + status int // Current status. See DownloadX. + sync.RWMutex // Mutext for changing the status + + // input + hash []byte // File hash + nodeID []byte // Node ID of the owner + + // runtime data + created time.Time // When the download was created. + ended time.Time // When the download was finished (only status = DownloadFinished). + + file apiFile // File metadata (only status >= DownloadWaitSwarm) + + DiskFile struct { // Target file on disk to store downloaded data + Name string // File name + Handle *os.File // Target file (on disk) to store downloaded data + StoredSize uint64 // Count of bytes downloaded and stored in the file + } + + Swarm struct { // Information about the swarm. Only valid for status >= DownloadActive. + CountPeers uint64 // Count of peers participating in the swarm. + } + + // live connections, to be changed + peer *core.PeerInfo + + api *WebapiInstance + backend *core.Backend +} + +func (api *WebapiInstance) downloadAdd(info *downloadInfo) { + api.downloadsMutex.Lock() + api.downloads[info.id] = info + api.downloadsMutex.Unlock() +} + +func (api *WebapiInstance) downloadDelete(id uuid.UUID) { + api.downloadsMutex.Lock() + delete(api.downloads, id) + api.downloadsMutex.Unlock() +} + +func (api *WebapiInstance) downloadLookup(id uuid.UUID) (info *downloadInfo) { + api.downloadsMutex.Lock() + info = api.downloads[id] + api.downloadsMutex.Unlock() + return info +} + +// DeleteDefer deletes the download from the downloads list after the given duration. +// It does not wait for the download to be finished. +func (info *downloadInfo) DeleteDefer(Duration time.Duration) { + go func() { + <-time.After(Duration) + info.api.downloadDelete(info.id) + }() +} + +// DecodeBlake3Hash decodes a blake3 hash that is hex encoded +func DecodeBlake3Hash(text string) (hash []byte, valid bool) { + hash, err := hex.DecodeString(text) + return hash, err == nil && len(hash) == 256/8 +} diff --git a/webapi/Download_test.go b/webapi/Download_test.go index 8437585..320bdf7 100644 --- a/webapi/Download_test.go +++ b/webapi/Download_test.go @@ -1,13 +1,13 @@ -package webapi - -import ( - "fmt" - "testing" -) - -// Test function -func TestDecodeBlake3Hash(t *testing.T) { - hash, bool := DecodeBlake3Hash("") - fmt.Println(hash) - fmt.Println(bool) -} +package webapi + +import ( + "fmt" + "testing" +) + +// Test function +func TestDecodeBlake3Hash(t *testing.T) { + hash, bool := DecodeBlake3Hash("") + fmt.Println(hash) + fmt.Println(bool) +} diff --git a/webapi/File Detection.go b/webapi/File Detection.go index 6222f3e..b20fc2f 100644 --- a/webapi/File Detection.go +++ b/webapi/File Detection.go @@ -1,221 +1,221 @@ -/* -File Name: File Detection.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "net/http" - "os" - "path" - "strings" - - "github.com/PeernetOfficial/core" -) - -// PathToExtension translates a path to a file extension, if possible. It also returns the second file extension if there is one (relevant for files like "test.tar.gz"). -func PathToExtension(Path string) (extension, extension2 string, valid bool) { - _, fileA := path.Split(Path) - - parts := strings.Split(fileA, ".") - if len(parts) <= 1 { - return "", "", false - } - extension = parts[len(parts)-1] - extension = strings.ToLower(extension) - - if len(parts) >= 3 { - extension2 = parts[len(parts)-2] - extension2 = strings.ToLower(extension2) - } - - return extension, extension2, true -} - -// FileTranslateExtension translates the extension to a File Type and File Format. If invalid, types are 0. -func FileTranslateExtension(extension string) (fileType, fileFormat uint16) { - switch extension { - case "txt", "log", "ini", "json", "md": - return core.TypeText, core.FormatText - - case "csv", "tsv": - return core.TypeText, core.FormatCSV - - case "html", "htm": - return core.TypeText, core.FormatHTML - - case "doc", "docx", "rtf", "odt": - return core.TypeDocument, core.FormatWord - - case "pdf": - return core.TypeDocument, core.FormatPDF - - case "xls", "xlsx", "ods": - return core.TypeDocument, core.FormatExcel - - case "gif", "jpg", "jpeg", "png", "svg", "bmp", "tif", "tiff", "jfif", "webp": - return core.TypePicture, core.FormatPicture - - case "mp4", "flv", "avi", "mov", "mpg", "mpeg", "h264", "3g2", "3gp", "mkv", "wmv", "webm", "ts": - return core.TypeVideo, core.FormatVideo - - case "mp3", "ogg", "flac": - return core.TypeAudio, core.FormatAudio - - case "zip", "rar", "7z", "tar": - return core.TypeContainer, core.FormatContainer - - case "ppt", "pptx", "odp": - return core.TypeDocument, core.FormatPowerpoint - - case "epub", "mobi", "prc": - return core.TypeEbook, core.FormatEbook - - case "gz", "bz", "bz2", "xz": - return core.TypeCompressed, core.FormatCompressed - - case "sql": - return core.TypeText, core.FormatDatabase - - case "eml", "mbox": - return core.TypeText, core.FormatEmail - - case "exe", "sys", "dll", "cmd", "bat": - return core.TypeExecutable, core.FormatExecutable - - case "msi": - return core.TypeExecutable, core.FormatInstaller - - case "apk": - return core.TypeExecutable, core.FormatAPK - - case "iso": - return core.TypeContainer, core.FormatISO - - default: - return core.TypeBinary, core.FormatBinary - } -} - -// HTTPContentTypeToCore translates the HTTP content type to the File Type and File Format used by the core package. -func HTTPContentTypeToCore(httpContentType string) (fileType, fileFormat uint16) { - switch httpContentType { - case "text/html", "application/xhtml+xml", "application/xml": - return core.TypeText, core.FormatHTML - - case "text/plain": - return core.TypeText, core.FormatText - - case "text/csv", "text/tsv", "text/tab-separated-values", "text/x-csv", "application/csv", "application/x-csv", "text/x-comma-separated-values": - return core.TypeText, core.FormatCSV - - case "application/pdf", "application/x-pdf": - return core.TypeDocument, core.FormatPDF - - case "application/msword", "application/rtf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.oasis.opendocument.text": - return core.TypeDocument, core.FormatWord - - case "application/excel", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.oasis.opendocument.spreadsheet": - return core.TypeDocument, core.FormatExcel - - case "application/vnd.ms-powerpoint", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.oasis.opendocument.presentation": - return core.TypeDocument, core.FormatPowerpoint - - case "image/png", "image/jpeg", "image/gif", "image/svg+xml", "image/tiff", "image/webp", "image/bmp", "image/x-bmp", "image/x-windows-bmp": // image/x-icon excluded - return core.TypePicture, core.FormatPicture - - case "audio/aac", "audio/midi", "audio/ogg", "audio/x-wav", "audio/webm", "audio/3gpp", "audio/3gpp2", "audio/mpeg", "audio/vorbis": - return core.TypeAudio, core.FormatAudio - - case "video/x-msvideo", "video/x-ms-wmv", "video/mpeg", "video/ogg", "video/webm", "video/3gpp", "video/3gpp2", "video/x-flv", "video/mp4": - return core.TypeVideo, core.FormatVideo - - case "application/zip", "application/x-rar-compressed", "application/x-tar", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed": - return core.TypeContainer, core.FormatContainer - - case "application/epub+zip", "application/x-mobipocket-ebook": - return core.TypeEbook, core.FormatEbook - - default: - return core.TypeBinary, core.FormatBinary - } -} - -// FileDataToHTTPContentType returns the HTTP content type based on the initial file data. It reads the first 512 bytes of the file. -func FileDataToHTTPContentType(Path string) (httpContentType string, err error) { - file, err := os.Open(Path) - if err != nil { - return "", err - } - - // Read up to 512 bytes. This specific number comes from http.DetectContentType which specifies it as constant "sniffLen". - buff := make([]byte, 512) - - if _, err := file.Read(buff); err != nil { - return "", err - } - - if err := file.Close(); err != nil { - return "", err - } - - httpContentType = http.DetectContentType(buff) - - // sanitize it first - httpContentType = strings.ToLower(strings.TrimSpace(httpContentType)) - if indexD := strings.IndexAny(httpContentType, ";"); indexD >= 0 { - httpContentType = httpContentType[:indexD] - } - - return httpContentType, nil -} - -// FileDetectType detects the File Type and File Format of a file. It uses the extension if available, otherwise the file data, for detection. -func FileDetectType(Path string) (fileType, fileFormat uint16, err error) { - // If a file extension is available, use that to detect the file type and file format. - // Otherwise, use the initial file data for detection. - if ext1, _, valid := PathToExtension(Path); valid { - fileType, fileFormat = FileTranslateExtension(ext1) - return fileType, fileFormat, nil - } - - httpContentType, err := FileDataToHTTPContentType(Path) - if err != nil { - return core.TypeBinary, core.FormatBinary, err - } - - fileType, fileFormat = HTTPContentTypeToCore(httpContentType) - return fileType, fileFormat, nil -} - -type apiResponseFileFormat struct { - Status int `json:"status"` // Status: 0 = Success, 1 = Error reading file - FileType uint16 `json:"filetype"` // File Type. - FileFormat uint16 `json:"fileformat"` // File Format. -} - -/* -apiFileFormat detects the file type and file format of the specified file. -It will primarily use the file extension for detection. If unavailable, it uses the first 512 bytes of the file data to detect the type. - -Request: GET /file/format?path=[file path on disk] -Result: 200 with JSON structure apiResponseFileFormat -*/ -func (api *WebapiInstance) apiFileFormat(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - filePath := r.Form.Get("path") - if filePath == "" { - http.Error(w, "", http.StatusBadRequest) - return - } - - fileType, fileFormat, err := FileDetectType(filePath) - if err != nil { - EncodeJSON(api.Backend, w, r, apiResponseFileFormat{Status: 1}) - return - } - - EncodeJSON(api.Backend, w, r, apiResponseFileFormat{Status: 0, FileType: fileType, FileFormat: fileFormat}) -} +/* +File Name: File Detection.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "net/http" + "os" + "path" + "strings" + + "github.com/PeernetOfficial/core" +) + +// PathToExtension translates a path to a file extension, if possible. It also returns the second file extension if there is one (relevant for files like "test.tar.gz"). +func PathToExtension(Path string) (extension, extension2 string, valid bool) { + _, fileA := path.Split(Path) + + parts := strings.Split(fileA, ".") + if len(parts) <= 1 { + return "", "", false + } + extension = parts[len(parts)-1] + extension = strings.ToLower(extension) + + if len(parts) >= 3 { + extension2 = parts[len(parts)-2] + extension2 = strings.ToLower(extension2) + } + + return extension, extension2, true +} + +// FileTranslateExtension translates the extension to a File Type and File Format. If invalid, types are 0. +func FileTranslateExtension(extension string) (fileType, fileFormat uint16) { + switch extension { + case "txt", "log", "ini", "json", "md": + return core.TypeText, core.FormatText + + case "csv", "tsv": + return core.TypeText, core.FormatCSV + + case "html", "htm": + return core.TypeText, core.FormatHTML + + case "doc", "docx", "rtf", "odt": + return core.TypeDocument, core.FormatWord + + case "pdf": + return core.TypeDocument, core.FormatPDF + + case "xls", "xlsx", "ods": + return core.TypeDocument, core.FormatExcel + + case "gif", "jpg", "jpeg", "png", "svg", "bmp", "tif", "tiff", "jfif", "webp": + return core.TypePicture, core.FormatPicture + + case "mp4", "flv", "avi", "mov", "mpg", "mpeg", "h264", "3g2", "3gp", "mkv", "wmv", "webm", "ts": + return core.TypeVideo, core.FormatVideo + + case "mp3", "ogg", "flac": + return core.TypeAudio, core.FormatAudio + + case "zip", "rar", "7z", "tar": + return core.TypeContainer, core.FormatContainer + + case "ppt", "pptx", "odp": + return core.TypeDocument, core.FormatPowerpoint + + case "epub", "mobi", "prc": + return core.TypeEbook, core.FormatEbook + + case "gz", "bz", "bz2", "xz": + return core.TypeCompressed, core.FormatCompressed + + case "sql": + return core.TypeText, core.FormatDatabase + + case "eml", "mbox": + return core.TypeText, core.FormatEmail + + case "exe", "sys", "dll", "cmd", "bat": + return core.TypeExecutable, core.FormatExecutable + + case "msi": + return core.TypeExecutable, core.FormatInstaller + + case "apk": + return core.TypeExecutable, core.FormatAPK + + case "iso": + return core.TypeContainer, core.FormatISO + + default: + return core.TypeBinary, core.FormatBinary + } +} + +// HTTPContentTypeToCore translates the HTTP content type to the File Type and File Format used by the core package. +func HTTPContentTypeToCore(httpContentType string) (fileType, fileFormat uint16) { + switch httpContentType { + case "text/html", "application/xhtml+xml", "application/xml": + return core.TypeText, core.FormatHTML + + case "text/plain": + return core.TypeText, core.FormatText + + case "text/csv", "text/tsv", "text/tab-separated-values", "text/x-csv", "application/csv", "application/x-csv", "text/x-comma-separated-values": + return core.TypeText, core.FormatCSV + + case "application/pdf", "application/x-pdf": + return core.TypeDocument, core.FormatPDF + + case "application/msword", "application/rtf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.oasis.opendocument.text": + return core.TypeDocument, core.FormatWord + + case "application/excel", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.oasis.opendocument.spreadsheet": + return core.TypeDocument, core.FormatExcel + + case "application/vnd.ms-powerpoint", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.oasis.opendocument.presentation": + return core.TypeDocument, core.FormatPowerpoint + + case "image/png", "image/jpeg", "image/gif", "image/svg+xml", "image/tiff", "image/webp", "image/bmp", "image/x-bmp", "image/x-windows-bmp": // image/x-icon excluded + return core.TypePicture, core.FormatPicture + + case "audio/aac", "audio/midi", "audio/ogg", "audio/x-wav", "audio/webm", "audio/3gpp", "audio/3gpp2", "audio/mpeg", "audio/vorbis": + return core.TypeAudio, core.FormatAudio + + case "video/x-msvideo", "video/x-ms-wmv", "video/mpeg", "video/ogg", "video/webm", "video/3gpp", "video/3gpp2", "video/x-flv", "video/mp4": + return core.TypeVideo, core.FormatVideo + + case "application/zip", "application/x-rar-compressed", "application/x-tar", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed": + return core.TypeContainer, core.FormatContainer + + case "application/epub+zip", "application/x-mobipocket-ebook": + return core.TypeEbook, core.FormatEbook + + default: + return core.TypeBinary, core.FormatBinary + } +} + +// FileDataToHTTPContentType returns the HTTP content type based on the initial file data. It reads the first 512 bytes of the file. +func FileDataToHTTPContentType(Path string) (httpContentType string, err error) { + file, err := os.Open(Path) + if err != nil { + return "", err + } + + // Read up to 512 bytes. This specific number comes from http.DetectContentType which specifies it as constant "sniffLen". + buff := make([]byte, 512) + + if _, err := file.Read(buff); err != nil { + return "", err + } + + if err := file.Close(); err != nil { + return "", err + } + + httpContentType = http.DetectContentType(buff) + + // sanitize it first + httpContentType = strings.ToLower(strings.TrimSpace(httpContentType)) + if indexD := strings.IndexAny(httpContentType, ";"); indexD >= 0 { + httpContentType = httpContentType[:indexD] + } + + return httpContentType, nil +} + +// FileDetectType detects the File Type and File Format of a file. It uses the extension if available, otherwise the file data, for detection. +func FileDetectType(Path string) (fileType, fileFormat uint16, err error) { + // If a file extension is available, use that to detect the file type and file format. + // Otherwise, use the initial file data for detection. + if ext1, _, valid := PathToExtension(Path); valid { + fileType, fileFormat = FileTranslateExtension(ext1) + return fileType, fileFormat, nil + } + + httpContentType, err := FileDataToHTTPContentType(Path) + if err != nil { + return core.TypeBinary, core.FormatBinary, err + } + + fileType, fileFormat = HTTPContentTypeToCore(httpContentType) + return fileType, fileFormat, nil +} + +type apiResponseFileFormat struct { + Status int `json:"status"` // Status: 0 = Success, 1 = Error reading file + FileType uint16 `json:"filetype"` // File Type. + FileFormat uint16 `json:"fileformat"` // File Format. +} + +/* +apiFileFormat detects the file type and file format of the specified file. +It will primarily use the file extension for detection. If unavailable, it uses the first 512 bytes of the file data to detect the type. + +Request: GET /file/format?path=[file path on disk] +Result: 200 with JSON structure apiResponseFileFormat +*/ +func (api *WebapiInstance) apiFileFormat(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + filePath := r.Form.Get("path") + if filePath == "" { + http.Error(w, "", http.StatusBadRequest) + return + } + + fileType, fileFormat, err := FileDetectType(filePath) + if err != nil { + EncodeJSON(api.Backend, w, r, apiResponseFileFormat{Status: 1}) + return + } + + EncodeJSON(api.Backend, w, r, apiResponseFileFormat{Status: 0, FileType: fileType, FileFormat: fileFormat}) +} diff --git a/webapi/File IO.go b/webapi/File IO.go index 0045691..48e4763 100644 --- a/webapi/File IO.go +++ b/webapi/File IO.go @@ -1,325 +1,325 @@ -/* -File Name: File IO.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "errors" - "io" - "net/http" - "strconv" - "time" - - "github.com/PeernetOfficial/core" - "github.com/PeernetOfficial/core/btcec" - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/warehouse" -) - -/* -apiFileRead reads a file immediately from a remote peer. Use the /download functions to download a file. -This endpoint supports the Range, Content-Range and Content-Length headers. Multipart ranges are not supported and result in HTTP 400. -Instead of providing the node ID, the peer ID is also accepted in the &node= parameter. -The default timeout for connecting to the peer is 10 seconds. - -Request: GET /file/read?hash=[hash]&node=[node ID] - - Optional: &offset=[offset]&limit=[limit] or via Range header. - Optional: &timeout=[seconds] - -Response: 200 with the content - - 206 with partial content - 400 if the parameters are invalid - 404 if the file was not found or other error on transfer initiate - 502 if unable to find or connect to the remote peer in time -*/ -func (api *WebapiInstance) apiFileRead(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - var err error - - // validate hashes (must be blake3) and other input - fileHash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) - nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) - publicKey, err3 := core.PublicKeyFromPeerID(r.Form.Get("node")) - if !valid1 || (!valid2 && err3 != nil) { - http.Error(w, "", http.StatusBadRequest) - return - } - - timeoutSeconds, _ := strconv.Atoi(r.Form.Get("timeout")) - if timeoutSeconds == 0 { - timeoutSeconds = 10 - } - timeout := time.Duration(timeoutSeconds) * time.Second - - offset, _ := strconv.Atoi(r.Form.Get("offset")) - limit, _ := strconv.Atoi(r.Form.Get("limit")) - - // Range header? - var ranges []HTTPRange - if ranges, err = ParseRangeHeader(r.Header.Get("Range"), -1, true); err != nil || len(ranges) > 1 { - http.Error(w, "", http.StatusBadRequest) - return - } else if len(ranges) == 1 { - if ranges[0].length != -1 { // if length is not specified, limit remains 0 which is maximum - limit = ranges[0].length - } - offset = ranges[0].start - } - - // Is the file available in the local warehouse? In that case requesting it from the remote is unnecessary. - if serveFileFromWarehouse(api.Backend, w, fileHash, uint64(offset), uint64(limit), ranges) { - return - } - - // try connecting via node ID or peer ID? - var peer *core.PeerInfo - - if valid2 { - peer, err = PeerConnectNode(api.Backend, nodeID, timeout) - } else if err3 == nil { - peer, err = PeerConnectPublicKey(api.Backend, publicKey, timeout) - } - if err != nil { - w.WriteHeader(http.StatusBadGateway) - return - } - - // Start the reader. If this HTTP request is canceled, r.Context().Done() acts as cancellation signal to the underlying UDT connection. - reader, fileSize, transferSize, err := FileStartReader(peer, fileHash, uint64(offset), uint64(limit), r.Context().Done()) - if reader != nil { - defer reader.Close() - } - if err != nil || reader == nil { - w.WriteHeader(http.StatusNotFound) - return - } - - // set the right headers - setContentLengthRangeHeader(w, uint64(offset), transferSize, fileSize, ranges) - - // Start sending the data! - io.Copy(w, io.LimitReader(reader, int64(transferSize))) -} - -// serveFileFromWarehouse serves the file from the warehouse. If it is not available, it returns false and does not use the writer. -// Limit is optional, 0 means the entire file. -func serveFileFromWarehouse(backend *core.Backend, w http.ResponseWriter, fileHash []byte, offset, limit uint64, ranges []HTTPRange) (valid bool) { - // Check if the file is available in the local warehouse. - _, fileSize, status, _ := backend.UserWarehouse.FileExists(fileHash) - if status != warehouse.StatusOK { - return false - } - - // validate offset and limit - if limit > 0 && offset+limit > fileSize { - http.Error(w, "invalid limit", http.StatusBadRequest) - return true - } else if offset > fileSize { - http.Error(w, "invalid offset", http.StatusBadRequest) - return true - } else if limit == 0 { - limit = fileSize - offset - } - - setContentLengthRangeHeader(w, offset, limit, fileSize, ranges) - - status, _, _ = backend.UserWarehouse.ReadFile(fileHash, int64(offset), int64(limit), w) - - // StatusErrorReadFile must be considered success, since parts of the file may have been transferred already and recovery is not possible. - return status == warehouse.StatusErrorReadFile || status == warehouse.StatusOK -} - -/* -apiFileView is similar to /file/read but but provides a format parameter. It sets the Content-Type and Accept-Ranges headers. -This endpoint supports the Range, Content-Range and Content-Length headers. Multipart ranges are not supported and result in HTTP 400. -Instead of providing the node ID, the peer ID is also accepted in the &node= parameter. -The default timeout for connecting to the peer is 10 seconds. -Formats: 14 = Video - -Request: GET /file/view?hash=[hash]&node=[node ID]&format=[format] - - Optional: &offset=[offset]&limit=[limit] or via Range header. - Optional: &timeout=[seconds] - -Response: 200 with the content - - 206 with partial content - 400 if the parameters are invalid - 404 if the file was not found or other error on transfer initiate - 502 if unable to find or connect to the remote peer in time -*/ -func (api *WebapiInstance) apiFileView(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - var err error - - // validate hashes (must be blake3) and other input - fileHash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) - nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) - publicKey, err3 := core.PublicKeyFromPeerID(r.Form.Get("node")) - if !valid1 || (!valid2 && err3 != nil) { - http.Error(w, "", http.StatusBadRequest) - return - } - - timeoutSeconds, _ := strconv.Atoi(r.Form.Get("timeout")) - if timeoutSeconds == 0 { - timeoutSeconds = 10 - } - timeout := time.Duration(timeoutSeconds) * time.Second - - offset, _ := strconv.Atoi(r.Form.Get("offset")) - limit, _ := strconv.Atoi(r.Form.Get("limit")) - format, _ := strconv.Atoi(r.Form.Get("format")) - localCacheDisable, _ := strconv.ParseBool(r.Form.Get("nocache")) - - // Range header? - var ranges []HTTPRange - if ranges, err = ParseRangeHeader(r.Header.Get("Range"), -1, true); err != nil || len(ranges) > 1 { - http.Error(w, "", http.StatusBadRequest) - return - } else if len(ranges) == 1 { - if ranges[0].length != -1 { // if length is not specified, limit remains 0 which is maximum - limit = ranges[0].length - } - offset = ranges[0].start - } - - w.Header().Set("Accept-Ranges", "bytes") // always indicate accepting of Range header - - switch format { - case 14: - // Video: Indicate MP4 always. There are tons of other MIME types that could be used. - w.Header().Set("Content-Type", "video/mp4") - } - - // Is the file available in the local warehouse? In that case requesting it from the remote is unnecessary. - if !localCacheDisable { - if serveFileFromWarehouse(api.Backend, w, fileHash, uint64(offset), uint64(limit), ranges) { - return - } - } - - // try connecting via node ID or peer ID? - var peer *core.PeerInfo - - if valid2 { - peer, err = PeerConnectNode(api.Backend, nodeID, timeout) - } else if err3 == nil { - peer, err = PeerConnectPublicKey(api.Backend, publicKey, timeout) - } - if err != nil { - w.WriteHeader(http.StatusBadGateway) - return - } - - // start the reader - reader, fileSize, transferSize, err := FileStartReader(peer, fileHash, uint64(offset), uint64(limit), r.Context().Done()) - if reader != nil { - defer reader.Close() - } - if err != nil || reader == nil { - w.WriteHeader(http.StatusNotFound) - return - } - - // set the right headers - setContentLengthRangeHeader(w, uint64(offset), transferSize, fileSize, ranges) - - // Start sending the data! - io.Copy(w, io.LimitReader(reader, int64(transferSize))) -} - -// PeerConnectPublicKey attempts to connect to the peer specified by its public key (= peer ID). -func PeerConnectPublicKey(backend *core.Backend, publicKey *btcec.PublicKey, timeout time.Duration) (peer *core.PeerInfo, err error) { - if publicKey == nil { - return nil, errors.New("invalid public key") - } - - // First look up in the peer list. - if peer = backend.PeerlistLookup(publicKey); peer != nil { - return peer, nil - } - - // Try to connect via DHT. - nodeID := protocol.PublicKey2NodeID(publicKey) - if _, peer, _ = backend.FindNode(nodeID, timeout); peer != nil { - return peer, nil - } - - // otherwise not found :( - return nil, errors.New("peer not found") -} - -// PeerConnectNode tries to connect via the node ID -func PeerConnectNode(backend *core.Backend, nodeID []byte, timeout time.Duration) (peer *core.PeerInfo, err error) { - if len(nodeID) != 256/8 { - return nil, errors.New("invalid node ID") - } - - // Try to connect via DHT. - if _, peer, _ = backend.FindNode(nodeID, timeout); peer != nil { - return peer, nil - } - - // otherwise not found :( - return nil, errors.New("peer not found") -} - -// FileStartReader providers a reader to a remote file. The reader must be closed by the caller. -// File Size is the full file size reported by the remote peer, regardless of the requested offset and limit. Limit is optional (0 means the entire file). -// Transfer Size is the size in bytes that is actually going to be transferred. The reader should be closed after reading that amount. -// The optional cancelChan can be used to stop the file transfer at any point. -func FileStartReader(peer *core.PeerInfo, hash []byte, offset, limit uint64, cancelChan <-chan struct{}) (reader io.ReadCloser, fileSize, transferSize uint64, err error) { - if peer == nil { - return nil, 0, 0, errors.New("peer not provided") - } else if !peer.IsConnectionActive() { - return nil, 0, 0, errors.New("no valid connection to peer") - } - - udtConn, virtualConn, err := peer.FileTransferRequestUDT(hash, offset, limit) - if err != nil { - return nil, 0, 0, err - } - - if cancelChan != nil { - go func() { - <-cancelChan - udtConn.Close() - }() - } - - fileSize, transferSize, err = protocol.FileTransferReadHeader(udtConn) - if err != nil { - udtConn.Close() - return nil, 0, 0, err - } - - virtualConn.Stats.(*core.FileTransferStats).FileSize = fileSize - - return udtConn, fileSize, transferSize, nil -} - -// FileReadAll downloads the file from the peer. -// This function should only be used for testing or as a basis to fork. The caller should develop a custom download function that handles timeouts and excessive file sizes. -// It allocates whatever size is reported by the remote peer. This could lead to an out of memory crash. -// This function is blocking and may take a long time depending on the remote peer and the network connection. -func FileReadAll(peer *core.PeerInfo, hash []byte) (data []byte, err error) { - reader, _, transferSize, err := FileStartReader(peer, hash, 0, 0, nil) - if err != nil { - return nil, err - } - defer reader.Close() - - // read all data - data = make([]byte, transferSize) // Warning: This could lead to an out of memory crash. - _, err = reader.Read(data) - - // Note: This function does not verify if the returned data matches the hash and expected size. - - return data, err -} +/* +File Name: File IO.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "errors" + "io" + "net/http" + "strconv" + "time" + + "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/btcec" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/warehouse" +) + +/* +apiFileRead reads a file immediately from a remote peer. Use the /download functions to download a file. +This endpoint supports the Range, Content-Range and Content-Length headers. Multipart ranges are not supported and result in HTTP 400. +Instead of providing the node ID, the peer ID is also accepted in the &node= parameter. +The default timeout for connecting to the peer is 10 seconds. + +Request: GET /file/read?hash=[hash]&node=[node ID] + + Optional: &offset=[offset]&limit=[limit] or via Range header. + Optional: &timeout=[seconds] + +Response: 200 with the content + + 206 with partial content + 400 if the parameters are invalid + 404 if the file was not found or other error on transfer initiate + 502 if unable to find or connect to the remote peer in time +*/ +func (api *WebapiInstance) apiFileRead(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + var err error + + // validate hashes (must be blake3) and other input + fileHash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) + publicKey, err3 := core.PublicKeyFromPeerID(r.Form.Get("node")) + if !valid1 || (!valid2 && err3 != nil) { + http.Error(w, "", http.StatusBadRequest) + return + } + + timeoutSeconds, _ := strconv.Atoi(r.Form.Get("timeout")) + if timeoutSeconds == 0 { + timeoutSeconds = 10 + } + timeout := time.Duration(timeoutSeconds) * time.Second + + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, _ := strconv.Atoi(r.Form.Get("limit")) + + // Range header? + var ranges []HTTPRange + if ranges, err = ParseRangeHeader(r.Header.Get("Range"), -1, true); err != nil || len(ranges) > 1 { + http.Error(w, "", http.StatusBadRequest) + return + } else if len(ranges) == 1 { + if ranges[0].length != -1 { // if length is not specified, limit remains 0 which is maximum + limit = ranges[0].length + } + offset = ranges[0].start + } + + // Is the file available in the local warehouse? In that case requesting it from the remote is unnecessary. + if serveFileFromWarehouse(api.Backend, w, fileHash, uint64(offset), uint64(limit), ranges) { + return + } + + // try connecting via node ID or peer ID? + var peer *core.PeerInfo + + if valid2 { + peer, err = PeerConnectNode(api.Backend, nodeID, timeout) + } else if err3 == nil { + peer, err = PeerConnectPublicKey(api.Backend, publicKey, timeout) + } + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + + // Start the reader. If this HTTP request is canceled, r.Context().Done() acts as cancellation signal to the underlying UDT connection. + reader, fileSize, transferSize, err := FileStartReader(peer, fileHash, uint64(offset), uint64(limit), r.Context().Done()) + if reader != nil { + defer reader.Close() + } + if err != nil || reader == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + // set the right headers + setContentLengthRangeHeader(w, uint64(offset), transferSize, fileSize, ranges) + + // Start sending the data! + io.Copy(w, io.LimitReader(reader, int64(transferSize))) +} + +// serveFileFromWarehouse serves the file from the warehouse. If it is not available, it returns false and does not use the writer. +// Limit is optional, 0 means the entire file. +func serveFileFromWarehouse(backend *core.Backend, w http.ResponseWriter, fileHash []byte, offset, limit uint64, ranges []HTTPRange) (valid bool) { + // Check if the file is available in the local warehouse. + _, fileSize, status, _ := backend.UserWarehouse.FileExists(fileHash) + if status != warehouse.StatusOK { + return false + } + + // validate offset and limit + if limit > 0 && offset+limit > fileSize { + http.Error(w, "invalid limit", http.StatusBadRequest) + return true + } else if offset > fileSize { + http.Error(w, "invalid offset", http.StatusBadRequest) + return true + } else if limit == 0 { + limit = fileSize - offset + } + + setContentLengthRangeHeader(w, offset, limit, fileSize, ranges) + + status, _, _ = backend.UserWarehouse.ReadFile(fileHash, int64(offset), int64(limit), w) + + // StatusErrorReadFile must be considered success, since parts of the file may have been transferred already and recovery is not possible. + return status == warehouse.StatusErrorReadFile || status == warehouse.StatusOK +} + +/* +apiFileView is similar to /file/read but but provides a format parameter. It sets the Content-Type and Accept-Ranges headers. +This endpoint supports the Range, Content-Range and Content-Length headers. Multipart ranges are not supported and result in HTTP 400. +Instead of providing the node ID, the peer ID is also accepted in the &node= parameter. +The default timeout for connecting to the peer is 10 seconds. +Formats: 14 = Video + +Request: GET /file/view?hash=[hash]&node=[node ID]&format=[format] + + Optional: &offset=[offset]&limit=[limit] or via Range header. + Optional: &timeout=[seconds] + +Response: 200 with the content + + 206 with partial content + 400 if the parameters are invalid + 404 if the file was not found or other error on transfer initiate + 502 if unable to find or connect to the remote peer in time +*/ +func (api *WebapiInstance) apiFileView(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + var err error + + // validate hashes (must be blake3) and other input + fileHash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node")) + publicKey, err3 := core.PublicKeyFromPeerID(r.Form.Get("node")) + if !valid1 || (!valid2 && err3 != nil) { + http.Error(w, "", http.StatusBadRequest) + return + } + + timeoutSeconds, _ := strconv.Atoi(r.Form.Get("timeout")) + if timeoutSeconds == 0 { + timeoutSeconds = 10 + } + timeout := time.Duration(timeoutSeconds) * time.Second + + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, _ := strconv.Atoi(r.Form.Get("limit")) + format, _ := strconv.Atoi(r.Form.Get("format")) + localCacheDisable, _ := strconv.ParseBool(r.Form.Get("nocache")) + + // Range header? + var ranges []HTTPRange + if ranges, err = ParseRangeHeader(r.Header.Get("Range"), -1, true); err != nil || len(ranges) > 1 { + http.Error(w, "", http.StatusBadRequest) + return + } else if len(ranges) == 1 { + if ranges[0].length != -1 { // if length is not specified, limit remains 0 which is maximum + limit = ranges[0].length + } + offset = ranges[0].start + } + + w.Header().Set("Accept-Ranges", "bytes") // always indicate accepting of Range header + + switch format { + case 14: + // Video: Indicate MP4 always. There are tons of other MIME types that could be used. + w.Header().Set("Content-Type", "video/mp4") + } + + // Is the file available in the local warehouse? In that case requesting it from the remote is unnecessary. + if !localCacheDisable { + if serveFileFromWarehouse(api.Backend, w, fileHash, uint64(offset), uint64(limit), ranges) { + return + } + } + + // try connecting via node ID or peer ID? + var peer *core.PeerInfo + + if valid2 { + peer, err = PeerConnectNode(api.Backend, nodeID, timeout) + } else if err3 == nil { + peer, err = PeerConnectPublicKey(api.Backend, publicKey, timeout) + } + if err != nil { + w.WriteHeader(http.StatusBadGateway) + return + } + + // start the reader + reader, fileSize, transferSize, err := FileStartReader(peer, fileHash, uint64(offset), uint64(limit), r.Context().Done()) + if reader != nil { + defer reader.Close() + } + if err != nil || reader == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + // set the right headers + setContentLengthRangeHeader(w, uint64(offset), transferSize, fileSize, ranges) + + // Start sending the data! + io.Copy(w, io.LimitReader(reader, int64(transferSize))) +} + +// PeerConnectPublicKey attempts to connect to the peer specified by its public key (= peer ID). +func PeerConnectPublicKey(backend *core.Backend, publicKey *btcec.PublicKey, timeout time.Duration) (peer *core.PeerInfo, err error) { + if publicKey == nil { + return nil, errors.New("invalid public key") + } + + // First look up in the peer list. + if peer = backend.PeerlistLookup(publicKey); peer != nil { + return peer, nil + } + + // Try to connect via DHT. + nodeID := protocol.PublicKey2NodeID(publicKey) + if _, peer, _ = backend.FindNode(nodeID, timeout); peer != nil { + return peer, nil + } + + // otherwise not found :( + return nil, errors.New("peer not found") +} + +// PeerConnectNode tries to connect via the node ID +func PeerConnectNode(backend *core.Backend, nodeID []byte, timeout time.Duration) (peer *core.PeerInfo, err error) { + if len(nodeID) != 256/8 { + return nil, errors.New("invalid node ID") + } + + // Try to connect via DHT. + if _, peer, _ = backend.FindNode(nodeID, timeout); peer != nil { + return peer, nil + } + + // otherwise not found :( + return nil, errors.New("peer not found") +} + +// FileStartReader providers a reader to a remote file. The reader must be closed by the caller. +// File Size is the full file size reported by the remote peer, regardless of the requested offset and limit. Limit is optional (0 means the entire file). +// Transfer Size is the size in bytes that is actually going to be transferred. The reader should be closed after reading that amount. +// The optional cancelChan can be used to stop the file transfer at any point. +func FileStartReader(peer *core.PeerInfo, hash []byte, offset, limit uint64, cancelChan <-chan struct{}) (reader io.ReadCloser, fileSize, transferSize uint64, err error) { + if peer == nil { + return nil, 0, 0, errors.New("peer not provided") + } else if !peer.IsConnectionActive() { + return nil, 0, 0, errors.New("no valid connection to peer") + } + + udtConn, virtualConn, err := peer.FileTransferRequestUDT(hash, offset, limit) + if err != nil { + return nil, 0, 0, err + } + + if cancelChan != nil { + go func() { + <-cancelChan + udtConn.Close() + }() + } + + fileSize, transferSize, err = protocol.FileTransferReadHeader(udtConn) + if err != nil { + udtConn.Close() + return nil, 0, 0, err + } + + virtualConn.Stats.(*core.FileTransferStats).FileSize = fileSize + + return udtConn, fileSize, transferSize, nil +} + +// FileReadAll downloads the file from the peer. +// This function should only be used for testing or as a basis to fork. The caller should develop a custom download function that handles timeouts and excessive file sizes. +// It allocates whatever size is reported by the remote peer. This could lead to an out of memory crash. +// This function is blocking and may take a long time depending on the remote peer and the network connection. +func FileReadAll(peer *core.PeerInfo, hash []byte) (data []byte, err error) { + reader, _, transferSize, err := FileStartReader(peer, hash, 0, 0, nil) + if err != nil { + return nil, err + } + defer reader.Close() + + // read all data + data = make([]byte, transferSize) // Warning: This could lead to an out of memory crash. + _, err = reader.Read(data) + + // Note: This function does not verify if the returned data matches the hash and expected size. + + return data, err +} diff --git a/webapi/File.go b/webapi/File.go index 60765cf..c6f6b0d 100644 --- a/webapi/File.go +++ b/webapi/File.go @@ -1,340 +1,340 @@ -/* -File Name: File.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "net/http" - "time" - - "github.com/PeernetOfficial/core" - "github.com/PeernetOfficial/core/blockchain" - "github.com/PeernetOfficial/core/merkle" - "github.com/PeernetOfficial/core/protocol" - "github.com/PeernetOfficial/core/warehouse" - "github.com/google/uuid" -) - -// apiFileMetadata contains metadata information. -type apiFileMetadata struct { - Type uint16 `json:"type"` // See core.TagX constants. - Name string `json:"name"` // User friendly name of the metadata type. Use the Type fields to identify the metadata as this name may change. - // Depending on the exact type, one of the below fields is used for proper encoding: - Text string `json:"text"` // Text value. UTF-8 encoding. - Blob []byte `json:"blob"` // Binary data - Date time.Time `json:"date"` // Date - Number uint64 `json:"number"` // Number -} - -// apiFile is the metadata of a file published on the blockchain -type apiFile struct { - ID uuid.UUID `json:"id"` // Unique ID. - Hash []byte `json:"hash"` // Blake3 hash of the file data - Type uint8 `json:"type"` // File Type. For example audio or document. See TypeX. - Format uint16 `json:"format"` // File Format. This is more granular, for example PDF or Word file. See FormatX. - Size uint64 `json:"size"` // Size of the file - Folder string `json:"folder"` // Folder, optional - Name string `json:"name"` // Name of the file - Description string `json:"description"` // Description. This is expected to be multiline and contain hashtags! - Date time.Time `json:"date"` // Date shared - NodeID []byte `json:"nodeid"` // Node ID, owner of the file. Read only. - Metadata []apiFileMetadata `json:"metadata"` // Additional metadata. -} - -// --- conversion from core to API data --- - -func blockRecordFileToAPI(input blockchain.BlockRecordFile) (output apiFile) { - output = apiFile{ID: input.ID, Hash: input.Hash, NodeID: input.NodeID, Type: input.Type, Format: input.Format, Size: input.Size, Metadata: []apiFileMetadata{}} - - for _, tag := range input.Tags { - switch tag.Type { - case blockchain.TagName: - output.Name = tag.Text() - - case blockchain.TagFolder: - output.Folder = tag.Text() - - case blockchain.TagDescription: - output.Description = tag.Text() - - case blockchain.TagDateShared: - output.Date, _ = tag.Date() - - case blockchain.TagDateCreated: - date, _ := tag.Date() - output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Date Created", Date: date}) - - case blockchain.TagSharedByCount: - output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By Count", Number: tag.Number()}) - - case blockchain.TagSharedByGeoIP: - output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By GeoIP", Text: tag.Text()}) - - default: - output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Blob: tag.Data}) - } - } - - return output -} - -func blockRecordFileFromAPI(input apiFile) (output blockchain.BlockRecordFile) { - output = blockchain.BlockRecordFile{ID: input.ID, Hash: input.Hash, Type: input.Type, Format: input.Format, Size: input.Size} - - if input.Name != "" { - output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagName, input.Name)) - } - if input.Folder != "" { - output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagFolder, input.Folder)) - } - if input.Description != "" { - output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagDescription, input.Description)) - } - - for _, meta := range input.Metadata { - if blockchain.IsTagVirtual(meta.Type) { // Virtual tags are not mapped back. They are read-only. - continue - } - - switch meta.Type { - case blockchain.TagName, blockchain.TagFolder, blockchain.TagDescription: // auto mapped tags - - case blockchain.TagDateCreated: - output.Tags = append(output.Tags, blockchain.TagFromDate(meta.Type, meta.Date)) - - default: - output.Tags = append(output.Tags, blockchain.BlockRecordFileTag{Type: meta.Type, Data: meta.Blob}) - } - } - - return output -} - -// --- File API --- - -// apiBlockAddFiles contains a list of files from the blockchain -type apiBlockAddFiles struct { - Files []apiFile `json:"files"` // List of files - Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API. -} - -/* -apiBlockchainFileAdd adds a file with the provided information to the blockchain. -Each file must be already stored in the Warehouse (virtual folders are exempt). -If any file is not stored in the Warehouse, the function aborts with the status code StatusNotInWarehouse. -If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord. -In case the function aborts, the blockchain remains unchanged. - -Request: POST /blockchain/file/add with JSON structure apiBlockAddFiles -Response: 200 with JSON structure apiBlockchainBlockStatus - - 400 if invalid input -*/ -func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.Request) { - var input apiBlockAddFiles - if err := DecodeJSON(w, r, &input); err != nil { - return - } - - var filesAdd []blockchain.BlockRecordFile - - for _, file := range input.Files { - if len(file.Hash) != protocol.HashSize { - http.Error(w, "", http.StatusBadRequest) - return - } - if file.ID == uuid.Nil { // if the ID is not provided by the caller, set it - file.ID = uuid.New() - } - - // Verify that the file exists in the warehouse. Folders are exempt from this check as they are only virtual. - if !file.IsVirtualFolder() { - if _, err := warehouse.ValidateHash(file.Hash); err != nil { - http.Error(w, "", http.StatusBadRequest) - return - } else if _, fileSize, status, _ := api.Backend.UserWarehouse.FileExists(file.Hash); status != warehouse.StatusOK { - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) - return - } else { - file.Size = fileSize - } - } else { - file.Hash = protocol.HashData(nil) - file.Size = 0 - } - - blockRecord := blockRecordFileFromAPI(file) - - // Set the merkle tree info as appropriate. - if !setFileMerkleInfo(api.Backend, &blockRecord) { - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) - return - } - - filesAdd = append(filesAdd, blockRecord) - } - - newHeight, newVersion, status := api.Backend.UserBlockchain.AddFiles(filesAdd) - - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) -} - -/* -apiBlockchainFileList lists all files stored on the blockchain. - -Request: GET /blockchain/file/list -Response: 200 with JSON structure apiBlockAddFiles -*/ -func (api *WebapiInstance) apiBlockchainFileList(w http.ResponseWriter, r *http.Request) { - files, status := api.Backend.UserBlockchain.ListFiles() - - var result apiBlockAddFiles - - for _, file := range files { - result.Files = append(result.Files, blockRecordFileToAPI(file)) - } - - result.Status = status - - EncodeJSON(api.Backend, w, r, result) -} - -/* -apiBlockchainFileDelete deletes files with the provided IDs. Other fields are ignored. -It will automatically delete the file in the Warehouse if there are no other references. - -Request: POST /blockchain/file/delete with JSON structure apiBlockAddFiles -Response: 200 with JSON structure apiBlockchainBlockStatus -*/ -func (api *WebapiInstance) apiBlockchainFileDelete(w http.ResponseWriter, r *http.Request) { - var input apiBlockAddFiles - if err := DecodeJSON(w, r, &input); err != nil { - return - } - - var deleteIDs []uuid.UUID - - for n := range input.Files { - deleteIDs = append(deleteIDs, input.Files[n].ID) - } - - newHeight, newVersion, deletedFiles, status := api.Backend.UserBlockchain.DeleteFiles(deleteIDs) - - // If successfully deleted from the blockchain, delete from the Warehouse in case there are no other references. - if status == blockchain.StatusOK { - for n := range deletedFiles { - if files, status := api.Backend.UserBlockchain.FileExists(deletedFiles[n].Hash); status == blockchain.StatusOK && len(files) == 0 { - api.Backend.UserWarehouse.DeleteFile(deletedFiles[n].Hash) - } - } - } - - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) -} - -/* -apiBlockchainSelfUpdateFile updates files that are already published on the blockchain. - -Request: POST /blockchain/file/update with JSON structure apiBlockAddFiles -Response: 200 with JSON structure apiBlockchainBlockStatus - - 400 if invalid input -*/ -func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *http.Request) { - var input apiBlockAddFiles - if err := DecodeJSON(w, r, &input); err != nil { - return - } - - var filesAdd []blockchain.BlockRecordFile - - for _, file := range input.Files { - if len(file.Hash) != protocol.HashSize { - http.Error(w, "", http.StatusBadRequest) - return - } else if file.ID == uuid.Nil { // if the ID is not provided by the caller, abort - http.Error(w, "", http.StatusBadRequest) - return - } - - // Verify that the file exists in the warehouse. Folders are exempt from this check as they are only virtual. - if !file.IsVirtualFolder() { - if _, err := warehouse.ValidateHash(file.Hash); err != nil { - http.Error(w, "", http.StatusBadRequest) - return - } else if _, fileSize, status, _ := api.Backend.UserWarehouse.FileExists(file.Hash); status != warehouse.StatusOK { - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) - return - } else { - file.Size = fileSize - } - } else { - file.Hash = protocol.HashData(nil) - file.Size = 0 - } - - blockRecord := blockRecordFileFromAPI(file) - - // Set the merkle tree info as appropriate. - if !setFileMerkleInfo(api.Backend, &blockRecord) { - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) - return - } - - filesAdd = append(filesAdd, blockRecord) - } - - newHeight, newVersion, status := api.Backend.UserBlockchain.ReplaceFiles(filesAdd) - - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) -} - -// ---- metadata functions ---- - -// GetMetadata returns the specified metadata or nil if not available. -func (file *apiFile) GetMetadata(Type uint16) (info *apiFileMetadata) { - for n := range file.Metadata { - if file.Metadata[n].Type == Type { - return &file.Metadata[n] - } - } - - return nil -} - -// GetNumber returns the data as number. 0 if not available. -func (info *apiFileMetadata) GetNumber() uint64 { - if info == nil { - return 0 - } - - return info.Number -} - -// IsVirtualFolder returns true if the file is a virtual folder -func (file *apiFile) IsVirtualFolder() bool { - return file.Type == core.TypeFolder && file.Format == core.FormatFolder -} - -// setFileMerkleInfo sets the merkle fields in the BlockRecordFile -func setFileMerkleInfo(backend *core.Backend, file *blockchain.BlockRecordFile) (valid bool) { - if file.Size <= merkle.MinimumFragmentSize { - // If smaller or equal than the minimum fragment size, the merkle tree is not used. - file.MerkleRootHash = file.Hash - file.FragmentSize = merkle.MinimumFragmentSize - } else { - // Get the information from the Warehouse .merkle companion file. - tree, status, _ := backend.UserWarehouse.ReadMerkleTree(file.Hash, true) - if status != warehouse.StatusOK { - return false - } - - file.MerkleRootHash = tree.RootHash - file.FragmentSize = tree.FragmentSize - } - - return true -} +/* +File Name: File.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "net/http" + "time" + + "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" + "github.com/PeernetOfficial/core/merkle" + "github.com/PeernetOfficial/core/protocol" + "github.com/PeernetOfficial/core/warehouse" + "github.com/google/uuid" +) + +// apiFileMetadata contains metadata information. +type apiFileMetadata struct { + Type uint16 `json:"type"` // See core.TagX constants. + Name string `json:"name"` // User friendly name of the metadata type. Use the Type fields to identify the metadata as this name may change. + // Depending on the exact type, one of the below fields is used for proper encoding: + Text string `json:"text"` // Text value. UTF-8 encoding. + Blob []byte `json:"blob"` // Binary data + Date time.Time `json:"date"` // Date + Number uint64 `json:"number"` // Number +} + +// apiFile is the metadata of a file published on the blockchain +type apiFile struct { + ID uuid.UUID `json:"id"` // Unique ID. + Hash []byte `json:"hash"` // Blake3 hash of the file data + Type uint8 `json:"type"` // File Type. For example audio or document. See TypeX. + Format uint16 `json:"format"` // File Format. This is more granular, for example PDF or Word file. See FormatX. + Size uint64 `json:"size"` // Size of the file + Folder string `json:"folder"` // Folder, optional + Name string `json:"name"` // Name of the file + Description string `json:"description"` // Description. This is expected to be multiline and contain hashtags! + Date time.Time `json:"date"` // Date shared + NodeID []byte `json:"nodeid"` // Node ID, owner of the file. Read only. + Metadata []apiFileMetadata `json:"metadata"` // Additional metadata. +} + +// --- conversion from core to API data --- + +func blockRecordFileToAPI(input blockchain.BlockRecordFile) (output apiFile) { + output = apiFile{ID: input.ID, Hash: input.Hash, NodeID: input.NodeID, Type: input.Type, Format: input.Format, Size: input.Size, Metadata: []apiFileMetadata{}} + + for _, tag := range input.Tags { + switch tag.Type { + case blockchain.TagName: + output.Name = tag.Text() + + case blockchain.TagFolder: + output.Folder = tag.Text() + + case blockchain.TagDescription: + output.Description = tag.Text() + + case blockchain.TagDateShared: + output.Date, _ = tag.Date() + + case blockchain.TagDateCreated: + date, _ := tag.Date() + output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Date Created", Date: date}) + + case blockchain.TagSharedByCount: + output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By Count", Number: tag.Number()}) + + case blockchain.TagSharedByGeoIP: + output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Name: "Shared By GeoIP", Text: tag.Text()}) + + default: + output.Metadata = append(output.Metadata, apiFileMetadata{Type: tag.Type, Blob: tag.Data}) + } + } + + return output +} + +func blockRecordFileFromAPI(input apiFile) (output blockchain.BlockRecordFile) { + output = blockchain.BlockRecordFile{ID: input.ID, Hash: input.Hash, Type: input.Type, Format: input.Format, Size: input.Size} + + if input.Name != "" { + output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagName, input.Name)) + } + if input.Folder != "" { + output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagFolder, input.Folder)) + } + if input.Description != "" { + output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagDescription, input.Description)) + } + + for _, meta := range input.Metadata { + if blockchain.IsTagVirtual(meta.Type) { // Virtual tags are not mapped back. They are read-only. + continue + } + + switch meta.Type { + case blockchain.TagName, blockchain.TagFolder, blockchain.TagDescription: // auto mapped tags + + case blockchain.TagDateCreated: + output.Tags = append(output.Tags, blockchain.TagFromDate(meta.Type, meta.Date)) + + default: + output.Tags = append(output.Tags, blockchain.BlockRecordFileTag{Type: meta.Type, Data: meta.Blob}) + } + } + + return output +} + +// --- File API --- + +// apiBlockAddFiles contains a list of files from the blockchain +type apiBlockAddFiles struct { + Files []apiFile `json:"files"` // List of files + Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API. +} + +/* +apiBlockchainFileAdd adds a file with the provided information to the blockchain. +Each file must be already stored in the Warehouse (virtual folders are exempt). +If any file is not stored in the Warehouse, the function aborts with the status code StatusNotInWarehouse. +If the block record encoding fails for any file, this function aborts with the status code StatusCorruptBlockRecord. +In case the function aborts, the blockchain remains unchanged. + +Request: POST /blockchain/file/add with JSON structure apiBlockAddFiles +Response: 200 with JSON structure apiBlockchainBlockStatus + + 400 if invalid input +*/ +func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.Request) { + var input apiBlockAddFiles + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var filesAdd []blockchain.BlockRecordFile + + for _, file := range input.Files { + if len(file.Hash) != protocol.HashSize { + http.Error(w, "", http.StatusBadRequest) + return + } + if file.ID == uuid.Nil { // if the ID is not provided by the caller, set it + file.ID = uuid.New() + } + + // Verify that the file exists in the warehouse. Folders are exempt from this check as they are only virtual. + if !file.IsVirtualFolder() { + if _, err := warehouse.ValidateHash(file.Hash); err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } else if _, fileSize, status, _ := api.Backend.UserWarehouse.FileExists(file.Hash); status != warehouse.StatusOK { + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return + } else { + file.Size = fileSize + } + } else { + file.Hash = protocol.HashData(nil) + file.Size = 0 + } + + blockRecord := blockRecordFileFromAPI(file) + + // Set the merkle tree info as appropriate. + if !setFileMerkleInfo(api.Backend, &blockRecord) { + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return + } + + filesAdd = append(filesAdd, blockRecord) + } + + newHeight, newVersion, status := api.Backend.UserBlockchain.AddFiles(filesAdd) + + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +/* +apiBlockchainFileList lists all files stored on the blockchain. + +Request: GET /blockchain/file/list +Response: 200 with JSON structure apiBlockAddFiles +*/ +func (api *WebapiInstance) apiBlockchainFileList(w http.ResponseWriter, r *http.Request) { + files, status := api.Backend.UserBlockchain.ListFiles() + + var result apiBlockAddFiles + + for _, file := range files { + result.Files = append(result.Files, blockRecordFileToAPI(file)) + } + + result.Status = status + + EncodeJSON(api.Backend, w, r, result) +} + +/* +apiBlockchainFileDelete deletes files with the provided IDs. Other fields are ignored. +It will automatically delete the file in the Warehouse if there are no other references. + +Request: POST /blockchain/file/delete with JSON structure apiBlockAddFiles +Response: 200 with JSON structure apiBlockchainBlockStatus +*/ +func (api *WebapiInstance) apiBlockchainFileDelete(w http.ResponseWriter, r *http.Request) { + var input apiBlockAddFiles + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var deleteIDs []uuid.UUID + + for n := range input.Files { + deleteIDs = append(deleteIDs, input.Files[n].ID) + } + + newHeight, newVersion, deletedFiles, status := api.Backend.UserBlockchain.DeleteFiles(deleteIDs) + + // If successfully deleted from the blockchain, delete from the Warehouse in case there are no other references. + if status == blockchain.StatusOK { + for n := range deletedFiles { + if files, status := api.Backend.UserBlockchain.FileExists(deletedFiles[n].Hash); status == blockchain.StatusOK && len(files) == 0 { + api.Backend.UserWarehouse.DeleteFile(deletedFiles[n].Hash) + } + } + } + + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +/* +apiBlockchainSelfUpdateFile updates files that are already published on the blockchain. + +Request: POST /blockchain/file/update with JSON structure apiBlockAddFiles +Response: 200 with JSON structure apiBlockchainBlockStatus + + 400 if invalid input +*/ +func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *http.Request) { + var input apiBlockAddFiles + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var filesAdd []blockchain.BlockRecordFile + + for _, file := range input.Files { + if len(file.Hash) != protocol.HashSize { + http.Error(w, "", http.StatusBadRequest) + return + } else if file.ID == uuid.Nil { // if the ID is not provided by the caller, abort + http.Error(w, "", http.StatusBadRequest) + return + } + + // Verify that the file exists in the warehouse. Folders are exempt from this check as they are only virtual. + if !file.IsVirtualFolder() { + if _, err := warehouse.ValidateHash(file.Hash); err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } else if _, fileSize, status, _ := api.Backend.UserWarehouse.FileExists(file.Hash); status != warehouse.StatusOK { + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return + } else { + file.Size = fileSize + } + } else { + file.Hash = protocol.HashData(nil) + file.Size = 0 + } + + blockRecord := blockRecordFileFromAPI(file) + + // Set the merkle tree info as appropriate. + if !setFileMerkleInfo(api.Backend, &blockRecord) { + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse}) + return + } + + filesAdd = append(filesAdd, blockRecord) + } + + newHeight, newVersion, status := api.Backend.UserBlockchain.ReplaceFiles(filesAdd) + + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +// ---- metadata functions ---- + +// GetMetadata returns the specified metadata or nil if not available. +func (file *apiFile) GetMetadata(Type uint16) (info *apiFileMetadata) { + for n := range file.Metadata { + if file.Metadata[n].Type == Type { + return &file.Metadata[n] + } + } + + return nil +} + +// GetNumber returns the data as number. 0 if not available. +func (info *apiFileMetadata) GetNumber() uint64 { + if info == nil { + return 0 + } + + return info.Number +} + +// IsVirtualFolder returns true if the file is a virtual folder +func (file *apiFile) IsVirtualFolder() bool { + return file.Type == core.TypeFolder && file.Format == core.FormatFolder +} + +// setFileMerkleInfo sets the merkle fields in the BlockRecordFile +func setFileMerkleInfo(backend *core.Backend, file *blockchain.BlockRecordFile) (valid bool) { + if file.Size <= merkle.MinimumFragmentSize { + // If smaller or equal than the minimum fragment size, the merkle tree is not used. + file.MerkleRootHash = file.Hash + file.FragmentSize = merkle.MinimumFragmentSize + } else { + // Get the information from the Warehouse .merkle companion file. + tree, status, _ := backend.UserWarehouse.ReadMerkleTree(file.Hash, true) + if status != warehouse.StatusOK { + return false + } + + file.MerkleRootHash = tree.RootHash + file.FragmentSize = tree.FragmentSize + } + + return true +} diff --git a/webapi/GeoIP.go b/webapi/GeoIP.go index aa551fb..3e19b94 100644 --- a/webapi/GeoIP.go +++ b/webapi/GeoIP.go @@ -1,50 +1,50 @@ -/* -File Name: GeoIP.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner - -Support for the free MaxMind database 'GeoLite2 City'. -Information about the database: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data - -Potential libraries: -* https://github.com/IncSW/geoip2 -* https://github.com/oschwald/maxminddb-golang - -The IncSW lib was chosen because it has 0 dependencies - awesome! - -*/ - -package webapi - -import ( - "net" - - "github.com/IncSW/geoip2" - "github.com/PeernetOfficial/core" -) - -func (api *WebapiInstance) InitGeoIPDatabase(filename string) (err error) { - api.geoipCityReader, err = geoip2.NewCityReaderFromFile(filename) - return err -} - -func (api *WebapiInstance) GeoIPLocation(IP net.IP) (latitude, longitude float64, valid bool) { - if api.geoipCityReader == nil { - return 0, 0, false - } - - record, err := api.geoipCityReader.Lookup(IP) - if err != nil { - return 0, 0, false - } - - return record.Location.Latitude, record.Location.Longitude, true -} - -func (api *WebapiInstance) Peer2GeoIP(peer *core.PeerInfo) (latitude, longitude float64, valid bool) { - if connection := peer.GetConnection2Share(false, true, true); connection != nil { - return api.GeoIPLocation(connection.Address.IP) - } - - return 0, 0, false -} +/* +File Name: GeoIP.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner + +Support for the free MaxMind database 'GeoLite2 City'. +Information about the database: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data + +Potential libraries: +* https://github.com/IncSW/geoip2 +* https://github.com/oschwald/maxminddb-golang + +The IncSW lib was chosen because it has 0 dependencies - awesome! + +*/ + +package webapi + +import ( + "net" + + "github.com/IncSW/geoip2" + "github.com/PeernetOfficial/core" +) + +func (api *WebapiInstance) InitGeoIPDatabase(filename string) (err error) { + api.geoipCityReader, err = geoip2.NewCityReaderFromFile(filename) + return err +} + +func (api *WebapiInstance) GeoIPLocation(IP net.IP) (latitude, longitude float64, valid bool) { + if api.geoipCityReader == nil { + return 0, 0, false + } + + record, err := api.geoipCityReader.Lookup(IP) + if err != nil { + return 0, 0, false + } + + return record.Location.Latitude, record.Location.Longitude, true +} + +func (api *WebapiInstance) Peer2GeoIP(peer *core.PeerInfo) (latitude, longitude float64, valid bool) { + if connection := peer.GetConnection2Share(false, true, true); connection != nil { + return api.GeoIPLocation(connection.Address.IP) + } + + return 0, 0, false +} diff --git a/webapi/HTTP Range.go b/webapi/HTTP Range.go index 6dbf473..cb8985f 100644 --- a/webapi/HTTP Range.go +++ b/webapi/HTTP Range.go @@ -1,109 +1,109 @@ -/* -File Name: HTTP Range.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "errors" - "net/http" - "net/textproto" - "strconv" - "strings" -) - -// Fork from https://golang.org/src/net/http/fs.go - -// HTTPRange represents an HTTP range -type HTTPRange struct { - start, length int -} - -// ParseRangeHeader parses a Range header string as per RFC 7233. -func ParseRangeHeader(s string, size int, noMultiRange bool) ([]HTTPRange, error) { - if s == "" { - return nil, nil // header not present - } - const b = "bytes=" - if !strings.HasPrefix(s, b) { - return nil, errors.New("invalid range") - } - var ranges []HTTPRange - for _, ra := range strings.Split(s[len(b):], ",") { - ra = textproto.TrimString(ra) - if ra == "" { - continue - } - i := strings.Index(ra, "-") - if i < 0 { - return nil, errors.New("invalid range") - } - start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:]) - var r HTTPRange - if start == "" { - if size < 0 { - return nil, errors.New("range start relative to end not supported") - } - // If no start is specified, end specifies the - // range start relative to the end of the file. - i, err := strconv.ParseInt(end, 10, 64) - if err != nil { - return nil, errors.New("invalid range") - } - if int(i) > size { - i = int64(size) - } - r.start = size - int(i) - r.length = size - r.start - } else { - i, err := strconv.ParseInt(start, 10, 64) - if err != nil || i < 0 { - return nil, errors.New("invalid range") - } - if size > 0 && int(i) >= size { - // If the range begins after the size of the content, then it does not overlap. -> always return error. - return nil, errors.New("start out of range") - } - r.start = int(i) - if end == "" { - // If no end is specified, range extends to end of the file. - if size < 0 { - //return nil, errors.New("open range not supported") - r.length = -1 - } else { - r.length = size - r.start - } - } else { - i, err := strconv.ParseInt(end, 10, 64) - if err != nil || r.start > int(i) { - return nil, errors.New("invalid range") - } - if size > 0 && int(i) >= size { - i = int64(size - 1) - } - r.length = int(i) - r.start + 1 - } - } - ranges = append(ranges, r) - if noMultiRange && len(ranges) > 1 { - return nil, errors.New("multiple ranges not supported") - } - } - return ranges, nil -} - -// setContentLengthRangeHeader sets the appropriate Content-Length and Content-Range headers -func setContentLengthRangeHeader(w http.ResponseWriter, offset, transferSize, fileSize uint64, ranges []HTTPRange) { - // Set the Content-Length header, always to the actual size of transferred data. - w.Header().Set("Content-Length", strconv.FormatUint(transferSize, 10)) - - // Set the Content-Range header if needed. - if len(ranges) == 1 { - w.Header().Set("Content-Range", "bytes "+strconv.FormatUint(offset, 10)+"-"+strconv.FormatUint(offset+transferSize-1, 10)+"/"+strconv.FormatUint(fileSize, 10)) - w.WriteHeader(http.StatusPartialContent) - } else { - w.WriteHeader(http.StatusOK) - } -} +/* +File Name: HTTP Range.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "errors" + "net/http" + "net/textproto" + "strconv" + "strings" +) + +// Fork from https://golang.org/src/net/http/fs.go + +// HTTPRange represents an HTTP range +type HTTPRange struct { + start, length int +} + +// ParseRangeHeader parses a Range header string as per RFC 7233. +func ParseRangeHeader(s string, size int, noMultiRange bool) ([]HTTPRange, error) { + if s == "" { + return nil, nil // header not present + } + const b = "bytes=" + if !strings.HasPrefix(s, b) { + return nil, errors.New("invalid range") + } + var ranges []HTTPRange + for _, ra := range strings.Split(s[len(b):], ",") { + ra = textproto.TrimString(ra) + if ra == "" { + continue + } + i := strings.Index(ra, "-") + if i < 0 { + return nil, errors.New("invalid range") + } + start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:]) + var r HTTPRange + if start == "" { + if size < 0 { + return nil, errors.New("range start relative to end not supported") + } + // If no start is specified, end specifies the + // range start relative to the end of the file. + i, err := strconv.ParseInt(end, 10, 64) + if err != nil { + return nil, errors.New("invalid range") + } + if int(i) > size { + i = int64(size) + } + r.start = size - int(i) + r.length = size - r.start + } else { + i, err := strconv.ParseInt(start, 10, 64) + if err != nil || i < 0 { + return nil, errors.New("invalid range") + } + if size > 0 && int(i) >= size { + // If the range begins after the size of the content, then it does not overlap. -> always return error. + return nil, errors.New("start out of range") + } + r.start = int(i) + if end == "" { + // If no end is specified, range extends to end of the file. + if size < 0 { + //return nil, errors.New("open range not supported") + r.length = -1 + } else { + r.length = size - r.start + } + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || r.start > int(i) { + return nil, errors.New("invalid range") + } + if size > 0 && int(i) >= size { + i = int64(size - 1) + } + r.length = int(i) - r.start + 1 + } + } + ranges = append(ranges, r) + if noMultiRange && len(ranges) > 1 { + return nil, errors.New("multiple ranges not supported") + } + } + return ranges, nil +} + +// setContentLengthRangeHeader sets the appropriate Content-Length and Content-Range headers +func setContentLengthRangeHeader(w http.ResponseWriter, offset, transferSize, fileSize uint64, ranges []HTTPRange) { + // Set the Content-Length header, always to the actual size of transferred data. + w.Header().Set("Content-Length", strconv.FormatUint(transferSize, 10)) + + // Set the Content-Range header if needed. + if len(ranges) == 1 { + w.Header().Set("Content-Range", "bytes "+strconv.FormatUint(offset, 10)+"-"+strconv.FormatUint(offset+transferSize-1, 10)+"/"+strconv.FormatUint(fileSize, 10)) + w.WriteHeader(http.StatusPartialContent) + } else { + w.WriteHeader(http.StatusOK) + } +} diff --git a/webapi/Profile.go b/webapi/Profile.go index be5ac1a..683b696 100644 --- a/webapi/Profile.go +++ b/webapi/Profile.go @@ -1,153 +1,153 @@ -/* -File Name: Profile.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "net/http" - "strconv" - - "github.com/PeernetOfficial/core/blockchain" -) - -// apiProfileData contains profile metadata stored on the blockchain. Any data is treated as untrusted and unverified by default. -type apiProfileData struct { - Fields []apiBlockRecordProfile `json:"fields"` // All fields - Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API. See blockchain.StatusX. -} - -// apiBlockRecordProfile provides information about the end user. Note that all profile data is arbitrary and shall be considered untrusted and unverified. -// To establish trust, the user must load Certificates into the blockchain that validate certain data. -type apiBlockRecordProfile struct { - Type uint16 `json:"type"` // See ProfileX constants. - // Depending on the exact type, one of the below fields is used for proper encoding: - Text string `json:"text"` // Text value. UTF-8 encoding. - Blob []byte `json:"blob"` // Binary data -} - -/* -apiProfileList lists all users profile fields. - -Request: GET /profile/list -Response: 200 with JSON structure apiProfileData -*/ -func (api *WebapiInstance) apiProfileList(w http.ResponseWriter, r *http.Request) { - fields, status := api.Backend.UserBlockchain.ProfileList() - - result := apiProfileData{Status: status} - for n := range fields { - result.Fields = append(result.Fields, blockRecordProfileToAPI(fields[n])) - } - - EncodeJSON(api.Backend, w, r, result) -} - -/* -apiProfileRead reads a specific users profile field. See core.ProfileX for recognized fields. - -Request: GET /profile/read?field=[index] -Response: 200 with JSON structure apiProfileData -*/ -func (api *WebapiInstance) apiProfileRead(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - fieldN, err1 := strconv.Atoi(r.Form.Get("field")) - - if err1 != nil || fieldN < 0 { - http.Error(w, "", http.StatusBadRequest) - return - } - - var result apiProfileData - - var data []byte - if data, result.Status = api.Backend.UserBlockchain.ProfileReadField(uint16(fieldN)); result.Status == blockchain.StatusOK { - result.Fields = append(result.Fields, blockRecordProfileToAPI(blockchain.BlockRecordProfile{Type: uint16(fieldN), Data: data})) - } - - EncodeJSON(api.Backend, w, r, result) -} - -/* -apiProfileWrite writes profile fields. See core.ProfileX for recognized fields. - -Request: POST /profile/write with JSON structure apiProfileData -Response: 200 with JSON structure apiBlockchainBlockStatus -*/ -func (api *WebapiInstance) apiProfileWrite(w http.ResponseWriter, r *http.Request) { - var input apiProfileData - if err := DecodeJSON(w, r, &input); err != nil { - return - } - - var fields []blockchain.BlockRecordProfile - - for n := range input.Fields { - fields = append(fields, blockRecordProfileFromAPI(input.Fields[n])) - } - - newHeight, newVersion, status := api.Backend.UserBlockchain.ProfileWrite(fields) - - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) -} - -/* -apiProfileDelete deletes profile fields identified by the types. See core.ProfileX for recognized fields. - -Request: POST /profile/delete with JSON structure apiProfileData -Response: 200 with JSON structure apiBlockchainBlockStatus -*/ -func (api *WebapiInstance) apiProfileDelete(w http.ResponseWriter, r *http.Request) { - var input apiProfileData - if err := DecodeJSON(w, r, &input); err != nil { - return - } - - var fields []uint16 - - for n := range input.Fields { - fields = append(fields, input.Fields[n].Type) - } - - newHeight, newVersion, status := api.Backend.UserBlockchain.ProfileDelete(fields) - - EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) -} - -// --- conversion from core to API data --- - -func blockRecordProfileToAPI(input blockchain.BlockRecordProfile) (output apiBlockRecordProfile) { - output.Type = input.Type - - switch input.Type { - case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress: - output.Text = input.Text() - - case blockchain.ProfilePicture: - output.Blob = input.Data - - default: - output.Blob = input.Data - } - - return output -} - -func blockRecordProfileFromAPI(input apiBlockRecordProfile) (output blockchain.BlockRecordProfile) { - output.Type = input.Type - - switch input.Type { - case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress: - output.Data = []byte(input.Text) - - case blockchain.ProfilePicture: - output.Data = input.Blob - - default: - output.Data = input.Blob - } - - return output -} +/* +File Name: Profile.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "net/http" + "strconv" + + "github.com/PeernetOfficial/core/blockchain" +) + +// apiProfileData contains profile metadata stored on the blockchain. Any data is treated as untrusted and unverified by default. +type apiProfileData struct { + Fields []apiBlockRecordProfile `json:"fields"` // All fields + Status int `json:"status"` // Status of the operation, only used when this structure is returned from the API. See blockchain.StatusX. +} + +// apiBlockRecordProfile provides information about the end user. Note that all profile data is arbitrary and shall be considered untrusted and unverified. +// To establish trust, the user must load Certificates into the blockchain that validate certain data. +type apiBlockRecordProfile struct { + Type uint16 `json:"type"` // See ProfileX constants. + // Depending on the exact type, one of the below fields is used for proper encoding: + Text string `json:"text"` // Text value. UTF-8 encoding. + Blob []byte `json:"blob"` // Binary data +} + +/* +apiProfileList lists all users profile fields. + +Request: GET /profile/list +Response: 200 with JSON structure apiProfileData +*/ +func (api *WebapiInstance) apiProfileList(w http.ResponseWriter, r *http.Request) { + fields, status := api.Backend.UserBlockchain.ProfileList() + + result := apiProfileData{Status: status} + for n := range fields { + result.Fields = append(result.Fields, blockRecordProfileToAPI(fields[n])) + } + + EncodeJSON(api.Backend, w, r, result) +} + +/* +apiProfileRead reads a specific users profile field. See core.ProfileX for recognized fields. + +Request: GET /profile/read?field=[index] +Response: 200 with JSON structure apiProfileData +*/ +func (api *WebapiInstance) apiProfileRead(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + fieldN, err1 := strconv.Atoi(r.Form.Get("field")) + + if err1 != nil || fieldN < 0 { + http.Error(w, "", http.StatusBadRequest) + return + } + + var result apiProfileData + + var data []byte + if data, result.Status = api.Backend.UserBlockchain.ProfileReadField(uint16(fieldN)); result.Status == blockchain.StatusOK { + result.Fields = append(result.Fields, blockRecordProfileToAPI(blockchain.BlockRecordProfile{Type: uint16(fieldN), Data: data})) + } + + EncodeJSON(api.Backend, w, r, result) +} + +/* +apiProfileWrite writes profile fields. See core.ProfileX for recognized fields. + +Request: POST /profile/write with JSON structure apiProfileData +Response: 200 with JSON structure apiBlockchainBlockStatus +*/ +func (api *WebapiInstance) apiProfileWrite(w http.ResponseWriter, r *http.Request) { + var input apiProfileData + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var fields []blockchain.BlockRecordProfile + + for n := range input.Fields { + fields = append(fields, blockRecordProfileFromAPI(input.Fields[n])) + } + + newHeight, newVersion, status := api.Backend.UserBlockchain.ProfileWrite(fields) + + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +/* +apiProfileDelete deletes profile fields identified by the types. See core.ProfileX for recognized fields. + +Request: POST /profile/delete with JSON structure apiProfileData +Response: 200 with JSON structure apiBlockchainBlockStatus +*/ +func (api *WebapiInstance) apiProfileDelete(w http.ResponseWriter, r *http.Request) { + var input apiProfileData + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + var fields []uint16 + + for n := range input.Fields { + fields = append(fields, input.Fields[n].Type) + } + + newHeight, newVersion, status := api.Backend.UserBlockchain.ProfileDelete(fields) + + EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion}) +} + +// --- conversion from core to API data --- + +func blockRecordProfileToAPI(input blockchain.BlockRecordProfile) (output apiBlockRecordProfile) { + output.Type = input.Type + + switch input.Type { + case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress: + output.Text = input.Text() + + case blockchain.ProfilePicture: + output.Blob = input.Data + + default: + output.Blob = input.Data + } + + return output +} + +func blockRecordProfileFromAPI(input apiBlockRecordProfile) (output blockchain.BlockRecordProfile) { + output.Type = input.Type + + switch input.Type { + case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress: + output.Data = []byte(input.Text) + + case blockchain.ProfilePicture: + output.Data = input.Blob + + default: + output.Data = input.Blob + } + + return output +} diff --git a/webapi/Search Dispatch.go b/webapi/Search Dispatch.go index 4c22ecf..a971446 100644 --- a/webapi/Search Dispatch.go +++ b/webapi/Search Dispatch.go @@ -1,83 +1,83 @@ -/* -File Name: Search Dispatch.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "bytes" - "fmt" - "time" - - "github.com/PeernetOfficial/core/blockchain" -) - -func (api *WebapiInstance) dispatchSearch(input SearchRequest) (job *SearchJob) { - Timeout := input.Parse() - Filter := input.ToSearchFilter() - - // create the search job - job = api.CreateSearchJob(Timeout, input.MaxResults, Filter) - - // todo: create actual search clients! - job.Status = SearchStatusLive - - go job.localSearch(api, input.Term) - - api.RemoveJobDefer(job, job.timeout+time.Minute*10) - - return job -} - -func (job *SearchJob) localSearch(api *WebapiInstance, term string) { - if api.Backend.SearchIndex == nil { - job.Status = SearchStatusNoIndex - return - } - - results := api.Backend.SearchIndex.Search(term) - - job.ResultSync.Lock() - -resultLoop: - for _, result := range results { - file, _, found, err := api.Backend.ReadFile(result.PublicKey, result.BlockchainVersion, result.BlockNumber, result.FileID) - if err != nil || !found { - continue - } - - // Deduplicate based on file hash from the same peer. - for n := range job.AllFiles { - if bytes.Equal(job.AllFiles[n].Hash, file.Hash) && bytes.Equal(job.AllFiles[n].NodeID, file.NodeID) { - continue resultLoop - } - } - - if bytes.Equal(file.NodeID, api.Backend.SelfNodeID()) { - // Indicates data from the current user. - file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1)) - } else if peer := api.Backend.NodelistLookup(file.NodeID); peer != nil { - // add the tags 'Shared By Count' and 'Shared By GeoIP' - file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1)) - if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { - sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) - file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP)) - } - } - - // new result - newFile := blockRecordFileToAPI(file) - - job.Files = append(job.Files, &newFile) - job.AllFiles = append(job.AllFiles, &newFile) - job.requireSort = true - job.statsAdd(&newFile) - } - - job.Status = SearchStatusTerminated - - job.ResultSync.Unlock() - job.Terminate() -} +/* +File Name: Search Dispatch.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "bytes" + "fmt" + "time" + + "github.com/PeernetOfficial/core/blockchain" +) + +func (api *WebapiInstance) dispatchSearch(input SearchRequest) (job *SearchJob) { + Timeout := input.Parse() + Filter := input.ToSearchFilter() + + // create the search job + job = api.CreateSearchJob(Timeout, input.MaxResults, Filter) + + // todo: create actual search clients! + job.Status = SearchStatusLive + + go job.localSearch(api, input.Term) + + api.RemoveJobDefer(job, job.timeout+time.Minute*10) + + return job +} + +func (job *SearchJob) localSearch(api *WebapiInstance, term string) { + if api.Backend.SearchIndex == nil { + job.Status = SearchStatusNoIndex + return + } + + results := api.Backend.SearchIndex.Search(term) + + job.ResultSync.Lock() + +resultLoop: + for _, result := range results { + file, _, found, err := api.Backend.ReadFile(result.PublicKey, result.BlockchainVersion, result.BlockNumber, result.FileID) + if err != nil || !found { + continue + } + + // Deduplicate based on file hash from the same peer. + for n := range job.AllFiles { + if bytes.Equal(job.AllFiles[n].Hash, file.Hash) && bytes.Equal(job.AllFiles[n].NodeID, file.NodeID) { + continue resultLoop + } + } + + if bytes.Equal(file.NodeID, api.Backend.SelfNodeID()) { + // Indicates data from the current user. + file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1)) + } else if peer := api.Backend.NodelistLookup(file.NodeID); peer != nil { + // add the tags 'Shared By Count' and 'Shared By GeoIP' + file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1)) + if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { + sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) + file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP)) + } + } + + // new result + newFile := blockRecordFileToAPI(file) + + job.Files = append(job.Files, &newFile) + job.AllFiles = append(job.AllFiles, &newFile) + job.requireSort = true + job.statsAdd(&newFile) + } + + job.Status = SearchStatusTerminated + + job.ResultSync.Unlock() + job.Terminate() +} diff --git a/webapi/Search Job.go b/webapi/Search Job.go index ffdf5cf..1f24435 100644 --- a/webapi/Search Job.go +++ b/webapi/Search Job.go @@ -1,469 +1,469 @@ -/* -File Name: Search Job.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "sort" - "sync" - "time" - - "github.com/PeernetOfficial/core/blockchain" - "github.com/google/uuid" -) - -// SearchFilter allows to filter search results based on the criteria. -type SearchFilter struct { - IsDates bool // Whether the from/to dates are valid, both are required. - DateFrom time.Time // Optional date from - DateTo time.Time // Optional date to - FileType int // File type such as binary, text document etc. See core.TypeX. -1 = not used. - FileFormat int // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. - Sort int // Sort order. See SortX. - SizeMin int // Min file size in bytes. -1 = not used. - SizeMax int // Max file size in bytes. -1 = not used. -} - -// SearchJob is a collection of search jobs -type SearchJob struct { - // input settings - id uuid.UUID // The job id - timeout time.Duration // timeout set for all searches - maxResult int // max results user-facing. - - filtersStart SearchFilter // Filters when starting the search. They cannot be changed later on. Any incoming file is checked against them, even if there are different runtime filters. - filtersRuntime SearchFilter // Runtime Filters. They allow filtering results after they were received. - - // File statistics (filters are ignored) of returned results. Map value is always count of files. - stats struct { - sync.RWMutex // Synced access to maps - date map[time.Time]int // Files per day (rounded down to midnight) - fileType map[uint8]int // Files per File Type - fileFormat map[uint16]int // Files per File Format - total int // Total count of files - } - - // -- result data -- - - // Status indicates the overall search status. This will be removed later when relying on search clients. - Status int - - // runtime data - //clients []*SearchClient // all search clients - clientsMutex sync.Mutex // mutex for manipulating client list - - // List of files found but not yet returned via API to the caller. They are subject to sorting. - Files []*apiFile - requireSort bool // if Files requires sort before returning the results - - // FreezeFiles is a list of files that were already finally delivered via the API. They may NOT change in sorting. - FreezeFiles []*apiFile - - // List of all files. Does not change based on sorting or runtime filters. This list only gets expanded. - AllFiles []*apiFile - - ResultSync sync.Mutex // ResultSync ensures unique access to the file results - - currentOffset int // for always getting the next results -} - -const ( - SearchStatusNotStarted = iota // Search was not yet started - SearchStatusLive // Search running - SearchStatusTerminated // Search is terminated. No more results are expected. - SearchStatusNoIndex // Search is terminated. No search index to use. -) - -// CreateSearchJob creates a new search job and adds it to the lookup list. -// Timeout and MaxResults must be set and must not be 0. -func (api *WebapiInstance) CreateSearchJob(Timeout time.Duration, MaxResults int, Filter SearchFilter) (job *SearchJob) { - job = &SearchJob{} - job.Status = SearchStatusNotStarted - job.id = uuid.New() - job.timeout = Timeout - job.maxResult = MaxResults - job.filtersStart = Filter - job.filtersRuntime = Filter // initialize the runtime filters as the same - - job.stats.date = make(map[time.Time]int) - job.stats.fileType = make(map[uint8]int) - job.stats.fileFormat = make(map[uint16]int) - - // add to the list of jobs - api.allJobsMutex.Lock() - api.allJobs[job.id] = job - api.allJobsMutex.Unlock() - - return -} - -// ReturnResult returns the selected results. -func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*apiFile) { - if Limit == 0 { - return Result - } - - job.ResultSync.Lock() - defer job.ResultSync.Unlock() - - // serve files from frozen list? - if Offset < len(job.FreezeFiles) { - countCopy := len(job.FreezeFiles) - Offset - if countCopy > Limit { - countCopy = Limit - } - Result = job.FreezeFiles[Offset : Offset+countCopy] - Limit -= countCopy - Offset = 0 - } else { - Offset -= len(job.FreezeFiles) - } - - if Limit == 0 { - return Result - } - - // go through the live results and fill the list - if Offset >= len(job.Files) { // offset wants to skip entire queue? - job.FreezeFiles = append(job.FreezeFiles, job.Files...) - job.Files = nil - return Result - } - - // check if a sort is required before using this queue - if job.requireSort { - job.requireSort = false - - job.Files = SortFiles(job.Files, job.filtersRuntime.Sort) - } - - // set the amount of files to copy - countCopy := len(job.Files) - Offset - if countCopy > Limit { - countCopy = Limit - } - - // copy the results and freeze them - Result = append(Result, job.Files[Offset:Offset+countCopy]...) - - // note that freeze disregards the offset, it has to freeze any elements before! - job.FreezeFiles = append(job.FreezeFiles, job.Files[:Offset+countCopy]...) - job.Files = job.Files[Offset+countCopy:] - - //Limit -= countCopy - - return Result -} - -// ReturnNext returns the next results. Call must be serialized. -func (job *SearchJob) ReturnNext(Limit int) (Result []*apiFile) { - Result = job.ReturnResult(job.currentOffset, Limit) - job.currentOffset += len(Result) - - return -} - -// PeekResult returns the selected results but will not change any frozen files or impact auto offset -func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*apiFile) { - job.ResultSync.Lock() - defer job.ResultSync.Unlock() - - // serve files from frozen list? - if Offset < len(job.FreezeFiles) { - countCopy := len(job.FreezeFiles) - Offset - if countCopy > Limit { - countCopy = Limit - } - Result = job.FreezeFiles[Offset : Offset+countCopy] - Limit -= countCopy - Offset = 0 - } else { - Offset -= len(job.FreezeFiles) - } - - if Limit == 0 || Offset >= len(job.Files) { // offset wants to skip entire queue? - return Result - } - - // check if a sort is required before using this queue - if job.requireSort { - job.requireSort = false - - job.Files = SortFiles(job.Files, job.filtersRuntime.Sort) - } - - countCopy := len(job.Files) - Offset - if countCopy > Limit { - countCopy = Limit - } - - // copy the results - Result = append(Result, job.Files[Offset:Offset+countCopy]...) - - return Result -} - -// RuntimeFilter allows to apply filters at runtime to search jobs that already started. To remove the filters, call this function without the filters set. -func (job *SearchJob) RuntimeFilter(Filter SearchFilter) { - job.ResultSync.Lock() - defer job.ResultSync.Unlock() - - job.filtersRuntime = Filter - - // FreezeFiles and current offset is reset - job.FreezeFiles = nil - job.currentOffset = 0 - - // files remain in AllFiles, but Files needs to be filtered based on the new filter - job.Files = []*apiFile{} - job.requireSort = false // Sorting is done immediately below - - // set Files based on AllFiles with the filter - for m := range job.AllFiles { - // only append if filter is matching - if job.isFileFiltered(job.AllFiles[m]) { - job.Files = append(job.Files, job.AllFiles[m]) - } - - // sort, if a sort order is defined - if job.filtersRuntime.Sort > 0 { - job.Files = SortFiles(job.Files, job.filtersRuntime.Sort) - } - } -} - -// isFileFiltered returns true if the file conforms to the runtime filter. If there is no runtime filter, it always returns true. -func (job *SearchJob) isFileFiltered(file *apiFile) bool { - if job.filtersRuntime.FileType >= 0 && file.Type != uint8(job.filtersRuntime.FileType) { - return false - } - - if job.filtersRuntime.FileFormat >= 0 && file.Format != uint16(job.filtersRuntime.FileFormat) { - return false - } - - // Note: If the date is not available in the file, it will be filtered out. Since this is the mapped Shared Date this should normally not occur though. - if job.filtersRuntime.IsDates && (file.Date.IsZero() || file.Date.Before(job.filtersRuntime.DateFrom) || file.Date.After(job.filtersRuntime.DateTo)) { - return false - } - - if job.filtersRuntime.SizeMin >= 0 && file.Size < uint64(job.filtersRuntime.SizeMin) || job.filtersRuntime.SizeMax >= 0 && file.Size > uint64(job.filtersRuntime.SizeMax) { - return false - } - - return true -} - -// SortFiles sorts a list of files. It returns a sorted list. 0 = no sorting, 1 = Relevance ASC, 2 = Relevance DESC, 3 = Date ASC, 4 = Date DESC, 5 = Name ASC, 6 = Name DESC -func SortFiles(files []*apiFile, Sort int) (sorted []*apiFile) { - switch Sort { - case SortRelevanceAsc: - sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) }) // first as date for secondary sorting - //sort.SliceStable(files, func(i, j int) bool { return files[i].Score < files[j].Score }) // TODO - case SortRelevanceDec: - sort.SliceStable(files, func(i, j int) bool { return files[j].Date.Before(files[i].Date) }) // first as date for secondary sorting - //sort.SliceStable(files, func(i, j int) bool { return files[i].Score > files[j].Score }) // TODO - - case SortDateAsc: - sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) }) - case SortDateDesc: - sort.SliceStable(files, func(i, j int) bool { return files[j].Date.Before(files[i].Date) }) - - case SortNameAsc: - sort.SliceStable(files, func(i, j int) bool { return files[i].Name < files[j].Name }) - case SortNameDesc: - sort.SliceStable(files, func(i, j int) bool { return files[i].Name > files[j].Name }) - - case SortSizeAsc: - sort.SliceStable(files, func(i, j int) bool { return files[i].Size < files[j].Size }) - case SortSizeDesc: - sort.SliceStable(files, func(i, j int) bool { return files[i].Size > files[j].Size }) - - case SortSharedByCountAsc: - sort.SliceStable(files, func(i, j int) bool { - return files[i].GetMetadata(blockchain.TagSharedByCount).Number < files[j].GetMetadata(blockchain.TagSharedByCount).Number - }) - case SortSharedByCountDesc: - sort.SliceStable(files, func(i, j int) bool { - return files[i].GetMetadata(blockchain.TagSharedByCount).Number > files[j].GetMetadata(blockchain.TagSharedByCount).Number - }) - - } - - return files -} - -// IsSearchResults checks if search results may be expected (either files are in queue or a search is running) -func (job *SearchJob) IsSearchResults() bool { - // check for any available results. Do not use any lock here as this is read only. - return len(job.Files) > 0 || !job.IsTerminated() -} - -// isFileReceived checks if a file was already received, preventing double results -func (job *SearchJob) isFileReceived(id uuid.UUID) (exists bool) { - // Future: A map would be likely faster than iterating over all results. - for m := range job.AllFiles { - if id == job.AllFiles[m].ID { - return true - } - } - - return false -} - -// ---- job list management ---- - -// RemoveJob removes the job structure from the list. Terminate should be called before. Unless the search is manually removed, it stays forever in the list. -func (api *WebapiInstance) RemoveJob(job *SearchJob) { - api.allJobsMutex.Lock() - delete(api.allJobs, job.id) // delete is safe to call multiple times, so auto-removal and manual one are fine and need no syncing - api.allJobsMutex.Unlock() -} - -// RemoveDefer removes the search job after a given time after all searches are terminated. This can be used for automated time delayed removal. Do not create additional search clients after deferal removing. -func (api *WebapiInstance) RemoveJobDefer(job *SearchJob, Duration time.Duration) { - go func() { - // for _, client := range job.clients { - // <-client.TerminateSignal - // } - - <-time.After(Duration) - api.RemoveJob(job) - }() -} - -// JobLookup looks up a job. Returns nil if not found. -func (api *WebapiInstance) JobLookup(id uuid.UUID) (job *SearchJob) { - api.allJobsMutex.RLock() - job = api.allJobs[id] - api.allJobsMutex.RUnlock() - - return job -} - -// IsTerminated checks if all searches are finished. The job itself does not record a termination signal. -func (job *SearchJob) IsTerminated() bool { - job.clientsMutex.Lock() - defer job.clientsMutex.Unlock() - - // for n := range job.clients { - // if !job.clients[n].IsTerminated { - // return false - // } - // } - - return job.Status == SearchStatusTerminated || job.Status == SearchStatusNoIndex -} - -// Terminate terminates all searches -func (job *SearchJob) Terminate() { - job.clientsMutex.Lock() - defer job.clientsMutex.Unlock() - - // for n := range job.clients { - // if !job.clients[n].IsTerminated { - // job.clients[n].Terminate(true) - // } - // } -} - -// WaitTerminate waits until all search clients are terminated. Do not create additional search clients after calling this function. -func (job *SearchJob) WaitTerminate() { - //for _, client := range job.clients { - // <-client.TerminateSignal - //} -} - -// ---- statistics ---- - -// SearchStatisticRecordDay is a single record containing date info. -type SearchStatisticRecordDay struct { - Date time.Time `json:"date"` // The day (which covers the full 24 hours). Always rounded down to midnight. - Count int `json:"count"` // Count of files. -} - -// SearchStatisticRecord is a single record. -type SearchStatisticRecord struct { - Key int `json:"key"` // Key index. The exact meaning depends on where this structure is used. - Count int `json:"count"` // Count of files for the given key -} - -// SearchStatisticData contains statistics on search results. -type SearchStatisticData struct { - Date []SearchStatisticRecordDay `json:"date"` // Files per date - FileType []SearchStatisticRecord `json:"filetype"` // Files per file type - FileFormat []SearchStatisticRecord `json:"fileformat"` // Files per file format - Total int `json:"total"` // Total count of files -} - -// Statistics generates statistics on all results, regardless of runtime filters. -func (job *SearchJob) Statistics() (result SearchStatisticData) { - job.stats.RLock() - defer job.stats.RUnlock() - - result.Total = job.stats.total - - // Files per date. Sort dates to date ASC. - for key, value := range job.stats.date { - result.Date = append(result.Date, SearchStatisticRecordDay{Date: key, Count: value}) - } - sort.SliceStable(result.Date, func(i, j int) bool { return result.Date[i].Date.Before(result.Date[j].Date) }) - - // File Type and Format - for key, value := range job.stats.fileType { - result.FileType = append(result.FileType, SearchStatisticRecord{Key: int(key), Count: value}) - } - for key, value := range job.stats.fileFormat { - result.FileFormat = append(result.FileFormat, SearchStatisticRecord{Key: int(key), Count: value}) - } - - return -} - -// statsAdd counts the files in the statistics -func (job *SearchJob) statsAdd(files ...*apiFile) { - job.stats.Lock() - defer job.stats.Unlock() - - for _, file := range files { - // Use file's Date field if available. - if !file.Date.IsZero() { - // Files per day - date := file.Date.Truncate(24 * time.Hour) - countDate := job.stats.date[date] - countDate++ - job.stats.date[date] = countDate - } - - // File Type and Format - countType := job.stats.fileType[file.Type] - countType++ - job.stats.fileType[file.Type] = countType - - countFormat := job.stats.fileFormat[file.Format] - countFormat++ - job.stats.fileFormat[file.Format] = countFormat - } - - job.stats.total += len(files) -} - -// ---- actual search & retrieving results ---- - -// appendSearchClient appends a search client to a job - -func (job *SearchJob) SearchAway() { - job.clientsMutex.Lock() - defer job.clientsMutex.Unlock() - - // TODO -} - -//func (job *SearchJob) appendSearchClient(search *dht.SearchClient) { -//} - -//func (job *SearchJob) receiveClientResults(client *dht.SearchClient) { -//} +/* +File Name: Search Job.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "sort" + "sync" + "time" + + "github.com/PeernetOfficial/core/blockchain" + "github.com/google/uuid" +) + +// SearchFilter allows to filter search results based on the criteria. +type SearchFilter struct { + IsDates bool // Whether the from/to dates are valid, both are required. + DateFrom time.Time // Optional date from + DateTo time.Time // Optional date to + FileType int // File type such as binary, text document etc. See core.TypeX. -1 = not used. + FileFormat int // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. + Sort int // Sort order. See SortX. + SizeMin int // Min file size in bytes. -1 = not used. + SizeMax int // Max file size in bytes. -1 = not used. +} + +// SearchJob is a collection of search jobs +type SearchJob struct { + // input settings + id uuid.UUID // The job id + timeout time.Duration // timeout set for all searches + maxResult int // max results user-facing. + + filtersStart SearchFilter // Filters when starting the search. They cannot be changed later on. Any incoming file is checked against them, even if there are different runtime filters. + filtersRuntime SearchFilter // Runtime Filters. They allow filtering results after they were received. + + // File statistics (filters are ignored) of returned results. Map value is always count of files. + stats struct { + sync.RWMutex // Synced access to maps + date map[time.Time]int // Files per day (rounded down to midnight) + fileType map[uint8]int // Files per File Type + fileFormat map[uint16]int // Files per File Format + total int // Total count of files + } + + // -- result data -- + + // Status indicates the overall search status. This will be removed later when relying on search clients. + Status int + + // runtime data + //clients []*SearchClient // all search clients + clientsMutex sync.Mutex // mutex for manipulating client list + + // List of files found but not yet returned via API to the caller. They are subject to sorting. + Files []*apiFile + requireSort bool // if Files requires sort before returning the results + + // FreezeFiles is a list of files that were already finally delivered via the API. They may NOT change in sorting. + FreezeFiles []*apiFile + + // List of all files. Does not change based on sorting or runtime filters. This list only gets expanded. + AllFiles []*apiFile + + ResultSync sync.Mutex // ResultSync ensures unique access to the file results + + currentOffset int // for always getting the next results +} + +const ( + SearchStatusNotStarted = iota // Search was not yet started + SearchStatusLive // Search running + SearchStatusTerminated // Search is terminated. No more results are expected. + SearchStatusNoIndex // Search is terminated. No search index to use. +) + +// CreateSearchJob creates a new search job and adds it to the lookup list. +// Timeout and MaxResults must be set and must not be 0. +func (api *WebapiInstance) CreateSearchJob(Timeout time.Duration, MaxResults int, Filter SearchFilter) (job *SearchJob) { + job = &SearchJob{} + job.Status = SearchStatusNotStarted + job.id = uuid.New() + job.timeout = Timeout + job.maxResult = MaxResults + job.filtersStart = Filter + job.filtersRuntime = Filter // initialize the runtime filters as the same + + job.stats.date = make(map[time.Time]int) + job.stats.fileType = make(map[uint8]int) + job.stats.fileFormat = make(map[uint16]int) + + // add to the list of jobs + api.allJobsMutex.Lock() + api.allJobs[job.id] = job + api.allJobsMutex.Unlock() + + return +} + +// ReturnResult returns the selected results. +func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*apiFile) { + if Limit == 0 { + return Result + } + + job.ResultSync.Lock() + defer job.ResultSync.Unlock() + + // serve files from frozen list? + if Offset < len(job.FreezeFiles) { + countCopy := len(job.FreezeFiles) - Offset + if countCopy > Limit { + countCopy = Limit + } + Result = job.FreezeFiles[Offset : Offset+countCopy] + Limit -= countCopy + Offset = 0 + } else { + Offset -= len(job.FreezeFiles) + } + + if Limit == 0 { + return Result + } + + // go through the live results and fill the list + if Offset >= len(job.Files) { // offset wants to skip entire queue? + job.FreezeFiles = append(job.FreezeFiles, job.Files...) + job.Files = nil + return Result + } + + // check if a sort is required before using this queue + if job.requireSort { + job.requireSort = false + + job.Files = SortFiles(job.Files, job.filtersRuntime.Sort) + } + + // set the amount of files to copy + countCopy := len(job.Files) - Offset + if countCopy > Limit { + countCopy = Limit + } + + // copy the results and freeze them + Result = append(Result, job.Files[Offset:Offset+countCopy]...) + + // note that freeze disregards the offset, it has to freeze any elements before! + job.FreezeFiles = append(job.FreezeFiles, job.Files[:Offset+countCopy]...) + job.Files = job.Files[Offset+countCopy:] + + //Limit -= countCopy + + return Result +} + +// ReturnNext returns the next results. Call must be serialized. +func (job *SearchJob) ReturnNext(Limit int) (Result []*apiFile) { + Result = job.ReturnResult(job.currentOffset, Limit) + job.currentOffset += len(Result) + + return +} + +// PeekResult returns the selected results but will not change any frozen files or impact auto offset +func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*apiFile) { + job.ResultSync.Lock() + defer job.ResultSync.Unlock() + + // serve files from frozen list? + if Offset < len(job.FreezeFiles) { + countCopy := len(job.FreezeFiles) - Offset + if countCopy > Limit { + countCopy = Limit + } + Result = job.FreezeFiles[Offset : Offset+countCopy] + Limit -= countCopy + Offset = 0 + } else { + Offset -= len(job.FreezeFiles) + } + + if Limit == 0 || Offset >= len(job.Files) { // offset wants to skip entire queue? + return Result + } + + // check if a sort is required before using this queue + if job.requireSort { + job.requireSort = false + + job.Files = SortFiles(job.Files, job.filtersRuntime.Sort) + } + + countCopy := len(job.Files) - Offset + if countCopy > Limit { + countCopy = Limit + } + + // copy the results + Result = append(Result, job.Files[Offset:Offset+countCopy]...) + + return Result +} + +// RuntimeFilter allows to apply filters at runtime to search jobs that already started. To remove the filters, call this function without the filters set. +func (job *SearchJob) RuntimeFilter(Filter SearchFilter) { + job.ResultSync.Lock() + defer job.ResultSync.Unlock() + + job.filtersRuntime = Filter + + // FreezeFiles and current offset is reset + job.FreezeFiles = nil + job.currentOffset = 0 + + // files remain in AllFiles, but Files needs to be filtered based on the new filter + job.Files = []*apiFile{} + job.requireSort = false // Sorting is done immediately below + + // set Files based on AllFiles with the filter + for m := range job.AllFiles { + // only append if filter is matching + if job.isFileFiltered(job.AllFiles[m]) { + job.Files = append(job.Files, job.AllFiles[m]) + } + + // sort, if a sort order is defined + if job.filtersRuntime.Sort > 0 { + job.Files = SortFiles(job.Files, job.filtersRuntime.Sort) + } + } +} + +// isFileFiltered returns true if the file conforms to the runtime filter. If there is no runtime filter, it always returns true. +func (job *SearchJob) isFileFiltered(file *apiFile) bool { + if job.filtersRuntime.FileType >= 0 && file.Type != uint8(job.filtersRuntime.FileType) { + return false + } + + if job.filtersRuntime.FileFormat >= 0 && file.Format != uint16(job.filtersRuntime.FileFormat) { + return false + } + + // Note: If the date is not available in the file, it will be filtered out. Since this is the mapped Shared Date this should normally not occur though. + if job.filtersRuntime.IsDates && (file.Date.IsZero() || file.Date.Before(job.filtersRuntime.DateFrom) || file.Date.After(job.filtersRuntime.DateTo)) { + return false + } + + if job.filtersRuntime.SizeMin >= 0 && file.Size < uint64(job.filtersRuntime.SizeMin) || job.filtersRuntime.SizeMax >= 0 && file.Size > uint64(job.filtersRuntime.SizeMax) { + return false + } + + return true +} + +// SortFiles sorts a list of files. It returns a sorted list. 0 = no sorting, 1 = Relevance ASC, 2 = Relevance DESC, 3 = Date ASC, 4 = Date DESC, 5 = Name ASC, 6 = Name DESC +func SortFiles(files []*apiFile, Sort int) (sorted []*apiFile) { + switch Sort { + case SortRelevanceAsc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) }) // first as date for secondary sorting + //sort.SliceStable(files, func(i, j int) bool { return files[i].Score < files[j].Score }) // TODO + case SortRelevanceDec: + sort.SliceStable(files, func(i, j int) bool { return files[j].Date.Before(files[i].Date) }) // first as date for secondary sorting + //sort.SliceStable(files, func(i, j int) bool { return files[i].Score > files[j].Score }) // TODO + + case SortDateAsc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) }) + case SortDateDesc: + sort.SliceStable(files, func(i, j int) bool { return files[j].Date.Before(files[i].Date) }) + + case SortNameAsc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Name < files[j].Name }) + case SortNameDesc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Name > files[j].Name }) + + case SortSizeAsc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Size < files[j].Size }) + case SortSizeDesc: + sort.SliceStable(files, func(i, j int) bool { return files[i].Size > files[j].Size }) + + case SortSharedByCountAsc: + sort.SliceStable(files, func(i, j int) bool { + return files[i].GetMetadata(blockchain.TagSharedByCount).Number < files[j].GetMetadata(blockchain.TagSharedByCount).Number + }) + case SortSharedByCountDesc: + sort.SliceStable(files, func(i, j int) bool { + return files[i].GetMetadata(blockchain.TagSharedByCount).Number > files[j].GetMetadata(blockchain.TagSharedByCount).Number + }) + + } + + return files +} + +// IsSearchResults checks if search results may be expected (either files are in queue or a search is running) +func (job *SearchJob) IsSearchResults() bool { + // check for any available results. Do not use any lock here as this is read only. + return len(job.Files) > 0 || !job.IsTerminated() +} + +// isFileReceived checks if a file was already received, preventing double results +func (job *SearchJob) isFileReceived(id uuid.UUID) (exists bool) { + // Future: A map would be likely faster than iterating over all results. + for m := range job.AllFiles { + if id == job.AllFiles[m].ID { + return true + } + } + + return false +} + +// ---- job list management ---- + +// RemoveJob removes the job structure from the list. Terminate should be called before. Unless the search is manually removed, it stays forever in the list. +func (api *WebapiInstance) RemoveJob(job *SearchJob) { + api.allJobsMutex.Lock() + delete(api.allJobs, job.id) // delete is safe to call multiple times, so auto-removal and manual one are fine and need no syncing + api.allJobsMutex.Unlock() +} + +// RemoveDefer removes the search job after a given time after all searches are terminated. This can be used for automated time delayed removal. Do not create additional search clients after deferal removing. +func (api *WebapiInstance) RemoveJobDefer(job *SearchJob, Duration time.Duration) { + go func() { + // for _, client := range job.clients { + // <-client.TerminateSignal + // } + + <-time.After(Duration) + api.RemoveJob(job) + }() +} + +// JobLookup looks up a job. Returns nil if not found. +func (api *WebapiInstance) JobLookup(id uuid.UUID) (job *SearchJob) { + api.allJobsMutex.RLock() + job = api.allJobs[id] + api.allJobsMutex.RUnlock() + + return job +} + +// IsTerminated checks if all searches are finished. The job itself does not record a termination signal. +func (job *SearchJob) IsTerminated() bool { + job.clientsMutex.Lock() + defer job.clientsMutex.Unlock() + + // for n := range job.clients { + // if !job.clients[n].IsTerminated { + // return false + // } + // } + + return job.Status == SearchStatusTerminated || job.Status == SearchStatusNoIndex +} + +// Terminate terminates all searches +func (job *SearchJob) Terminate() { + job.clientsMutex.Lock() + defer job.clientsMutex.Unlock() + + // for n := range job.clients { + // if !job.clients[n].IsTerminated { + // job.clients[n].Terminate(true) + // } + // } +} + +// WaitTerminate waits until all search clients are terminated. Do not create additional search clients after calling this function. +func (job *SearchJob) WaitTerminate() { + //for _, client := range job.clients { + // <-client.TerminateSignal + //} +} + +// ---- statistics ---- + +// SearchStatisticRecordDay is a single record containing date info. +type SearchStatisticRecordDay struct { + Date time.Time `json:"date"` // The day (which covers the full 24 hours). Always rounded down to midnight. + Count int `json:"count"` // Count of files. +} + +// SearchStatisticRecord is a single record. +type SearchStatisticRecord struct { + Key int `json:"key"` // Key index. The exact meaning depends on where this structure is used. + Count int `json:"count"` // Count of files for the given key +} + +// SearchStatisticData contains statistics on search results. +type SearchStatisticData struct { + Date []SearchStatisticRecordDay `json:"date"` // Files per date + FileType []SearchStatisticRecord `json:"filetype"` // Files per file type + FileFormat []SearchStatisticRecord `json:"fileformat"` // Files per file format + Total int `json:"total"` // Total count of files +} + +// Statistics generates statistics on all results, regardless of runtime filters. +func (job *SearchJob) Statistics() (result SearchStatisticData) { + job.stats.RLock() + defer job.stats.RUnlock() + + result.Total = job.stats.total + + // Files per date. Sort dates to date ASC. + for key, value := range job.stats.date { + result.Date = append(result.Date, SearchStatisticRecordDay{Date: key, Count: value}) + } + sort.SliceStable(result.Date, func(i, j int) bool { return result.Date[i].Date.Before(result.Date[j].Date) }) + + // File Type and Format + for key, value := range job.stats.fileType { + result.FileType = append(result.FileType, SearchStatisticRecord{Key: int(key), Count: value}) + } + for key, value := range job.stats.fileFormat { + result.FileFormat = append(result.FileFormat, SearchStatisticRecord{Key: int(key), Count: value}) + } + + return +} + +// statsAdd counts the files in the statistics +func (job *SearchJob) statsAdd(files ...*apiFile) { + job.stats.Lock() + defer job.stats.Unlock() + + for _, file := range files { + // Use file's Date field if available. + if !file.Date.IsZero() { + // Files per day + date := file.Date.Truncate(24 * time.Hour) + countDate := job.stats.date[date] + countDate++ + job.stats.date[date] = countDate + } + + // File Type and Format + countType := job.stats.fileType[file.Type] + countType++ + job.stats.fileType[file.Type] = countType + + countFormat := job.stats.fileFormat[file.Format] + countFormat++ + job.stats.fileFormat[file.Format] = countFormat + } + + job.stats.total += len(files) +} + +// ---- actual search & retrieving results ---- + +// appendSearchClient appends a search client to a job + +func (job *SearchJob) SearchAway() { + job.clientsMutex.Lock() + defer job.clientsMutex.Unlock() + + // TODO +} + +//func (job *SearchJob) appendSearchClient(search *dht.SearchClient) { +//} + +//func (job *SearchJob) receiveClientResults(client *dht.SearchClient) { +//} diff --git a/webapi/Search.go b/webapi/Search.go index 43b2fed..9e16860 100644 --- a/webapi/Search.go +++ b/webapi/Search.go @@ -1,456 +1,456 @@ -/* -File Name: Search.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner - -/search Submit a search request -/search/result Return search results -/search/terminate Terminate a search -/search/result/ws Websocket to receive results as stream -/search/statistic Statistics about the results - -/explore List recently shared files - -*/ - -package webapi - -import ( - "net/http" - "strconv" - "time" - - "github.com/google/uuid" -) - -// SearchRequest is the information from the end-user for the search. Filters and sort order may be applied when starting the search, or at runtime when getting the results. -type SearchRequest struct { - Term string `json:"term"` // Search term. - Timeout int `json:"timeout"` // Timeout in seconds. 0 means default. This is the entire time the search may take. Found results are still available after this timeout. - MaxResults int `json:"maxresults"` // Total number of max results. 0 means default. - DateFrom string `json:"datefrom"` // Date from, both from/to are required if set. Format "2006-01-02 15:04:05". - DateTo string `json:"dateto"` // Date to, both from/to are required if set. Format "2006-01-02 15:04:05". - Sort int `json:"sort"` // See SortX. - TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate. - FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. -1 = not used. - FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. - SizeMin int `json:"sizemin"` // Min file size in bytes. -1 = not used. - SizeMax int `json:"sizemax"` // Max file size in bytes. -1 = not used. -} - -// Sort orders -const ( - SortNone = 0 // No sorting. Results are returned as they come in. - SortRelevanceAsc = 1 // Least relevant results first. - SortRelevanceDec = 2 // Most relevant results first. - SortDateAsc = 3 // Oldest first. - SortDateDesc = 4 // Newest first. - SortNameAsc = 5 // File name ascending. The folder name is not used for sorting. - SortNameDesc = 6 // File name descending. The folder name is not used for sorting. - SortSizeAsc = 7 // File size ascending. Smallest files first. - SortSizeDesc = 8 // File size descending. Largest files first. - SortSharedByCountAsc = 9 // Shared by count ascending. Files that are shared by the least count of peers first. - SortSharedByCountDesc = 10 // Shared by count descending. Files that are shared by the most count of peers first. -) - -// SearchRequestResponse is the result to the initial search request -type SearchRequestResponse struct { - ID uuid.UUID `json:"id"` // ID of the search job. This is used to get the results. - Status int `json:"status"` // Status of the search: 0 = Success (ID valid), 1 = Invalid Term, 2 = Error Max Concurrent Searches -} - -// SearchResult contains the search results. -type SearchResult struct { - Status int `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying - Files []apiFile `json:"files"` // List of files found - Statistic interface{} `json:"statistic"` // Statistics of all results (independent from applied filters), if requested. Only set if files are returned (= if statistics changed). See SearchStatisticData. -} - -// SearchStatistic contains statistics on search results. Statistics are always calculated over all results, regardless of any applied runtime filters. -type SearchStatistic struct { - SearchStatisticData - Status int `json:"status"` // Status: 0 = Success - IsTerminated bool `json:"terminated"` // Whether the search is terminated, meaning that statistics won't change -} - -const apiDateFormat = "2006-01-02 15:04:05" - -/* -apiSearch submits a search request - -Request: POST /search with JSON SearchRequest -Result: 200 on success with JSON SearchRequestResponse - - 400 on invalid JSON -*/ -func (api *WebapiInstance) apiSearch(w http.ResponseWriter, r *http.Request) { - var input SearchRequest - if err := DecodeJSON(w, r, &input); err != nil { - return - } - - if input.Timeout <= 0 { - input.Timeout = 20 - } - if input.MaxResults <= 0 { - input.MaxResults = 200 - } - - // Terminate previous searches, if their IDs were supplied. This allows terminating the old search immediately without making a separate /search/terminate request. - for _, terminate := range input.TerminateID { - if job := api.JobLookup(terminate); job != nil { - job.Terminate() - api.RemoveJob(job) - } - } - - job := api.dispatchSearch(input) - - EncodeJSON(api.backend, w, r, SearchRequestResponse{Status: 0, ID: job.id}) -} - -/* -apiSearchResult returns results. The default limit is 100. -If reset is set, all results will be filtered and sorted according to the settings. This means that the new first result will be returned again and internal result offset is set to 0. - -Request: GET /search/result?id=[UUID]&limit=[max records] -Optional parameters: - - &reset=[0|1] to reset the filters or sort orders with any of the below parameters (all required): - &filetype=[File Type] - &fileformat=[File Format] - &from=[Date From]&to=[Date To] - &sizemin=[Minimum file size] - &sizemax=[Maximum file size] - &sort=[sort order] - &offset=[absolute offset] with &limit=[records] to get items pagination style. Returned items (and ones before) are automatically frozen. - -Result: 200 with JSON structure SearchResult. Check the field status. -*/ -func (api *WebapiInstance) apiSearchResult(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - jobID, err := uuid.Parse(r.Form.Get("id")) - if err != nil { - http.Error(w, "", http.StatusBadRequest) - return - } - limit, err := strconv.Atoi(r.Form.Get("limit")) - if err != nil { - limit = 100 - } - offset, errOffset := strconv.Atoi(r.Form.Get("offset")) - - // find the job ID - job := api.JobLookup(jobID) - if job == nil { - EncodeJSON(api.backend, w, r, SearchResult{Status: 2}) - return - } - - // filters and sort parameter - if filterReset, _ := strconv.ParseBool(r.Form.Get("reset")); filterReset { - fileType, _ := strconv.Atoi(r.Form.Get("filetype")) - fileFormat, _ := strconv.Atoi(r.Form.Get("fileformat")) - dateFrom := r.Form.Get("from") - dateTo := r.Form.Get("to") - sort, _ := strconv.Atoi(r.Form.Get("sort")) - sizeMin, _ := strconv.Atoi(r.Form.Get("sizemin")) - sizeMax, _ := strconv.Atoi(r.Form.Get("sizemax")) - - filter := inputToSearchFilter(sort, fileType, fileFormat, dateFrom, dateTo, sizeMin, sizeMax) - - job.RuntimeFilter(filter) - } - - // query all results - var resultFiles []*apiFile - if errOffset == nil { - resultFiles = job.ReturnResult(offset, limit) - } else { - resultFiles = job.ReturnNext(limit) - } - - var result SearchResult - result.Files = []apiFile{} - - // loop over results - for n := range resultFiles { - result.Files = append(result.Files, *resultFiles[n]) - } - - // set the status - if len(result.Files) > 0 { - if job.IsSearchResults() { - result.Status = 0 // 0 = Success with results - } else { - result.Status = 1 // No more results to expect - } - } else { - switch job.Status { - case SearchStatusLive: - result.Status = 3 // No results yet available keep trying - - case SearchStatusTerminated: - result.Status = 1 // No more results to expect - - default: // SearchStatusNoIndex, SearchStatusNotStarted - result.Status = 1 // No more results to expect - } - } - - // embedded statistics? - if returnStats, _ := strconv.ParseBool(r.Form.Get("stats")); returnStats { - result.Statistic = job.Statistics() - } - - EncodeJSON(api.backend, w, r, result) -} - -/* -apiSearchResultStream provides a websocket to receive results as stream. - -Request: GET /search/result/ws?id=[UUID]&limit=[optional max records] -Result: If successful, upgrades to a websocket and sends JSON structure SearchResult messages. - - Limit is optional. Not used if ommitted or 0. -*/ -func (api *WebapiInstance) apiSearchResultStream(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - jobID, err := uuid.Parse(r.Form.Get("id")) - if err != nil { - http.Error(w, "", http.StatusBadRequest) - return - } - limit, err := strconv.Atoi(r.Form.Get("limit")) - useLimit := err == nil - - // look up the job - job := api.JobLookup(jobID) - if job == nil { - EncodeJSON(api.backend, w, r, SearchResult{Status: 2}) - return - } - - // upgrade to websocket - conn, err := WSUpgrader.Upgrade(w, r, nil) - if err != nil { - // gorilla will automatically respond with "400 Bad Request", no other response is therefore necessary - return - } - - defer conn.Close() - - // loop to get new results and send out via the web socket. - // Only exit if limit is reached if used, otherwise only if there are no result or the connection breaks. - for { - // query all results - var resultFiles []*apiFile - - queryCount := 1 - if useLimit { - queryCount = limit - } - resultFiles = job.ReturnNext(queryCount) - - if useLimit { - limit -= len(resultFiles) - } - - // loop over results - var result SearchResult - result.Files = []apiFile{} - - for n := range resultFiles { - result.Files = append(result.Files, *resultFiles[n]) - } - - if !job.IsSearchResults() { - result.Status = 1 // No more results to expect - - if len(result.Files) == 0 { - conn.WriteJSON(result) // final message - return - } - } - - // if no results, stall - if len(result.Files) == 0 { - time.Sleep(time.Millisecond * 100) - continue - } - - // send out the results via the websocket - if err := conn.WriteJSON(result); err != nil { - return - } - - // Check whether to continue. If the limit is used break once all done. - if (useLimit && limit <= 0) || result.Status == 1 { - break - } - } -} - -/* -apiSearchTerminate terminates a search - -Request: GET /search/terminate?id=[UUID] -Response: 204 Empty - - 400 Invalid input - 404 ID not found -*/ -func (api *WebapiInstance) apiSearchTerminate(w http.ResponseWriter, r *http.Request) { - - r.ParseForm() - jobID, err := uuid.Parse(r.Form.Get("id")) - if err != nil { - http.Error(w, "", http.StatusBadRequest) - return - } - - // look up the job - job := api.JobLookup(jobID) - if job == nil { - http.Error(w, "", http.StatusNotFound) - return - } - - // terminate and remove it from the list - job.Terminate() - api.RemoveJob(job) - - w.WriteHeader(http.StatusNoContent) -} - -/* -apiSearchStatistic returns search result statistics. Statistics are always calculated over all results, regardless of any applied runtime filters. - -Request: GET /search/result?id=[UUID] -Result: 200 with JSON structure SearchStatistic. Check the field status (0 = Success, 2 = ID not found). -*/ -func (api *WebapiInstance) apiSearchStatistic(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - jobID, err := uuid.Parse(r.Form.Get("id")) - if err != nil { - http.Error(w, "", http.StatusBadRequest) - return - } - - // find the job ID - job := api.JobLookup(jobID) - if job == nil { - EncodeJSON(api.backend, w, r, SearchStatistic{Status: 2}) - return - } - - stats := job.Statistics() - - EncodeJSON(api.backend, w, r, SearchStatistic{SearchStatisticData: stats, Status: 0, IsTerminated: job.IsTerminated()}) -} - -/* -apiExplore returns recently shared files in Peernet. Results are returned in real-time. The file type is an optional filter. See TypeX. -Special type -2 = Binary, Compressed, Container, Executable. This special type includes everything except Documents, Video, Audio, Ebooks, Picture, Text. - -Request: GET /explore?limit=[max records]&type=[file type]&offset=[offset] -Result: 200 with JSON structure SearchResult. Check the field status. -*/ -func (api *WebapiInstance) apiExplore(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - offset, _ := strconv.Atoi(r.Form.Get("offset")) - limit, err := strconv.Atoi(r.Form.Get("limit")) - if err != nil { - limit = 100 - } - - fileType, err := strconv.Atoi(r.Form.Get("type")) - if err != nil { - fileType = -1 - } - - result := api.ExploreHelper(fileType,limit,offset,[]byte{},false) - - EncodeJSON(api.backend, w, r, result) -} - -/* -apiExploreNodeID returns the shared files of a particular node in Peernet. Results are returned in real-time. The file type is an optional filter. See TypeX. -Special type -2 = Binary, Compressed, Container, Executable. This special type includes everything except Documents, Video, Audio, Ebooks, Picture, Text. - -Request: GET /explore/node?limit=[max records]&type=[file type]&offset=[offset]&NodeID=[node id] -Result: 200 with JSON structure SearchResult. Check the field status. -*/ -func (api *WebapiInstance) apiExploreNodeID(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - offset, _ := strconv.Atoi(r.Form.Get("offset")) - limit, err := strconv.Atoi(r.Form.Get("limit")) - if err != nil { - limit = 100 - } - // ID fields for results for a specific node ID. - NodeId, _ := DecodeBlake3Hash(r.Form.Get("NodeID")) - - fileType, err := strconv.Atoi(r.Form.Get("type")) - if err != nil { - fileType = -1 - } - - result := api.ExploreHelper(fileType,limit,offset,NodeId,true) - - EncodeJSON(api.backend, w, r, result) -} - -// ExploreHelper Helper function for the explore route with the possibility search based on a node ID -func (api *WebapiInstance) ExploreHelper(fileType int, limit, offset int, nodeID []byte, nodeIDState bool) *SearchResult{ - resultFiles := api.queryRecentShared(api.backend, fileType, uint64(limit*20/100), uint64(offset), uint64(limit), nodeID, nodeIDState) - - var result SearchResult - result.Files = []apiFile{} - - // loop over results - for n := range resultFiles { - result.Files = append(result.Files, blockRecordFileToAPI(resultFiles[n])) - } - - if len(result.Files) == 0 { - result.Status = 3 // No results yet available keep trying - } - - result.Status = 1 // No more results to expect - - return &result -} - -func (input *SearchRequest) Parse() (Timeout time.Duration) { - if input.Timeout == 0 { - Timeout = time.Second * 20 // default timeout: 20 seconds - } else { - Timeout = time.Duration(input.Timeout) * time.Second - } - - return -} - -// ToSearchFilter converts the user input to a valid search filter -func (input *SearchRequest) ToSearchFilter() (output SearchFilter) { - return inputToSearchFilter(input.Sort, input.FileType, input.FileFormat, input.DateFrom, input.DateTo, input.SizeMin, input.SizeMax) -} - -func inputToSearchFilter(Sort, FileType, FileFormat int, DateFrom, DateTo string, SizeMin, SizeMax int) (output SearchFilter) { - output.Sort = Sort - output.FileType = FileType - output.FileFormat = FileFormat - output.SizeMin = SizeMin - output.SizeMax = SizeMax - - dateFrom, errFrom := time.Parse(apiDateFormat, DateFrom) - dateTo, errTo := time.Parse(apiDateFormat, DateTo) - if errFrom == nil && errTo == nil && dateFrom.Before(dateTo) { - output.DateFrom = dateFrom - output.DateTo = dateTo - output.IsDates = true - } - - return -} +/* +File Name: Search.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner + +/search Submit a search request +/search/result Return search results +/search/terminate Terminate a search +/search/result/ws Websocket to receive results as stream +/search/statistic Statistics about the results + +/explore List recently shared files + +*/ + +package webapi + +import ( + "net/http" + "strconv" + "time" + + "github.com/google/uuid" +) + +// SearchRequest is the information from the end-user for the search. Filters and sort order may be applied when starting the search, or at runtime when getting the results. +type SearchRequest struct { + Term string `json:"term"` // Search term. + Timeout int `json:"timeout"` // Timeout in seconds. 0 means default. This is the entire time the search may take. Found results are still available after this timeout. + MaxResults int `json:"maxresults"` // Total number of max results. 0 means default. + DateFrom string `json:"datefrom"` // Date from, both from/to are required if set. Format "2006-01-02 15:04:05". + DateTo string `json:"dateto"` // Date to, both from/to are required if set. Format "2006-01-02 15:04:05". + Sort int `json:"sort"` // See SortX. + TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate. + FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. -1 = not used. + FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used. + SizeMin int `json:"sizemin"` // Min file size in bytes. -1 = not used. + SizeMax int `json:"sizemax"` // Max file size in bytes. -1 = not used. +} + +// Sort orders +const ( + SortNone = 0 // No sorting. Results are returned as they come in. + SortRelevanceAsc = 1 // Least relevant results first. + SortRelevanceDec = 2 // Most relevant results first. + SortDateAsc = 3 // Oldest first. + SortDateDesc = 4 // Newest first. + SortNameAsc = 5 // File name ascending. The folder name is not used for sorting. + SortNameDesc = 6 // File name descending. The folder name is not used for sorting. + SortSizeAsc = 7 // File size ascending. Smallest files first. + SortSizeDesc = 8 // File size descending. Largest files first. + SortSharedByCountAsc = 9 // Shared by count ascending. Files that are shared by the least count of peers first. + SortSharedByCountDesc = 10 // Shared by count descending. Files that are shared by the most count of peers first. +) + +// SearchRequestResponse is the result to the initial search request +type SearchRequestResponse struct { + ID uuid.UUID `json:"id"` // ID of the search job. This is used to get the results. + Status int `json:"status"` // Status of the search: 0 = Success (ID valid), 1 = Invalid Term, 2 = Error Max Concurrent Searches +} + +// SearchResult contains the search results. +type SearchResult struct { + Status int `json:"status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying + Files []apiFile `json:"files"` // List of files found + Statistic interface{} `json:"statistic"` // Statistics of all results (independent from applied filters), if requested. Only set if files are returned (= if statistics changed). See SearchStatisticData. +} + +// SearchStatistic contains statistics on search results. Statistics are always calculated over all results, regardless of any applied runtime filters. +type SearchStatistic struct { + SearchStatisticData + Status int `json:"status"` // Status: 0 = Success + IsTerminated bool `json:"terminated"` // Whether the search is terminated, meaning that statistics won't change +} + +const apiDateFormat = "2006-01-02 15:04:05" + +/* +apiSearch submits a search request + +Request: POST /search with JSON SearchRequest +Result: 200 on success with JSON SearchRequestResponse + + 400 on invalid JSON +*/ +func (api *WebapiInstance) apiSearch(w http.ResponseWriter, r *http.Request) { + var input SearchRequest + if err := DecodeJSON(w, r, &input); err != nil { + return + } + + if input.Timeout <= 0 { + input.Timeout = 20 + } + if input.MaxResults <= 0 { + input.MaxResults = 200 + } + + // Terminate previous searches, if their IDs were supplied. This allows terminating the old search immediately without making a separate /search/terminate request. + for _, terminate := range input.TerminateID { + if job := api.JobLookup(terminate); job != nil { + job.Terminate() + api.RemoveJob(job) + } + } + + job := api.dispatchSearch(input) + + EncodeJSON(api.backend, w, r, SearchRequestResponse{Status: 0, ID: job.id}) +} + +/* +apiSearchResult returns results. The default limit is 100. +If reset is set, all results will be filtered and sorted according to the settings. This means that the new first result will be returned again and internal result offset is set to 0. + +Request: GET /search/result?id=[UUID]&limit=[max records] +Optional parameters: + + &reset=[0|1] to reset the filters or sort orders with any of the below parameters (all required): + &filetype=[File Type] + &fileformat=[File Format] + &from=[Date From]&to=[Date To] + &sizemin=[Minimum file size] + &sizemax=[Maximum file size] + &sort=[sort order] + &offset=[absolute offset] with &limit=[records] to get items pagination style. Returned items (and ones before) are automatically frozen. + +Result: 200 with JSON structure SearchResult. Check the field status. +*/ +func (api *WebapiInstance) apiSearchResult(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + limit, err := strconv.Atoi(r.Form.Get("limit")) + if err != nil { + limit = 100 + } + offset, errOffset := strconv.Atoi(r.Form.Get("offset")) + + // find the job ID + job := api.JobLookup(jobID) + if job == nil { + EncodeJSON(api.backend, w, r, SearchResult{Status: 2}) + return + } + + // filters and sort parameter + if filterReset, _ := strconv.ParseBool(r.Form.Get("reset")); filterReset { + fileType, _ := strconv.Atoi(r.Form.Get("filetype")) + fileFormat, _ := strconv.Atoi(r.Form.Get("fileformat")) + dateFrom := r.Form.Get("from") + dateTo := r.Form.Get("to") + sort, _ := strconv.Atoi(r.Form.Get("sort")) + sizeMin, _ := strconv.Atoi(r.Form.Get("sizemin")) + sizeMax, _ := strconv.Atoi(r.Form.Get("sizemax")) + + filter := inputToSearchFilter(sort, fileType, fileFormat, dateFrom, dateTo, sizeMin, sizeMax) + + job.RuntimeFilter(filter) + } + + // query all results + var resultFiles []*apiFile + if errOffset == nil { + resultFiles = job.ReturnResult(offset, limit) + } else { + resultFiles = job.ReturnNext(limit) + } + + var result SearchResult + result.Files = []apiFile{} + + // loop over results + for n := range resultFiles { + result.Files = append(result.Files, *resultFiles[n]) + } + + // set the status + if len(result.Files) > 0 { + if job.IsSearchResults() { + result.Status = 0 // 0 = Success with results + } else { + result.Status = 1 // No more results to expect + } + } else { + switch job.Status { + case SearchStatusLive: + result.Status = 3 // No results yet available keep trying + + case SearchStatusTerminated: + result.Status = 1 // No more results to expect + + default: // SearchStatusNoIndex, SearchStatusNotStarted + result.Status = 1 // No more results to expect + } + } + + // embedded statistics? + if returnStats, _ := strconv.ParseBool(r.Form.Get("stats")); returnStats { + result.Statistic = job.Statistics() + } + + EncodeJSON(api.backend, w, r, result) +} + +/* +apiSearchResultStream provides a websocket to receive results as stream. + +Request: GET /search/result/ws?id=[UUID]&limit=[optional max records] +Result: If successful, upgrades to a websocket and sends JSON structure SearchResult messages. + + Limit is optional. Not used if ommitted or 0. +*/ +func (api *WebapiInstance) apiSearchResultStream(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + limit, err := strconv.Atoi(r.Form.Get("limit")) + useLimit := err == nil + + // look up the job + job := api.JobLookup(jobID) + if job == nil { + EncodeJSON(api.backend, w, r, SearchResult{Status: 2}) + return + } + + // upgrade to websocket + conn, err := WSUpgrader.Upgrade(w, r, nil) + if err != nil { + // gorilla will automatically respond with "400 Bad Request", no other response is therefore necessary + return + } + + defer conn.Close() + + // loop to get new results and send out via the web socket. + // Only exit if limit is reached if used, otherwise only if there are no result or the connection breaks. + for { + // query all results + var resultFiles []*apiFile + + queryCount := 1 + if useLimit { + queryCount = limit + } + resultFiles = job.ReturnNext(queryCount) + + if useLimit { + limit -= len(resultFiles) + } + + // loop over results + var result SearchResult + result.Files = []apiFile{} + + for n := range resultFiles { + result.Files = append(result.Files, *resultFiles[n]) + } + + if !job.IsSearchResults() { + result.Status = 1 // No more results to expect + + if len(result.Files) == 0 { + conn.WriteJSON(result) // final message + return + } + } + + // if no results, stall + if len(result.Files) == 0 { + time.Sleep(time.Millisecond * 100) + continue + } + + // send out the results via the websocket + if err := conn.WriteJSON(result); err != nil { + return + } + + // Check whether to continue. If the limit is used break once all done. + if (useLimit && limit <= 0) || result.Status == 1 { + break + } + } +} + +/* +apiSearchTerminate terminates a search + +Request: GET /search/terminate?id=[UUID] +Response: 204 Empty + + 400 Invalid input + 404 ID not found +*/ +func (api *WebapiInstance) apiSearchTerminate(w http.ResponseWriter, r *http.Request) { + + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + // look up the job + job := api.JobLookup(jobID) + if job == nil { + http.Error(w, "", http.StatusNotFound) + return + } + + // terminate and remove it from the list + job.Terminate() + api.RemoveJob(job) + + w.WriteHeader(http.StatusNoContent) +} + +/* +apiSearchStatistic returns search result statistics. Statistics are always calculated over all results, regardless of any applied runtime filters. + +Request: GET /search/result?id=[UUID] +Result: 200 with JSON structure SearchStatistic. Check the field status (0 = Success, 2 = ID not found). +*/ +func (api *WebapiInstance) apiSearchStatistic(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + jobID, err := uuid.Parse(r.Form.Get("id")) + if err != nil { + http.Error(w, "", http.StatusBadRequest) + return + } + + // find the job ID + job := api.JobLookup(jobID) + if job == nil { + EncodeJSON(api.backend, w, r, SearchStatistic{Status: 2}) + return + } + + stats := job.Statistics() + + EncodeJSON(api.backend, w, r, SearchStatistic{SearchStatisticData: stats, Status: 0, IsTerminated: job.IsTerminated()}) +} + +/* +apiExplore returns recently shared files in Peernet. Results are returned in real-time. The file type is an optional filter. See TypeX. +Special type -2 = Binary, Compressed, Container, Executable. This special type includes everything except Documents, Video, Audio, Ebooks, Picture, Text. + +Request: GET /explore?limit=[max records]&type=[file type]&offset=[offset] +Result: 200 with JSON structure SearchResult. Check the field status. +*/ +func (api *WebapiInstance) apiExplore(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, err := strconv.Atoi(r.Form.Get("limit")) + if err != nil { + limit = 100 + } + + fileType, err := strconv.Atoi(r.Form.Get("type")) + if err != nil { + fileType = -1 + } + + result := api.ExploreHelper(fileType, limit, offset, []byte{}, false) + + EncodeJSON(api.backend, w, r, result) +} + +/* +apiExploreNodeID returns the shared files of a particular node in Peernet. Results are returned in real-time. The file type is an optional filter. See TypeX. +Special type -2 = Binary, Compressed, Container, Executable. This special type includes everything except Documents, Video, Audio, Ebooks, Picture, Text. + +Request: GET /explore/node?limit=[max records]&type=[file type]&offset=[offset]&NodeID=[node id] +Result: 200 with JSON structure SearchResult. Check the field status. +*/ +func (api *WebapiInstance) apiExploreNodeID(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, err := strconv.Atoi(r.Form.Get("limit")) + if err != nil { + limit = 100 + } + // ID fields for results for a specific node ID. + NodeId, _ := DecodeBlake3Hash(r.Form.Get("NodeID")) + + fileType, err := strconv.Atoi(r.Form.Get("type")) + if err != nil { + fileType = -1 + } + + result := api.ExploreHelper(fileType, limit, offset, NodeId, true) + + EncodeJSON(api.backend, w, r, result) +} + +// ExploreHelper Helper function for the explore route with the possibility search based on a node ID +func (api *WebapiInstance) ExploreHelper(fileType int, limit, offset int, nodeID []byte, nodeIDState bool) *SearchResult { + resultFiles := api.queryRecentShared(api.backend, fileType, uint64(limit*20/100), uint64(offset), uint64(limit), nodeID, nodeIDState) + + var result SearchResult + result.Files = []apiFile{} + + // loop over results + for n := range resultFiles { + result.Files = append(result.Files, blockRecordFileToAPI(resultFiles[n])) + } + + if len(result.Files) == 0 { + result.Status = 3 // No results yet available keep trying + } + + result.Status = 1 // No more results to expect + + return &result +} + +func (input *SearchRequest) Parse() (Timeout time.Duration) { + if input.Timeout == 0 { + Timeout = time.Second * 20 // default timeout: 20 seconds + } else { + Timeout = time.Duration(input.Timeout) * time.Second + } + + return +} + +// ToSearchFilter converts the user input to a valid search filter +func (input *SearchRequest) ToSearchFilter() (output SearchFilter) { + return inputToSearchFilter(input.Sort, input.FileType, input.FileFormat, input.DateFrom, input.DateTo, input.SizeMin, input.SizeMax) +} + +func inputToSearchFilter(Sort, FileType, FileFormat int, DateFrom, DateTo string, SizeMin, SizeMax int) (output SearchFilter) { + output.Sort = Sort + output.FileType = FileType + output.FileFormat = FileFormat + output.SizeMin = SizeMin + output.SizeMax = SizeMax + + dateFrom, errFrom := time.Parse(apiDateFormat, DateFrom) + dateTo, errTo := time.Parse(apiDateFormat, DateTo) + if errFrom == nil && errTo == nil && dateFrom.Before(dateTo) { + output.DateFrom = dateFrom + output.DateTo = dateTo + output.IsDates = true + } + + return +} diff --git a/webapi/Shared Recent.go b/webapi/Shared Recent.go index ed44227..32487d9 100644 --- a/webapi/Shared Recent.go +++ b/webapi/Shared Recent.go @@ -1,103 +1,103 @@ -/* -File Name: Shared Recent.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ -package webapi - -import ( - "fmt" - "time" - - "github.com/PeernetOfficial/core" - "github.com/PeernetOfficial/core/blockchain" -) - -// queryRecentShared returns recently shared files on the network from random peers until the limit is reached. -func (api *WebapiInstance) queryRecentShared(backend *core.Backend, fileType int, limitPeer, offsetTotal, limitTotal uint64, nodeID []byte, nodeIDState bool) (files []blockchain.BlockRecordFile) { - if limitPeer == 0 { - limitPeer = 1 - } - - // Assign peer list as an empty array - var peerList []*core.PeerInfo - - // check if the NodeID is provided or not - if len(nodeID) == 0 && !nodeIDState { - // Use the peer list to know about active peers. Random order! - peerList = api.backend.PeerlistGet() - } else { - // Get peer information based on the NodeID provided - _, peer, err := api.backend.FindNode(nodeID, time.Second*5) - if err != nil { - return - } - peerList = append(peerList, peer) - } - - // Files from peers exceeding the limit. It is used if from all peers the total limit is not reached. - var filesSeconday []blockchain.BlockRecordFile - - for _, peer := range peerList { - if peer.BlockchainHeight == 0 { - continue - } - - var filesFromPeer uint64 - - // decode blocks from top down - blockLoop: - for blockN := peer.BlockchainHeight - 1; blockN > 0; blockN-- { - blockDecoded, _, found, _ := backend.ReadBlock(peer.PublicKey, peer.BlockchainVersion, blockN) - if !found { - continue - } - - for _, record := range blockDecoded.RecordsDecoded { - if file, ok := record.(blockchain.BlockRecordFile); ok && isFileTypeMatchBlock(&file, fileType) { - // add the tags 'Shared By Count' and 'Shared By GeoIP' - file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1)) - if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { - sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) - file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP)) - } - - // found a new file! append. - if filesFromPeer < limitPeer { - filesFromPeer++ - - if offsetTotal > 0 { - offsetTotal-- - continue - } - - files = append(files, file) - - if uint64(len(files)) >= limitTotal { - return - } - } else if uint64(len(filesSeconday)) < limitTotal-uint64(len(files)) { - filesSeconday = append(filesSeconday, file) - } else { - break blockLoop - } - } - } - } - } - - files = append(files, filesSeconday...) - - return -} - -// isFileTypeMatchBlock checks if the file type matches. -1 = accept any. -2 = core.TypeBinary, core.TypeCompressed, core.TypeContainer, core.TypeExecutable. -func isFileTypeMatchBlock(file *blockchain.BlockRecordFile, fileType int) bool { - if fileType == -1 { - return true - } else if fileType == -2 { - return file.Type == core.TypeBinary || file.Type == core.TypeCompressed || file.Type == core.TypeContainer || file.Type == core.TypeExecutable - } - - return file.Type == uint8(fileType) -} +/* +File Name: Shared Recent.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ +package webapi + +import ( + "fmt" + "time" + + "github.com/PeernetOfficial/core" + "github.com/PeernetOfficial/core/blockchain" +) + +// queryRecentShared returns recently shared files on the network from random peers until the limit is reached. +func (api *WebapiInstance) queryRecentShared(backend *core.Backend, fileType int, limitPeer, offsetTotal, limitTotal uint64, nodeID []byte, nodeIDState bool) (files []blockchain.BlockRecordFile) { + if limitPeer == 0 { + limitPeer = 1 + } + + // Assign peer list as an empty array + var peerList []*core.PeerInfo + + // check if the NodeID is provided or not + if len(nodeID) == 0 && !nodeIDState { + // Use the peer list to know about active peers. Random order! + peerList = api.backend.PeerlistGet() + } else { + // Get peer information based on the NodeID provided + _, peer, err := api.backend.FindNode(nodeID, time.Second*5) + if err != nil { + return + } + peerList = append(peerList, peer) + } + + // Files from peers exceeding the limit. It is used if from all peers the total limit is not reached. + var filesSeconday []blockchain.BlockRecordFile + + for _, peer := range peerList { + if peer.BlockchainHeight == 0 { + continue + } + + var filesFromPeer uint64 + + // decode blocks from top down + blockLoop: + for blockN := peer.BlockchainHeight - 1; blockN > 0; blockN-- { + blockDecoded, _, found, _ := backend.ReadBlock(peer.PublicKey, peer.BlockchainVersion, blockN) + if !found { + continue + } + + for _, record := range blockDecoded.RecordsDecoded { + if file, ok := record.(blockchain.BlockRecordFile); ok && isFileTypeMatchBlock(&file, fileType) { + // add the tags 'Shared By Count' and 'Shared By GeoIP' + file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1)) + if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { + sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) + file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP)) + } + + // found a new file! append. + if filesFromPeer < limitPeer { + filesFromPeer++ + + if offsetTotal > 0 { + offsetTotal-- + continue + } + + files = append(files, file) + + if uint64(len(files)) >= limitTotal { + return + } + } else if uint64(len(filesSeconday)) < limitTotal-uint64(len(files)) { + filesSeconday = append(filesSeconday, file) + } else { + break blockLoop + } + } + } + } + } + + files = append(files, filesSeconday...) + + return +} + +// isFileTypeMatchBlock checks if the file type matches. -1 = accept any. -2 = core.TypeBinary, core.TypeCompressed, core.TypeContainer, core.TypeExecutable. +func isFileTypeMatchBlock(file *blockchain.BlockRecordFile, fileType int) bool { + if fileType == -1 { + return true + } else if fileType == -2 { + return file.Type == core.TypeBinary || file.Type == core.TypeCompressed || file.Type == core.TypeContainer || file.Type == core.TypeExecutable + } + + return file.Type == uint8(fileType) +} diff --git a/webapi/Status.go b/webapi/Status.go index 5c0d0c8..e63735a 100644 --- a/webapi/Status.go +++ b/webapi/Status.go @@ -1,126 +1,126 @@ -/* -File Name: Status.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "encoding/hex" - "fmt" - "net/http" - "strconv" -) - -func apiTest(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - w.Write([]byte("ok")) -} - -type apiResponseStatus struct { - Status int `json:"status"` // Status code: 0 = Ok. - IsConnected bool `json:"isconnected"` // Whether connected to Peernet. - CountPeerList int `json:"countpeerlist"` // Count of peers in the peer list. Note that this contains peers that are considered inactive, but have not yet been removed from the list. - CountNetwork int `json:"countnetwork"` // Count of total peers in the network. - // This is usually a higher number than CountPeerList, which just represents the current number of connected peers. - // The CountNetwork number is going to be queried from root peers which may or may not have a limited view. -} - -/* -apiStatus returns the current connectivity status to the network -Request: GET /status -Result: 200 with JSON structure Status -*/ -func (api *WebapiInstance) apiStatus(w http.ResponseWriter, r *http.Request) { - status := apiResponseStatus{Status: 0, CountPeerList: api.Backend.PeerlistCount()} - status.CountNetwork = status.CountPeerList // For now always same as CountPeerList, until native Statistics message to root peers is available. - - // Connected: If at leat 2 peers. - // This metric needs to be improved in the future, as root peers never disconnect. - // Instead, the core should keep a count of "active peers". - status.IsConnected = status.CountPeerList >= 2 - - EncodeJSON(api.Backend, w, r, status) -} - -type apiResponsePeerSelf struct { - PeerID string `json:"peerid"` // Peer ID. This is derived from the public in compressed form. - NodeID string `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. -} - -/* -apiAccountInfo provides information about the current account. -Request: GET /account/info -Result: 200 with JSON structure apiResponsePeerSelf -*/ -func (api *WebapiInstance) apiAccountInfo(w http.ResponseWriter, r *http.Request) { - response := apiResponsePeerSelf{} - response.NodeID = hex.EncodeToString(api.Backend.SelfNodeID()) - - _, publicKey := api.Backend.ExportPrivateKey() - response.PeerID = hex.EncodeToString(publicKey.SerializeCompressed()) - - EncodeJSON(api.Backend, w, r, response) -} - -/* -apiAccountDelete deletes the current account. The confirm parameter must include the user's choice. -Request: GET /account/delete?confirm=[0 or 1] -Result: 204 if the user choses not to delete the account - - 200 if successfully deleted -*/ -func (api *WebapiInstance) apiAccountDelete(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - if confirm, _ := strconv.ParseBool(r.Form.Get("confirm")); !confirm { - w.WriteHeader(http.StatusNoContent) - return - } - - api.Backend.DeleteAccount() - - w.WriteHeader(http.StatusOK) -} - -/* -apiStatusPeers returns the information about peers currently connected. -The GeoIP information may not alawys be available, for example if the GeoIP file is not available or the mapping from IP address to location is not available. -Peers that are connected only via local network will not have a geo location. - -Request: GET /status/peers -Result: 200 with JSON array apiResponsePeerInfo -*/ -func (api *WebapiInstance) apiStatusPeers(w http.ResponseWriter, r *http.Request) { - var peers []apiResponsePeerInfo - - // query all nodes - for _, peer := range api.Backend.PeerlistGet() { - peerInfo := apiResponsePeerInfo{ - PeerID: peer.PublicKey.SerializeCompressed(), - NodeID: peer.NodeID, - UserAgent: peer.UserAgent, - IsRoot: peer.IsRootPeer, - BlockchainHeight: peer.BlockchainHeight, - BlockchainVersion: peer.BlockchainVersion, - } - - if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { - peerInfo.GeoIP = fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) - } - - peers = append(peers, peerInfo) - } - - EncodeJSON(api.Backend, w, r, peers) -} - -type apiResponsePeerInfo struct { - PeerID []byte `json:"peerid"` // Peer ID. This is derived from the public in compressed form. - NodeID []byte `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. - GeoIP string `json:"geoip"` // GeoIP location as "Latitude,Longitude" CSV format. Empty if location not available. - UserAgent string `json:"useragent"` // User Agent. - IsRoot bool `json:"isroot"` // If the peer is a root peer. - BlockchainHeight uint64 `json:"blockchainheight"` // Blockchain height - BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version -} +/* +File Name: Status.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "encoding/hex" + "fmt" + "net/http" + "strconv" +) + +func apiTest(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("ok")) +} + +type apiResponseStatus struct { + Status int `json:"status"` // Status code: 0 = Ok. + IsConnected bool `json:"isconnected"` // Whether connected to Peernet. + CountPeerList int `json:"countpeerlist"` // Count of peers in the peer list. Note that this contains peers that are considered inactive, but have not yet been removed from the list. + CountNetwork int `json:"countnetwork"` // Count of total peers in the network. + // This is usually a higher number than CountPeerList, which just represents the current number of connected peers. + // The CountNetwork number is going to be queried from root peers which may or may not have a limited view. +} + +/* +apiStatus returns the current connectivity status to the network +Request: GET /status +Result: 200 with JSON structure Status +*/ +func (api *WebapiInstance) apiStatus(w http.ResponseWriter, r *http.Request) { + status := apiResponseStatus{Status: 0, CountPeerList: api.Backend.PeerlistCount()} + status.CountNetwork = status.CountPeerList // For now always same as CountPeerList, until native Statistics message to root peers is available. + + // Connected: If at leat 2 peers. + // This metric needs to be improved in the future, as root peers never disconnect. + // Instead, the core should keep a count of "active peers". + status.IsConnected = status.CountPeerList >= 2 + + EncodeJSON(api.Backend, w, r, status) +} + +type apiResponsePeerSelf struct { + PeerID string `json:"peerid"` // Peer ID. This is derived from the public in compressed form. + NodeID string `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. +} + +/* +apiAccountInfo provides information about the current account. +Request: GET /account/info +Result: 200 with JSON structure apiResponsePeerSelf +*/ +func (api *WebapiInstance) apiAccountInfo(w http.ResponseWriter, r *http.Request) { + response := apiResponsePeerSelf{} + response.NodeID = hex.EncodeToString(api.Backend.SelfNodeID()) + + _, publicKey := api.Backend.ExportPrivateKey() + response.PeerID = hex.EncodeToString(publicKey.SerializeCompressed()) + + EncodeJSON(api.Backend, w, r, response) +} + +/* +apiAccountDelete deletes the current account. The confirm parameter must include the user's choice. +Request: GET /account/delete?confirm=[0 or 1] +Result: 204 if the user choses not to delete the account + + 200 if successfully deleted +*/ +func (api *WebapiInstance) apiAccountDelete(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + if confirm, _ := strconv.ParseBool(r.Form.Get("confirm")); !confirm { + w.WriteHeader(http.StatusNoContent) + return + } + + api.Backend.DeleteAccount() + + w.WriteHeader(http.StatusOK) +} + +/* +apiStatusPeers returns the information about peers currently connected. +The GeoIP information may not alawys be available, for example if the GeoIP file is not available or the mapping from IP address to location is not available. +Peers that are connected only via local network will not have a geo location. + +Request: GET /status/peers +Result: 200 with JSON array apiResponsePeerInfo +*/ +func (api *WebapiInstance) apiStatusPeers(w http.ResponseWriter, r *http.Request) { + var peers []apiResponsePeerInfo + + // query all nodes + for _, peer := range api.Backend.PeerlistGet() { + peerInfo := apiResponsePeerInfo{ + PeerID: peer.PublicKey.SerializeCompressed(), + NodeID: peer.NodeID, + UserAgent: peer.UserAgent, + IsRoot: peer.IsRootPeer, + BlockchainHeight: peer.BlockchainHeight, + BlockchainVersion: peer.BlockchainVersion, + } + + if latitude, longitude, valid := api.Peer2GeoIP(peer); valid { + peerInfo.GeoIP = fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude) + } + + peers = append(peers, peerInfo) + } + + EncodeJSON(api.Backend, w, r, peers) +} + +type apiResponsePeerInfo struct { + PeerID []byte `json:"peerid"` // Peer ID. This is derived from the public in compressed form. + NodeID []byte `json:"nodeid"` // Node ID. This is the blake3 hash of the peer ID and used in the DHT. + GeoIP string `json:"geoip"` // GeoIP location as "Latitude,Longitude" CSV format. Empty if location not available. + UserAgent string `json:"useragent"` // User Agent. + IsRoot bool `json:"isroot"` // If the peer is a root peer. + BlockchainHeight uint64 `json:"blockchainheight"` // Blockchain height + BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version +} diff --git a/webapi/Warehouse.go b/webapi/Warehouse.go index bb92a9e..95c5c2b 100644 --- a/webapi/Warehouse.go +++ b/webapi/Warehouse.go @@ -1,156 +1,156 @@ -/* -File Name: Warehouse.go -Copyright: 2021 Peernet Foundation s.r.o. -Author: Peter Kleissner -*/ - -package webapi - -import ( - "net/http" - "strconv" - - "github.com/PeernetOfficial/core/warehouse" -) - -// WarehouseResult is the response to creating a new file in the warehouse -type WarehouseResult struct { - Status int `json:"status"` // See warehouse.StatusX. - Hash []byte `json:"hash"` // Hash of the file. -} - -/* -apiWarehouseCreateFile creates a file in the warehouse. - -Request: POST /warehouse/create with raw data to create as new file -Response: 200 with JSON structure WarehouseResult -*/ -func (api *WebapiInstance) apiWarehouseCreateFile(w http.ResponseWriter, r *http.Request) { - hash, status, err := api.Backend.UserWarehouse.CreateFile(r.Body, 0) - - if err != nil { - api.Backend.LogError("warehouse.CreateFile", "status %d error: %v", status, err) - } - - EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash}) -} - -/* -apiWarehouseCreateFilePath creates a file in the warehouse by copying it from an existing file. -Warning: An attacker could supply any local file using this function, put them into storage and read them! No input path verification or limitation is done. -In the future the API should be secured using a random API key and setting the CORS header prohibiting regular browsers to access the API. - -Request: GET /warehouse/create/path?path=[target path on disk] -Response: 200 with JSON structure WarehouseResult -*/ -func (api *WebapiInstance) apiWarehouseCreateFilePath(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - filePath := r.Form.Get("path") - if filePath == "" { - http.Error(w, "", http.StatusBadRequest) - return - } - - hash, status, err := api.Backend.UserWarehouse.CreateFileFromPath(filePath) - - if err != nil { - api.Backend.LogError("warehouse.CreateFile", "status %d error: %v", status, err) - } - - EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash}) -} - -/* -apiWarehouseReadFile reads a file in the warehouse. - -Request: GET /warehouse/read?hash=[hash] - - Optional parameters &offset=[file offset]&limit=[read limit in bytes] - -Response: 200 with the raw file data - - 404 if file was not found - 500 in case of internal error opening the file -*/ -func (api *WebapiInstance) apiWarehouseReadFile(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) - if !valid1 { - http.Error(w, "", http.StatusBadRequest) - return - } - - offset, _ := strconv.Atoi(r.Form.Get("offset")) - limit, _ := strconv.Atoi(r.Form.Get("limit")) - - status, bytesRead, err := api.Backend.UserWarehouse.ReadFile(hash, int64(offset), int64(limit), w) - - switch status { - case warehouse.StatusFileNotFound: - w.WriteHeader(http.StatusNotFound) - return - case warehouse.StatusInvalidHash, warehouse.StatusErrorOpenFile, warehouse.StatusErrorSeekFile: - w.WriteHeader(http.StatusInternalServerError) - return - // Cannot catch warehouse.StatusErrorReadFile since data may have been already returned. - // In the future a special header indicating the expected file length could be sent (would require a callback in ReadFile), although the caller should already know the file size based on metadata. - } - - if err != nil { - api.Backend.LogError("warehouse.ReadFile", "status %d read %d error: %v", status, bytesRead, err) - } -} - -/* -apiWarehouseDeleteFile deletes a file in the warehouse. - -Request: GET /warehouse/delete?hash=[hash] -Response: 200 with JSON structure WarehouseResult -*/ -func (api *WebapiInstance) apiWarehouseDeleteFile(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) - if !valid1 { - http.Error(w, "", http.StatusBadRequest) - return - } - - status, err := api.Backend.UserWarehouse.DeleteFile(hash) - - if err != nil { - api.Backend.LogError("warehouse.DeleteFile", "status %d error: %v", status, err) - } - - EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash}) -} - -/* -apiWarehouseReadFilePath reads a file from the warehouse and stores it to the target file. It fails with StatusErrorTargetExists if the target file already exists. -The path must include the full directory and file name. - -Request: GET /warehouse/read/path?hash=[hash]&path=[target path on disk] - - Optional parameters &offset=[file offset]&limit=[read limit in bytes] - -Response: 200 with JSON structure WarehouseResult -*/ -func (api *WebapiInstance) apiWarehouseReadFilePath(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) - if !valid1 { - http.Error(w, "", http.StatusBadRequest) - return - } - - targetFile := r.Form.Get("path") - offset, _ := strconv.Atoi(r.Form.Get("offset")) - limit, _ := strconv.Atoi(r.Form.Get("limit")) - - status, bytesRead, err := api.Backend.UserWarehouse.ReadFileToDisk(hash, int64(offset), int64(limit), targetFile) - - if err != nil { - api.Backend.LogError("warehouse.ReadFileToDisk", "status %d read %d error: %v", status, bytesRead, err) - } - - EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash}) -} +/* +File Name: Warehouse.go +Copyright: 2021 Peernet Foundation s.r.o. +Author: Peter Kleissner +*/ + +package webapi + +import ( + "net/http" + "strconv" + + "github.com/PeernetOfficial/core/warehouse" +) + +// WarehouseResult is the response to creating a new file in the warehouse +type WarehouseResult struct { + Status int `json:"status"` // See warehouse.StatusX. + Hash []byte `json:"hash"` // Hash of the file. +} + +/* +apiWarehouseCreateFile creates a file in the warehouse. + +Request: POST /warehouse/create with raw data to create as new file +Response: 200 with JSON structure WarehouseResult +*/ +func (api *WebapiInstance) apiWarehouseCreateFile(w http.ResponseWriter, r *http.Request) { + hash, status, err := api.Backend.UserWarehouse.CreateFile(r.Body, 0) + + if err != nil { + api.Backend.LogError("warehouse.CreateFile", "status %d error: %v", status, err) + } + + EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash}) +} + +/* +apiWarehouseCreateFilePath creates a file in the warehouse by copying it from an existing file. +Warning: An attacker could supply any local file using this function, put them into storage and read them! No input path verification or limitation is done. +In the future the API should be secured using a random API key and setting the CORS header prohibiting regular browsers to access the API. + +Request: GET /warehouse/create/path?path=[target path on disk] +Response: 200 with JSON structure WarehouseResult +*/ +func (api *WebapiInstance) apiWarehouseCreateFilePath(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + filePath := r.Form.Get("path") + if filePath == "" { + http.Error(w, "", http.StatusBadRequest) + return + } + + hash, status, err := api.Backend.UserWarehouse.CreateFileFromPath(filePath) + + if err != nil { + api.Backend.LogError("warehouse.CreateFile", "status %d error: %v", status, err) + } + + EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash}) +} + +/* +apiWarehouseReadFile reads a file in the warehouse. + +Request: GET /warehouse/read?hash=[hash] + + Optional parameters &offset=[file offset]&limit=[read limit in bytes] + +Response: 200 with the raw file data + + 404 if file was not found + 500 in case of internal error opening the file +*/ +func (api *WebapiInstance) apiWarehouseReadFile(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + if !valid1 { + http.Error(w, "", http.StatusBadRequest) + return + } + + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, _ := strconv.Atoi(r.Form.Get("limit")) + + status, bytesRead, err := api.Backend.UserWarehouse.ReadFile(hash, int64(offset), int64(limit), w) + + switch status { + case warehouse.StatusFileNotFound: + w.WriteHeader(http.StatusNotFound) + return + case warehouse.StatusInvalidHash, warehouse.StatusErrorOpenFile, warehouse.StatusErrorSeekFile: + w.WriteHeader(http.StatusInternalServerError) + return + // Cannot catch warehouse.StatusErrorReadFile since data may have been already returned. + // In the future a special header indicating the expected file length could be sent (would require a callback in ReadFile), although the caller should already know the file size based on metadata. + } + + if err != nil { + api.Backend.LogError("warehouse.ReadFile", "status %d read %d error: %v", status, bytesRead, err) + } +} + +/* +apiWarehouseDeleteFile deletes a file in the warehouse. + +Request: GET /warehouse/delete?hash=[hash] +Response: 200 with JSON structure WarehouseResult +*/ +func (api *WebapiInstance) apiWarehouseDeleteFile(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + if !valid1 { + http.Error(w, "", http.StatusBadRequest) + return + } + + status, err := api.Backend.UserWarehouse.DeleteFile(hash) + + if err != nil { + api.Backend.LogError("warehouse.DeleteFile", "status %d error: %v", status, err) + } + + EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash}) +} + +/* +apiWarehouseReadFilePath reads a file from the warehouse and stores it to the target file. It fails with StatusErrorTargetExists if the target file already exists. +The path must include the full directory and file name. + +Request: GET /warehouse/read/path?hash=[hash]&path=[target path on disk] + + Optional parameters &offset=[file offset]&limit=[read limit in bytes] + +Response: 200 with JSON structure WarehouseResult +*/ +func (api *WebapiInstance) apiWarehouseReadFilePath(w http.ResponseWriter, r *http.Request) { + r.ParseForm() + hash, valid1 := DecodeBlake3Hash(r.Form.Get("hash")) + if !valid1 { + http.Error(w, "", http.StatusBadRequest) + return + } + + targetFile := r.Form.Get("path") + offset, _ := strconv.Atoi(r.Form.Get("offset")) + limit, _ := strconv.Atoi(r.Form.Get("limit")) + + status, bytesRead, err := api.Backend.UserWarehouse.ReadFileToDisk(hash, int64(offset), int64(limit), targetFile) + + if err != nil { + api.Backend.LogError("warehouse.ReadFileToDisk", "status %d read %d error: %v", status, bytesRead, err) + } + + EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash}) +}