webapi: Implement GeoIP.

Config Default.yaml

@@ -4,6 +4,7 @@ BlockchainMain:   "data/blockchain main/"       # Blockchain main stores the end
 BlockchainGlobal: "data/blockchain global/"     # Blockchain global caches blockchain data from global users. Empty to disable.
 WarehouseMain:    "data/warehouse main/"        # Warehouse main stores the actual data of files shared by the end-user.
 SearchIndex:      "data/search index/"          # Local search index of blockchain records. Empty to disable.
+GeoIPDatabase:    "data/GeoLite2-City.mmdb"     # GeoLite2 City database to provide GeoIP information.
 
 # Listen defines all IP:Port combinations to listen on. If empty, it will listen on all IPs automatically on available ports.
 # IPv6 must be in the form "[IPv6]:Port". This setting is only recommended to be set on servers.

go.mod

@@ -3,6 +3,7 @@ module github.com/PeernetOfficial/core
 go 1.16
 
 require (
+	github.com/IncSW/geoip2 v0.1.1
 	github.com/akrylysov/pogreb v0.10.1
 	github.com/enfipy/locker v1.1.0
 	github.com/google/uuid v1.3.0

go.sum

@@ -1,3 +1,5 @@
+github.com/IncSW/geoip2 v0.1.1 h1:afzzYF7n9JbdcPy8aiBSgBJuXi4mTWXZ3z6V3o6Vg34=
+github.com/IncSW/geoip2 v0.1.1/go.mod h1:adcasR40vXiUBjtzdaTTKL/6wSf+fgO4M8Gve/XzPUk=
 github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
 github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
 github.com/enfipy/locker v1.1.0 h1:2zVJ0ky7cS1Vjs0x6OQWFiT2dSEiHrI5/O2KCz1fgGc=

webapi/API.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/IncSW/geoip2"
 	"github.com/PeernetOfficial/core"
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
@@ -20,7 +21,8 @@ import (
 )
 
 type WebapiInstance struct {
-	backend *core.Backend
+	backend         *core.Backend
+	geoipCityReader *geoip2.CityReader
 
 	// Router can be used to register additional API functions
 	Router          *mux.Router

webapi/GeoIP.go

@@ -0,0 +1,50 @@
+/*
+File Name:  GeoIP.go
+Copyright:  2021 Peernet Foundation s.r.o.
+Author:     Peter Kleissner
+
+Support for the free MaxMind database 'GeoLite2 City'.
+Information about the database: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data
+
+Potential libraries:
+* https://github.com/IncSW/geoip2
+* https://github.com/oschwald/maxminddb-golang
+
+The IncSW lib was chosen because it has 0 dependencies - awesome!
+
+*/
+
+package webapi
+
+import (
+	"net"
+
+	"github.com/IncSW/geoip2"
+	"github.com/PeernetOfficial/core"
+)
+
+func (api *WebapiInstance) InitGeoIPDatabase(filename string) (err error) {
+	api.geoipCityReader, err = geoip2.NewCityReaderFromFile(filename)
+	return err
+}
+
+func (api *WebapiInstance) GeoIPLocation(IP net.IP) (latitude, longitude float64, valid bool) {
+	if api.geoipCityReader == nil {
+		return 0, 0, false
+	}
+
+	record, err := api.geoipCityReader.Lookup(IP)
+	if err != nil {
+		return 0, 0, false
+	}
+
+	return record.Location.Latitude, record.Location.Longitude, true
+}
+
+func (api *WebapiInstance) Peer2GeoIP(peer *core.PeerInfo) (latitude, longitude float64, valid bool) {
+	if connection := peer.GetConnection2Share(false, true, true); connection != nil {
+		return api.GeoIPLocation(connection.Address.IP)
+	}
+
+	return 0, 0, false
+}

webapi/Search Dispatch.go

@@ -3,10 +3,12 @@ File Name:  Search Dispatch.go
 Copyright:  2021 Peernet Foundation s.r.o.
 Author:     Peter Kleissner
 */
+
 package webapi
 
 import (
 	"bytes"
+	"fmt"
 	"time"
 
 	"github.com/PeernetOfficial/core"
@@ -22,25 +24,25 @@ func (api *WebapiInstance) dispatchSearch(input SearchRequest) (job *SearchJob)
 
 	// todo: create actual search clients!
 
-	go job.localSearch(api.backend, input.Term)
+	go job.localSearch(api, input.Term)
 
 	job.RemoveDefer(job.timeout + time.Minute*10)
 
 	return job
 }
 
-func (job *SearchJob) localSearch(backend *core.Backend, term string) {
+func (job *SearchJob) localSearch(api *WebapiInstance, term string) {
-	if backend.SearchIndex == nil || backend.GlobalBlockchainCache == nil {
+	if api.backend.SearchIndex == nil || api.backend.GlobalBlockchainCache == nil {
 		return
 	}
 
-	results := backend.SearchIndex.Search(term)
+	results := api.backend.SearchIndex.Search(term)
 
 	job.ResultSync.Lock()
 
 resultLoop:
 	for _, result := range results {
-		file, _, found, err := backend.GlobalBlockchainCache.ReadFile(result.PublicKey, result.BlockchainVersion, result.BlockNumber, result.FileID)
+		file, _, found, err := api.backend.GlobalBlockchainCache.ReadFile(result.PublicKey, result.BlockchainVersion, result.BlockNumber, result.FileID)
 		if err != nil || !found {
 			continue
 		}
@@ -53,7 +55,12 @@ resultLoop:
 		}
 
 		if peer := core.NodelistLookup(file.NodeID); peer != nil {
+			// add the tags 'Shared By Count' and 'Shared By GeoIP'
 			file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1))
+			if latitude, longitude, valid := api.Peer2GeoIP(peer); valid {
+				sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude)
+				file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP))
+			}
 		}
 
 		// new result

webapi/Search.go

@@ -343,7 +343,7 @@ func (api *WebapiInstance) apiExplore(w http.ResponseWriter, r *http.Request) {
 		fileType = -1
 	}
 
-	resultFiles := queryRecentShared(api.backend, fileType, uint64(limit*20/100), uint64(limit))
+	resultFiles := api.queryRecentShared(api.backend, fileType, uint64(limit*20/100), uint64(limit))
 
 	var result SearchResult
 	result.Files = []apiFile{}

webapi/Shared Recent.go

@@ -6,12 +6,14 @@ Author:     Peter Kleissner
 package webapi
 
 import (
+	"fmt"
+
 	"github.com/PeernetOfficial/core"
 	"github.com/PeernetOfficial/core/blockchain"
 )
 
 // queryRecentShared returns recently shared files on the network from random peers until the limit is reached.
-func queryRecentShared(backend *core.Backend, fileType int, limitPeer, limitTotal uint64) (files []blockchain.BlockRecordFile) {
+func (api *WebapiInstance) queryRecentShared(backend *core.Backend, fileType int, limitPeer, limitTotal uint64) (files []blockchain.BlockRecordFile) {
 	if limitPeer == 0 {
 		limitPeer = 1
 	}
@@ -39,7 +41,12 @@ func queryRecentShared(backend *core.Backend, fileType int, limitPeer, limitTota
 
 			for _, record := range blockDecoded.RecordsDecoded {
 				if file, ok := record.(blockchain.BlockRecordFile); ok && isFileTypeMatchBlock(&file, fileType) {
+					// add the tags 'Shared By Count' and 'Shared By GeoIP'
 					file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1))
+					if latitude, longitude, valid := api.Peer2GeoIP(peer); valid {
+						sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude)
+						file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP))
+					}
 
 					// found a new file! append.
 					if filesFromPeer < limitPeer {