Merge pull request #108 from PeernetOfficial/api-merge-directory

Api merge directory
2026-07-17 02:47:51 +01:00 · 2023-04-04 18:46:41 +01:00
parent b34b2e2b7c a739485931
commit 97c3001596
5 changed files with 250 additions and 88 deletions
--- a/search/Search
+++ b/search/Search
@@ -0,0 +1,24 @@
+package search
+
+import (
+	"github.com/PeernetOfficial/core/protocol"
+	"github.com/google/uuid"
+)
+
+// SearchNodeIDBasedOnHash Provides a list of NodeIDs
+// based on the hash provided
+// This is used to find out which nodes are hosting
+// which files based on the hash provided
+func (index *SearchIndexStore) SearchNodeIDBasedOnHash(hash []byte) (NodeIDs [][]byte, err error) {
+	var resultMap map[uuid.UUID]*SearchIndexRecord
+	err = index.LookupHash(SearchSelector{Hash: hash}, resultMap)
+	if err != nil {
+		return
+	}
+
+	for i := range resultMap {
+		NodeIDs = append(NodeIDs, protocol.PublicKey2NodeID(resultMap[i].PublicKey))
+	}
+
+	return
+}
--- a/webapi/API.go
+++ b/webapi/API.go
@@ -81,6 +81,7 @@ func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, Certifi
 	api.Router.HandleFunc("/blockchain/file/delete", api.apiBlockchainFileDelete).Methods("POST")
 	api.Router.HandleFunc("/blockchain/file/update", api.apiBlockchainFileUpdate).Methods("POST")
 	api.Router.HandleFunc("/blockchain/view", api.apiExploreNodeID).Methods("GET")
+	api.Router.HandleFunc("/merge/directory", api.apiMergeDirectory).Methods("GET")
 	api.Router.HandleFunc("/profile/list", api.apiProfileList).Methods("GET")
 	api.Router.HandleFunc("/profile/read", api.apiProfileRead).Methods("GET")
 	api.Router.HandleFunc("/profile/write", api.apiProfileWrite).Methods("POST")
--- a/webapi/Blockchain.go
+++ b/webapi/Blockchain.go
@@ -7,18 +7,17 @@ Author:     Peter Kleissner
 package webapi

 import (
-	"bytes"
 	"encoding/hex"
 	"net/http"
 	"strconv"

-    "github.com/PeernetOfficial/core/blockchain"
+	"github.com/PeernetOfficial/core/blockchain"
 )

 type apiBlockchainHeader struct {
-    PeerID  string `json:"peerid"`  // Peer ID hex encoded.
-    Version uint64 `json:"version"` // Current version number of the blockchain.
-    Height  uint64 `json:"height"`  // Height of the blockchain (number of blocks). If 0, no data exists.
+	PeerID  string `json:"peerid"`  // Peer ID hex encoded.
+	Version uint64 `json:"version"` // Current version number of the blockchain.
+	Height  uint64 `json:"height"`  // Height of the blockchain (number of blocks). If 0, no data exists.
 }

 /*
@@ -28,25 +27,25 @@ Request:    GET /blockchain/header
 Result:     200 with JSON structure apiResponsePeerSelf
 */
 func (api *WebapiInstance) apiBlockchainHeaderFunc(w http.ResponseWriter, r *http.Request) {
-    publicKey, height, version := api.Backend.UserBlockchain.Header()
+	publicKey, height, version := api.Backend.UserBlockchain.Header()

-    EncodeJSON(api.Backend, w, r, apiBlockchainHeader{Version: version, Height: height, PeerID: hex.EncodeToString(publicKey.SerializeCompressed())})
+	EncodeJSON(api.Backend, w, r, apiBlockchainHeader{Version: version, Height: height, PeerID: hex.EncodeToString(publicKey.SerializeCompressed())})
 }

 type apiBlockRecordRaw struct {
-    Type uint8  `json:"type"` // Record Type. See core.RecordTypeX.
-    Data []byte `json:"data"` // Data according to the type.
+	Type uint8  `json:"type"` // Record Type. See core.RecordTypeX.
+	Data []byte `json:"data"` // Data according to the type.
 }

 // apiBlockchainBlockRaw contains a raw block of the blockchain via API
 type apiBlockchainBlockRaw struct {
-    Records []apiBlockRecordRaw `json:"records"` // Block records in encoded raw format.
+	Records []apiBlockRecordRaw `json:"records"` // Block records in encoded raw format.
 }

 type apiBlockchainBlockStatus struct {
-    Status  int    `json:"status"`  // See blockchain.StatusX.
-    Height  uint64 `json:"height"`  // Height of the blockchain (number of blocks).
-    Version uint64 `json:"version"` // Version of the blockchain.
+	Status  int    `json:"status"`  // See blockchain.StatusX.
+	Height  uint64 `json:"height"`  // Height of the blockchain (number of blocks).
+	Version uint64 `json:"version"` // Version of the blockchain.
 }

 /*
@@ -57,30 +56,30 @@ Request:    POST /blockchain/append with JSON structure apiBlockchainBlockRaw
 Response:   200 with JSON structure apiBlockchainBlockStatus
 */
 func (api *WebapiInstance) apiBlockchainAppend(w http.ResponseWriter, r *http.Request) {
-    var input apiBlockchainBlockRaw
-    if err := DecodeJSON(w, r, &input); err != nil {
-        return
-    }
+	var input apiBlockchainBlockRaw
+	if err := DecodeJSON(w, r, &input); err != nil {
+		return
+	}

-    var records []blockchain.BlockRecordRaw
+	var records []blockchain.BlockRecordRaw

-    for _, record := range input.Records {
-        records = append(records, blockchain.BlockRecordRaw{Type: record.Type, Data: record.Data})
-    }
+	for _, record := range input.Records {
+		records = append(records, blockchain.BlockRecordRaw{Type: record.Type, Data: record.Data})
+	}

-    newHeight, newVersion, status := api.Backend.UserBlockchain.Append(records)
+	newHeight, newVersion, status := api.Backend.UserBlockchain.Append(records)

-    EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion})
+	EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion})
 }

 type apiBlockchainBlock struct {
-    Status            int                 `json:"status"`            // See blockchain.StatusX.
-    PeerID            string              `json:"peerid"`            // Peer ID hex encoded.
-    LastBlockHash     []byte              `json:"lastblockhash"`     // Hash of the last block. Blake3.
-    BlockchainVersion uint64              `json:"blockchainversion"` // Blockchain version
-    Number            uint64              `json:"blocknumber"`       // Block number
-    RecordsRaw        []apiBlockRecordRaw `json:"recordsraw"`        // Records raw. Successfully decoded records are parsed into the below fields.
-    RecordsDecoded    []interface{}       `json:"recordsdecoded"`    // Records decoded. The encoding for each record depends on its type.
+	Status            int                 `json:"status"`            // See blockchain.StatusX.
+	PeerID            string              `json:"peerid"`            // Peer ID hex encoded.
+	LastBlockHash     []byte              `json:"lastblockhash"`     // Hash of the last block. Blake3.
+	BlockchainVersion uint64              `json:"blockchainversion"` // Blockchain version
+	Number            uint64              `json:"blocknumber"`       // Block number
+	RecordsRaw        []apiBlockRecordRaw `json:"recordsraw"`        // Records raw. Successfully decoded records are parsed into the below fields.
+	RecordsDecoded    []interface{}       `json:"recordsdecoded"`    // Records decoded. The encoding for each record depends on its type.
 }

 /*
@@ -90,63 +89,36 @@ Request:    GET /blockchain/read?block=[number]
 Result:     200 with JSON structure apiBlockchainBlock
 */
 func (api *WebapiInstance) apiBlockchainRead(w http.ResponseWriter, r *http.Request) {
-    r.ParseForm()
-    blockN, err := strconv.Atoi(r.Form.Get("block"))
-    if err != nil || blockN < 0 {
-        http.Error(w, "", http.StatusBadRequest)
-        return
-    }
+	r.ParseForm()
+	blockN, err := strconv.Atoi(r.Form.Get("block"))
+	if err != nil || blockN < 0 {
+		http.Error(w, "", http.StatusBadRequest)
+		return
+	}

-    block, status, _ := api.Backend.UserBlockchain.Read(uint64(blockN))
-    result := apiBlockchainBlock{Status: status}
+	block, status, _ := api.Backend.UserBlockchain.Read(uint64(blockN))
+	result := apiBlockchainBlock{Status: status}

-    if status == 0 {
-        for _, record := range block.RecordsRaw {
-            result.RecordsRaw = append(result.RecordsRaw, apiBlockRecordRaw{Type: record.Type, Data: record.Data})
-        }
+	if status == 0 {
+		for _, record := range block.RecordsRaw {
+			result.RecordsRaw = append(result.RecordsRaw, apiBlockRecordRaw{Type: record.Type, Data: record.Data})
+		}

-        result.PeerID = hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())
+		result.PeerID = hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())

-        for _, record := range block.RecordsDecoded {
-            switch v := record.(type) {
-            case blockchain.BlockRecordFile:
-                result.RecordsDecoded = append(result.RecordsDecoded, blockRecordFileToAPI(v))
+		for _, record := range block.RecordsDecoded {
+			switch v := record.(type) {
+			case blockchain.BlockRecordFile:
+				result.RecordsDecoded = append(result.RecordsDecoded, blockRecordFileToAPI(v))

-            case blockchain.BlockRecordProfile:
-                result.RecordsDecoded = append(result.RecordsDecoded, blockRecordProfileToAPI(v))
+			case blockchain.BlockRecordProfile:
+				result.RecordsDecoded = append(result.RecordsDecoded, blockRecordProfileToAPI(v))

-            }
-        }
-    }
+			}
+		}
+	}

-    EncodeJSON(api.Backend, w, r, result)
-}
-
-/*
-apiExploreNodeID returns the shared files of a particular node in Peernet. Results are returned in real-time. The file type is an optional filter. See TypeX.
-Special type -2 = Binary, Compressed, Container, Executable. This special type includes everything except Documents, Video, Audio, Ebooks, Picture, Text.
-
-Request:    GET /blockchain/view?limit=[max records]&type=[file type]&offset=[offset]&node=[node id]
-Result:     200 with JSON structure SearchResult. Check the field status.
-*/
-func (api *WebapiInstance) apiExploreNodeID(w http.ResponseWriter, r *http.Request) {
-    r.ParseForm()
-    offset, _ := strconv.Atoi(r.Form.Get("offset"))
-    limit, err := strconv.Atoi(r.Form.Get("limit"))
-    if err != nil {
-        limit = 100
-    }
-    // ID fields for results for a specific node ID.
-    NodeId, _ := DecodeBlake3Hash(r.Form.Get("node"))
-
-    fileType, err := strconv.Atoi(r.Form.Get("type"))
-    if err != nil {
-        fileType = -1
-    }
-
-    result := api.ExploreHelper(fileType, limit, offset, NodeId, true)
-
-    EncodeJSON(api.Backend, w, r, result)
+	EncodeJSON(api.Backend, w, r, result)
 }

 /*
@@ -171,14 +143,6 @@ func (api *WebapiInstance) apiExploreNodeID(w http.ResponseWriter, r *http.Reque
 		fileType = -1
 	}

-	// check if the node IDs are the same
-	if bytes.Compare(NodeId, api.Backend.SelfNodeID()) == 0 {
-		// setting the value of block to 0
-		r.Form.Add("block", "0")
-		api.apiBlockchainRead(w, r)
-		return
-	}
-
 	result := api.ExploreHelper(fileType, limit, offset, NodeId, true)

 	EncodeJSON(api.Backend, w, r, result)
--- a/directory.go
+++ b/directory.go
@@ -0,0 +1,143 @@
+package webapi
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/PeernetOfficial/core/blockchain"
+	"net/http"
+	"strconv"
+)
+
+// SearchResultMergedDirectory contains results for the merged directory.
+type SearchResultMergedDirectory struct {
+	Status    int         `json:"status"`    // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying
+	Files     []apiFile   `json:"files"`     // List of files found
+	Statistic interface{} `json:"statistic"` // Statistics of all results (independent from applied filters), if requested. Only set if files are returned (= if statistics changed). See SearchStatisticData.
+}
+
+/*
+apiMergeDirectory Shows the recent files of peers that shared
+the same file as the one provided in the GET request.
+Currently searches through Memory for Nodes currently
+identified in the network and then checks if the files
+they shared match with the hash that is provided
+in the search parameter and the queries the recent
+file that node shared and then returns that result
+back.
+
+Request:    GET /merge/directory with JSON structure apiMergeDirectory
+Response:   200 with JSON structure SearchResultMergedDirectory
+
+	400 if invalid input
+*/
+func (api *WebapiInstance) apiMergeDirectory(w http.ResponseWriter, r *http.Request) {
+	r.ParseForm()
+	offset, _ := strconv.Atoi(r.Form.Get("offset"))
+	limit, err := strconv.Atoi(r.Form.Get("limit"))
+	if err != nil {
+		limit = 100
+	}
+	// ID fields for results for a specific node ID.
+	//NodeId, _ := DecodeBlake3Hash(r.Form.Get("node"))
+	hash, _ := DecodeBlake3Hash(r.Form.Get("hash"))
+
+	fileType, err := strconv.Atoi(r.Form.Get("type"))
+	if err != nil {
+		fileType = -1
+	}
+
+	result := api.ExploreFileSharedByNodeThatSharedSimilarFile(fileType, limit, offset, hash, true)
+
+	EncodeJSON(api.Backend, w, r, result)
+}
+
+// ExploreFileSharedByNodeThatSharedSimilarFile lists files shared by a nodes which share the same file as a common point
+// Currently this is a greedy search and requires work for optimization
+func (api *WebapiInstance) ExploreFileSharedByNodeThatSharedSimilarFile(fileType int, limit, offset int, hash []byte, nodeIDState bool) *SearchResultMergedDirectory {
+	// lookup all NodeID blockchains which have the similar hash
+	// do a search to get all the node IDs sharing the particular file
+	var NodeIDs [][]byte
+	NodeIDs, _ = api.Backend.SearchIndex.SearchNodeIDBasedOnHash(hash)
+
+	// Does a greedy search to find node serving similar files
+	api.GreedySearchMergeDirection(&NodeIDs, fileType, hash)
+
+	var result SearchResultMergedDirectory
+
+	for i := range NodeIDs {
+		resultFiles := api.queryRecentShared(api.Backend, fileType, uint64(limit*20/100), uint64(offset), uint64(limit), NodeIDs[i], nodeIDState)
+
+		// loop over results
+		for n := range resultFiles {
+			result.Files = append(result.Files, blockRecordFileToAPI(resultFiles[n]))
+		}
+
+	}
+
+	result.Status = 1 // No more results to expect
+
+	return &result
+}
+
+// GreedySearchMergeDirection This function is implemented since the local index tables do not
+// always consist of the required hashes of the NodeIDs.
+func (api *WebapiInstance) GreedySearchMergeDirection(nodeID *[][]byte, fileType int, hash []byte) {
+
+	// get all NodeIDs
+	peerList := api.Backend.PeerlistGet()
+
+	//var tags []blockchain.BlockRecordFileTag
+
+	// search with AllNodes which have a match of the NodeID.
+	for _, peer := range peerList {
+		if peer.BlockchainHeight == 0 {
+			continue
+		}
+
+		var filesFromPeer uint64
+
+		// decode blocks from top down
+		for blockN := peer.BlockchainHeight - 1; blockN > 0; blockN-- {
+			blockDecoded, _, found, _ := api.Backend.ReadBlock(peer.PublicKey, peer.BlockchainVersion, blockN)
+			if !found {
+				continue
+			}
+
+			for _, record := range blockDecoded.RecordsDecoded {
+				if file, ok := record.(blockchain.BlockRecordFile); ok && isFileTypeMatchBlock(&file, fileType) {
+					// add the tags 'Shared By Count' and 'Shared By GeoIP'
+					file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1))
+					if latitude, longitude, valid := api.Peer2GeoIP(peer); valid {
+						sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude)
+						file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP))
+					}
+
+					// found a new file! append.
+					filesFromPeer++
+
+					// if both the hashes match
+					if bytes.Equal(file.Hash, hash) {
+						// set the Tags needed for the filter parameter
+						//tags = file.Tags
+						*nodeID = append(*nodeID, file.NodeID)
+					}
+
+					//for _, tag := range file.Tags {
+					//	// checks if any of tags from
+					//	// the NodeID provided matches
+					//	// Requires better search parameters
+					//	// So that it's more narrow
+					//	for i := range tags {
+					//		if tag.Text() == tags[i].Text() {
+					//			*nodeID = append(*nodeID, file.NodeID)
+					//			break
+					//		}
+					//	}
+					//}
+				}
+			}
+
+		}
+	}
+
+}
--- a/webapi/readme.md
+++ b/webapi/readme.md
@@ -72,6 +72,9 @@ These are the functions provided by the API:
 /warehouse/read                 Read a file in the warehouse
 /warehouse/read/path            Read a file in the warehouse to disk
 /warehouse/delete               Delete a file in the warehouse
+
+/merge/directory                List all recent files shared by peers based 
+                                on the similar file shared
 ```

 # API Documentation
@@ -1099,3 +1102,30 @@ Response:   200 with JSON structure WarehouseResult
 ```

 Example request: `http://127.0.0.1:112/warehouse/delete?hash=dbf344f23e7820261329883ae26f64929a7c9977549001d28dc40c9202d7651e`
+
+### Merge Directory
+Shows the recent files of peers that shared
+the same file as the one provided in the GET request.
+Currently searches through Memory for Nodes currently 
+identified in the network and then checks if the files 
+they shared match with the hash that is provided 
+in the search parameter and the queries the recent 
+file that node shared and then returns that result 
+back.
+
+```
+Request:    GET /merge/directory?hash=[hash]
+Response:   200 with JSON structure SearchResultMergedDirectory
+```
+
+Example request: `http://127.0.0.1:112/merge/directory?hash=dbf344f23e7820261329883ae26f64929a7c9977549001d28dc40c9202d7651e`
+
+```go
+// SearchResultMergedDirectory contains results for the merged directory.
+type SearchResultMergedDirectory struct {
+	Status    int         `json:"status"`    // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying
+	Files     []apiFile   `json:"files"`     // List of files found
+	Statistic interface{} `json:"statistic"` // Statistics of all results (independent from applied filters), if requested. Only set if files are returned (= if statistics changed). See SearchStatisticData.
+}
+```
+