Peernet-Archive/Abstraction
1
0
Fork 0
mirror of https://github.com/PeernetOfficial/Abstraction.git synced 2026-07-17 02:57:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

added .gitignore changes

Browse Source
This commit is contained in:
Akilan Selvacoumar
2022-12-10 06:35:26 +00:00
parent 653c56e275
commit be7ea26502
21 changed files with 0 additions and 4733 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -1,2 +0,0 @@
.idea/
.DS_Store

302
files.go
View File

@@ -1,302 +0,0 @@
/*
File Name: abstractions.go
Copyright: 2021 Peernet s.r.o.
Authors: Peter Kleissner, Akilan Selvacoumar
*/
package Abstrations
import (
"encoding/hex"
"errors"
"github.com/PeernetOfficial/Abstraction/webapi"
"github.com/PeernetOfficial/core/blockchain"
"github.com/PeernetOfficial/core/protocol"
"github.com/PeernetOfficial/core/warehouse"
"github.com/google/uuid"
"math"
"path/filepath"
"time"
)
/*
Library description
to about abstracted function to easily add and remove files.
*/
type TouchReturn struct {
BlockchainHeight uint64
BlockchainVersion uint64
}
// Touch abstracted function that creates a file
// and adds the file to the warehouse and
// blockchain
// returns blockchain version and height
func Touch(api *webapi.WebapiInstance, filePath string) (*TouchReturn, error) {
// Creates a File in the warehouse
hash, _, err := api.Backend.UserWarehouse.CreateFileFromPath(filePath)
if err != nil {
return nil, err
}
// Add the File to the local blockchain
var input webapi.ApiBlockAddFiles
var inputFiles []webapi.ApiFile
var inputFile webapi.ApiFile
// Write File information to the input File
inputFile.Date = time.Now()
// Folder and File name
dir, file := filepath.Split(filePath)
inputFile.Folder = dir
inputFile.Name = file
inputFile.ID = uuid.New()
inputFile.Hash = hash
// Get the public key of the current node
_, publicKey := api.Backend.ExportPrivateKey()
inputFile.NodeID = []byte(hex.EncodeToString(publicKey.SerializeCompressed()))
inputFiles = append(inputFiles, inputFile)
input.Files = inputFiles
var filesAdd []blockchain.BlockRecordFile
for _, File := range input.Files {
if len(File.Hash) != protocol.HashSize {
return nil, errors.New("bad request")
}
if File.ID == uuid.Nil { // if the ID is not provided by the caller, set it
File.ID = uuid.New()
}
// Verify that the File exists in the warehouse. Folders are exempt from this check as they are only virtual.
if !File.IsVirtualFolder() {
if _, err := warehouse.ValidateHash(File.Hash); err != nil {
return nil, errors.New("bad request when validating hash")
} else if _, fileInfo, status, _ := api.Backend.UserWarehouse.FileExists(File.Hash); status != warehouse.StatusOK {
//EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse})
return nil, errors.New("file not in warehouse")
} else {
File.Size = fileInfo
}
} else {
File.Hash = protocol.HashData(nil)
File.Size = 0
}
blockRecord := webapi.BlockRecordFileFromAPI(File)
// Set the merkle tree info as appropriate.
if !webapi.SetFileMerkleInfo(api.Backend, &blockRecord) {
return nil, errors.New("merkle information not set")
}
filesAdd = append(filesAdd, blockRecord)
}
newHeight, newVersion, _ := api.Backend.UserBlockchain.AddFiles(filesAdd)
// Creating object for custom return type
var touchReturn TouchReturn
touchReturn.BlockchainHeight = newHeight
touchReturn.BlockchainVersion = newVersion
return &touchReturn, nil
}
// Rm Abstracted function that
// removes file from the blockchain and warehouse
func Rm(api *webapi.WebapiInstance, hashStr string) error {
ID, err := uuid.FromBytes([]byte(hashStr))
if err != nil {
return err
}
var UUIDs []uuid.UUID
UUIDs = append(UUIDs, ID)
_, _, deletedFiles, status := api.Backend.UserBlockchain.DeleteFiles(UUIDs)
// If successfully deleted from the blockchain, delete from the Warehouse in case there are no other references.
if status == blockchain.StatusOK {
for n := range deletedFiles {
if files, status := api.Backend.UserBlockchain.FileExists(deletedFiles[n].Hash); status == blockchain.StatusOK && len(files) == 0 {
api.Backend.UserWarehouse.DeleteFile(deletedFiles[n].Hash)
}
}
}
return nil
}
// Search Abstracted function
// to query for files available
// in the p2p network (i.e the
// Peernet protocol)
// Since it's default it's ran for
// 5 seconds as the default timeout
// (This will be changed on later
// iterations)
func Search(api *webapi.WebapiInstance, term string) (*webapi.SearchResult, error) {
var input webapi.SearchRequest
input.Term = term
input.Timeout = 5
jobID, err := StartSearch(api, &input)
if err != nil {
return nil, err
}
// 6 seconds
time.Sleep(1000 * 6)
result, err := SearchResult(api, jobID)
if err != nil {
return nil, err
}
return result, nil
}
// StartSearch Abstracted function that
// starts the search job based on specified
// parameters and return the job ID
// for a reference
func StartSearch(api *webapi.WebapiInstance, input *webapi.SearchRequest) (uuid.UUID, error) {
if input.Timeout <= 0 {
input.Timeout = 20
}
if input.MaxResults <= 0 {
input.MaxResults = 200
}
// Terminate previous searches, if their IDs were supplied. This allows terminating the old search immediately without making a separate /search/terminate request.
for _, terminate := range input.TerminateID {
if job := api.JobLookup(terminate); job != nil {
job.Terminate()
api.RemoveJob(job)
}
}
job := api.DispatchSearch(*input)
return job.ID, nil
}
func SearchResult(api *webapi.WebapiInstance, jobID uuid.UUID) (*webapi.SearchResult, error) {
// find the job ID
job := api.JobLookup(jobID)
if job == nil {
return nil, errors.New("job id not found")
}
limit := 100
// query all results
var resultFiles []*webapi.ApiFile
resultFiles = job.ReturnNext(limit)
var result webapi.SearchResult
result.Files = []webapi.ApiFile{}
// loop over results
for n := range resultFiles {
result.Files = append(result.Files, *resultFiles[n])
}
// set the status
if len(result.Files) > 0 {
if job.IsSearchResults() {
result.Status = 0 // 0 = Success with results
return &result, nil
} else {
result.Status = 1 // No more results to expect
return nil, errors.New("no more results to expect (Search still running)")
}
} else {
switch job.Status {
case webapi.SearchStatusLive:
result.Status = 3 // No results yet available keep trying
return nil, errors.New("no results yet available keep trying")
case webapi.SearchStatusTerminated:
result.Status = 1 // No more results to expect
return nil, errors.New("no more results to expect (Search terminated)")
default: // SearchStatusNoIndex, SearchStatusNotStarted
result.Status = 1 // No more results to expect
return nil, errors.New("no more results to expect (Search not started)")
}
}
return nil, errors.New("search not successful")
}
// Download and abstracted function that starts downloading a file
// and returns the ID which can be used to track the files
// download status
func Download(api *webapi.WebapiInstance, hashStr string, nodeIDStr string, path string) (*uuid.UUID, error) {
// validate hashes, must be blake3
hash, valid1 := webapi.DecodeBlake3Hash(hashStr)
nodeID, valid2 := webapi.DecodeBlake3Hash(nodeIDStr)
if !valid1 || !valid2 {
//http.Error(w, "", http.StatusBadRequest)
return nil, errors.New("hash or node ID was not valid")
}
filePath := path
if filePath == "" {
// http.Error(w, "", http.StatusBadRequest)
return nil, errors.New("file path not provided")
}
ID := uuid.New()
info := &webapi.DownloadInfo{Backend: api.Backend, Api: api, ID: ID, Created: time.Now(), Hash: hash, NodeID: nodeID}
// create the file immediately
err := info.InitDiskFile(filePath)
if err != nil {
return nil, err
}
// add the download to the list
api.DownloadAdd(info)
// start the download!
go info.Start()
return &ID, nil
}
// DownloadStatus Abstracted function that finds the status of a downloaded files
// based on the download ID provided and returns with the appropriate information
func DownloadStatus(api *webapi.WebapiInstance, DownloadID *uuid.UUID) (*webapi.ApiResponseDownloadStatus, error) {
info := api.DownloadLookup(*DownloadID)
if info == nil {
//EncodeJSON(api.Backend, w, r, apiResponseDownloadStatus{APIStatus: DownloadResponseIDNotFound})
return nil, errors.New("download ID not found")
}
info.RLock()
response := webapi.ApiResponseDownloadStatus{APIStatus: webapi.DownloadResponseSuccess, ID: info.ID, DownloadStatus: info.Status}
if info.Status >= webapi.DownloadWaitSwarm {
response.File = info.File
response.Progress.TotalSize = info.File.Size
response.Progress.DownloadedSize = info.DiskFile.StoredSize
response.Progress.Percentage = math.Round(float64(info.DiskFile.StoredSize)/float64(info.File.Size)*100*100) / 100
}
if info.Status >= webapi.DownloadActive {
response.Swarm.CountPeers = info.Swarm.CountPeers
}
info.RUnlock()
return &response, nil
}

22
go.mod
View File

@@ -1,22 +0,0 @@
module github.com/PeernetOfficial/Abstraction
go 1.19
require (
github.com/PeernetOfficial/core v0.0.0-20221101165801-6989ef4a19c5
github.com/google/uuid v1.3.0
)
require (
github.com/IncSW/geoip2 v0.1.2 // indirect
github.com/akrylysov/pogreb v0.10.1 // indirect
github.com/enfipy/locker v1.1.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/klauspost/cpuid/v2 v2.1.2 // indirect
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a // indirect
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
)

30
go.sum
View File

@@ -1,30 +0,0 @@
github.com/IncSW/geoip2 v0.1.2 h1:v7iAyDiNZjHES45P1JPM3SMvkw0VNeJtz0XSVxkRwOY=
github.com/IncSW/geoip2 v0.1.2/go.mod h1:adcasR40vXiUBjtzdaTTKL/6wSf+fgO4M8Gve/XzPUk=
github.com/PeernetOfficial/core v0.0.0-20221101165801-6989ef4a19c5 h1:9AQcZ535+vpMZm8vky527b/G4jyCuO1FSjoGld0ufSk=
github.com/PeernetOfficial/core v0.0.0-20221101165801-6989ef4a19c5/go.mod h1:UGZqseyO3H9uA4Mm8OQDHP6Gs4z+q7R9VF7R+WzzJbY=
github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/enfipy/locker v1.1.0 h1:2zVJ0ky7cS1Vjs0x6OQWFiT2dSEiHrI5/O2KCz1fgGc=
github.com/enfipy/locker v1.1.0/go.mod h1:uuj+dvWHECshK8rkHcw+ZOb9SLo16yc0Em/JGUqRqko=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.1.2 h1:XhdX4fqAJUA0yj+kUwMavO0hHrSPAecYdYf1ZmxHvak=
github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a h1:NmSIgad6KjE6VvHciPZuNRTKxGhlPfD6OA87W/PLkqg=
golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU=
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 h1:OK7RB6t2WQX54srQQYSXMW8dF5C6/8+oA/s5QBmmto4=
golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=

197
webapi/API.go
View File

@@ -1,197 +0,0 @@
/*
File Name: API.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"crypto/tls"
"encoding/json"
"errors"
"net/http"
"sync"
"time"
"github.com/IncSW/geoip2"
"github.com/PeernetOfficial/core"
"github.com/google/uuid"
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
)
type WebapiInstance struct {
Backend *core.Backend
geoipCityReader *geoip2.CityReader
// Router can be used to register additional API functions
Router *mux.Router
AllowKeyInParam []string // List of paths that accept the API key as &k= parameter
// search jobs
allJobs map[uuid.UUID]*SearchJob
allJobsMutex sync.RWMutex
// download info
downloads map[uuid.UUID]*DownloadInfo
downloadsMutex sync.RWMutex
}
// WSUpgrader is used for websocket functionality. It allows all requests.
var WSUpgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
CheckOrigin: func(r *http.Request) bool {
// allow all connections by default
return true
},
}
// Start starts the API. ListenAddresses is a list of IP:Ports.
// The certificate File and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout.
// The API key may be uuid.Nil to disable it although this is not recommended for security reasons.
func Start(Backend *core.Backend, ListenAddresses []string, UseSSL bool, CertificateFile, CertificateKey string, TimeoutRead, TimeoutWrite time.Duration, APIKey uuid.UUID) (api *WebapiInstance) {
if len(ListenAddresses) == 0 {
return nil
}
api = &WebapiInstance{
Backend: Backend,
Router: mux.NewRouter(),
AllowKeyInParam: []string{"/File/read", "/File/view"},
allJobs: make(map[uuid.UUID]*SearchJob),
downloads: make(map[uuid.UUID]*DownloadInfo),
}
if APIKey != uuid.Nil {
api.Router.Use(api.authenticateMiddleware(APIKey))
}
api.Router.HandleFunc("/test", apiTest).Methods("GET")
api.Router.HandleFunc("/Status", api.apiStatus).Methods("GET")
api.Router.HandleFunc("/Status/peers", api.apiStatusPeers).Methods("GET")
api.Router.HandleFunc("/account/info", api.apiAccountInfo).Methods("GET")
api.Router.HandleFunc("/account/delete", api.apiAccountDelete).Methods("GET")
api.Router.HandleFunc("/blockchain/header", api.apiBlockchainHeaderFunc).Methods("GET")
api.Router.HandleFunc("/blockchain/append", api.apiBlockchainAppend).Methods("POST")
api.Router.HandleFunc("/blockchain/read", api.apiBlockchainRead).Methods("GET")
api.Router.HandleFunc("/blockchain/File/add", api.apiBlockchainFileAdd).Methods("POST")
api.Router.HandleFunc("/blockchain/File/list", api.apiBlockchainFileList).Methods("GET")
api.Router.HandleFunc("/blockchain/File/delete", api.apiBlockchainFileDelete).Methods("POST")
api.Router.HandleFunc("/blockchain/File/update", api.apiBlockchainFileUpdate).Methods("POST")
api.Router.HandleFunc("/profile/list", api.apiProfileList).Methods("GET")
api.Router.HandleFunc("/profile/read", api.apiProfileRead).Methods("GET")
api.Router.HandleFunc("/profile/write", api.apiProfileWrite).Methods("POST")
api.Router.HandleFunc("/profile/delete", api.apiProfileDelete).Methods("POST")
api.Router.HandleFunc("/search", api.apiSearch).Methods("POST")
api.Router.HandleFunc("/search/result", api.apiSearchResult).Methods("GET")
api.Router.HandleFunc("/search/result/ws", api.apiSearchResultStream).Methods("GET")
api.Router.HandleFunc("/search/statistic", api.apiSearchStatistic).Methods("GET")
api.Router.HandleFunc("/search/terminate", api.apiSearchTerminate).Methods("GET")
api.Router.HandleFunc("/explore", api.apiExplore).Methods("GET")
api.Router.HandleFunc("/File/format", api.apiFileFormat).Methods("GET")
api.Router.HandleFunc("/download/start", api.apiDownloadStart).Methods("GET")
api.Router.HandleFunc("/download/Status", api.apiDownloadStatus).Methods("GET")
api.Router.HandleFunc("/download/action", api.apiDownloadAction).Methods("GET")
api.Router.HandleFunc("/warehouse/create", api.apiWarehouseCreateFile).Methods("POST")
api.Router.HandleFunc("/warehouse/create/path", api.apiWarehouseCreateFilePath).Methods("GET")
api.Router.HandleFunc("/warehouse/read", api.apiWarehouseReadFile).Methods("GET")
api.Router.HandleFunc("/warehouse/read/path", api.apiWarehouseReadFilePath).Methods("GET")
api.Router.HandleFunc("/warehouse/delete", api.apiWarehouseDeleteFile).Methods("GET")
api.Router.HandleFunc("/File/read", api.apiFileRead).Methods("GET")
api.Router.HandleFunc("/File/view", api.apiFileView).Methods("GET")
for _, listen := range ListenAddresses {
go startWebAPI(Backend, listen, UseSSL, CertificateFile, CertificateKey, api.Router, "API", TimeoutRead, TimeoutWrite)
}
return api
}
// startWebAPI starts a web-server with given parameters and logs the Status. If may block forever and only returns if there is an error.
// The certificate File and key are only used if SSL is enabled. The read and write timeout may be 0 for no timeout.
func startWebAPI(Backend *core.Backend, WebListen string, UseSSL bool, CertificateFile, CertificateKey string, Handler http.Handler, Info string, ReadTimeout, WriteTimeout time.Duration) {
Backend.LogError("startWebAPI", "Start API at '%s'\n", WebListen)
tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} // for security reasons disable TLS 1.0/1.1
server := &http.Server{
Addr: WebListen,
Handler: Handler,
ReadTimeout: ReadTimeout, // ReadTimeout is the maximum duration for reading the entire request, including the body.
WriteTimeout: WriteTimeout, // WriteTimeout is the maximum duration before timing out writes of the response. This includes processing time and is therefore the max time any HTTP function may take.
//IdleTimeout: IdleTimeout, // IdleTimeout is the maximum amount of time to wait for the next request when keep-alives are enabled.
TLSConfig: tlsConfig,
}
if UseSSL {
// HTTPS
if err := server.ListenAndServeTLS(CertificateFile, CertificateKey); err != nil {
Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err)
}
} else {
// HTTP
if err := server.ListenAndServe(); err != nil {
Backend.LogError("startWebAPI", "Error listening on '%s': %v\n", WebListen, err)
}
}
}
// EncodeJSON encodes the data as JSON
func EncodeJSON(Backend *core.Backend, w http.ResponseWriter, r *http.Request, data interface{}) (err error) {
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(data)
if err != nil {
Backend.LogError("EncodeJSON", "Error writing data for route '%s': %v\n", r.URL.Path, err)
}
return err
}
// DecodeJSON decodes input JSON data server side sent either via GET or POST. It does not limit the maximum amount to read.
// In case of error it will automatically send an error to the client.
func DecodeJSON(w http.ResponseWriter, r *http.Request, data interface{}) (err error) {
if r.Body == nil {
http.Error(w, "", http.StatusBadRequest)
return errors.New("no data")
}
err = json.NewDecoder(r.Body).Decode(data)
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return err
}
return nil
}
// authenticateMiddleware returns a middleware function to be used with mux.Router.Use(). It handles all authentication functionality.
func (api *WebapiInstance) authenticateMiddleware(APIKey uuid.UUID) func(http.Handler) http.Handler {
return (func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
keyID, err := uuid.Parse(r.Header.Get("x-api-key"))
if err != nil { // special case for some paths
for _, exceptPath := range api.AllowKeyInParam {
if exceptPath == r.URL.Path {
r.ParseForm()
keyID, err = uuid.Parse(r.Form.Get("k"))
break
}
}
}
if err != nil { // Invalid key format
w.WriteHeader(http.StatusUnauthorized)
return
}
if keyID != APIKey {
w.WriteHeader(http.StatusUnauthorized)
return
}
next.ServeHTTP(w, r)
})
})
}

122
webapi/Blockchain.go
View File

@@ -1,122 +0,0 @@
/*
File Name: Blockchain.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"encoding/hex"
"net/http"
"strconv"
"github.com/PeernetOfficial/core/blockchain"
)
type apiBlockchainHeader struct {
PeerID string `json:"peerid"` // Peer ID hex encoded.
Version uint64 `json:"version"` // Current version number of the blockchain.
Height uint64 `json:"height"` // Height of the blockchain (number of blocks). If 0, no data exists.
}
/*
apiBlockchainHeaderFunc returns the current blockchain header information
Request: GET /blockchain/header
Result: 200 with JSON structure apiResponsePeerSelf
*/
func (api *WebapiInstance) apiBlockchainHeaderFunc(w http.ResponseWriter, r *http.Request) {
publicKey, height, version := api.Backend.UserBlockchain.Header()
EncodeJSON(api.Backend, w, r, apiBlockchainHeader{Version: version, Height: height, PeerID: hex.EncodeToString(publicKey.SerializeCompressed())})
}
type apiBlockRecordRaw struct {
Type uint8 `json:"type"` // Record Type. See core.RecordTypeX.
Data []byte `json:"data"` // Data according to the type.
}
// apiBlockchainBlockRaw contains a raw block of the blockchain via API
type apiBlockchainBlockRaw struct {
Records []apiBlockRecordRaw `json:"records"` // Block records in encoded raw format.
}
type apiBlockchainBlockStatus struct {
Status int `json:"Status"` // See blockchain.StatusX.
Height uint64 `json:"height"` // Height of the blockchain (number of blocks).
Version uint64 `json:"version"` // Version of the blockchain.
}
/*
apiBlockchainAppend appends a block to the blockchain. This is a low-level function for already encoded blocks.
Do not use this function. Adding invalid data to the blockchain may corrupt it which might result in blacklisting by other peers.
Request: POST /blockchain/append with JSON structure apiBlockchainBlockRaw
Response: 200 with JSON structure apiBlockchainBlockStatus
*/
func (api *WebapiInstance) apiBlockchainAppend(w http.ResponseWriter, r *http.Request) {
var input apiBlockchainBlockRaw
if err := DecodeJSON(w, r, &input); err != nil {
return
}
var records []blockchain.BlockRecordRaw
for _, record := range input.Records {
records = append(records, blockchain.BlockRecordRaw{Type: record.Type, Data: record.Data})
}
newHeight, newVersion, status := api.Backend.UserBlockchain.Append(records)
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion})
}
type apiBlockchainBlock struct {
Status int `json:"Status"` // See blockchain.StatusX.
PeerID string `json:"peerid"` // Peer ID hex encoded.
LastBlockHash []byte `json:"lastblockhash"` // Hash of the last block. Blake3.
BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version
Number uint64 `json:"blocknumber"` // Block number
RecordsRaw []apiBlockRecordRaw `json:"recordsraw"` // Records raw. Successfully decoded records are parsed into the below fields.
RecordsDecoded []interface{} `json:"recordsdecoded"` // Records decoded. The encoding for each record depends on its type.
}
/*
apiBlockchainRead reads a block and returns the decoded information.
Request: GET /blockchain/read?block=[number]
Result: 200 with JSON structure apiBlockchainBlock
*/
func (api *WebapiInstance) apiBlockchainRead(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
blockN, err := strconv.Atoi(r.Form.Get("block"))
if err != nil || blockN < 0 {
http.Error(w, "", http.StatusBadRequest)
return
}
block, status, _ := api.Backend.UserBlockchain.Read(uint64(blockN))
result := apiBlockchainBlock{Status: status}
if status == 0 {
for _, record := range block.RecordsRaw {
result.RecordsRaw = append(result.RecordsRaw, apiBlockRecordRaw{Type: record.Type, Data: record.Data})
}
result.PeerID = hex.EncodeToString(block.OwnerPublicKey.SerializeCompressed())
for _, record := range block.RecordsDecoded {
switch v := record.(type) {
case blockchain.BlockRecordFile:
result.RecordsDecoded = append(result.RecordsDecoded, blockRecordFileToAPI(v))
case blockchain.BlockRecordProfile:
result.RecordsDecoded = append(result.RecordsDecoded, blockRecordProfileToAPI(v))
}
}
}
EncodeJSON(api.Backend, w, r, result)
}

202
webapi/Download Transfer.go
View File

@@ -1,202 +0,0 @@
/*
File Name: Download Transfer.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
Temporary download code to provide dummy results for testing. To be replaced!
*/
package webapi
import (
"bytes"
"os"
"time"
"github.com/PeernetOfficial/core/warehouse"
)
// Starts the download.
func (info *DownloadInfo) Start() {
// current user?
if bytes.Equal(info.NodeID, info.Backend.SelfNodeID()) {
info.DownloadSelf()
return
}
for n := 0; n < 3 && info.Peer == nil; n++ {
_, info.Peer, _ = info.Backend.FindNode(info.NodeID, time.Second*5)
if info.Status == DownloadCanceled {
return
}
}
if info.Peer != nil {
info.Download()
} else {
info.Status = DownloadCanceled
}
}
func (info *DownloadInfo) Download() {
//fmt.Printf("Download start of %s\n", hex.EncodeToString(info.Hash))
// try to download the entire File
reader, fileSize, transferSize, err := FileStartReader(info.Peer, info.Hash, 0, 0, nil)
if reader != nil {
defer reader.Close()
}
if err != nil {
info.Status = DownloadCanceled
return
} else if fileSize != transferSize {
info.Status = DownloadCanceled
return
}
info.File.Size = fileSize
info.Status = DownloadActive
// download in a loop
var fileOffset, totalRead uint64
dataRemaining := fileSize
readSize := uint64(4096)
for dataRemaining > 0 {
//fmt.Printf("data remaining: downloaded %d from total %d = %d %%\n", totalRead, fileSize, totalRead*100/fileSize)
if dataRemaining < readSize {
readSize = dataRemaining
}
data := make([]byte, readSize)
n, err := reader.Read(data)
totalRead += uint64(n)
dataRemaining -= uint64(n)
data = data[:n]
if err != nil {
info.Status = DownloadCanceled
return
}
info.storeDownloadData(data[:n], fileOffset)
fileOffset += uint64(n)
}
//fmt.Printf("data finished: downloaded %d from total %d = %d %%\n", totalRead, fileSize, totalRead*100/fileSize)
info.Finish()
info.DeleteDefer(time.Hour * 1) // cache the details for 1 hour before removing
}
// Pause pauses the download. Status is DownloadResponseX.
func (info *DownloadInfo) Pause() (status int) {
info.Lock()
defer info.Unlock()
if info.Status != DownloadActive { // The download must be active to be paused.
return DownloadResponseActionInvalid
}
info.Status = DownloadPause
return DownloadResponseSuccess
}
// Resume resumes the download. Status is DownloadResponseX.
func (info *DownloadInfo) Resume() (status int) {
info.Lock()
defer info.Unlock()
if info.Status != DownloadPause { // The download must be paused to resume.
return DownloadResponseActionInvalid
}
info.Status = DownloadActive
return DownloadResponseSuccess
}
// Cancel cancels the download. Status is DownloadResponseX.
func (info *DownloadInfo) Cancel() (status int) {
info.Lock()
defer info.Unlock()
if info.Status >= DownloadCanceled { // The download must not be already canceled or finished.
return DownloadResponseActionInvalid
}
info.Status = DownloadCanceled
info.DiskFile.Handle.Close()
return DownloadResponseSuccess
}
// Finish marks the download as finished.
func (info *DownloadInfo) Finish() (status int) {
info.Lock()
defer info.Unlock()
if info.Status != DownloadActive { // The download must be active.
return DownloadResponseActionInvalid
}
info.Status = DownloadFinished
info.DiskFile.Handle.Close()
return DownloadResponseSuccess
}
// InitDiskFile creates the target File
func (info *DownloadInfo) InitDiskFile(path string) (err error) {
info.DiskFile.Name = path
info.DiskFile.Handle, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) // 666 : All uses can read/write
return err
}
// storeDownloadData stores downloaded data. It does not change the download Status.
func (info *DownloadInfo) storeDownloadData(data []byte, offset uint64) (status int) {
info.Lock()
defer info.Unlock()
if info.Status != DownloadActive { // The download must be active.
return DownloadResponseActionInvalid
}
if _, err := info.DiskFile.Handle.WriteAt(data, int64(offset)); err != nil {
return DownloadResponseFileWrite
}
info.DiskFile.StoredSize += uint64(len(data))
return DownloadResponseSuccess
}
func (info *DownloadInfo) DownloadSelf() {
// Check if the File is available in the local warehouse.
_, fileSize, status, _ := info.Backend.UserWarehouse.FileExists(info.Hash)
if status != warehouse.StatusOK {
info.Status = DownloadCanceled
return
}
info.File.Size = fileSize
info.Status = DownloadActive
// read the File
status, bytesRead, _ := info.Backend.UserWarehouse.ReadFile(info.Hash, 0, int64(info.File.Size), info.DiskFile.Handle)
info.DiskFile.StoredSize = uint64(bytesRead)
if status != warehouse.StatusOK {
info.Status = DownloadCanceled
return
}
info.Finish()
info.DeleteDefer(time.Hour * 1) // cache the details for 1 hour before removing}
}

243
webapi/Download.go
View File

@@ -1,243 +0,0 @@
/*
File Name: Download.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"encoding/hex"
"math"
"net/http"
"os"
"strconv"
"sync"
"time"
"github.com/PeernetOfficial/core"
"github.com/google/uuid"
)
type ApiResponseDownloadStatus struct {
APIStatus int `json:"apistatus"` // Status of the API call. See DownloadResponseX.
ID uuid.UUID `json:"ID"` // Download ID. This can be used to query the latest Status and take actions.
DownloadStatus int `json:"downloadstatus"` // Status of the download. See DownloadX.
File ApiFile `json:"File"` // File information. Only available for Status >= DownloadWaitSwarm.
Progress struct {
TotalSize uint64 `json:"totalsize"` // Total size in bytes.
DownloadedSize uint64 `json:"downloadedsize"` // Count of bytes download so far.
Percentage float64 `json:"percentage"` // Percentage downloaded. Rounded to 2 decimal points. Between 0.00 and 100.00.
} `json:"progress"` // Progress of the download. Only valid for Status >= DownloadWaitSwarm.
Swarm struct {
CountPeers uint64 `json:"countpeers"` // Count of peers participating in the swarm.
} `json:"swarm"` // Information about the swarm. Only valid for Status >= DownloadActive.
}
const (
DownloadResponseSuccess = 0 // Success
DownloadResponseIDNotFound = 1 // Error: Download ID not found.
DownloadResponseFileInvalid = 2 // Error: Target File cannot be used. For example, permissions denied to create it.
DownloadResponseActionInvalid = 4 // Error: Invalid action. Pausing a non-active download, resuming a non-paused download, or canceling already canceled or finished download.
DownloadResponseFileWrite = 5 // Error writing File.
)
// Download Status list
const (
DownloadWaitMetadata = 0 // Wait for File metadata.
DownloadWaitSwarm = 1 // Wait to join swarm.
DownloadActive = 2 // Active downloading. It could still be stuck at any percentage (including 0%) if no seeders are available.
DownloadPause = 3 // Paused by the user.
DownloadCanceled = 4 // Canceled by the user before the download finished. Once canceled, a new download has to be started if the File shall be downloaded.
DownloadFinished = 5 // Download finished 100%.
)
/*
apiDownloadStart starts the download of a File. The path is the full path on disk to store the File.
The Hash parameter identifies the File to download. The node ID identifies the blockchain (i.e., the "owner" of the File).
Request: GET /download/start?path=[target path on disk]&Hash=[File Hash to download]&node=[node ID]
Result: 200 with JSON structure ApiResponseDownloadStatus
*/
func (api *WebapiInstance) apiDownloadStart(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
// validate hashes, must be blake3
hash, valid1 := DecodeBlake3Hash(r.Form.Get("Hash"))
nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node"))
if !valid1 || !valid2 {
http.Error(w, "", http.StatusBadRequest)
return
}
filePath := r.Form.Get("path")
if filePath == "" {
http.Error(w, "", http.StatusBadRequest)
return
}
info := &DownloadInfo{Backend: api.Backend, Api: api, ID: uuid.New(), Created: time.Now(), Hash: hash, NodeID: nodeID}
// create the File immediately
if info.InitDiskFile(filePath) != nil {
EncodeJSON(api.Backend, w, r, ApiResponseDownloadStatus{APIStatus: DownloadResponseFileInvalid})
return
}
// add the download to the list
api.DownloadAdd(info)
// start the download!
go info.Start()
EncodeJSON(api.Backend, w, r, ApiResponseDownloadStatus{APIStatus: DownloadResponseSuccess, ID: info.ID, DownloadStatus: DownloadWaitMetadata})
}
/*
apiDownloadStatus returns the Status of an active download.
Request: GET /download/Status?ID=[download ID]
Result: 200 with JSON structure ApiResponseDownloadStatus
*/
func (api *WebapiInstance) apiDownloadStatus(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
id, err := uuid.Parse(r.Form.Get("ID"))
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return
}
info := api.DownloadLookup(id)
if info == nil {
EncodeJSON(api.Backend, w, r, ApiResponseDownloadStatus{APIStatus: DownloadResponseIDNotFound})
return
}
info.RLock()
response := ApiResponseDownloadStatus{APIStatus: DownloadResponseSuccess, ID: info.ID, DownloadStatus: info.Status}
if info.Status >= DownloadWaitSwarm {
response.File = info.File
response.Progress.TotalSize = info.File.Size
response.Progress.DownloadedSize = info.DiskFile.StoredSize
response.Progress.Percentage = math.Round(float64(info.DiskFile.StoredSize)/float64(info.File.Size)*100*100) / 100
}
if info.Status >= DownloadActive {
response.Swarm.CountPeers = info.Swarm.CountPeers
}
info.RUnlock()
EncodeJSON(api.Backend, w, r, response)
}
/*
apiDownloadAction pauses, resumes, and cancels a download. Once canceled, a new download has to be started if the File shall be downloaded.
Only active downloads can be paused. While a download is in discovery phase (querying metadata, joining swarm), it can only be canceled.
Action: 0 = Pause, 1 = Resume, 2 = Cancel.
Request: GET /download/action?ID=[download ID]&action=[action]
Result: 200 with JSON structure ApiResponseDownloadStatus (using APIStatus and DownloadStatus)
*/
func (api *WebapiInstance) apiDownloadAction(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
id, err := uuid.Parse(r.Form.Get("ID"))
action, err2 := strconv.Atoi(r.Form.Get("action"))
if err != nil || err2 != nil || action < 0 || action > 2 {
http.Error(w, "", http.StatusBadRequest)
return
}
info := api.DownloadLookup(id)
if info == nil {
EncodeJSON(api.Backend, w, r, ApiResponseDownloadStatus{APIStatus: DownloadResponseIDNotFound})
return
}
apiStatus := 0
switch action {
case 0: // Pause
apiStatus = info.Pause()
case 1: // Resume
apiStatus = info.Resume()
case 2: // Cancel
apiStatus = info.Cancel()
}
EncodeJSON(api.Backend, w, r, ApiResponseDownloadStatus{APIStatus: apiStatus, ID: info.ID, DownloadStatus: info.Status})
}
// ---- download tracking ----
type DownloadInfo struct {
ID uuid.UUID // Download ID
Status int // Current Status. See DownloadX.
sync.RWMutex // Mutext for changing the Status
// input
Hash []byte // File Hash
NodeID []byte // Node ID of the owner
// runtime data
Created time.Time // When the download was Created.
Ended time.Time // When the download was finished (only Status = DownloadFinished).
File ApiFile // File metadata (only Status >= DownloadWaitSwarm)
DiskFile struct { // Target File on disk to store downloaded data
Name string // File name
Handle *os.File // Target File (on disk) to store downloaded data
StoredSize uint64 // Count of bytes downloaded and stored in the File
}
Swarm struct { // Information about the swarm. Only valid for Status >= DownloadActive.
CountPeers uint64 // Count of peers participating in the swarm.
}
// live connections, to be changed
Peer *core.PeerInfo
Api *WebapiInstance
Backend *core.Backend
}
func (api *WebapiInstance) DownloadAdd(info *DownloadInfo) {
api.downloadsMutex.Lock()
api.downloads[info.ID] = info
api.downloadsMutex.Unlock()
}
func (api *WebapiInstance) DownloadDelete(id uuid.UUID) {
api.downloadsMutex.Lock()
delete(api.downloads, id)
api.downloadsMutex.Unlock()
}
func (api *WebapiInstance) DownloadLookup(id uuid.UUID) (info *DownloadInfo) {
api.downloadsMutex.Lock()
info = api.downloads[id]
api.downloadsMutex.Unlock()
return info
}
// DeleteDefer deletes the download from the downloads list after the given duration.
// It does not wait for the download to be finished.
func (info *DownloadInfo) DeleteDefer(Duration time.Duration) {
go func() {
<-time.After(Duration)
info.Api.DownloadDelete(info.ID)
}()
}
// DecodeBlake3Hash decodes a blake3 Hash that is hex encoded
func DecodeBlake3Hash(text string) (hash []byte, valid bool) {
hash, err := hex.DecodeString(text)
return hash, err == nil && len(hash) == 256/8
}

221
webapi/File Detection.go
View File

@@ -1,221 +0,0 @@
/*
File Name: File Detection.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"net/http"
"os"
"path"
"strings"
"github.com/PeernetOfficial/core"
)
// PathToExtension translates a path to a File extension, if possible. It also returns the second File extension if there is one (relevant for files like "test.tar.gz").
func PathToExtension(Path string) (extension, extension2 string, valid bool) {
_, fileA := path.Split(Path)
parts := strings.Split(fileA, ".")
if len(parts) <= 1 {
return "", "", false
}
extension = parts[len(parts)-1]
extension = strings.ToLower(extension)
if len(parts) >= 3 {
extension2 = parts[len(parts)-2]
extension2 = strings.ToLower(extension2)
}
return extension, extension2, true
}
// FileTranslateExtension translates the extension to a File Type and File Format. If invalid, types are 0.
func FileTranslateExtension(extension string) (fileType, fileFormat uint16) {
switch extension {
case "txt", "log", "ini", "json", "md":
return core.TypeText, core.FormatText
case "csv", "tsv":
return core.TypeText, core.FormatCSV
case "html", "htm":
return core.TypeText, core.FormatHTML
case "doc", "docx", "rtf", "odt":
return core.TypeDocument, core.FormatWord
case "pdf":
return core.TypeDocument, core.FormatPDF
case "xls", "xlsx", "ods":
return core.TypeDocument, core.FormatExcel
case "gif", "jpg", "jpeg", "png", "svg", "bmp", "tif", "tiff", "jfif", "webp":
return core.TypePicture, core.FormatPicture
case "mp4", "flv", "avi", "mov", "mpg", "mpeg", "h264", "3g2", "3gp", "mkv", "wmv", "webm", "ts":
return core.TypeVideo, core.FormatVideo
case "mp3", "ogg", "flac":
return core.TypeAudio, core.FormatAudio
case "zip", "rar", "7z", "tar":
return core.TypeContainer, core.FormatContainer
case "ppt", "pptx", "odp":
return core.TypeDocument, core.FormatPowerpoint
case "epub", "mobi", "prc":
return core.TypeEbook, core.FormatEbook
case "gz", "bz", "bz2", "xz":
return core.TypeCompressed, core.FormatCompressed
case "sql":
return core.TypeText, core.FormatDatabase
case "eml", "mbox":
return core.TypeText, core.FormatEmail
case "exe", "sys", "dll", "cmd", "bat":
return core.TypeExecutable, core.FormatExecutable
case "msi":
return core.TypeExecutable, core.FormatInstaller
case "apk":
return core.TypeExecutable, core.FormatAPK
case "iso":
return core.TypeContainer, core.FormatISO
default:
return core.TypeBinary, core.FormatBinary
}
}
// HTTPContentTypeToCore translates the HTTP content type to the File Type and File Format used by the core package.
func HTTPContentTypeToCore(httpContentType string) (fileType, fileFormat uint16) {
switch httpContentType {
case "text/html", "application/xhtml+xml", "application/xml":
return core.TypeText, core.FormatHTML
case "text/plain":
return core.TypeText, core.FormatText
case "text/csv", "text/tsv", "text/tab-separated-values", "text/x-csv", "application/csv", "application/x-csv", "text/x-comma-separated-values":
return core.TypeText, core.FormatCSV
case "application/pdf", "application/x-pdf":
return core.TypeDocument, core.FormatPDF
case "application/msword", "application/rtf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.oasis.opendocument.text":
return core.TypeDocument, core.FormatWord
case "application/excel", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.oasis.opendocument.spreadsheet":
return core.TypeDocument, core.FormatExcel
case "application/vnd.ms-powerpoint", "application/vnd.openxmlformats-officedocument.presentationml.presentation", "application/vnd.oasis.opendocument.presentation":
return core.TypeDocument, core.FormatPowerpoint
case "image/png", "image/jpeg", "image/gif", "image/svg+xml", "image/tiff", "image/webp", "image/bmp", "image/x-bmp", "image/x-windows-bmp": // image/x-icon excluded
return core.TypePicture, core.FormatPicture
case "audio/aac", "audio/midi", "audio/ogg", "audio/x-wav", "audio/webm", "audio/3gpp", "audio/3gpp2", "audio/mpeg", "audio/vorbis":
return core.TypeAudio, core.FormatAudio
case "video/x-msvideo", "video/x-ms-wmv", "video/mpeg", "video/ogg", "video/webm", "video/3gpp", "video/3gpp2", "video/x-flv", "video/mp4":
return core.TypeVideo, core.FormatVideo
case "application/zip", "application/x-rar-compressed", "application/x-tar", "application/x-bzip", "application/x-bzip2", "application/x-7z-compressed":
return core.TypeContainer, core.FormatContainer
case "application/epub+zip", "application/x-mobipocket-ebook":
return core.TypeEbook, core.FormatEbook
default:
return core.TypeBinary, core.FormatBinary
}
}
// FileDataToHTTPContentType returns the HTTP content type based on the initial File data. It reads the first 512 bytes of the File.
func FileDataToHTTPContentType(Path string) (httpContentType string, err error) {
file, err := os.Open(Path)
if err != nil {
return "", err
}
// Read up to 512 bytes. This specific number comes from http.DetectContentType which specifies it as constant "sniffLen".
buff := make([]byte, 512)
if _, err := file.Read(buff); err != nil {
return "", err
}
if err := file.Close(); err != nil {
return "", err
}
httpContentType = http.DetectContentType(buff)
// sanitize it first
httpContentType = strings.ToLower(strings.TrimSpace(httpContentType))
if indexD := strings.IndexAny(httpContentType, ";"); indexD >= 0 {
httpContentType = httpContentType[:indexD]
}
return httpContentType, nil
}
// FileDetectType detects the File Type and File Format of a File. It uses the extension if available, otherwise the File data, for detection.
func FileDetectType(Path string) (fileType, fileFormat uint16, err error) {
// If a File extension is available, use that to detect the File type and File format.
// Otherwise, use the initial File data for detection.
if ext1, _, valid := PathToExtension(Path); valid {
fileType, fileFormat = FileTranslateExtension(ext1)
return fileType, fileFormat, nil
}
httpContentType, err := FileDataToHTTPContentType(Path)
if err != nil {
return core.TypeBinary, core.FormatBinary, err
}
fileType, fileFormat = HTTPContentTypeToCore(httpContentType)
return fileType, fileFormat, nil
}
type apiResponseFileFormat struct {
Status int `json:"Status"` // Status: 0 = Success, 1 = Error reading File
FileType uint16 `json:"filetype"` // File Type.
FileFormat uint16 `json:"fileformat"` // File Format.
}
/*
apiFileFormat detects the File type and File format of the specified File.
It will primarily use the File extension for detection. If unavailable, it uses the first 512 bytes of the File data to detect the type.
Request: GET /File/format?path=[File path on disk]
Result: 200 with JSON structure apiResponseFileFormat
*/
func (api *WebapiInstance) apiFileFormat(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
filePath := r.Form.Get("path")
if filePath == "" {
http.Error(w, "", http.StatusBadRequest)
return
}
fileType, fileFormat, err := FileDetectType(filePath)
if err != nil {
EncodeJSON(api.Backend, w, r, apiResponseFileFormat{Status: 1})
return
}
EncodeJSON(api.Backend, w, r, apiResponseFileFormat{Status: 0, FileType: fileType, FileFormat: fileFormat})
}

319
webapi/File IO.go
View File

@@ -1,319 +0,0 @@
/*
File Name: File IO.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"errors"
"io"
"net/http"
"strconv"
"time"
"github.com/PeernetOfficial/core"
"github.com/PeernetOfficial/core/btcec"
"github.com/PeernetOfficial/core/protocol"
"github.com/PeernetOfficial/core/warehouse"
)
/*
apiFileRead reads a File immediately from a remote Peer. Use the /download functions to download a File.
This endpoint supports the Range, Content-Range and Content-Length headers. Multipart ranges are not supported and result in HTTP 400.
Instead of providing the node ID, the Peer ID is also accepted in the &node= parameter.
The default timeout for connecting to the Peer is 10 seconds.
Request: GET /File/read?Hash=[Hash]&node=[node ID]
Optional: &offset=[offset]&limit=[limit] or via Range header.
Optional: &timeout=[seconds]
Response: 200 with the content
206 with partial content
400 if the parameters are invalid
404 if the File was not found or other error on transfer initiate
502 if unable to find or connect to the remote Peer in time
*/
func (api *WebapiInstance) apiFileRead(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
var err error
// validate hashes (must be blake3) and other input
fileHash, valid1 := DecodeBlake3Hash(r.Form.Get("Hash"))
nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node"))
publicKey, err3 := core.PublicKeyFromPeerID(r.Form.Get("node"))
if !valid1 || (!valid2 && err3 != nil) {
http.Error(w, "", http.StatusBadRequest)
return
}
timeoutSeconds, _ := strconv.Atoi(r.Form.Get("timeout"))
if timeoutSeconds == 0 {
timeoutSeconds = 10
}
timeout := time.Duration(timeoutSeconds) * time.Second
offset, _ := strconv.Atoi(r.Form.Get("offset"))
limit, _ := strconv.Atoi(r.Form.Get("limit"))
// Range header?
var ranges []HTTPRange
if ranges, err = ParseRangeHeader(r.Header.Get("Range"), -1, true); err != nil || len(ranges) > 1 {
http.Error(w, "", http.StatusBadRequest)
return
} else if len(ranges) == 1 {
if ranges[0].length != -1 { // if length is not specified, limit remains 0 which is maximum
limit = ranges[0].length
}
offset = ranges[0].start
}
// Is the File available in the local warehouse? In that case requesting it from the remote is unnecessary.
if serveFileFromWarehouse(api.Backend, w, fileHash, uint64(offset), uint64(limit), ranges) {
return
}
// try connecting via node ID or Peer ID?
var peer *core.PeerInfo
if valid2 {
peer, err = PeerConnectNode(api.Backend, nodeID, timeout)
} else if err3 == nil {
peer, err = PeerConnectPublicKey(api.Backend, publicKey, timeout)
}
if err != nil {
w.WriteHeader(http.StatusBadGateway)
return
}
// Start the reader. If this HTTP request is canceled, r.Context().Done() acts as cancellation signal to the underlying UDT connection.
reader, fileSize, transferSize, err := FileStartReader(peer, fileHash, uint64(offset), uint64(limit), r.Context().Done())
if reader != nil {
defer reader.Close()
}
if err != nil || reader == nil {
w.WriteHeader(http.StatusNotFound)
return
}
// set the right headers
setContentLengthRangeHeader(w, uint64(offset), transferSize, fileSize, ranges)
// Start sending the data!
io.Copy(w, io.LimitReader(reader, int64(transferSize)))
}
// serveFileFromWarehouse serves the File from the warehouse. If it is not available, it returns false and does not use the writer.
// Limit is optional, 0 means the entire File.
func serveFileFromWarehouse(backend *core.Backend, w http.ResponseWriter, fileHash []byte, offset, limit uint64, ranges []HTTPRange) (valid bool) {
// Check if the File is available in the local warehouse.
_, fileSize, status, _ := backend.UserWarehouse.FileExists(fileHash)
if status != warehouse.StatusOK {
return false
}
// validate offset and limit
if limit > 0 && offset+limit > fileSize {
http.Error(w, "invalid limit", http.StatusBadRequest)
return true
} else if offset > fileSize {
http.Error(w, "invalid offset", http.StatusBadRequest)
return true
} else if limit == 0 {
limit = fileSize - offset
}
setContentLengthRangeHeader(w, offset, limit, fileSize, ranges)
status, _, _ = backend.UserWarehouse.ReadFile(fileHash, int64(offset), int64(limit), w)
// StatusErrorReadFile must be considered success, since parts of the File may have been transferred already and recovery is not possible.
return status == warehouse.StatusErrorReadFile || status == warehouse.StatusOK
}
/*
apiFileView is similar to /File/read but but provides a format parameter. It sets the Content-Type and Accept-Ranges headers.
This endpoint supports the Range, Content-Range and Content-Length headers. Multipart ranges are not supported and result in HTTP 400.
Instead of providing the node ID, the Peer ID is also accepted in the &node= parameter.
The default timeout for connecting to the Peer is 10 seconds.
Formats: 14 = Video
Request: GET /File/view?Hash=[Hash]&node=[node ID]&format=[format]
Optional: &offset=[offset]&limit=[limit] or via Range header.
Optional: &timeout=[seconds]
Response: 200 with the content
206 with partial content
400 if the parameters are invalid
404 if the File was not found or other error on transfer initiate
502 if unable to find or connect to the remote Peer in time
*/
func (api *WebapiInstance) apiFileView(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
var err error
// validate hashes (must be blake3) and other input
fileHash, valid1 := DecodeBlake3Hash(r.Form.Get("Hash"))
nodeID, valid2 := DecodeBlake3Hash(r.Form.Get("node"))
publicKey, err3 := core.PublicKeyFromPeerID(r.Form.Get("node"))
if !valid1 || (!valid2 && err3 != nil) {
http.Error(w, "", http.StatusBadRequest)
return
}
timeoutSeconds, _ := strconv.Atoi(r.Form.Get("timeout"))
if timeoutSeconds == 0 {
timeoutSeconds = 10
}
timeout := time.Duration(timeoutSeconds) * time.Second
offset, _ := strconv.Atoi(r.Form.Get("offset"))
limit, _ := strconv.Atoi(r.Form.Get("limit"))
format, _ := strconv.Atoi(r.Form.Get("format"))
localCacheDisable, _ := strconv.ParseBool(r.Form.Get("nocache"))
// Range header?
var ranges []HTTPRange
if ranges, err = ParseRangeHeader(r.Header.Get("Range"), -1, true); err != nil || len(ranges) > 1 {
http.Error(w, "", http.StatusBadRequest)
return
} else if len(ranges) == 1 {
if ranges[0].length != -1 { // if length is not specified, limit remains 0 which is maximum
limit = ranges[0].length
}
offset = ranges[0].start
}
w.Header().Set("Accept-Ranges", "bytes") // always indicate accepting of Range header
switch format {
case 14:
// Video: Indicate MP4 always. There are tons of other MIME types that could be used.
w.Header().Set("Content-Type", "video/mp4")
}
// Is the File available in the local warehouse? In that case requesting it from the remote is unnecessary.
if !localCacheDisable {
if serveFileFromWarehouse(api.Backend, w, fileHash, uint64(offset), uint64(limit), ranges) {
return
}
}
// try connecting via node ID or Peer ID?
var peer *core.PeerInfo
if valid2 {
peer, err = PeerConnectNode(api.Backend, nodeID, timeout)
} else if err3 == nil {
peer, err = PeerConnectPublicKey(api.Backend, publicKey, timeout)
}
if err != nil {
w.WriteHeader(http.StatusBadGateway)
return
}
// start the reader
reader, fileSize, transferSize, err := FileStartReader(peer, fileHash, uint64(offset), uint64(limit), r.Context().Done())
if reader != nil {
defer reader.Close()
}
if err != nil || reader == nil {
w.WriteHeader(http.StatusNotFound)
return
}
// set the right headers
setContentLengthRangeHeader(w, uint64(offset), transferSize, fileSize, ranges)
// Start sending the data!
io.Copy(w, io.LimitReader(reader, int64(transferSize)))
}
// PeerConnectPublicKey attempts to connect to the Peer specified by its public key (= Peer ID).
func PeerConnectPublicKey(backend *core.Backend, publicKey *btcec.PublicKey, timeout time.Duration) (peer *core.PeerInfo, err error) {
if publicKey == nil {
return nil, errors.New("invalid public key")
}
// First look up in the Peer list.
if peer = backend.PeerlistLookup(publicKey); peer != nil {
return peer, nil
}
// Try to connect via DHT.
nodeID := protocol.PublicKey2NodeID(publicKey)
if _, peer, _ = backend.FindNode(nodeID, timeout); peer != nil {
return peer, nil
}
// otherwise not found :(
return nil, errors.New("Peer not found")
}
// PeerConnectNode tries to connect via the node ID
func PeerConnectNode(backend *core.Backend, nodeID []byte, timeout time.Duration) (peer *core.PeerInfo, err error) {
if len(nodeID) != 256/8 {
return nil, errors.New("invalid node ID")
}
// Try to connect via DHT.
if _, peer, _ = backend.FindNode(nodeID, timeout); peer != nil {
return peer, nil
}
// otherwise not found :(
return nil, errors.New("Peer not found")
}
// FileStartReader providers a reader to a remote File. The reader must be closed by the caller.
// File Size is the full File size reported by the remote Peer, regardless of the requested offset and limit. Limit is optional (0 means the entire File).
// Transfer Size is the size in bytes that is actually going to be transferred. The reader should be closed after reading that amount.
// The optional cancelChan can be used to stop the File transfer at any point.
func FileStartReader(peer *core.PeerInfo, hash []byte, offset, limit uint64, cancelChan <-chan struct{}) (reader io.ReadCloser, fileSize, transferSize uint64, err error) {
if peer == nil {
return nil, 0, 0, errors.New("Peer not provided")
} else if !peer.IsConnectionActive() {
return nil, 0, 0, errors.New("no valid connection to Peer")
}
udtConn, virtualConn, err := peer.FileTransferRequestUDT(hash, offset, limit)
if err != nil {
return nil, 0, 0, err
}
if cancelChan != nil {
go func() {
<-cancelChan
udtConn.Close()
}()
}
fileSize, transferSize, err = protocol.FileTransferReadHeader(udtConn)
if err != nil {
udtConn.Close()
return nil, 0, 0, err
}
virtualConn.Stats.(*core.FileTransferStats).FileSize = fileSize
return udtConn, fileSize, transferSize, nil
}
// FileReadAll downloads the File from the Peer.
// This function should only be used for testing or as a basis to fork. The caller should develop a custom download function that handles timeouts and excessive File sizes.
// It allocates whatever size is reported by the remote Peer. This could lead to an out of memory crash.
// This function is blocking and may take a long time depending on the remote Peer and the network connection.
func FileReadAll(peer *core.PeerInfo, hash []byte) (data []byte, err error) {
reader, _, transferSize, err := FileStartReader(peer, hash, 0, 0, nil)
if err != nil {
return nil, err
}
defer reader.Close()
// read all data
data = make([]byte, transferSize) // Warning: This could lead to an out of memory crash.
_, err = reader.Read(data)
// Note: This function does not verify if the returned data matches the Hash and expected size.
return data, err
}

338
webapi/File.go
View File

@@ -1,338 +0,0 @@
/*
File Name: File.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"net/http"
"time"
"github.com/PeernetOfficial/core"
"github.com/PeernetOfficial/core/blockchain"
"github.com/PeernetOfficial/core/merkle"
"github.com/PeernetOfficial/core/protocol"
"github.com/PeernetOfficial/core/warehouse"
"github.com/google/uuid"
)
// ApiFileMetadata contains metadata information.
type ApiFileMetadata struct {
Type uint16 `json:"type"` // See core.TagX constants.
Name string `json:"name"` // User friendly name of the metadata type. Use the Type fields to identify the metadata as this name may change.
// Depending on the exact type, one of the below fields is used for proper encoding:
Text string `json:"text"` // Text value. UTF-8 encoding.
Blob []byte `json:"blob"` // Binary data
Date time.Time `json:"date"` // Date
Number uint64 `json:"number"` // Number
}
// ApiFile is the metadata of a File published on the blockchain
type ApiFile struct {
ID uuid.UUID `json:"ID"` // Unique ID.
Hash []byte `json:"Hash"` // Blake3 Hash of the File data
Type uint8 `json:"type"` // File Type. For example audio or document. See TypeX.
Format uint16 `json:"format"` // File Format. This is more granular, for example PDF or Word File. See FormatX.
Size uint64 `json:"size"` // Size of the File
Folder string `json:"folder"` // Folder, optional
Name string `json:"name"` // Name of the File
Description string `json:"description"` // Description. This is expected to be multiline and contain hashtags!
Date time.Time `json:"date"` // Date shared
NodeID []byte `json:"nodeid"` // Node ID, owner of the File. Read only.
Metadata []ApiFileMetadata `json:"metadata"` // Additional metadata.
}
// --- conversion from core to API data ---
func blockRecordFileToAPI(input blockchain.BlockRecordFile) (output ApiFile) {
output = ApiFile{ID: input.ID, Hash: input.Hash, NodeID: input.NodeID, Type: input.Type, Format: input.Format, Size: input.Size, Metadata: []ApiFileMetadata{}}
for _, tag := range input.Tags {
switch tag.Type {
case blockchain.TagName:
output.Name = tag.Text()
case blockchain.TagFolder:
output.Folder = tag.Text()
case blockchain.TagDescription:
output.Description = tag.Text()
case blockchain.TagDateShared:
output.Date, _ = tag.Date()
case blockchain.TagDateCreated:
date, _ := tag.Date()
output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Date Created", Date: date})
case blockchain.TagSharedByCount:
output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Shared By Count", Number: tag.Number()})
case blockchain.TagSharedByGeoIP:
output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Name: "Shared By GeoIP", Text: tag.Text()})
default:
output.Metadata = append(output.Metadata, ApiFileMetadata{Type: tag.Type, Blob: tag.Data})
}
}
return output
}
func BlockRecordFileFromAPI(input ApiFile) (output blockchain.BlockRecordFile) {
output = blockchain.BlockRecordFile{ID: input.ID, Hash: input.Hash, Type: input.Type, Format: input.Format, Size: input.Size}
if input.Name != "" {
output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagName, input.Name))
}
if input.Folder != "" {
output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagFolder, input.Folder))
}
if input.Description != "" {
output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagDescription, input.Description))
}
for _, meta := range input.Metadata {
if blockchain.IsTagVirtual(meta.Type) { // Virtual tags are not mapped back. They are read-only.
continue
}
switch meta.Type {
case blockchain.TagName, blockchain.TagFolder, blockchain.TagDescription: // auto mapped tags
case blockchain.TagDateCreated:
output.Tags = append(output.Tags, blockchain.TagFromDate(meta.Type, meta.Date))
default:
output.Tags = append(output.Tags, blockchain.BlockRecordFileTag{Type: meta.Type, Data: meta.Blob})
}
}
return output
}
// --- File API ---
// ApiBlockAddFiles contains a list of files from the blockchain
type ApiBlockAddFiles struct {
Files []ApiFile `json:"files"` // List of files
Status int `json:"Status"` // Status of the operation, only used when this structure is returned from the API.
}
/*
apiBlockchainFileAdd adds a File with the provided information to the blockchain.
Each File must be already stored in the Warehouse (virtual folders are exempt).
If any File is not stored in the Warehouse, the function aborts with the Status code StatusNotInWarehouse.
If the block record encoding fails for any File, this function aborts with the Status code StatusCorruptBlockRecord.
In case the function aborts, the blockchain remains unchanged.
Request: POST /blockchain/File/add with JSON structure ApiBlockAddFiles
Response: 200 with JSON structure apiBlockchainBlockStatus
400 if invalid input
*/
func (api *WebapiInstance) apiBlockchainFileAdd(w http.ResponseWriter, r *http.Request) {
var input ApiBlockAddFiles
if err := DecodeJSON(w, r, &input); err != nil {
return
}
var filesAdd []blockchain.BlockRecordFile
for _, file := range input.Files {
if len(file.Hash) != protocol.HashSize {
http.Error(w, "", http.StatusBadRequest)
return
}
if file.ID == uuid.Nil { // if the ID is not provided by the caller, set it
file.ID = uuid.New()
}
// Verify that the File exists in the warehouse. Folders are exempt from this check as they are only virtual.
if !file.IsVirtualFolder() {
if _, err := warehouse.ValidateHash(file.Hash); err != nil {
http.Error(w, "", http.StatusBadRequest)
return
} else if _, fileSize, status, _ := api.Backend.UserWarehouse.FileExists(file.Hash); status != warehouse.StatusOK {
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse})
return
} else {
file.Size = fileSize
}
} else {
file.Hash = protocol.HashData(nil)
file.Size = 0
}
blockRecord := BlockRecordFileFromAPI(file)
// Set the merkle tree info as appropriate.
if !SetFileMerkleInfo(api.Backend, &blockRecord) {
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse})
return
}
filesAdd = append(filesAdd, blockRecord)
}
newHeight, newVersion, status := api.Backend.UserBlockchain.AddFiles(filesAdd)
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion})
}
/*
apiBlockchainFileList lists all files stored on the blockchain.
Request: GET /blockchain/File/list
Response: 200 with JSON structure ApiBlockAddFiles
*/
func (api *WebapiInstance) apiBlockchainFileList(w http.ResponseWriter, r *http.Request) {
files, status := api.Backend.UserBlockchain.ListFiles()
var result ApiBlockAddFiles
for _, file := range files {
result.Files = append(result.Files, blockRecordFileToAPI(file))
}
result.Status = status
EncodeJSON(api.Backend, w, r, result)
}
/*
apiBlockchainFileDelete deletes files with the provided IDs. Other fields are ignored.
It will automatically delete the File in the Warehouse if there are no other references.
Request: POST /blockchain/File/delete with JSON structure ApiBlockAddFiles
Response: 200 with JSON structure apiBlockchainBlockStatus
*/
func (api *WebapiInstance) apiBlockchainFileDelete(w http.ResponseWriter, r *http.Request) {
var input ApiBlockAddFiles
if err := DecodeJSON(w, r, &input); err != nil {
return
}
var deleteIDs []uuid.UUID
for n := range input.Files {
deleteIDs = append(deleteIDs, input.Files[n].ID)
}
newHeight, newVersion, deletedFiles, status := api.Backend.UserBlockchain.DeleteFiles(deleteIDs)
// If successfully deleted from the blockchain, delete from the Warehouse in case there are no other references.
if status == blockchain.StatusOK {
for n := range deletedFiles {
if files, status := api.Backend.UserBlockchain.FileExists(deletedFiles[n].Hash); status == blockchain.StatusOK && len(files) == 0 {
api.Backend.UserWarehouse.DeleteFile(deletedFiles[n].Hash)
}
}
}
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion})
}
/*
apiBlockchainSelfUpdateFile updates files that are already published on the blockchain.
Request: POST /blockchain/File/update with JSON structure ApiBlockAddFiles
Response: 200 with JSON structure apiBlockchainBlockStatus
400 if invalid input
*/
func (api *WebapiInstance) apiBlockchainFileUpdate(w http.ResponseWriter, r *http.Request) {
var input ApiBlockAddFiles
if err := DecodeJSON(w, r, &input); err != nil {
return
}
var filesAdd []blockchain.BlockRecordFile
for _, file := range input.Files {
if len(file.Hash) != protocol.HashSize {
http.Error(w, "", http.StatusBadRequest)
return
} else if file.ID == uuid.Nil { // if the ID is not provided by the caller, abort
http.Error(w, "", http.StatusBadRequest)
return
}
// Verify that the File exists in the warehouse. Folders are exempt from this check as they are only virtual.
if !file.IsVirtualFolder() {
if _, err := warehouse.ValidateHash(file.Hash); err != nil {
http.Error(w, "", http.StatusBadRequest)
return
} else if _, fileSize, status, _ := api.Backend.UserWarehouse.FileExists(file.Hash); status != warehouse.StatusOK {
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse})
return
} else {
file.Size = fileSize
}
} else {
file.Hash = protocol.HashData(nil)
file.Size = 0
}
blockRecord := BlockRecordFileFromAPI(file)
// Set the merkle tree info as appropriate.
if !SetFileMerkleInfo(api.Backend, &blockRecord) {
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse})
return
}
filesAdd = append(filesAdd, blockRecord)
}
newHeight, newVersion, status := api.Backend.UserBlockchain.ReplaceFiles(filesAdd)
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion})
}
// ---- metadata functions ----
// GetMetadata returns the specified metadata or nil if not available.
func (file *ApiFile) GetMetadata(Type uint16) (info *ApiFileMetadata) {
for n := range file.Metadata {
if file.Metadata[n].Type == Type {
return &file.Metadata[n]
}
}
return nil
}
// GetNumber returns the data as number. 0 if not available.
func (info *ApiFileMetadata) GetNumber() uint64 {
if info == nil {
return 0
}
return info.Number
}
// IsVirtualFolder returns true if the File is a virtual folder
func (file *ApiFile) IsVirtualFolder() bool {
return file.Type == core.TypeFolder && file.Format == core.FormatFolder
}
// SetFileMerkleInfo sets the merkle fields in the BlockRecordFile
func SetFileMerkleInfo(backend *core.Backend, file *blockchain.BlockRecordFile) (valid bool) {
if file.Size <= merkle.MinimumFragmentSize {
// If smaller or equal than the minimum fragment size, the merkle tree is not used.
file.MerkleRootHash = file.Hash
file.FragmentSize = merkle.MinimumFragmentSize
} else {
// Get the information from the Warehouse .merkle companion File.
tree, status, _ := backend.UserWarehouse.ReadMerkleTree(file.Hash, true)
if status != warehouse.StatusOK {
return false
}
file.MerkleRootHash = tree.RootHash
file.FragmentSize = tree.FragmentSize
}
return true
}

50
webapi/GeoIP.go
View File

@@ -1,50 +0,0 @@
/*
File Name: GeoIP.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
Support for the free MaxMind database 'GeoLite2 City'.
Information about the database: https://dev.maxmind.com/geoip/geolite2-free-geolocation-data
Potential libraries:
* https://github.com/IncSW/geoip2
* https://github.com/oschwald/maxminddb-golang
The IncSW lib was chosen because it has 0 dependencies - awesome!
*/
package webapi
import (
"net"
"github.com/IncSW/geoip2"
"github.com/PeernetOfficial/core"
)
func (api *WebapiInstance) InitGeoIPDatabase(filename string) (err error) {
api.geoipCityReader, err = geoip2.NewCityReaderFromFile(filename)
return err
}
func (api *WebapiInstance) GeoIPLocation(IP net.IP) (latitude, longitude float64, valid bool) {
if api.geoipCityReader == nil {
return 0, 0, false
}
record, err := api.geoipCityReader.Lookup(IP)
if err != nil {
return 0, 0, false
}
return record.Location.Latitude, record.Location.Longitude, true
}
func (api *WebapiInstance) Peer2GeoIP(peer *core.PeerInfo) (latitude, longitude float64, valid bool) {
if connection := peer.GetConnection2Share(false, true, true); connection != nil {
return api.GeoIPLocation(connection.Address.IP)
}
return 0, 0, false
}

109
webapi/HTTP Range.go
View File

@@ -1,109 +0,0 @@
/*
File Name: HTTP Range.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"errors"
"net/http"
"net/textproto"
"strconv"
"strings"
)
// Fork from https://golang.org/src/net/http/fs.go
// HTTPRange represents an HTTP range
type HTTPRange struct {
start, length int
}
// ParseRangeHeader parses a Range header string as per RFC 7233.
func ParseRangeHeader(s string, size int, noMultiRange bool) ([]HTTPRange, error) {
if s == "" {
return nil, nil // header not present
}
const b = "bytes="
if !strings.HasPrefix(s, b) {
return nil, errors.New("invalid range")
}
var ranges []HTTPRange
for _, ra := range strings.Split(s[len(b):], ",") {
ra = textproto.TrimString(ra)
if ra == "" {
continue
}
i := strings.Index(ra, "-")
if i < 0 {
return nil, errors.New("invalid range")
}
start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:])
var r HTTPRange
if start == "" {
if size < 0 {
return nil, errors.New("range start relative to end not supported")
}
// If no start is specified, end specifies the
// range start relative to the end of the File.
i, err := strconv.ParseInt(end, 10, 64)
if err != nil {
return nil, errors.New("invalid range")
}
if int(i) > size {
i = int64(size)
}
r.start = size - int(i)
r.length = size - r.start
} else {
i, err := strconv.ParseInt(start, 10, 64)
if err != nil || i < 0 {
return nil, errors.New("invalid range")
}
if size > 0 && int(i) >= size {
// If the range begins after the size of the content, then it does not overlap. -> always return error.
return nil, errors.New("start out of range")
}
r.start = int(i)
if end == "" {
// If no end is specified, range extends to end of the File.
if size < 0 {
//return nil, errors.New("open range not supported")
r.length = -1
} else {
r.length = size - r.start
}
} else {
i, err := strconv.ParseInt(end, 10, 64)
if err != nil || r.start > int(i) {
return nil, errors.New("invalid range")
}
if size > 0 && int(i) >= size {
i = int64(size - 1)
}
r.length = int(i) - r.start + 1
}
}
ranges = append(ranges, r)
if noMultiRange && len(ranges) > 1 {
return nil, errors.New("multiple ranges not supported")
}
}
return ranges, nil
}
// setContentLengthRangeHeader sets the appropriate Content-Length and Content-Range headers
func setContentLengthRangeHeader(w http.ResponseWriter, offset, transferSize, fileSize uint64, ranges []HTTPRange) {
// Set the Content-Length header, always to the actual size of transferred data.
w.Header().Set("Content-Length", strconv.FormatUint(transferSize, 10))
// Set the Content-Range header if needed.
if len(ranges) == 1 {
w.Header().Set("Content-Range", "bytes "+strconv.FormatUint(offset, 10)+"-"+strconv.FormatUint(offset+transferSize-1, 10)+"/"+strconv.FormatUint(fileSize, 10))
w.WriteHeader(http.StatusPartialContent)
} else {
w.WriteHeader(http.StatusOK)
}
}

153
webapi/Profile.go
View File

@@ -1,153 +0,0 @@
/*
File Name: Profile.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"net/http"
"strconv"
"github.com/PeernetOfficial/core/blockchain"
)
// apiProfileData contains profile metadata stored on the blockchain. Any data is treated as untrusted and unverified by default.
type apiProfileData struct {
Fields []apiBlockRecordProfile `json:"fields"` // All fields
Status int `json:"Status"` // Status of the operation, only used when this structure is returned from the API. See blockchain.StatusX.
}
// apiBlockRecordProfile provides information about the end user. Note that all profile data is arbitrary and shall be considered untrusted and unverified.
// To establish trust, the user must load Certificates into the blockchain that validate certain data.
type apiBlockRecordProfile struct {
Type uint16 `json:"type"` // See ProfileX constants.
// Depending on the exact type, one of the below fields is used for proper encoding:
Text string `json:"text"` // Text value. UTF-8 encoding.
Blob []byte `json:"blob"` // Binary data
}
/*
apiProfileList lists all users profile fields.
Request: GET /profile/list
Response: 200 with JSON structure apiProfileData
*/
func (api *WebapiInstance) apiProfileList(w http.ResponseWriter, r *http.Request) {
fields, status := api.Backend.UserBlockchain.ProfileList()
result := apiProfileData{Status: status}
for n := range fields {
result.Fields = append(result.Fields, blockRecordProfileToAPI(fields[n]))
}
EncodeJSON(api.Backend, w, r, result)
}
/*
apiProfileRead reads a specific users profile field. See core.ProfileX for recognized fields.
Request: GET /profile/read?field=[index]
Response: 200 with JSON structure apiProfileData
*/
func (api *WebapiInstance) apiProfileRead(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
fieldN, err1 := strconv.Atoi(r.Form.Get("field"))
if err1 != nil || fieldN < 0 {
http.Error(w, "", http.StatusBadRequest)
return
}
var result apiProfileData
var data []byte
if data, result.Status = api.Backend.UserBlockchain.ProfileReadField(uint16(fieldN)); result.Status == blockchain.StatusOK {
result.Fields = append(result.Fields, blockRecordProfileToAPI(blockchain.BlockRecordProfile{Type: uint16(fieldN), Data: data}))
}
EncodeJSON(api.Backend, w, r, result)
}
/*
apiProfileWrite writes profile fields. See core.ProfileX for recognized fields.
Request: POST /profile/write with JSON structure apiProfileData
Response: 200 with JSON structure apiBlockchainBlockStatus
*/
func (api *WebapiInstance) apiProfileWrite(w http.ResponseWriter, r *http.Request) {
var input apiProfileData
if err := DecodeJSON(w, r, &input); err != nil {
return
}
var fields []blockchain.BlockRecordProfile
for n := range input.Fields {
fields = append(fields, blockRecordProfileFromAPI(input.Fields[n]))
}
newHeight, newVersion, status := api.Backend.UserBlockchain.ProfileWrite(fields)
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion})
}
/*
apiProfileDelete deletes profile fields identified by the types. See core.ProfileX for recognized fields.
Request: POST /profile/delete with JSON structure apiProfileData
Response: 200 with JSON structure apiBlockchainBlockStatus
*/
func (api *WebapiInstance) apiProfileDelete(w http.ResponseWriter, r *http.Request) {
var input apiProfileData
if err := DecodeJSON(w, r, &input); err != nil {
return
}
var fields []uint16
for n := range input.Fields {
fields = append(fields, input.Fields[n].Type)
}
newHeight, newVersion, status := api.Backend.UserBlockchain.ProfileDelete(fields)
EncodeJSON(api.Backend, w, r, apiBlockchainBlockStatus{Status: status, Height: newHeight, Version: newVersion})
}
// --- conversion from core to API data ---
func blockRecordProfileToAPI(input blockchain.BlockRecordProfile) (output apiBlockRecordProfile) {
output.Type = input.Type
switch input.Type {
case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress:
output.Text = input.Text()
case blockchain.ProfilePicture:
output.Blob = input.Data
default:
output.Blob = input.Data
}
return output
}
func blockRecordProfileFromAPI(input apiBlockRecordProfile) (output blockchain.BlockRecordProfile) {
output.Type = input.Type
switch input.Type {
case blockchain.ProfileName, blockchain.ProfileEmail, blockchain.ProfileWebsite, blockchain.ProfileTwitter, blockchain.ProfileYouTube, blockchain.ProfileAddress:
output.Data = []byte(input.Text)
case blockchain.ProfilePicture:
output.Data = input.Blob
default:
output.Data = input.Blob
}
return output
}

83
webapi/Search Dispatch.go
View File

@@ -1,83 +0,0 @@
/*
File Name: Search Dispatch.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"bytes"
"fmt"
"time"
"github.com/PeernetOfficial/core/blockchain"
)
func (api *WebapiInstance) DispatchSearch(input SearchRequest) (job *SearchJob) {
Timeout := input.Parse()
Filter := input.ToSearchFilter()
// create the search job
job = api.CreateSearchJob(Timeout, input.MaxResults, Filter)
// todo: create actual search clients!
job.Status = SearchStatusLive
go job.localSearch(api, input.Term)
api.RemoveJobDefer(job, job.timeout+time.Minute*10)
return job
}
func (job *SearchJob) localSearch(api *WebapiInstance, term string) {
if api.Backend.SearchIndex == nil {
job.Status = SearchStatusNoIndex
return
}
results := api.Backend.SearchIndex.Search(term)
job.ResultSync.Lock()
resultLoop:
for _, result := range results {
file, _, found, err := api.Backend.ReadFile(result.PublicKey, result.BlockchainVersion, result.BlockNumber, result.FileID)
if err != nil || !found {
continue
}
// Deduplicate based on File Hash from the same Peer.
for n := range job.AllFiles {
if bytes.Equal(job.AllFiles[n].Hash, file.Hash) && bytes.Equal(job.AllFiles[n].NodeID, file.NodeID) {
continue resultLoop
}
}
if bytes.Equal(file.NodeID, api.Backend.SelfNodeID()) {
// Indicates data from the current user.
file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1))
} else if peer := api.Backend.NodelistLookup(file.NodeID); peer != nil {
// add the tags 'Shared By Count' and 'Shared By GeoIP'
file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1))
if latitude, longitude, valid := api.Peer2GeoIP(peer); valid {
sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude)
file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP))
}
}
// new result
newFile := blockRecordFileToAPI(file)
job.Files = append(job.Files, &newFile)
job.AllFiles = append(job.AllFiles, &newFile)
job.requireSort = true
job.statsAdd(&newFile)
}
job.Status = SearchStatusTerminated
job.ResultSync.Unlock()
job.Terminate()
}

469
webapi/Search Job.go
View File

@@ -1,469 +0,0 @@
/*
File Name: Search Job.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"sort"
"sync"
"time"
"github.com/PeernetOfficial/core/blockchain"
"github.com/google/uuid"
)
// SearchFilter allows to filter search results based on the criteria.
type SearchFilter struct {
IsDates bool // Whether the from/to dates are valid, both are required.
DateFrom time.Time // Optional date from
DateTo time.Time // Optional date to
FileType int // File type such as binary, text document etc. See core.TypeX. -1 = not used.
FileFormat int // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used.
Sort int // Sort order. See SortX.
SizeMin int // Min File size in bytes. -1 = not used.
SizeMax int // Max File size in bytes. -1 = not used.
}
// SearchJob is a collection of search jobs
type SearchJob struct {
// input settings
ID uuid.UUID // The job ID
timeout time.Duration // timeout set for all searches
maxResult int // max results user-facing.
filtersStart SearchFilter // Filters when starting the search. They cannot be changed later on. Any incoming File is checked against them, even if there are different runtime filters.
filtersRuntime SearchFilter // Runtime Filters. They allow filtering results after they were received.
// File statistics (filters are ignored) of returned results. Map value is always count of files.
stats struct {
sync.RWMutex // Synced access to maps
date map[time.Time]int // Files per day (rounded down to midnight)
fileType map[uint8]int // Files per File Type
fileFormat map[uint16]int // Files per File Format
total int // Total count of files
}
// -- result data --
// Status indicates the overall search Status. This will be removed later when relying on search clients.
Status int
// runtime data
//clients []*SearchClient // all search clients
clientsMutex sync.Mutex // mutex for manipulating client list
// List of files found but not yet returned via API to the caller. They are subject to sorting.
Files []*ApiFile
requireSort bool // if Files requires sort before returning the results
// FreezeFiles is a list of files that were already finally delivered via the API. They may NOT change in sorting.
FreezeFiles []*ApiFile
// List of all files. Does not change based on sorting or runtime filters. This list only gets expanded.
AllFiles []*ApiFile
ResultSync sync.Mutex // ResultSync ensures unique access to the File results
currentOffset int // for always getting the next results
}
const (
SearchStatusNotStarted = iota // Search was not yet started
SearchStatusLive // Search running
SearchStatusTerminated // Search is terminated. No more results are expected.
SearchStatusNoIndex // Search is terminated. No search index to use.
)
// CreateSearchJob creates a new search job and adds it to the lookup list.
// Timeout and MaxResults must be set and must not be 0.
func (api *WebapiInstance) CreateSearchJob(Timeout time.Duration, MaxResults int, Filter SearchFilter) (job *SearchJob) {
job = &SearchJob{}
job.Status = SearchStatusNotStarted
job.ID = uuid.New()
job.timeout = Timeout
job.maxResult = MaxResults
job.filtersStart = Filter
job.filtersRuntime = Filter // initialize the runtime filters as the same
job.stats.date = make(map[time.Time]int)
job.stats.fileType = make(map[uint8]int)
job.stats.fileFormat = make(map[uint16]int)
// add to the list of jobs
api.allJobsMutex.Lock()
api.allJobs[job.ID] = job
api.allJobsMutex.Unlock()
return
}
// ReturnResult returns the selected results.
func (job *SearchJob) ReturnResult(Offset, Limit int) (Result []*ApiFile) {
if Limit == 0 {
return Result
}
job.ResultSync.Lock()
defer job.ResultSync.Unlock()
// serve files from frozen list?
if Offset < len(job.FreezeFiles) {
countCopy := len(job.FreezeFiles) - Offset
if countCopy > Limit {
countCopy = Limit
}
Result = job.FreezeFiles[Offset : Offset+countCopy]
Limit -= countCopy
Offset = 0
} else {
Offset -= len(job.FreezeFiles)
}
if Limit == 0 {
return Result
}
// go through the live results and fill the list
if Offset >= len(job.Files) { // offset wants to skip entire queue?
job.FreezeFiles = append(job.FreezeFiles, job.Files...)
job.Files = nil
return Result
}
// check if a sort is required before using this queue
if job.requireSort {
job.requireSort = false
job.Files = SortFiles(job.Files, job.filtersRuntime.Sort)
}
// set the amount of files to copy
countCopy := len(job.Files) - Offset
if countCopy > Limit {
countCopy = Limit
}
// copy the results and freeze them
Result = append(Result, job.Files[Offset:Offset+countCopy]...)
// note that freeze disregards the offset, it has to freeze any elements before!
job.FreezeFiles = append(job.FreezeFiles, job.Files[:Offset+countCopy]...)
job.Files = job.Files[Offset+countCopy:]
//Limit -= countCopy
return Result
}
// ReturnNext returns the next results. Call must be serialized.
func (job *SearchJob) ReturnNext(Limit int) (Result []*ApiFile) {
Result = job.ReturnResult(job.currentOffset, Limit)
job.currentOffset += len(Result)
return
}
// PeekResult returns the selected results but will not change any frozen files or impact auto offset
func (job *SearchJob) PeekResult(Offset, Limit int) (Result []*ApiFile) {
job.ResultSync.Lock()
defer job.ResultSync.Unlock()
// serve files from frozen list?
if Offset < len(job.FreezeFiles) {
countCopy := len(job.FreezeFiles) - Offset
if countCopy > Limit {
countCopy = Limit
}
Result = job.FreezeFiles[Offset : Offset+countCopy]
Limit -= countCopy
Offset = 0
} else {
Offset -= len(job.FreezeFiles)
}
if Limit == 0 || Offset >= len(job.Files) { // offset wants to skip entire queue?
return Result
}
// check if a sort is required before using this queue
if job.requireSort {
job.requireSort = false
job.Files = SortFiles(job.Files, job.filtersRuntime.Sort)
}
countCopy := len(job.Files) - Offset
if countCopy > Limit {
countCopy = Limit
}
// copy the results
Result = append(Result, job.Files[Offset:Offset+countCopy]...)
return Result
}
// RuntimeFilter allows to apply filters at runtime to search jobs that already started. To remove the filters, call this function without the filters set.
func (job *SearchJob) RuntimeFilter(Filter SearchFilter) {
job.ResultSync.Lock()
defer job.ResultSync.Unlock()
job.filtersRuntime = Filter
// FreezeFiles and current offset is reset
job.FreezeFiles = nil
job.currentOffset = 0
// files remain in AllFiles, but Files needs to be filtered based on the new filter
job.Files = []*ApiFile{}
job.requireSort = false // Sorting is done immediately below
// set Files based on AllFiles with the filter
for m := range job.AllFiles {
// only append if filter is matching
if job.isFileFiltered(job.AllFiles[m]) {
job.Files = append(job.Files, job.AllFiles[m])
}
// sort, if a sort order is defined
if job.filtersRuntime.Sort > 0 {
job.Files = SortFiles(job.Files, job.filtersRuntime.Sort)
}
}
}
// isFileFiltered returns true if the File conforms to the runtime filter. If there is no runtime filter, it always returns true.
func (job *SearchJob) isFileFiltered(file *ApiFile) bool {
if job.filtersRuntime.FileType >= 0 && file.Type != uint8(job.filtersRuntime.FileType) {
return false
}
if job.filtersRuntime.FileFormat >= 0 && file.Format != uint16(job.filtersRuntime.FileFormat) {
return false
}
// Note: If the date is not available in the File, it will be filtered out. Since this is the mapped Shared Date this should normally not occur though.
if job.filtersRuntime.IsDates && (file.Date.IsZero() || file.Date.Before(job.filtersRuntime.DateFrom) || file.Date.After(job.filtersRuntime.DateTo)) {
return false
}
if job.filtersRuntime.SizeMin >= 0 && file.Size < uint64(job.filtersRuntime.SizeMin) || job.filtersRuntime.SizeMax >= 0 && file.Size > uint64(job.filtersRuntime.SizeMax) {
return false
}
return true
}
// SortFiles sorts a list of files. It returns a sorted list. 0 = no sorting, 1 = Relevance ASC, 2 = Relevance DESC, 3 = Date ASC, 4 = Date DESC, 5 = Name ASC, 6 = Name DESC
func SortFiles(files []*ApiFile, Sort int) (sorted []*ApiFile) {
switch Sort {
case SortRelevanceAsc:
sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) }) // first as date for secondary sorting
//sort.SliceStable(files, func(i, j int) bool { return files[i].Score < files[j].Score }) // TODO
case SortRelevanceDec:
sort.SliceStable(files, func(i, j int) bool { return files[j].Date.Before(files[i].Date) }) // first as date for secondary sorting
//sort.SliceStable(files, func(i, j int) bool { return files[i].Score > files[j].Score }) // TODO
case SortDateAsc:
sort.SliceStable(files, func(i, j int) bool { return files[i].Date.Before(files[j].Date) })
case SortDateDesc:
sort.SliceStable(files, func(i, j int) bool { return files[j].Date.Before(files[i].Date) })
case SortNameAsc:
sort.SliceStable(files, func(i, j int) bool { return files[i].Name < files[j].Name })
case SortNameDesc:
sort.SliceStable(files, func(i, j int) bool { return files[i].Name > files[j].Name })
case SortSizeAsc:
sort.SliceStable(files, func(i, j int) bool { return files[i].Size < files[j].Size })
case SortSizeDesc:
sort.SliceStable(files, func(i, j int) bool { return files[i].Size > files[j].Size })
case SortSharedByCountAsc:
sort.SliceStable(files, func(i, j int) bool {
return files[i].GetMetadata(blockchain.TagSharedByCount).Number < files[j].GetMetadata(blockchain.TagSharedByCount).Number
})
case SortSharedByCountDesc:
sort.SliceStable(files, func(i, j int) bool {
return files[i].GetMetadata(blockchain.TagSharedByCount).Number > files[j].GetMetadata(blockchain.TagSharedByCount).Number
})
}
return files
}
// IsSearchResults checks if search results may be expected (either files are in queue or a search is running)
func (job *SearchJob) IsSearchResults() bool {
// check for any available results. Do not use any lock here as this is read only.
return len(job.Files) > 0 || !job.IsTerminated()
}
// isFileReceived checks if a File was already received, preventing double results
func (job *SearchJob) isFileReceived(id uuid.UUID) (exists bool) {
// Future: A map would be likely faster than iterating over all results.
for m := range job.AllFiles {
if id == job.AllFiles[m].ID {
return true
}
}
return false
}
// ---- job list management ----
// RemoveJob removes the job structure from the list. Terminate should be called before. Unless the search is manually removed, it stays forever in the list.
func (api *WebapiInstance) RemoveJob(job *SearchJob) {
api.allJobsMutex.Lock()
delete(api.allJobs, job.ID) // delete is safe to call multiple times, so auto-removal and manual one are fine and need no syncing
api.allJobsMutex.Unlock()
}
// RemoveDefer removes the search job after a given time after all searches are terminated. This can be used for automated time delayed removal. Do not create additional search clients after deferal removing.
func (api *WebapiInstance) RemoveJobDefer(job *SearchJob, Duration time.Duration) {
go func() {
// for _, client := range job.clients {
// <-client.TerminateSignal
// }
<-time.After(Duration)
api.RemoveJob(job)
}()
}
// JobLookup looks up a job. Returns nil if not found.
func (api *WebapiInstance) JobLookup(id uuid.UUID) (job *SearchJob) {
api.allJobsMutex.RLock()
job = api.allJobs[id]
api.allJobsMutex.RUnlock()
return job
}
// IsTerminated checks if all searches are finished. The job itself does not record a termination signal.
func (job *SearchJob) IsTerminated() bool {
job.clientsMutex.Lock()
defer job.clientsMutex.Unlock()
// for n := range job.clients {
// if !job.clients[n].IsTerminated {
// return false
// }
// }
return job.Status == SearchStatusTerminated || job.Status == SearchStatusNoIndex
}
// Terminate terminates all searches
func (job *SearchJob) Terminate() {
job.clientsMutex.Lock()
defer job.clientsMutex.Unlock()
// for n := range job.clients {
// if !job.clients[n].IsTerminated {
// job.clients[n].Terminate(true)
// }
// }
}
// WaitTerminate waits until all search clients are terminated. Do not create additional search clients after calling this function.
func (job *SearchJob) WaitTerminate() {
//for _, client := range job.clients {
// <-client.TerminateSignal
//}
}
// ---- statistics ----
// SearchStatisticRecordDay is a single record containing date info.
type SearchStatisticRecordDay struct {
Date time.Time `json:"date"` // The day (which covers the full 24 hours). Always rounded down to midnight.
Count int `json:"count"` // Count of files.
}
// SearchStatisticRecord is a single record.
type SearchStatisticRecord struct {
Key int `json:"key"` // Key index. The exact meaning depends on where this structure is used.
Count int `json:"count"` // Count of files for the given key
}
// SearchStatisticData contains statistics on search results.
type SearchStatisticData struct {
Date []SearchStatisticRecordDay `json:"date"` // Files per date
FileType []SearchStatisticRecord `json:"filetype"` // Files per File type
FileFormat []SearchStatisticRecord `json:"fileformat"` // Files per File format
Total int `json:"total"` // Total count of files
}
// Statistics generates statistics on all results, regardless of runtime filters.
func (job *SearchJob) Statistics() (result SearchStatisticData) {
job.stats.RLock()
defer job.stats.RUnlock()
result.Total = job.stats.total
// Files per date. Sort dates to date ASC.
for key, value := range job.stats.date {
result.Date = append(result.Date, SearchStatisticRecordDay{Date: key, Count: value})
}
sort.SliceStable(result.Date, func(i, j int) bool { return result.Date[i].Date.Before(result.Date[j].Date) })
// File Type and Format
for key, value := range job.stats.fileType {
result.FileType = append(result.FileType, SearchStatisticRecord{Key: int(key), Count: value})
}
for key, value := range job.stats.fileFormat {
result.FileFormat = append(result.FileFormat, SearchStatisticRecord{Key: int(key), Count: value})
}
return
}
// statsAdd counts the files in the statistics
func (job *SearchJob) statsAdd(files ...*ApiFile) {
job.stats.Lock()
defer job.stats.Unlock()
for _, file := range files {
// Use File's Date field if available.
if !file.Date.IsZero() {
// Files per day
date := file.Date.Truncate(24 * time.Hour)
countDate := job.stats.date[date]
countDate++
job.stats.date[date] = countDate
}
// File Type and Format
countType := job.stats.fileType[file.Type]
countType++
job.stats.fileType[file.Type] = countType
countFormat := job.stats.fileFormat[file.Format]
countFormat++
job.stats.fileFormat[file.Format] = countFormat
}
job.stats.total += len(files)
}
// ---- actual search & retrieving results ----
// appendSearchClient appends a search client to a job
func (job *SearchJob) SearchAway() {
job.clientsMutex.Lock()
defer job.clientsMutex.Unlock()
// TODO
}
//func (job *SearchJob) appendSearchClient(search *dht.SearchClient) {
//}
//func (job *SearchJob) receiveClientResults(client *dht.SearchClient) {
//}

417
webapi/Search.go
View File

@@ -1,417 +0,0 @@
/*
File Name: Search.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
/search Submit a search request
/search/result Return search results
/search/terminate Terminate a search
/search/result/ws Websocket to receive results as stream
/search/statistic Statistics about the results
/explore List recently shared files
*/
package webapi
import (
"net/http"
"strconv"
"time"
"github.com/google/uuid"
)
// SearchRequest is the information from the end-user for the search. Filters and sort order may be applied when starting the search, or at runtime when getting the results.
type SearchRequest struct {
Term string `json:"term"` // Search term.
Timeout int `json:"timeout"` // Timeout in seconds. 0 means default. This is the entire time the search may take. Found results are still available after this timeout.
MaxResults int `json:"maxresults"` // Total number of max results. 0 means default.
DateFrom string `json:"datefrom"` // Date from, both from/to are required if set. Format "2006-01-02 15:04:05".
DateTo string `json:"dateto"` // Date to, both from/to are required if set. Format "2006-01-02 15:04:05".
Sort int `json:"sort"` // See SortX.
TerminateID []uuid.UUID `json:"terminate"` // Optional: Previous search IDs to terminate. This is if the user makes a new search from the same tab. Same as first calling /search/terminate.
FileType int `json:"filetype"` // File type such as binary, text document etc. See core.TypeX. -1 = not used.
FileFormat int `json:"fileformat"` // File format such as PDF, Word, Ebook, etc. See core.FormatX. -1 = not used.
SizeMin int `json:"sizemin"` // Min File size in bytes. -1 = not used.
SizeMax int `json:"sizemax"` // Max File size in bytes. -1 = not used.
}
// Sort orders
const (
SortNone = 0 // No sorting. Results are returned as they come in.
SortRelevanceAsc = 1 // Least relevant results first.
SortRelevanceDec = 2 // Most relevant results first.
SortDateAsc = 3 // Oldest first.
SortDateDesc = 4 // Newest first.
SortNameAsc = 5 // File name ascending. The folder name is not used for sorting.
SortNameDesc = 6 // File name descending. The folder name is not used for sorting.
SortSizeAsc = 7 // File size ascending. Smallest files first.
SortSizeDesc = 8 // File size descending. Largest files first.
SortSharedByCountAsc = 9 // Shared by count ascending. Files that are shared by the least count of peers first.
SortSharedByCountDesc = 10 // Shared by count descending. Files that are shared by the most count of peers first.
)
// SearchRequestResponse is the result to the initial search request
type SearchRequestResponse struct {
ID uuid.UUID `json:"ID"` // ID of the search job. This is used to get the results.
Status int `json:"Status"` // Status of the search: 0 = Success (ID valid), 1 = Invalid Term, 2 = Error Max Concurrent Searches
}
// SearchResult contains the search results.
type SearchResult struct {
Status int `json:"Status"` // Status: 0 = Success with results, 1 = No more results available, 2 = Search ID not found, 3 = No results yet available keep trying
Files []ApiFile `json:"files"` // List of files found
Statistic interface{} `json:"statistic"` // Statistics of all results (independent from applied filters), if requested. Only set if files are returned (= if statistics changed). See SearchStatisticData.
}
// SearchStatistic contains statistics on search results. Statistics are always calculated over all results, regardless of any applied runtime filters.
type SearchStatistic struct {
SearchStatisticData
Status int `json:"Status"` // Status: 0 = Success
IsTerminated bool `json:"terminated"` // Whether the search is terminated, meaning that statistics won't change
}
const apiDateFormat = "2006-01-02 15:04:05"
/*
apiSearch submits a search request
Request: POST /search with JSON SearchRequest
Result: 200 on success with JSON SearchRequestResponse
400 on invalid JSON
*/
func (api *WebapiInstance) apiSearch(w http.ResponseWriter, r *http.Request) {
var input SearchRequest
if err := DecodeJSON(w, r, &input); err != nil {
return
}
if input.Timeout <= 0 {
input.Timeout = 20
}
if input.MaxResults <= 0 {
input.MaxResults = 200
}
// Terminate previous searches, if their IDs were supplied. This allows terminating the old search immediately without making a separate /search/terminate request.
for _, terminate := range input.TerminateID {
if job := api.JobLookup(terminate); job != nil {
job.Terminate()
api.RemoveJob(job)
}
}
job := api.DispatchSearch(input)
EncodeJSON(api.Backend, w, r, SearchRequestResponse{Status: 0, ID: job.ID})
}
/*
apiSearchResult returns results. The default limit is 100.
If reset is set, all results will be filtered and sorted according to the settings. This means that the new first result will be returned again and internal result offset is set to 0.
Request: GET /search/result?ID=[UUID]&limit=[max records]
Optional parameters:
&reset=[0|1] to reset the filters or sort orders with any of the below parameters (all required):
&filetype=[File Type]
&fileformat=[File Format]
&from=[Date From]&to=[Date To]
&sizemin=[Minimum File size]
&sizemax=[Maximum File size]
&sort=[sort order]
&offset=[absolute offset] with &limit=[records] to get items pagination style. Returned items (and ones before) are automatically frozen.
Result: 200 with JSON structure SearchResult. Check the field Status.
*/
func (api *WebapiInstance) apiSearchResult(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
jobID, err := uuid.Parse(r.Form.Get("ID"))
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return
}
limit, err := strconv.Atoi(r.Form.Get("limit"))
if err != nil {
limit = 100
}
offset, errOffset := strconv.Atoi(r.Form.Get("offset"))
// find the job ID
job := api.JobLookup(jobID)
if job == nil {
EncodeJSON(api.Backend, w, r, SearchResult{Status: 2})
return
}
// filters and sort parameter
if filterReset, _ := strconv.ParseBool(r.Form.Get("reset")); filterReset {
fileType, _ := strconv.Atoi(r.Form.Get("filetype"))
fileFormat, _ := strconv.Atoi(r.Form.Get("fileformat"))
dateFrom := r.Form.Get("from")
dateTo := r.Form.Get("to")
sort, _ := strconv.Atoi(r.Form.Get("sort"))
sizeMin, _ := strconv.Atoi(r.Form.Get("sizemin"))
sizeMax, _ := strconv.Atoi(r.Form.Get("sizemax"))
filter := inputToSearchFilter(sort, fileType, fileFormat, dateFrom, dateTo, sizeMin, sizeMax)
job.RuntimeFilter(filter)
}
// query all results
var resultFiles []*ApiFile
if errOffset == nil {
resultFiles = job.ReturnResult(offset, limit)
} else {
resultFiles = job.ReturnNext(limit)
}
var result SearchResult
result.Files = []ApiFile{}
// loop over results
for n := range resultFiles {
result.Files = append(result.Files, *resultFiles[n])
}
// set the Status
if len(result.Files) > 0 {
if job.IsSearchResults() {
result.Status = 0 // 0 = Success with results
} else {
result.Status = 1 // No more results to expect
}
} else {
switch job.Status {
case SearchStatusLive:
result.Status = 3 // No results yet available keep trying
case SearchStatusTerminated:
result.Status = 1 // No more results to expect
default: // SearchStatusNoIndex, SearchStatusNotStarted
result.Status = 1 // No more results to expect
}
}
// embedded statistics?
if returnStats, _ := strconv.ParseBool(r.Form.Get("stats")); returnStats {
result.Statistic = job.Statistics()
}
EncodeJSON(api.Backend, w, r, result)
}
/*
apiSearchResultStream provides a websocket to receive results as stream.
Request: GET /search/result/ws?ID=[UUID]&limit=[optional max records]
Result: If successful, upgrades to a websocket and sends JSON structure SearchResult messages.
Limit is optional. Not used if ommitted or 0.
*/
func (api *WebapiInstance) apiSearchResultStream(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
jobID, err := uuid.Parse(r.Form.Get("ID"))
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return
}
limit, err := strconv.Atoi(r.Form.Get("limit"))
useLimit := err == nil
// look up the job
job := api.JobLookup(jobID)
if job == nil {
EncodeJSON(api.Backend, w, r, SearchResult{Status: 2})
return
}
// upgrade to websocket
conn, err := WSUpgrader.Upgrade(w, r, nil)
if err != nil {
// gorilla will automatically respond with "400 Bad Request", no other response is therefore necessary
return
}
defer conn.Close()
// loop to get new results and send out via the web socket.
// Only exit if limit is reached if used, otherwise only if there are no result or the connection breaks.
for {
// query all results
var resultFiles []*ApiFile
queryCount := 1
if useLimit {
queryCount = limit
}
resultFiles = job.ReturnNext(queryCount)
if useLimit {
limit -= len(resultFiles)
}
// loop over results
var result SearchResult
result.Files = []ApiFile{}
for n := range resultFiles {
result.Files = append(result.Files, *resultFiles[n])
}
if !job.IsSearchResults() {
result.Status = 1 // No more results to expect
if len(result.Files) == 0 {
conn.WriteJSON(result) // final message
return
}
}
// if no results, stall
if len(result.Files) == 0 {
time.Sleep(time.Millisecond * 100)
continue
}
// send out the results via the websocket
if err := conn.WriteJSON(result); err != nil {
return
}
// Check whether to continue. If the limit is used break once all done.
if (useLimit && limit <= 0) || result.Status == 1 {
break
}
}
}
/*
apiSearchTerminate terminates a search
Request: GET /search/terminate?ID=[UUID]
Response: 204 Empty
400 Invalid input
404 ID not found
*/
func (api *WebapiInstance) apiSearchTerminate(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
jobID, err := uuid.Parse(r.Form.Get("ID"))
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return
}
// look up the job
job := api.JobLookup(jobID)
if job == nil {
http.Error(w, "", http.StatusNotFound)
return
}
// terminate and remove it from the list
job.Terminate()
api.RemoveJob(job)
w.WriteHeader(http.StatusNoContent)
}
/*
apiSearchStatistic returns search result statistics. Statistics are always calculated over all results, regardless of any applied runtime filters.
Request: GET /search/result?ID=[UUID]
Result: 200 with JSON structure SearchStatistic. Check the field Status (0 = Success, 2 = ID not found).
*/
func (api *WebapiInstance) apiSearchStatistic(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
jobID, err := uuid.Parse(r.Form.Get("ID"))
if err != nil {
http.Error(w, "", http.StatusBadRequest)
return
}
// find the job ID
job := api.JobLookup(jobID)
if job == nil {
EncodeJSON(api.Backend, w, r, SearchStatistic{Status: 2})
return
}
stats := job.Statistics()
EncodeJSON(api.Backend, w, r, SearchStatistic{SearchStatisticData: stats, Status: 0, IsTerminated: job.IsTerminated()})
}
/*
apiExplore returns recently shared files in Peernet. Results are returned in real-time. The File type is an optional filter. See TypeX.
Special type -2 = Binary, Compressed, Container, Executable. This special type includes everything except Documents, Video, Audio, Ebooks, Picture, Text.
Request: GET /explore?limit=[max records]&type=[File type]&offset=[offset]
Result: 200 with JSON structure SearchResult. Check the field Status.
*/
func (api *WebapiInstance) apiExplore(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
offset, _ := strconv.Atoi(r.Form.Get("offset"))
limit, err := strconv.Atoi(r.Form.Get("limit"))
if err != nil {
limit = 100
}
fileType, err := strconv.Atoi(r.Form.Get("type"))
if err != nil {
fileType = -1
}
resultFiles := api.queryRecentShared(api.Backend, fileType, uint64(limit*20/100), uint64(offset), uint64(limit))
var result SearchResult
result.Files = []ApiFile{}
// loop over results
for n := range resultFiles {
result.Files = append(result.Files, blockRecordFileToAPI(resultFiles[n]))
}
if len(result.Files) == 0 {
result.Status = 3 // No results yet available keep trying
}
result.Status = 1 // No more results to expect
EncodeJSON(api.Backend, w, r, result)
}
func (input *SearchRequest) Parse() (Timeout time.Duration) {
if input.Timeout == 0 {
Timeout = time.Second * 20 // default timeout: 20 seconds
} else {
Timeout = time.Duration(input.Timeout) * time.Second
}
return
}
// ToSearchFilter converts the user input to a valid search filter
func (input *SearchRequest) ToSearchFilter() (output SearchFilter) {
return inputToSearchFilter(input.Sort, input.FileType, input.FileFormat, input.DateFrom, input.DateTo, input.SizeMin, input.SizeMax)
}
func inputToSearchFilter(Sort, FileType, FileFormat int, DateFrom, DateTo string, SizeMin, SizeMax int) (output SearchFilter) {
output.Sort = Sort
output.FileType = FileType
output.FileFormat = FileFormat
output.SizeMin = SizeMin
output.SizeMax = SizeMax
dateFrom, errFrom := time.Parse(apiDateFormat, DateFrom)
dateTo, errTo := time.Parse(apiDateFormat, DateTo)
if errFrom == nil && errTo == nil && dateFrom.Before(dateTo) {
output.DateFrom = dateFrom
output.DateTo = dateTo
output.IsDates = true
}
return
}

89
webapi/Shared Recent.go
View File

@@ -1,89 +0,0 @@
/*
File Name: Shared Recent.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"fmt"
"github.com/PeernetOfficial/core"
"github.com/PeernetOfficial/core/blockchain"
)
// queryRecentShared returns recently shared files on the network from random peers until the limit is reached.
func (api *WebapiInstance) queryRecentShared(backend *core.Backend, fileType int, limitPeer, offsetTotal, limitTotal uint64) (files []blockchain.BlockRecordFile) {
if limitPeer == 0 {
limitPeer = 1
}
// Use the Peer list to know about active peers. Random order!
peerList := api.Backend.PeerlistGet()
// Files from peers exceeding the limit. It is used if from all peers the total limit is not reached.
var filesSeconday []blockchain.BlockRecordFile
for _, peer := range peerList {
if peer.BlockchainHeight == 0 {
continue
}
var filesFromPeer uint64
// decode blocks from top down
blockLoop:
for blockN := peer.BlockchainHeight - 1; blockN > 0; blockN-- {
blockDecoded, _, found, _ := backend.ReadBlock(peer.PublicKey, peer.BlockchainVersion, blockN)
if !found {
continue
}
for _, record := range blockDecoded.RecordsDecoded {
if file, ok := record.(blockchain.BlockRecordFile); ok && isFileTypeMatchBlock(&file, fileType) {
// add the tags 'Shared By Count' and 'Shared By GeoIP'
file.Tags = append(file.Tags, blockchain.TagFromNumber(blockchain.TagSharedByCount, 1))
if latitude, longitude, valid := api.Peer2GeoIP(peer); valid {
sharedByGeoIP := fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude)
file.Tags = append(file.Tags, blockchain.TagFromText(blockchain.TagSharedByGeoIP, sharedByGeoIP))
}
// found a new File! append.
if filesFromPeer < limitPeer {
filesFromPeer++
if offsetTotal > 0 {
offsetTotal--
continue
}
files = append(files, file)
if uint64(len(files)) >= limitTotal {
return
}
} else if uint64(len(filesSeconday)) < limitTotal-uint64(len(files)) {
filesSeconday = append(filesSeconday, file)
} else {
break blockLoop
}
}
}
}
}
files = append(files, filesSeconday...)
return
}
// isFileTypeMatchBlock checks if the File type matches. -1 = accept any. -2 = core.TypeBinary, core.TypeCompressed, core.TypeContainer, core.TypeExecutable.
func isFileTypeMatchBlock(file *blockchain.BlockRecordFile, fileType int) bool {
if fileType == -1 {
return true
} else if fileType == -2 {
return file.Type == core.TypeBinary || file.Type == core.TypeCompressed || file.Type == core.TypeContainer || file.Type == core.TypeExecutable
}
return file.Type == uint8(fileType)
}

126
webapi/Status.go
View File

@@ -1,126 +0,0 @@
/*
File Name: Status.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"encoding/hex"
"fmt"
"net/http"
"strconv"
)
func apiTest(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}
type apiResponseStatus struct {
Status int `json:"Status"` // Status code: 0 = Ok.
IsConnected bool `json:"isconnected"` // Whether connected to Peernet.
CountPeerList int `json:"countpeerlist"` // Count of peers in the Peer list. Note that this contains peers that are considered inactive, but have not yet been removed from the list.
CountNetwork int `json:"countnetwork"` // Count of total peers in the network.
// This is usually a higher number than CountPeerList, which just represents the current number of connected peers.
// The CountNetwork number is going to be queried from root peers which may or may not have a limited view.
}
/*
apiStatus returns the current connectivity Status to the network
Request: GET /Status
Result: 200 with JSON structure Status
*/
func (api *WebapiInstance) apiStatus(w http.ResponseWriter, r *http.Request) {
status := apiResponseStatus{Status: 0, CountPeerList: api.Backend.PeerlistCount()}
status.CountNetwork = status.CountPeerList // For now always same as CountPeerList, until native Statistics message to root peers is available.
// Connected: If at leat 2 peers.
// This metric needs to be improved in the future, as root peers never disconnect.
// Instead, the core should keep a count of "active peers".
status.IsConnected = status.CountPeerList >= 2
EncodeJSON(api.Backend, w, r, status)
}
type apiResponsePeerSelf struct {
PeerID string `json:"peerid"` // Peer ID. This is derived from the public in compressed form.
NodeID string `json:"nodeid"` // Node ID. This is the blake3 Hash of the Peer ID and used in the DHT.
}
/*
apiAccountInfo provides information about the current account.
Request: GET /account/info
Result: 200 with JSON structure apiResponsePeerSelf
*/
func (api *WebapiInstance) apiAccountInfo(w http.ResponseWriter, r *http.Request) {
response := apiResponsePeerSelf{}
response.NodeID = hex.EncodeToString(api.Backend.SelfNodeID())
_, publicKey := api.Backend.ExportPrivateKey()
response.PeerID = hex.EncodeToString(publicKey.SerializeCompressed())
EncodeJSON(api.Backend, w, r, response)
}
/*
apiAccountDelete deletes the current account. The confirm parameter must include the user's choice.
Request: GET /account/delete?confirm=[0 or 1]
Result: 204 if the user choses not to delete the account
200 if successfully deleted
*/
func (api *WebapiInstance) apiAccountDelete(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
if confirm, _ := strconv.ParseBool(r.Form.Get("confirm")); !confirm {
w.WriteHeader(http.StatusNoContent)
return
}
api.Backend.DeleteAccount()
w.WriteHeader(http.StatusOK)
}
/*
apiStatusPeers returns the information about peers currently connected.
The GeoIP information may not alawys be available, for example if the GeoIP File is not available or the mapping from IP address to location is not available.
Peers that are connected only via local network will not have a geo location.
Request: GET /Status/peers
Result: 200 with JSON array apiResponsePeerInfo
*/
func (api *WebapiInstance) apiStatusPeers(w http.ResponseWriter, r *http.Request) {
var peers []apiResponsePeerInfo
// query all nodes
for _, peer := range api.Backend.PeerlistGet() {
peerInfo := apiResponsePeerInfo{
PeerID: peer.PublicKey.SerializeCompressed(),
NodeID: peer.NodeID,
UserAgent: peer.UserAgent,
IsRoot: peer.IsRootPeer,
BlockchainHeight: peer.BlockchainHeight,
BlockchainVersion: peer.BlockchainVersion,
}
if latitude, longitude, valid := api.Peer2GeoIP(peer); valid {
peerInfo.GeoIP = fmt.Sprintf("%.4f", latitude) + "," + fmt.Sprintf("%.4f", longitude)
}
peers = append(peers, peerInfo)
}
EncodeJSON(api.Backend, w, r, peers)
}
type apiResponsePeerInfo struct {
PeerID []byte `json:"peerid"` // Peer ID. This is derived from the public in compressed form.
NodeID []byte `json:"nodeid"` // Node ID. This is the blake3 Hash of the Peer ID and used in the DHT.
GeoIP string `json:"geoip"` // GeoIP location as "Latitude,Longitude" CSV format. Empty if location not available.
UserAgent string `json:"useragent"` // User Agent.
IsRoot bool `json:"isroot"` // If the Peer is a root Peer.
BlockchainHeight uint64 `json:"blockchainheight"` // Blockchain height
BlockchainVersion uint64 `json:"blockchainversion"` // Blockchain version
}

151
webapi/Warehouse.go
View File

@@ -1,151 +0,0 @@
/*
File Name: Warehouse.go
Copyright: 2021 Peernet Foundation s.r.o.
Author: Peter Kleissner
*/
package webapi
import (
"net/http"
"strconv"
"github.com/PeernetOfficial/core/warehouse"
)
// WarehouseResult is the response to creating a new File in the warehouse
type WarehouseResult struct {
Status int `json:"Status"` // See warehouse.StatusX.
Hash []byte `json:"Hash"` // Hash of the File.
}
/*
apiWarehouseCreateFile creates a File in the warehouse.
Request: POST /warehouse/create with raw data to create as new File
Response: 200 with JSON structure WarehouseResult
*/
func (api *WebapiInstance) apiWarehouseCreateFile(w http.ResponseWriter, r *http.Request) {
hash, status, err := api.Backend.UserWarehouse.CreateFile(r.Body, 0)
if err != nil {
api.Backend.LogError("warehouse.CreateFile", "Status %d error: %v", status, err)
}
EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash})
}
/*
apiWarehouseCreateFilePath creates a File in the warehouse by copying it from an existing File.
Warning: An attacker could supply any local File using this function, put them into storage and read them! No input path verification or limitation is done.
In the future the API should be secured using a random API key and setting the CORS header prohibiting regular browsers to access the API.
Request: GET /warehouse/create/path?path=[target path on disk]
Response: 200 with JSON structure WarehouseResult
*/
func (api *WebapiInstance) apiWarehouseCreateFilePath(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
filePath := r.Form.Get("path")
if filePath == "" {
http.Error(w, "", http.StatusBadRequest)
return
}
hash, status, err := api.Backend.UserWarehouse.CreateFileFromPath(filePath)
if err != nil {
api.Backend.LogError("warehouse.CreateFile", "Status %d error: %v", status, err)
}
EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash})
}
/*
apiWarehouseReadFile reads a File in the warehouse.
Request: GET /warehouse/read?Hash=[Hash]
Optional parameters &offset=[File offset]&limit=[read limit in bytes]
Response: 200 with the raw File data
404 if File was not found
500 in case of internal error opening the File
*/
func (api *WebapiInstance) apiWarehouseReadFile(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
hash, valid1 := DecodeBlake3Hash(r.Form.Get("Hash"))
if !valid1 {
http.Error(w, "", http.StatusBadRequest)
return
}
offset, _ := strconv.Atoi(r.Form.Get("offset"))
limit, _ := strconv.Atoi(r.Form.Get("limit"))
status, bytesRead, err := api.Backend.UserWarehouse.ReadFile(hash, int64(offset), int64(limit), w)
switch status {
case warehouse.StatusFileNotFound:
w.WriteHeader(http.StatusNotFound)
return
case warehouse.StatusInvalidHash, warehouse.StatusErrorOpenFile, warehouse.StatusErrorSeekFile:
w.WriteHeader(http.StatusInternalServerError)
return
// Cannot catch warehouse.StatusErrorReadFile since data may have been already returned.
// In the future a special header indicating the expected File length could be sent (would require a callback in ReadFile), although the caller should already know the File size based on metadata.
}
if err != nil {
api.Backend.LogError("warehouse.ReadFile", "Status %d read %d error: %v", status, bytesRead, err)
}
}
/*
apiWarehouseDeleteFile deletes a File in the warehouse.
Request: GET /warehouse/delete?Hash=[Hash]
Response: 200 with JSON structure WarehouseResult
*/
func (api *WebapiInstance) apiWarehouseDeleteFile(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
hash, valid1 := DecodeBlake3Hash(r.Form.Get("Hash"))
if !valid1 {
http.Error(w, "", http.StatusBadRequest)
return
}
status, err := api.Backend.UserWarehouse.DeleteFile(hash)
if err != nil {
api.Backend.LogError("warehouse.DeleteFile", "Status %d error: %v", status, err)
}
EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash})
}
/*
apiWarehouseReadFilePath reads a File from the warehouse and stores it to the target File. It fails with StatusErrorTargetExists if the target File already exists.
The path must include the full directory and File name.
Request: GET /warehouse/read/path?Hash=[Hash]&path=[target path on disk]
Optional parameters &offset=[File offset]&limit=[read limit in bytes]
Response: 200 with JSON structure WarehouseResult
*/
func (api *WebapiInstance) apiWarehouseReadFilePath(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
hash, valid1 := DecodeBlake3Hash(r.Form.Get("Hash"))
if !valid1 {
http.Error(w, "", http.StatusBadRequest)
return
}
targetFile := r.Form.Get("path")
offset, _ := strconv.Atoi(r.Form.Get("offset"))
limit, _ := strconv.Atoi(r.Form.Get("limit"))
status, bytesRead, err := api.Backend.UserWarehouse.ReadFileToDisk(hash, int64(offset), int64(limit), targetFile)
if err != nil {
api.Backend.LogError("warehouse.ReadFileToDisk", "Status %d read %d error: %v", status, bytesRead, err)
}
EncodeJSON(api.Backend, w, r, WarehouseResult{Status: status, Hash: hash})
}

1088
webapi/readme.md
View File

File diff suppressed because it is too large Load Diff