Peernet-Archive/WebGatewayUpload
1
0
Fork 0
mirror of https://github.com/PeernetOfficial/WebGatewayUpload.git synced 2026-07-16 19:47:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

removed usecase for backend port

Browse Source
This commit is contained in:
Akilan Selvacoumar
2022-11-22 00:02:02 +00:00
parent 0992098809
commit 200f3ea675
1 changed files with 341 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

547
main.go
View File

@@ -1,42 +1,44 @@
package main package main
import ( import (
"bytes" "bytes"
"encoding/hex" "encoding/hex"
"encoding/json" "errors"
"errors" "flag"
"flag" "fmt"
"fmt" "github.com/PeernetOfficial/core"
"github.com/PeernetOfficial/core" "github.com/PeernetOfficial/core/blockchain"
"github.com/PeernetOfficial/core/btcec" "github.com/PeernetOfficial/core/btcec"
"github.com/PeernetOfficial/core/webapi" "github.com/PeernetOfficial/core/merkle"
"github.com/gin-gonic/gin" "github.com/PeernetOfficial/core/protocol"
"github.com/google/uuid" "github.com/PeernetOfficial/core/warehouse"
limiter "github.com/julianshen/gin-limiter" "github.com/PeernetOfficial/core/webapi"
"io" "github.com/gin-gonic/gin"
"io/ioutil" "github.com/google/uuid"
"mime/multipart" limiter "github.com/julianshen/gin-limiter"
"net/http" "io"
"time" "mime/multipart"
"net/http"
"time"
) )
// Variables for the flags to get the address // Variables for the flags to get the address
var ( var (
// BackEndApiAddress Refers to the Peernet address ex: <address>:<port no> // BackEndApiAddress Refers to the Peernet address ex: <address>:<port no>
BackEndApiAddress *string //BackEndApiAddress *string
// WebpageAddress Refers to the address for upload webgate server // Port Refers to the address for upload webgate server
WebpageAddress *string Port *string
// SSL To ensure SSL is required checks if SSL is required and subsequently // SSL To ensure SSL is required checks if SSL is required and subsequently
// checks if the certificate is provided // checks if the certificate is provided
SSL *bool SSL *bool
// Certificate SSL Certificate file // Certificate SSL Certificate file
Certificate *string Certificate *string
// Key SSL Key file // Key SSL Key file
Key *string Key *string
// BackendAddressWithHTTP ex: http://<address>:<port no> // BackendAddressWithHTTP ex: http://<address>:<port no>
BackendAddressWithHTTP string //BackendAddressWithHTTP string
// Production mode // Production mode
Production *bool Production *bool
) )
// ------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------
@@ -45,33 +47,31 @@ var (
// init reading flags before any part of the code is executed // init reading flags before any part of the code is executed
func init() { func init() {
BackEndApiAddress = flag.String("BackEndApiAddress", "localhost:8088", "current environment") //BackEndApiAddress = flag.String("BackEndApiAddress", "localhost:8088", "current environment")
WebpageAddress = flag.String("WebpageAddress", "localhost:8098", "current environment") Port = flag.String("Port", "8098", "current environment")
SSL = flag.Bool("SSL", false, "Flag to check if the SSL certificate is enabled or not") SSL = flag.Bool("SSL", false, "Flag to check if the SSL certificate is enabled or not")
Certificate = flag.String("Certificate", "server.crt", "SSL Certificate file") Certificate = flag.String("Certificate", "server.crt", "SSL Certificate file")
Key = flag.String("Key", "server.key", "SSL Key file") Key = flag.String("Key", "server.key", "SSL Key file")
Production = flag.Bool("Production", false, "Flag to check if required to run on production mode") Production = flag.Bool("Production", false, "Flag to check if required to run on production mode")
} }
// InitPeernet Initializes Peernet backend // InitPeernet Initializes Peernet backend
func InitPeernet() *core.Backend { func InitPeernet() *core.Backend {
backend, status, err := core.Init("Peernet Upload Application/1.0", "Config.yaml", nil, nil) backend, status, err := core.Init("Peernet Upload Application/1.0", "Config.yaml", nil, nil)
if status != core.ExitSuccess { if status != core.ExitSuccess {
fmt.Printf("Error %d initializing backend: %s\n", status, err.Error()) fmt.Printf("Error %d initializing backend: %s\n", status, err.Error())
return nil return nil
} }
return backend return backend
} }
// RunPeernet Starts the WebAPI and peernet // RunPeernet Starts the WebAPI and peernet
func RunPeernet(backend *core.Backend) { func RunPeernet(backend *core.Backend) *webapi.WebapiInstance {
webapi.Start(backend, []string{*BackEndApiAddress}, false, "", "", 10*time.Second, 10*time.Second, uuid.Nil) api := webapi.Start(backend, []string{}, false, "", "", 10*time.Second, 10*time.Second, uuid.Nil)
backend.Connect() backend.Connect()
for { return api
}
} }
// ------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------
@@ -84,8 +84,8 @@ func RunPeernet(backend *core.Backend) {
// ---------------------------------- Warehouse related structs --------------------------- // ---------------------------------- Warehouse related structs ---------------------------
type WarehouseResult struct { type WarehouseResult struct {
Status int `json:"status"` Status int `json:"status"`
Hash []byte `json:"hash"` Hash []byte `json:"hash"`
} }
// ----------------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------------
@@ -93,20 +93,38 @@ type WarehouseResult struct {
// BlockchainRequest blockchain backend API request struct // BlockchainRequest blockchain backend API request struct
type BlockchainRequest struct { type BlockchainRequest struct {
Files []File `json:"files"` Files []File `json:"files"`
} }
type File struct { type File struct {
Hash []byte `json:"hash"` ID uuid.UUID `json:"id"` // Unique ID.
Type uint16 `json:"type"` Hash []byte `json:"hash"` // Blake3 hash of the file data
Name string `json:"name"` Type uint8 `json:"type"` // File Type. For example audio or document. See TypeX.
Format uint16 `json:"format"` // File Format. This is more granular, for example PDF or Word file. See FormatX.
Size uint64 `json:"size"` // Size of the file
Folder string `json:"folder"` // Folder, optional
Name string `json:"name"` // Name of the file
Description string `json:"description"` // Description. This is expected to be multiline and contain hashtags!
Date time.Time `json:"date"` // Date shared
NodeID []byte `json:"nodeid"` // Node ID, owner of the file. Read only.
Metadata []apiFileMetadata `json:"metadata"` // Additional metadata.
}
type apiFileMetadata struct {
Type uint16 `json:"type"` // See core.TagX constants.
Name string `json:"name"` // User friendly name of the metadata type. Use the Type fields to identify the metadata as this name may change.
// Depending on the exact type, one of the below fields is used for proper encoding:
Text string `json:"text"` // Text value. UTF-8 encoding.
Blob []byte `json:"blob"` // Binary data
Date time.Time `json:"date"` // Date
Number uint64 `json:"number"` // Number
} }
// BlockchainResponse blockchain backend API response struct // BlockchainResponse blockchain backend API response struct
type BlockchainResponse struct { type BlockchainResponse struct {
Status int `json:"status"` Status int `json:"status"`
Height int `json:"height"` Height int `json:"height"`
Version int `json:"version"` Version int `json:"version"`
} }
// ----------------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------------
@@ -122,100 +140,211 @@ type BlockchainResponse struct {
// ---------------------------------- Warehouse related ------------------------------------ // ---------------------------------- Warehouse related ------------------------------------
// AddFileWarehouse API call for (Storing file in the warehouse) // AddFileWarehouse API call for (Storing file in the warehouse)
func AddFileWarehouse(file io.Reader) *WarehouseResult { func AddFileWarehouse(file io.Reader, backend *core.Backend) (*WarehouseResult, error) {
url := BackendAddressWithHTTP + "/warehouse/create"
req, err := http.NewRequest("POST", url, file) var warehouse WarehouseResult
//req.Header.Set("X-Custom-Header", "myvalue") buf := new(bytes.Buffer)
//req.Header.Set("Content-Type", "application/json") buf.ReadFrom(file)
client := &http.Client{} hash, status, err := backend.UserWarehouse.CreateFile(file, uint64(buf.Len()))
resp, err := client.Do(req) if err != nil {
if err != nil { return nil, err
panic(err) }
} warehouse.Hash = hash
defer resp.Body.Close() warehouse.Status = status
body, err := ioutil.ReadAll(resp.Body) return &warehouse, nil
if err != nil {
fmt.Println(err)
}
var result WarehouseResult
err = json.Unmarshal(body, &result)
if err != nil {
fmt.Println(err)
}
return &result //url := BackendAddressWithHTTP + "/warehouse/create"
//
//req, err := http.NewRequest("POST", url, file)
////req.Header.Set("X-Custom-Header", "myvalue")
////req.Header.Set("Content-Type", "application/json")
//
//client := &http.Client{}
//resp, err := client.Do(req)
//if err != nil {
// panic(err)
//}
//defer resp.Body.Close()
//
//body, err := ioutil.ReadAll(resp.Body)
//if err != nil {
// fmt.Println(err)
//}
//var result WarehouseResult
//err = json.Unmarshal(body, &result)
//if err != nil {
// fmt.Println(err)
//}
//
//return &result
} }
// UploadFile Simple abstracted function to add files to peernet core // UploadFile Simple abstracted function to add files to peernet core
func UploadFile(backend *core.Backend, file *multipart.File, header *multipart.FileHeader) (*btcec.PublicKey, *WarehouseResult, error) { func UploadFile(backend *core.Backend, file *multipart.File, header *multipart.FileHeader) (*btcec.PublicKey, *WarehouseResult, error) {
buf := bytes.NewBuffer(nil) buf := bytes.NewBuffer(nil)
if _, err := io.Copy(buf, *file); err != nil { if _, err := io.Copy(buf, *file); err != nil {
return nil, nil, errors.New("io.Copy not successful") return nil, nil, errors.New("io.Copy not successful")
} }
// adds file to warehouse // adds file to warehouse
warehouseResult := AddFileWarehouse(buf) warehouseResult, err := AddFileWarehouse(buf, backend)
// current using default port for Peernet api which is 8080 if err != nil {
// First add file to warehouse return nil, nil, err
}
// current using default port for Peernet api which is 8080
// First add file to warehouse
// Adds the file to a blockchain // Adds the file to a blockchain
Blockchainfo := AddFileToBlockchain(warehouseResult.Hash, header.Filename) Blockchainfo := AddFileToBlockchain(warehouseResult.Hash, header.Filename, backend)
if Blockchainfo == nil { if Blockchainfo == nil {
return nil, nil, errors.New("add file to blockchain not successful") return nil, nil, errors.New("add file to blockchain not successful")
} }
_, publicKey := backend.ExportPrivateKey() _, publicKey := backend.ExportPrivateKey()
return publicKey, warehouseResult, nil return publicKey, warehouseResult, nil
} }
// ----------------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------------
// ----------------------------------- Blockchain related --------------------------------- // ----------------------------------- Blockchain related ---------------------------------
// AddFileToBlockchain The follwoing function adds the filename and hash to the blockchain func blockRecordFileFromAPI(input File) (output blockchain.BlockRecordFile) {
func AddFileToBlockchain(hash []byte, filename string) *BlockchainResponse { output = blockchain.BlockRecordFile{ID: input.ID, Hash: input.Hash, Type: input.Type, Format: input.Format, Size: input.Size}
url := BackendAddressWithHTTP + "/blockchain/file/add"
// Get file type if input.Name != "" {
detectType, _, err := webapi.FileDetectType(filename) output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagName, input.Name))
if err != nil { }
panic(err) if input.Folder != "" {
} output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagFolder, input.Folder))
}
if input.Description != "" {
output.Tags = append(output.Tags, blockchain.TagFromText(blockchain.TagDescription, input.Description))
}
// Create file object for post for _, meta := range input.Metadata {
var blockchainRequest BlockchainRequest if blockchain.IsTagVirtual(meta.Type) { // Virtual tags are not mapped back. They are read-only.
var files File continue
files.Name = filename }
files.Hash = hash
files.Type = detectType
blockchainRequest.Files = append(blockchainRequest.Files, files)
Byte, err := json.Marshal(blockchainRequest) switch meta.Type {
case blockchain.TagName, blockchain.TagFolder, blockchain.TagDescription: // auto mapped tags
// convert bytes case blockchain.TagDateCreated:
req, err := http.NewRequest("POST", url, bytes.NewBuffer(Byte)) output.Tags = append(output.Tags, blockchain.TagFromDate(meta.Type, meta.Date))
//req.Header.Set("X-Custom-Header", "myvalue")
req.Header.Set("Content-Type", "application/json")
client := &http.Client{} default:
resp, err := client.Do(req) output.Tags = append(output.Tags, blockchain.BlockRecordFileTag{Type: meta.Type, Data: meta.Blob})
if err != nil { }
panic(err) }
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body) return output
}
var result BlockchainResponse // setFileMerkleInfo sets the merkle fields in the BlockRecordFile
err = json.Unmarshal(body, &result) func setFileMerkleInfo(backend *core.Backend, file *blockchain.BlockRecordFile) (valid bool) {
if err != nil { if file.Size <= merkle.MinimumFragmentSize {
return nil // If smaller or equal than the minimum fragment size, the merkle tree is not used.
} file.MerkleRootHash = file.Hash
file.FragmentSize = merkle.MinimumFragmentSize
} else {
// Get the information from the Warehouse .merkle companion file.
tree, status, _ := backend.UserWarehouse.ReadMerkleTree(file.Hash, true)
if status != warehouse.StatusOK {
return false
}
return &result file.MerkleRootHash = tree.RootHash
file.FragmentSize = tree.FragmentSize
}
return true
}
// AddFileToBlockchain The following function adds the filename and hash to the blockchain
func AddFileToBlockchain(hash []byte, filename string, backend *core.Backend) *BlockchainResponse {
// Get file type
detectType, _, err := webapi.FileDetectType(filename)
if err != nil {
panic(err)
}
// Create file object for post
var blockchainRequest BlockchainRequest
var files File
files.Name = filename
files.Hash = hash
files.Type = uint8(detectType)
blockchainRequest.Files = append(blockchainRequest.Files, files)
var filesAdd []blockchain.BlockRecordFile
for _, file := range blockchainRequest.Files {
if len(file.Hash) != protocol.HashSize {
//http.Error(w, "", http.StatusBadRequest)
//return
}
if file.ID == uuid.Nil { // if the ID is not provided by the caller, set it
file.ID = uuid.New()
}
// Verify that the file exists in the warehouse. Folders are exempt from this check as they are only virtual.
//if !file.IsVirtualFolder() {
// if _, err := warehouse.ValidateHash(file.Hash); err != nil {
// //http.Error(w, "", http.StatusBadRequest)
// //return
// } else if _, fileSize, status, _ := backend.UserWarehouse.FileExists(file.Hash); status != warehouse.StatusOK {
// //EncodeJSON(api.backend, w, r, apiBlockchainBlockStatus{Status: blockchain.StatusNotInWarehouse})
// //return
// } else {
// file.Size = fileSize
// }
//} else {
// file.Hash = protocol.HashData(nil)
// file.Size = 0
//}
blockRecord := blockRecordFileFromAPI(file)
// Set the merkle tree info as appropriate.
setFileMerkleInfo(backend, &blockRecord)
filesAdd = append(filesAdd, blockRecord)
}
newHeight, newVersion, status := backend.UserBlockchain.AddFiles(filesAdd)
//Byte, err := json.Marshal(blockchainRequest)
//
//// convert bytes
//req, err := http.NewRequest("POST", url, bytes.NewBuffer(Byte))
////req.Header.Set("X-Custom-Header", "myvalue")
//req.Header.Set("Content-Type", "application/json")
//
//client := &http.Client{}
//resp, err := client.Do(req)
//if err != nil {
// panic(err)
//}
//defer resp.Body.Close()
//
//body, _ := ioutil.ReadAll(resp.Body)
//
//var result BlockchainResponse
//err = json.Unmarshal(body, &result)
//if err != nil {
// return nil
//}
var result BlockchainResponse
result.Status = status
fmt.Println(status)
result.Version = int(newVersion)
result.Height = int(newHeight)
return &result
} }
// ----------------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------------
@@ -226,99 +355,103 @@ func AddFileToBlockchain(hash []byte, filename string) *BlockchainResponse {
// ------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------
func main() { func main() {
// Parsing flags // Parsing flags
flag.Parse() flag.Parse()
// Start peernet // Start peernet
backend := InitPeernet() backend := InitPeernet()
go RunPeernet(backend) RunPeernet(backend)
var r *gin.Engine var r *gin.Engine
if *Production { if *Production {
gin.SetMode(gin.ReleaseMode) gin.SetMode(gin.ReleaseMode)
r = gin.New() r = gin.New()
} else { } else {
r = gin.Default() r = gin.Default()
} }
r.LoadHTMLGlob("templates/*.html") // Set of trusted proxies which can be IPV4 or IPV6
r.Static("/templates", "./templates") r.SetTrustedProxies([]string{""})
// --------------------------------- Middleware rate limiter ----------------------------------- r.LoadHTMLGlob("templates/*.html")
lm := limiter.NewRateLimiter(time.Minute, 10, func(ctx *gin.Context) (string, error) { r.Static("/templates", "./templates")
return "", nil
})
// ---------------------------------------------------------------------------------------------
// ---------------------------------------- Routes --------------------------------------------- // --------------------------------- Middleware rate limiter -----------------------------------
// GET /upload to open upload page from webgateway lm := limiter.NewRateLimiter(time.Minute, 10, func(ctx *gin.Context) (string, error) {
r.GET("/upload", lm.Middleware(), func(c *gin.Context) { return "", nil
c.HTML(http.StatusOK, "upload.html", nil) })
}) // ---------------------------------------------------------------------------------------------
// POST /uploadFile Uploads file to peernet from Webgateway // ---------------------------------------- Routes ---------------------------------------------
r.POST("/upload", lm.Middleware(), func(c *gin.Context) { // GET /upload to open upload page from webgateway
file, header, err := c.Request.FormFile("file") r.GET("/upload", lm.Middleware(), func(c *gin.Context) {
defer file.Close() c.HTML(http.StatusOK, "upload.html", nil)
})
if err != nil { // POST /uploadFile Uploads file to peernet from Webgateway
c.HTML(http.StatusBadRequest, "upload.html", gin.H{ r.POST("/upload", lm.Middleware(), func(c *gin.Context) {
"error": "File not added during upload", file, header, err := c.Request.FormFile("file")
}) defer file.Close()
return
}
publicKey, warehouseResult, err := UploadFile(backend, &file, header) if err != nil {
if err != nil { c.HTML(http.StatusBadRequest, "upload.html", gin.H{
c.HTML(http.StatusBadRequest, "upload.html", gin.H{ "error": "File not added during upload",
"error": "File not added during upload", })
}) return
return }
//fmt.Println(err)
}
c.HTML(http.StatusOK, "upload.html", gin.H{ publicKey, warehouseResult, err := UploadFile(backend, &file, header)
"hash": hex.EncodeToString(warehouseResult.Hash), if err != nil {
"filename": header.Filename, c.HTML(http.StatusBadRequest, "upload.html", gin.H{
"size": header.Size, "error": "File not added during upload",
"link": "https://peer.ae/" + hex.EncodeToString(publicKey.SerializeCompressed()) + "/" + hex.EncodeToString(warehouseResult.Hash), })
"address": *WebpageAddress, return
}) //fmt.Println(err)
}
}) c.HTML(http.StatusOK, "upload.html", gin.H{
"hash": hex.EncodeToString(warehouseResult.Hash),
"filename": header.Filename,
"size": header.Size,
"link": "https://peer.ae/" + hex.EncodeToString(publicKey.SerializeCompressed()) + "/" + hex.EncodeToString(warehouseResult.Hash),
"address": *Port,
})
// Implement CURL script to ensure linux users can upload directly })
// the Cli like https://bashupload.com
// Ex: curl http://localhost:8080/uploadCurl -F add=@<file name>
r.POST("/uploadCurl", lm.Middleware(), func(c *gin.Context) {
file, header, err := c.Request.FormFile("add")
defer file.Close()
if err != nil { // Implement CURL script to ensure linux users can upload directly
fmt.Println(err) // the Cli like https://bashupload.com
} // Ex: curl http://localhost:8080/uploadCurl -F add=@<file name>
r.POST("/uploadCurl", lm.Middleware(), func(c *gin.Context) {
file, header, err := c.Request.FormFile("add")
defer file.Close()
publicKey, warehouseResult, err := UploadFile(backend, &file, header) if err != nil {
if err != nil { fmt.Println(err)
fmt.Println(err) }
}
link := "https://peer.ae/" + hex.EncodeToString(publicKey.SerializeCompressed()) + "/" + hex.EncodeToString(warehouseResult.Hash) publicKey, warehouseResult, err := UploadFile(backend, &file, header)
c.Data(http.StatusOK, "plain/text", []byte(link)) if err != nil {
}) fmt.Println(err)
}
// --------------------------------------------------------------------------------------------- link := "https://peer.ae/" + hex.EncodeToString(publicKey.SerializeCompressed()) + "/" + hex.EncodeToString(warehouseResult.Hash)
c.Data(http.StatusOK, "plain/text", []byte(link))
})
// ---------------------------------- Start Gin server ----------------------------------------- // ---------------------------------------------------------------------------------------------
// check if SSL is used or not
if *SSL { // ---------------------------------- Start Gin server -----------------------------------------
BackendAddressWithHTTP = "https://" + *BackEndApiAddress // check if SSL is used or not
r.RunTLS(*WebpageAddress, *Certificate, *Key) if *SSL {
*WebpageAddress = "https://" + *WebpageAddress //BackendAddressWithHTTP = "https://" + *BackEndApiAddress
} else { r.RunTLS(":"+*Port, *Certificate, *Key)
BackendAddressWithHTTP = "http://" + *BackEndApiAddress //*Port = "https://" + *Port
r.Run(*WebpageAddress) } else {
*WebpageAddress = "http://" + *WebpageAddress //BackendAddressWithHTTP = "http://" + *BackEndApiAddress
} //r.Run(*Port)
// --------------------------------------------------------------------------------------------- r.Run(":" + *Port)
//*Port = "http://" + *Port
}
// ---------------------------------------------------------------------------------------------
} }