diff --git a/.DS_Store b/.DS_Store index 6a4aff2..441d6f5 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/Bindings/docs.md b/Bindings/docs.md new file mode 100644 index 0000000..8c87483 --- /dev/null +++ b/Bindings/docs.md @@ -0,0 +1,113 @@ + + +# Bindings + +```go +import "github.com/Akilan1999/p2p-rendering-computation/Bindings" +``` + +## Index + +- [func ConvertStructToJSONString\(Struct interface\{\}\) \*C.char](<#ConvertStructToJSONString>) +- [func EscapeFirewall\(HostOutsideNATIP string, HostOutsideNATPort string, internalPort string\) \(output \*C.char\)](<#EscapeFirewall>) +- [func GetSpecs\(IP string\) \(output \*C.char\)](<#GetSpecs>) +- [func Init\(customConfig string\) \(output \*C.char\)](<#Init>) +- [func MapPort\(Port string, DomainName string, ServerAddress string\) \*C.char](<#MapPort>) +- [func RemoveContainer\(IP string, ID string\) \(output \*C.char\)](<#RemoveContainer>) +- [func Server\(\) \(output \*C.char\)](<#Server>) +- [func StartContainer\(IP string\) \(output \*C.char\)](<#StartContainer>) +- [func UpdateIPTable\(\) \(output \*C.char\)](<#UpdateIPTable>) +- [func ViewIPTable\(\) \(output \*C.char\)](<#ViewIPTable>) + + + +## func [ConvertStructToJSONString]() + +```go +func ConvertStructToJSONString(Struct interface{}) *C.char +``` + + + + +## func [EscapeFirewall]() + +```go +func EscapeFirewall(HostOutsideNATIP string, HostOutsideNATPort string, internalPort string) (output *C.char) +``` + + + + +## func [GetSpecs]() + +```go +func GetSpecs(IP string) (output *C.char) +``` + + + + +## func [Init]() + +```go +func Init(customConfig string) (output *C.char) +``` + + + + +## func [MapPort]() + +```go +func MapPort(Port string, DomainName string, ServerAddress string) *C.char +``` + + + + +## func [RemoveContainer]() + +```go +func RemoveContainer(IP string, ID string) (output *C.char) +``` + + + + +## func [Server]() + +```go +func Server() (output *C.char) +``` + + + + +## func [StartContainer]() + +```go +func StartContainer(IP string) (output *C.char) +``` + + + + +## func [UpdateIPTable]() + +```go +func UpdateIPTable() (output *C.char) +``` + + + + +## func [ViewIPTable]() + +```go +func ViewIPTable() (output *C.char) +``` + + + +Generated by [gomarkdoc]() diff --git a/CONTRIBUTE.md.org b/CONTRIBUTE.md.org new file mode 100644 index 0000000..c87ecca --- /dev/null +++ b/CONTRIBUTE.md.org @@ -0,0 +1,93 @@ +* Contributing +:PROPERTIES: +:CUSTOM_ID: contributing +:END: +When contributing to this repository, please first discuss the change +you wish to make via issue, email, or any other method with the owners +of this repository before making a change. + +Please note we have a code of conduct, please follow it in all your +interactions with the project. + +** Pull Request Process +:PROPERTIES: +:CUSTOM_ID: pull-request-process +:END: +1. Ensure any install or build dependencies are removed before the end + of the layer when doing a build. +2. Update the appropriate docs before stating the changes you did. +3. You may merge the Pull Request in once you have the approval of + CODEOWNER, or if you do not have permission to do that, you may + request the second reviewer to merge it for you. + +** Code of conduct +:PROPERTIES: +:CUSTOM_ID: code-of-conduct +:END: +All projects(including mailing lists) at HWTechClub is required to +follow [[https://lunduke.com/pages/codeofconduct/][The Lunduke Code of +Conduct]]. + +*** The Lunduke Code of Conduct +:PROPERTIES: +:CUSTOM_ID: the-lunduke-code-of-conduct +:END: + +#+begin_quote +**** "Be Excellent to Each Other." +:PROPERTIES: +:CUSTOM_ID: be-excellent-to-each-other. +:END: +#+end_quote + +***** FAQ +:PROPERTIES: +:CUSTOM_ID: faq +:END: +*Q*: Wait. That's it? + +A: Yup. + +*Q*: But... but... it's only 5 words! + +A: It is? Cool! + +*Q*: Isn't that too short to be a viable Code of Conduct? + +A: Nah. + +*Q*: It's too vague! How would you enforce this?!? + +A: By being excellent to each other. + +*Q*: What if someone violates this Code of Conduct? + +A: I trust you to figure it out. + +*Q*: What if someone talks about politics I disagree with?!?!!? + +A: I know a guy who votes for Papa Smurf in every Presidential election. +He's a pretty cool dude. + +*Q*: I'm super angry about something! And a person likes the thing +making me angry! What do I do? + +A: I dunno. Try /hugs/? + +*Q*: Are virtual hugs (such as /"hugs"/ and animated .GIFs of teddy +bears hugging) excellent? + +A: Most excellent. + +*Q*: Huh. Seems so simple! + +A: I know, right? + +*Q*: I like you! + +A: I like you, too! + +/The Lunduke Code of Conduct is free for you to use however you like./ + +*Source*: [[https://lunduke.com/pages/codeofconduct/][The Lunduke Code +of Conduct]] diff --git a/Docs/.gitignore b/Docs/.gitignore new file mode 100644 index 0000000..e70658e --- /dev/null +++ b/Docs/.gitignore @@ -0,0 +1,73 @@ +# Don't bother tracking a bunch of stuff when building and installing +# Org from the master git repository. + +# ...by ignoring everything created by 'make', 'make doc', `make info' +# `make html_manual', `make release' + +*.aux +*.bak +*.cp +*.cps +*.diff +*.dvi +*.elc +*.fn +*.fns +*.html +*.info +*.ky +*.kys +*.log +*.patch +*.pdf +*.pg +*.pgs +*.ps +*.toc +*.tex +*.tp +*.vr +*.vrs +orgcard_letter.tex +orgcard.txt +org +org-loaddefs.el +org-version.el +doc/org-version.inc +org-*.tar* +orgplus-*.tar* +org-*.zip +version.mk +manual +org_dual_license.texi +ORGWEBPAGE/Changes.txt +local*.mk +.gitattributes +mk/x11idle + +# texi2pdf --tidy + +doc/*.t2d + +# aspell word and replacement lists +.aspell.org.pws +.aspell.org.prepl + +# allow tmp and test directories that will not be tracked + +test +t +auto +tmp +TODO + +# and collateral damage from Emacs + +*~ +.DS_Store +*# +.#* + +# +# Local variables: +# End: diff --git a/Docs/Abstractions.md b/Docs/DocsDeprecated/Abstractions.md similarity index 100% rename from Docs/Abstractions.md rename to Docs/DocsDeprecated/Abstractions.md diff --git a/Docs/DocsDeprecated/Abstractions.org b/Docs/DocsDeprecated/Abstractions.org new file mode 100644 index 0000000..d0d8e6f --- /dev/null +++ b/Docs/DocsDeprecated/Abstractions.org @@ -0,0 +1,35 @@ +* Abstractions +:PROPERTIES: +:CUSTOM_ID: abstractions +:END: +| [[file:Installation.md][◀ Previous]] | [[file:Implementation.md][Next ▶]] | +|--------------------------------------+------------------------------------| + +The Abstractions package consists of black-boxed functions for P2PRC. + +** Functions +:PROPERTIES: +:CUSTOM_ID: functions +:END: +- =Init()=: Initializes P2PRC with all the needed + configurations. +- =Start()=: Starts p2prc as a server and makes it possible to extend by + adding other routes and functionality to P2PRC. +- =MapPort()=: On the local machine the port you want to export + to world. +- =StartContainer()=: The machine on the p2p network where + you want to spin up a docker container. +- =RemoveContainer(,)=: Terminate container + based on the IP address and container name. +- =GetSpecs()=: Get specs of a machine on the network based + on the IP address. +- =ViewIPTable()=: View the IP table which about nodes in the network. +- =UpdateIPTable()=: Force update IP table to learn about new nodes + faster. + +-------------- + +*** Next Chapter: [[file:Implementation.org][Implementation]] +:PROPERTIES: +:CUSTOM_ID: next-chapter-implementation +:END: diff --git a/Docs/Bindings.md b/Docs/DocsDeprecated/Bindings.md similarity index 100% rename from Docs/Bindings.md rename to Docs/DocsDeprecated/Bindings.md diff --git a/Docs/DocsDeprecated/Bindings.org b/Docs/DocsDeprecated/Bindings.org new file mode 100644 index 0000000..fa77c64 --- /dev/null +++ b/Docs/DocsDeprecated/Bindings.org @@ -0,0 +1,180 @@ +* Language Bindings +:PROPERTIES: +:CUSTOM_ID: language-bindings +:END: +[[https://en.wikipedia.org/wiki/Language_binding][Language bindings]] +refers to wrappers to bridge 2 programming languages. This is used in +P2PRC to extend calling P2PRC functions in other programming languages. +Currently this is done by generating =.so= and =.h= from the Go +compiler. + +** How to build shared object files +:PROPERTIES: +:CUSTOM_ID: how-to-build-shared-object-files +:END: +**** The easier way +:PROPERTIES: +:CUSTOM_ID: the-easier-way +:END: +#+begin_src sh +# Run +make sharedObjects +#+end_src + +**** Or the direct way +:PROPERTIES: +:CUSTOM_ID: or-the-direct-way +:END: +#+begin_src sh +# Run +cd Bindings && go build -buildmode=c-shared -o p2prc.so +#+end_src + +**** If successfully built: +:PROPERTIES: +:CUSTOM_ID: if-successfully-built +:END: +#+begin_src sh +# Enter into the Bindings directory +cd Bindings +# List files +ls +# Find files +p2prc.h p2prc.so +#+end_src + +** Workings under the hood +:PROPERTIES: +:CUSTOM_ID: workings-under-the-hood +:END: +Below are a sample set of commands to open the bindings implementation. + +#+begin_example +# run +cd Bindings/ +# list files +ls +# search for file +Client.go +#+end_example + +*** In Client go +:PROPERTIES: +:CUSTOM_ID: in-client-go +:END: +There a few things to notice which are different from your standard Go +programs: + +**** 1. We import "C" which means [[https://pkg.go.dev/cmd/cgo][Cgo]] is required. +:PROPERTIES: +:CUSTOM_ID: we-import-c-which-means-cgo-is-required. +:END: +#+begin_src go +import "C" +#+end_src + +**** 2. All functions which are required to be called from other programming languages have comment such as. +:PROPERTIES: +:CUSTOM_ID: all-functions-which-are-required-to-be-called-from-other-programming-languages-have-comment-such-as. +:END: +#+begin_src go +//export + +// ------------ Example ---------------- +// The function below allows to externally +// to call the P2PRC function to start containers +// in a specific node in the know list of nodes +// in the p2p network. +// Note: the comment "//export StartContainer". + +//export StartContainer +func StartContainer(IP string) (output *C.char) { + container, err := client.StartContainer(IP, 0, false, "", "") + if err != nil { + return C.CString(err.Error()) + } + return ConvertStructToJSONString(container) + } +#+end_src + +**** 3. While looking through the file (If 2 files are compared it is pretty trivial to notice a common structure). +:PROPERTIES: +:CUSTOM_ID: while-looking-through-the-file-if-2-files-are-compared-it-is-pretty-trivial-to-notice-a-common-structure. +:END: +#+begin_src go +// --------- Example ------------ + +//export StartContainer +func StartContainer(IP string) (output *C.char) { + container, err := client.StartContainer(IP, 0, false, "", "") + if err != nil { + return C.CString(err.Error()) + } + return ConvertStructToJSONString(container) +} + +//export ViewPlugin +func ViewPlugin() (output *C.char) { + plugins, err := plugin.DetectPlugins() + if err != nil { + return C.CString(err.Error()) + } + return ConvertStructToJSONString(plugins) +} +#+end_src + +**** It is easy to notice that: +:PROPERTIES: +:CUSTOM_ID: it-is-easy-to-notice-that +:END: +- =ConvertStructToJSONString()=: This is a helper function + that convert a go object to JSON string initially and converts it to + =CString=. +- =(output *C.char)=: This is the return type for most of the functions. + +**** A Pseudo code to refer to the common function implementation shape could be represented as: +:PROPERTIES: +:CUSTOM_ID: a-pseudo-code-to-refer-to-the-common-function-implementation-shape-could-be-represented-as +:END: +#+begin_example +func (output *C.char) { + , := () + if != nil { + return C.CString(.Error()) + } + return ConvertStructToJSONString() +} +#+end_example + +** Current languages supported +:PROPERTIES: +:CUSTOM_ID: current-languages-supported +:END: +- Python + +*** Build sample python program +:PROPERTIES: +:CUSTOM_ID: build-sample-python-program +:END: +The easier way + +#+begin_src sh +# Run +make python +# Expected ouput +Output is in the Directory Bindings/python/export/ +# Run +cd Bindings/python/export/ +# list files +ls +# Expected output +SharedObjects/ p2prc.py +#+end_src + +Above shows a generated folder which consists of a folder called +"SharedObjects/" which consists of =p2prc.so= and =p2prc.h= files. +=p2prc.py= refers to a sample python script calling P2PRC go functions. +To start an any project to extend P2PRC with python, This generated +folder can copied and created as a new git repo for P2PRC extensions +scripted or used a reference point as proof of concept that P2PRC can be +called from other programming languages. diff --git a/Docs/CliImplementation.md b/Docs/DocsDeprecated/CliImplementation.md similarity index 100% rename from Docs/CliImplementation.md rename to Docs/DocsDeprecated/CliImplementation.md diff --git a/Docs/DocsDeprecated/CliImplementation.org b/Docs/DocsDeprecated/CliImplementation.org new file mode 100644 index 0000000..c9940c4 --- /dev/null +++ b/Docs/DocsDeprecated/CliImplementation.org @@ -0,0 +1,24 @@ +* Cli module +:PROPERTIES: +:CUSTOM_ID: cli-module +:END: +The Cli (i.e Command Line Interface) is the only one in which the user +can directly interact with the modules in the project. The objective +when building the Cli was to have the least amount of commands as +possible. The cli was built using the library called urfave cli v2 . +They were 2 major files created named as flags.go and actions.go. ### +Flags.go The flags .go file is responsible to create the appropriate +flags for the cli. There are 2 types of flags called boolean and string. +Each of the flags outputs are assigned to a variable to be handled. The +flags can also detect environment variables set. This feature is useful +because if the user wants to call certain flags in a repeated sequence +it only has to be initialized once. + +*** Actions.go +:PROPERTIES: +:CUSTOM_ID: actions.go +:END: +The actions.go file is implemented to call the appropriate functions +when the flags are called. It interacts directly with the modules in the +project. Action.go checks if variables are not empty string or the +boolean value is true. diff --git a/Docs/Client.md b/Docs/DocsDeprecated/Client.md similarity index 100% rename from Docs/Client.md rename to Docs/DocsDeprecated/Client.md diff --git a/Docs/DocsDeprecated/Client.org b/Docs/DocsDeprecated/Client.org new file mode 100644 index 0000000..0142853 --- /dev/null +++ b/Docs/DocsDeprecated/Client.org @@ -0,0 +1,33 @@ +* Client Module +:PROPERTIES: +:CUSTOM_ID: client-module +:END: +This module is incharge of communicating with the server and receiving +the appropriate information back from the server. + +** Functions of the Client Module +:PROPERTIES: +:CUSTOM_ID: functions-of-the-client-module +:END: + +#+begin_html + +#+end_html + +- [[#decision-maker-on-how-the-ip-table-is-created-or-updated][Decision + maker on how the ip table is created or updated]] + +** Decision maker on how the IP table is created or updated +:PROPERTIES: +:CUSTOM_ID: decision-maker-on-how-the-ip-table-is-created-or-updated +:END: +- Does a local speedtest to verify and see if the server IP's in the IP + table are pingable. +- Tries to ping the servers IP Table addresses. +- If it's pingable then it's added as a new entry in the IP table. +- The following steps occurs in the clients IP table. +- To ensure that the same servers are not being called to update the IP + table. There is a temporary list of IP address which have already been + called in relation to updating the IP table. +- Based on the current implementation there will 3 hops done to update + the IP table. diff --git a/Docs/ClientArchitecture.md b/Docs/DocsDeprecated/ClientArchitecture.md similarity index 100% rename from Docs/ClientArchitecture.md rename to Docs/DocsDeprecated/ClientArchitecture.md diff --git a/Docs/DocsDeprecated/ClientArchitecture.org b/Docs/DocsDeprecated/ClientArchitecture.org new file mode 100644 index 0000000..daf1ca6 --- /dev/null +++ b/Docs/DocsDeprecated/ClientArchitecture.org @@ -0,0 +1,15 @@ +* Client Module Architecture +:PROPERTIES: +:CUSTOM_ID: client-module-architecture +:END: +The Client Module interacts with the P2P module and Server Module. It is +responsible for interacting with the server module and appropriately +updating the IP table on the client side. It connects to the server +using the server's REST Apis. It is also the primary decision maker on +how the IP table is updated is on the client side. This is because each +user can have requirements like how many number of hops they would want +to do to update their IP table. Hops is the number of times the client +is going to download the IP table from different servers ,once it gets +the IP tables from the previous servers. + +[[file:images/NumOfHops.png]] [[file:images/clientmoduleArch.png]] diff --git a/Docs/ClientImplementation.md b/Docs/DocsDeprecated/ClientImplementation.md similarity index 100% rename from Docs/ClientImplementation.md rename to Docs/DocsDeprecated/ClientImplementation.md diff --git a/Docs/DocsDeprecated/ClientImplementation.org b/Docs/DocsDeprecated/ClientImplementation.org new file mode 100644 index 0000000..0300525 --- /dev/null +++ b/Docs/DocsDeprecated/ClientImplementation.org @@ -0,0 +1,124 @@ +* Client Module Implementation +:PROPERTIES: +:CUSTOM_ID: client-module-implementation +:END: +The Client Module interacts with the P2P module and Server Module. It is +responsible for interacting with the server module and appropriately +updating the IP table on the client side. It connects to the server +using the server's REST Apis. It is also the primary decision maker on +how the IP table is updated is on the client side. This is because each +user can have requirements like how many number of hops they would want +to do to update their IP table. Hops is the number of times the client +is going to download the IP table from different servers ,once it gets +the IP tables from the previous servers. + +[[file:images/NumOfHops.png]] [[file:images/clientmoduleArch.png]] + +This section focuses in depth on how the client module works. The client +module is incharge of communicating with different servers based on the +IP addresses provided to the user. The IP addresses are derived from +peer to peer modules. The objective here is how the client module +interacts with peer to peer module and server module. + +*** Updating the IP table +:PROPERTIES: +:CUSTOM_ID: updating-the-ip-table +:END: +The client module calls the peer to peer module to get the local IP +table initially, Based on the servers IP addresses available it calls +the speedtest function from the peer to peer module to update IP +addresses with information such as latencies, download and upload +speeds. Once this is done the client module does a Rest Api call to the +server to download its IP Table. Once the hops are done it writes the +appropriate results to the Local IP table. Once this is done it prints +out the results. To derive parameters such as current the public IP +address the url "http://ip-api.com/json/" was called. This url returns +json response of the current public IP address. This feature will be +used in the future to ensure that the user's current IP address will not +be used for a speed test. Clients IP table is updated to the server +using a form of type multipart. + +*** Reading server specifications +:PROPERTIES: +:CUSTOM_ID: reading-server-specifications +:END: +The client module calls the route /server_specs and reads the json +response. If the json response was successful then it just calls the +pretty print function which just prints the json output in the terminal. + +*** Client creating and removing container +:PROPERTIES: +:CUSTOM_ID: client-creating-and-removing-container +:END: +The client module uses the servers Rest apis to create and delete +containers. To create a container the client requires 3 parameters being +the server ip address, the number of the ports the user wants to open +and if the user wants it connected to the GPU or not. The 3 parameters +are sent as a GET request to the server and the server responds with a +json file which has information such as the container ID, ports open , +SSH username, SSH password, VNC username and VNC password. At the moment +the username and password are hard coded from the server side for both +SSH and VNC. To remove a container the client module only requires the +server IP address and the container ID. The client prints the response +from the server Rest api. + +*** Tracking Containers +:PROPERTIES: +:CUSTOM_ID: tracking-containers +:END: +Clients create docker images in multiple machines. This means if the +client (i.e user) has many containers created there needs to be a way to +track them. To track containers there is a file called +=trackcontainers.json= which tracks all the containers running. The +snippet below show a sample structure of file =trackcontainer.json=. + +#+begin_example +{ + "TrackContainer": [ + { + "ID": "", + "Container": {}, + "IpAddress": "" + } + ] +} +#+end_example + +The default path to the container tracker is +=client/trackcontainers/trackcontainers.json=. + +*** Grouping Containers +:PROPERTIES: +:CUSTOM_ID: grouping-containers +:END: +When starting a set container possibility to be able to group them. The +benefit this would be that when executing plugins the group ID would be +enough to execute plugin in a set of containers. This provides the +possibility to execute repetitive tasks in containers in a single cli +command. To store groups there is a file called +=grouptrackcontainer.json= which tracks all the groups currently present +set by the client. The snippet below show a sample structure of file +=grouptrackcontainer.json=. + +#+begin_example +{ + "Groups": [ + { + "ID": "grp", + "TrackContainer": [{client.TrackContainers struct}] + } + ] +} +#+end_example + +The default path to the container tracker is +=client/trackcontainers/grouptrackcontainer.json=. + +#+begin_quote +[!NOTE] The group id will be auto-generated and will have its own prefix +in the start which will mostly be =grp=.\\ +When a container is removed using the command. +=p2prc --rm --id =. It will be automatically +deleted from the groups it exists in. + +#+end_quote diff --git a/Docs/ConfigImplementation.md b/Docs/DocsDeprecated/ConfigImplementation.md similarity index 100% rename from Docs/ConfigImplementation.md rename to Docs/DocsDeprecated/ConfigImplementation.md diff --git a/Docs/DocsDeprecated/ConfigImplementation.org b/Docs/DocsDeprecated/ConfigImplementation.org new file mode 100644 index 0000000..960093d --- /dev/null +++ b/Docs/DocsDeprecated/ConfigImplementation.org @@ -0,0 +1,35 @@ +* Config Implementation +:PROPERTIES: +:CUSTOM_ID: config-implementation +:END: +The configuration module is responsible to store basic information of +absolute paths of files being called in the Go code. In a full-fledged +Cli the configuration file can be found in the directory /etc/ and from +there points to location such as where the IP table file is located. In +the future implementation the config file will have information such as +number of hops and other parameters to tweak and to improve the +effectiveness of the peer to peer network. The configuration module was +implemented using the library Viper. The Viper library automates +features such as searching in default paths to find out if the +configuration file is present. If the configuration file is not present +in the default paths then it auto generates the configuration file. The +configurations file can be in any format. In this project the +configuration file was generated using JSON format. + +#+begin_src json +{ + "MachineName": "pc-74-120.customer.ask4.lan", + "IPTable": "/Users/akilan/Documents/p2p-rendering-computation/p2p/iptable/ip_table.json", + "DockerContainers": "/Users/akilan/Documents/p2p-rendering-computation/server/docker/containers/", + "DefaultDockerFile": "/Users/akilan/Documents/p2p-rendering-computation/server/docker/containers/docker-ubuntu-sshd/", + "SpeedTestFile": "/Users/akilan/Documents/p2p-rendering-computation/p2p/50.bin", + "IPV6Address": "", + "PluginPath": "/Users/akilan/Documents/p2p-rendering-computation/plugin/deploy", + "TrackContainersPath": "/Users/akilan/Documents/p2p-rendering-computation/client/trackcontainers/trackcontainers.json", + "ServerPort": "8088", + "GroupTrackContainersPath": "/Users/akilan/Documents/p2p-rendering-computation/client/trackcontainers/grouptrackcontainers.json", + "FRPServerPort": "True", + "BehindNAT": "True", + "CustomConfig": null +} +#+end_src diff --git a/Docs/DesignArchtectureIntro.md b/Docs/DocsDeprecated/DesignArchtectureIntro.md similarity index 100% rename from Docs/DesignArchtectureIntro.md rename to Docs/DocsDeprecated/DesignArchtectureIntro.md diff --git a/Docs/DocsDeprecated/DesignArchtectureIntro.org b/Docs/DocsDeprecated/DesignArchtectureIntro.org new file mode 100644 index 0000000..013d61b --- /dev/null +++ b/Docs/DocsDeprecated/DesignArchtectureIntro.org @@ -0,0 +1,26 @@ +* Design Architecture +:PROPERTIES: +:CUSTOM_ID: design-architecture +:END: +This chapter focuses on architecture of the dissertation. The objective +would be to have a good understanding on the purpose of each module and +how they interact with each other. The design architecture was inspired +and based on the linux kernel design. The project is segmented into +various modules. Each module is responsible for certain tasks in the +project. The modules are highly dependent on each other hence the entire +codebase can be considered as a huge monolithic chuck which acts as its +own library. The following sub topics below talk about the main modules +and how they function with appropriate diagrams. + +*** 1. [[file:ClientArchitecture.md][Client Module]] +:PROPERTIES: +:CUSTOM_ID: client-module +:END: +*** 2. [[file:P2PArchitecture.md][P2P Module]] +:PROPERTIES: +:CUSTOM_ID: p2p-module +:END: +*** 3. [[file:ServerArchitecture.md][Server Module]] +:PROPERTIES: +:CUSTOM_ID: server-module +:END: diff --git a/Docs/DomainNameMappingsImplementation.md b/Docs/DocsDeprecated/DomainNameMappingsImplementation.md similarity index 100% rename from Docs/DomainNameMappingsImplementation.md rename to Docs/DocsDeprecated/DomainNameMappingsImplementation.md diff --git a/Docs/DocsDeprecated/DomainNameMappingsImplementation.org b/Docs/DocsDeprecated/DomainNameMappingsImplementation.org new file mode 100644 index 0000000..69f504a --- /dev/null +++ b/Docs/DocsDeprecated/DomainNameMappingsImplementation.org @@ -0,0 +1,6 @@ +* Domain name mappings +:PROPERTIES: +:CUSTOM_ID: domain-name-mappings +:END: + +Todo be written. diff --git a/Docs/Implementation.md b/Docs/DocsDeprecated/Implementation.md similarity index 100% rename from Docs/Implementation.md rename to Docs/DocsDeprecated/Implementation.md diff --git a/Docs/DocsDeprecated/Implementation.org b/Docs/DocsDeprecated/Implementation.org new file mode 100644 index 0000000..5c10ab2 --- /dev/null +++ b/Docs/DocsDeprecated/Implementation.org @@ -0,0 +1,21 @@ +* Implementation +:PROPERTIES: +:CUSTOM_ID: implementation +:END +This chapter describes how the project was built. It talks in depth of +the implementation performed to give a better understanding of the +project. + +** Programming language used +:PROPERTIES: +:CUSTOM_ID: programming-langauge-used +:END: +The programming language used for this project was +[[https://go.dev/][Golang]]. The reason Go lang was chosen was because +it is a compiled language. The entire codebase is just a single binary +file. When distributing to other linux distributing the only requirement +would be the binary file to run the code. It is easy to write +independant modules and be monolithic at the sametime using Go. Using +Go.mod makes it very easy to handle external libraries and modularise +code. The go.mod name for the project is +[[https://git.sr.ht/~akilan1999/p2p-rendering-computation][git.sr.ht/~akilan1999/p2p-rendering-computation]]. diff --git a/Docs/Installation.md b/Docs/DocsDeprecated/Installation.md similarity index 100% rename from Docs/Installation.md rename to Docs/DocsDeprecated/Installation.md diff --git a/Docs/DocsDeprecated/Installation.org b/Docs/DocsDeprecated/Installation.org new file mode 100644 index 0000000..27adec3 --- /dev/null +++ b/Docs/DocsDeprecated/Installation.org @@ -0,0 +1,352 @@ +* Installation +:PROPERTIES: +:CUSTOM_ID: installation +:END: + +Over here we will cover the basic steps to get the server and client +side running. + +** Latest release install +:PROPERTIES: +:CUSTOM_ID: latest-release-install +:END: +https://github.com/Akilan1999/p2p-rendering-computation/releases + +** Install from Github master branch +:PROPERTIES: +:CUSTOM_ID: install-from-github-master-branch +:END: +*** Install Go lang +:PROPERTIES: +:CUSTOM_ID: install-go-lang +:END: +The entire the implementation of this project is done using Go lang. +Thus, we need go lang to compile to code to a binary file. +[[https://golang.org/doc/install][Instructions to install Go lang]] + +*** Install Docker +:PROPERTIES: +:CUSTOM_ID: install-docker +:END: +In this project the choice of virtualization is Docker due to it's wide +usage in the developer community. In the server module we use the Docker +Go API to create and interact with the containers. + +[[https://docs.docker.com/get-docker/][Instructions to install docker]] + +[[https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker][Instructions +to install docker GPU]] + +#+begin_example +// Do ensure that the docker command does not need sudo to run +sudo chmod 666 /var/run/docker.sock +#+end_example + +*** Build Project and install project +:PROPERTIES: +:CUSTOM_ID: build-project-and-install-project +:END: +To set up the internal dependencies and build the entire go code into a +single binary + +#+begin_example +make install +#+end_example + +**** For Windows +:PROPERTIES: +:CUSTOM_ID: for-windows +:END: +To set up P2PRC on Windows, simply run this batch file. *Make sure you +are not in admin mode when running this.* + +#+begin_example +.\install.bat +#+end_example + +*** Add appropriate paths to =.bashrc= +:PROPERTIES: +:CUSTOM_ID: add-appropriate-paths-to-.bashrc +:END: +#+begin_example +export P2PRC=//p2p-rendering-computation +export PATH=//p2p-rendering-computation:${PATH} +#+end_example + +*** Set up configuration file +:PROPERTIES: +:CUSTOM_ID: set-up-configuration-file +:END: +#+begin_example +make configfile +#+end_example + +Open the config file =config.json= and add the IPv6 address if you have +one. + +*** Test if binary works +:PROPERTIES: +:CUSTOM_ID: test-if-binary-works +:END: +#+begin_example +p2prc --help +#+end_example + +**** Output: +:PROPERTIES: +:CUSTOM_ID: output +:END: +#+begin_example +NAME: + p2p-rendering-computation - p2p cli application to create and access VMs in other servers + +USAGE: + p2prc [global options] command [command options] [arguments...] + +VERSION: + + +COMMANDS: + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --Server, -s Starts server (default: false) [$SERVER] + --UpdateServerList, --us Update List of Server available based on servers iptables (default: false) [$UPDATE_SERVER_LIST] + --ListServers, --ls List servers which can render tasks (default: false) [$LIST_SERVERS] + --AddServer value, --as value Adds server IP address to iptables [$ADD_SERVER] + --ViewImages value, --vi value View images available on the server IP address [$VIEW_IMAGES] + --CreateVM value, --touch value Creates Docker container on the selected server [$CREATE_VM] + --ContainerName value, --cn value Specifying the container run on the server side [$CONTAINER_NAME] + --RemoveVM value, --rm value Stop and Remove Docker container (IP:port) accompanied by container ID via --ID or --id [$REMOVE_VM] + --ID value, --id value Docker Container ID [$ID] + --Ports value, -p value Number of ports to open for the Docker Container [$NUM_PORTS] + --GPU, --gpu Create Docker Containers to access GPU (default: false) [$USE_GPU] + --Specification value, --specs value Specs of the server node [$SPECS] + --SetDefaultConfig, --dc Sets a default configuration file (default: false) [$SET_DEFAULT_CONFIG] + --NetworkInterfaces, --ni Shows the network interface in your computer (default: false) [$NETWORK_INTERFACE] + --ViewPlugins, --vp Shows plugins available to be executed (default: false) [$VIEW_PLUGIN] + --TrackedContainers, --tc View (currently running) containers which have been created from the client side (default: false) [$TRACKED_CONTAINERS] + --ExecutePlugin value, --plugin value Plugin which needs to be executed [$EXECUTE_PLUGIN] + --CreateGroup, --cgroup Creates a new group (default: false) [$CREATE_GROUP] + --Group value, --group value group flag with argument group ID [$GROUP] + --Groups, --groups View all groups (default: false) [$GROUPS] + --RemoveContainerGroup, --rmcgroup Remove specific container in the group (default: false) [$REMOVE_CONTAINER_GROUP] + --RemoveGroup value, --rmgroup value Removes the entire group [$REMOVE_GROUP] + --Generate value, --gen value Generates a new copy of P2PRC which can be modified based on your needs [$GENERATE] + --ModuleName value, --mod value New go project module name [$MODULENAME] + --PullPlugin value, --pp value Pulls plugin from git repos [$PULLPLUGIN] + --RemovePlugin value, --rp value Removes plugin [$REMOVEPLUGIN] + --help, -h show help (default: false) + --version, -v print the version (default: false) +#+end_example + +-------------- + +* Using basic commands +:PROPERTIES: +:CUSTOM_ID: using-basic-commands +:END: +*** Start as a server +:PROPERTIES: +:CUSTOM_ID: start-as-a-server +:END: +Do ensure you have Docker installed for this + +#+begin_example +p2prc -s +#+end_example + +*** View server Specification +:PROPERTIES: +:CUSTOM_ID: view-server-specification +:END: +#+begin_example +p2prc --specs= +#+end_example + +*** Run container +:PROPERTIES: +:CUSTOM_ID: run-container +:END: +use the =--gpu= if you know the other machine has a gpu. + +#+begin_example +p2prc --touch= -p --gpu +#+end_example + +*** Remove container +:PROPERTIES: +:CUSTOM_ID: remove-container +:END: +The docker id is present in the output where you create a container + +#+begin_example +p2prc --rm= --id= +#+end_example + +*** Adding servers to ip table +:PROPERTIES: +:CUSTOM_ID: adding-servers-to-ip-table +:END: +#+begin_example +p2prc --as= +#+end_example + +*** Update ip table +:PROPERTIES: +:CUSTOM_ID: update-ip-table +:END: +#+begin_example +p2prc --us +#+end_example + +*** List Servers +:PROPERTIES: +:CUSTOM_ID: list-servers +:END: +#+begin_example +p2prc --ls +#+end_example + +*** View Network interfaces +:PROPERTIES: +:CUSTOM_ID: view-network-interfaces +:END: +#+begin_example +p2prc --ni +#+end_example + +*** Viewing Containers created Client side +:PROPERTIES: +:CUSTOM_ID: viewing-containers-created-client-side +:END: +#+begin_example +p2prc --tc +#+end_example + +[[file:ClientImplementation.md#tracking-containers][read more on +tracking containers]] + +*** Running plugin +:PROPERTIES: +:CUSTOM_ID: running-plugin +:END: +#+begin_example +p2prc --plugin --id +#+end_example + +*** Create group +:PROPERTIES: +:CUSTOM_ID: create-group +:END: +#+begin_example +p2prc --cgroup +#+end_example + +*** Add container to group +:PROPERTIES: +:CUSTOM_ID: add-container-to-group +:END: +#+begin_example +p2prc --group --id +#+end_example + +*** View groups +:PROPERTIES: +:CUSTOM_ID: view-groups +:END: +#+begin_example +p2prc --groups +#+end_example + +*** View specific group +:PROPERTIES: +:CUSTOM_ID: view-specific-group +:END: +#+begin_example +p2prc --group +#+end_example + +*** Delete container from group +:PROPERTIES: +:CUSTOM_ID: delete-container-from-group +:END: +#+begin_example +p2prc --rmcgroup --group --id +#+end_example + +*** Delete entire group +:PROPERTIES: +:CUSTOM_ID: delete-entire-group +:END: +#+begin_example +p2prc --rmgroup +#+end_example + +[[file:ClientImplementation.md#Grouping-Containers][read more on +grouping containers]] ### Extending usecase of P2PRC (Requires a go +compiler to run) + +#+begin_example +p2prc --gen --mod +#+end_example + +[[file:GenerateImplementation.md][read more about the generate module]] + +*** Pulling plugin from a remote repo +:PROPERTIES: +:CUSTOM_ID: pulling-plugin-from-a-remote-repo +:END: +#+begin_example +p2prc --pp +#+end_example + +*** Deleting plugin from the plugin directory +:PROPERTIES: +:CUSTOM_ID: deleting-plugin-from-the-plugin-directory +:END: +#+begin_example +p2prc --rp +#+end_example + +*** Added custom metadata about the current node +:PROPERTIES: +:CUSTOM_ID: added-custom-metadata-about-the-current-node +:END: +#+begin_example +p2prc --amd "custom metadata" +#+end_example + +-------------- + +* Using Plugins +:PROPERTIES: +:CUSTOM_ID: using-plugins +:END: +This feature is still Under Development: +[[file:PluginImplementation.md][Read more on the implementation]] + +**** Dependencies +:PROPERTIES: +:CUSTOM_ID: dependencies +:END: +- Ansible: + - Debian/ubuntu: =sudo apt install ansible= + - Others: + [[https://ansible-tips-and-tricks.readthedocs.io/en/latest/ansible/install/][Installation + link]] + +**** Run Test Cases +:PROPERTIES: +:CUSTOM_ID: run-test-cases +:END: +- Generate Test Case Ansible file + - =make testcases= +- Enter inside plugin directory and run tests. + +#+begin_quote +[!NOTE] That docker needs to installed and needs to run without sudo. +Refer the section [[#install-docker][Install Docker]]. - =cd plugin= - +=go test .= + +#+end_quote \ No newline at end of file diff --git a/Docs/Introduction.md b/Docs/DocsDeprecated/Introduction.md similarity index 100% rename from Docs/Introduction.md rename to Docs/DocsDeprecated/Introduction.md diff --git a/Docs/DocsDeprecated/Introduction.org b/Docs/DocsDeprecated/Introduction.org new file mode 100644 index 0000000..00b46d7 --- /dev/null +++ b/Docs/DocsDeprecated/Introduction.org @@ -0,0 +1,39 @@ +* Chapter 1: Introduction +:PROPERTIES: +:CUSTOM_ID: chapter-1-introduction +:END: + +** Abstract +:PROPERTIES: +:CUSTOM_ID: abstract +:END: +This project focuses on creating a framework on running heavy tasks that +a regular computer cannot run easily such as graphically demanding video +games, rendering 3D animations , protein folding simulations. In this +project the major focus will not be on the financial incentive part. A +peer to peer network will be created to help run tasks decentrally, +increasing bandwidth for running tasks. To ensure the tasks in the peer +to peer network do not corrupt the server 0S (Operating System), they +will be executed in a virtual environment in the server. + +The main aim of this project was to create a custom peer to peer +network. The user acting as the client has total flexibility on how to +batch the tasks and the user acting as the server has complete +flexibility on tracking the container's usages and killing the +containers at any point of time. + +** Motivation +:PROPERTIES: +:CUSTOM_ID: motivation +:END: +Many of the users rely on our PC / Laptop or servers that belong to a +server farm to run heavy tasks and with the demand of high creativity +requires higher computing power. Buying a powerful computer every few +years to run a bunch of heavy tasks which are not executed as frequently +to reap the benefits can be inefficient utilization of hardware. On the +other end, renting servers to run these heavy tasks can be really +useful. Ethically speaking this is leading to monopolisation of +computing power similar to what is happening in the web server area. By +using peer to peer principles it is possible to remove the +monopolisation factor and increase the bandwidth between the client and +server. \ No newline at end of file diff --git a/Docs/NAT-Traveral.md b/Docs/DocsDeprecated/NAT-Traveral.md similarity index 100% rename from Docs/NAT-Traveral.md rename to Docs/DocsDeprecated/NAT-Traveral.md diff --git a/Docs/DocsDeprecated/NAT-Traveral.org b/Docs/DocsDeprecated/NAT-Traveral.org new file mode 100644 index 0000000..d96ea8b --- /dev/null +++ b/Docs/DocsDeprecated/NAT-Traveral.org @@ -0,0 +1,50 @@ +* NAT Traversal +:PROPERTIES: +:CUSTOM_ID: nat-traversal +:END: +P2PRC currently supports TURN for NAT traversal. + +** TURN +:PROPERTIES: +:CUSTOM_ID: turn +:END: +The current TURN implementation used is FRP. The TURN server is also +required when a P2PRC node is acting as a Server. The TURN server is +determined based on the Node with the least amount of latency based on +the Nodes available on the IPTable. Once a TURN server is determined +there are 2 actions performed. The first one is =/FRPPort= to the TURN +server to receive a port which is used to generate the external port +from the TURN server. The flow below describes the workflow. + +*** Client mode +:PROPERTIES: +:CUSTOM_ID: client-mode +:END: +- Call =/FRPPort= + +#+begin_example +http://:/FRPport +#+end_example + +- Call the TURN server in the following manner. The following is a + sample code snippet below. + +#+begin_src go +import ( + "github.com/Akilan1999/p2p-rendering-computation/p2p/frp" +) + +func main() { + serverPort, err := frp.GetFRPServerPort("http://" + + ":" + lowestLatencyIpAddress.ServerPort) + if err != nil { + return nil, err + } + // Create 1 second delay to allow FRP server to start + time.Sleep(1 * time.Second) + // Starts FRP as a client with + proxyPort, err := frp.StartFRPClientForServer(, serverPort, ) + if err != nil { + return nil, err + } +} +#+end_src diff --git a/Docs/P2P-testing.md b/Docs/DocsDeprecated/P2P-testing.md similarity index 100% rename from Docs/P2P-testing.md rename to Docs/DocsDeprecated/P2P-testing.md diff --git a/Docs/DocsDeprecated/P2P-testing.org b/Docs/DocsDeprecated/P2P-testing.org new file mode 100644 index 0000000..67c5be9 --- /dev/null +++ b/Docs/DocsDeprecated/P2P-testing.org @@ -0,0 +1,136 @@ +* Testing P2P network +:PROPERTIES: +:CUSTOM_ID: testing-p2p-network +:END: +The objective would be to test the p2p network, and the effectiveness of +updating the ip tables. The objective of would be to give the impression +to the client and server of a Zero configuration setting. For testing +there will be a test network set. In the testing scenario all will be +client and server because the IP table does not store clients IP +addresses. At current number of hopes would be 3 as default. + +*** Test Network Scenario 1 +:PROPERTIES: +:CUSTOM_ID: test-network-scenario-1 +:END: +The test network consists of 5 nodes acting as a client and server. The +objective would be to have the entire IP table Updated in each node with +interacting with only 1 node once. Each node has knowledge of one node +only. + +[[https://user-images.githubusercontent.com/31743758/115069627-e4aa8c80-9f04-11eb-8402-706a3407f0e8.png]] +Fig 1.0 Visual Representation of testnet scenario 1 + +**** Result +:PROPERTIES: +:CUSTOM_ID: result +:END: +All nodes except node 1 where able to have information of IP addresses +in the test net. This was due to the reason of 3 hops set as default. +Node 1 had in it's IP table IP addresses of Node 2, Node 3, Node 4. Once +the number of hops was set to 4 objective of the test was acheived. + +*** Test Network Scenario 2 +:PROPERTIES: +:CUSTOM_ID: test-network-scenario-2 +:END: +The second test network has a scenario of a single peer which all the +other nodes connect too. The scenario being when the other nodes connect +to the single server they download information about nodes that have +connected to the server node before. + +*** Testing Broadcast Module +:PROPERTIES: +:CUSTOM_ID: testing-broadcast-module +:END: +For testing the broadcast module 2 types of servers will be tested. One +with a CPU only , another one with a CPU and GPU. The expected result +being that the appropriate results are visible. + +**** Results (CPU and GPU): +:PROPERTIES: +:CUSTOM_ID: results-cpu-and-gpu +:END: +#+begin_example +{ + "Hostname": "akilan-Lenovo-IdeaPad-Y510P", + "Platform": "ubuntu", + "CPU": "Intel(R) Core(TM) i7-4700MQ CPU @ 2.40GHz", + "RAM": 7872, + "Disk": 937367, + "GPU": { + "DriveVersion": "390.141", + "Gpu": { + "GpuName": "GeForce GT 755M", + "BiosVersion": "80.07.A8.00.0F", + "FanSpeed": "N/A", + "Utilization": { + "GpuUsage": "N/A", + "MemoryUsage": "N/A" + }, + "Temperature": { + "GpuTemp": "66 C" + }, + "Clock": { + "GpuClock": "N/A", + "GpuMemClock": "N/A" + } + } + } +} +#+end_example + +At the moment of the current implementation v1.0. Nvidia GPU are only +compatible. As the Go code calls the command =nvidia-smi= to get +information about the GPU available. + +**** Results (CPU only) +:PROPERTIES: +:CUSTOM_ID: results-cpu-only +:END: +#+begin_example +{ + "Hostname": "sv-t1.small.x86-01", + "Platform": "ubuntu", + "CPU": "Intel(R) Atom(TM) CPU C2750 @ 2.40GHz", + "RAM": 7944, + "Disk": 138793, + "GPU": null +} +#+end_example + +As the =nvidia-smi= interface was not detected it only broadcasts the +CPU specs available. + +*** SpeedTests +:PROPERTIES: +:CUSTOM_ID: speedtests +:END: +The speed test has 3 parameters which are Ping , upload and download. +The tests check if the results returned are approximately correct. The +ping at the moment returns the correct result. The upload and download +returned are inccorect at the moment, This is due incorrect +implementation in for timer and will be patched in future versions. + +*** Unit tests +:PROPERTIES: +:CUSTOM_ID: unit-tests +:END: +All functions implemented on the P2P module returns type error. The +units test call certain functions and check if the functions return an +error or not. This proved sufficient as the point of the units tests was +code coverage to check if certain functions return an error. + +**** Functions tested +:PROPERTIES: +:CUSTOM_ID: functions-tested +:END: +This sections talks about the function called and represents code +coverage. + +1. =TestServer_SpeedTest=: Function called LocalSpeedTestIpTable() +2. =TestReadIpTable=: Function called ReadIpTable() + +The P2P module has a 100% code coverage in unit tests as both the unit +tests call directly or call within the function all the functions used +in the P2P module. diff --git a/Docs/P2P.md b/Docs/DocsDeprecated/P2P.md similarity index 100% rename from Docs/P2P.md rename to Docs/DocsDeprecated/P2P.md diff --git a/Docs/DocsDeprecated/P2P.org b/Docs/DocsDeprecated/P2P.org new file mode 100644 index 0000000..8d34d62 --- /dev/null +++ b/Docs/DocsDeprecated/P2P.org @@ -0,0 +1,45 @@ +* P2P (Peer to Peer module) +:PROPERTIES: +:CUSTOM_ID: p2p-peer-to-peer-module +:END: +In this repository the P2P module has been designed from sratch at the +point of this implementation. +[[https://pkg.go.dev/git.sr.ht/~akilan1999/p2p-rendering-computation@v0.0.0-20210404191839-6a046babcb02/p2p][More +about function implementation]] + +** Terminology +:PROPERTIES: +:CUSTOM_ID: terminology +:END: +1. IPTable: Refers to a json file which stores information about the + current servers avaliable with the speedtest results ran from the + Node that triggered it. + +#+begin_example +{ +"ip_address": [ + { + "ipv4": "localhost", + "latency": 14981051, + "download": 8142.122540206258, + "upload": 3578.766512629995, + } +] +} +#+end_example + +** Responsibility +:PROPERTIES: +:CUSTOM_ID: responsibility +:END: +- To ensure the IP table has nodes which are pingable +- Taking to nodes behind NAT. [[file:NAT-Traversal][More about the + implementation]]... + +#+begin_quote +[!NOTE] If you are running in server mode it is recommended to use +[[https://routerguide.net/when-and-how-to-setup-dmz-host-for-home-use/][DMZ]] +to bypass the +[[https://en.wikipedia.org/wiki/Network_address_translation][NAT]]. + +#+end_quote diff --git a/Docs/P2PArchitecture.md b/Docs/DocsDeprecated/P2PArchitecture.md similarity index 100% rename from Docs/P2PArchitecture.md rename to Docs/DocsDeprecated/P2PArchitecture.md diff --git a/Docs/DocsDeprecated/P2PArchitecture.org b/Docs/DocsDeprecated/P2PArchitecture.org new file mode 100644 index 0000000..c7a3ad4 --- /dev/null +++ b/Docs/DocsDeprecated/P2PArchitecture.org @@ -0,0 +1,16 @@ +* P2P Module Architecture +:PROPERTIES: +:CUSTOM_ID: p2p-module-architecture +:END: +The P2P module (i.e Peer to Peer Module) is responsible for storing the +IP table and interacting with the IP table. In the following +implementation of the P2P module ,the IP table stores information about +servers available in the network. The other functionality the P2P module +takes care of is doing the appropriate speed tests to the servers in the +IP table. This is for informing the users about nodes which are close by +and nodes which have quicker uploads and downloads speeds. The module is +responsible to ensure that there are no duplicate server IPs in the IP +table and to remove all server IPs which are not pingable. + +#+caption: UML diagram of P2P module +[[file:images/p2pmoduleArch.png]] diff --git a/Docs/P2PImplementation.md b/Docs/DocsDeprecated/P2PImplementation.md similarity index 100% rename from Docs/P2PImplementation.md rename to Docs/DocsDeprecated/P2PImplementation.md diff --git a/Docs/DocsDeprecated/P2PImplementation.org b/Docs/DocsDeprecated/P2PImplementation.org new file mode 100644 index 0000000..344fff8 --- /dev/null +++ b/Docs/DocsDeprecated/P2PImplementation.org @@ -0,0 +1,111 @@ +* P2P Module Implementation +:PROPERTIES: +:CUSTOM_ID: p2p-module-implementation +:END: +The P2P module (i.e Peer to Peer Module) is responsible for storing the +IP table and interacting with the IP table. In the following +implementation of the P2P module ,the IP table stores information about +servers available in the network. The other functionality the P2P module +takes care of is doing the appropriate speed tests to the servers in the +IP table. This is for informing the users about nodes which are close by +and nodes which have quicker uploads and downloads speeds. The module is +responsible to ensure that there are no duplicate server IPs in the IP +table and to remove all server IPs which are not pingable. + +#+caption: UML diagram of P2P module +[[file:images/p2pmoduleArch.png]] + +The peer to peer implementation was built from scratch. This is because +other peer to peer libraries were on the implementation of the +Distributed hash table. At the current moment all those heavy features +are not needed because the objective is to search and list all possible +servers available. The limitation being that to be a part of the network +the user has to know at least 1 server. The advantage of building from +scratch makes the module super light and possibility for custom +functions and structs. The sub topics below will mention the +implementations of each functionality in depth. + +** IP Table +:PROPERTIES: +:CUSTOM_ID: ip-table +:END: +The ip table file is a json as the format with a list of servers ip +addresses, latencies, downloads and uploads speeds. The functions +implemented include read file, write file and remove duplicate IP +addresses. The remove duplicate IP address function exists because +sometimes servers IP tables can have the same ip addresses as what the +client has. The path of the IP table json file is received from the +configuration module. + +#+begin_src json +{ + "ip_address": [ + { + "ipv4": "", + "latency": "", + "download": "", + "upload": "" + "port no": "", + } + ] +} +#+end_src + +*** Latency +:PROPERTIES: +:CUSTOM_ID: latency +:END: +The latency is measured in milliseconds. The route /server_info is +called from the server and time it takes to provide a json response is +recorded. + +** NAT Traversal +:PROPERTIES: +:CUSTOM_ID: nat-traversal +:END: +P2PRC currently supports TURN for NAT traversal. + +** TURN +:PROPERTIES: +:CUSTOM_ID: turn +:END: +The current TURN implementation used is FRP. The TURN server is also +required when a P2PRC node is acting as a Server. The TURN server is +determined based on the Node with the least amount of latency based on +the Nodes available on the IPTable. Once a TURN server is determined +there are 2 actions performed. The first one is =/FRPPort= to the TURN +server to receive a port which is used to generate the external port +from the TURN server. The flow below describes the workflow. + +*** Client mode +:PROPERTIES: +:CUSTOM_ID: client-mode +:END: +- Call =/FRPPort= + +#+begin_example +http://:/FRPport +#+end_example + +- Call the TURN server in the following manner. The following is a + sample code snippet below. + +#+begin_src go +import ( + "github.com/Akilan1999/p2p-rendering-computation/p2p/frp" +) + +func main() { + serverPort, err := frp.GetFRPServerPort("http://" + + ":" + lowestLatencyIpAddress.ServerPort) + if err != nil { + return nil, err + } + // Create 1 second delay to allow FRP server to start + time.Sleep(1 * time.Second) + // Starts FRP as a client with + proxyPort, err := frp.StartFRPClientForServer(, serverPort, ) + if err != nil { + return nil, err + } +} +#+end_src diff --git a/Docs/PluginImplementation.md b/Docs/DocsDeprecated/PluginImplementation.md similarity index 100% rename from Docs/PluginImplementation.md rename to Docs/DocsDeprecated/PluginImplementation.md diff --git a/Docs/DocsDeprecated/PluginImplementation.org b/Docs/DocsDeprecated/PluginImplementation.org new file mode 100644 index 0000000..fb456a0 --- /dev/null +++ b/Docs/DocsDeprecated/PluginImplementation.org @@ -0,0 +1,203 @@ +* Plugin Module Implementation +:PROPERTIES: +:CUSTOM_ID: plugin-module-implementation +:END: + +** Introduction +:PROPERTIES: +:CUSTOM_ID: introduction +:END: +The plugin module is designed to ensure clients can execute instructions +in a declarative manner across different containers created. This means +the user (i.e client) needs to write the instruction only once, and +these instructions can be executed across different nodes in a +repetitive manner. + +In the scenario of this project Ansibles will be used as the way the +users can create these instructions. + +- [[file:Installation.md#Using-Plugins][Setup instruction]] + +The plugin module introduces a new path to the config file known as +pluginpath. This path by defaults points to =${P2PRC}/plugin/deploy=. +Any file/folder inside =plugin/deploy= is part of the .gitginore. +Plugins are detected by folder names inside the =plugin/deploy=. + +#+begin_example +plugin +|___ Deploy + |___ + |___ site.yml + |___ hosts + |___ ports.json + |___ description.txt + . + . + . + n: n number of plugins possible +#+end_example + +** Site File Template +:PROPERTIES: +:CUSTOM_ID: site-file-template +:END: +The site file is also known as the Ansible playbook and is incharge of +executing instructions in a declarative manner. The below example +specifies how to make one. + +#+begin_example +- hosts: all + + tasks: + - name: + + debug: + msg: +#+end_example + +Read more about ansible tasks: +https://docs.ansible.com/ansible/latest/user_guide/playbooks_intro.html#about-playbooks + +** Hosts file +:PROPERTIES: +:CUSTOM_ID: hosts-file +:END: +hosts file is also known as the inventory file. This file consists of +all the information required to connect to other nodes to execute +Ansible instructions. In this project this file needs to be set in a +certain configuration because the go code or binary will populate this +file automatically with the appropriate information required to connect +to local or remote containers. + +#+begin_quote +[!NOTE] Add as exactly specified below: + +#+begin_example +all: + vars: + ansible_python_interpreter: /usr/bin/python3 // Path to your python 3 interpreter +main: + hosts: + host1: + // Note: These values will be automatically overwritten + // by the Go functions + ansible_host: 0.0.0.0 + ansible_port: 39269 + ansible_user: master + ansible_ssh_pass: password + ansible_sudo_pass: password +#+end_example + +#+end_quote + +** Ports.json +:PROPERTIES: +:CUSTOM_ID: ports.json +:END: +The =ports.json= file is intended to mention the number of ports +required by the plugin. + +#+begin_example +{ + "NumOfPorts": +} +#+end_example + +** Description file +:PROPERTIES: +:CUSTOM_ID: description-file +:END: +This is a simple text file used to describe what the module does. When +the client is looking at various commands via the ClI. The description +is displayed along-side the plugin name. + +Ex: When the flag =--ViewPlugins= or =--vp= is called + +#+begin_example +{ + "PluginsDetected": [ + { + "FolderName": "", + "PluginDescription": "" + } + ] +} +#+end_example + +** Automatic port allocations +:PROPERTIES: +:CUSTOM_ID: automatic-port-allocations +:END: +P2PRC would be in-charge to set to the ports to various TCP ports +opened. Due to this implementation the plugin being executed is copied +to the tmp directory with a unique UUID. + +#+begin_example +Command: ls /tmp +output: Semantic _ +2e6d76c4-0ed1-4b55-9385-79a58d4f0492_p2prc-vscode-browser +7b631e08-62ee-4c1c-a2a4-c05857b9aa7d_p2prc-vscode-browser +#+end_example + +Once the copy of the plugin is added to the /tmp directory the site.yml +file inside the appropriate yaml is modified with the appropriate ports +assigned to the container. + +*** Ex: +:PROPERTIES: +:CUSTOM_ID: ex +:END: +1. Create container called c1 with an automatic generated TCP port 3313 + (external) - 3313 (internal) +2. Assumption of plugin p1 exists. p1 has one server which needs to be + mapped to a free open TCP port in container c1. Below shows an + implementation of a sample site.yml file. + +#+begin_example +--- +- hosts: all + tasks: + - name: start vscode code server + shell: sh server.sh 0.0.0.0:{{index . 0}} +#+end_example + +Notice there is the following {{index . 0}}. {{index . 0}} does not +belong to Ansible but rather is a way to mention where to add the +external free port of the container. We use the golang +[[https://pkg.go.dev/text/template][template library]] to parse and +populate the site.yml with the appropriate open ports. An array of ints +which consists of open free ports are sent to the site.yml. 0 in +{{index . 0}} refers to the index in the int array passed on. + +After the port is automatically it's ready to run ! + +#+begin_example +--- +- hosts: all + tasks: + - name: start vscode code server + shell: sh server.sh 0.0.0.0:3313 +#+end_example + +*** Sample plugins implemented: +:PROPERTIES: +:CUSTOM_ID: sample-plugins-implemented +:END: +- [[https://github.com/Akilan1999/p2prc-vscode-browser][VSCode Plugin]] + +** Pull Plugins +:PROPERTIES: +:CUSTOM_ID: pull-plugins +:END: +The following allows us to pull plugins from a remote git repository and +store them in the default plugins directory. The implementation uses a +Go git library to pull the git repo and automatically save it as a +folder in the plugin path. + +** Delete Plugins +:PROPERTIES: +:CUSTOM_ID: delete-plugins +:END: +We delete the plugin folder based on the plugin name provided as an +argument on the cli command. Once the folder is deleted, the plugin +manager automatically knows that the plugin does not exist anymore. diff --git a/Docs/DocsDeprecated/README.md b/Docs/DocsDeprecated/README.md new file mode 100644 index 0000000..2dcacbf --- /dev/null +++ b/Docs/DocsDeprecated/README.md @@ -0,0 +1,170 @@ +> [!TIP] +> Haskell bindings supported!: [Bindings documentaton](https://p2prc.akilan.io/Docs/haskell/P2PRC.html) + +> [!NOTE] +> Fixing documentation to latest changes. If you have any questions setting up P2PRC either [create an issue](https://github.com/Akilan1999/p2p-rendering-computation/issues/new/choose) or send me an email (me AT akilan dot io). +> Currently HEAD is always intended to stay on a working state. It is recommended to always use HEAD in your go.mod file. + +

+
+ p2prc +
+

+ + + +
+ GPLv2 license + Maintenance + made-with-Go + GoDoc reference example +
+ +The main aim of this project was to create a custom peer to peer network. The user acting as the +client has total flexibility on how to batch the tasks and the user acting as the server has complete +flexibility on tracking the container's usages and killing the containers at any point of time. + +## Latest tutorial +[![IMAGE ALT TEXT](https://i.ytimg.com/vi/OMwCpedu5cs/hqdefault.jpg)](https://www.youtube.com/watch?v=OMwCpedu5cs") + +
+ +## Table of contents in the current README +1. [Introduction](#Introduction) +2. [Installation](#extend-your-application-with-p2prc) +3. [Design Architecture](#Design-Architecture) +4. [Implementation](#Implementation) +5. [Find out more](#Find-out-more) + +
+ +# Table of contents in the Docs folder +1. [Introduction](Docs/Introduction.md) +2. [Installation](Docs/Installation.md) +3. [Abstractions](Docs/Abstractions.md) + +4. [Implementation](Docs/Implementation.md) + 1. [Client Module](Docs/ClientImplementation.md) + 2. [P2P Module](Docs/P2PImplementation.md) + 3. [Server Module](Docs/ServerImplementation.md) + 4. [Config Module](Docs/ConfigImplementation.md) + 5. [Cli Module](Docs/CliImplementation.md) + 6. [Plugin Module](Docs/PluginImplementation.md) + 7. [Language bindings](Docs/Bindings.md) + 8. [Domain name mappings](Docs/Bindings.md) +5. Language bindings + 1. [Haskell](Docs/haskell/) + + +
+ +## Introduction +This project aims to create a peer to peer (p2p) network, where a user can use the p2p network to act as a client (i.e sending tasks) or the server (i.e executing the tasks). A prototype application will be developed, which comes bundled with a p2p module and possible to execute docker containers or virtual environments across selected nodes. + +### Objectives +- Background review on peer to peer network, virtual environments, decentralized rendering tools and tools to batch any sort of tasks. +- Creating p2p network +- Server to create a containerised environment +- The client node to run tasks on Server containerised node + +[Read more on the introduction](Docs/Introduction.md) + +
+ +## Extend your application with P2PRC +```go +package main + +import ( + "fmt" + "github.com/Akilan1999/p2p-rendering-computation/abstractions" +) + +func main() { + _, err := abstractions.Init(nil) + if err != nil { + fmt.Println(err) + return + } + + // start p2prc + _, err = abstractions.Start() + if err != nil { + fmt.Println(err) + return + } + + // Run server till termination + for { + + } +} + +``` + +### Export once this is added export P2PRC as environment paths +``` +export P2PRC= +export PATH=:${PATH} +``` +[Read more](Docs/Abstractions.md) ... + +## Installation from source +1. Ensure the Go compiler is installed + ``` + go version + ``` +3. Ensure docker is installed (Should run without sudo) + ``` + docker ps + ``` +3. Clone this repository + ``` + git clone https://github.com/Akilan1999/p2p-rendering-computation + ``` +4. Install and build the project + ``` + make install + ``` +- If you look closely you will get outputs such as: + ``` + // Add them to your .bashrc file + export P2PRC=//p2p-rendering-computation + export PATH=//p2p-rendering-computation:${PATH} + ``` + +5. Test if it works + ``` + p2prc -h + ``` + or + ``` + ./p2prc -h + ``` +[Read more on the installation and usage](Docs/Installation.md) + +
+ +## Design Architecture +The design architecture was inspired and based on the linux kernel design. The project is segmented into various modules. Each module is responsible for certain tasks in the project. The modules are highly dependent on each other hence the entire codebase can be considered as a huge monolithic chuck which acts as its own library + +[Read more on the Design Architecture](Docs/DesignArchtectureIntro.md) + +
+ +## Implementation +The programming language used for this project was Golang. The reason Go lang was chosen was because it is a compiled language. The entire codebase is just a single binary file. When distributing to other linux distributing the only requirement would be the binary file to run the code. It is easy to write independant modules and be monolithic at the sametime using Go. Using Go.mod makes it very easy to handle external libraries and modularise code. The go.mod name for the project is git.sr.ht/~akilan1999/p2p-rendering-computation. + +[Read more on the Implementation](Docs/Implementation.md) + +
+ +## Find out more +As we are working on the open source project p2prc (i.e p2p network designed for computation).If you are interested in participating as a contributor +or just providing feedback on new features to build or even just curious about new features added to the project. We have decided to create a discord group. +[![Support Server](https://discordapp.com/api/guilds/854397492795277322/widget.png?style=banner2)](https://discord.gg/b4nRGTjYqy) + +[![Star History Chart](https://api.star-history.com/svg?repos=Akilan1999/p2p-rendering-computation&type=Date)](https://github.com/Gaurav-Gosain) diff --git a/Docs/DocsDeprecated/README.org b/Docs/DocsDeprecated/README.org new file mode 100644 index 0000000..ebe24e0 --- /dev/null +++ b/Docs/DocsDeprecated/README.org @@ -0,0 +1,18 @@ +* Table of contents +:PROPERTIES: +:CUSTOM_ID: table-of-contents +:END: +1.[[file:Introduction.org][ Introduction]] +2. [[file:Installation.org][Installation]] +3. [[file:Abstractions.org][Abstractions]] +4. [[file:Implementation.org][Implementation]] + 1. [[file:ClientImplementation.org][Client Module]] + 2. [[file:P2PImplementation.org][P2P Module]] + 3. [[file:ServerImplementation.org][Server Module]] + 4. [[file:ConfigImplementation.org][Config Module]] + 5. [[file:CliImplementation.org][Cli Module]] + 6. [[file:PluginImplementation.org][Plugin Module]] + 7. [[file:Bindings.org][Language bindings]] + 8. [[file:Bindings.org][Domain name mappings]] +5. Language bindings + 1. [[file:haskell/][Haskell]] diff --git a/Docs/ServerArchitecture.md b/Docs/DocsDeprecated/ServerArchitecture.md similarity index 100% rename from Docs/ServerArchitecture.md rename to Docs/DocsDeprecated/ServerArchitecture.md diff --git a/Docs/DocsDeprecated/ServerArchitecture.md.org b/Docs/DocsDeprecated/ServerArchitecture.md.org new file mode 100644 index 0000000..caf1cca --- /dev/null +++ b/Docs/DocsDeprecated/ServerArchitecture.md.org @@ -0,0 +1,14 @@ +* Server Module Architecture +:PROPERTIES: +:CUSTOM_ID: server-module-architecture +:END: +The server module takes care of setting and removing the virtualization +environment (i.e containers) for accessing and doing the appropriate +computation. It also interacts with the peer to peer module to update +the IP table on the server side. The server module accesses information +regarding CPU and GPU specifications of the machine running the server +module. To do Speed tests the server has routes which allows it to +upload and download a 50mb. + +#+caption: UML diagram of server module +[[file:images/servermoduleArch.png]] diff --git a/Docs/ServerImplementation.md b/Docs/DocsDeprecated/ServerImplementation.md similarity index 100% rename from Docs/ServerImplementation.md rename to Docs/DocsDeprecated/ServerImplementation.md diff --git a/Docs/DocsDeprecated/ServerImplementation.md.org b/Docs/DocsDeprecated/ServerImplementation.md.org new file mode 100644 index 0000000..5e7713c --- /dev/null +++ b/Docs/DocsDeprecated/ServerImplementation.md.org @@ -0,0 +1,278 @@ +* Server Module Implementation +:PROPERTIES: +:CUSTOM_ID: server-module-implementation +:END: +This section focuses on an in-depth understanding of the server module +implementation. To understand the architecture of the server module +refer. The server module can be split into various sections. Each +section will provide information on how a certain feature works. + +The server module takes care of setting and removing the virtualization +environment (i.e containers) for accessing and doing the appropriate +computation. It also interacts with the peer to peer module to update +the IP table on the server side. The server module accesses information +regarding CPU and GPU specifications of the machine running the server +module. To do Speed tests the server has routes which allows it to +upload and download a 50mb. + +#+caption: UML diagram of server module +[[file:images/servermoduleArch.png]] + +** Web framework +:PROPERTIES: +:CUSTOM_ID: web-framework +:END: +The web framework used for the server module is called Gin. The reason +Gin was chosen is due to its wide use and strong documentation available +on the official github repository. The default port used is 8088. For +version 1.0 of the project ,the server needs to keep port 8088 open to +ensure that other clients and servers can detect it. The possible +requests available are GET and POST for this implementation. The +possible responses are either a string or json response or a file. In +the majority of routes a string response refers to an error when calling +the following routes. The following sub topics below will talk about the +route implemented: + +*** /server_info +:PROPERTIES: +:CUSTOM_ID: server_info +:END: +This route is responsible to get information about the specifications of +the server. The response of this route is in json if the call was +successful. + +*** /50 +:PROPERTIES: +:CUSTOM_ID: section +:END: +This route is responsible for returning a randomly generated 50mb file. +This is used to calculate the download speed from the p2p module. + +*** /IpTable +:PROPERTIES: +:CUSTOM_ID: iptable +:END: +This route is a POST request that is responsible to update the server IP +table based on the IP table the client provides. Once the server gets +the IP table it checks if the client is also a server. This is done by +calling the url http://:8088/server_info. If the server_info route from +the client responds back with computer specifications of the client. +Then the server initially appends the clients IP to the struct. After +that the IP table received from the client is uploaded to the struct. +Once this is done the server passes the struct to the peer to peer +module function. The peer to peer module function will return the back +with the new struct with the valid server nodes. The server responds +back to the new struct as a json format. If a string is present in the +response then there is probably an error on the server side. + +*** /startcontainer +:PROPERTIES: +:CUSTOM_ID: startcontainer +:END: +This route takes in a GET request with the number of TCP ports to open +and checks whether the docker container should be hooked to the GPU or +not. This route talks to the docker module implemented as a sub module +in the server module. More information on the docker module in section +5.4.3. This route calls docker the module to start the container for the +client. The docker module returns back a struct. This struct is returned +back to the client as the json response. This struct consists of +information such as docker id, ports numbers open , information +regarding SSH and VNC connections to the docker container created when +the client created this request. + +*** /RemoveContainer +:PROPERTIES: +:CUSTOM_ID: removecontainer +:END: +This route takes in a GET request as the container ID. Based on the +container ID provided ,it calls the docker module which deletes the +container. If the deletion is successful it returns back a string which +says success. + +** Server information/ Specification +:PROPERTIES: +:CUSTOM_ID: server-information-specification +:END: +This section provides information on how the server specifications are +read. There are 2 major implementations. The first implementation +mentions how basic information such as RAM usage, CPU specification are +detected and the second implementation mentions how the GPU drivers are +detected and information is extracted. The client has to assume that the +server is using default docker settings in terms of CPU cycles and other +parameters. + +*** Basic Information +:PROPERTIES: +:CUSTOM_ID: basic-information +:END: +The file name for these functions is called gopsutil.go. This codebase +uses the library gopsutil. Gopsutil has various packages or modules +within the library which have functions implemented to get system +information. The following information is stored in a struct and the +function returns that struct. + +#+begin_src go +type SysInfo struct { + Hostname string `bson:hostname` + Platform string `bson:platform` + CPU string `bson:cpu` + RAM uint64 `bson:ram` + Disk uint64 `bson:disk` + GPU *Query `xml: GpuInfo` +} +#+end_src + +*** GPU Information +:PROPERTIES: +:CUSTOM_ID: gpu-information +:END: +The file name for these functions is called GPU.go. This codebase checks +if the Nvidia driver exists and returns the driver information. To do +this a shell command called nvidia-smi is executed. This shell command +is executed with a --xml as flag to ensure that the output is in the XML +format. If there is an output as a xml format, that means there is an +nvidia driver installed, and the function just reads the output and +stores it to the struct and returns the GPU information. + +#+begin_src go + +type Query struct { + DriveVersion string `xml:"driver_version"` + Gpu Gpu `xml:"gpu"` +} + +type Gpu struct{ + GpuName string `xml:"product_name"` + BiosVersion string `xml:"vbios_version"` + FanSpeed string `xml:"fan_speed"` + Utilization GpuUtilization `xml:"utilization"` + Temperature GpuTemperature `xml:"temperature"` + Clock GpuClock `xml:"clocks"` +} + +type GpuUtilization struct { + GpuUsage string `xml:"gpu_util"` + MemoryUsage string `xml:"memory_util"` +} + +type GpuTemperature struct { + GpuTemp string `xml:"gpu_temp"` +} + +type GpuClock struct { + GpuClock string `xml:"graphics_clock"` + GpuMemClock string `xml:"mem_clock"` +} +#+end_src + +** Docker Module +:PROPERTIES: +:CUSTOM_ID: docker-module +:END: +This section provides information on how the server module interacts +with the docker containers. The server calls 2 routes which either +creates or removes the docker container. Docker has a huge advantage +because it takes less than 20 seconds to spin up a new container once +it's built and executed at least once. For docker operations a separate +module/package has been created. The following subtopics will provide +more information on how this package works. + +*** Docker Api +:PROPERTIES: +:CUSTOM_ID: docker-api +:END: +For this the api has been taken from the official docker repository. To +be more specific it is the client module in the official docker +repository. Docker was built using Go. During this project Docker +functions could be directly called from the docker repository. The +Docker api initially ensures that it can detect the docker environment +variables. Once detected, it can execute various functions from the +docker client module. The reason the docker api was selected was to +detect and handle errors better. + +*** Docker Image +:PROPERTIES: +:CUSTOM_ID: docker-image +:END: +The docker image used to spin up the containers is called +ConSol/docker-headless-vnc-container. The following container was +modified to open SSH ports for an SSH connection. The following docker +image runs ubuntu 16. The reason this image was chosen as a default is +because if the client wants to access the container in the form of a +desktop environment. This image would allow the client to do so from +just a browser. + +*** Build container +:PROPERTIES: +:CUSTOM_ID: build-container +:END: +This function pulls the docker image locally and builds the image. +Initially there is a timeout function to ensure that building the image +does not take too long to build. The next phase would be based on the +path to get the DockerFile. The tag name of the container is set as +p2p-ubuntu as default. Once the following is set then the docker build +command is executed. + +*** Run container +:PROPERTIES: +:CUSTOM_ID: run-container +:END: +After building the container it needs to be executed for the user to +access the container and do certain operations. The docker +package/module has a function to do this. The function takes in the +docker environment as a parameter and also the docker struct. The docker +struct has information such as the TCP ports which are supposed to be +open and whether the docker container should have the GPU hooked to it +or not. Based on the appropriate information provided ,the docker image +gets started. The Image gets started by interacting with the docker +client modules. When hooking the GPU the docker run command is called +from the shell. This is because the docker Api does not support the GPU +module yet. When the container is executed for the first time it takes +more than 10 minutes to build. From the second time onwards it takes +only 10 seconds to run. + +*** Stop and remove container +:PROPERTIES: +:CUSTOM_ID: stop-and-remove-container +:END: +This implementation here ensures that the docker is stopped, and the +container is removed. This is to ensure it does not utilize server +resources when it is not being used, or the task that is intended to be +executed is complete. To run this function all that is needed is the +docker container ID. If the function is successful it returns a string +that says success. + +*** Ports json file +:PROPERTIES: +:CUSTOM_ID: ports-json-file +:END: +This file will help map internal ports inside a container to external +ports inside a container. A common example would be the SSH port which +is port 22 inside the docker container and is mapped to random TCP port +outside container so that any external machines can directly connect +into the container. The below representation mentions of where the +ports.json file is located and also the format of that file. + +#+begin_example +|_ + |_ Dockerfile + |_ description.txt + |_ ports.json // The ports file +#+end_example + +Format of the ports.json file + +#+begin_example +{ + "Port": [ + { + "PortName": "", + "InternalPort": , + "Type": "", + "ExternalPort": , + "IsUsed": "", + "Description": "" + }, ... n + ] +} +#+end_example diff --git a/Docs/Virtualization b/Docs/DocsDeprecated/Virtualization similarity index 100% rename from Docs/Virtualization rename to Docs/DocsDeprecated/Virtualization diff --git a/Docs/DocsDeprecated/docs.org b/Docs/DocsDeprecated/docs.org new file mode 100644 index 0000000..47b2386 --- /dev/null +++ b/Docs/DocsDeprecated/docs.org @@ -0,0 +1,800 @@ +yes* Chapter 1: Introduction +:PROPERTIES: +:CUSTOM_ID: chapter-1-introduction +:END: + +** Abstract +:PROPERTIES: +:CUSTOM_ID: abstract +:END: +This project focuses on creating a framework on running heavy tasks that +a regular computer cannot run easily such as graphically demanding video +games, rendering 3D animations , protein folding simulations. In this +project the major focus will not be on the financial incentive part. A +peer to peer network will be created to help run tasks decentrally, +increasing bandwidth for running tasks. To ensure the tasks in the peer +to peer network do not corrupt the server 0S (Operating System), they +will be executed in a virtual environment in the server. + +The main aim of this project was to create a custom peer to peer +network. The user acting as the client has total flexibility on how to +batch the tasks and the user acting as the server has complete +flexibility on tracking the container's usages and killing the +containers at any point of time. + +** Motivation +:PROPERTIES: +:CUSTOM_ID: motivation +:END: +Many of the users rely on our PC / Laptop or servers that belong to a +server farm to run heavy tasks and with the demand of high creativity +requires higher computing power. Buying a powerful computer every few +years to run a bunch of heavy tasks which are not executed as frequently +to reap the benefits can be inefficient utilization of hardware. On the +other end, renting servers to run these heavy tasks can be really +useful. Ethically speaking this is leading to monopolisation of +computing power similar to what is happening in the web server area. By +using peer to peer principles it is possible to remove the +monopolisation factor and increase the bandwidth between the client and +server. + +* Installation +:PROPERTIES: +:CUSTOM_ID: installation +:END: + +Over here we will cover the basic steps to get the server and client +side running. + +** Latest release install +:PROPERTIES: +:CUSTOM_ID: latest-release-install +:END: +https://github.com/Akilan1999/p2p-rendering-computation/releases + +** Install from Github master branch +:PROPERTIES: +:CUSTOM_ID: install-from-github-master-branch +:END: +*** Install Go lang +:PROPERTIES: +:CUSTOM_ID: install-go-lang +:END: +The entire the implementation of this project is done using Go lang. +Thus, we need go lang to compile to code to a binary file. +[[https://golang.org/doc/install][Instructions to install Go lang]] + +*** Install Docker +:PROPERTIES: +:CUSTOM_ID: install-docker +:END: +In this project the choice of virtualization is Docker due to it's wide +usage in the developer community. In the server module we use the Docker +Go API to create and interact with the containers. + +[[https://docs.docker.com/get-docker/][Instructions to install docker]] + +[[https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker][Instructions +to install docker GPU]] + +#+begin_example +// Do ensure that the docker command does not need sudo to run +sudo chmod 666 /var/run/docker.sock +#+end_example + +*** Build Project and install project +:PROPERTIES: +:CUSTOM_ID: build-project-and-install-project +:END: +To set up the internal dependencies and build the entire go code into a +single binary + +#+begin_example +make install +#+end_example + +**** For Windows +:PROPERTIES: +:CUSTOM_ID: for-windows +:END: +To set up P2PRC on Windows, simply run this batch file. *Make sure you +are not in admin mode when running this.* + +#+begin_example +.\install.bat +#+end_example + +*** Add appropriate paths to =.bashrc= +:PROPERTIES: +:CUSTOM_ID: add-appropriate-paths-to-.bashrc +:END: +#+begin_example +export P2PRC=//p2p-rendering-computation +export PATH=//p2p-rendering-computation:${PATH} +#+end_example + +*** Set up configuration file +:PROPERTIES: +:CUSTOM_ID: set-up-configuration-file +:END: +#+begin_example +make configfile +#+end_example + +Open the config file =config.json= and add the IPv6 address if you have +one. + +*** Test if binary works +:PROPERTIES: +:CUSTOM_ID: test-if-binary-works +:END: +#+begin_example +p2prc --help +#+end_example + +**** Output: +:PROPERTIES: +:CUSTOM_ID: output +:END: +#+begin_example +NAME: + p2p-rendering-computation - p2p cli application to create and access VMs in other servers + +USAGE: + p2prc [global options] command [command options] [arguments...] + +VERSION: + + +COMMANDS: + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --Server, -s Starts server (default: false) [$SERVER] + --UpdateServerList, --us Update List of Server available based on servers iptables (default: false) [$UPDATE_SERVER_LIST] + --ListServers, --ls List servers which can render tasks (default: false) [$LIST_SERVERS] + --AddServer value, --as value Adds server IP address to iptables [$ADD_SERVER] + --ViewImages value, --vi value View images available on the server IP address [$VIEW_IMAGES] + --CreateVM value, --touch value Creates Docker container on the selected server [$CREATE_VM] + --ContainerName value, --cn value Specifying the container run on the server side [$CONTAINER_NAME] + --RemoveVM value, --rm value Stop and Remove Docker container (IP:port) accompanied by container ID via --ID or --id [$REMOVE_VM] + --ID value, --id value Docker Container ID [$ID] + --Ports value, -p value Number of ports to open for the Docker Container [$NUM_PORTS] + --GPU, --gpu Create Docker Containers to access GPU (default: false) [$USE_GPU] + --Specification value, --specs value Specs of the server node [$SPECS] + --SetDefaultConfig, --dc Sets a default configuration file (default: false) [$SET_DEFAULT_CONFIG] + --NetworkInterfaces, --ni Shows the network interface in your computer (default: false) [$NETWORK_INTERFACE] + --ViewPlugins, --vp Shows plugins available to be executed (default: false) [$VIEW_PLUGIN] + --TrackedContainers, --tc View (currently running) containers which have been created from the client side (default: false) [$TRACKED_CONTAINERS] + --ExecutePlugin value, --plugin value Plugin which needs to be executed [$EXECUTE_PLUGIN] + --CreateGroup, --cgroup Creates a new group (default: false) [$CREATE_GROUP] + --Group value, --group value group flag with argument group ID [$GROUP] + --Groups, --groups View all groups (default: false) [$GROUPS] + --RemoveContainerGroup, --rmcgroup Remove specific container in the group (default: false) [$REMOVE_CONTAINER_GROUP] + --RemoveGroup value, --rmgroup value Removes the entire group [$REMOVE_GROUP] + --Generate value, --gen value Generates a new copy of P2PRC which can be modified based on your needs [$GENERATE] + --ModuleName value, --mod value New go project module name [$MODULENAME] + --PullPlugin value, --pp value Pulls plugin from git repos [$PULLPLUGIN] + --RemovePlugin value, --rp value Removes plugin [$REMOVEPLUGIN] + --help, -h show help (default: false) + --version, -v print the version (default: false) +#+end_example + +-------------- + +* Using basic commands +:PROPERTIES: +:CUSTOM_ID: using-basic-commands +:END: +*** Start as a server +:PROPERTIES: +:CUSTOM_ID: start-as-a-server +:END: +Do ensure you have Docker installed for this + +#+begin_example +p2prc -s +#+end_example + +*** View server Specification +:PROPERTIES: +:CUSTOM_ID: view-server-specification +:END: +#+begin_example +p2prc --specs= +#+end_example + +*** Run container +:PROPERTIES: +:CUSTOM_ID: run-container +:END: +use the =--gpu= if you know the other machine has a gpu. + +#+begin_example +p2prc --touch= -p --gpu +#+end_example + +*** Remove container +:PROPERTIES: +:CUSTOM_ID: remove-container +:END: +The docker id is present in the output where you create a container + +#+begin_example +p2prc --rm= --id= +#+end_example + +*** Adding servers to ip table +:PROPERTIES: +:CUSTOM_ID: adding-servers-to-ip-table +:END: +#+begin_example +p2prc --as= +#+end_example + +*** Update ip table +:PROPERTIES: +:CUSTOM_ID: update-ip-table +:END: +#+begin_example +p2prc --us +#+end_example + +*** List Servers +:PROPERTIES: +:CUSTOM_ID: list-servers +:END: +#+begin_example +p2prc --ls +#+end_example + +*** View Network interfaces +:PROPERTIES: +:CUSTOM_ID: view-network-interfaces +:END: +#+begin_example +p2prc --ni +#+end_example + +*** Viewing Containers created Client side +:PROPERTIES: +:CUSTOM_ID: viewing-containers-created-client-side +:END: +#+begin_example +p2prc --tc +#+end_example + +[[file:ClientImplementation.md#tracking-containers][read more on +tracking containers]] + +*** Running plugin +:PROPERTIES: +:CUSTOM_ID: running-plugin +:END: +#+begin_example +p2prc --plugin --id +#+end_example + +*** Create group +:PROPERTIES: +:CUSTOM_ID: create-group +:END: +#+begin_example +p2prc --cgroup +#+end_example + +*** Add container to group +:PROPERTIES: +:CUSTOM_ID: add-container-to-group +:END: +#+begin_example +p2prc --group --id +#+end_example + +*** View groups +:PROPERTIES: +:CUSTOM_ID: view-groups +:END: +#+begin_example +p2prc --groups +#+end_example + +*** View specific group +:PROPERTIES: +:CUSTOM_ID: view-specific-group +:END: +#+begin_example +p2prc --group +#+end_example + +*** Delete container from group +:PROPERTIES: +:CUSTOM_ID: delete-container-from-group +:END: +#+begin_example +p2prc --rmcgroup --group --id +#+end_example + +*** Delete entire group +:PROPERTIES: +:CUSTOM_ID: delete-entire-group +:END: +#+begin_example +p2prc --rmgroup +#+end_example + +[[file:ClientImplementation.md#Grouping-Containers][read more on +grouping containers]] ### Extending usecase of P2PRC (Requires a go +compiler to run) + +#+begin_example +p2prc --gen --mod +#+end_example + +[[file:GenerateImplementation.md][read more about the generate module]] + +*** Pulling plugin from a remote repo +:PROPERTIES: +:CUSTOM_ID: pulling-plugin-from-a-remote-repo +:END: +#+begin_example +p2prc --pp +#+end_example + +*** Deleting plugin from the plugin directory +:PROPERTIES: +:CUSTOM_ID: deleting-plugin-from-the-plugin-directory +:END: +#+begin_example +p2prc --rp +#+end_example + +*** Added custom metadata about the current node +:PROPERTIES: +:CUSTOM_ID: added-custom-metadata-about-the-current-node +:END: +#+begin_example +p2prc --amd "custom metadata" +#+end_example + +-------------- + +* Using Plugins +:PROPERTIES: +:CUSTOM_ID: using-plugins +:END: +This feature is still Under Development: +[[file:PluginImplementation.md][Read more on the implementation]] + +**** Dependencies +:PROPERTIES: +:CUSTOM_ID: dependencies +:END: +- Ansible: + - Debian/ubuntu: =sudo apt install ansible= + - Others: + [[https://ansible-tips-and-tricks.readthedocs.io/en/latest/ansible/install/][Installation + link]] + +**** Run Test Cases +:PROPERTIES: +:CUSTOM_ID: run-test-cases +:END: +- Generate Test Case Ansible file + - =make testcases= +- Enter inside plugin directory and run tests. + +#+begin_quote +[!NOTE] That docker needs to installed and needs to run without sudo. +Refer the section [[#install-docker][Install Docker]]. - =cd plugin= - +=go test .= + +#+end_quote + +* P2P Module Implementation +:PROPERTIES: +:CUSTOM_ID: p2p-module-implementation +:END: +The P2P module (i.e Peer to Peer Module) is responsible for storing the +IP table and interacting with the IP table. In the following +implementation of the P2P module ,the IP table stores information about +servers available in the network. The other functionality the P2P module +takes care of is doing the appropriate speed tests to the servers in the +IP table. This is for informing the users about nodes which are close by +and nodes which have quicker uploads and downloads speeds. The module is +responsible to ensure that there are no duplicate server IPs in the IP +table and to remove all server IPs which are not pingable. + +#+caption: UML diagram of P2P module +[[file:images/p2pmoduleArch.png]] + +The peer to peer implementation was built from scratch. This is because +other peer to peer libraries were on the implementation of the +Distributed hash table. At the current moment all those heavy features +are not needed because the objective is to search and list all possible +servers available. The limitation being that to be a part of the network +the user has to know at least 1 server. The advantage of building from +scratch makes the module super light and possibility for custom +functions and structs. The sub topics below will mention the +implementations of each functionality in depth. + +** IP Table +:PROPERTIES: +:CUSTOM_ID: ip-table +:END: +The ip table file is a json as the format with a list of servers ip +addresses, latencies, downloads and uploads speeds. The functions +implemented include read file, write file and remove duplicate IP +addresses. The remove duplicate IP address function exists because +sometimes servers IP tables can have the same ip addresses as what the +client has. The path of the IP table json file is received from the +configuration module. + +#+begin_src json +{ + "ip_address": [ + { + "ipv4": "", + "latency": "", + "download": "", + "upload": "" + "port no": "", + } + ] +} +#+end_src + +*** Latency +:PROPERTIES: +:CUSTOM_ID: latency +:END: +The latency is measured in milliseconds. The route /server_info is +called from the server and time it takes to provide a json response is +recorded. + +** NAT Traversal +:PROPERTIES: +:CUSTOM_ID: nat-traversal +:END: +P2PRC currently supports TURN for NAT traversal. + +** TURN +:PROPERTIES: +:CUSTOM_ID: turn +:END: +The current TURN implementation used is FRP. The TURN server is also +required when a P2PRC node is acting as a Server. The TURN server is +determined based on the Node with the least amount of latency based on +the Nodes available on the IPTable. Once a TURN server is determined +there are 2 actions performed. The first one is =/FRPPort= to the TURN +server to receive a port which is used to generate the external port +from the TURN server. The flow below describes the workflow. + +*** Client mode +:PROPERTIES: +:CUSTOM_ID: client-mode +:END: +- Call =/FRPPort= + +#+begin_example +http://:/FRPport +#+end_example + +- Call the TURN server in the following manner. The following is a + sample code snippet below. + +#+begin_src go +import ( + "github.com/Akilan1999/p2p-rendering-computation/p2p/frp" +) + +func main() { + serverPort, err := frp.GetFRPServerPort("http://" + + ":" + lowestLatencyIpAddress.ServerPort) + if err != nil { + return nil, err + } + // Create 1 second delay to allow FRP server to start + time.Sleep(1 * time.Second) + // Starts FRP as a client with + proxyPort, err := frp.StartFRPClientForServer(, serverPort, ) + if err != nil { + return nil, err + } +} +#+end_src + +* Language Bindings +:PROPERTIES: +:CUSTOM_ID: language-bindings +:END: +[[https://en.wikipedia.org/wiki/Language_binding][Language bindings]] +refers to wrappers to bridge 2 programming languages. This is used in +P2PRC to extend calling P2PRC functions in other programming languages. +Currently this is done by generating =.so= and =.h= from the Go +compiler. + +** How to build shared object files +:PROPERTIES: +:CUSTOM_ID: how-to-build-shared-object-files +:END: +**** The easier way +:PROPERTIES: +:CUSTOM_ID: the-easier-way +:END: +#+begin_src sh +# Run +make sharedObjects +#+end_src + +**** Or the direct way +:PROPERTIES: +:CUSTOM_ID: or-the-direct-way +:END: +#+begin_src sh +# Run +cd Bindings && go build -buildmode=c-shared -o p2prc.so +#+end_src + +**** If successfully built: +:PROPERTIES: +:CUSTOM_ID: if-successfully-built +:END: +#+begin_src sh +# Enter into the Bindings directory +cd Bindings +# List files +ls +# Find files +p2prc.h p2prc.so +#+end_src + +** Workings under the hood +:PROPERTIES: +:CUSTOM_ID: workings-under-the-hood +:END: +Below are a sample set of commands to open the bindings implementation. + +#+begin_example +# run +cd Bindings/ +# list files +ls +# search for file +Client.go +#+end_example + +*** In Client go +:PROPERTIES: +:CUSTOM_ID: in-client-go +:END: +There a few things to notice which are different from your standard Go +programs: + +**** 1. We import "C" which means [[https://pkg.go.dev/cmd/cgo][Cgo]] is required. +:PROPERTIES: +:CUSTOM_ID: we-import-c-which-means-cgo-is-required. +:END: +#+begin_src go +import "C" +#+end_src + +**** 2. All functions which are required to be called from other programming languages have comment such as. +:PROPERTIES: +:CUSTOM_ID: all-functions-which-are-required-to-be-called-from-other-programming-languages-have-comment-such-as. +:END: +#+begin_src go +//export + +// ------------ Example ---------------- +// The function below allows to externally +// to call the P2PRC function to start containers +// in a specific node in the know list of nodes +// in the p2p network. +// Note: the comment "//export StartContainer". + +//export StartContainer +func StartContainer(IP string) (output *C.char) { + container, err := client.StartContainer(IP, 0, false, "", "") + if err != nil { + return C.CString(err.Error()) + } + return ConvertStructToJSONString(container) + } +#+end_src + +**** 3. While looking through the file (If 2 files are compared it is pretty trivial to notice a common structure). +:PROPERTIES: +:CUSTOM_ID: while-looking-through-the-file-if-2-files-are-compared-it-is-pretty-trivial-to-notice-a-common-structure. +:END: +#+begin_src go +// --------- Example ------------ + +//export StartContainer +func StartContainer(IP string) (output *C.char) { + container, err := client.StartContainer(IP, 0, false, "", "") + if err != nil { + return C.CString(err.Error()) + } + return ConvertStructToJSONString(container) +} + +//export ViewPlugin +func ViewPlugin() (output *C.char) { + plugins, err := plugin.DetectPlugins() + if err != nil { + return C.CString(err.Error()) + } + return ConvertStructToJSONString(plugins) +} +#+end_src + +**** It is easy to notice that: +:PROPERTIES: +:CUSTOM_ID: it-is-easy-to-notice-that +:END: +- =ConvertStructToJSONString()=: This is a helper function + that convert a go object to JSON string initially and converts it to + =CString=. +- =(output *C.char)=: This is the return type for most of the functions. + +**** A Pseudo code to refer to the common function implementation shape could be represented as: +:PROPERTIES: +:CUSTOM_ID: a-pseudo-code-to-refer-to-the-common-function-implementation-shape-could-be-represented-as +:END: +#+begin_example +func (output *C.char) { + , := () + if != nil { + return C.CString(.Error()) + } + return ConvertStructToJSONString() +} +#+end_example + +** Current languages supported +:PROPERTIES: +:CUSTOM_ID: current-languages-supported +:END: +- Python + +*** Build sample python program +:PROPERTIES: +:CUSTOM_ID: build-sample-python-program +:END: +The easier way + +#+begin_src sh +# Run +make python +# Expected ouput +Output is in the Directory Bindings/python/export/ +# Run +cd Bindings/python/export/ +# list files +ls +# Expected output +SharedObjects/ p2prc.py +#+end_src + +Above shows a generated folder which consists of a folder called +"SharedObjects/" which consists of =p2prc.so= and =p2prc.h= files. +=p2prc.py= refers to a sample python script calling P2PRC go functions. +To start an any project to extend P2PRC with python, This generated +folder can copied and created as a new git repo for P2PRC extensions +scripted or used a reference point as proof of concept that P2PRC can be +called from other programming languages. + + +* Config Implementation +:PROPERTIES: +:CUSTOM_ID: config-implementation +:END: +The configuration module is responsible to store basic information of +absolute paths of files being called in the Go code. In a full-fledged +Cli the configuration file can be found in the directory /etc/ and from +there points to location such as where the IP table file is located. In +the future implementation the config file will have information such as +number of hops and other parameters to tweak and to improve the +effectiveness of the peer to peer network. The configuration module was +implemented using the library Viper. The Viper library automates +features such as searching in default paths to find out if the +configuration file is present. If the configuration file is not present +in the default paths then it auto generates the configuration file. The +configurations file can be in any format. In this project the +configuration file was generated using JSON format. + +#+begin_src json +{ + "MachineName": "pc-74-120.customer.ask4.lan", + "IPTable": "/Users/akilan/Documents/p2p-rendering-computation/p2p/iptable/ip_table.json", + "DockerContainers": "/Users/akilan/Documents/p2p-rendering-computation/server/docker/containers/", + "DefaultDockerFile": "/Users/akilan/Documents/p2p-rendering-computation/server/docker/containers/docker-ubuntu-sshd/", + "SpeedTestFile": "/Users/akilan/Documents/p2p-rendering-computation/p2p/50.bin", + "IPV6Address": "", + "PluginPath": "/Users/akilan/Documents/p2p-rendering-computation/plugin/deploy", + "TrackContainersPath": "/Users/akilan/Documents/p2p-rendering-computation/client/trackcontainers/trackcontainers.json", + "ServerPort": "8088", + "GroupTrackContainersPath": "/Users/akilan/Documents/p2p-rendering-computation/client/trackcontainers/grouptrackcontainers.json", + "FRPServerPort": "True", + "BehindNAT": "True", + "CustomConfig": null +} +#+end_src + +* Abstractions +:PROPERTIES: +:CUSTOM_ID: abstractions +:END: + +The Abstractions package consists of black-boxed functions for P2PRC. + +** Functions +:PROPERTIES: +:CUSTOM_ID: functions +:END: +- =Init()=: Initializes P2PRC with all the needed + configurations. +- =Start()=: Starts p2prc as a server and makes it possible to extend by + adding other routes and functionality to P2PRC. +- =MapPort()=: On the local machine the port you want to export + to world. +- =StartContainer()=: The machine on the p2p network where + you want to spin up a docker container. +- =RemoveContainer(,)=: Terminate container + based on the IP address and container name. +- =GetSpecs()=: Get specs of a machine on the network based + on the IP address. +- =ViewIPTable()=: View the IP table which about nodes in the network. +- =UpdateIPTable()=: Force update IP table to learn about new nodes + faster. + +* NAT Traversal +:PROPERTIES: +:CUSTOM_ID: nat-traversal +:END: +P2PRC currently supports TURN for NAT traversal. + +** TURN +:PROPERTIES: +:CUSTOM_ID: turn +:END: +The current TURN implementation used is FRP. The TURN server is also +required when a P2PRC node is acting as a Server. The TURN server is +determined based on the Node with the least amount of latency based on +the Nodes available on the IPTable. Once a TURN server is determined +there are 2 actions performed. The first one is =/FRPPort= to the TURN +server to receive a port which is used to generate the external port +from the TURN server. The flow below describes the workflow. + +*** Client mode +:PROPERTIES: +:CUSTOM_ID: client-mode +:END: +- Call =/FRPPort= + +#+begin_example +http://:/FRPport +#+end_example + +- Call the TURN server in the following manner. The following is a + sample code snippet below. + +#+begin_src go +import ( + "github.com/Akilan1999/p2p-rendering-computation/p2p/frp" +) + +func main() { + serverPort, err := frp.GetFRPServerPort("http://" + + ":" + lowestLatencyIpAddress.ServerPort) + if err != nil { + return nil, err + } + // Create 1 second delay to allow FRP server to start + time.Sleep(1 * time.Second) + // Starts FRP as a client with + proxyPort, err := frp.StartFRPClientForServer(, serverPort, ) + if err != nil { + return nil, err + } +} +#+end_src diff --git a/Docs/README.md b/Docs/README.md deleted file mode 100644 index 9c85313..0000000 --- a/Docs/README.md +++ /dev/null @@ -1,21 +0,0 @@ - -# Table of contents -1. [Introduction](Introduction.md) -2. [Installation](Installation.md) -3. [Abstractions](Abstractions.md) - -4. [Implementation](Implementation.md) - 1. [Client Module](ClientImplementation.md) - 2. [P2P Module](P2PImplementation.md) - 3. [Server Module](ServerImplementation.md) - 4. [Config Module](ConfigImplementation.md) - 5. [Cli Module](CliImplementation.md) - 6. [Plugin Module](PluginImplementation.md) - 7. [Language bindings](Bindings.md) - 8. [Domain name mappings](Bindings.md) -5. Language bindings - 1. [Haskell](haskell/) - diff --git a/Docs/diagrams/P2PRCRemoteNodes.drawio b/Docs/diagrams/P2PRCRemoteNodes.drawio new file mode 100644 index 0000000..cd57727 --- /dev/null +++ b/Docs/diagrams/P2PRCRemoteNodes.drawio @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Docs/docs.org b/Docs/docs.org new file mode 100644 index 0000000..9e81540 --- /dev/null +++ b/Docs/docs.org @@ -0,0 +1,825 @@ +#+SETUPFILE: https://fniessen.github.io/org-html-themes/org/theme-readtheorg.setup + +#+HTML_HEAD: +#+HTML_HEAD: + + +* Introduction +:PROPERTIES: +:CUSTOM_ID: chapter1-introduction +:END: + +** Abstract +:PROPERTIES: +:CUSTOM_ID: abstract +:END: +This project focuses on creating a framework for running heavy computational tasks that a regular +computer cannot handle easily. These tasks may include graphically demanding video games, rendering +3D animations, and performing complex protein folding simulations. The major focus of this project +is not on financial incentives but rather on building a robust and efficient peer-to-peer (P2P) +network to decentralise task execution and increase the computational bandwidth available for +such tasks. + +The P2PRC framework serves as a foundation for decentralised rendering and computation, +providing insights into how tasks can be distributed efficiently across a network of peers. +Leveraging the P2PRC approach, this project extends its capabilities to handle a +wider range of computationally intensive tasks. + +** Motivation +:PROPERTIES: +:CUSTOM_ID: motivation +:END: +Many of the users rely on our PC / Laptop or servers that belong to a +server farm to run heavy tasks and with the demand of high creativity +requires higher computing power. Buying a powerful computer every few +years to run a bunch of heavy tasks which are not executed as frequently +to reap the benefits can be inefficient utilization of hardware. On the +other end, renting servers to run these heavy tasks can be really +useful. Ethically speaking this is leading to monopolisation of +computing power similar to what is happening in the web server area. By +using peer to peer principles it is possible to remove the +monopolisation factor and increase the bandwidth between the client and +server. + +* Installation +:PROPERTIES: +:CUSTOM_ID: installation +:END: + +Over here we will cover the basic steps to get the server and client +side running. + +** Latest release install +:PROPERTIES: +:CUSTOM_ID: latest-release-install +:END: +https://github.com/Akilan1999/p2p-rendering-computation/releases + +** Install from Github master branch +:PROPERTIES: +:CUSTOM_ID: install-from-github-master-branch +:END: +*** Install Go lang +:PROPERTIES: +:CUSTOM_ID: install-go-lang +:END: +The entire the implementation of this project is done using Go lang. +Thus, we need go lang to compile to code to a binary file. +[[https://golang.org/doc/install][Instructions to install Go lang]] + +*** Install Docker +:PROPERTIES: +:CUSTOM_ID: install-docker +:END: +In this project the choice of virtualization is Docker due to it's wide +usage in the developer community. In the server module we use the Docker +Go API to create and interact with the containers. + +[[https://docs.docker.com/get-docker/][Instructions to install docker]] + +[[https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker][Instructions +to install docker GPU]] + +#+begin_example +// Do ensure that the docker command does not need sudo to run +sudo chmod 666 /var/run/docker.sock +#+end_example + +*** Build Project and install project +:PROPERTIES: +:CUSTOM_ID: build-project-and-install-project +:END: +To set up the internal dependencies and build the entire go code into a +single binary + +#+begin_example +make install +#+end_example + +**** For Windows +:PROPERTIES: +:CUSTOM_ID: for-windows +:END: +To set up P2PRC on Windows, simply run this batch file. *Make sure you +are not in admin mode when running this.* + +#+begin_example +.\install.bat +#+end_example + +*** Add appropriate paths to =.bashrc= +:PROPERTIES: +:CUSTOM_ID: add-appropriate-paths-to-.bashrc +:END: +#+begin_example +export P2PRC=//p2p-rendering-computation +export PATH=//p2p-rendering-computation:${PATH} +#+end_example + +*** Set up configuration file +:PROPERTIES: +:CUSTOM_ID: set-up-configuration-file +:END: +#+begin_example +make configfile +#+end_example + +Open the config file =config.json= and add the IPv6 address if you have +one. + +*** Test if binary works +:PROPERTIES: +:CUSTOM_ID: test-if-binary-works +:END: +#+begin_example +p2prc --help +#+end_example + +**** Output: +:PROPERTIES: +:CUSTOM_ID: output +:END: +#+begin_example +NAME: + p2p-rendering-computation - p2p cli application to create and access VMs in other servers + +USAGE: + p2prc [global options] command [command options] [arguments...] + +VERSION: + 2.0.0 + +COMMANDS: + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --Server, -s Starts server (default: false) [$SERVER] + --UpdateServerList, --us Update List of Server available based on servers iptables (default: false) [$UPDATE_SERVER_LIST] + --ListServers, --ls List servers which can render tasks (default: false) [$LIST_SERVERS] + --AddServer value, --as value Adds server IP address to iptables [$ADD_SERVER] + --ViewImages value, --vi value View images available on the server IP address [$VIEW_IMAGES] + --CreateVM value, --touch value Creates Docker container on the selected server [$CREATE_VM] + --ContainerName value, --cn value Specifying the container run on the server side [$CONTAINER_NAME] + --BaseImage value, --bi value Specifying the docker base image to template the dockerfile [$CONTAINER_NAME] + --RemoveVM value, --rm value Stop and Remove Docker container (IP:port) accompanied by container ID via --ID or --id [$REMOVE_VM] + --ID value, --id value Docker Container ID [$ID] + --Ports value, -p value Number of ports to open for the Docker Container [$NUM_PORTS] + --GPU, --gpu Create Docker Containers to access GPU (default: false) [$USE_GPU] + --Specification value, --specs value Specs of the server node [$SPECS] + --SetDefaultConfig, --dc Sets a default configuration file (default: false) [$SET_DEFAULT_CONFIG] + --NetworkInterfaces, --ni Shows the network interface in your computer (default: false) [$NETWORK_INTERFACE] + --ViewPlugins, --vp Shows plugins available to be executed (default: false) [$VIEW_PLUGIN] + --TrackedContainers, --tc View (currently running) containers which have been created from the client side (default: false) [$TRACKED_CONTAINERS] + --ExecutePlugin value, --plugin value Plugin which needs to be executed [$EXECUTE_PLUGIN] + --CreateGroup, --cgroup Creates a new group (default: false) [$CREATE_GROUP] + --Group value, --group value group flag with argument group ID [$GROUP] + --Groups, --groups View all groups (default: false) [$GROUPS] + --RemoveContainerGroup, --rmcgroup Remove specific container in the group (default: false) [$REMOVE_CONTAINER_GROUP] + --RemoveGroup value, --rmgroup value Removes the entire group [$REMOVE_GROUP] + --MAPPort value, --mp value Maps port for a specific port provided as the parameter [$MAPPORT] + --DomainName value, --dn value While mapping ports allows to set a domain name to create a mapping in the proxy server [$DOMAINNAME] + --Generate value, --gen value Generates a new copy of P2PRC which can be modified based on your needs [$GENERATE] + --ModuleName value, --mod value New go project module name [$MODULENAME] + --PullPlugin value, --pp value Pulls plugin from git repos [$PULLPLUGIN] + --RemovePlugin value, --rp value Removes plugin [$REMOVEPLUGIN] + --AddMetaData value, --amd value Adds metadata about the current node in the p2p network which is then propagated through the network [$ADDMETADATA] + --help, -h show help (default: false) + --version, -v print the version (default: false) +#+end_example + +-------------- + +* Using basic commands +:PROPERTIES: +:CUSTOM_ID: using-basic-commands +:END: +*** Start as a server +:PROPERTIES: +:CUSTOM_ID: start-as-a-server +:END: + +#+begin_example +p2prc -s +#+end_example + +*** View server Specification +:PROPERTIES: +:CUSTOM_ID: view-server-specification +:END: +#+begin_example +p2prc --specs= +#+end_example + +*** Run container +:PROPERTIES: +:CUSTOM_ID: run-container +:END: +use the =--gpu= if you know the other machine has a gpu. + +#+begin_example +p2prc --touch= -p --gpu +#+end_example + +*** Remove container +:PROPERTIES: +:CUSTOM_ID: remove-container +:END: +The docker id is present in the output where you create a container + +#+begin_example +p2prc --rm= --id= +#+end_example + +*** Adding servers to ip table +:PROPERTIES: +:CUSTOM_ID: adding-servers-to-ip-table +:END: +#+begin_example +p2prc --as= +#+end_example + +*** Update ip table +:PROPERTIES: +:CUSTOM_ID: update-ip-table +:END: +#+begin_example +p2prc --us +#+end_example + +*** List Servers +:PROPERTIES: +:CUSTOM_ID: list-servers +:END: +#+begin_example +p2prc --ls +#+end_example + +*** View Network interfaces +:PROPERTIES: +:CUSTOM_ID: view-network-interfaces +:END: +#+begin_example +p2prc --ni +#+end_example + +*** Viewing Containers created Client side +:PROPERTIES: +:CUSTOM_ID: viewing-containers-created-client-side +:END: +#+begin_example +p2prc --tc +#+end_example + +*** Running plugin +:PROPERTIES: +:CUSTOM_ID: running-plugin +:END: +#+begin_example +p2prc --plugin --id +#+end_example + +*** Create group +:PROPERTIES: +:CUSTOM_ID: create-group +:END: +#+begin_example +p2prc --cgroup +#+end_example + +*** Add container to group +:PROPERTIES: +:CUSTOM_ID: add-container-to-group +:END: +#+begin_example +p2prc --group --id +#+end_example + +*** View groups +:PROPERTIES: +:CUSTOM_ID: view-groups +:END: +#+begin_example +p2prc --groups +#+end_example + +*** View specific group +:PROPERTIES: +:CUSTOM_ID: view-specific-group +:END: +#+begin_example +p2prc --group +#+end_example + +*** Delete container from group +:PROPERTIES: +:CUSTOM_ID: delete-container-from-group +:END: +#+begin_example +p2prc --rmcgroup --group --id +#+end_example + +*** Delete entire group +:PROPERTIES: +:CUSTOM_ID: delete-entire-group +:END: +#+begin_example +p2prc --rmgroup +#+end_example + +*** Pulling plugin from a remote repo +:PROPERTIES: +:CUSTOM_ID: pulling-plugin-from-a-remote-repo +:END: +#+begin_example +p2prc --pp +#+end_example + +*** Deleting plugin from the plugin directory +:PROPERTIES: +:CUSTOM_ID: deleting-plugin-from-the-plugin-directory +:END: +#+begin_example +p2prc --rp +#+end_example + +*** Added custom metadata about the current node +:PROPERTIES: +:CUSTOM_ID: added-custom-metadata-about-the-current-node +:END: +#+begin_example +p2prc --amd "custom metadata" +#+end_example + +*** MapPort and link to domain name +#+begin_example +p2prc --mp --dn +#+end_example + +-------------- + +* P2P Module Implementation +:PROPERTIES: +:CUSTOM_ID: p2p-module-implementation +:END: +The P2P module is for managing server information within the network. +It maintains and updates the IP table, ensuring accuracy by preventing duplicates and removing +entries for unreachable servers. Furthermore, the module conducts speed tests on the listed servers +to determine upload and download speeds. This valuable information enables users to identify nearby +servers with optimal performance, enhancing their overall network experience. + +#+caption: UML diagram of P2P module +[[file:images/p2pmoduleArch.png]] + +The peer to peer implementation was built from scratch. This is because +other peer to peer libraries were on the implementation of the +Distributed hash table. At the current moment all those heavy features +are not needed because the objective is to search and list all possible +servers available. The limitation being that to be a part of the network +the user has to know at least 1 server. The advantage of building from +scratch makes the module super light and possibility for custom +functions and structs. The sub topics below will mention the +implementations of each functionality in depth. + +** IP Table +:PROPERTIES: +:CUSTOM_ID: ip-table +:END: +The ip table file is a json as the format with a list of servers ip +addresses, latencies, downloads and uploads speeds. The functions +implemented include read file, write file and remove duplicate IP +addresses. The remove duplicate IP address function exists because +sometimes servers IP tables can have the same ip addresses as what the +client has. The path of the IP table json file is received from the +configuration module. + +#+begin_src json +{ + "ip_address": [ + { + "Name": "", + "MachineUsername": "", + "IPV4": "", + "IPV6": "", + "Latency": , + "Download": , + "Upload": , + "ServerPort": "", + "BareMetalSSHPort": "", + "NAT": "", + "EscapeImplementation": "", + "ProxyServer": "", + "UnSafeMode": ", + "PublicKey": "", + "CustomInformation": "" + } + ] +} +#+end_src + +*** Latency +:PROPERTIES: +:CUSTOM_ID: latency +:END: +The latency is measured in milliseconds. The route /server_info is +called from the server and time it takes to provide a json response is +recorded. + +** NAT Traversal +:PROPERTIES: +:CUSTOM_ID: nat-traversal +:END: +P2PRC currently supports TURN for NAT traversal. + +** TURN +:PROPERTIES: +:CUSTOM_ID: turn +:END: +The current TURN implementation used is FRP. The TURN server is also +required when a P2PRC node is acting as a Server. The TURN server is +determined based on the Node with the least amount of latency based on +the Nodes available on the IPTable. Once a TURN server is determined +there are 2 actions performed. The first one is =/FRPPort= to the TURN +server to receive a port which is used to generate the external port +from the TURN server. The flow below describes the workflow. + +*** Client mode +:PROPERTIES: +:CUSTOM_ID: client-mode +:END: +- Call =/FRPPort= + +#+begin_example +http://:/FRPport +#+end_example + +- Call the TURN server in the following manner. The following is a + sample code snippet below. + +#+begin_src go +import ( + "github.com/Akilan1999/p2p-rendering-computation/p2p/frp" +) + +func main() { + serverPort, err := frp.GetFRPServerPort("http://" + + ":" + lowestLatencyIpAddress.ServerPort) + if err != nil { + return nil, err + } + // Create 1 second delay to allow FRP server to start + time.Sleep(1 * time.Second) + // Starts FRP as a client with + proxyPort, err := frp.StartFRPClientForServer(, serverPort, ) + if err != nil { + return nil, err + } +} +#+end_src + +* Language Bindings +:PROPERTIES: +:CUSTOM_ID: language-bindings +:END: +[[https://en.wikipedia.org/wiki/Language_binding][Language bindings]] +refers to wrappers to bridge 2 programming languages. This is used in +P2PRC to extend calling P2PRC functions in other programming languages. +Currently this is done by generating =.so= and =.h= from the Go +compiler. + +** How to build shared object files +:PROPERTIES: +:CUSTOM_ID: how-to-build-shared-object-files +:END: +**** The easier way +:PROPERTIES: +:CUSTOM_ID: the-easier-way +:END: +#+begin_src sh +# Run +make sharedObjects +#+end_src + +**** Or the direct way +:PROPERTIES: +:CUSTOM_ID: or-the-direct-way +:END: +#+begin_src sh +# Run +cd Bindings && go build -buildmode=c-shared -o p2prc.so +#+end_src + +**** If successfully built: +:PROPERTIES: +:CUSTOM_ID: if-successfully-built +:END: +#+begin_src sh +# Enter into the Bindings directory +cd Bindings +# List files +ls +# Find files +p2prc.h p2prc.so +#+end_src + +** Workings under the hood +:PROPERTIES: +:CUSTOM_ID: workings-under-the-hood +:END: +Below are a sample set of commands to open the bindings implementation. + +#+begin_example +# run +cd Bindings/ +# list files +ls +# search for file +Client.go +#+end_example + +*** In Client go +:PROPERTIES: +:CUSTOM_ID: in-client-go +:END: +There a few things to notice which are different from your standard Go +programs: + +**** 1. We import "C" which means [[https://pkg.go.dev/cmd/cgo][Cgo]] is required. +:PROPERTIES: +:CUSTOM_ID: we-import-c-which-means-cgo-is-required. +:END: +#+begin_src go +import "C" +#+end_src + +**** 2. All functions which are required to be called from other programming languages have comment such as. +:PROPERTIES: +:CUSTOM_ID: all-functions-which-are-required-to-be-called-from-other-programming-languages-have-comment-such-as. +:END: +#+begin_src go +//export + +// ------------ Example ---------------- +// The function below allows to externally +// to call the P2PRC function to start containers +// in a specific node in the know list of nodes +// in the p2p network. +// Note: the comment "//export StartContainer". + +//export StartContainer +func StartContainer(IP string) (output *C.char) { + container, err := client.StartContainer(IP, 0, false, "", "") + if err != nil { + return C.CString(err.Error()) + } + return ConvertStructToJSONString(container) + } +#+end_src + +**** 3. While looking through the file (If 2 files are compared it is pretty trivial to notice a common structure). +:PROPERTIES: +:CUSTOM_ID: while-looking-through-the-file-if-2-files-are-compared-it-is-pretty-trivial-to-notice-a-common-structure. +:END: +#+begin_src go +// --------- Example ------------ + +//export StartContainer +func StartContainer(IP string) (output *C.char) { + container, err := client.StartContainer(IP, 0, false, "", "") + if err != nil { + return C.CString(err.Error()) + } + return ConvertStructToJSONString(container) +} + +//export ViewPlugin +func ViewPlugin() (output *C.char) { + plugins, err := plugin.DetectPlugins() + if err != nil { + return C.CString(err.Error()) + } + return ConvertStructToJSONString(plugins) +} +#+end_src + +**** It is easy to notice that: +:PROPERTIES: +:CUSTOM_ID: it-is-easy-to-notice-that +:END: +- =ConvertStructToJSONString()=: This is a helper function + that convert a go object to JSON string initially and converts it to + =CString=. +- =(output *C.char)=: This is the return type for most of the functions. + +**** A Pseudo code to refer to the common function implementation shape could be represented as: +:PROPERTIES: +:CUSTOM_ID: a-pseudo-code-to-refer-to-the-common-function-implementation-shape-could-be-represented-as +:END: +#+begin_example +func (output *C.char) { + , := () + if != nil { + return C.CString(.Error()) + } + return ConvertStructToJSONString() +} +#+end_example + +** Current languages supported +:PROPERTIES: +:CUSTOM_ID: current-languages-supported +:END: +- Python + +*** Build sample python program +:PROPERTIES: +:CUSTOM_ID: build-sample-python-program +:END: +The easier way + +#+begin_src sh +# Run +make python +# Expected ouput +Output is in the Directory Bindings/python/export/ +# Run +cd Bindings/python/export/ +# list files +ls +# Expected output +SharedObjects/ p2prc.py +#+end_src + +Above shows a generated folder which consists of a folder called +"SharedObjects/" which consists of =p2prc.so= and =p2prc.h= files. +=p2prc.py= refers to a sample python script calling P2PRC go functions. +To start an any project to extend P2PRC with python, This generated +folder can copied and created as a new git repo for P2PRC extensions +scripted or used a reference point as proof of concept that P2PRC can be +called from other programming languages. + + +* Config Implementation +:PROPERTIES: +:CUSTOM_ID: config-implementation +:END: +The configuration module is responsible to store basic information of +absolute paths of files being called in the Go code. In a full-fledged +Cli the configuration file can be found in the directory /etc/ and from +there points to location such as where the IP table file is located. In +the future implementation the config file will have information such as +number of hops and other parameters to tweak and to improve the +effectiveness of the peer to peer network. The configuration module was +implemented using the library Viper. The Viper library automates +features such as searching in default paths to find out if the +configuration file is present. If the configuration file is not present +in the default paths then it auto generates the configuration file. The +configurations file can be in any format. In this project the +configuration file was generated using JSON format. + +#+begin_src json +{ + "MachineName": "pc-74-120.customer.ask4.lan", + "IPTable": "/Users/akilan/Documents/p2p-rendering-computation/p2p/iptable/ip_table.json", + "DockerContainers": "/Users/akilan/Documents/p2p-rendering-computation/server/docker/containers/", + "DefaultDockerFile": "/Users/akilan/Documents/p2p-rendering-computation/server/docker/containers/docker-ubuntu-sshd/", + "SpeedTestFile": "/Users/akilan/Documents/p2p-rendering-computation/p2p/50.bin", + "IPV6Address": "", + "PluginPath": "/Users/akilan/Documents/p2p-rendering-computation/plugin/deploy", + "TrackContainersPath": "/Users/akilan/Documents/p2p-rendering-computation/client/trackcontainers/trackcontainers.json", + "ServerPort": "8088", + "GroupTrackContainersPath": "/Users/akilan/Documents/p2p-rendering-computation/client/trackcontainers/grouptrackcontainers.json", + "FRPServerPort": "True", + "BehindNAT": "True", + "CustomConfig": null +} +#+end_src + +* Abstractions +:PROPERTIES: +:CUSTOM_ID: abstractions +:END: + +The Abstractions package consists of black-boxed functions for P2PRC. + +** Functions +:PROPERTIES: +:CUSTOM_ID: functions +:END: +- =Init()=: Initializes P2PRC with all the needed + configurations. +- =Start()=: Starts p2prc as a server and makes it possible to extend by + adding other routes and functionality to P2PRC. +- =MapPort()=: On the local machine the port you want to export + to world. +- =StartContainer()=: The machine on the p2p network where + you want to spin up a docker container. +- =RemoveContainer(,)=: Terminate container + based on the IP address and container name. +- =GetSpecs()=: Get specs of a machine on the network based + on the IP address. +- =ViewIPTable()=: View the IP table which about nodes in the network. +- =UpdateIPTable()=: Force update IP table to learn about new nodes + faster. + +* NAT Traversal +:PROPERTIES: +:CUSTOM_ID: nat-traversal +:END: +P2PRC currently supports TURN for NAT traversal. + +** TURN +:PROPERTIES: +:CUSTOM_ID: turn +:END: +The current TURN implementation used is FRP. The TURN server is also +required when a P2PRC node is acting as a Server. The TURN server is +determined based on the Node with the least amount of latency based on +the Nodes available on the IPTable. Once a TURN server is determined +there are 2 actions performed. The first one is =/FRPPort= to the TURN +server to receive a port which is used to generate the external port +from the TURN server. The flow below describes the workflow. + +* Client mode +:PROPERTIES: +:CUSTOM_ID: client-mode +:END: +- Call =/FRPPort= + +#+begin_example +http://:/FRPport +#+end_example + +- Call the TURN server in the following manner. The following is a + sample code snippet below. + +#+begin_src go +import ( + "github.com/Akilan1999/p2p-rendering-computation/p2p/frp" +) + +func main() { + serverPort, err := frp.GetFRPServerPort("http://" + + ":" + lowestLatencyIpAddress.ServerPort) + if err != nil { + return nil, err + } + // Create 1 second delay to allow FRP server to start + time.Sleep(1 * time.Second) + // Starts FRP as a client with + proxyPort, err := frp.StartFRPClientForServer(, serverPort, ) + if err != nil { + return nil, err + } +} +#+end_src + + + + + +* Ideas for future potencial features +Consists of personal loideas for the future of P2PRC. +At moment only has main contributors writiing to this. + +** To support hetrogenous set of Nodes that cannot run P2PRC +This stems from a personal issue I have when doing research +on [[https://github.com/CTSRD-CHERI/cheribsd][CheriBSD]] kernel. For my research I am using the ARM morello +which is a 128bit ARMv8 processor. At the moment Go programs +can cannot compile and run inside the CPU. This means I cannot +run P2PRC at the moment inside the ARM morello to remotely access +it when it's behind NAT using P2PRC. This would indeed be a common +problem when running against various Architectures that do not +support running P2PRC. As you will see soon this also creates +oppurtunity space to scale faster to nodes in a local network +and would introduce a new layer fault tolerance within a local +network nodes. + +*** Assumptions: + - I have a Morello board that cannot run P2PRC + - The Morello has a local IP address (ex: 192.168.0.10) + - I have 2 laptops running P2PRC in that local network. + - This means I have 2 ways to access the Morello board: Which is to SSH + into either 2 laptops and then SSH into 192.168.0.10 to gain access + to the board. Wouldn't it be great to automate this whole layer and + as well look into custom tasks into the hetrogenous hardware. +*** Set of interesting possible: + We build a cool set possibilities before and use this to build up the implementation + plan. + - We can use P2PRC access the morello board remotely in a single command. + - We can use the P2PRC protocol to run servers inside the morello board via traversed + node locally which can access that Node. + - Spin servers on node not running P2PRC using the P2PRC standard abstractions. + - Auto-setup P2PRC nodes with just SSH access via potencially a DSL. + - A neat use case for CHERI for instance would be use the architecture to run light + weight hypervisors. +*** Implementation + - To use implementations similar to [[https://linux.die.net/man/1/socat][socat]] to ensure we can bind address of local + nodes to a node running P2PRC and the node running P2PRC can do a local map port. + - We are working on hardening the implementation of the --mp (Map port) to even + map ports to machines which remotely running P2PRC. This means of instance I + can issue a command to the Morello board without the morello board being in + my local network. + - We would want to implement the exsisting P2PRC public key mechanism as well so that + other nodes can access the Morello board who have permission access. + +#+CAPTION: Implementation idea (To be improved upon) + [[./images/P2PRCRemoteNodes.png]] diff --git a/Docs/images/P2PRCRemoteNodes.png b/Docs/images/P2PRCRemoteNodes.png new file mode 100644 index 0000000..d19e3e6 Binary files /dev/null and b/Docs/images/P2PRCRemoteNodes.png differ diff --git a/Docs/style.css b/Docs/style.css new file mode 100644 index 0000000..8e9554d --- /dev/null +++ b/Docs/style.css @@ -0,0 +1,25 @@ +#table-of-contents h2{ + z-index: 200; + background-color: #b96c29; + text-align: center; + padding: 0.809em; + display: block; + color: #fcfcfc; + font-size: 100%; + margin-bottom: 0.809em; +} + +#search-results li { + background-color: #b96c29; + color: white; + margin-bottom: 5px; + padding: 5px; + border-radius: 4px; + cursor: pointer; + font-size: 14px; +} + +h4,h5,h6{ + color: #b96c29; + font-weight:300; +} diff --git a/README.md b/README.md index 2dcacbf..7ef4c35 100644 --- a/README.md +++ b/README.md @@ -20,33 +20,56 @@ GoDoc reference example -The main aim of this project was to create a custom peer to peer network. The user acting as the + + + +This project aims to simplify self-hosting servers and streamline the creation of distributed systems. The primary focus is to enable users to design custom peer-to-peer networks for distributed computing, offering full flexibility and control while abstracting away the complexities of networking. + +## Key Features +Simplified Self-Hosting +Empower users to easily host nodes, whether on personal computers behind NAT, custom hardware, or cloud-based infrastructure. + +### Custom Peer-to-Peer Networks +Build a decentralised network tailored to user requirements, enabling distributed task execution without the need for in-depth networking knowledge. + +### Heterogeneous Node Support +Harness a diverse array of nodes from personal computers to specialised hardware—seamlessly integrated into a global node list. + +### Abstracted Networking Layer +Networking details are completely abstracted, enabling users to focus on developing bespoke task orchestration systems. No more manual configuration of IPs, ports, or connection logic. + +### Flexible Task Distribution +Users acting as clients retain full control over how tasks are batched and assigned to nodes, creating endless possibilities for optimising performance based on specific use cases. ## Latest tutorial [![IMAGE ALT TEXT](https://i.ytimg.com/vi/OMwCpedu5cs/hqdefault.jpg)](https://www.youtube.com/watch?v=OMwCpedu5cs")
-## Table of contents in the current README -1. [Introduction](#Introduction) + +
-# Table of contents in the Docs folder + -4. [Implementation](Docs/Implementation.md) + 3. [Server Module](ServerArchitecture.md) +4. [Implementation](Docs/Implementation.md) 1. [Client Module](Docs/ClientImplementation.md) 2. [P2P Module](Docs/P2PImplementation.md) 3. [Server Module](Docs/ServerImplementation.md) @@ -57,10 +80,11 @@ flexibility on tracking the container's usages and killing the containers at any 8. [Domain name mappings](Docs/Bindings.md) 5. Language bindings 1. [Haskell](Docs/haskell/) - +5. [Problems](https://github.com/Akilan1999/p2p-rendering-computation/issues) -->
+ -[Read more on the introduction](Docs/Introduction.md) + -
+ -## Extend your application with P2PRC +## Extend your application with P2PRC ```go package main @@ -105,7 +129,7 @@ func main() { ``` -### Export once this is added export P2PRC as environment paths + ## Find out more As we are working on the open source project p2prc (i.e p2p network designed for computation).If you are interested in participating as a contributor -or just providing feedback on new features to build or even just curious about new features added to the project. We have decided to create a discord group. -[![Support Server](https://discordapp.com/api/guilds/854397492795277322/widget.png?style=banner2)](https://discord.gg/b4nRGTjYqy) +or just providing feedback on new features to build or even just curious about new features added to the project do file an Issue or send an email to me@akilan.io. [![Star History Chart](https://api.star-history.com/svg?repos=Akilan1999/p2p-rendering-computation&type=Date)](https://github.com/Gaurav-Gosain) diff --git a/README.md.org b/README.md.org new file mode 100644 index 0000000..c282bc8 --- /dev/null +++ b/README.md.org @@ -0,0 +1,252 @@ +#+begin_quote +[!TIP] Haskell bindings supported!: +[[https://p2prc.akilan.io/Docs/haskell/P2PRC.html][Bindings +documentaton]] + +#+end_quote + +#+begin_quote +[!NOTE] Fixing documentation to latest changes. If you have any +questions setting up P2PRC either +[[https://github.com/Akilan1999/p2p-rendering-computation/issues/new/choose][create +an issue]] or send me an email (me AT akilan dot io). Currently HEAD is +always intended to stay on a working state. It is recommended to always +use HEAD in your go.mod file. + +#+end_quote + +#+begin_html +

+#+end_html + +#+begin_html +

+#+end_html + +#+begin_html + +#+end_html + +#+begin_html + +#+end_html + +#+begin_html + +#+end_html + +This project aims to simplify self-hosting servers and streamline the +creation of distributed systems. The primary focus is to enable users to +design custom peer-to-peer networks for distributed computing, offering +full flexibility and control while abstracting away the complexities of +networking. + +** Key Features +:PROPERTIES: +:CUSTOM_ID: key-features +:END: +Simplified Self-Hosting Empower users to easily host nodes, whether on +personal computers behind NAT, custom hardware, or cloud-based +infrastructure. + +*** Custom Peer-to-Peer Networks +:PROPERTIES: +:CUSTOM_ID: custom-peer-to-peer-networks +:END: +Build a decentralised network tailored to user requirements, enabling +distributed task execution without the need for in-depth networking +knowledge. + +*** Heterogeneous Node Support +:PROPERTIES: +:CUSTOM_ID: heterogeneous-node-support +:END: +Harness a diverse array of nodes from personal computers to specialised +hardware---seamlessly integrated into a global node list. + +*** Abstracted Networking Layer +:PROPERTIES: +:CUSTOM_ID: abstracted-networking-layer +:END: +Networking details are completely abstracted, enabling users to focus on +developing bespoke task orchestration systems. No more manual +configuration of IPs, ports, or connection logic. + +*** Flexible Task Distribution +:PROPERTIES: +:CUSTOM_ID: flexible-task-distribution +:END: +Users acting as clients retain full control over how tasks are batched +and assigned to nodes, creating endless possibilities for optimising +performance based on specific use cases. + +** Latest tutorial +:PROPERTIES: +:CUSTOM_ID: latest-tutorial +:END: +[[https://www.youtube.com/watch?v=OMwCpedu5cs%22][[[https://i.ytimg.com/vi/OMwCpedu5cs/hqdefault.jpg]]]] + +#+begin_html + +#+end_html + +#+begin_html + +#+end_html + +#+begin_html + +#+end_html + +#+begin_html + +#+end_html + +#+begin_html + +#+end_html + +#+begin_html + +#+end_html + +** Extend your application with P2PRC +:PROPERTIES: +:CUSTOM_ID: extend-your-application-with-p2prc +:END: +#+begin_src go +package main + +import ( + "fmt" + "github.com/Akilan1999/p2p-rendering-computation/abstractions" +) + +func main() { + _, err := abstractions.Init(nil) + if err != nil { + fmt.Println(err) + return + } + + // start p2prc + _, err = abstractions.Start() + if err != nil { + fmt.Println(err) + return + } + + // Run server till termination + for { + + } +} +#+end_src + +#+begin_html + +#+end_html + +** Find out more +:PROPERTIES: +:CUSTOM_ID: find-out-more +:END: +As we are working on the open source project p2prc (i.e p2p network +designed for computation).If you are interested in participating as a +contributor or just providing feedback on new features to build or even +just curious about new features added to the project do file an Issue or +send an email to me@akilan.io. + +[[https://github.com/Gaurav-Gosain][[[https://api.star-history.com/svg?repos=Akilan1999/p2p-rendering-computation&type=Date]]]] diff --git a/abstractions/docs.md b/abstractions/docs.md new file mode 100644 index 0000000..7afcfae --- /dev/null +++ b/abstractions/docs.md @@ -0,0 +1,93 @@ + + +# abstractions + +```go +import "github.com/Akilan1999/p2p-rendering-computation/abstractions" +``` + +## Index + +- [func GetSpecs\(IP string\) \(specs \*server.SysInfo, err error\)](<#GetSpecs>) +- [func Init\(customConfig interface\{\}\) \(config \*Config.Config, err error\)](<#Init>) +- [func MapPort\(port string, domainName string, serverAddress string\) \(entireAddress string, mapPort string, err error\)](<#MapPort>) +- [func RemoveContainer\(IP string, ID string\) error](<#RemoveContainer>) +- [func Start\(\) \(\*gin.Engine, error\)](<#Start>) +- [func StartContainer\(IP string\) \(container \*docker.DockerVM, err error\)](<#StartContainer>) +- [func UpdateIPTable\(\) \(err error\)](<#UpdateIPTable>) +- [func ViewIPTable\(\) \(table \*p2p.IpAddresses, err error\)](<#ViewIPTable>) + + + +## func [GetSpecs]() + +```go +func GetSpecs(IP string) (specs *server.SysInfo, err error) +``` + +GetSpecs Get spec information about the remote server + + +## func [Init]() + +```go +func Init(customConfig interface{}) (config *Config.Config, err error) +``` + +Init Initialises p2prc + + +## func [MapPort]() + +```go +func MapPort(port string, domainName string, serverAddress string) (entireAddress string, mapPort string, err error) +``` + +MapPort Creates a reverse proxy connection and maps the appropriate port + + +## func [RemoveContainer]() + +```go +func RemoveContainer(IP string, ID string) error +``` + +RemoveContainer Removes docker container based on the IP address and ID provided + + +## func [Start]() + +```go +func Start() (*gin.Engine, error) +``` + +Start p2prc in a server mode + + +## func [StartContainer]() + +```go +func StartContainer(IP string) (container *docker.DockerVM, err error) +``` + +StartContainer Starts docker container on the remote machine + + +## func [UpdateIPTable]() + +```go +func UpdateIPTable() (err error) +``` + +UpdateIPTable Force updates IP tables based on new new nodes discovered in the network + + +## func [ViewIPTable]() + +```go +func ViewIPTable() (table *p2p.IpAddresses, err error) +``` + +ViewIPTable View information of nodes in the network + +Generated by [gomarkdoc]() diff --git a/client/clientIPTable/docs.md b/client/clientIPTable/docs.md new file mode 100644 index 0000000..7f9d173 --- /dev/null +++ b/client/clientIPTable/docs.md @@ -0,0 +1,63 @@ + + +# clientIPTable + +```go +import "github.com/Akilan1999/p2p-rendering-computation/client/clientIPTable" +``` + +## Index + +- [func AddCustomInformationToIPTable\(text string\) error](<#AddCustomInformationToIPTable>) +- [func RemoveOfflineNodes\(\) error](<#RemoveOfflineNodes>) +- [func UpdateIpTable\(IpAddress string, serverPort string, wg \*sync.WaitGroup\) error](<#UpdateIpTable>) +- [func UpdateIpTableListClient\(\) error](<#UpdateIpTableListClient>) +- [func UploadMultipartFile\(client http.Client, uri, key, path string\) \(\[\]byte, error\)](<#UploadMultipartFile>) + + + +## func [AddCustomInformationToIPTable]() + +```go +func AddCustomInformationToIPTable(text string) error +``` + + + + +## func [RemoveOfflineNodes]() + +```go +func RemoveOfflineNodes() error +``` + + + + +## func [UpdateIpTable]() + +```go +func UpdateIpTable(IpAddress string, serverPort string, wg *sync.WaitGroup) error +``` + +UpdateIpTable Does the following to update it's IP table + + +## func [UpdateIpTableListClient]() + +```go +func UpdateIpTableListClient() error +``` + +UpdateIpTableListClient updates IP tables \(Default 3 hops\) based on server information available on the ip tables + + +## func [UploadMultipartFile]() + +```go +func UploadMultipartFile(client http.Client, uri, key, path string) ([]byte, error) +``` + + + +Generated by [gomarkdoc]() diff --git a/client/docs.md b/client/docs.md new file mode 100644 index 0000000..cbce893 --- /dev/null +++ b/client/docs.md @@ -0,0 +1,357 @@ + + +# client + +```go +import "github.com/Akilan1999/p2p-rendering-computation/client" +``` + +## Index + +- [func AddTrackContainer\(d \*docker.DockerVM, ipAddress string\) error](<#AddTrackContainer>) +- [func CheckID\(ID string\) \(string, error\)](<#CheckID>) +- [func GetServerPort\(IpAddress string\) \(string, error\)](<#GetServerPort>) +- [func GetSpecs\(IP string\) \(\*server.SysInfo, error\)](<#GetSpecs>) +- [func PrettyPrint\(data interface\{\}\)](<#PrettyPrint>) +- [func RemoveContainerGroups\(ContainerID string\) error](<#RemoveContainerGroups>) +- [func RemoveContianer\(IP string, ID string\) error](<#RemoveContianer>) +- [func RemoveGroup\(GroupID string\) error](<#RemoveGroup>) +- [func RemoveTrackedContainer\(id string\) error](<#RemoveTrackedContainer>) +- [func StartContainer\(IP string, NumPorts int, GPU bool, ContainerName string, baseImage string\) \(\*docker.DockerVM, error\)](<#StartContainer>) +- [func ViewContainers\(IP string\) \(\*docker.DockerContainers, error\)](<#ViewContainers>) +- [type Group](<#Group>) + - [func AddContainerToGroup\(ContainerID string, GroupID string\) \(\*Group, error\)](<#AddContainerToGroup>) + - [func CreateGroup\(\) \(\*Group, error\)](<#CreateGroup>) + - [func GetGroup\(GroupID string\) \(\*Group, error\)](<#GetGroup>) + - [func RemoveContainerGroup\(ContainerID string, GroupID string\) \(\*Group, error\)](<#RemoveContainerGroup>) + - [func \(grp \*Group\) AddContainer\(Container \*TrackContainer\) error](<#Group.AddContainer>) + - [func \(grp \*Group\) AddGroupToFile\(\) error](<#Group.AddGroupToFile>) + - [func \(grp \*Group\) RemoveContainerGroup\(Container \*TrackContainer\) error](<#Group.RemoveContainerGroup>) +- [type Groups](<#Groups>) + - [func ReadGroup\(\) \(\*Groups, error\)](<#ReadGroup>) + - [func \(grp \*Groups\) RemoveContainerGroups\(Container \*TrackContainer\) error](<#Groups.RemoveContainerGroups>) + - [func \(grp \*Groups\) WriteGroup\(\) error](<#Groups.WriteGroup>) +- [type ResponseMAPPort](<#ResponseMAPPort>) + - [func MAPPort\(port string, domainName string\) \(\*ResponseMAPPort, error\)](<#MAPPort>) +- [type TrackContainer](<#TrackContainer>) + - [func GetContainerInformation\(ID string\) \(\*TrackContainer, error\)](<#GetContainerInformation>) + - [func \(TC \*TrackContainer\) ModifyContainerGroups\(\) error](<#TrackContainer.ModifyContainerGroups>) + - [func \(TC \*TrackContainer\) ModifyContainerInformation\(\) error](<#TrackContainer.ModifyContainerInformation>) +- [type TrackContainers](<#TrackContainers>) + - [func ReadTrackContainers\(filename string\) \(\*TrackContainers, error\)](<#ReadTrackContainers>) + - [func ViewTrackedContainers\(\) \(error, \*TrackContainers\)](<#ViewTrackedContainers>) + - [func \(TC \*TrackContainers\) WriteContainers\(\) error](<#TrackContainers.WriteContainers>) + + + +## func [AddTrackContainer]() + +```go +func AddTrackContainer(d *docker.DockerVM, ipAddress string) error +``` + +AddTrackContainer Adds new container which has been added to the track container + + +## func [CheckID]() + +```go +func CheckID(ID string) (string, error) +``` + +CheckID Checks if the ID belongs to a group or a single container + + +## func [GetServerPort]() + +```go +func GetServerPort(IpAddress string) (string, error) +``` + +GetServerPort Helper function to do find out server port information + + +## func [GetSpecs]() + +```go +func GetSpecs(IP string) (*server.SysInfo, error) +``` + +GetSpecs Gets Specs from the server such CPU, GPU usage and other basic information which helps set a cluster of computer + + +## func [PrettyPrint]() + +```go +func PrettyPrint(data interface{}) +``` + +PrettyPrint print the contents of the obj \( Reference: https://stackoverflow.com/questions/24512112/how-to-print-struct-variables-in-console + + +## func [RemoveContainerGroups]() + +```go +func RemoveContainerGroups(ContainerID string) error +``` + +RemoveContainerGroups Remove Container from groups \(i.e which ever groups has information about that container\). This is mostly called when a container is deleted or removed from the tracked container list + + +## func [RemoveContianer]() + +```go +func RemoveContianer(IP string, ID string) error +``` + +RemoveContianer Stops and removes container from the server + + +## func [RemoveGroup]() + +```go +func RemoveGroup(GroupID string) error +``` + +RemoveGroup Removes group based on the group ID provided + + +## func [RemoveTrackedContainer]() + +```go +func RemoveTrackedContainer(id string) error +``` + +RemoveTrackedContainer This function removos tracked container from the trackcontainer JSON file + + +## func [StartContainer]() + +```go +func StartContainer(IP string, NumPorts int, GPU bool, ContainerName string, baseImage string) (*docker.DockerVM, error) +``` + +StartContainer Start container using REST api Implementation From the selected server IP address TODO: Test cases for this function Calls URL ex: http://0.0.0.0:8088/startcontainer?ports=0&GPU=false&ContainerName=docker-ubuntu-sshd + + +## func [ViewContainers]() + +```go +func ViewContainers(IP string) (*docker.DockerContainers, error) +``` + +ViewContainers This function displays all containers available on server side + + +## type [Group]() + +Group Information about a single group + +```go +type Group struct { + ID string `json:"ID"` + TrackContainerList []*TrackContainer `json:"TrackContainer"` + // contains filtered or unexported fields +} +``` + + +### func [AddContainerToGroup]() + +```go +func AddContainerToGroup(ContainerID string, GroupID string) (*Group, error) +``` + +AddContainerToGroup Adds container information to the Group based on the Group ID + + +### func [CreateGroup]() + +```go +func CreateGroup() (*Group, error) +``` + +CreateGroup Creates a new group to add a set of track containers + + +### func [GetGroup]() + +```go +func GetGroup(GroupID string) (*Group, error) +``` + +GetGroup Gets group information based on group id provided + + +### func [RemoveContainerGroup]() + +```go +func RemoveContainerGroup(ContainerID string, GroupID string) (*Group, error) +``` + +RemoveContainerGroup Remove Container from the group ID specified + + +### func \(\*Group\) [AddContainer]() + +```go +func (grp *Group) AddContainer(Container *TrackContainer) error +``` + +AddContainer Adds a container to the Tracked container list of the group + + +### func \(\*Group\) [AddGroupToFile]() + +```go +func (grp *Group) AddGroupToFile() error +``` + +AddGroupToFile Adds Group struct to the GroupTrackContainer File + + +### func \(\*Group\) [RemoveContainerGroup]() + +```go +func (grp *Group) RemoveContainerGroup(Container *TrackContainer) error +``` + +RemoveContainerGroup Removes container information from the group + + +## type [Groups]() + +Groups Data Structure type + +```go +type Groups struct { + GroupList []*Group `json:"Groups"` +} +``` + + +### func [ReadGroup]() + +```go +func ReadGroup() (*Groups, error) +``` + +ReadGroup Function reads grouptrackcontainers.json and converts result to Groups + + +### func \(\*Groups\) [RemoveContainerGroups]() + +```go +func (grp *Groups) RemoveContainerGroups(Container *TrackContainer) error +``` + +RemoveContainerGroups removes container found in all groups + + +### func \(\*Groups\) [WriteGroup]() + +```go +func (grp *Groups) WriteGroup() error +``` + +WriteGroup Function to write type Groups to the grouptrackcontainers.json file + + +## type [ResponseMAPPort]() + + + +```go +type ResponseMAPPort struct { + IPAddress string +} +``` + + +### func [MAPPort]() + +```go +func MAPPort(port string, domainName string) (*ResponseMAPPort, error) +``` + + + + +## type [TrackContainer]() + +TrackContainer Stores information of current containers + +```go +type TrackContainer struct { + Id string `json:"ID"` + Container *docker.DockerVM `json:"Container"` + IpAddress string `json:"IpAddress"` +} +``` + + +### func [GetContainerInformation]() + +```go +func GetContainerInformation(ID string) (*TrackContainer, error) +``` + +GetContainerInformation gets information about container based on container ID provided + + +### func \(\*TrackContainer\) [ModifyContainerGroups]() + +```go +func (TC *TrackContainer) ModifyContainerGroups() error +``` + +ModifyContainerGroups Modifies container information is all groups available + + +### func \(\*TrackContainer\) [ModifyContainerInformation]() + +```go +func (TC *TrackContainer) ModifyContainerInformation() error +``` + +ModifyContainerInformation Modifies information inside the container + + +## type [TrackContainers]() + +TrackContainers This struct stores arrays of current containers running + +```go +type TrackContainers struct { + TrackContainerList []TrackContainer `json:"TrackContainer"` +} +``` + + +### func [ReadTrackContainers]() + +```go +func ReadTrackContainers(filename string) (*TrackContainers, error) +``` + +ReadTrackContainers Reads containers which are currently tracked + + +### func [ViewTrackedContainers]() + +```go +func ViewTrackedContainers() (error, *TrackContainers) +``` + +ViewTrackedContainers View Containers currently tracked + + +### func \(\*TrackContainers\) [WriteContainers]() + +```go +func (TC *TrackContainers) WriteContainers() error +``` + +WriteContainers Write information back to the config file + +Generated by [gomarkdoc]() diff --git a/cmd/docs.md b/cmd/docs.md new file mode 100644 index 0000000..bf0d990 --- /dev/null +++ b/cmd/docs.md @@ -0,0 +1,523 @@ + + +# cmd + +```go +import "github.com/Akilan1999/p2p-rendering-computation/cmd" +``` + +## Index + +- [Variables](<#variables>) + + +## Variables + +Variables declared for CLI + +```go +var ( + AddServer string + ViewImages string + CreateVM string + ContainerName string + BaseImage string + Ports string + Server bool + RemoveVM string + ID string + Specs string + GPU bool + UpdateServerList bool + ServerList bool + SetDefaultConfig bool + NetworkInterface bool + ViewPlugin bool + TrackedContainers bool + ExecutePlugin string + CreateGroup bool + Group string + Groups bool + RemoveContainerGroup bool + RemoveGroup string + MAPPort string + DomainName string + //FRPProxy bool + // Generate only allowed in dev release + // -- REMOVE ON REGULAR RELEASE -- + Generate string + Modulename string + //-------------------------------- + PullPlugin string + RemovePlugin string + AddMetaData string +) +``` + + + +```go +var AppConfigFlags = []cli.Flag{ + + &cli.BoolFlag{ + Name: "Server", + Aliases: []string{"s"}, + Usage: "Starts server", + EnvVars: []string{"SERVER"}, + Destination: &Server, + }, + &cli.BoolFlag{ + Name: "UpdateServerList", + Aliases: []string{"us"}, + Usage: "Update List of Server available based on servers iptables", + EnvVars: []string{"UPDATE_SERVER_LIST"}, + Destination: &UpdateServerList, + }, + &cli.BoolFlag{ + Name: "ListServers", + Aliases: []string{"ls"}, + Usage: "List servers which can render tasks", + EnvVars: []string{"LIST_SERVERS"}, + Destination: &ServerList, + }, + &cli.StringFlag{ + Name: "AddServer", + Aliases: []string{"as"}, + Usage: "Adds server IP address to iptables", + EnvVars: []string{"ADD_SERVER"}, + Destination: &AddServer, + }, + &cli.StringFlag{ + Name: "ViewImages", + Aliases: []string{"vi"}, + Usage: "View images available on the server IP address", + EnvVars: []string{"VIEW_IMAGES"}, + Destination: &ViewImages, + }, + &cli.StringFlag{ + Name: "CreateVM", + Aliases: []string{"touch"}, + Usage: "Creates Docker container on the selected server", + EnvVars: []string{"CREATE_VM"}, + Destination: &CreateVM, + }, + &cli.StringFlag{ + Name: "ContainerName", + Aliases: []string{"cn"}, + Usage: "Specifying the container run on the server side", + EnvVars: []string{"CONTAINER_NAME"}, + Destination: &ContainerName, + }, + &cli.StringFlag{ + Name: "BaseImage", + Aliases: []string{"bi"}, + Usage: "Specifying the docker base image to template the dockerfile", + EnvVars: []string{"CONTAINER_NAME"}, + Destination: &BaseImage, + }, + &cli.StringFlag{ + Name: "RemoveVM", + Aliases: []string{"rm"}, + Usage: "Stop and Remove Docker container (IP:port) accompanied by container ID via --ID or --id", + EnvVars: []string{"REMOVE_VM"}, + Destination: &RemoveVM, + }, + &cli.StringFlag{ + Name: "ID", + Aliases: []string{"id"}, + Usage: "Docker Container ID", + EnvVars: []string{"ID"}, + Destination: &ID, + }, + &cli.StringFlag{ + Name: "Ports", + Aliases: []string{"p"}, + Usage: "Number of ports to open for the Docker Container", + EnvVars: []string{"NUM_PORTS"}, + Destination: &Ports, + }, + &cli.BoolFlag{ + Name: "GPU", + Aliases: []string{"gpu"}, + Usage: "Create Docker Containers to access GPU", + EnvVars: []string{"USE_GPU"}, + Destination: &GPU, + }, + &cli.StringFlag{ + Name: "Specification", + Aliases: []string{"specs"}, + Usage: "Specs of the server node", + EnvVars: []string{"SPECS"}, + Destination: &Specs, + }, + &cli.BoolFlag{ + Name: "SetDefaultConfig", + Aliases: []string{"dc"}, + Usage: "Sets a default configuration file", + EnvVars: []string{"SET_DEFAULT_CONFIG"}, + Destination: &SetDefaultConfig, + }, + &cli.BoolFlag{ + Name: "NetworkInterfaces", + Aliases: []string{"ni"}, + Usage: "Shows the network interface in your computer", + EnvVars: []string{"NETWORK_INTERFACE"}, + Destination: &NetworkInterface, + }, + &cli.BoolFlag{ + Name: "ViewPlugins", + Aliases: []string{"vp"}, + Usage: "Shows plugins available to be executed", + EnvVars: []string{"VIEW_PLUGIN"}, + Destination: &ViewPlugin, + }, + &cli.BoolFlag{ + Name: "TrackedContainers", + Aliases: []string{"tc"}, + Usage: "View (currently running) containers which have " + + "been created from the client side ", + EnvVars: []string{"TRACKED_CONTAINERS"}, + Destination: &TrackedContainers, + }, + &cli.StringFlag{ + Name: "ExecutePlugin", + Aliases: []string{"plugin"}, + Usage: "Plugin which needs to be executed", + EnvVars: []string{"EXECUTE_PLUGIN"}, + Destination: &ExecutePlugin, + }, + &cli.BoolFlag{ + Name: "CreateGroup", + Aliases: []string{"cgroup"}, + Usage: "Creates a new group", + EnvVars: []string{"CREATE_GROUP"}, + Destination: &CreateGroup, + }, + &cli.StringFlag{ + Name: "Group", + Aliases: []string{"group"}, + Usage: "group flag with argument group ID", + EnvVars: []string{"GROUP"}, + Destination: &Group, + }, + &cli.BoolFlag{ + Name: "Groups", + Aliases: []string{"groups"}, + Usage: "View all groups", + EnvVars: []string{"GROUPS"}, + Destination: &Groups, + }, + &cli.BoolFlag{ + Name: "RemoveContainerGroup", + Aliases: []string{"rmcgroup"}, + Usage: "Remove specific container in the group", + EnvVars: []string{"REMOVE_CONTAINER_GROUP"}, + Destination: &RemoveContainerGroup, + }, + &cli.StringFlag{ + Name: "RemoveGroup", + Aliases: []string{"rmgroup"}, + Usage: "Removes the entire group", + EnvVars: []string{"REMOVE_GROUP"}, + Destination: &RemoveGroup, + }, + &cli.StringFlag{ + Name: "MAPPort", + Aliases: []string{"mp"}, + Usage: "Maps port for a specific port provided as the parameter", + EnvVars: []string{"MAPPORT"}, + Destination: &MAPPort, + }, + &cli.StringFlag{ + Name: "DomainName", + Aliases: []string{"dn"}, + Usage: "While mapping ports allows to set a domain name to create a mapping in the proxy server", + EnvVars: []string{"DOMAINNAME"}, + Destination: &DomainName, + }, + + &cli.StringFlag{ + Name: "Generate", + Aliases: []string{"gen"}, + Usage: "Generates a new copy of P2PRC which can be modified based on your needs", + EnvVars: []string{"GENERATE"}, + Destination: &Generate, + }, + &cli.StringFlag{ + Name: "ModuleName", + Aliases: []string{"mod"}, + Usage: "New go project module name", + EnvVars: []string{"MODULENAME"}, + Destination: &Modulename, + }, + + &cli.StringFlag{ + Name: "PullPlugin", + Aliases: []string{"pp"}, + Usage: "Pulls plugin from git repos", + EnvVars: []string{"PULLPLUGIN"}, + Destination: &PullPlugin, + }, + &cli.StringFlag{ + Name: "RemovePlugin", + Aliases: []string{"rp"}, + Usage: "Removes plugin", + EnvVars: []string{"REMOVEPLUGIN"}, + Destination: &RemovePlugin, + }, + &cli.StringFlag{ + Name: "AddMetaData", + Aliases: []string{"amd"}, + Usage: "Adds metadata about the current node in the p2p network which is then propagated through the network", + EnvVars: []string{"ADDMETADATA"}, + Destination: &AddMetaData, + }, +} +``` + + + +```go +var CliAction = func(ctx *cli.Context) error { + if Server { + _, err := server.Server() + if err != nil { + fmt.Print(err) + } + + for { + + } + } + + if UpdateServerList { + err := clientIPTable.UpdateIpTableListClient() + if err != nil { + fmt.Print(err) + } + + p2p.PrintIpTable() + } + + if ServerList { + + p2p.PrintIpTable() + } + + if AddServer != "" { + res, err := p2p.ReadIpTable() + if err != nil { + fmt.Println(err) + } + + //Create variable of type IpAddress and set IP address + // to it + var IpAddr p2p.IpAddress + + ip4Orip6 := p2p.Ip4or6(AddServer) + if ip4Orip6 == "version 6" { + IpAddr.Ipv6 = AddServer + } else { + IpAddr.Ipv4 = AddServer + } + + if Ports != "" { + IpAddr.ServerPort = Ports + } else { + IpAddr.ServerPort = "8088" + } + + res.IpAddress = append(res.IpAddress, IpAddr) + + res.WriteIpTable() + + } + + if ViewImages != "" { + imageRes, err := client.ViewContainers(ViewImages) + + if err != nil { + fmt.Print(err) + } + client.PrettyPrint(imageRes) + } + + if RemoveVM != "" { + if ID == "" { + fmt.Println("provide container ID via --ID or --id") + } else { + err := client.RemoveContianer(RemoveVM, ID) + if err != nil { + fmt.Print(err) + } + } + } + + if CreateVM != "" { + + var PortsInt int + + if Ports != "" { + + fmt.Sscanf(Ports, "%d", &PortsInt) + } + + imageRes, err := client.StartContainer(CreateVM, PortsInt, GPU, ContainerName, BaseImage) + + if err != nil { + fmt.Print(err) + } + client.PrettyPrint(imageRes) + } + + if Specs != "" { + specs, err := client.GetSpecs(Specs) + if err != nil { + return err + } + + client.PrettyPrint(specs) + } + + if SetDefaultConfig { + _, err := generate.SetDefaults("P2PRC", false, nil, false) + if err != nil { + fmt.Print(err) + } + } + + if NetworkInterface { + err := p2p.ViewNetworkInterface() + if err != nil { + fmt.Print(err) + } + } + + if ViewPlugin { + plugins, err := plugin.DetectPlugins() + if err != nil { + fmt.Print(err) + } + client.PrettyPrint(plugins) + } + + if TrackedContainers { + err, trackedContainers := client.ViewTrackedContainers() + if err != nil { + fmt.Print(err) + } + client.PrettyPrint(trackedContainers) + } + + if ExecutePlugin != "" { + + if ID != "" { + err := plugin.CheckRunPlugin(ExecutePlugin, ID) + if err != nil { + fmt.Println(err) + } else { + fmt.Println("Success") + } + } else { + fmt.Println("provide container ID via --ID or --id") + } + + } + + if CreateGroup { + group, err := client.CreateGroup() + if err != nil { + return err + } + client.PrettyPrint(group) + } + + if Group != "" { + + if RemoveContainerGroup && ID != "" { + group, err := client.RemoveContainerGroup(ID, Group) + if err != nil { + fmt.Println(err) + } else { + client.PrettyPrint(group) + } + } else if ID != "" { + + group, err := client.AddContainerToGroup(ID, Group) + if err != nil { + fmt.Println(err) + } else { + client.PrettyPrint(group) + } + } else { + group, err := client.GetGroup(Group) + if err != nil { + fmt.Println(err) + } else { + client.PrettyPrint(group) + } + } + } + + if RemoveGroup != "" { + err := client.RemoveGroup(RemoveGroup) + if err != nil { + fmt.Println(err) + } else { + fmt.Println("Group Removed") + } + } + + if Groups { + groups, err := client.ReadGroup() + if err != nil { + fmt.Println(err) + } else { + client.PrettyPrint(groups) + } + } + + if PullPlugin != "" { + err := plugin.DownloadPlugin(PullPlugin) + if err != nil { + fmt.Println(err) + } else { + fmt.Println("Success") + } + } + + if RemovePlugin != "" { + err := plugin.DeletePlugin(RemovePlugin) + if err != nil { + fmt.Println(err) + } else { + fmt.Println("Success") + } + } + + if AddMetaData != "" { + err := clientIPTable.AddCustomInformationToIPTable(AddMetaData) + if err != nil { + fmt.Println(err) + } else { + fmt.Println("Success") + } + } + + if MAPPort != "" { + address, err := client.MAPPort(MAPPort, DomainName) + if err != nil { + return err + } + if err != nil { + fmt.Println(err) + } else { + client.PrettyPrint(address) + } + } + + return nil +} +``` + +Generated by [gomarkdoc]() diff --git a/config/docs.md b/config/docs.md new file mode 100644 index 0000000..85356eb --- /dev/null +++ b/config/docs.md @@ -0,0 +1,116 @@ + + +# config + +```go +import "github.com/Akilan1999/p2p-rendering-computation/config" +``` + +## Index + +- [func GetCurrentPath\(\) \(string, error\)](<#GetCurrentPath>) +- [func GetEnvName\(\) string](<#GetEnvName>) +- [func GetPathP2PRC\(Envname string\) \(string, error\)](<#GetPathP2PRC>) +- [func SetEnvName\(EnvName string\) error](<#SetEnvName>) +- [type Config](<#Config>) + - [func ConfigInit\(defaultsParameter map\[string\]interface\{\}, CustomConfig interface\{\}, envNameOptional ...string\) \(\*Config, error\)](<#ConfigInit>) + - [func \(c \*Config\) GetPublicKey\(\) \(string, error\)](<#Config.GetPublicKey>) + - [func \(c \*Config\) WriteConfig\(\) error](<#Config.WriteConfig>) + + + +## func [GetCurrentPath]() + +```go +func GetCurrentPath() (string, error) +``` + +GetCurrentPath Getting P2PRC Directory from environment variable + + +## func [GetEnvName]() + +```go +func GetEnvName() string +``` + + + + +## func [GetPathP2PRC]() + +```go +func GetPathP2PRC(Envname string) (string, error) +``` + +GetPathP2PRC Getting P2PRC Directory from environment variable + + +## func [SetEnvName]() + +```go +func SetEnvName(EnvName string) error +``` + +SetEnvName Sets the environment name This is to ensure that the Path of your project is detected from your environment variable This is useful when extending the use case of P2PRC + + +## type [Config]() + + + +```go +type Config struct { + MachineName string + IPTable string + DockerContainers string + DefaultDockerFile string + DockerRunLogs string + SpeedTestFile string + IPV6Address string + PluginPath string + TrackContainersPath string + ServerPort string + ProxyPort string + GroupTrackContainersPath string + FRPServerPort string + BehindNAT string + IPTableKey string + PublicKeyFile string + PrivateKeyFile string + PemFile string + KeyFile string + BareMetal bool + UnsafeMode bool + CustomConfig interface{} +} +``` + + +### func [ConfigInit]() + +```go +func ConfigInit(defaultsParameter map[string]interface{}, CustomConfig interface{}, envNameOptional ...string) (*Config, error) +``` + +ConfigInit Pass environment name as an optional parameter + + +### func \(\*Config\) [GetPublicKey]() + +```go +func (c *Config) GetPublicKey() (string, error) +``` + +GetPublicKey Gets public key of the current machine based on the path provided on the config file + + +### func \(\*Config\) [WriteConfig]() + +```go +func (c *Config) WriteConfig() error +``` + + + +Generated by [gomarkdoc]() diff --git a/config/generate/docs.md b/config/generate/docs.md new file mode 100644 index 0000000..e6af169 --- /dev/null +++ b/config/generate/docs.md @@ -0,0 +1,176 @@ + + +# generate + +```go +import "github.com/Akilan1999/p2p-rendering-computation/config/generate" +``` + +## Index + +- [func Copy\(src, dst string\) error](<#Copy>) +- [func CreateIPTableFolderStructure\(\) \(err error\)](<#CreateIPTableFolderStructure>) +- [func FileExists\(path string\) \(bool, error\)](<#FileExists>) +- [func GenerateCertificate\(\) error](<#GenerateCertificate>) +- [func GenerateClientTrackContainers\(\) \(err error\)](<#GenerateClientTrackContainers>) +- [func GenerateDockerFiles\(\) \(err error\)](<#GenerateDockerFiles>) +- [func GenerateFiles\(rootNodes ...p2p.IpAddress\) \(err error\)](<#GenerateFiles>) +- [func GenerateIPTableFile\(rootNodes \[\]p2p.IpAddress\) \(err error\)](<#GenerateIPTableFile>) +- [func GeneratePluginDirectory\(\) \(err error\)](<#GeneratePluginDirectory>) +- [func GetPathP2PRC\(\) \(string, error\)](<#GetPathP2PRC>) +- [func MakeSSHKeyPair\(pubKeyPath, privateKeyPath string\) error](<#MakeSSHKeyPair>) +- [func SetDefaults\(envName string, forceDefault bool, CustomConfig interface\{\}, NoBoilerPlate bool, ConfigUpdate ...\*config.Config\) \(\*config.Config, error\)](<#SetDefaults>) +- [func SetEnvName\(EnvName string\) error](<#SetEnvName>) +- [func String\(length int\) string](<#String>) +- [func StringWithCharset\(length int, charset string\) string](<#StringWithCharset>) +- [type AuthorizedKey](<#AuthorizedKey>) + + + +## func [Copy]() + +```go +func Copy(src, dst string) error +``` + +Copy the src file to dst. Any existing file will be overwritten and will not copy file attributes. Source: https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file + + +## func [CreateIPTableFolderStructure]() + +```go +func CreateIPTableFolderStructure() (err error) +``` + +CreateIPTableFolderStructure Create folder structure for IPTable + + +## func [FileExists]() + +```go +func FileExists(path string) (bool, error) +``` + +FileExists exists returns whether the given file or directory exists source: https://stackoverflow.com/questions/10510691/how-to-check-whether-a-file-or-directory-exists + + +## func [GenerateCertificate]() + +```go +func GenerateCertificate() error +``` + + + + +## func [GenerateClientTrackContainers]() + +```go +func GenerateClientTrackContainers() (err error) +``` + + + + +## func [GenerateDockerFiles]() + +```go +func GenerateDockerFiles() (err error) +``` + +GenerateDockerFiles Generate default docker files + + +## func [GenerateFiles]() + +```go +func GenerateFiles(rootNodes ...p2p.IpAddress) (err error) +``` + +GenerateFiles Generates all the files needed to setup P2PRC + + +## func [GenerateIPTableFile]() + +```go +func GenerateIPTableFile(rootNodes []p2p.IpAddress) (err error) +``` + +GenerateIPTableFile Generates the IPTable file with the appropirate root node + + +## func [GeneratePluginDirectory]() + +```go +func GeneratePluginDirectory() (err error) +``` + +GeneratePluginDirectory Generates plugin directory structure + + +## func [GetPathP2PRC]() + +```go +func GetPathP2PRC() (string, error) +``` + +GetPathP2PRC Getting P2PRC Directory from environment variable + + +## func [MakeSSHKeyPair]() + +```go +func MakeSSHKeyPair(pubKeyPath, privateKeyPath string) error +``` + +MakeSSHKeyPair make a pair of public and private keys for SSH access. Public key is encoded in the format for inclusion in an OpenSSH authorized\_keys file. Private Key generated is PEM encoded source: https://gist.github.com/goliatone/e9c13e5f046e34cef6e150d06f20a34c + + +## func [SetDefaults]() + +```go +func SetDefaults(envName string, forceDefault bool, CustomConfig interface{}, NoBoilerPlate bool, ConfigUpdate ...*config.Config) (*config.Config, error) +``` + +SetDefaults This function to be called only during a make install + + +## func [SetEnvName]() + +```go +func SetEnvName(EnvName string) error +``` + +SetEnvName Sets the environment name This is to ensure that the Path of your project is detected from your environment variable This is useful when extending the use case of P2PRC + + +## func [String]() + +```go +func String(length int) string +``` + + + + +## func [StringWithCharset]() + +```go +func StringWithCharset(length int, charset string) string +``` + + + + +## type [AuthorizedKey]() + +AuthorizedKey struct represents the structure of an authorized key + +```go +type AuthorizedKey struct { + Username string + Key string +} +``` + +Generated by [gomarkdoc]() diff --git a/main.md b/main.md new file mode 100644 index 0000000..72aa90b --- /dev/null +++ b/main.md @@ -0,0 +1,34 @@ + + +# p2p\-rendering\-computation + +```go +import "github.com/Akilan1999/p2p-rendering-computation" +``` + +## Index + +- [Variables](<#variables>) + + +## Variables + + + +```go +var List_servers, Ip_table bool +``` + +Varaibles if mode is client + +```go +var OS, Pull_location, Run_script string +``` + +VERSION specifies the version of the platform + +```go +var VERSION = "2.0.0" +``` + +Generated by [gomarkdoc]() diff --git a/main.md.org b/main.md.org new file mode 100644 index 0000000..606d3f2 --- /dev/null +++ b/main.md.org @@ -0,0 +1,40 @@ +#+begin_html + +#+end_html + +* p2p-rendering-computation +:PROPERTIES: +:CUSTOM_ID: p2p-rendering-computation +:END: +#+begin_src go +import "github.com/Akilan1999/p2p-rendering-computation" +#+end_src + +** Index +:PROPERTIES: +:CUSTOM_ID: index +:END: +- [[#variables][Variables]] + +** Variables +:PROPERTIES: +:CUSTOM_ID: variables +:END: + +#+begin_src go +var List_servers, Ip_table bool +#+end_src + +Varaibles if mode is client + +#+begin_src go +var OS, Pull_location, Run_script string +#+end_src + +VERSION specifies the version of the platform + +#+begin_src go +var VERSION = "2.0.0" +#+end_src + +Generated by [[https://github.com/princjef/gomarkdoc][gomarkdoc]] diff --git a/p2p/docs.md b/p2p/docs.md new file mode 100644 index 0000000..47aabea --- /dev/null +++ b/p2p/docs.md @@ -0,0 +1,303 @@ + + +# p2p + +```go +import "github.com/Akilan1999/p2p-rendering-computation/p2p" +``` + +## Index + +- [func AddAuthorisationKey\(PublicKey string\) error](<#AddAuthorisationKey>) +- [func AddKeyToAuthorizedKeys\(path, newKey string\) error](<#AddKeyToAuthorizedKeys>) +- [func CurrentPublicIP\(\) \(string, error\)](<#CurrentPublicIP>) +- [func ForwardPort\(port int\) error](<#ForwardPort>) +- [func GenerateHashSHA256\(text string\) \[\]byte](<#GenerateHashSHA256>) +- [func GetAuthorizedKeysPath\(\) \(string, error\)](<#GetAuthorizedKeysPath>) +- [func GetCurrentIPV6\(\) \(string, error\)](<#GetCurrentIPV6>) +- [func Ip4or6\(s string\) string](<#Ip4or6>) +- [func LocalSpeedTestIpTable\(\) error](<#LocalSpeedTestIpTable>) +- [func PrettyPrint\(data interface\{\}\)](<#PrettyPrint>) +- [func PrintIpTable\(\) error](<#PrintIpTable>) +- [func ReadAuthorizedKeys\(path string\) \(map\[string\]bool, error\)](<#ReadAuthorizedKeys>) +- [func RemoveKeyFromAuthorizedKeys\(path, keyToRemove string\) error](<#RemoveKeyFromAuthorizedKeys>) +- [func UnForwardPort\(port int\) error](<#UnForwardPort>) +- [func ValidateHashSHA256\(text string, Hash \[\]byte\) bool](<#ValidateHashSHA256>) +- [func ViewNetworkInterface\(\) error](<#ViewNetworkInterface>) +- [type IP](<#IP>) +- [type IpAddress](<#IpAddress>) + - [func \(s \*IpAddress\) DownloadSpeed\(\) error](<#IpAddress.DownloadSpeed>) + - [func \(s \*IpAddress\) PingTest\(\) error](<#IpAddress.PingTest>) + - [func \(s \*IpAddress\) UploadSpeed\(\) error](<#IpAddress.UploadSpeed>) +- [type IpAddresses](<#IpAddresses>) + - [func ReadIpTable\(\) \(\*IpAddresses, error\)](<#ReadIpTable>) + - [func \(table \*IpAddresses\) RemoveDuplicates\(\) error](<#IpAddresses.RemoveDuplicates>) + - [func \(ip \*IpAddresses\) SpeedTest\(\) error](<#IpAddresses.SpeedTest>) + - [func \(ip \*IpAddresses\) SpeedTestUpdatedIPTable\(\) error](<#IpAddresses.SpeedTestUpdatedIPTable>) + - [func \(i \*IpAddresses\) WriteIpTable\(\) error](<#IpAddresses.WriteIpTable>) + + + +## func [AddAuthorisationKey]() + +```go +func AddAuthorisationKey(PublicKey string) error +``` + +AddAuthorisationKey Adds public key provided to the authorization file so that nodes can SSH into the + + +## func [AddKeyToAuthorizedKeys]() + +```go +func AddKeyToAuthorizedKeys(path, newKey string) error +``` + +AddKeyToAuthorizedKeys adds a new key to the authorized\_keys file if it doesn’t already exist + + +## func [CurrentPublicIP]() + +```go +func CurrentPublicIP() (string, error) +``` + +CurrentPublicIP Get Current Public IP address + + +## func [ForwardPort]() + +```go +func ForwardPort(port int) error +``` + +Port forwarding to the router + + +## func [GenerateHashSHA256]() + +```go +func GenerateHashSHA256(text string) []byte +``` + + + + +## func [GetAuthorizedKeysPath]() + +```go +func GetAuthorizedKeysPath() (string, error) +``` + +GetAuthorizedKeysPath returns the path to the authorized\_keys file + + +## func [GetCurrentIPV6]() + +```go +func GetCurrentIPV6() (string, error) +``` + +GetCurrentIPV6 gets the current IPV6 address based on the interface specified in the config file + + +## func [Ip4or6]() + +```go +func Ip4or6(s string) string +``` + +Ip4or6 Helper function to check if the IP address is IPV4 or IPV6 \(https://socketloop.com/tutorials/golang-check-if-ip-address-is-version-4-or-6\) + + +## func [LocalSpeedTestIpTable]() + +```go +func LocalSpeedTestIpTable() error +``` + +LocalSpeedTestIpTable Runs speed test in iptables locally only + + +## func [PrettyPrint]() + +```go +func PrettyPrint(data interface{}) +``` + + + + +## func [PrintIpTable]() + +```go +func PrintIpTable() error +``` + +PrintIpTable Print Ip table data for Cli + + +## func [ReadAuthorizedKeys]() + +```go +func ReadAuthorizedKeys(path string) (map[string]bool, error) +``` + +ReadAuthorizedKeys reads and returns the current contents of the authorized\_keys file as a map + + +## func [RemoveKeyFromAuthorizedKeys]() + +```go +func RemoveKeyFromAuthorizedKeys(path, keyToRemove string) error +``` + + + + +## func [UnForwardPort]() + +```go +func UnForwardPort(port int) error +``` + +UnForwardPort from router + + +## func [ValidateHashSHA256]() + +```go +func ValidateHashSHA256(text string, Hash []byte) bool +``` + +ValidateHashSHA256 CustomInformationKey the text and check if the text and the hash are the same, if they are then return true SHA256 is the current hashing algorthm used. + + +## func [ViewNetworkInterface]() + +```go +func ViewNetworkInterface() error +``` + +ViewNetworkInterface This function is created to view the network interfaces available + + +## type [IP]() + + + +```go +type IP struct { + Query string +} +``` + + +## type [IpAddress]() + + + +```go +type IpAddress struct { + Name string `json:"Name"` + MachineUsername string `json:"MachineUsername"` + Ipv4 string `json:"IPV4"` + Ipv6 string `json:"IPV6"` + Latency time.Duration `json:"Latency"` + Download float64 `json:"Download"` + Upload float64 `json:"Upload"` + ServerPort string `json:"ServerPort"` + BareMetalSSHPort string `json:"BareMetalSSHPort"` + NAT string `json:"NAT"` + EscapeImplementation string `json:"EscapeImplementation"` + ProxyServer string `json:"ProxyServer"` + UnSafeMode bool `json:"UnSafeMode"` + PublicKey string `json:"PublicKey"` + CustomInformation string `json:"CustomInformation"` +} +``` + + +### func \(\*IpAddress\) [DownloadSpeed]() + +```go +func (s *IpAddress) DownloadSpeed() error +``` + +Download Speed + + +### func \(\*IpAddress\) [PingTest]() + +```go +func (s *IpAddress) PingTest() error +``` + +PingTest executes test to measure latency + + +### func \(\*IpAddress\) [UploadSpeed]() + +```go +func (s *IpAddress) UploadSpeed() error +``` + + + + +## type [IpAddresses]() + + + +```go +type IpAddresses struct { + IpAddress []IpAddress `json:"ip_address"` +} +``` + + +### func [ReadIpTable]() + +```go +func ReadIpTable() (*IpAddresses, error) +``` + +ReadIpTable Read data from Ip tables from json file + + +### func \(\*IpAddresses\) [RemoveDuplicates]() + +```go +func (table *IpAddresses) RemoveDuplicates() error +``` + +RemoveDuplicates This is a temporary fix current functions failing to remove Duplicate IP addresses from local IP table + + +### func \(\*IpAddresses\) [SpeedTest]() + +```go +func (ip *IpAddresses) SpeedTest() error +``` + +SpeedTest Runs a speed test and does updates IP tables accordingly + + +### func \(\*IpAddresses\) [SpeedTestUpdatedIPTable]() + +```go +func (ip *IpAddresses) SpeedTestUpdatedIPTable() error +``` + +SpeedTestUpdatedIPTable Called when ip tables from httpclient/server is also passed on + + +### func \(\*IpAddresses\) [WriteIpTable]() + +```go +func (i *IpAddresses) WriteIpTable() error +``` + +WriteIpTable Write to IP table json file + +Generated by [gomarkdoc]() diff --git a/p2p/frp/docs.md b/p2p/frp/docs.md new file mode 100644 index 0000000..5d723d0 --- /dev/null +++ b/p2p/frp/docs.md @@ -0,0 +1,113 @@ + + +# frp + +```go +import "github.com/Akilan1999/p2p-rendering-computation/p2p/frp" +``` + +## Index + +- [func GetFRPServerPort\(host string\) \(string, error\)](<#GetFRPServerPort>) +- [func StartFRPCDockerContainer\(ipaddress string, port string, Docker \*docker.DockerVM\) \(\*docker.DockerVM, error\)](<#StartFRPCDockerContainer>) +- [func StartFRPClientForServer\(ipaddress string, port string, localport string, remoteport string\) \(string, error\)](<#StartFRPClientForServer>) +- [func StartFRPProxyFromRandom\(\) \(int, error\)](<#StartFRPProxyFromRandom>) +- [type Client](<#Client>) + - [func \(c \*Client\) StartFRPClient\(\) error](<#Client.StartFRPClient>) +- [type ClientMapping](<#ClientMapping>) +- [type Server](<#Server>) + - [func \(s \*Server\) StartFRPServer\(\) error](<#Server.StartFRPServer>) + + + +## func [GetFRPServerPort]() + +```go +func GetFRPServerPort(host string) (string, error) +``` + +GetFRPServerPort Gets the port no from the FRPServer to establish the FRP connection needed. + + +## func [StartFRPCDockerContainer]() + +```go +func StartFRPCDockerContainer(ipaddress string, port string, Docker *docker.DockerVM) (*docker.DockerVM, error) +``` + + + + +## func [StartFRPClientForServer]() + +```go +func StartFRPClientForServer(ipaddress string, port string, localport string, remoteport string) (string, error) +``` + +StartFRPClientForServer Starts Server using FRP server returns back a port remote port is a custom external port a user would want to open. This under the assumption the user knows the exact port available in server doing the TURN connection. + + +## func [StartFRPProxyFromRandom]() + +```go +func StartFRPProxyFromRandom() (int, error) +``` + +StartFRPProxyFromRandom starts reverse proxy server based on a random port generated + + +## type [Client]() + +Client This struct stores client information with server proxy connected + +```go +type Client struct { + Name string + Server *Server + ClientMappings []ClientMapping +} +``` + + +### func \(\*Client\) [StartFRPClient]() + +```go +func (c *Client) StartFRPClient() error +``` + +StartFRPClient Starts FRP client + + +## type [ClientMapping]() + +ClientMapping Stores client mapping ports to proxy server + +```go +type ClientMapping struct { + LocalIP string + LocalPort int + RemotePort int +} +``` + + +## type [Server]() + + + +```go +type Server struct { + // contains filtered or unexported fields +} +``` + + +### func \(\*Server\) [StartFRPServer]() + +```go +func (s *Server) StartFRPServer() error +``` + +StartFRPServer The initial plan is only support reverse proxy for TCP ports This function starts a server that can act as a reverse proxy for nodes behind NAT. + +Generated by [gomarkdoc]() diff --git a/plugin/docs.md b/plugin/docs.md new file mode 100644 index 0000000..e3914c1 --- /dev/null +++ b/plugin/docs.md @@ -0,0 +1,224 @@ + + +# plugin + +```go +import "github.com/Akilan1999/p2p-rendering-computation/plugin" +``` + +## Index + +- [func CheckRunPlugin\(PluginName string, ID string\) error](<#CheckRunPlugin>) +- [func DeletePlugin\(pluginname string\) error](<#DeletePlugin>) +- [func DownloadPlugin\(pluginurl string\) error](<#DownloadPlugin>) +- [func RunPluginContainer\(PluginName string, ContainerID string\) error](<#RunPluginContainer>) +- [type ExecuteIP](<#ExecuteIP>) + - [func \(e \*ExecuteIP\) ModifyHost\(p \*Plugin\) error](<#ExecuteIP.ModifyHost>) + - [func \(e \*ExecuteIP\) RunAnsible\(p \*Plugin\) error](<#ExecuteIP.RunAnsible>) +- [type Host](<#Host>) + - [func ReadHost\(filename string\) \(\*Host, error\)](<#ReadHost>) +- [type Plugin](<#Plugin>) + - [func RunPlugin\(pluginName string, IPAddresses \[\]\*ExecuteIP\) \(\*Plugin, error\)](<#RunPlugin>) + - [func SearchPlugin\(pluginname string\) \(\*Plugin, error\)](<#SearchPlugin>) + - [func \(p \*Plugin\) AutoSetPorts\(containerID string\) error](<#Plugin.AutoSetPorts>) + - [func \(p \*Plugin\) CopyToTmpPlugin\(\) error](<#Plugin.CopyToTmpPlugin>) + - [func \(p \*Plugin\) ExecutePlugin\(\) error](<#Plugin.ExecutePlugin>) + - [func \(p \*Plugin\) NumPorts\(\) error](<#Plugin.NumPorts>) +- [type Plugins](<#Plugins>) + - [func DetectPlugins\(\) \(\*Plugins, error\)](<#DetectPlugins>) + + + +## func [CheckRunPlugin]() + +```go +func CheckRunPlugin(PluginName string, ID string) error +``` + +CheckRunPlugin Checks if the ID belongs to the group or container calls the plugin function the appropriate amount of times + + +## func [DeletePlugin]() + +```go +func DeletePlugin(pluginname string) error +``` + +DeletePlugin The following function deletes a plugin based on the plugin name provided. + + +## func [DownloadPlugin]() + +```go +func DownloadPlugin(pluginurl string) error +``` + +DownloadPlugin This functions downloads package from a git repo. + + +## func [RunPluginContainer]() + +```go +func RunPluginContainer(PluginName string, ContainerID string) error +``` + +RunPluginContainer Runs ansible plugin based on plugin name and container name which is derived from the tracked containers file We pass in the group ID as a parameter because when we modify the ports taken + + +## type [ExecuteIP]() + +ExecuteIP IP Address to execute Ansible instruction + +```go +type ExecuteIP struct { + ContainerID string + IPAddress string + SSHPortNo string + Success bool +} +``` + + +### func \(\*ExecuteIP\) [ModifyHost]() + +```go +func (e *ExecuteIP) ModifyHost(p *Plugin) error +``` + +ModifyHost adds IP address , port no to the config file + + +### func \(\*ExecuteIP\) [RunAnsible]() + +```go +func (e *ExecuteIP) RunAnsible(p *Plugin) error +``` + +RunAnsible Executes based on credentials on the struct + + +## type [Host]() + +Host Struct for ansible host Generated from https://zhwt.github.io/yaml-to-go/ + +```go +type Host struct { + All struct { + Vars struct { + AnsiblePythonInterpreter string `yaml:"ansible_python_interpreter"` + } `yaml:"vars"` + } `yaml:"all"` + Main struct { + Hosts struct { + Host1 struct { + AnsibleHost string `yaml:"ansible_host"` + AnsiblePort int `yaml:"ansible_port"` + AnsibleUser string `yaml:"ansible_user"` + AnsibleSSHPass string `yaml:"ansible_ssh_pass"` + AnsibleSudoPass string `yaml:"ansible_sudo_pass"` + } `yaml:"host1"` + } `yaml:"hosts"` + } `yaml:"main"` +} +``` + + +### func [ReadHost]() + +```go +func ReadHost(filename string) (*Host, error) +``` + +ReadHost Reads host file and adds + + +## type [Plugin]() + +Plugin Information about the plugins available + +```go +type Plugin struct { + FolderName string + PluginDescription string + + Execute []*ExecuteIP + NumOfPorts int + // contains filtered or unexported fields +} +``` + + +### func [RunPlugin]() + +```go +func RunPlugin(pluginName string, IPAddresses []*ExecuteIP) (*Plugin, error) +``` + +RunPlugin Executes plugins based on the plugin name provided + + +### func [SearchPlugin]() + +```go +func SearchPlugin(pluginname string) (*Plugin, error) +``` + +SearchPlugin Detects plugin information based on the name provided on the parameter + + +### func \(\*Plugin\) [AutoSetPorts]() + +```go +func (p *Plugin) AutoSetPorts(containerID string) error +``` + +AutoSetPorts Automatically maps free ports to site.yml file + + +### func \(\*Plugin\) [CopyToTmpPlugin]() + +```go +func (p *Plugin) CopyToTmpPlugin() error +``` + +CopyToTmpPlugin This function would ensure that we create a copy of the plugin in the tmp directory, and it would be executed from there. This due to the reason of automating port allocation when running plugins + + +### func \(\*Plugin\) [ExecutePlugin]() + +```go +func (p *Plugin) ExecutePlugin() error +``` + +ExecutePlugin Function to execute plugins that are called + + +### func \(\*Plugin\) [NumPorts]() + +```go +func (p *Plugin) NumPorts() error +``` + +NumPorts Gets the Number the ports the plugin requires + + +## type [Plugins]() + +Plugins Array of all plugins detected + +```go +type Plugins struct { + PluginsDetected []*Plugin +} +``` + + +### func [DetectPlugins]() + +```go +func DetectPlugins() (*Plugins, error) +``` + +DetectPlugins Detects all the plugins available + +Generated by [gomarkdoc]() diff --git a/server/docker/docs.md b/server/docker/docs.md new file mode 100644 index 0000000..0185ff3 --- /dev/null +++ b/server/docker/docs.md @@ -0,0 +1,174 @@ + + +# docker + +```go +import "github.com/Akilan1999/p2p-rendering-computation/server/docker" +``` + +## Index + +- [func StopAndRemoveContainer\(containername string\) error](<#StopAndRemoveContainer>) +- [type DockerContainer](<#DockerContainer>) +- [type DockerContainers](<#DockerContainers>) + - [func ViewAllContainers\(\) \(\*DockerContainers, error\)](<#ViewAllContainers>) +- [type DockerVM](<#DockerVM>) + - [func BuildRunContainer\(NumPorts int, GPU string, ContainerName string, baseImage string, publicKey string\) \(\*DockerVM, error\)](<#BuildRunContainer>) + - [func \(d \*DockerVM\) CopyToTmpContainer\(\) error](<#DockerVM.CopyToTmpContainer>) + - [func \(d \*DockerVM\) TemplateDockerContainer\(\) error](<#DockerVM.TemplateDockerContainer>) +- [type ErrorDetail](<#ErrorDetail>) +- [type ErrorLine](<#ErrorLine>) +- [type Port](<#Port>) +- [type Ports](<#Ports>) + - [func OpenPortsFile\(filename string\) \(\*Ports, error\)](<#OpenPortsFile>) + + + +## func [StopAndRemoveContainer]() + +```go +func StopAndRemoveContainer(containername string) error +``` + +StopAndRemoveContainer Stop and remove a container Reference \(https://gist.github.com/frikky/e2efcea6c733ea8d8d015b7fe8a91bf6\) + + +## type [DockerContainer]() + + + +```go +type DockerContainer struct { + ContainerName string `json:"DockerContainerName"` + ContainerDescription string `json:"ContainerDescription"` +} +``` + + +## type [DockerContainers]() + + + +```go +type DockerContainers struct { + DockerContainer []DockerContainer `json:"DockerContainer"` +} +``` + + +### func [ViewAllContainers]() + +```go +func ViewAllContainers() (*DockerContainers, error) +``` + +ViewAllContainers returns all containers runnable and which can be built + + +## type [DockerVM]() + + + +```go +type DockerVM struct { + SSHUsername string `json:"SSHUsername"` + SSHPublcKey string `json:"SSHPublicKey"` + ID string `json:"ID"` + TagName string `json:"TagName"` + ImagePath string `json:"ImagePath"` + Ports Ports `json:"Ports"` + GPU string `json:"GPU"` + TempPath string + BaseImage string + LogsPath string + SSHCommand string `json:"SSHCommand"` +} +``` + + +### func [BuildRunContainer]() + +```go +func BuildRunContainer(NumPorts int, GPU string, ContainerName string, baseImage string, publicKey string) (*DockerVM, error) +``` + +BuildRunContainer Function is incharge to invoke building and running contianer and also allocating external ports + + +### func \(\*DockerVM\) [CopyToTmpContainer]() + +```go +func (d *DockerVM) CopyToTmpContainer() error +``` + +CopyToTmpContainer Creates a copy of the docker folder + + +### func \(\*DockerVM\) [TemplateDockerContainer]() + +```go +func (d *DockerVM) TemplateDockerContainer() error +``` + +TemplateDockerContainer This function templates the docker container with the base docker image to use + + +## type [ErrorDetail]() + + + +```go +type ErrorDetail struct { + Message string `json:"message"` +} +``` + + +## type [ErrorLine]() + + + +```go +type ErrorLine struct { + Error string `json:"error"` + ErrorDetail ErrorDetail `json:"errorDetail"` +} +``` + + +## type [Port]() + + + +```go +type Port struct { + PortName string `json:"PortName"` + InternalPort int `json:"InternalPort"` + Type string `json:"Type"` + ExternalPort int `json:"ExternalPort"` + IsUsed bool `json:"IsUsed"` + Description string `json:"Description"` +} +``` + + +## type [Ports]() + + + +```go +type Ports struct { + PortSet []Port `json:"Port"` +} +``` + + +### func [OpenPortsFile]() + +```go +func OpenPortsFile(filename string) (*Ports, error) +``` + + + +Generated by [gomarkdoc]() diff --git a/server/docs.md b/server/docs.md new file mode 100644 index 0000000..8cb94a5 --- /dev/null +++ b/server/docs.md @@ -0,0 +1,227 @@ + + +# server + +```go +import "github.com/Akilan1999/p2p-rendering-computation/server" +``` + +source:https://github.com/afoley587/go-rev-proxy/tree/main + +## Index + +- [Variables](<#variables>) +- [func GetScheme\(c \*gin.Context\) string](<#GetScheme>) +- [func MapPort\(port string, domainName string, serverAddress string\) \(string, string, error\)](<#MapPort>) +- [func Proxy\(c \*gin.Context\)](<#Proxy>) +- [func ProxyRun\(port string\)](<#ProxyRun>) +- [func SaveRegistration\(OutsideHost string, InsideHost string\) error](<#SaveRegistration>) +- [func Server\(\) \(\*gin.Engine, error\)](<#Server>) +- [type Gpu](<#Gpu>) +- [type GpuClock](<#GpuClock>) +- [type GpuTemperature](<#GpuTemperature>) +- [type GpuUtilization](<#GpuUtilization>) +- [type Query](<#Query>) + - [func GPUInfo\(\) \(\*Query, error\)](<#GPUInfo>) +- [type RegistrationRequest](<#RegistrationRequest>) +- [type ReverseProxy](<#ReverseProxy>) +- [type SysInfo](<#SysInfo>) + - [func ServerInfo\(\) \*SysInfo](<#ServerInfo>) + + +## Variables + + + +```go +var ( + //RunPort = 2002 // The server port to run on + //ReverseServerAddr = fmt.Sprint("0.0.0.0:", RunPort) // this is our reverse server ip address + //InsideProxyHostname = fmt.Sprint("proxy:", RunPort) // Requests from private network + //OutsideProxyHostname = fmt.Sprint("registration.localhost:", RunPort) // Requests from public network + KnownAddresses = map[string]string{} // Known Addresses +) +``` + +ReverseProxies Reverse to the map such as ReverseProxies\[\\]ReverseProxy type + +```go +var ReverseProxies map[string]ReverseProxy +``` + + +## func [GetScheme]() + +```go +func GetScheme(c *gin.Context) string +``` + +This function checks if TLS was enabled on the request and translates it to the proper scheme \(http or https\) + + +## func [MapPort]() + +```go +func MapPort(port string, domainName string, serverAddress string) (string, string, error) +``` + + + + +## func [Proxy]() + +```go +func Proxy(c *gin.Context) +``` + +Proxy runs the actual proxy and will look at the hostnames requested from the received request. It will then translate that to the inside hostname and forward the request + + +## func [ProxyRun]() + +```go +func ProxyRun(port string) +``` + + + + +## func [SaveRegistration]() + +```go +func SaveRegistration(OutsideHost string, InsideHost string) error +``` + +SaveRegistration Creates registration of proxy node + + +## func [Server]() + +```go +func Server() (*gin.Engine, error) +``` + + + + +## type [Gpu]() + + + +```go +type Gpu struct { + GpuName string `xml:"product_name"` + BiosVersion string `xml:"vbios_version"` + FanSpeed string `xml:"fan_speed"` + Utilization GpuUtilization `xml:"utilization"` + Temperature GpuTemperature `xml:"temperature"` + Clock GpuClock `xml:"clocks"` +} +``` + + +## type [GpuClock]() + + + +```go +type GpuClock struct { + GpuClock string `xml:"graphics_clock"` + GpuMemClock string `xml:"mem_clock"` +} +``` + + +## type [GpuTemperature]() + + + +```go +type GpuTemperature struct { + GpuTemp string `xml:"gpu_temp"` +} +``` + + +## type [GpuUtilization]() + + + +```go +type GpuUtilization struct { + GpuUsage string `xml:"gpu_util"` + MemoryUsage string `xml:"memory_util"` +} +``` + + +## type [Query]() + + + +```go +type Query struct { + DriveVersion string `xml:"driver_version"` + Gpu Gpu `xml:"gpu"` +} +``` + + +### func [GPUInfo]() + +```go +func GPUInfo() (*Query, error) +``` + +Gets GPU information by calling nvidia\-smi in XML output + + +## type [RegistrationRequest]() + + + +```go +type RegistrationRequest struct { + OutsideHost string // Outside host address. The hostname or IP on the public network. For example foo.bar.com or foo.localhost + InsideHost string // Inside host address. The hostname or IP on the internal network. For example 192.168.10.5 +} +``` + + +## type [ReverseProxy]() + + + +```go +type ReverseProxy struct { + IPAddress string + Port string +} +``` + + +## type [SysInfo]() + +SysInfo saves the basic system information + +```go +type SysInfo struct { + Hostname string `bson:hostname` + Platform string `bson:platform` + CPU string `bson:cpu` + RAM uint64 `bson:ram` + Disk uint64 `bson:disk` + GPU *Query `xml: GpuInfo` +} +``` + + +### func [ServerInfo]() + +```go +func ServerInfo() *SysInfo +``` + + + +Generated by [gomarkdoc]()