From 861f746ff982b826e23adb6c26a6dc2b05e415fc Mon Sep 17 00:00:00 2001 From: Akilan Date: Tue, 11 May 2021 06:18:10 +0400 Subject: [PATCH] added docs for server implementation --- Docs/ConfigImplementation.md | 21 +++++ Docs/ServerImplementation.md | 165 +++++++++++++++++++++++++++++++++ requirements.txt | 1 - server/docker/docker_vm.py | 96 ------------------- server/docker/requirements.txt | 2 - 5 files changed, 186 insertions(+), 99 deletions(-) create mode 100644 Docs/ConfigImplementation.md create mode 100644 Docs/ServerImplementation.md delete mode 100644 requirements.txt delete mode 100644 server/docker/docker_vm.py delete mode 100644 server/docker/requirements.txt diff --git a/Docs/ConfigImplementation.md b/Docs/ConfigImplementation.md new file mode 100644 index 0000000..8480715 --- /dev/null +++ b/Docs/ConfigImplementation.md @@ -0,0 +1,21 @@ +# Config Implementation + +The configuration module is responsible to store basic information of absolute paths of files being +called in the Go code. In a full-fledged Cli the configuration file can be found in the directory +/etc/ and from there points to location such as where the IP table file is located. In +the future implementation the config file will have information such as number of hops and other +parameters to tweak and to improve the effectiveness of the peer to peer network. The +configuration module was implemented using the library Viper. The Viper library automates +features such as searching in default paths to find out if the configuration file is present. If the +configuration file is not present in the default paths then it auto generates the configuration file. +The configurations file can be in any format. In this project the configuration file was generated using +JSON format. + +```json +{ + "dockerfile": "//p2p-rendering-computation/server/docker/containers/docker-ubuntu-sshd/", + "iptable": "//p2p-rendering-computation/p2p/ip_table.json", + "speedtestfile": "//p2p-rendering-computation/p2p/50.bin" +} +``` + diff --git a/Docs/ServerImplementation.md b/Docs/ServerImplementation.md new file mode 100644 index 0000000..0f38281 --- /dev/null +++ b/Docs/ServerImplementation.md @@ -0,0 +1,165 @@ +# Server Module Implementation + +This section focuses on an in-depth understanding of the server module implementation. To +understand the architecture of the server module refer. The server module can be split +into various sections. Each section will provide information on how a certain feature works. + +## Web framework +The web framework used for the server module is called Gin. The reason Gin was chosen is due to +its wide use and strong documentation available on the official github repository. The default +port used is 8088. For version 1.0 of the project ,the server needs to keep port 8088 open to +ensure that other clients and servers can detect it. The possible requests available are GET and +POST for this implementation. The possible responses are either a string or json response or a file. +In the majority of routes a string response refers to an error when calling the following routes. +The following sub topics below will talk about the route implemented: + +### /server_info +This route is responsible to get information about the specifications of the +server. The response of this route is in json if the call was successful. + +### /50 +This route is responsible for returning a randomly generated 50mb file. This is used to +calculate the download speed from the p2p module. + +### /IpTable +This route is a POST request that is responsible to update the server IP table +based on the IP table the client provides. Once the server gets the IP table it checks if the +client is also a server. This is done by calling the url http://:8088/server_info. If +the server_info route from the client responds back with computer specifications of the +client. Then the server initially appends the clients IP to the struct. After that the IP table +received from the client is uploaded to the struct. Once this is done the server passes the +struct to the peer to peer module function. The peer to peer module function will return the back with the +new struct with the valid server nodes. The server responds back to the new struct as a +json format. If a string is present in the response then there is probably an error on the +server side. + +### /startcontainer +This route takes in a GET request with the number of TCP ports to open and +checks whether the docker container should be hooked to the GPU or not. This route talks +to the docker module implemented as a sub module in the server module. More +information on the docker module in section 5.4.3. This route calls docker the module to +start the container for the client. The docker module returns back a struct. This struct is +returned back to the client as the json response. This struct consists of information such as +docker id, ports numbers open , information regarding SSH and VNC connections to the +docker container created when the client created this request. + +### /RemoveContainer +This route takes in a GET request as the container ID. Based on the +container ID provided ,it calls the docker module which deletes the container. If the +deletion is successful it returns back a string which says success. + +## Server information/ Specification +This section provides information on how the server specifications are read. There are 2 major +implementations. The first implementation mentions how basic information such as RAM usage, +CPU specification are detected and the second implementation mentions how the GPU drivers are +detected and information is extracted. The client has to assume that the server is using default +docker settings in terms of CPU cycles and other parameters. + +### Basic Information +The file name for these functions is called gopsutil.go. This codebase +uses the library gopsutil. Gopsutil has various packages or modules within the library +which have functions implemented to get system information. The following information is +stored in a struct and the function returns that struct. + +```go +type SysInfo struct { + Hostname string `bson:hostname` + Platform string `bson:platform` + CPU string `bson:cpu` + RAM uint64 `bson:ram` + Disk uint64 `bson:disk` + GPU *Query `xml: GpuInfo` +} +``` +### GPU Information +The file name for these functions is called GPU.go. This codebase checks +if the Nvidia driver exists and returns the driver information. To do this a shell +command called nvidia-smi is executed. This shell command is executed with a --xml as flag +to ensure that the output is in the XML format. If there is an output as a xml format, that +means there is an nvidia driver installed, and the function just reads the output and stores it +to the struct and returns the GPU information. + +```go + +type Query struct { + DriveVersion string `xml:"driver_version"` + Gpu Gpu `xml:"gpu"` +} + +type Gpu struct{ + GpuName string `xml:"product_name"` + BiosVersion string `xml:"vbios_version"` + FanSpeed string `xml:"fan_speed"` + Utilization GpuUtilization `xml:"utilization"` + Temperature GpuTemperature `xml:"temperature"` + Clock GpuClock `xml:"clocks"` +} + +type GpuUtilization struct { + GpuUsage string `xml:"gpu_util"` + MemoryUsage string `xml:"memory_util"` +} + +type GpuTemperature struct { + GpuTemp string `xml:"gpu_temp"` +} + +type GpuClock struct { + GpuClock string `xml:"graphics_clock"` + GpuMemClock string `xml:"mem_clock"` +} +``` + +## Docker Module +This section provides information on how the server module interacts with the docker containers. +The server calls 2 routes which either creates or removes the docker container. Docker has a huge +advantage because it takes less than 20 seconds to spin up a new container once it’s built and +executed at least once. For docker operations a separate module/package has been created. The +following subtopics will provide more information on how this package works. + +### Docker Api +For this the api has been taken from the official docker repository. To be more +specific it is the client module in the official docker repository. Docker was built using Go. +During this project Docker functions could be directly called from the docker repository. +The Docker api initially ensures that it can detect the docker environment variables. Once +detected, it can execute various functions from the docker client module. The reason the +docker api was selected was to detect and handle errors better. + +### Docker Image +The docker image used to spin up the containers is called +ConSol/docker-headless-vnc-container. The following container was modified to open +SSH ports for an SSH connection. The following docker image runs ubuntu 16. The reason +this image was chosen as a default is because if the client wants to access the container in +the form of a desktop environment. This image would allow the client to do so from just a +browser. + +### Build container +This function pulls the docker image locally and builds the image. Initially +there is a timeout function to ensure that building the image does not take too long to +build. The next phase would be based on the path to get the DockerFile. The tag name of +the container is set as p2p-ubuntu as default. Once the following is set then the docker +build command is executed. + +### Run container +After building the container it needs to be executed for the user to access +the container and do certain operations. The docker package/module has a function to do +this. The function takes in the docker environment as a parameter and also the docker +struct. The docker struct has information such as the TCP ports which are supposed to be +open and whether the docker container should have the GPU hooked to it or not. Based on +the appropriate information provided ,the docker image gets started. The Image gets +started by interacting with the docker client modules. When hooking the GPU the docker +run command is called from the shell. This is because the docker Api does not support the +GPU module yet. When the container is executed for the first time it takes +more than 10 minutes to build. From the second time onwards it takes only 10 seconds to +run. + +### Stop and remove container +This implementation here ensures that the docker is stopped, and the container is removed. This is to ensure +it does not utilize server resources when it is not being used, or the task that is intended to be executed is complete. +To run this function all that is needed is the docker container ID. If the function is successful it returns +a string that says success. + + + + + diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 770052b..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -ipfs-api \ No newline at end of file diff --git a/server/docker/docker_vm.py b/server/docker/docker_vm.py deleted file mode 100644 index 7269d7e..0000000 --- a/server/docker/docker_vm.py +++ /dev/null @@ -1,96 +0,0 @@ -''' -The follwing file is a docker module to interact with the p2p -system computation/redering system. - -Functionality -1. --createvm: Creates a ubuntu virtualmachine using docker settings and takes name as the parameter -''' -#!/usr/bin/env python3 - -''' -TODO: FUTURE RELEASE TO BE CONVERTED TO GO LANG -''' - -import click -import sys -import docker -import socket -import uuid -import os - -# ----------------------------------------------- CLI flags -------------------------------------------------------- - -@click.command() -@click.option("--createvm", help="Creates docker default VM") - -# ------------------------------------------------------------------------------------------------------------------- -# -------------------------------------- Actions when flags are called ---------------------------------------------- - -def main(createvm): - - # creates docker virtual machine - if createvm: - #name = str(uuid.uuid4()) - build_run_contianer(createvm) - - else: - ctx = click.get_current_context() - click.echo(ctx.get_help()) - -# ------------------------------------------------------------------------------------------------------------------- -# ------------------------------------------ build and run Contianer ------------------------------------------------ - -def build_run_contianer(name): - # Get docker information from environment variables - client = docker.from_env() - - image_path = "./server/docker/containers/docker-ubuntu-sshd/" - tag_name = "p2p-ubuntu" - free_port = str(get_free_tcp_port()) - - # Change to JSON response - vnc_free_port = str(get_free_tcp_port()) - - # Check if image is already exists (If already exists then delete and rebuild) - ''' - Note: - Since system dependant functionality is being used here. The - Error handleing is yet to be implemented. - ''' - try: - client.images.get(tag_name) - #print("------ Image exists (Running Docker container " + name + ") --------") - - # Run the docker continer - # Running interactive version until log files for results are created - os.system("docker run -d=true --name="+ name +" --restart=always -p "+ vnc_free_port +":6901 -p "+ free_port +":22 -v=/opt/data:/data "+ tag_name +" /start > /dev/null") - - except: - #print("------ Image does not exists (building " + name + " and running image) ---------") - - # client.images.build(path=framework_image_path,tag=tag_name,rm=True) - os.system("docker build -t " + tag_name + " " + image_path) - - #print("------ Running Docker contianer " + name + " ---------") - # Run the docker continer - # Running interactive version until log files for results are created - os.system("docker run -d=true --name="+ name +" --restart=always -p "+ vnc_free_port +":6901 -p "+ free_port +":22 -v=/opt/data:/data "+ tag_name +" /start > /dev/null") - - result = {"ssh_port":free_port,"vnc_port":vnc_free_port,"id": name} - print(result) - -# ------------------------------------------------------------------------------------------------------------------- -# ------------------------------------------ Get free TCP port ------------------------------------------------------ - -def get_free_tcp_port(): - tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - tcp.bind(('', 0)) - addr, port = tcp.getsockname() - tcp.close() - return port - - -# ------------------------------------------------------------------------------------------------------------------- - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/server/docker/requirements.txt b/server/docker/requirements.txt deleted file mode 100644 index f97f7c1..0000000 --- a/server/docker/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -docker -click