From e5334cbd3c87b74c7912294fd12eaac180f29b52 Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Wed, 25 Aug 2021 13:27:31 +0400 Subject: [PATCH 01/11] buit base functions --- .gitignore | 4 +- Makefile | 11 +- README.md | 6 +- generate/generate.go | 72 +++++++ generate/generate_test.go | 26 +++ go.mod | 3 +- go.sum | 8 + install.sh | 11 + sample-run/tensorflow/LICENSE | 201 ------------------ sample-run/tensorflow/README.md | 147 ------------- sample-run/tensorflow/ToolSelection.md | 93 -------- sample-run/tensorflow/ansible.cfg | 6 - sample-run/tensorflow/hosts | 12 -- sample-run/tensorflow/hosts.example | 25 --- sample-run/tensorflow/packages.yml | 38 ---- sample-run/tensorflow/roles/cuda/README.md | 23 -- .../tensorflow/roles/cuda/defaults/.gitkeep | 0 .../cuda/files/cuda-11-3_11.3.1-1_amd64.deb | Bin 2450 -> 0 bytes .../cuda-runtime-11-3_11.3.1-1_amd64.deb | Bin 2424 -> 0 bytes .../tensorflow/roles/cuda/handlers/main.yml | 5 - .../tensorflow/roles/cuda/meta/.gitkeep | 0 .../tensorflow/roles/cuda/tasks/debian.yml | 22 -- .../tensorflow/roles/cuda/tasks/main.yml | 3 - .../tensorflow/roles/cuda/templates/.gitkeep | 0 .../tensorflow/roles/cuda/vars/main.yml | 10 - sample-run/tensorflow/roles/docker/README.md | 33 --- .../tensorflow/roles/docker/defaults/.gitkeep | 0 .../tensorflow/roles/docker/files/.gitkeep | 0 .../tensorflow/roles/docker/handlers/.gitkeep | 0 .../tensorflow/roles/docker/meta/.gitkeep | 0 .../tensorflow/roles/docker/tasks/debian.yml | 17 -- .../tensorflow/roles/docker/tasks/main.yml | 23 -- .../roles/docker/templates/.gitkeep | 0 .../tensorflow/roles/docker/vars/main.yml | 8 - sample-run/tensorflow/roles/nvidia/README.md | 31 --- .../tensorflow/roles/nvidia/defaults/.gitkeep | 0 .../tensorflow/roles/nvidia/files/.gitkeep | 0 .../tensorflow/roles/nvidia/handlers/main.yml | 6 - .../tensorflow/roles/nvidia/meta/.gitkeep | 0 .../tensorflow/roles/nvidia/tasks/debian.yml | 31 --- .../tensorflow/roles/nvidia/tasks/main.yml | 4 - .../roles/nvidia/templates/.gitkeep | 0 .../tensorflow/roles/nvidia/vars/main.yml | 10 - .../tensorflow/roles/packages/README.md | 26 --- .../roles/packages/defaults/.gitkeep | 0 .../tensorflow/roles/packages/files/.gitkeep | 0 .../roles/packages/handlers/.gitkeep | 0 .../tensorflow/roles/packages/meta/.gitkeep | 0 .../roles/packages/tasks/debian.yml | 12 -- .../tensorflow/roles/packages/tasks/main.yml | 3 - .../roles/packages/templates/.gitkeep | 0 .../tensorflow/roles/packages/vars/main.yml | 15 -- .../tensorflow/roles/rsync/defaults/.gitkeep | 0 .../roles/rsync/files/rsyncd.example.conf | 16 -- .../tensorflow/roles/rsync/handlers/main.yml | 5 - .../tensorflow/roles/rsync/meta/.gitkeep | 0 .../tensorflow/roles/rsync/tasks/main.yml | 26 --- .../tensorflow/roles/rsync/templates/.gitkeep | 0 .../tensorflow/roles/rsync/vars/main.yml | 10 - sample-run/tensorflow/secrets.example.yml | 15 -- sample-run/tensorflow/secrets.yml | 15 -- 61 files changed, 124 insertions(+), 908 deletions(-) create mode 100644 generate/generate.go create mode 100644 generate/generate_test.go create mode 100644 install.sh delete mode 100644 sample-run/tensorflow/LICENSE delete mode 100644 sample-run/tensorflow/README.md delete mode 100644 sample-run/tensorflow/ToolSelection.md delete mode 100644 sample-run/tensorflow/ansible.cfg delete mode 100644 sample-run/tensorflow/hosts delete mode 100644 sample-run/tensorflow/hosts.example delete mode 100644 sample-run/tensorflow/packages.yml delete mode 100644 sample-run/tensorflow/roles/cuda/README.md delete mode 100644 sample-run/tensorflow/roles/cuda/defaults/.gitkeep delete mode 100644 sample-run/tensorflow/roles/cuda/files/cuda-11-3_11.3.1-1_amd64.deb delete mode 100644 sample-run/tensorflow/roles/cuda/files/cuda-runtime-11-3_11.3.1-1_amd64.deb delete mode 100644 sample-run/tensorflow/roles/cuda/handlers/main.yml delete mode 100644 sample-run/tensorflow/roles/cuda/meta/.gitkeep delete mode 100644 sample-run/tensorflow/roles/cuda/tasks/debian.yml delete mode 100644 sample-run/tensorflow/roles/cuda/tasks/main.yml delete mode 100644 sample-run/tensorflow/roles/cuda/templates/.gitkeep delete mode 100644 sample-run/tensorflow/roles/cuda/vars/main.yml delete mode 100644 sample-run/tensorflow/roles/docker/README.md delete mode 100644 sample-run/tensorflow/roles/docker/defaults/.gitkeep delete mode 100644 sample-run/tensorflow/roles/docker/files/.gitkeep delete mode 100644 sample-run/tensorflow/roles/docker/handlers/.gitkeep delete mode 100644 sample-run/tensorflow/roles/docker/meta/.gitkeep delete mode 100644 sample-run/tensorflow/roles/docker/tasks/debian.yml delete mode 100644 sample-run/tensorflow/roles/docker/tasks/main.yml delete mode 100644 sample-run/tensorflow/roles/docker/templates/.gitkeep delete mode 100644 sample-run/tensorflow/roles/docker/vars/main.yml delete mode 100644 sample-run/tensorflow/roles/nvidia/README.md delete mode 100644 sample-run/tensorflow/roles/nvidia/defaults/.gitkeep delete mode 100644 sample-run/tensorflow/roles/nvidia/files/.gitkeep delete mode 100644 sample-run/tensorflow/roles/nvidia/handlers/main.yml delete mode 100644 sample-run/tensorflow/roles/nvidia/meta/.gitkeep delete mode 100644 sample-run/tensorflow/roles/nvidia/tasks/debian.yml delete mode 100644 sample-run/tensorflow/roles/nvidia/tasks/main.yml delete mode 100644 sample-run/tensorflow/roles/nvidia/templates/.gitkeep delete mode 100644 sample-run/tensorflow/roles/nvidia/vars/main.yml delete mode 100644 sample-run/tensorflow/roles/packages/README.md delete mode 100644 sample-run/tensorflow/roles/packages/defaults/.gitkeep delete mode 100644 sample-run/tensorflow/roles/packages/files/.gitkeep delete mode 100644 sample-run/tensorflow/roles/packages/handlers/.gitkeep delete mode 100644 sample-run/tensorflow/roles/packages/meta/.gitkeep delete mode 100644 sample-run/tensorflow/roles/packages/tasks/debian.yml delete mode 100644 sample-run/tensorflow/roles/packages/tasks/main.yml delete mode 100644 sample-run/tensorflow/roles/packages/templates/.gitkeep delete mode 100644 sample-run/tensorflow/roles/packages/vars/main.yml delete mode 100644 sample-run/tensorflow/roles/rsync/defaults/.gitkeep delete mode 100644 sample-run/tensorflow/roles/rsync/files/rsyncd.example.conf delete mode 100644 sample-run/tensorflow/roles/rsync/handlers/main.yml delete mode 100644 sample-run/tensorflow/roles/rsync/meta/.gitkeep delete mode 100644 sample-run/tensorflow/roles/rsync/tasks/main.yml delete mode 100644 sample-run/tensorflow/roles/rsync/templates/.gitkeep delete mode 100644 sample-run/tensorflow/roles/rsync/vars/main.yml delete mode 100644 sample-run/tensorflow/secrets.example.yml delete mode 100644 sample-run/tensorflow/secrets.yml diff --git a/.gitignore b/.gitignore index 9d1c1fe..da58fd3 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,6 @@ p2p/iptable/ #ignore plugins added plugin/deploy/ #ignore track container file -client/trackcontainers/ \ No newline at end of file +client/trackcontainers/ +# Test generated files +generate/p2prctest \ No newline at end of file diff --git a/Makefile b/Makefile index 4bd8a68..a32c494 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,7 @@ SHELL := /bin/bash install: - go build -o p2prc - echo '# Paths for p2p rendering and computation' - echo 'export P2PRC=${PWD}' - echo 'export PATH=${PWD}:$${PATH}' - -build: - go build -o p2prc - -configfile: - ./p2prc --SetDefaultConfig + sh install.sh p2prc testcases: sh plugin/generate_test_case.sh diff --git a/README.md b/README.md index 2e1fac7..2de36b0 100644 --- a/README.md +++ b/README.md @@ -63,11 +63,7 @@ This project aims to create a peer to peer (p2p) network, where a user can use t // Add them to your .bashrc file export P2PRC=//p2p-rendering-computation export PATH=//p2p-rendering-computation:${PATH} - -5. Generate Config file - ``` - make configfile - ``` + 5. Test if it works ``` diff --git a/generate/generate.go b/generate/generate.go new file mode 100644 index 0000000..b65824c --- /dev/null +++ b/generate/generate.go @@ -0,0 +1,72 @@ +// Package generate The purpose of this package is to ensure that we can extend the use-case of P2PRC. +// We will create a project directory with the template to extend the use-case of P2PRC +package generate + +import ( + "fmt" + "github.com/otiai10/copy" + "os" + "strings" +) + +func GenerateNewProject(name string) error { + // Get path of the current directory + curDir := os.Getenv("PWD") + // Add slash to the end + curDir = curDir + "/" + // Folder name of the new generated project + NewProject := curDir + name + "/" + fmt.Println(NewProject) + // Create a new folder based on name entered + err := CreateFolder(name, curDir) + if err != nil { + return err + } + // get path of P2PRC + P2PRCPATH := os.Getenv("P2PRC") + // Add slash to the end + P2PRCPATH = P2PRCPATH + "/" + // Steps: + // - copy all files from P2PRC + // - remove go.mod and go.sum and create new ones + + // Files we require to skip + var Options copy.Options + + // Skips the appropriate files and directories not needed + Options.Skip = func(src string) (bool, error) { + switch { + case strings.HasSuffix(src, "go.mod"): + return true, nil + case strings.HasSuffix(src, "go.sum"): + return true, nil + case strings.HasSuffix(src, name): + return true, nil + default: + return false, nil + } + } + + // Copies all files from P2PRC to the new project created + err = copy.Copy(P2PRCPATH, NewProject,Options) + if err != nil { + return err + } + // Installing new project + //cmd := exec.Command("sh","install.sh",name) + //if err := cmd.Run(); err != nil { + // return err + //} + + return nil +} + +// CreateFolder Creates a new folder based on the name and path provided +func CreateFolder(name string,path string) error { + //Create a folder/directory at a full qualified path + err := os.Mkdir(path + name, 0755) + if err != nil { + return err + } + return nil +} \ No newline at end of file diff --git a/generate/generate_test.go b/generate/generate_test.go new file mode 100644 index 0000000..9da7c01 --- /dev/null +++ b/generate/generate_test.go @@ -0,0 +1,26 @@ +package generate + +import ( + "fmt" + "testing" +) + +// Tests the create folder function creates a folder +// This test will create a folder in the temporary +// directory +func TestCreateFolder(t *testing.T) { + err := CreateFolder("test", "/tmp/") + if err != nil { + t.Error(err) + } +} + +// Testing if a new project is created successfully +func TestGenerateNewProject(t *testing.T) { + // Checking if a new project is created successfully + err := GenerateNewProject("p2prctest") + if err != nil { + fmt.Println(err) + t.Error(err) + } +} \ No newline at end of file diff --git a/go.mod b/go.mod index 19b3e8b..06019f1 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/moby/sys/mount v0.2.0 // indirect github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf // indirect github.com/morikuni/aec v1.0.0 // indirect + github.com/otiai10/copy v1.6.0 github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/shirou/gopsutil v3.20.12+incompatible github.com/spf13/viper v1.4.0 @@ -29,6 +30,6 @@ require ( golang.org/x/text v0.3.5 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect google.golang.org/grpc v1.35.0 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.0.3 // indirect ) diff --git a/go.sum b/go.sum index d7e8059..3213490 100644 --- a/go.sum +++ b/go.sum @@ -462,6 +462,14 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.m github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/otiai10/copy v1.6.0 h1:IinKAryFFuPONZ7cm6T6E2QX/vcJwSnlaA5lfoaXIiQ= +github.com/otiai10/copy v1.6.0/go.mod h1:XWfuS3CrI0R6IE0FbgHsEazaXO8G0LpMp9o8tos0x4E= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/curr v1.0.0 h1:TJIWdbX0B+kpNagQrjgq8bCMrbhiuX73M2XwgtDMoOI= +github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/otiai10/mint v1.3.2 h1:VYWnrP5fXmz1MXvjuUvcBrXSjGE6xjON+axB/UrpO3E= +github.com/otiai10/mint v1.3.2/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..a3d7dc1 --- /dev/null +++ b/install.sh @@ -0,0 +1,11 @@ +# This script setups up the project P2PRC +# Call: sh install.sh +echo '# Add the following paths to .bashrc or .zshrc based on the configuration you have set' +echo export P2PRC=$PWD +echo export PATH=$PWD:\${PATH} +export P2PRC=${PWD} +export PATH=${PWD}:${PATH} + +# Expects an argument of the name of the binary +go build -o ${1} +./p2prc --dc \ No newline at end of file diff --git a/sample-run/tensorflow/LICENSE b/sample-run/tensorflow/LICENSE deleted file mode 100644 index 261eeb9..0000000 --- a/sample-run/tensorflow/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/sample-run/tensorflow/README.md b/sample-run/tensorflow/README.md deleted file mode 100644 index 9cd45d1..0000000 --- a/sample-run/tensorflow/README.md +++ /dev/null @@ -1,147 +0,0 @@ -# Ansible p2p-rendering-computation - -This repository contains an Ansible playbook and instructions to create and manage a single (or many) bare metal deep learning machines. For a description of why Ansible was chosen and what other alternatives were considered, please see [ToolSelection.md](ToolSelection.md) - -## Quick Reference - -If you've already [installed Ansible](#Installation), you can execute the entire playbook by running: - -```bash -$ ansible-playbook packages.yml -``` - -You can also execute only the pieces you need by passing tags on the command line: - -- Install only apt/pip [pre-requisites](roles/packages) to execute the other roles: - ```bash - $ ansible-playbook packages.yml --tags "packages" - ``` -- Install [Docker CE](roles/docker): - ```bash - $ ansible-playbook packages.yml --tags "docker" - ``` -- Install the [Nvidia CUDA GPU drivers](roles/cuda): - ```bash - $ ansible-playbook packages.yml --tags "cuda" - ``` -- Install the [Nvidia Docker Runtime](roles/nvidia): - ```bash - $ ansible-playbook packages.yml --tags "nvidia" - ``` - -## What's Included - -After running the ansible script your machines will be loaded with the following: - -1. Docker -2. Nvidia CUDA GPU Drivers -3. Nvidia Docker Runtime -4. TensorFlow GPU Python3 Docker Container -5. JupyterLab - -## Using This Repository to Configure Your Environment - -1. [Installation](#Installation) -2. [Configuration](#Configuration) -3. [Running](#Running) - ---- - -### Installation - -Ansible runs on your local machine and sends commands to the remote (machine learning) machines. You'll need ansible installed locally (not on the machine learning boxes). -For macOS users, the easiest way to install Ansible is via [Homebrew](https://brew.sh/): - -```bash -$ brew install ansible -``` - -If that's not your cup of tea, install Ansible by following the directions for your machine [here](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#installing-the-control-machine). - ---- - -### Configuration - -Gather the following: - -- SSH key or user credentials for the remote account - - **Note:** Ansible does not expose a channel to allow communication between the user and the ssh process to accept a password manually to decrypt an ssh key when using the ssh connection plugin (which is the default). The use of `ssh-agent` is highly recommended. - -- List of servers you wish to manage: - - hostnames/IP addresses - - SSH port - - usernames - -Copy [hosts.example] to `/etc/ansible/hosts` (if it does not already exist). Populate the `hosts` file (no extension) with the information about the servers you gathered above. - -Confirm that you have populated your Ansible hosts file correctly: - -```bash -$ ansible-inventory --list -``` - ---- - -### Running - -Once you're satisfied that you correctly populated your `hosts` file, update the `- hosts:` line of [tensorflow.yml] to reflect the hosts or groups you want to configure. - -Examples: - -- Apply against a single host defined as `ml2` in `/etc/ansible/hosts`: - ```yaml - - hosts: ml2 - ``` -- Apply against a group of hosts defined as `production` in `/etc/ansible/hosts`: - ```yaml - - hosts: production - ``` -- Apply against all hosts defined in `/etc/ansible/hosts`: - ```yaml - - hosts: all - ``` - -Then, when you're ready, run the playbook: - -```bash -$ ansible-playbook packages.yml --ask-become-pass -``` - -**Note:** You must have `sudo` access to run the playbook! - -Review the output: - -- `[ok]` means no change (this task was already completed) -- `[changed]` means the task successfully ran and the change was applied -- `[unreachable]` means the host could not be reached -- `[failed]` means the task ran but failed to complete - -`[ok]` and `[changed]` are successful outcomes. Any `[unreachable]` and `[failed]` outputs should be investigated and resolved. - -**Note:** This Ansible playbook is idempotent; once a configuration has been successfully applied, if you apply it again, all actions will report `[ok]`. - -## Executing Tensorflow Jobs in Your New Environment - -1. Point your browser to http://<hostname>:8888 and login with the password you provided. -2. The `jupyter.volumes.source` folder will be mounted as the `notebooks` folder. -3. Edit and execute your Jupyter notebooks as normal! - -### Command Line Access - -If you need to drop into a GPU-powered TensorFlow environment, SSH into the remote machine and execute the following: - -```bash -$ docker run --runtime=nvidia -it --rm tensorflow/tensorflow:latest-gpu-py3 bash -``` - -**Note:** You must be a member of the `docker` group or have `sudo` access on the _remote machine_ to execute docker commands. - ---- - -## Additional Files - -- [ansible.cfg](ansible.cfg) enables SSH credential forwarding. This is a necessary step during data synchronization, as Ansible delegates those credentials to the master/writer host to push the data folder out to each of the mirrors. -- [Dockerfile](Dockerfile) is used to build the Arricor TensorFlow image. See [Docker.md](Docker.md) for additional details. -- [hosts.example](hosts.example) is an example of the Ansible inventory hosts file saved in `/etc/ansible/hosts` -- [secrets.example.yml](secrets.example.yml) is an example of the expected structure of the `secrets.yml` file diff --git a/sample-run/tensorflow/ToolSelection.md b/sample-run/tensorflow/ToolSelection.md deleted file mode 100644 index 117fa83..0000000 --- a/sample-run/tensorflow/ToolSelection.md +++ /dev/null @@ -1,93 +0,0 @@ -# Configuration Management Tool Selection - -## Summary - -Ansible is proposed as the configuration management tool for this project due to the following characteristics: - -- no agent installation on the managed nodes -- push-based configuration changes do not require a separate machine to be provisioned -- commands can be pushed from any developer/operator workstation -- all updates are performed over SSH - -## Tools Considered - -Where a tool offers an open source and an enterprise version (e.g., Puppet, Terraform), only the open source version was considered. Web or other graphical user interfaces (GUIs) were not considered, as the goal of this phase is to produce infrastructure as code which can be run (and re-run) from the command line interface (CLI). - -- [Ansible](#Ansible) -- [Chef](#Chef) -- [Puppet](#Puppet) -- [Salt](#Salt) -- [Terraform](#Terraform) - -### Ansible - -[Ansible's simple interface and usability fit right into the sys admin mindset, and in a shop with lots of Linux and Unix systems, Ansible is quick and easy to run right out of the gate](https://www.infoworld.com/article/2609482/data-center/data-center-review-puppet-vs-chef-vs-ansible-vs-Salt.html?page=4). - -#### Ansible Pros - -- Requires no agent installation on the managed nodes. -- Push-based. -- Does not require a separate machine to be provisioned. Commands can be pushed from any developer/operator workstation. -- All updates are performed over SSH. - -#### Ansible Cons - -- Push-based synchronization means that the configuration is only refreshed on demand. - -### Chef - -[Chef has a stable and well-designed layout, and while it's not quite up to the level of Puppet in terms of raw features, it's a very capable solution. Chef may pose the most difficult learning curve to administrators who lack significant programming experience, but it could be the most natural fit for development-minded admins and development shops](https://www.infoworld.com/article/2609482/data-center/data-center-review-puppet-vs-chef-vs-ansible-vs-Salt.html?page=4). - -#### Chef Pros - -- AWS OpsWorks is based on Chef; any future migration to AWS could benefit from the existing configuration files. - -#### Chef Cons - -- **(Fatal)** Requires a master server and a workstation that will need to be maintained separately from the already provisioned deep learning machines. -- **(Fatal)** Requires an agent (client) to be installed on each machine. -- Requires an on-premises Chef server and [Push Jobs](https://docs.chef.io/push_jobs.html) plugin to push configurations manually. Otherwise, managed machines must check in with the master on a schedule. - -### Puppet - -[Puppet is the most mature and probably the most approachable of the four from a usability standpoint, though a solid knowledge of Ruby is highly recommended. Puppet is not as streamlined as Ansible or Salt, and its configuration can get Byzantine at times. Puppet is the safest bet for heterogeneous environments, but you may find Ansible or Salt to be a better fit in a larger or more homogenous infrastructure](https://www.infoworld.com/article/2609482/data-center/data-center-review-puppet-vs-chef-vs-ansible-vs-salt.html?page=4). - -#### Puppet Pros - -- Configuration can be kept up to date via a schedule or manual configuration pushes. - -#### Puppet Cons - -- **(Fatal)** Requires a "puppet master" (server) that will need to be maintained separately from the already provisioned machines. -- **(Fatal)** Requires an agent (client) to be installed on each machine. - -### Salt - -[Salt is the sleekest and most robust of the four, and like Ansible it will resonate with sys admins. Highly scalable and quite capable, Salt is hamstrung only by the Web UI](https://www.infoworld.com/article/2609482/data-center/data-center-review-puppet-vs-chef-vs-ansible-vs-salt.html?page=4). - -#### Salt Pros - -- Push-based. -- Can function via regular SSH or via client agents called "minions". - -#### Salt Cons - -- When running agentless, push-based synchronization means that the configuration is only refreshed on demand. - -### Terraform - -The initial hope was that [Terraform](https://terraform.io) could be used for configuration management for this project. However, after reviewing the [available Terraform provider plugins](https://terraform.io/docs/providers) and other available material, it was agreed on 23 January 2019 that Terraform was not suitable for this stage of the project. - -In the future, if Arricor migrates its instances to a cloud service provider (CSP) such as AWS, Azure, or Google Cloud Platform, this configuration may be reusable as a first step via the [terraform-provisioner-ansible](https://github.com/jonmorehouse/terraform-provisioner-ansible) plugin. This plugin allows the user to create CSP instances, then execute the given Ansible playbook against those instances. - -#### Terraform Pros - -- Cloud native with support for multiple service providers. - -#### Terraform Cons - -- **(Fatal)** No support for provisioning bare metal instances. - -## Conclusion - -Ansible allows for management of a server from scratch, with only the requirement to have a valid username and password. Additionally, its agentless approach avoids having to configure and maintain an admin server, further simplifying operations at the scale Arricor requires. diff --git a/sample-run/tensorflow/ansible.cfg b/sample-run/tensorflow/ansible.cfg deleted file mode 100644 index 4ac23a5..0000000 --- a/sample-run/tensorflow/ansible.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[defaults] -host_key_checking = False -inventory = hosts - -[ssh_connection] -ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s diff --git a/sample-run/tensorflow/hosts b/sample-run/tensorflow/hosts deleted file mode 100644 index 79b0fef..0000000 --- a/sample-run/tensorflow/hosts +++ /dev/null @@ -1,12 +0,0 @@ ---- -all: - vars: - ansible_python_interpreter: /usr/bin/python3 -test: - hosts: - test1: - ansible_host: 0.0.0.0 # Replace with your remote IP - ansible_port: 44003 # Replace with your remote SSH port - ansible_user: master # Replace wtih your username - ansible_ssh_pass: password - ansible_sudo_pass: password \ No newline at end of file diff --git a/sample-run/tensorflow/hosts.example b/sample-run/tensorflow/hosts.example deleted file mode 100644 index bf4edf8..0000000 --- a/sample-run/tensorflow/hosts.example +++ /dev/null @@ -1,25 +0,0 @@ ---- -all: - vars: - ansible_python_interpreter: /usr/bin/python3 -test: - hosts: - test1: - ansible_host: 127.0.0.1 # Replace with your remote IP - ansible_port: 2222 # Replace with your remote SSH port - ansible_user: vagrant # Replace wtih your username - -production: - hosts: - writer: - ansible_host: 192.168.1.100 # Replace with your remote IP - ansible_port: 2202 # Replace with your remote SSH port - ansible_user: ansible # Replace wtih your username - mirror1: - ansible_host: 192.168.1.101 # Replace with your remote IP - ansible_port: 2202 # Replace with your remote SSH port - ansible_user: ansible # Replace wtih your username - mirror2: - ansible_host: 192.168.1.102 # Replace with your remote IP - ansible_port: 2202 # Replace with your remote SSH port - ansible_user: ansible # Replace wtih your username diff --git a/sample-run/tensorflow/packages.yml b/sample-run/tensorflow/packages.yml deleted file mode 100644 index 4afd81a..0000000 --- a/sample-run/tensorflow/packages.yml +++ /dev/null @@ -1,38 +0,0 @@ -################################################################################ -# This file consists of three main sections: -# 1. vars - variables used throughout the playbook are defined here. If you want to change packages or version, this is the place to do it. -# 2. tasks - these are the steps that will be executed by the playbook. This is a straightforward, sequential playbook. -# 3. handlers - these steps run only in response to specific events. If the event does not occur, the handler does not run. -################################################################################ ---- -- hosts: test # Alternatively, 'canary', 'blue'|'green', 'all' according to your deployment strategy and your /etc/ansible/hosts file - become: yes # All commands must run as root - strategy: free # Allows all hosts to run to the end of the playbook as fast as they can - - vars_files: - - secrets.yml - - roles: - - role: packages # General Dependencies - tags: [packages] - - # - role: rsync # Rsync Setup - # tags: [rsync] - # vars: - # sync: "{{ secrets.sync }}" - - # - role: docker # General Docker setup - # tags: [docker] - # vars: - # users: "{{ secrets.users }}" - - - role: cuda # CUDA Driver Setup - tags: [cuda] - - # - role: nvidia # NVIDIA-Docker Runtime Setup - # tags: [nvidia] - - # - role: jupyter # Launch Jupyter Container in Docker - # tags: [jupyter] - # vars: - # volumes: "{{ secrets.volumes }}" diff --git a/sample-run/tensorflow/roles/cuda/README.md b/sample-run/tensorflow/roles/cuda/README.md deleted file mode 100644 index 752ecda..0000000 --- a/sample-run/tensorflow/roles/cuda/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# The `cuda` Role - -The `cuda` role installs the Nvidia CUDA GPU repository and drivers. - -This role may be executed independently by running: - -```bash -$ ansible-playbook packages.yml --tags "cuda" -``` - -On execution, this role installs the following directly to the remote machine's operating system: - -### Apt Repositories (Debian/Ubuntu) - -- Nvidia CUDA - -### Apt GPG Signing Keys (Debian/Ubuntu) - -- cudatools - -### Operating System Packages (Debian/Ubuntu) - -- cuda v10.x diff --git a/sample-run/tensorflow/roles/cuda/defaults/.gitkeep b/sample-run/tensorflow/roles/cuda/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/cuda/files/cuda-11-3_11.3.1-1_amd64.deb b/sample-run/tensorflow/roles/cuda/files/cuda-11-3_11.3.1-1_amd64.deb deleted file mode 100644 index 04dc7af27a6fefac633f0eb9eeedbc13ad564bca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2450 zcmah~36K+28V-lRG%K*MtRRduuE5C1Oy9>a97(2=$vL?v2?+F=bSCM}vbkp2J$4+1U+u6_x_0milbE&@VEU>XOkZGIw&N{4eMql9M zjrHcl)0}0+GhL5N9CN(Ks_vI=EPTCrO0Ubi4z~a8>X1+WylI>v?X?}fc&+N>p+w-w zy77mjw?A6neaYBGO)oWfy`t$Iihb9*W$LKAb>h|yL#fskTkpL6-sFvU(%w50rkeYg z_x5u?9Jn%f&ZsZe;U3hTe{IB=rV=YgXwfGHg^ZMn-+W%71Z~VZ$BexFUJL2fj zUGoxG+a|>>o!NQlt6|B{U01EguFRfHe7msKxA)qFF?)L9$NxT|fBU>k+nPI{Tq94( zWv_p`<=SrU#p*6?m%cg3uc=!4{a`7F{B>IscoY6gCQp9eWy-7<+*>_e4=)(G`kg~( zE<70>)8kBR|Gw_4Z{>Hie9&_JcU`Oc=1v?K#+<%!ob31EuoW*6<7N$bd4qd<&+#AJ zUg?bw{IYq?pcxsnjxX&u`aC>y7_^GJWncVSbIZW}b?n@w6Q*5w{)^%N(zNfnJ$*pQ z!P)GxogaJ9@3`F)OdYHD2c|B+u!F5QH~Gmg5B%`O-hDlnPu+QEbt--0?qhFnygO09 z@{KQTPGa5Z-JL=YueZ#J@ywlTyB)?ouX&<-)4(Ym7HuyzkIwhqv47~BZ1wBU zy*gt2++)5~p5>m?(?9Z@KGJ8qw5RDA%cbAGKk?jC6R(fF?O)kp+?MRLk9F(vmH%0& zpRc@U+eyc|ZNsmeJ&P}k^*_Gsu;~e`Y{e08^*=&O&YbUmwDq5z28=sct$x=1(A5Rg zz1h#2hOTY7+&<|iHs@>4s_l!LhfI6rpxpdaUvk+N&+-fJP9J#E`9dVUtJ`Do#)UhiYkwN$ zk~dWhdMcvKxx8V(iF3(@fFU^Ga!<=(PnRB5ZEv1D_u8g6df7jI_f~T9{;RA1uy5_o z(o;QKn}+Wy`D1!hS?l|A2AqmShP>5uwoi|l9TzP%mTYM6{q!i^;fwumE>1SfC9X5N zBbl8#%c()G7s`I)GJZPu^o&<_(47jwU4se(M-M8^#fn@vytHBW^+n0~nhtl1{A)Dd zZ{FOtaaZV8$9p<-4g@EL;vp_AM+GtQu)ZWh{G2NH_53+oV1y`E?qL!z;vRI2(PRf5 zMhECN8SP9BXk*-Nt&#b$2-LF4a24pT)#5M$06sC1lvSk)!~nM(Qp7|R$ft`AQ`Kk^ ztbXt~R>{bM%vSJfY&1~Aro<}H6HbFRHV?um2vb!sQUwtp0wFNqQlrK4Pd5Mza#T!K zfj|k$OEgR&BqB*HE{HIVlQ<%fsDuhED&itY(m2i21T8`|A|Wi!QUrvF2ttt*M6x{2 z!5m6qtWXToN8%SKfhiIZA%ZOKL_8?RS&0$}1eOHwUd}$6%|Hny;4sfa7*A7>$Uz86QW8P4BEr%rNs0)Fu{1}( z93)`~hI0^wLJ|d`#eqr$j`B1{6EFcwEQl35@uBYhCAG93uj?nF9!O(U(heh-DgHMq z5CkkU+P#%lSxM)f1pwJ;;&oa^rw?h3I;T0L@OVtCRn3ee;BXhu5ANwbpP> zveK&GQ(r)>-pa-XHD|y@8z5T(HX|xUazf7SsE3MlN%?TfZzr75oHorg*3|1Fj2;d% zvMGiJQ|TN@Vu}&Q5qEu58>KS<5+hKj7?0ZORKG9A6C|q-ge_#g-s4efjItM(7}%Xn z2Ynt_v=U24bx}m{gbiG64nb|slobdeqEB#!ER;jG)!FiPA64+jqIOxaat=zSD4fc> z#Egj}Yr;n8zRQWaL9{PCL0)i5ETGZ!f$ulB7p>#gj0EIdAXLO z7)~Kwj3G(65eo9+ZU>f$*z6dKG~yUL^uhI^5=xi{XSvrH4ifFp)UD9gDifE diff --git a/sample-run/tensorflow/roles/cuda/files/cuda-runtime-11-3_11.3.1-1_amd64.deb b/sample-run/tensorflow/roles/cuda/files/cuda-runtime-11-3_11.3.1-1_amd64.deb deleted file mode 100644 index ed7d4b322a19cc2631d1beee567e059c93818a56..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2424 zcma)732+nF8J02ULZ=W6se!-}z=aB|-IZ3mYm7mXb)VL;vM?A{dq}JMv?QklLrX&& z2$RGm)OG9-2%ZFHf*I@*Ou-2dAR#FOHUk#B8y))ByP?s7rHyxmy{A?!Z0$2B+7f)> zs>0dlT2sp71uwVeO((YBzqYk}V{L6~)8@S1+zVah6W0Ff#LI2FkgMHQCF;wUf2c#* z6J5rxu0xlXEomNg<~mR?WmoRu`j+$N=<-(v8sA^Fy#SxPC_T`&<%b*nsm{~=V$$gt zI^*WR!FiG93lm!g!`gwpZP{b??xnSK`O4@!woe>r3j&Z%`Iwd50*vx?x*KW&RcO?O#^k*mdHI zf8@7s?M^iv30F1=uH2cr1>MK*UcUp|$zT`ZUP)7tNwHy?eQKxd7Z*R}uO-rzE6+_) zd_TJB&SzJa{n)d=tMjGbn!aqfys~Ua!*@2<@UsJnVTZ^+>N~mI=(xy7<2P?>JfGLQ zV&VvD>+zmhtgh$nadT&^c_N3`{AOFq-)OA9q+t}bIzs(AhE zwR0A|af5gfi7y?NmRS|7|mUEisFQ8KgPq@%dSQ+K{MzpHz1?!=1Y3$~o!3JJb0wdaHK zpE|yO!ARCsPB=KOxN*>C*H~(C%-Vcdh*eK-K=*2tf4IARm~Zx={L1dXo%;ND(TUM* zLq2to?X%(s|EkznL4VSExc0uQ3CX;XgM$tG;LFQ~y#3{XQMBt{mu=X)Icxs8R!~_X z8s`~JORDs1V-*uR_iz7;s^wVUn>`=)9k$$>KWBC7=)Cf2?^R#i)TXn& z`szDq-pI3)u6K-A>^P?;gJV>iz>$+K*1M z33Arv)>s6B8n~pX`GDvCpaIRj786p2!(ob%;y4Sl5W~Q%ghD_`)=i&xU|<9$CLlMA!VD}COKF^tvIv8U z35LSN5{Sickb!UrrCBPwos_^>il(wNl-uoQafuruAc|uUR)S{_o>k^EYW_L2WOj%3 zF;GvTQHPQiHCUDXI}{=aI8$wMhzxuvnVtmze3_P(%an43N2ZpS={zAC4asC-ozmj6 z*s5VsI4YEBo#8T>P8f7SiHy+&8*xdg&%xN zG6^a)>+y)t9HfH_6>7c7rWdAMexFR{xBAQCrCc>vj>m)sz)ku?8oAiz6ye68%}%8_ zjb0W|=A$k(hlhD!j1DqQ&eEm)wWa5g^3{ zu!xSan>Dn858@F&nNcgnWE0-{6%nApKY0Wy^;IX+#p*874Qz9)vgkjhn%&3Kg zFD3_6S{qSjlB#*7PwgOZTa{B~P5X&>GN4y!;u?|FMXA!V(x}LwQ&tFlU`FYZXQ~36*U1=NK`KoF^B@^r{YWQDQ|VJrdqySy1z(16 ACjbBd diff --git a/sample-run/tensorflow/roles/cuda/handlers/main.yml b/sample-run/tensorflow/roles/cuda/handlers/main.yml deleted file mode 100644 index 6dfc62c..0000000 --- a/sample-run/tensorflow/roles/cuda/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# CUDA Driver Setup -- name: reboot the machine - reboot: - listen: "reboot the machine" diff --git a/sample-run/tensorflow/roles/cuda/meta/.gitkeep b/sample-run/tensorflow/roles/cuda/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/cuda/tasks/debian.yml b/sample-run/tensorflow/roles/cuda/tasks/debian.yml deleted file mode 100644 index 5290be8..0000000 --- a/sample-run/tensorflow/roles/cuda/tasks/debian.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Cuda Driver Package Installation -- name: copy the CUDA repo onto the machine - copy: - src: "{{ cuda.apt.repo }}" - dest: "{{ general.working_dir }}/{{ cuda.apt.repo }}" - -- name: get the CUDA signing key - apt_key: - state: present - url: "{{ cuda.apt.signing_key_url }}" - -- name: add the CUDA repo - apt: - deb: "{{ general.working_dir }}/{{ cuda.apt.repo }}" - state: present - -- name: install the CUDA drivers - apt: - name: "{{ cuda.apt.package }}" - update_cache: yes - notify: "reboot the machine" diff --git a/sample-run/tensorflow/roles/cuda/tasks/main.yml b/sample-run/tensorflow/roles/cuda/tasks/main.yml deleted file mode 100644 index bd512db..0000000 --- a/sample-run/tensorflow/roles/cuda/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- import_tasks: debian.yml - when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu diff --git a/sample-run/tensorflow/roles/cuda/templates/.gitkeep b/sample-run/tensorflow/roles/cuda/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/cuda/vars/main.yml b/sample-run/tensorflow/roles/cuda/vars/main.yml deleted file mode 100644 index dfb2793..0000000 --- a/sample-run/tensorflow/roles/cuda/vars/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -cuda: - apt: - package: cuda=11.* # known good with 10.0.130-1 - repo: cuda-11-3_11.3.1-1_amd64.deb # Network install - dependency: cuda-runtime-11-3_11.3.1-1_amd64.deb - signing_key_url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub - -general: - working_dir: /tmp diff --git a/sample-run/tensorflow/roles/docker/README.md b/sample-run/tensorflow/roles/docker/README.md deleted file mode 100644 index e4c625c..0000000 --- a/sample-run/tensorflow/roles/docker/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# The `docker` Role - -The `docker` role installs the Docker CE repository and runtime packages. - -This role may be executed independently by running: - -```bash -$ ansible-playbook packages.yml --tags "docker" -``` - -On execution, this role installs the following directly to the remote machine's operating system: - -### Apt Repositories (Debian/Ubuntu) - -- Docker CE - -### Apt GPG Signing Keys (Debian/Ubuntu) - -- Docker Release (CE deb) - -### Operating System Packages (Debian/Ubuntu) - -- docker-ce v5:18.x - -### System Services - -- `docker` (enabled and started) - -### User Groups - -- docker - -All users listed under `secrets.users` in `secrets.yml` will be added to the `docker` group. diff --git a/sample-run/tensorflow/roles/docker/defaults/.gitkeep b/sample-run/tensorflow/roles/docker/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/docker/files/.gitkeep b/sample-run/tensorflow/roles/docker/files/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/docker/handlers/.gitkeep b/sample-run/tensorflow/roles/docker/handlers/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/docker/meta/.gitkeep b/sample-run/tensorflow/roles/docker/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/docker/tasks/debian.yml b/sample-run/tensorflow/roles/docker/tasks/debian.yml deleted file mode 100644 index 4228c04..0000000 --- a/sample-run/tensorflow/roles/docker/tasks/debian.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# Docker Package Installation -- name: add Docker's official GPG key - apt_key: - state: present - url: "{{ docker.apt.signing_key_url }}" - -- name: add Docker apt repository - apt_repository: - repo: "{{ docker.apt.repo }}" - state: present - -- name: install Docker CE - apt: - name: "{{ docker.apt.package }}" - state: present - update_cache: yes diff --git a/sample-run/tensorflow/roles/docker/tasks/main.yml b/sample-run/tensorflow/roles/docker/tasks/main.yml deleted file mode 100644 index 9cebc69..0000000 --- a/sample-run/tensorflow/roles/docker/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# General Docker Setup -- import_tasks: debian.yml - when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu - -- name: ensure the docker service starts at boot - systemd: - name: "{{ docker.service_name }}" - enabled: yes - state: started - -- name: create Docker group to avoid running as root - group: - name: "{{ docker.group_name }}" - state: present - -- name: add users to Docker group - user: - append: yes - groups: - - "{{ docker.group_name }}" - name: "{{ item }}" - loop: "{{ users }}" diff --git a/sample-run/tensorflow/roles/docker/templates/.gitkeep b/sample-run/tensorflow/roles/docker/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/docker/vars/main.yml b/sample-run/tensorflow/roles/docker/vars/main.yml deleted file mode 100644 index 6506998..0000000 --- a/sample-run/tensorflow/roles/docker/vars/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -docker: - apt: - package: docker-ce=5:18.* # known good with 5:18.09.1~3-0~ubuntu-bionic - repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable - signing_key_url: https://download.docker.com/linux/ubuntu/gpg - group_name: docker - service_name: docker diff --git a/sample-run/tensorflow/roles/nvidia/README.md b/sample-run/tensorflow/roles/nvidia/README.md deleted file mode 100644 index b5c4446..0000000 --- a/sample-run/tensorflow/roles/nvidia/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# The `nvidia` Role - -The `nvidia` role installs the Nvidia Docker runtime, enabling access to Nvidia CUDA GPU hardware from within Docker containers. - -This role may be executed independently by running: - -```bash -$ ansible-playbook packages.yml --tags "nvidia" -``` - -On execution, this role installs the following directly to the remote machine's operating system: - -### Apt Repositories (Debian/Ubuntu) - -- libnvidia_container -- nvidia_container_runtime -- nvidia_docker - -### Apt GPG Signing Keys (Debian/Ubuntu) - -- NVIDIA CORPORATION (Open Source Projects) - -### Operating System Packages (Debian/Ubuntu) - -- nvidia-docker2 v2.x - -Containers are launched with the Nvidia Docker runtime by passing the `--runtime=nvidia` flag, e.g., - -```bash -$ docker run --rm --runtime=nvidia nvidia/cuda nvidia-smi -``` diff --git a/sample-run/tensorflow/roles/nvidia/defaults/.gitkeep b/sample-run/tensorflow/roles/nvidia/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/nvidia/files/.gitkeep b/sample-run/tensorflow/roles/nvidia/files/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/nvidia/handlers/main.yml b/sample-run/tensorflow/roles/nvidia/handlers/main.yml deleted file mode 100644 index 18e548b..0000000 --- a/sample-run/tensorflow/roles/nvidia/handlers/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: reload the docker service - systemd: - name: docker - state: reloaded - listen: "reload the docker service" diff --git a/sample-run/tensorflow/roles/nvidia/meta/.gitkeep b/sample-run/tensorflow/roles/nvidia/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/nvidia/tasks/debian.yml b/sample-run/tensorflow/roles/nvidia/tasks/debian.yml deleted file mode 100644 index dc1911a..0000000 --- a/sample-run/tensorflow/roles/nvidia/tasks/debian.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -# Nvidia Docker Runtime Installation -- name: get the NVIDIA signing key - apt_key: - state: present - url: "{{ nvidia.apt.signing_key_url }}" - -- name: add the libnvidia-container repo - apt_repository: - filename: "{{ nvidia.apt.repo_filename }}" - repo: "{{ nvidia.apt.repos.libnvidia_container }}" - state: present - -- name: add the nvidia-container-runtime repo - apt_repository: - filename: "{{ nvidia.apt.repo_filename }}" - repo: "{{ nvidia.apt.repos.nvidia_container_runtime }}" - state: present - -- name: add the nvidia-docker repo - apt_repository: - filename: "{{ nvidia.apt.repo_filename }}" - repo: "{{ nvidia.apt.repos.nvidia_docker }}" - state: present - -- name: install the nvidia-docker2 runtime - apt: - name: "{{ nvidia.apt.package }}" - state: present - update_cache: yes - notify: "reload the docker service" diff --git a/sample-run/tensorflow/roles/nvidia/tasks/main.yml b/sample-run/tensorflow/roles/nvidia/tasks/main.yml deleted file mode 100644 index b7581b4..0000000 --- a/sample-run/tensorflow/roles/nvidia/tasks/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -# Nvidia Docker Runtime Setup -- import_tasks: debian.yml - when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu diff --git a/sample-run/tensorflow/roles/nvidia/templates/.gitkeep b/sample-run/tensorflow/roles/nvidia/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/nvidia/vars/main.yml b/sample-run/tensorflow/roles/nvidia/vars/main.yml deleted file mode 100644 index 83064d2..0000000 --- a/sample-run/tensorflow/roles/nvidia/vars/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -nvidia: - apt: - package: nvidia-docker2=2.* # known good with 2.0.3+docker18.09.1-1 - repo_filename: nvidia_github_io_nvidia_docker_ubuntu18_04_nvidia_docker.list - repos: - libnvidia_container: deb https://nvidia.github.io/libnvidia-container/ubuntu18.04/$(ARCH) / - nvidia_container_runtime: deb https://nvidia.github.io/nvidia-container-runtime/ubuntu18.04/$(ARCH) / - nvidia_docker: deb https://nvidia.github.io/nvidia-docker/ubuntu18.04/$(ARCH) / - signing_key_url: https://nvidia.github.io/nvidia-docker/gpgkey diff --git a/sample-run/tensorflow/roles/packages/README.md b/sample-run/tensorflow/roles/packages/README.md deleted file mode 100644 index 9d46317..0000000 --- a/sample-run/tensorflow/roles/packages/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# The `packages` Role - -The `packages` role installs the operating system and Python pre-requisites that are required to execute the other roles. - -This role may be executed independently by running: - -```bash -$ ansible-playbook packages.yml --tags "packages" -``` - -On execution, this role installs the following directly to the remote machine's operating system: - -### Operating System Packages (Debian/Ubuntu) - -- apt-transport-https v1.x -- ca-certificates v20180409 -- curl v7.x -- gnupg-agent v2.x -- python-apt v1.x -- python-pip v9.x -- python-setuptools v39.x -- software-properties-common v0.96.24.32.7 - -### Python (pip) Packages - -- docker v3.7.0 or greater diff --git a/sample-run/tensorflow/roles/packages/defaults/.gitkeep b/sample-run/tensorflow/roles/packages/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/packages/files/.gitkeep b/sample-run/tensorflow/roles/packages/files/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/packages/handlers/.gitkeep b/sample-run/tensorflow/roles/packages/handlers/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/packages/meta/.gitkeep b/sample-run/tensorflow/roles/packages/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/packages/tasks/debian.yml b/sample-run/tensorflow/roles/packages/tasks/debian.yml deleted file mode 100644 index e7e0830..0000000 --- a/sample-run/tensorflow/roles/packages/tasks/debian.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# General Dependencies -- name: install general apt dependencies - apt: - name: "{{ general.dependencies.apt }}" - state: present - update_cache: yes - -- name: install general pip dependencies for Ansible - pip: - name: "{{ general.dependencies.pip }}" - state: present diff --git a/sample-run/tensorflow/roles/packages/tasks/main.yml b/sample-run/tensorflow/roles/packages/tasks/main.yml deleted file mode 100644 index bd512db..0000000 --- a/sample-run/tensorflow/roles/packages/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- import_tasks: debian.yml - when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu diff --git a/sample-run/tensorflow/roles/packages/templates/.gitkeep b/sample-run/tensorflow/roles/packages/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/packages/vars/main.yml b/sample-run/tensorflow/roles/packages/vars/main.yml deleted file mode 100644 index 2e8186b..0000000 --- a/sample-run/tensorflow/roles/packages/vars/main.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -# General Dependencies -general: - dependencies: - apt: - - apt-transport-https # known good with 1.6.6ubuntu0.1 all - - ca-certificates # known good with 20180409 - - curl # known good with 7.58.0-2ubuntu3.5 - - gnupg-agent # known good with 2.2.4-1ubuntu1.2 - - python3-pip # known good with 9.0.1-2.3~ubuntu1 - - software-properties-common # known good with 0.96.24.32.7 - - git - - golang - pip: - - docker>=3.7.0 # known good with 3.7.0 diff --git a/sample-run/tensorflow/roles/rsync/defaults/.gitkeep b/sample-run/tensorflow/roles/rsync/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/rsync/files/rsyncd.example.conf b/sample-run/tensorflow/roles/rsync/files/rsyncd.example.conf deleted file mode 100644 index 1f34942..0000000 --- a/sample-run/tensorflow/roles/rsync/files/rsyncd.example.conf +++ /dev/null @@ -1,16 +0,0 @@ -motd file = /etc/rsyncd.motd -log file = /var/log/rsyncd.log -pid file = /var/run/rsyncd.pid -lock file = /var/run/rsync.lock - -[jupyter_data] - path = /nfs/jupyter/data - comment = Jupyter data files - uid = nobody - gid = nobody - read only = yes - list = yes - auth users = @rsync:ro - secrets file = /etc/rsyncd.secrets - hosts allow = 192.168.1.0/24 - hosts deny = 0.0.0.0/0 diff --git a/sample-run/tensorflow/roles/rsync/handlers/main.yml b/sample-run/tensorflow/roles/rsync/handlers/main.yml deleted file mode 100644 index ce7e8e5..0000000 --- a/sample-run/tensorflow/roles/rsync/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: restart the rsync service - systemd: - name: "{{ rsync.service_name }}" - state: restarted diff --git a/sample-run/tensorflow/roles/rsync/meta/.gitkeep b/sample-run/tensorflow/roles/rsync/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/rsync/tasks/main.yml b/sample-run/tensorflow/roles/rsync/tasks/main.yml deleted file mode 100644 index 211ade0..0000000 --- a/sample-run/tensorflow/roles/rsync/tasks/main.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# Configure the rsync daemon on the master -- name: copy rsync.conf up to the server - copy: - src: "{{ rsync.config_file.src }}" - dest: "{{ rsync.config_file.dest }}" - notify: - - restart the rsync service - when: inventory_hostname == rsync.master - -- name: configure rsync on the server - service: - enabled: yes - name: "{{ rsync.service_name }}" - state: started - when: inventory_hostname == rsync.master - -# Synchronize the data deploy across hosts -- name: synchronize the data deploy - delegate_to: "{{ rsync.master }}" - synchronize: - src: "{{ sync.source }}" - dest: "{{ sync.destination }}" - rsync_opts: "{{ rsync.options }}" - use_ssh_args: yes - when: inventory_hostname != rsync.master diff --git a/sample-run/tensorflow/roles/rsync/templates/.gitkeep b/sample-run/tensorflow/roles/rsync/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/rsync/vars/main.yml b/sample-run/tensorflow/roles/rsync/vars/main.yml deleted file mode 100644 index 3f61189..0000000 --- a/sample-run/tensorflow/roles/rsync/vars/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -rsync: - config_file: - src: rsyncd.conf - dest: /etc/rsyncd.conf - master: ml1 - options: - - "-e ssh" - - "--no-motd" - service_name: rsync diff --git a/sample-run/tensorflow/secrets.example.yml b/sample-run/tensorflow/secrets.example.yml deleted file mode 100644 index bc64cd4..0000000 --- a/sample-run/tensorflow/secrets.example.yml +++ /dev/null @@ -1,15 +0,0 @@ -################################################################################ -# This file contains secrets that should not be checked into VCS. -################################################################################ ---- -secrets: - # Cachux7I - password_hash: sha1:b9bbcb5e92ad:0358d2ce7c34192afa50fc5e5143ed91d75eda0d - sync: - source: /nfs/data - destination: /nfs - users: - - ansible - volumes: - data: /nfs/data:/data - source: /nfs/src:/tf diff --git a/sample-run/tensorflow/secrets.yml b/sample-run/tensorflow/secrets.yml deleted file mode 100644 index bc64cd4..0000000 --- a/sample-run/tensorflow/secrets.yml +++ /dev/null @@ -1,15 +0,0 @@ -################################################################################ -# This file contains secrets that should not be checked into VCS. -################################################################################ ---- -secrets: - # Cachux7I - password_hash: sha1:b9bbcb5e92ad:0358d2ce7c34192afa50fc5e5143ed91d75eda0d - sync: - source: /nfs/data - destination: /nfs - users: - - ansible - volumes: - data: /nfs/data:/data - source: /nfs/src:/tf From 85bf56ec0160c61ec960cd2e6df0028e1c44501a Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Fri, 27 Aug 2021 14:01:06 +0400 Subject: [PATCH 02/11] added generate function to the ClI --- cmd/action.go | 19 +++++++++++++++ cmd/flags.go | 14 +++++++++++ config/config.go | 35 +++++++++++++++++----------- config/config_test.go | 19 +++++++++++++++ generate/generate.go | 54 +++++++++++++++++++++++++++++-------------- 5 files changed, 111 insertions(+), 30 deletions(-) diff --git a/cmd/action.go b/cmd/action.go index b8807b2..8665f07 100644 --- a/cmd/action.go +++ b/cmd/action.go @@ -4,6 +4,7 @@ import ( "fmt" "git.sr.ht/~akilan1999/p2p-rendering-computation/client" "git.sr.ht/~akilan1999/p2p-rendering-computation/config" + "git.sr.ht/~akilan1999/p2p-rendering-computation/generate" "git.sr.ht/~akilan1999/p2p-rendering-computation/p2p" "git.sr.ht/~akilan1999/p2p-rendering-computation/plugin" "git.sr.ht/~akilan1999/p2p-rendering-computation/server" @@ -243,6 +244,24 @@ var CliAction = func(ctx *cli.Context) error { client.PrettyPrint(groups) } } + // -- REMOVE ON REGULAR RELEASE -- + // when flag --gen is called an extension + // of the project is created to repurpose + // the project for custom purpose + if Generate != "" { + err := generate.GenerateNewProject(Generate) + if err != nil { + fmt.Println(err) + } else { + fmt.Println("Created new folder: " + Generate) + fmt.Println("1. Enter inside " + Generate + "directory") + fmt.Println("2. go mod init ") + fmt.Println("3. sh install.sh " + Generate) + fmt.Println("4. ./" + Generate + " -h (This is to test if the binary is working") + } + } + //-------------------------------- + return nil diff --git a/cmd/flags.go b/cmd/flags.go index 0fe8fc9..07ec2a5 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -28,6 +28,10 @@ var ( Groups bool RemoveContainerGroup bool RemoveGroup string + // Generate only allowed in dev release + // -- REMOVE ON REGULAR RELEASE -- + Generate string + //-------------------------------- ) var AppConfigFlags = []cli.Flag{ @@ -187,4 +191,14 @@ var AppConfigFlags = []cli.Flag{ EnvVars: []string{"REMOVE_GROUP"}, Destination: &RemoveGroup, }, + // Generate only allowed in dev release + // -- REMOVE ON REGULAR RELEASE -- + &cli.StringFlag{ + Name: "Generate", + Aliases: []string{"gen"}, + Usage: "Generates a new copy of P2PRC which can be modified based on your needs", + EnvVars: []string{"GENERATE"}, + Destination: &Generate, + }, + //-------------------------------- } \ No newline at end of file diff --git a/config/config.go b/config/config.go index 72c34a7..15e6f9b 100644 --- a/config/config.go +++ b/config/config.go @@ -8,12 +8,7 @@ import ( var ( defaultPath string - defaults = map[string]interface{}{ - "IPTable": "/etc/p2p-rendering/ip_table.json", - "DockerContainers": "/home/akilan/Documents/p2prendering/p2p-redering-computation/server/docker/containers/", - "DefaultDockerFile": "/home/akilan/Documents/p2prendering/p2p-redering-computation/server/docker/containers/docker-ubuntu-sshd/", - "SpeedTestFile":"/etc/p2p-rendering/50.bin", - } + defaults = map[string]interface{}{} configName = "config" configType = "json" configFile = "config.json" @@ -67,18 +62,30 @@ func Copy(src, dst string) error { return out.Close() } +// GetPathP2PRC Getting P2PRC Directory from environment variable +func GetPathP2PRC()(string,error) { + curDir := os.Getenv("P2PRC") + return curDir + "/", nil +} + +// GetCurrentPath Getting P2PRC Directory from environment variable +func GetCurrentPath()(string,error) { + curDir := os.Getenv("PWD") + return curDir + "/", nil +} + // SetDefaults This function to be called only during a // make install func SetDefaults() error { - //Getting Current Directory from environment variable - curDir := os.Getenv("P2PRC") - //Setting current directory to default path - defaultPath = curDir + "/" + defaultPath, err := GetPathP2PRC() + if err != nil{ + return err + } //Creates ip_table.json in the json directory - err := Copy("p2p/ip_table.json","p2p/iptable/ip_table.json") + err = Copy("p2p/ip_table.json","p2p/iptable/ip_table.json") if err != nil { return err } @@ -129,9 +136,11 @@ func SetDefaults() error { func ConfigInit()(*Config,error) { - curDir := os.Getenv("P2PRC") //Setting current directory to default path - defaultPath = curDir + "/" + defaultPath, err := GetPathP2PRC() + if err != nil{ + return nil, err + } //Paths to search for config file configPaths = append(configPaths, defaultPath) diff --git a/config/config_test.go b/config/config_test.go index af4e5d9..c88ffb8 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "testing" ) @@ -17,3 +18,21 @@ func TestSetDefaults(t *testing.T) { t.Error(err) } } + +func TestGetCurrentPath(t *testing.T) { + path, err := GetCurrentPath() + if err != nil { + fmt.Println(err) + t.Error(err) + } + fmt.Println(path) +} + +func TestGetPathP2PRC(t *testing.T) { + path, err := GetPathP2PRC() + if err != nil { + fmt.Println(err) + t.Error(err) + } + fmt.Println(path) +} diff --git a/generate/generate.go b/generate/generate.go index b65824c..42434a4 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -3,29 +3,43 @@ package generate import ( - "fmt" + "git.sr.ht/~akilan1999/p2p-rendering-computation/config" "github.com/otiai10/copy" "os" "strings" ) +type NewProject struct { + Name string + NewDir string + P2PRCPath string + Option *copy.Options +} + +// GenerateNewProject creates a new copy of the P2PRC +// project for custom modification func GenerateNewProject(name string) error { + // Create new variable of type NewProject + var newProject NewProject // Get path of the current directory - curDir := os.Getenv("PWD") - // Add slash to the end - curDir = curDir + "/" + curDir, err := config.GetCurrentPath() + if err != nil { + return err + } // Folder name of the new generated project - NewProject := curDir + name + "/" - fmt.Println(NewProject) + newProject.NewDir = curDir + name + "/" // Create a new folder based on name entered - err := CreateFolder(name, curDir) + err = CreateFolder(name, curDir) if err != nil { return err } // get path of P2PRC - P2PRCPATH := os.Getenv("P2PRC") - // Add slash to the end - P2PRCPATH = P2PRCPATH + "/" + P2PRCPATH, err := config.GetPathP2PRC() + if err != nil { + return err + } + // Assign P2PRC path to the newly generated project + newProject.P2PRCPath = P2PRCPATH // Steps: // - copy all files from P2PRC // - remove go.mod and go.sum and create new ones @@ -33,13 +47,21 @@ func GenerateNewProject(name string) error { // Files we require to skip var Options copy.Options - // Skips the appropriate files and directories not needed + // Skip or have the appropriate files and directories not needed Options.Skip = func(src string) (bool, error) { switch { + case strings.HasSuffix(src, "main.go"): + return false, nil + case strings.HasSuffix(src, ".go"): + return true, nil case strings.HasSuffix(src, "go.mod"): return true, nil case strings.HasSuffix(src, "go.sum"): return true, nil + case strings.HasSuffix(src, ".idea"): + return true, nil + case strings.HasSuffix(src, "Makefile"): + return true, nil case strings.HasSuffix(src, name): return true, nil default: @@ -47,16 +69,14 @@ func GenerateNewProject(name string) error { } } + // Storing type option in the struct new project + newProject.Option = &Options + // Copies all files from P2PRC to the new project created - err = copy.Copy(P2PRCPATH, NewProject,Options) + err = copy.Copy(newProject.P2PRCPath, newProject.NewDir,*newProject.Option) if err != nil { return err } - // Installing new project - //cmd := exec.Command("sh","install.sh",name) - //if err := cmd.Run(); err != nil { - // return err - //} return nil } From 8fd9ccfa2dba08e85cec91bbfd7f2d98237191f8 Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Sat, 28 Aug 2021 21:46:36 +0400 Subject: [PATCH 03/11] added AST feature and test case --- .gitignore | 3 ++- cmd/action.go | 2 +- generate/Test/.gitkeep | 0 generate/generate.go | 28 +++++++++++++++++++--- generate/generate_test.go | 45 ++++++++++++++++++++++++++++++++++- generate/modfyGenerate.go | 49 +++++++++++++++++++++++++++++++++++++++ generate/testcaseAST.go | 9 +++++++ go.mod | 1 + go.sum | 2 ++ 9 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 generate/Test/.gitkeep create mode 100644 generate/modfyGenerate.go create mode 100644 generate/testcaseAST.go diff --git a/.gitignore b/.gitignore index da58fd3..62cbcde 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ plugin/deploy/ #ignore track container file client/trackcontainers/ # Test generated files -generate/p2prctest \ No newline at end of file +generate/p2prctest +generate/Test \ No newline at end of file diff --git a/cmd/action.go b/cmd/action.go index 8665f07..4d17a82 100644 --- a/cmd/action.go +++ b/cmd/action.go @@ -249,7 +249,7 @@ var CliAction = func(ctx *cli.Context) error { // of the project is created to repurpose // the project for custom purpose if Generate != "" { - err := generate.GenerateNewProject(Generate) + err := generate.GenerateNewProject(Generate,"test") if err != nil { fmt.Println(err) } else { diff --git a/generate/Test/.gitkeep b/generate/Test/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/generate/generate.go b/generate/generate.go index 42434a4..0e8131b 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -5,22 +5,31 @@ package generate import ( "git.sr.ht/~akilan1999/p2p-rendering-computation/config" "github.com/otiai10/copy" + "go/ast" + "go/token" "os" + "os/exec" "strings" ) type NewProject struct { Name string + Module string NewDir string P2PRCPath string - Option *copy.Options + Option *copy.Options + Token *token.FileSet + AST *ast.File + FileNameAST string } // GenerateNewProject creates a new copy of the P2PRC // project for custom modification -func GenerateNewProject(name string) error { +func GenerateNewProject(name string, module string) error { // Create new variable of type NewProject var newProject NewProject + //Setting module name to the new project + newProject.Module = module // Get path of the current directory curDir, err := config.GetCurrentPath() if err != nil { @@ -78,6 +87,8 @@ func GenerateNewProject(name string) error { return err } + // Creating a new go.mod file in the appropriate directory + return nil } @@ -89,4 +100,15 @@ func CreateFolder(name string,path string) error { return err } return nil -} \ No newline at end of file +} + +// CreateGoMod Creates a new go module for the new project created +func (a *NewProject)CreateGoMod() error { + // Create new go.mod in the appropriate directory + cmd := exec.Command("go","mod","init",a.Module) + cmd.Dir = a.NewDir + if err := cmd.Start(); err != nil { + return err + } + return nil +} diff --git a/generate/generate_test.go b/generate/generate_test.go index 9da7c01..5853075 100644 --- a/generate/generate_test.go +++ b/generate/generate_test.go @@ -2,6 +2,7 @@ package generate import ( "fmt" + "git.sr.ht/~akilan1999/p2p-rendering-computation/config" "testing" ) @@ -18,9 +19,51 @@ func TestCreateFolder(t *testing.T) { // Testing if a new project is created successfully func TestGenerateNewProject(t *testing.T) { // Checking if a new project is created successfully - err := GenerateNewProject("p2prctest") + err := GenerateNewProject("p2prctest","p2prctest") if err != nil { fmt.Println(err) t.Error(err) } +} + +// Testing AST function to ensure imports are +// working as intended +func TestChangingImportAST(t *testing.T) { + // Create a new variable of type + var np NewProject + // Get current directory + path, err := config.GetCurrentPath() + if err != nil { + fmt.Println(err) + t.Error(err) + } + // Create testcase scenario + err = config.Copy(path + "testcaseAST.go", path + "/Test/testcaseAST.go") + if err != nil { + fmt.Println(err) + t.Error(err) + } + // Sets new directory to the folder test + np.NewDir = path + "Test/" + // Sets file name to be opened and modified in the AST + np.FileNameAST = path + "Test/" + "testcaseAST.go" + // Call the Read AST function + err = np.GetASTGoFile() + if err != nil { + fmt.Println(err) + t.Error(err) + } + // Change an import + err = np.ChangeImports("fmt", "lolol") + if err != nil { + fmt.Println(err) + t.Error(err) + } + // Write those saved changes + err = np.WriteGoAst() + if err != nil { + fmt.Println(err) + t.Error(err) + } + } \ No newline at end of file diff --git a/generate/modfyGenerate.go b/generate/modfyGenerate.go new file mode 100644 index 0000000..cc66061 --- /dev/null +++ b/generate/modfyGenerate.go @@ -0,0 +1,49 @@ +package generate + +import ( + "go/parser" + "go/printer" + "go/token" + "os" +) + + +// GetASTGoFile Gets AST of the Go file provided +func (np *NewProject)GetASTGoFile() error{ + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, np.FileNameAST, nil, parser.ParseComments) + if err != nil { + return err + } + //Write Token information to the struct + np.Token = fset + // Write AST information the struct + np.AST = node + return nil +} + +// ChangeImports Changes import of the AST +func (np *NewProject)ChangeImports(CurrentImport string,ChangedImport string) error { + // Iterating through the loop and changing the appropriate import + for i, spec := range np.AST.Imports { + // If the current import is found then change it + if spec.Path.Value == "\"" + CurrentImport + "\"" { + np.AST.Imports[i].Path.Value = "\"" + ChangedImport + "\"" + } + } + return nil +} + +// WriteGoAst Write changed imports back to the AST +func (np *NewProject)WriteGoAst() error { + // write new AST to file + f, err := os.Create(np.FileNameAST) + if err != nil { + return nil + } + defer f.Close() + if err := printer.Fprint(f, np.Token, np.AST); err != nil { + return err + } + return nil +} \ No newline at end of file diff --git a/generate/testcaseAST.go b/generate/testcaseAST.go new file mode 100644 index 0000000..0d1dfbb --- /dev/null +++ b/generate/testcaseAST.go @@ -0,0 +1,9 @@ +package generate + +import ( + "fmt" +) + +func TestCaseAST() { + fmt.Println("lol") +} \ No newline at end of file diff --git a/go.mod b/go.mod index 06019f1..d5d86b2 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/apenella/go-ansible v1.1.0 github.com/containerd/continuity v0.0.0-20210315143101-93e15499afd5 // indirect + github.com/dave/jennifer v1.4.1 github.com/docker/docker v20.10.0-beta1.0.20201113105859-b6bfff2a628f+incompatible github.com/docker/go-connections v0.4.0 github.com/gin-gonic/gin v1.6.3 diff --git a/go.sum b/go.sum index 3213490..db2fe1e 100644 --- a/go.sum +++ b/go.sum @@ -177,6 +177,8 @@ github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1S github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw= +github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 5c2b36c08602fb0b1ed0b8e3192ab367a59d26c7 Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Sat, 28 Aug 2021 21:54:09 +0400 Subject: [PATCH 04/11] created test case for go.mod --- generate/generate.go | 2 +- generate/generate_test.go | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 0e8131b..04a6a62 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -107,7 +107,7 @@ func (a *NewProject)CreateGoMod() error { // Create new go.mod in the appropriate directory cmd := exec.Command("go","mod","init",a.Module) cmd.Dir = a.NewDir - if err := cmd.Start(); err != nil { + if err := cmd.Run(); err != nil { return err } return nil diff --git a/generate/generate_test.go b/generate/generate_test.go index 5853075..a6e0c5b 100644 --- a/generate/generate_test.go +++ b/generate/generate_test.go @@ -29,7 +29,7 @@ func TestGenerateNewProject(t *testing.T) { // Testing AST function to ensure imports are // working as intended func TestChangingImportAST(t *testing.T) { - // Create a new variable of type + // Create a new variable of type NewProject var np NewProject // Get current directory path, err := config.GetCurrentPath() @@ -66,4 +66,27 @@ func TestChangingImportAST(t *testing.T) { t.Error(err) } +} + +// Testing the if Go Mod is created +func TestNewProject_CreateGoMod(t *testing.T) { + // Create a new variable of type NewProject + var np NewProject + path, err := config.GetCurrentPath() + if err != nil { + fmt.Println(err) + t.Error(err) + } + // Set new project name as Test + np.Name = "Test" + // Set new project module as github.com/Test + np.Module = "github.com/Test" + // Set Path of the new project + np.NewDir = path + "Test/" + // Creating a go.mod file + err = np.CreateGoMod() + if err != nil { + fmt.Println(err) + t.Error(err) + } } \ No newline at end of file From 5823cf2028a5d5ac09c7d8a140e5b1b46bb879c1 Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Sat, 28 Aug 2021 23:32:05 +0400 Subject: [PATCH 05/11] added function to get current go module and added test case --- generate/generate.go | 73 ++++++++++++++++++++++++++++++++++----- generate/generate_test.go | 20 +++++++++++ go.mod | 2 +- go.sum | 3 +- 4 files changed, 87 insertions(+), 11 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index 04a6a62..c45f679 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -3,24 +3,28 @@ package generate import ( + "fmt" "git.sr.ht/~akilan1999/p2p-rendering-computation/config" "github.com/otiai10/copy" "go/ast" "go/token" + modfile "golang.org/x/mod/modfile" + "io/ioutil" "os" "os/exec" "strings" ) type NewProject struct { - Name string - Module string - NewDir string - P2PRCPath string - Option *copy.Options - Token *token.FileSet - AST *ast.File - FileNameAST string + Name string + Module string + NewDir string + P2PRCPath string + CurrentModule string + Option *copy.Options + Token *token.FileSet + AST *ast.File + FileNameAST string } // GenerateNewProject creates a new copy of the P2PRC @@ -54,9 +58,22 @@ func GenerateNewProject(name string, module string) error { // - remove go.mod and go.sum and create new ones // Files we require to skip + var Options copy.Options + // IF YOU ARE RELEASING AN EXTENSION OF P2PRC + // MODIFY HERE (AN EXTENSION FROM YOUR EXTENSION :P) // Skip or have the appropriate files and directories not needed + //---------------------------------------------------------------- + // Action performed: + // - Ensuring main.go file exists + // - Skipping all .go files + // - Skipping go.mod file + // - Skipping go.sum file + // - Skipping .idea/ directory + // - Skipping Makefile file + // - Skipping / directory + //---------------------------------------------------------------- Options.Skip = func(src string) (bool, error) { switch { case strings.HasSuffix(src, "main.go"): @@ -88,6 +105,10 @@ func GenerateNewProject(name string, module string) error { } // Creating a new go.mod file in the appropriate directory + err = newProject.CreateGoMod() + if err != nil { + return err + } return nil } @@ -112,3 +133,39 @@ func (a *NewProject)CreateGoMod() error { } return nil } + +// ChangeImportFiles Changes Appropriate imports in the appropriate file +func (a *NewProject)ChangeImportFiles() error { + // IF YOU ARE RELEASING AN EXTENSION OF P2PRC + // MODIFY HERE (AN EXTENSION FROM YOUR EXTENSION :P) + //---------------------------------------------------------------- + // Action performed: + // Files we would need to modify the imports in + // generate/generate.go -> config module + // cmd/action.go -> config module, server module, generate module + // cmd/flags.go -> config module, server module, generate module + // server/server.go -> config module + // main.go -> cmd module + //----------------------------------------------------------------- + a.FileNameAST = a.NewDir + "generate/generator.go" + err := a.ChangeImports("", a.Module+"config") + if err != nil { + return err + } + return nil +} + +// GetCurrentGoModule Gets the current go module name +func (a *NewProject)GetCurrentGoModule() (error) { + goModBytes, err := ioutil.ReadFile(a.P2PRCPath + "go.mod") + if err != nil { + return err + } + modName := modfile.ModulePath(goModBytes) + fmt.Fprintf(os.Stdout, "modName=%+v\n", modName) + + // Set current module to struct of file NewProject + a.CurrentModule = modName + + return nil +} diff --git a/generate/generate_test.go b/generate/generate_test.go index a6e0c5b..0074dcc 100644 --- a/generate/generate_test.go +++ b/generate/generate_test.go @@ -89,4 +89,24 @@ func TestNewProject_CreateGoMod(t *testing.T) { fmt.Println(err) t.Error(err) } +} + +// Testing if the current go module is returned +func TestNewProject_GetCurrentGoModule(t *testing.T) { + // Create a new variable of type NewProject + var np NewProject + path, err := config.GetPathP2PRC() + if err != nil { + fmt.Println(err) + t.Error(err) + } + // Set Current project path + np.P2PRCPath = path + // Get module name + err = np.GetCurrentGoModule() + if err != nil { + fmt.Println(err) + t.Error(err) + } + fmt.Println(np.CurrentModule) } \ No newline at end of file diff --git a/go.mod b/go.mod index d5d86b2..332b647 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/apenella/go-ansible v1.1.0 github.com/containerd/continuity v0.0.0-20210315143101-93e15499afd5 // indirect - github.com/dave/jennifer v1.4.1 github.com/docker/docker v20.10.0-beta1.0.20201113105859-b6bfff2a628f+incompatible github.com/docker/go-connections v0.4.0 github.com/gin-gonic/gin v1.6.3 @@ -26,6 +25,7 @@ require ( github.com/urfave/cli/v2 v2.3.0 gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 // indirect gitlab.com/NebulousLabs/go-upnp v0.0.0-20181011194642-3a71999ed0d3 + golang.org/x/mod v0.3.0 golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 // indirect golang.org/x/text v0.3.5 // indirect diff --git a/go.sum b/go.sum index db2fe1e..be5176c 100644 --- a/go.sum +++ b/go.sum @@ -177,8 +177,6 @@ github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1S github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= -github.com/dave/jennifer v1.4.1 h1:XyqG6cn5RQsTj3qlWQTKlRGAyrTcsk1kUmWdZBzRjDw= -github.com/dave/jennifer v1.4.1/go.mod h1:7jEdnm+qBcxl8PC0zyp7vxcpSRnzXSt9r39tpTVGlwA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -657,6 +655,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= From e4b1aaf208e113fa3400094a67cd1f76410a61ca Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Sun, 29 Aug 2021 00:13:18 +0400 Subject: [PATCH 06/11] generator import change complete --- generate/generate.go | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index c45f679..df6577e 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -15,6 +15,7 @@ import ( "strings" ) +// NewProject Struct information required when creating a new project type NewProject struct { Name string Module string @@ -78,6 +79,8 @@ func GenerateNewProject(name string, module string) error { switch { case strings.HasSuffix(src, "main.go"): return false, nil + case strings.HasSuffix(src, "generate.go"): + return false, nil case strings.HasSuffix(src, ".go"): return true, nil case strings.HasSuffix(src, "go.mod"): @@ -110,6 +113,18 @@ func GenerateNewProject(name string, module string) error { return err } + // Get current project mod name + err = newProject.GetCurrentGoModule() + if err != nil { + return err + } + + // Change the appropriate imports + err = newProject.ChangeImportFiles() + if err != nil { + return err + } + return nil } @@ -141,18 +156,33 @@ func (a *NewProject)ChangeImportFiles() error { //---------------------------------------------------------------- // Action performed: // Files we would need to modify the imports in - // generate/generate.go -> config module - // cmd/action.go -> config module, server module, generate module - // cmd/flags.go -> config module, server module, generate module - // server/server.go -> config module - // main.go -> cmd module + // - generate/generate.go -> config module + // - cmd/action.go -> config module, server module, generate module + // - cmd/flags.go -> config module, server module, generate module + // - server/server.go -> config module + // - main.go -> cmd module //----------------------------------------------------------------- - a.FileNameAST = a.NewDir + "generate/generator.go" - err := a.ChangeImports("", a.Module+"config") + + //----------------------------------------------------------------- + // 1.0 - generate/generate.go -> config module + a.FileNameAST = a.NewDir + "generate/generate.go" + // Get AST information of the file + err := a.GetASTGoFile() + if err != nil { + return err + } + // Change the appropriate Go file + err = a.ChangeImports(a.CurrentModule + "/config" , a.Module + "/config") + if err != nil { + return err + } + // Writes the change to the appropriate file + err = a.WriteGoAst() if err != nil { return err } return nil + //----------------------------------------------------------------- } // GetCurrentGoModule Gets the current go module name From ce5b7aa1af2aab7887bc2d2515b39f05fd7d60b4 Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Mon, 30 Aug 2021 14:38:27 +0400 Subject: [PATCH 07/11] saving current changes for project extension generation --- generate/generate.go | 190 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 170 insertions(+), 20 deletions(-) diff --git a/generate/generate.go b/generate/generate.go index df6577e..7a505d2 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -17,15 +17,15 @@ import ( // NewProject Struct information required when creating a new project type NewProject struct { - Name string - Module string - NewDir string - P2PRCPath string - CurrentModule string - Option *copy.Options - Token *token.FileSet - AST *ast.File - FileNameAST string + Name string + Module string + NewDir string + P2PRCPath string + CurrentModule string + Option *copy.Options + Token *token.FileSet + AST *ast.File + FileNameAST string } // GenerateNewProject creates a new copy of the P2PRC @@ -68,7 +68,11 @@ func GenerateNewProject(name string, module string) error { //---------------------------------------------------------------- // Action performed: // - Ensuring main.go file exists - // - Skipping all .go files + // - Ensuring generate.go file exists + // - Ensuring server/server.go file exists + // - Ensuring cmd/action.go file exists + // - Ensuring cmd/flags.go file exists + // - Skipping all .go files apart from the ones listed above // - Skipping go.mod file // - Skipping go.sum file // - Skipping .idea/ directory @@ -81,6 +85,12 @@ func GenerateNewProject(name string, module string) error { return false, nil case strings.HasSuffix(src, "generate.go"): return false, nil + case strings.HasSuffix(src, "server/server.go"): + return false, nil + case strings.HasSuffix(src, "cmd/action.go"): + return false, nil + case strings.HasSuffix(src, "cmd/flags.go"): + return false, nil case strings.HasSuffix(src, ".go"): return true, nil case strings.HasSuffix(src, "go.mod"): @@ -102,7 +112,7 @@ func GenerateNewProject(name string, module string) error { newProject.Option = &Options // Copies all files from P2PRC to the new project created - err = copy.Copy(newProject.P2PRCPath, newProject.NewDir,*newProject.Option) + err = copy.Copy(newProject.P2PRCPath, newProject.NewDir, *newProject.Option) if err != nil { return err } @@ -125,13 +135,30 @@ func GenerateNewProject(name string, module string) error { return err } + // Add changes inside the new project + err = newProject.GitAdd() + if err != nil { + return err + } + + // commit changes inside the new project + err = newProject.GitCommit() + if err != nil { + return err + } + // Creates go.sum file + //err = newProject.CreateGoModTidy() + //if err != nil { + // return err + //} + return nil } // CreateFolder Creates a new folder based on the name and path provided -func CreateFolder(name string,path string) error { +func CreateFolder(name string, path string) error { //Create a folder/directory at a full qualified path - err := os.Mkdir(path + name, 0755) + err := os.Mkdir(path+name, 0755) if err != nil { return err } @@ -139,9 +166,19 @@ func CreateFolder(name string,path string) error { } // CreateGoMod Creates a new go module for the new project created -func (a *NewProject)CreateGoMod() error { +func (a *NewProject) CreateGoMod() error { // Create new go.mod in the appropriate directory - cmd := exec.Command("go","mod","init",a.Module) + cmd := exec.Command("go", "mod", "init", a.Module) + cmd.Dir = a.NewDir + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (a *NewProject) CreateGoModTidy() error { + // Installs all the appropriate dependencies and creates go.sum + cmd := exec.Command("go", "mod", "tidy") cmd.Dir = a.NewDir if err := cmd.Run(); err != nil { return err @@ -150,7 +187,7 @@ func (a *NewProject)CreateGoMod() error { } // ChangeImportFiles Changes Appropriate imports in the appropriate file -func (a *NewProject)ChangeImportFiles() error { +func (a *NewProject) ChangeImportFiles() error { // IF YOU ARE RELEASING AN EXTENSION OF P2PRC // MODIFY HERE (AN EXTENSION FROM YOUR EXTENSION :P) //---------------------------------------------------------------- @@ -172,7 +209,7 @@ func (a *NewProject)ChangeImportFiles() error { return err } // Change the appropriate Go file - err = a.ChangeImports(a.CurrentModule + "/config" , a.Module + "/config") + err = a.ChangeImports(a.CurrentModule+"/config", a.Module+"/config") if err != nil { return err } @@ -181,12 +218,105 @@ func (a *NewProject)ChangeImportFiles() error { if err != nil { return err } - return nil + //----------------------------------------------------------------- + //----------------------------------------------------------------- + // 2.0 - cmd/action.go -> config module, server module, generate module + a.FileNameAST = a.NewDir + "cmd/action.go" + // Get AST information of the file + err = a.GetASTGoFile() + if err != nil { + return err + } + // Change the appropriate Go file + err = a.ChangeImports(a.CurrentModule+"/config", a.Module+"/config") + if err != nil { + return err + } + err = a.ChangeImports(a.CurrentModule+"/server", a.Module+"/server") + if err != nil { + return err + } + err = a.ChangeImports(a.CurrentModule+"/generate", a.Module+"/generate") + if err != nil { + return err + } + // Writes the change to the appropriate file + err = a.WriteGoAst() + if err != nil { + return err + } + //----------------------------------------------------------------- + // 2.1 - cmd/flags.go -> config module, server module, generate module + a.FileNameAST = a.NewDir + "cmd/flags.go" + // Get AST information of the file + err = a.GetASTGoFile() + if err != nil { + return err + } + // Change the appropriate Go file + err = a.ChangeImports(a.CurrentModule+"/config", a.Module+"/config") + if err != nil { + return err + } + err = a.ChangeImports(a.CurrentModule+"/server", a.Module+"/server") + if err != nil { + return err + } + err = a.ChangeImports(a.CurrentModule+"/generate", a.Module+"/generate") + if err != nil { + return err + } + // Writes the change to the appropriate file + err = a.WriteGoAst() + if err != nil { + return err + } + //----------------------------------------------------------------- + //----------------------------------------------------------------- + // 3.0 - server/server.go -> config module + a.FileNameAST = a.NewDir + "server/server.go" + // Get AST information of the file + err = a.GetASTGoFile() + if err != nil { + return err + } + // Change the appropriate Go file + err = a.ChangeImports(a.CurrentModule+"/config", a.Module+"/config") + if err != nil { + return err + } + // Writes the change to the appropriate file + err = a.WriteGoAst() + if err != nil { + return err + } + //----------------------------------------------------------------- + //----------------------------------------------------------------- + // 4.0 - server/server.go -> config module + a.FileNameAST = a.NewDir + "main.go" + // Get AST information of the file + err = a.GetASTGoFile() + if err != nil { + return err + } + // Change the appropriate Go file + err = a.ChangeImports(a.CurrentModule+"/cmd", a.Module+"/cmd") + if err != nil { + return err + } + // Writes the change to the appropriate file + err = a.WriteGoAst() + if err != nil { + return err + } + //----------------------------------------------------------------- + + return nil } // GetCurrentGoModule Gets the current go module name -func (a *NewProject)GetCurrentGoModule() (error) { +func (a *NewProject) GetCurrentGoModule() error { goModBytes, err := ioutil.ReadFile(a.P2PRCPath + "go.mod") if err != nil { return err @@ -195,7 +325,27 @@ func (a *NewProject)GetCurrentGoModule() (error) { fmt.Fprintf(os.Stdout, "modName=%+v\n", modName) // Set current module to struct of file NewProject - a.CurrentModule = modName + a.CurrentModule = modName return nil } + +func (a *NewProject) GitAdd() error { + // Installs all the appropriate dependencies and creates go.sum + cmd := exec.Command("git", "add", ".") + cmd.Dir = a.NewDir + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (a *NewProject) GitCommit() error { + // Installs all the appropriate dependencies and creates go.sum + cmd := exec.Command("git", "commit", "-m=removed appropriate go files") + cmd.Dir = a.NewDir + if err := cmd.Run(); err != nil { + return err + } + return nil +} From c3a3a21132b16d8eafccf9f467ccd69ff0bf2b59 Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Mon, 30 Aug 2021 16:59:33 +0400 Subject: [PATCH 08/11] fixing errors to ensure generate works --- cmd/action.go | 18 +++++++++++++----- cmd/flags.go | 8 ++++++++ generate/generate.go | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 5 deletions(-) diff --git a/cmd/action.go b/cmd/action.go index 4d17a82..cd7b12d 100644 --- a/cmd/action.go +++ b/cmd/action.go @@ -249,15 +249,23 @@ var CliAction = func(ctx *cli.Context) error { // of the project is created to repurpose // the project for custom purpose if Generate != "" { - err := generate.GenerateNewProject(Generate,"test") + var err error + // If the module name is provided + if Modulename != "" { + err = generate.GenerateNewProject(Generate,Modulename) + } else { + err = generate.GenerateNewProject(Generate,Generate) + } if err != nil { fmt.Println(err) } else { fmt.Println("Created new folder: " + Generate) - fmt.Println("1. Enter inside " + Generate + "directory") - fmt.Println("2. go mod init ") - fmt.Println("3. sh install.sh " + Generate) - fmt.Println("4. ./" + Generate + " -h (This is to test if the binary is working") + fmt.Println("1. Enter inside " + Generate + " directory") + fmt.Println("2. git remote add " + Generate + " ") + fmt.Println("3. git push " + Generate + " ") + fmt.Println("4. go mod tidy") + fmt.Println("5. sh install.sh " + Generate) + fmt.Println("6. ./" + Generate + " -h (This is to test if the binary is working)") } } //-------------------------------- diff --git a/cmd/flags.go b/cmd/flags.go index 07ec2a5..f3e724a 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -31,6 +31,7 @@ var ( // Generate only allowed in dev release // -- REMOVE ON REGULAR RELEASE -- Generate string + Modulename string //-------------------------------- ) @@ -200,5 +201,12 @@ var AppConfigFlags = []cli.Flag{ EnvVars: []string{"GENERATE"}, Destination: &Generate, }, + &cli.StringFlag{ + Name: "ModuleName", + Aliases: []string{"mod"}, + Usage: "New go project module name", + EnvVars: []string{"MODULENAME"}, + Destination: &Modulename, + }, //-------------------------------- } \ No newline at end of file diff --git a/generate/generate.go b/generate/generate.go index 7a505d2..e4fb8e4 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -69,7 +69,11 @@ func GenerateNewProject(name string, module string) error { // Action performed: // - Ensuring main.go file exists // - Ensuring generate.go file exists + // - Ensuring modifyGenerate.go file exists + // - Ensuring generate_test.go file exists // - Ensuring server/server.go file exists + // - Ensuring server/gopsutil.go file exists + // - Ensuring server/gpu.go file exists // - Ensuring cmd/action.go file exists // - Ensuring cmd/flags.go file exists // - Skipping all .go files apart from the ones listed above @@ -85,12 +89,24 @@ func GenerateNewProject(name string, module string) error { return false, nil case strings.HasSuffix(src, "generate.go"): return false, nil + case strings.HasSuffix(src, "modifyGenerate.go"): + return false, nil + case strings.HasSuffix(src, "generate_test.go"): + return false, nil case strings.HasSuffix(src, "server/server.go"): return false, nil + case strings.HasSuffix(src, "server/gopsutil.go"): + return false, nil + case strings.HasSuffix(src, "server/gpu.go"): + return false, nil case strings.HasSuffix(src, "cmd/action.go"): return false, nil case strings.HasSuffix(src, "cmd/flags.go"): return false, nil + case strings.HasSuffix(src, "config/config.go"): + return false, nil + case strings.HasSuffix(src, "config/config_test.go"): + return false, nil case strings.HasSuffix(src, ".go"): return true, nil case strings.HasSuffix(src, "go.mod"): @@ -194,6 +210,7 @@ func (a *NewProject) ChangeImportFiles() error { // Action performed: // Files we would need to modify the imports in // - generate/generate.go -> config module + // - generate/generate_test.go -> config module // - cmd/action.go -> config module, server module, generate module // - cmd/flags.go -> config module, server module, generate module // - server/server.go -> config module @@ -218,6 +235,24 @@ func (a *NewProject) ChangeImportFiles() error { if err != nil { return err } + //----------------------------------------------------------------- + // 1.1 - generate/generate_test.go -> config module + a.FileNameAST = a.NewDir + "generate/generate_test.go" + // Get AST information of the file + err = a.GetASTGoFile() + if err != nil { + return err + } + // Change the appropriate Go file + err = a.ChangeImports(a.CurrentModule+"/config", a.Module+"/config") + if err != nil { + return err + } + // Writes the change to the appropriate file + err = a.WriteGoAst() + if err != nil { + return err + } //----------------------------------------------------------------- //----------------------------------------------------------------- From 65de9dda1768bbb276d54907efa60a99accb73d4 Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Mon, 30 Aug 2021 17:57:54 +0400 Subject: [PATCH 09/11] fixed errors with generate --- deploy/LICENSE | 201 ------------------ deploy/README.md | 148 ------------- deploy/TestAnsible/description.txt | 1 - deploy/TestAnsible/hosts | 12 -- deploy/TestAnsible/site.yml | 8 - deploy/ToolSelection.md | 93 -------- deploy/ansible.cfg | 5 - deploy/hosts | 0 deploy/hosts.example | 25 --- deploy/packages.yml | 38 ---- deploy/roles/cuda/README.md | 23 -- deploy/roles/cuda/defaults/.gitkeep | 0 .../cuda-repo-ubuntu1804_10.0.130-1_amd64.deb | Bin 2940 -> 0 bytes deploy/roles/cuda/handlers/main.yml | 5 - deploy/roles/cuda/meta/.gitkeep | 0 deploy/roles/cuda/tasks/debian.yml | 22 -- deploy/roles/cuda/tasks/main.yml | 3 - deploy/roles/cuda/templates/.gitkeep | 0 deploy/roles/cuda/vars/main.yml | 9 - deploy/roles/docker/README.md | 33 --- deploy/roles/docker/defaults/.gitkeep | 0 deploy/roles/docker/files/.gitkeep | 0 deploy/roles/docker/handlers/.gitkeep | 0 deploy/roles/docker/meta/.gitkeep | 0 deploy/roles/docker/tasks/debian.yml | 17 -- deploy/roles/docker/tasks/main.yml | 23 -- deploy/roles/docker/templates/.gitkeep | 0 deploy/roles/docker/vars/main.yml | 8 - deploy/roles/nvidia/README.md | 31 --- deploy/roles/nvidia/defaults/.gitkeep | 0 deploy/roles/nvidia/files/.gitkeep | 0 deploy/roles/nvidia/handlers/main.yml | 6 - deploy/roles/nvidia/meta/.gitkeep | 0 deploy/roles/nvidia/tasks/debian.yml | 31 --- deploy/roles/nvidia/tasks/main.yml | 4 - deploy/roles/nvidia/templates/.gitkeep | 0 deploy/roles/nvidia/vars/main.yml | 10 - deploy/roles/packages/README.md | 26 --- deploy/roles/packages/defaults/.gitkeep | 0 deploy/roles/packages/files/.gitkeep | 0 deploy/roles/packages/handlers/.gitkeep | 0 deploy/roles/packages/meta/.gitkeep | 0 deploy/roles/packages/tasks/debian.yml | 12 -- deploy/roles/packages/tasks/main.yml | 3 - deploy/roles/packages/templates/.gitkeep | 0 deploy/roles/packages/vars/main.yml | 17 -- deploy/roles/rsync/defaults/.gitkeep | 0 deploy/roles/rsync/files/rsyncd.example.conf | 16 -- deploy/roles/rsync/handlers/main.yml | 5 - deploy/roles/rsync/meta/.gitkeep | 0 deploy/roles/rsync/tasks/main.yml | 26 --- deploy/roles/rsync/templates/.gitkeep | 0 deploy/roles/rsync/vars/main.yml | 10 - deploy/secrets.example.yml | 15 -- .../{modfyGenerate.go => modifyGenerate.go} | 0 55 files changed, 886 deletions(-) delete mode 100644 deploy/LICENSE delete mode 100644 deploy/README.md delete mode 100644 deploy/TestAnsible/description.txt delete mode 100644 deploy/TestAnsible/hosts delete mode 100644 deploy/TestAnsible/site.yml delete mode 100644 deploy/ToolSelection.md delete mode 100644 deploy/ansible.cfg delete mode 100644 deploy/hosts delete mode 100644 deploy/hosts.example delete mode 100644 deploy/packages.yml delete mode 100644 deploy/roles/cuda/README.md delete mode 100644 deploy/roles/cuda/defaults/.gitkeep delete mode 100644 deploy/roles/cuda/files/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb delete mode 100644 deploy/roles/cuda/handlers/main.yml delete mode 100644 deploy/roles/cuda/meta/.gitkeep delete mode 100644 deploy/roles/cuda/tasks/debian.yml delete mode 100644 deploy/roles/cuda/tasks/main.yml delete mode 100644 deploy/roles/cuda/templates/.gitkeep delete mode 100644 deploy/roles/cuda/vars/main.yml delete mode 100644 deploy/roles/docker/README.md delete mode 100644 deploy/roles/docker/defaults/.gitkeep delete mode 100644 deploy/roles/docker/files/.gitkeep delete mode 100644 deploy/roles/docker/handlers/.gitkeep delete mode 100644 deploy/roles/docker/meta/.gitkeep delete mode 100644 deploy/roles/docker/tasks/debian.yml delete mode 100644 deploy/roles/docker/tasks/main.yml delete mode 100644 deploy/roles/docker/templates/.gitkeep delete mode 100644 deploy/roles/docker/vars/main.yml delete mode 100644 deploy/roles/nvidia/README.md delete mode 100644 deploy/roles/nvidia/defaults/.gitkeep delete mode 100644 deploy/roles/nvidia/files/.gitkeep delete mode 100644 deploy/roles/nvidia/handlers/main.yml delete mode 100644 deploy/roles/nvidia/meta/.gitkeep delete mode 100644 deploy/roles/nvidia/tasks/debian.yml delete mode 100644 deploy/roles/nvidia/tasks/main.yml delete mode 100644 deploy/roles/nvidia/templates/.gitkeep delete mode 100644 deploy/roles/nvidia/vars/main.yml delete mode 100644 deploy/roles/packages/README.md delete mode 100644 deploy/roles/packages/defaults/.gitkeep delete mode 100644 deploy/roles/packages/files/.gitkeep delete mode 100644 deploy/roles/packages/handlers/.gitkeep delete mode 100644 deploy/roles/packages/meta/.gitkeep delete mode 100644 deploy/roles/packages/tasks/debian.yml delete mode 100644 deploy/roles/packages/tasks/main.yml delete mode 100644 deploy/roles/packages/templates/.gitkeep delete mode 100644 deploy/roles/packages/vars/main.yml delete mode 100644 deploy/roles/rsync/defaults/.gitkeep delete mode 100644 deploy/roles/rsync/files/rsyncd.example.conf delete mode 100644 deploy/roles/rsync/handlers/main.yml delete mode 100644 deploy/roles/rsync/meta/.gitkeep delete mode 100644 deploy/roles/rsync/tasks/main.yml delete mode 100644 deploy/roles/rsync/templates/.gitkeep delete mode 100644 deploy/roles/rsync/vars/main.yml delete mode 100644 deploy/secrets.example.yml rename generate/{modfyGenerate.go => modifyGenerate.go} (100%) diff --git a/deploy/LICENSE b/deploy/LICENSE deleted file mode 100644 index 261eeb9..0000000 --- a/deploy/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/deploy/README.md b/deploy/README.md deleted file mode 100644 index 7b4b26a..0000000 --- a/deploy/README.md +++ /dev/null @@ -1,148 +0,0 @@ -# Ansible p2p-rendering-computation -### Modified from: - -This repository contains an Ansible playbook and instructions to create and manage a single (or many) bare metal deep learning machines. For a description of why Ansible was chosen and what other alternatives were considered, please see [ToolSelection.md](ToolSelection.md) - -## Quick Reference - -If you've already [installed Ansible](#Installation), you can execute the entire playbook by running: - -```bash -$ ansible-playbook packages.yml -``` - -You can also execute only the pieces you need by passing tags on the command line: - -- Install only apt/pip [pre-requisites](roles/packages) to execute the other roles: - ```bash - $ ansible-playbook packages.yml --tags "packages" - ``` -- Install [Docker CE](roles/docker): - ```bash - $ ansible-playbook packages.yml --tags "docker" - ``` -- Install the [Nvidia CUDA GPU drivers](roles/cuda): - ```bash - $ ansible-playbook packages.yml --tags "cuda" - ``` -- Install the [Nvidia Docker Runtime](roles/nvidia): - ```bash - $ ansible-playbook packages.yml --tags "nvidia" - ``` - -## What's Included - -After running the ansible script your machines will be loaded with the following: - -1. Docker -2. Nvidia CUDA GPU Drivers -3. Nvidia Docker Runtime -4. TensorFlow GPU Python3 Docker Container -5. JupyterLab - -## Using This Repository to Configure Your Environment - -1. [Installation](#Installation) -2. [Configuration](#Configuration) -3. [Running](#Running) - ---- - -### Installation - -Ansible runs on your local machine and sends commands to the remote (machine learning) machines. You'll need ansible installed locally (not on the machine learning boxes). -For macOS users, the easiest way to install Ansible is via [Homebrew](https://brew.sh/): - -```bash -$ brew install ansible -``` - -If that's not your cup of tea, install Ansible by following the directions for your machine [here](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#installing-the-control-machine). - ---- - -### Configuration - -Gather the following: - -- SSH key or user credentials for the remote account - - **Note:** Ansible does not expose a channel to allow communication between the user and the ssh process to accept a password manually to decrypt an ssh key when using the ssh connection plugin (which is the default). The use of `ssh-agent` is highly recommended. - -- List of servers you wish to manage: - - hostnames/IP addresses - - SSH port - - usernames - -Copy [hosts.example] to `/etc/ansible/hosts` (if it does not already exist). Populate the `hosts` file (no extension) with the information about the servers you gathered above. - -Confirm that you have populated your Ansible hosts file correctly: - -```bash -$ ansible-inventory --list -``` - ---- - -### Running - -Once you're satisfied that you correctly populated your `hosts` file, update the `- hosts:` line of [tensorflow.yml] to reflect the hosts or groups you want to configure. - -Examples: - -- Apply against a single host defined as `ml2` in `/etc/ansible/hosts`: - ```yaml - - hosts: ml2 - ``` -- Apply against a group of hosts defined as `production` in `/etc/ansible/hosts`: - ```yaml - - hosts: production - ``` -- Apply against all hosts defined in `/etc/ansible/hosts`: - ```yaml - - hosts: all - ``` - -Then, when you're ready, run the playbook: - -```bash -$ ansible-playbook packages.yml --ask-become-pass -``` - -**Note:** You must have `sudo` access to run the playbook! - -Review the output: - -- `[ok]` means no change (this task was already completed) -- `[changed]` means the task successfully ran and the change was applied -- `[unreachable]` means the host could not be reached -- `[failed]` means the task ran but failed to complete - -`[ok]` and `[changed]` are successful outcomes. Any `[unreachable]` and `[failed]` outputs should be investigated and resolved. - -**Note:** This Ansible playbook is idempotent; once a configuration has been successfully applied, if you apply it again, all actions will report `[ok]`. - -## Executing Tensorflow Jobs in Your New Environment - -1. Point your browser to http://<hostname>:8888 and login with the password you provided. -2. The `jupyter.volumes.source` folder will be mounted as the `notebooks` folder. -3. Edit and execute your Jupyter notebooks as normal! - -### Command Line Access - -If you need to drop into a GPU-powered TensorFlow environment, SSH into the remote machine and execute the following: - -```bash -$ docker run --runtime=nvidia -it --rm tensorflow/tensorflow:latest-gpu-py3 bash -``` - -**Note:** You must be a member of the `docker` group or have `sudo` access on the _remote machine_ to execute docker commands. - ---- - -## Additional Files - -- [ansible.cfg](ansible.cfg) enables SSH credential forwarding. This is a necessary step during data synchronization, as Ansible delegates those credentials to the master/writer host to push the data folder out to each of the mirrors. -- [Dockerfile](Dockerfile) is used to build the Arricor TensorFlow image. See [Docker.md](Docker.md) for additional details. -- [hosts.example](hosts.example) is an example of the Ansible inventory hosts file saved in `/etc/ansible/hosts` -- [secrets.example.yml](secrets.example.yml) is an example of the expected structure of the `secrets.yml` file diff --git a/deploy/TestAnsible/description.txt b/deploy/TestAnsible/description.txt deleted file mode 100644 index b0a0a8f..0000000 --- a/deploy/TestAnsible/description.txt +++ /dev/null @@ -1 +0,0 @@ -Test if it's detected \ No newline at end of file diff --git a/deploy/TestAnsible/hosts b/deploy/TestAnsible/hosts deleted file mode 100644 index 07e4869..0000000 --- a/deploy/TestAnsible/hosts +++ /dev/null @@ -1,12 +0,0 @@ ---- -all: - vars: - ansible_python_interpreter: /usr/bin/python3 -main: - hosts: - host1: - ansible_host: 0.0.0.0 # Replace with your remote IP - ansible_port: 44003 # Replace with your remote SSH port - ansible_user: master # Replace wtih your username - ansible_ssh_pass: password - ansible_sudo_pass: password \ No newline at end of file diff --git a/deploy/TestAnsible/site.yml b/deploy/TestAnsible/site.yml deleted file mode 100644 index 3dc78ac..0000000 --- a/deploy/TestAnsible/site.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- - -- hosts: all - - tasks: - - name: simple-ansibleplaybook - debug: - msg: Your are running 'simple-ansibleplaybook' example \ No newline at end of file diff --git a/deploy/ToolSelection.md b/deploy/ToolSelection.md deleted file mode 100644 index 117fa83..0000000 --- a/deploy/ToolSelection.md +++ /dev/null @@ -1,93 +0,0 @@ -# Configuration Management Tool Selection - -## Summary - -Ansible is proposed as the configuration management tool for this project due to the following characteristics: - -- no agent installation on the managed nodes -- push-based configuration changes do not require a separate machine to be provisioned -- commands can be pushed from any developer/operator workstation -- all updates are performed over SSH - -## Tools Considered - -Where a tool offers an open source and an enterprise version (e.g., Puppet, Terraform), only the open source version was considered. Web or other graphical user interfaces (GUIs) were not considered, as the goal of this phase is to produce infrastructure as code which can be run (and re-run) from the command line interface (CLI). - -- [Ansible](#Ansible) -- [Chef](#Chef) -- [Puppet](#Puppet) -- [Salt](#Salt) -- [Terraform](#Terraform) - -### Ansible - -[Ansible's simple interface and usability fit right into the sys admin mindset, and in a shop with lots of Linux and Unix systems, Ansible is quick and easy to run right out of the gate](https://www.infoworld.com/article/2609482/data-center/data-center-review-puppet-vs-chef-vs-ansible-vs-Salt.html?page=4). - -#### Ansible Pros - -- Requires no agent installation on the managed nodes. -- Push-based. -- Does not require a separate machine to be provisioned. Commands can be pushed from any developer/operator workstation. -- All updates are performed over SSH. - -#### Ansible Cons - -- Push-based synchronization means that the configuration is only refreshed on demand. - -### Chef - -[Chef has a stable and well-designed layout, and while it's not quite up to the level of Puppet in terms of raw features, it's a very capable solution. Chef may pose the most difficult learning curve to administrators who lack significant programming experience, but it could be the most natural fit for development-minded admins and development shops](https://www.infoworld.com/article/2609482/data-center/data-center-review-puppet-vs-chef-vs-ansible-vs-Salt.html?page=4). - -#### Chef Pros - -- AWS OpsWorks is based on Chef; any future migration to AWS could benefit from the existing configuration files. - -#### Chef Cons - -- **(Fatal)** Requires a master server and a workstation that will need to be maintained separately from the already provisioned deep learning machines. -- **(Fatal)** Requires an agent (client) to be installed on each machine. -- Requires an on-premises Chef server and [Push Jobs](https://docs.chef.io/push_jobs.html) plugin to push configurations manually. Otherwise, managed machines must check in with the master on a schedule. - -### Puppet - -[Puppet is the most mature and probably the most approachable of the four from a usability standpoint, though a solid knowledge of Ruby is highly recommended. Puppet is not as streamlined as Ansible or Salt, and its configuration can get Byzantine at times. Puppet is the safest bet for heterogeneous environments, but you may find Ansible or Salt to be a better fit in a larger or more homogenous infrastructure](https://www.infoworld.com/article/2609482/data-center/data-center-review-puppet-vs-chef-vs-ansible-vs-salt.html?page=4). - -#### Puppet Pros - -- Configuration can be kept up to date via a schedule or manual configuration pushes. - -#### Puppet Cons - -- **(Fatal)** Requires a "puppet master" (server) that will need to be maintained separately from the already provisioned machines. -- **(Fatal)** Requires an agent (client) to be installed on each machine. - -### Salt - -[Salt is the sleekest and most robust of the four, and like Ansible it will resonate with sys admins. Highly scalable and quite capable, Salt is hamstrung only by the Web UI](https://www.infoworld.com/article/2609482/data-center/data-center-review-puppet-vs-chef-vs-ansible-vs-salt.html?page=4). - -#### Salt Pros - -- Push-based. -- Can function via regular SSH or via client agents called "minions". - -#### Salt Cons - -- When running agentless, push-based synchronization means that the configuration is only refreshed on demand. - -### Terraform - -The initial hope was that [Terraform](https://terraform.io) could be used for configuration management for this project. However, after reviewing the [available Terraform provider plugins](https://terraform.io/docs/providers) and other available material, it was agreed on 23 January 2019 that Terraform was not suitable for this stage of the project. - -In the future, if Arricor migrates its instances to a cloud service provider (CSP) such as AWS, Azure, or Google Cloud Platform, this configuration may be reusable as a first step via the [terraform-provisioner-ansible](https://github.com/jonmorehouse/terraform-provisioner-ansible) plugin. This plugin allows the user to create CSP instances, then execute the given Ansible playbook against those instances. - -#### Terraform Pros - -- Cloud native with support for multiple service providers. - -#### Terraform Cons - -- **(Fatal)** No support for provisioning bare metal instances. - -## Conclusion - -Ansible allows for management of a server from scratch, with only the requirement to have a valid username and password. Additionally, its agentless approach avoids having to configure and maintain an admin server, further simplifying operations at the scale Arricor requires. diff --git a/deploy/ansible.cfg b/deploy/ansible.cfg deleted file mode 100644 index 04c0df4..0000000 --- a/deploy/ansible.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[defaults] -host_key_checking = False - -[ssh_connection] -ssh_args = -o ForwardAgent=yes -o ControlMaster=auto -o ControlPersist=60s diff --git a/deploy/hosts b/deploy/hosts deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/hosts.example b/deploy/hosts.example deleted file mode 100644 index bf4edf8..0000000 --- a/deploy/hosts.example +++ /dev/null @@ -1,25 +0,0 @@ ---- -all: - vars: - ansible_python_interpreter: /usr/bin/python3 -test: - hosts: - test1: - ansible_host: 127.0.0.1 # Replace with your remote IP - ansible_port: 2222 # Replace with your remote SSH port - ansible_user: vagrant # Replace wtih your username - -production: - hosts: - writer: - ansible_host: 192.168.1.100 # Replace with your remote IP - ansible_port: 2202 # Replace with your remote SSH port - ansible_user: ansible # Replace wtih your username - mirror1: - ansible_host: 192.168.1.101 # Replace with your remote IP - ansible_port: 2202 # Replace with your remote SSH port - ansible_user: ansible # Replace wtih your username - mirror2: - ansible_host: 192.168.1.102 # Replace with your remote IP - ansible_port: 2202 # Replace with your remote SSH port - ansible_user: ansible # Replace wtih your username diff --git a/deploy/packages.yml b/deploy/packages.yml deleted file mode 100644 index f5b35b7..0000000 --- a/deploy/packages.yml +++ /dev/null @@ -1,38 +0,0 @@ -################################################################################ -# This file consists of three main sections: -# 1. vars - variables used throughout the playbook are defined here. If you want to change packages or version, this is the place to do it. -# 2. tasks - these are the steps that will be executed by the playbook. This is a straightforward, sequential playbook. -# 3. handlers - these steps run only in response to specific events. If the event does not occur, the handler does not run. -################################################################################ ---- -- hosts: test # Alternatively, 'canary', 'blue'|'green', 'all' according to your deployment strategy and your /etc/ansible/hosts file - become: yes # All commands must run as root - strategy: free # Allows all hosts to run to the end of the playbook as fast as they can - - vars_files: - - secrets.yml - - roles: - - role: packages # General Dependencies - tags: [packages] - - # - role: rsync # Rsync Setup - # tags: [rsync] - # vars: - # sync: "{{ secrets.sync }}" - - - role: docker # General Docker setup - tags: [docker] - vars: - users: "{{ secrets.users }}" - - - role: cuda # CUDA Driver Setup - tags: [cuda] - - - role: nvidia # NVIDIA-Docker Runtime Setup - tags: [nvidia] - - - role: jupyter # Launch Jupyter Container in Docker - tags: [jupyter] - vars: - volumes: "{{ secrets.volumes }}" diff --git a/deploy/roles/cuda/README.md b/deploy/roles/cuda/README.md deleted file mode 100644 index 752ecda..0000000 --- a/deploy/roles/cuda/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# The `cuda` Role - -The `cuda` role installs the Nvidia CUDA GPU repository and drivers. - -This role may be executed independently by running: - -```bash -$ ansible-playbook packages.yml --tags "cuda" -``` - -On execution, this role installs the following directly to the remote machine's operating system: - -### Apt Repositories (Debian/Ubuntu) - -- Nvidia CUDA - -### Apt GPG Signing Keys (Debian/Ubuntu) - -- cudatools - -### Operating System Packages (Debian/Ubuntu) - -- cuda v10.x diff --git a/deploy/roles/cuda/defaults/.gitkeep b/deploy/roles/cuda/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/cuda/files/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb b/deploy/roles/cuda/files/cuda-repo-ubuntu1804_10.0.130-1_amd64.deb deleted file mode 100644 index 85eaa00362ab82ac71e498123f58f1b02ecd1dfb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2940 zcmbuAc~BHr9>>8mwkEEkl2gaqtx@Rd>6xBJJm$Wbn_-SbW1lm_%=8R%&)|`ms4T9l zM$rutl@N_a3}HjWtHi)+3>w8FVieJG70)1uM${;a8ZDFBs{FCPs_yFkz1Qzm*ZY3H z-;tSEfN)L)ND6mh%p}~!V1YP`#p3X|d<5n2VVK1NyWgMPIUx9?*WLcDW&uzd2oOFd z6!3Y{LRcUzmiY7U1^lP0vRJc#he*O|PfF_5agrnht?h%S7t*}^vXb~sjztN!qE0dWdl2ZvCgaI465l`w@w>L z>`u!6cGMfTiK7xHM(tp?ufr-+Z#8NYiXFqRWs{{RT&w;WT@1qj;uwCrOQ8TxV@|9r~IA6Bf%jx zt#9|p?<2lpf+gCG%%**!r(-y^XY)6j2I=4I4TMH~{^gxzpIu!OEx9KxD#{!%XVFaAe+rQJ zfXUOkn5BIiGwCT%a@&$9)3UC9?60@Futkkow`D_*O}IY9gYKV@bH4nPWMRrG%YvT= z^D2(^R&Lv+DXz2FH??`rT_6^0U2_mlNm{+J!cbk^D4m|0d}_O*i!Kg&zjzHXr(7bl2^>_LGaNg-d3fSozaOQ}gTAKoBF;2;x`httxdA_ zaUHEMoj#53tbD0&NyUXRzmBrECz;&`SGY%^%fqwnbKmNbD%d%2`q!g3*RQRf@P0UB zTu;?nCv&{*`%YUMd_sKs?xQ-&edT5_&%C$va5FRVo4%*2PW*TueO)Shb>tRD{`Nkp z+~{(kuy)jqSE|@o#Q8T4-Mle8Qog|2RRaxnl?R4al+D+r4C|B9`|ix8*9JNU+e_Yo zn=9R`Wy8a>7q4#Vj0Q_Co!_iHdg)+Ag{*Bd_uz^`)xF}=7pL#-(fP@HrqSj00Y~ni z-SJD4LRa$FiASOjlJ^uyCZ+(I+VOc~52y9hpy8UDPgX3bUNs}@+V$$S9sS=ezWi3l zu9A*RI6I}rZ|jJCp1x~l+O@$=yA6>C9V3EkYUwejwV4mnuO=m(i$9!xap(S&q~z*u zZjzEF7f-9N{I}ylkH`4*ve`*2ggu9{o7g|&n#X@e_8+-y@A)E^hqut@ySYrh`u}iw z_hdu=!>O}^1>e=qDI6y{-TUR-iP+laVJ*=M&Ak<~8&a>%xjX+l_|>hu&0%KX(Sob( z>C^VQTL;XG{h*y+tUg?BoPS{%{DpDF7xJm@#^s7gmbOq`S5UX_ZszXfhO>18?9jgB z4Vu+3M^lNLAHfflQ#$rfKGZ6^zxkc2nk_9?z@kls2hI=s;7)1jh(cVYF2;Y1Wc1vg z{=>m_BQ|_idsU%3cjt!Y+edz*_pAMpzszbndE|}C*S5XiTDUk>cCcV21` z;jza@TY$vEQ)6ajR@65pdhLl89JpLUOdE3Y=GasI&$}v@w7-%eJ376vsK>Acwu0~L zhr4x5;PsU1z?x#m^@@#_;WpKfXv0jiqaE(*-@E+I>Xxl`ahqUL{M_4}7F}_)X49DY z!e!P&W?e(>>U(FVjn*G?_dB`lvkrdlqMb)S-t&5GzA>Dh+O{(E=CxIQdhe1i$SK}> z=A!sATf3rv!=fYWS%*JrJ`d(;x2!3@PV_Pl;Vj+!#E^IFuDB>jwbdsjFX{DIZ_9oj zKJ-Psz3cEh@UY86Qh_J=@LX@mvjlnS`_FQO1M#^}gJc#k>2Zmqa;27~m+M(3rCckO zuryMWNhp^-83Ytq(3!z9DTF)@1OPcyAn5Wj87vquxf~1?$Y2rSZpHh2o**mp`QvmZ z;v!vG8sYO!1teIA%3xV2l4YX&EDqF7Lb-SU^86 zL8BZ2gg`Wn@kok;@(>o!GVFX2Jgkn9;)ZgbY(2s{>7>R)06A3Ox@Ck%~a1O-5_*@KwNr=lr z1sFweNQ?qOiXeGB8YFOnrn)Z(f+2z;AOX!oP%i7SGrZs?|Le4b*=FN!2L-qc%7jQL z6^R`}Qs$CM0}h65LO_?!XNo8EiColyX6dbLR{$2ugr2b7BNGV44!7Z19|*uAu05L0 zRvER<0BrGcIDRV$Q+9hmVWs&x3}-uiUQ$mf8A*^YkE05kOQdH=!j2$e+(WYgv(*;N z6{&Tg1{NV`JcL<6NDPy1hstHl1|v{_r;&+bY%U#fMFI(#$)1N~Nu?;1m9BEi9dxHl;@Yaal#wNjdr`w0EWG2OhPC%@@!w+ zot+zDJSbq+!7dV}4Sb10%D4=K8S(jze32kbn)o&;!=9*$rwr%8QneWg2=Hvb-mO-nI(arn z-K{eT&nZQC2Cpdx4+D0SlPz}S#qv}#OCrR}6%eX)zcw8S=U8JgCG1X^gfY+&5$XJ- ziyMgfq+*=S(SW&no8RM9%Hx1gC{F~{!E`I%BvlAQVoXKR2DaBj${2@8A@&)vj8-nj zh_~MXWPDmt#Edpr7q6iyd`=ownc05O|<3t&e-3RgzO!*Z$J r9QBw4#vmEWmBKQ_9>=tLyFsE)cT54Mkn%j!Q%Iy*$#X+{5t07_Ev$WZ diff --git a/deploy/roles/cuda/handlers/main.yml b/deploy/roles/cuda/handlers/main.yml deleted file mode 100644 index 6dfc62c..0000000 --- a/deploy/roles/cuda/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# CUDA Driver Setup -- name: reboot the machine - reboot: - listen: "reboot the machine" diff --git a/deploy/roles/cuda/meta/.gitkeep b/deploy/roles/cuda/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/cuda/tasks/debian.yml b/deploy/roles/cuda/tasks/debian.yml deleted file mode 100644 index 5290be8..0000000 --- a/deploy/roles/cuda/tasks/debian.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Cuda Driver Package Installation -- name: copy the CUDA repo onto the machine - copy: - src: "{{ cuda.apt.repo }}" - dest: "{{ general.working_dir }}/{{ cuda.apt.repo }}" - -- name: get the CUDA signing key - apt_key: - state: present - url: "{{ cuda.apt.signing_key_url }}" - -- name: add the CUDA repo - apt: - deb: "{{ general.working_dir }}/{{ cuda.apt.repo }}" - state: present - -- name: install the CUDA drivers - apt: - name: "{{ cuda.apt.package }}" - update_cache: yes - notify: "reboot the machine" diff --git a/deploy/roles/cuda/tasks/main.yml b/deploy/roles/cuda/tasks/main.yml deleted file mode 100644 index bd512db..0000000 --- a/deploy/roles/cuda/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- import_tasks: debian.yml - when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu diff --git a/deploy/roles/cuda/templates/.gitkeep b/deploy/roles/cuda/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/cuda/vars/main.yml b/deploy/roles/cuda/vars/main.yml deleted file mode 100644 index 054d5fa..0000000 --- a/deploy/roles/cuda/vars/main.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -cuda: - apt: - package: cuda=10.* # known good with 10.0.130-1 - repo: cuda-repo-ubuntu1804_10.0.130-1_amd64.deb # Network install - signing_key_url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub - -general: - working_dir: /tmp diff --git a/deploy/roles/docker/README.md b/deploy/roles/docker/README.md deleted file mode 100644 index e4c625c..0000000 --- a/deploy/roles/docker/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# The `docker` Role - -The `docker` role installs the Docker CE repository and runtime packages. - -This role may be executed independently by running: - -```bash -$ ansible-playbook packages.yml --tags "docker" -``` - -On execution, this role installs the following directly to the remote machine's operating system: - -### Apt Repositories (Debian/Ubuntu) - -- Docker CE - -### Apt GPG Signing Keys (Debian/Ubuntu) - -- Docker Release (CE deb) - -### Operating System Packages (Debian/Ubuntu) - -- docker-ce v5:18.x - -### System Services - -- `docker` (enabled and started) - -### User Groups - -- docker - -All users listed under `secrets.users` in `secrets.yml` will be added to the `docker` group. diff --git a/deploy/roles/docker/defaults/.gitkeep b/deploy/roles/docker/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/docker/files/.gitkeep b/deploy/roles/docker/files/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/docker/handlers/.gitkeep b/deploy/roles/docker/handlers/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/docker/meta/.gitkeep b/deploy/roles/docker/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/docker/tasks/debian.yml b/deploy/roles/docker/tasks/debian.yml deleted file mode 100644 index 4228c04..0000000 --- a/deploy/roles/docker/tasks/debian.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# Docker Package Installation -- name: add Docker's official GPG key - apt_key: - state: present - url: "{{ docker.apt.signing_key_url }}" - -- name: add Docker apt repository - apt_repository: - repo: "{{ docker.apt.repo }}" - state: present - -- name: install Docker CE - apt: - name: "{{ docker.apt.package }}" - state: present - update_cache: yes diff --git a/deploy/roles/docker/tasks/main.yml b/deploy/roles/docker/tasks/main.yml deleted file mode 100644 index 9cebc69..0000000 --- a/deploy/roles/docker/tasks/main.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# General Docker Setup -- import_tasks: debian.yml - when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu - -- name: ensure the docker service starts at boot - systemd: - name: "{{ docker.service_name }}" - enabled: yes - state: started - -- name: create Docker group to avoid running as root - group: - name: "{{ docker.group_name }}" - state: present - -- name: add users to Docker group - user: - append: yes - groups: - - "{{ docker.group_name }}" - name: "{{ item }}" - loop: "{{ users }}" diff --git a/deploy/roles/docker/templates/.gitkeep b/deploy/roles/docker/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/docker/vars/main.yml b/deploy/roles/docker/vars/main.yml deleted file mode 100644 index 6506998..0000000 --- a/deploy/roles/docker/vars/main.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -docker: - apt: - package: docker-ce=5:18.* # known good with 5:18.09.1~3-0~ubuntu-bionic - repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable - signing_key_url: https://download.docker.com/linux/ubuntu/gpg - group_name: docker - service_name: docker diff --git a/deploy/roles/nvidia/README.md b/deploy/roles/nvidia/README.md deleted file mode 100644 index b5c4446..0000000 --- a/deploy/roles/nvidia/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# The `nvidia` Role - -The `nvidia` role installs the Nvidia Docker runtime, enabling access to Nvidia CUDA GPU hardware from within Docker containers. - -This role may be executed independently by running: - -```bash -$ ansible-playbook packages.yml --tags "nvidia" -``` - -On execution, this role installs the following directly to the remote machine's operating system: - -### Apt Repositories (Debian/Ubuntu) - -- libnvidia_container -- nvidia_container_runtime -- nvidia_docker - -### Apt GPG Signing Keys (Debian/Ubuntu) - -- NVIDIA CORPORATION (Open Source Projects) - -### Operating System Packages (Debian/Ubuntu) - -- nvidia-docker2 v2.x - -Containers are launched with the Nvidia Docker runtime by passing the `--runtime=nvidia` flag, e.g., - -```bash -$ docker run --rm --runtime=nvidia nvidia/cuda nvidia-smi -``` diff --git a/deploy/roles/nvidia/defaults/.gitkeep b/deploy/roles/nvidia/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/nvidia/files/.gitkeep b/deploy/roles/nvidia/files/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/nvidia/handlers/main.yml b/deploy/roles/nvidia/handlers/main.yml deleted file mode 100644 index 18e548b..0000000 --- a/deploy/roles/nvidia/handlers/main.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -- name: reload the docker service - systemd: - name: docker - state: reloaded - listen: "reload the docker service" diff --git a/deploy/roles/nvidia/meta/.gitkeep b/deploy/roles/nvidia/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/nvidia/tasks/debian.yml b/deploy/roles/nvidia/tasks/debian.yml deleted file mode 100644 index dc1911a..0000000 --- a/deploy/roles/nvidia/tasks/debian.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- -# Nvidia Docker Runtime Installation -- name: get the NVIDIA signing key - apt_key: - state: present - url: "{{ nvidia.apt.signing_key_url }}" - -- name: add the libnvidia-container repo - apt_repository: - filename: "{{ nvidia.apt.repo_filename }}" - repo: "{{ nvidia.apt.repos.libnvidia_container }}" - state: present - -- name: add the nvidia-container-runtime repo - apt_repository: - filename: "{{ nvidia.apt.repo_filename }}" - repo: "{{ nvidia.apt.repos.nvidia_container_runtime }}" - state: present - -- name: add the nvidia-docker repo - apt_repository: - filename: "{{ nvidia.apt.repo_filename }}" - repo: "{{ nvidia.apt.repos.nvidia_docker }}" - state: present - -- name: install the nvidia-docker2 runtime - apt: - name: "{{ nvidia.apt.package }}" - state: present - update_cache: yes - notify: "reload the docker service" diff --git a/deploy/roles/nvidia/tasks/main.yml b/deploy/roles/nvidia/tasks/main.yml deleted file mode 100644 index b7581b4..0000000 --- a/deploy/roles/nvidia/tasks/main.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -# Nvidia Docker Runtime Setup -- import_tasks: debian.yml - when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu diff --git a/deploy/roles/nvidia/templates/.gitkeep b/deploy/roles/nvidia/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/nvidia/vars/main.yml b/deploy/roles/nvidia/vars/main.yml deleted file mode 100644 index 83064d2..0000000 --- a/deploy/roles/nvidia/vars/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -nvidia: - apt: - package: nvidia-docker2=2.* # known good with 2.0.3+docker18.09.1-1 - repo_filename: nvidia_github_io_nvidia_docker_ubuntu18_04_nvidia_docker.list - repos: - libnvidia_container: deb https://nvidia.github.io/libnvidia-container/ubuntu18.04/$(ARCH) / - nvidia_container_runtime: deb https://nvidia.github.io/nvidia-container-runtime/ubuntu18.04/$(ARCH) / - nvidia_docker: deb https://nvidia.github.io/nvidia-docker/ubuntu18.04/$(ARCH) / - signing_key_url: https://nvidia.github.io/nvidia-docker/gpgkey diff --git a/deploy/roles/packages/README.md b/deploy/roles/packages/README.md deleted file mode 100644 index 9d46317..0000000 --- a/deploy/roles/packages/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# The `packages` Role - -The `packages` role installs the operating system and Python pre-requisites that are required to execute the other roles. - -This role may be executed independently by running: - -```bash -$ ansible-playbook packages.yml --tags "packages" -``` - -On execution, this role installs the following directly to the remote machine's operating system: - -### Operating System Packages (Debian/Ubuntu) - -- apt-transport-https v1.x -- ca-certificates v20180409 -- curl v7.x -- gnupg-agent v2.x -- python-apt v1.x -- python-pip v9.x -- python-setuptools v39.x -- software-properties-common v0.96.24.32.7 - -### Python (pip) Packages - -- docker v3.7.0 or greater diff --git a/deploy/roles/packages/defaults/.gitkeep b/deploy/roles/packages/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/packages/files/.gitkeep b/deploy/roles/packages/files/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/packages/handlers/.gitkeep b/deploy/roles/packages/handlers/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/packages/meta/.gitkeep b/deploy/roles/packages/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/packages/tasks/debian.yml b/deploy/roles/packages/tasks/debian.yml deleted file mode 100644 index e7e0830..0000000 --- a/deploy/roles/packages/tasks/debian.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -# General Dependencies -- name: install general apt dependencies - apt: - name: "{{ general.dependencies.apt }}" - state: present - update_cache: yes - -- name: install general pip dependencies for Ansible - pip: - name: "{{ general.dependencies.pip }}" - state: present diff --git a/deploy/roles/packages/tasks/main.yml b/deploy/roles/packages/tasks/main.yml deleted file mode 100644 index bd512db..0000000 --- a/deploy/roles/packages/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- import_tasks: debian.yml - when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu diff --git a/deploy/roles/packages/templates/.gitkeep b/deploy/roles/packages/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/packages/vars/main.yml b/deploy/roles/packages/vars/main.yml deleted file mode 100644 index 800eb36..0000000 --- a/deploy/roles/packages/vars/main.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -# General Dependencies -general: - dependencies: - apt: - - apt-transport-https=1.* # known good with 1.6.6ubuntu0.1 all - - ca-certificates=20180409 # known good with 20180409 - - curl=7.* # known good with 7.58.0-2ubuntu3.5 - - gnupg-agent=2.* # known good with 2.2.4-1ubuntu1.2 - - python3-apt=1.* # known good with 1.6.3 - - python3-pip=9.* # known good with 9.0.1-2.3~ubuntu1 - - python3-setuptools=39.* # known good with 39.0.1-2 - - software-properties-common=0.96.24.32.7 # known good with 0.96.24.32.7 - - git - - golang - pip: - - docker>=3.7.0 # known good with 3.7.0 diff --git a/deploy/roles/rsync/defaults/.gitkeep b/deploy/roles/rsync/defaults/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/rsync/files/rsyncd.example.conf b/deploy/roles/rsync/files/rsyncd.example.conf deleted file mode 100644 index 1f34942..0000000 --- a/deploy/roles/rsync/files/rsyncd.example.conf +++ /dev/null @@ -1,16 +0,0 @@ -motd file = /etc/rsyncd.motd -log file = /var/log/rsyncd.log -pid file = /var/run/rsyncd.pid -lock file = /var/run/rsync.lock - -[jupyter_data] - path = /nfs/jupyter/data - comment = Jupyter data files - uid = nobody - gid = nobody - read only = yes - list = yes - auth users = @rsync:ro - secrets file = /etc/rsyncd.secrets - hosts allow = 192.168.1.0/24 - hosts deny = 0.0.0.0/0 diff --git a/deploy/roles/rsync/handlers/main.yml b/deploy/roles/rsync/handlers/main.yml deleted file mode 100644 index ce7e8e5..0000000 --- a/deploy/roles/rsync/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- name: restart the rsync service - systemd: - name: "{{ rsync.service_name }}" - state: restarted diff --git a/deploy/roles/rsync/meta/.gitkeep b/deploy/roles/rsync/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/rsync/tasks/main.yml b/deploy/roles/rsync/tasks/main.yml deleted file mode 100644 index 211ade0..0000000 --- a/deploy/roles/rsync/tasks/main.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# Configure the rsync daemon on the master -- name: copy rsync.conf up to the server - copy: - src: "{{ rsync.config_file.src }}" - dest: "{{ rsync.config_file.dest }}" - notify: - - restart the rsync service - when: inventory_hostname == rsync.master - -- name: configure rsync on the server - service: - enabled: yes - name: "{{ rsync.service_name }}" - state: started - when: inventory_hostname == rsync.master - -# Synchronize the data deploy across hosts -- name: synchronize the data deploy - delegate_to: "{{ rsync.master }}" - synchronize: - src: "{{ sync.source }}" - dest: "{{ sync.destination }}" - rsync_opts: "{{ rsync.options }}" - use_ssh_args: yes - when: inventory_hostname != rsync.master diff --git a/deploy/roles/rsync/templates/.gitkeep b/deploy/roles/rsync/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deploy/roles/rsync/vars/main.yml b/deploy/roles/rsync/vars/main.yml deleted file mode 100644 index 3f61189..0000000 --- a/deploy/roles/rsync/vars/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -rsync: - config_file: - src: rsyncd.conf - dest: /etc/rsyncd.conf - master: ml1 - options: - - "-e ssh" - - "--no-motd" - service_name: rsync diff --git a/deploy/secrets.example.yml b/deploy/secrets.example.yml deleted file mode 100644 index bc64cd4..0000000 --- a/deploy/secrets.example.yml +++ /dev/null @@ -1,15 +0,0 @@ -################################################################################ -# This file contains secrets that should not be checked into VCS. -################################################################################ ---- -secrets: - # Cachux7I - password_hash: sha1:b9bbcb5e92ad:0358d2ce7c34192afa50fc5e5143ed91d75eda0d - sync: - source: /nfs/data - destination: /nfs - users: - - ansible - volumes: - data: /nfs/data:/data - source: /nfs/src:/tf diff --git a/generate/modfyGenerate.go b/generate/modifyGenerate.go similarity index 100% rename from generate/modfyGenerate.go rename to generate/modifyGenerate.go From 9d69aed8ce0fe5273aaff2828f7d51c3d5ac2ce4 Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Mon, 30 Aug 2021 18:00:54 +0400 Subject: [PATCH 10/11] added usage docs --- Docs/Installation.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Docs/Installation.md b/Docs/Installation.md index 116ade4..56aec37 100644 --- a/Docs/Installation.md +++ b/Docs/Installation.md @@ -171,8 +171,13 @@ p2prc --rmcgroup --group --id ``` p2prc --rmgroup ``` - [read more on grouping containers](ClientImplementation.md#Grouping-Containers) +### Extending usecase of P2PRC (Requires a go compiler to run) +``` +p2prc --gen --mod +``` + +
From 6b43d94e8d199593cbf36a042412e5c0748b678f Mon Sep 17 00:00:00 2001 From: Akilan Selvacoumar Date: Tue, 31 Aug 2021 23:44:20 +0400 Subject: [PATCH 11/11] added docs for generate module --- Docs/GenerateImplementation.md | 116 +++++++++++++++++++++++++++++++++ Docs/Implementation.md | 3 +- Docs/Installation.md | 1 + Docs/README.md | 3 +- 4 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 Docs/GenerateImplementation.md diff --git a/Docs/GenerateImplementation.md b/Docs/GenerateImplementation.md new file mode 100644 index 0000000..0ea644c --- /dev/null +++ b/Docs/GenerateImplementation.md @@ -0,0 +1,116 @@ +# Generate Module +P2PRC is a great layer of abstraction. This means that in many cases it is not an end product but rather +a tool that customized as an end product. An example would be writing your own billing module to monetize +the computation power available. The generate module copies the current with the appropriate git histories +and keeps only the go files which would be useful to edit. To use the generate module the user will need +to have a go compiler present in his computer. Due to the introduction of this module there will 2 releases: + +- Regular Release (Consists of only the build binary and cli command cannot access the generate module) +- Developer Release (Consists of important Go files and the cli can access the generate module) + +## How does this work ? + +### [Struct information](https://github.com/Akilan1999/p2p-rendering-computation/blob/9d69aed8ce0fe5273aaff2828f7d51c3d5ac2ce4/generate/generate.go#L19) +- ###```Generate.go```: +This file creates a local copy of P2PRC from where the CLI was called from. +This go file also does various stuff like instruction of file should be ignored when copying and +which of should not be. Now let's understand this. Below is a sample code which does the following: + +```go + //---------------------------------------------------------------- + // Action performed: + // - Ensuring main.go file exists + // - Skipping all .go files apart from the ones listed above + // - Skipping .idea/ directory + // - Skipping Makefile file + //---------------------------------------------------------------- + Options.Skip = func(src string) (bool, error) { + switch { + case strings.HasSuffix(src, "main.go"): + return false, nil + case strings.HasSuffix(src, ".go"): + return true, nil + case strings.HasSuffix(src, ".idea"): + return true, nil + case strings.HasSuffix(src, "Makefile"): + return true, nil + default: + return false, nil + } + } + + // Doing the copy + err = copy.Copy("", "", Options) +``` + +Unfortunately currently this will have to be manually edited in the ```Generate.go``` file. When using the generate +module the user also creates their own Go module which is the modified version of P2PRC. This means +if the 1 modified package is using another modified package then the appropriate import have to be modified +in the file where the import is called: + +Ex: +```go + //Sample Project module name = Test + //Package names: + //- Test/Genius + //- Test/GeGeGenuis + // + // When we call the generate function with the new project with the module name = MicDrop + // The new package name would be: + // - MicDrop/Genius + // - MicDrop/GeGeGenuis + + // Test/Genius code depends on the package Test/GeGeGenuis + import ( + "Test/GeGeGenuis" + ) + +// When we create a new module with the copy of the +// existing project we need change: +import ( + "MicDrop/GeGeGenuis" +) +``` + +To do this we have built functions which can modify import names in the Go file provided. +To customize the use case of your generate module you would need to manually add your own +imports which are supposed to be replaced and in which files they are supposed to be replaced +in. + +```go +// 1.0 - Test/Genius.go -> GeGeGenuis module +// a is struct of type NewProject + a.FileNameAST = "/Test/Genius.go" + // Get AST information of the file + err := a.GetASTGoFile() + if err != nil { + return err + } + // Change the appropriate Go file + err = a.ChangeImports("Test/GeGeGenuis", "MicDrop/GeGeGenuis") + if err != nil { + return err + } + // Writes the change to the appropriate file + err = a.WriteGoAst() + if err != nil { + return err + } +``` + +Higher order of execution of ```Generate.go```: +1. Copy entire P2PRC project and ignores files which are not meant to be copied +2. The folder name will be based on the new project name and the module name based on the new + module name provided. +3. Modifies the appropriate imports in the project as instructed in the code. +4. Creates a commit with the new changes in the new project. + + +- ### ``` modifyGenerate.go```: + This a really simple implementation where we replace the imports + in certain files as instructed from ```generate.go```. To do we create an AST (i.e Abstract Syntax tree) + from new file we want to change the imports in. AST create a tree structure of expression. To change the + import we can just traverse to the appropriate expression and change the value of that expression in + the case of modifying imports. This approach is more simple than using templates. + + diff --git a/Docs/Implementation.md b/Docs/Implementation.md index 619385c..9a45341 100644 --- a/Docs/Implementation.md +++ b/Docs/Implementation.md @@ -16,4 +16,5 @@ the project is git.sr.ht/~akilan1999/p2p-rendering-computation. ## [Server Module](ServerImplementation.md) ## [Client Module](ClientImplementation.md) ## [P2P Module](P2PImplementation.md) -## [Plugin Module](PluginImplementation.md) \ No newline at end of file +## [Plugin Module](PluginImplementation.md) +## [Generate Module](GenerateImplementation.md) \ No newline at end of file diff --git a/Docs/Installation.md b/Docs/Installation.md index 56aec37..436ebd7 100644 --- a/Docs/Installation.md +++ b/Docs/Installation.md @@ -176,6 +176,7 @@ p2prc --rmgroup ``` p2prc --gen --mod ``` +[read more about the generate module](GenerateImplementation.md) diff --git a/Docs/README.md b/Docs/README.md index 26b4b20..e7b0f9b 100644 --- a/Docs/README.md +++ b/Docs/README.md @@ -13,5 +13,6 @@ 4. [Config Module](ConfigImplementation.md) 5. [Cli Module](CliImplementation.md) 6. [Plugin Module](PluginImplementation.md) -5. [Problems](Problems.md) + 7. [Generate Module](GenerateImplementation.md) +5. [Problems](https://github.com/Akilan1999/p2p-rendering-computation/issues)