diff --git a/.gitignore b/.gitignore index 531dfcf..ff62927 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ server/docker/__pycache__ p2p-rendering-computation p2prc config.json +*.tgz .vscode/ \ No newline at end of file diff --git a/go.mod b/go.mod index f693d0a..d5eb59c 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 // indirect golang.org/x/text v0.3.5 // indirect golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - golang.org/x/tools v0.0.0-20210106214847-113979e3529a + golang.org/x/tools v0.0.0-20210106214847-113979e3529a // indirect google.golang.org/grpc v1.35.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools/v3 v3.0.3 // indirect diff --git a/sample-run/run.txt b/sample-run/run.txt new file mode 100644 index 0000000..9262f1e --- /dev/null +++ b/sample-run/run.txt @@ -0,0 +1,27 @@ +Create 3 vm using ./p2prc --CreateVM 0.0.0.0 --Ports=1 --GPU + +then on each run + +sudo apt clean && sudo apt update && sudo apt -y install cmake ccache nano +sudo ln -s /usr/local/cuda-11.2 /usr/local/cuda +export PATH=${HOME}/.local/bin:$PATH && export PATH=/usr/local/cuda-11.2/bin${PATH:+:${PATH}} && export LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} +wget -c https://raw.githubusercontent.com/horovod/horovod/master/examples/tensorflow2/tensorflow2_synthetic_benchmark.py + +pip install --no-cache-dir horovod[tensorflow] + +# this will test if its locally install properly +horovodrun -np 1 python3 tensorflow2_synthetic_benchmark.py + +then configure machine1 ssh/config file like +Host host1 + HostName 0.0.0.0 + Port 39751 + +Host host2 + HostName 0.0.0.0 + Port 34779 + +and ensure that the ssh can be connected and ssh keys are copied using ssh-copy-id command + +then run +horovodrun -np 2 -H host1:1,host2:1 python3 tensorflow2_synthetic_benchmark.py diff --git a/sample-run/tensorflow/hosts b/sample-run/tensorflow/hosts index 79b0fef..7d48a1c 100644 --- a/sample-run/tensorflow/hosts +++ b/sample-run/tensorflow/hosts @@ -6,7 +6,7 @@ test: hosts: test1: ansible_host: 0.0.0.0 # Replace with your remote IP - ansible_port: 44003 # Replace with your remote SSH port + ansible_port: 38775 # Replace with your remote SSH port ansible_user: master # Replace wtih your username ansible_ssh_pass: password ansible_sudo_pass: password \ No newline at end of file diff --git a/sample-run/tensorflow/packages.yml b/sample-run/tensorflow/packages.yml index 4afd81a..edabffd 100644 --- a/sample-run/tensorflow/packages.yml +++ b/sample-run/tensorflow/packages.yml @@ -26,8 +26,11 @@ # vars: # users: "{{ secrets.users }}" - - role: cuda # CUDA Driver Setup - tags: [cuda] +# - role: cuda # CUDA Driver Setup +# tags: [cuda] + + - role: cudnn + tags: [cudnn] # - role: nvidia # NVIDIA-Docker Runtime Setup # tags: [nvidia] diff --git a/sample-run/tensorflow/roles/cuda/README.md b/sample-run/tensorflow/roles/cuda/README.md deleted file mode 100644 index 752ecda..0000000 --- a/sample-run/tensorflow/roles/cuda/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# The `cuda` Role - -The `cuda` role installs the Nvidia CUDA GPU repository and drivers. - -This role may be executed independently by running: - -```bash -$ ansible-playbook packages.yml --tags "cuda" -``` - -On execution, this role installs the following directly to the remote machine's operating system: - -### Apt Repositories (Debian/Ubuntu) - -- Nvidia CUDA - -### Apt GPG Signing Keys (Debian/Ubuntu) - -- cudatools - -### Operating System Packages (Debian/Ubuntu) - -- cuda v10.x diff --git a/sample-run/tensorflow/roles/cuda/files/cuda-11-3_11.3.1-1_amd64.deb b/sample-run/tensorflow/roles/cuda/files/cuda-11-3_11.3.1-1_amd64.deb deleted file mode 100644 index 04dc7af..0000000 Binary files a/sample-run/tensorflow/roles/cuda/files/cuda-11-3_11.3.1-1_amd64.deb and /dev/null differ diff --git a/sample-run/tensorflow/roles/cuda/files/cuda-runtime-11-3_11.3.1-1_amd64.deb b/sample-run/tensorflow/roles/cuda/files/cuda-runtime-11-3_11.3.1-1_amd64.deb deleted file mode 100644 index ed7d4b3..0000000 Binary files a/sample-run/tensorflow/roles/cuda/files/cuda-runtime-11-3_11.3.1-1_amd64.deb and /dev/null differ diff --git a/sample-run/tensorflow/roles/cuda/handlers/main.yml b/sample-run/tensorflow/roles/cuda/handlers/main.yml deleted file mode 100644 index 6dfc62c..0000000 --- a/sample-run/tensorflow/roles/cuda/handlers/main.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -# CUDA Driver Setup -- name: reboot the machine - reboot: - listen: "reboot the machine" diff --git a/sample-run/tensorflow/roles/cuda/meta/.gitkeep b/sample-run/tensorflow/roles/cuda/meta/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/cuda/tasks/debian.yml b/sample-run/tensorflow/roles/cuda/tasks/debian.yml deleted file mode 100644 index 5290be8..0000000 --- a/sample-run/tensorflow/roles/cuda/tasks/debian.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Cuda Driver Package Installation -- name: copy the CUDA repo onto the machine - copy: - src: "{{ cuda.apt.repo }}" - dest: "{{ general.working_dir }}/{{ cuda.apt.repo }}" - -- name: get the CUDA signing key - apt_key: - state: present - url: "{{ cuda.apt.signing_key_url }}" - -- name: add the CUDA repo - apt: - deb: "{{ general.working_dir }}/{{ cuda.apt.repo }}" - state: present - -- name: install the CUDA drivers - apt: - name: "{{ cuda.apt.package }}" - update_cache: yes - notify: "reboot the machine" diff --git a/sample-run/tensorflow/roles/cuda/tasks/main.yml b/sample-run/tensorflow/roles/cuda/tasks/main.yml deleted file mode 100644 index bd512db..0000000 --- a/sample-run/tensorflow/roles/cuda/tasks/main.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -- import_tasks: debian.yml - when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu diff --git a/sample-run/tensorflow/roles/cuda/templates/.gitkeep b/sample-run/tensorflow/roles/cuda/templates/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/sample-run/tensorflow/roles/cuda/vars/main.yml b/sample-run/tensorflow/roles/cuda/vars/main.yml deleted file mode 100644 index dfb2793..0000000 --- a/sample-run/tensorflow/roles/cuda/vars/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -cuda: - apt: - package: cuda=11.* # known good with 10.0.130-1 - repo: cuda-11-3_11.3.1-1_amd64.deb # Network install - dependency: cuda-runtime-11-3_11.3.1-1_amd64.deb - signing_key_url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub - -general: - working_dir: /tmp diff --git a/sample-run/tensorflow/roles/cudnn/.travis.yml b/sample-run/tensorflow/roles/cudnn/.travis.yml new file mode 100644 index 0000000..36bbf62 --- /dev/null +++ b/sample-run/tensorflow/roles/cudnn/.travis.yml @@ -0,0 +1,29 @@ +--- +language: python +python: "2.7" + +# Use the new container infrastructure +sudo: false + +# Install ansible +addons: + apt: + packages: + - python-pip + +install: + # Install ansible + - pip install ansible + + # Check ansible version + - ansible --version + + # Create ansible.cfg with correct roles_path + - printf '[defaults]\nroles_path=../' >ansible.cfg + +script: + # Basic role syntax check + - ansible-playbook tests/test.yml -i tests/inventory --syntax-check + +notifications: + webhooks: https://galaxy.ansible.com/api/v1/notifications/ \ No newline at end of file diff --git a/sample-run/tensorflow/roles/cudnn/README.md b/sample-run/tensorflow/roles/cudnn/README.md new file mode 100644 index 0000000..411b7db --- /dev/null +++ b/sample-run/tensorflow/roles/cudnn/README.md @@ -0,0 +1,43 @@ +Ansible Role: cuDNN +========= + +Install [cuDNN](https://developer.nvidia.com/cudnn) on Linux System. + +Requirements +------------ + +- [CUDA Toolkit](https://developer.nvidia.com/cuda-toolkit) installed. +- Download [cuDNN](https://developer.nvidia.com/cudnn) to [`files`](files) folder with your NVIDIA Developer account. + +Role Variables +-------------- + +`cuda_version`: The version of CUDA Toolkit installed. (default: 8.0) + +`cudnn_version`: The version cuDNN. (default: v5.1) + +Dependencies +------------ + +None. + +Example Playbook +---------------- + +```yaml +- hosts: gpu + become: yes + + roles: + - { role: cudnn, cuda_version: 8.0, cudnn_version: v5.1} +``` + +License +------- + +BSD + +Author Information +------------------ + +This role was created by [Daniel D](https://github.com/djx339). diff --git a/sample-run/tensorflow/roles/cudnn/defaults/main.yml b/sample-run/tensorflow/roles/cudnn/defaults/main.yml new file mode 100644 index 0000000..5f7484f --- /dev/null +++ b/sample-run/tensorflow/roles/cudnn/defaults/main.yml @@ -0,0 +1,6 @@ +--- +# defaults file for cudnn +cuda_version: 8.0 +cuda_version_str: "v{{ cuda_version }}" + +cudnn_version: v5.1 diff --git a/sample-run/tensorflow/roles/cuda/defaults/.gitkeep b/sample-run/tensorflow/roles/cudnn/files/.gitkeep similarity index 100% rename from sample-run/tensorflow/roles/cuda/defaults/.gitkeep rename to sample-run/tensorflow/roles/cudnn/files/.gitkeep diff --git a/sample-run/tensorflow/roles/cudnn/handlers/main.yml b/sample-run/tensorflow/roles/cudnn/handlers/main.yml new file mode 100644 index 0000000..bdafa9a --- /dev/null +++ b/sample-run/tensorflow/roles/cudnn/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for cudnn diff --git a/sample-run/tensorflow/roles/cudnn/meta/main.yml b/sample-run/tensorflow/roles/cudnn/meta/main.yml new file mode 100644 index 0000000..97309e2 --- /dev/null +++ b/sample-run/tensorflow/roles/cudnn/meta/main.yml @@ -0,0 +1,26 @@ +galaxy_info: + author: Daniel D + description: An engineer + company: + + license: BSD + + min_ansible_version: 1.2 + + platforms: + - name: Ubuntu + versions: + - all + - trusty + - xenial + - name: CentOS + versions: + - all + + + galaxy_tags: + - cudnn + - nvidia + - cuda + +dependencies: [] diff --git a/sample-run/tensorflow/roles/cudnn/tasks/main.yml b/sample-run/tensorflow/roles/cudnn/tasks/main.yml new file mode 100644 index 0000000..c0a08bf --- /dev/null +++ b/sample-run/tensorflow/roles/cudnn/tasks/main.yml @@ -0,0 +1,15 @@ +--- +# tasks file for cudnn +- name: Create installer dir + file: "path={{ cudnn_installer_dir }} state=directory mode=0755" + +- name: Uncompressing cudnn lib + unarchive: "src={{ cudnn_installer.local_file }} dest={{ cudnn_installer_dir }}" + register: __cudnn_result + +- name: Install cudnn + shell: cp -rf {{ item.src }} {{ item.dest }} + with_items: + - { src: "{{ cudnn_installer_dir }}/cuda/lib64/*", dest: "/usr/local/cuda/lib64/" } + - { src: "{{ cudnn_installer_dir }}/cuda/include/cudnn.h", dest: "/usr/local/cuda/include/" } + when: __cudnn_result|changed diff --git a/sample-run/tensorflow/roles/cudnn/tests/inventory b/sample-run/tensorflow/roles/cudnn/tests/inventory new file mode 100644 index 0000000..d18580b --- /dev/null +++ b/sample-run/tensorflow/roles/cudnn/tests/inventory @@ -0,0 +1 @@ +localhost \ No newline at end of file diff --git a/sample-run/tensorflow/roles/cudnn/tests/test.yml b/sample-run/tensorflow/roles/cudnn/tests/test.yml new file mode 100644 index 0000000..ee49f39 --- /dev/null +++ b/sample-run/tensorflow/roles/cudnn/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - cudnn \ No newline at end of file diff --git a/sample-run/tensorflow/roles/cudnn/vars/main.yml b/sample-run/tensorflow/roles/cudnn/vars/main.yml new file mode 100644 index 0000000..c10dd31 --- /dev/null +++ b/sample-run/tensorflow/roles/cudnn/vars/main.yml @@ -0,0 +1,31 @@ +--- +# vars file for cudnn +cudnn_installer_map: + v11.5: + v8.2.0.53: + url: https://developer.nvidia.com/rdp/assets/cudnn-11.3-linux-x64-v8.2.53.tgz + local_file: cudnn-8.0-linux-x64-v5.1.tgz + v8.0: + v6.0: + url: https://developer.nvidia.com/compute/machine-learning/cudnn/secure/v6/prod/8.0_20170427/cudnn-8.0-linux-x64-v6.0-tgz + local_file: cudnn-8.0-linux-x64-v6.0.tgz + v5.1: + url: https://developer.nvidia.com/compute/machine-learning/cudnn/secure/v5.1/prod_20161129/8.0/cudnn-8.0-linux-x64-v5.1-tgz + local_file: cudnn-8.0-linux-x64-v5.1.tgz + v5.0: + url: https://developer.nvidia.com/rdp/assets/cudnn-8.0-linux-x64-v5.0-ga-tgz + local_file: cudnn-8.0-linux-x64-v5.0-ga.tgz + v7.5: + v6.0: + url: https://developer.nvidia.com/compute/machine-learning/cudnn/secure/v6/prod/7.5_20170427/cudnn-7.5-linux-x64-v6.0-tgz + local_file: cudnn-7.5-linux-x64-v6.0.tgz + v5.1: + url: https://developer.nvidia.com/compute/machine-learning/cudnn/secure/v5.1/prod_20161129/7.5/cudnn-7.5-linux-x64-v5.1-tgz + local_file: cudnn-7.5-linux-x64-v5.1.tgz + v5.0: + url: https://developer.nvidia.com/rdp/assets/cudnn-7.5-linux-x64-v5.0-ga.tgz + local_file: cudnn-7.5-linux-x64-v5.0-ga.tgz + + +cudnn_installer: "{{ cudnn_installer_map[cuda_version_str][cudnn_version] }}" +cudnn_installer_dir: "/tmp/cudnn_installer" diff --git a/server/docker/containers/docker-ubuntu-sshd-gpu/Dockerfile b/server/docker/containers/docker-ubuntu-sshd-gpu/Dockerfile new file mode 100644 index 0000000..f3fe442 --- /dev/null +++ b/server/docker/containers/docker-ubuntu-sshd-gpu/Dockerfile @@ -0,0 +1,50 @@ +# ----------------------------------------------------------------------------- +# This is base image of Ubuntu LTS with SSHD service. +# +# Authors: Art567 +# Updated: Sep 20th, 2015 +# Require: Docker (http://www.docker.io/) +# ----------------------------------------------------------------------------- + +# Base system is the latest LTS version of Ubuntu. +# from consol/ubuntu-xfce-vnc + +# due to dependency issues vnc is still work in progress +from nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 + +# Switch to root user to install additional software +USER 0 + +# Make sure we don't get notifications we can't answer during building. +env DEBIAN_FRONTEND noninteractive + +# Prepare scripts and configs +add ./scripts/start /start + +# Download and install everything from the repos. +run apt-get -q -y update; apt-get -q -y upgrade && \ + apt-get -q -y install sudo openssh-server python3-pip && \ + mkdir /var/run/sshd + +# Set root password +run echo 'root:password' >> /root/passwdfile + +# Create user and it's password +run useradd -m -G sudo master && \ + echo 'master:password' >> /root/passwdfile + +# Apply root password +run chpasswd -c SHA512 < /root/passwdfile && \ + rm /root/passwdfile + +# Port 22 is used for ssh +expose 22 + +# Assign /data as static volume. +volume ["/data"] + +# Fix all permissions +run chmod +x /start + +# Starting sshd +cmd ["/start"] diff --git a/server/docker/containers/docker-ubuntu-sshd-gpu/README.md b/server/docker/containers/docker-ubuntu-sshd-gpu/README.md new file mode 100644 index 0000000..ee62e54 --- /dev/null +++ b/server/docker/containers/docker-ubuntu-sshd-gpu/README.md @@ -0,0 +1,72 @@ +Ubuntu LTS with SSH (Docker) +========= + +A Docker file to build a [docker.io] base container with Ubuntu LTS and a sshd service +[docker.io]: http://docker.io +Nice and easy way to get any server up and running using docker + + +Instructions +----------- + - Install Docker first. + Read [installation instructions] (http://docs.docker.io/en/latest/installation/). + + + - Clone this repository: + + `$ git clone https://github.com/art567/docker-ubuntu-sshd.git` + + + - Enter local directory: + + `$ cd docker-ubuntu-sshd` + + - Change passwords in Dockerfile: + + `$ vi Dockerfile` + + - Build the container: + + `$ sudo docker build -t art567/ubuntu .` + + + - Run the container: + + `$ sudo docker run -d=true --name=ubuntu --restart=always -p=2222:22 -v=/opt/data:/data art567/ubuntu /start` + + + - Your container will start and bind ssh to 2222 port. + + + - Find your local IP Address: + + `$ ifconfig` + + + - Connect to the SSH server: + + `$ ssh root@ -p 2222` + + + - If you don't want to deal with root: + + `$ ssh master@ -p 2222` + + +**VERY IMPORTANT!!!** +----------- + + **Don't forget to change passwords in Dockerfile before building image!** + + +Versions +----------- +Some branches represents Ubuntu versions. + +Branch `master` is always represented by latest Ubuntu LTS + + For example: + - 12.04 - Ubuntu 12.04 LTS + - 14.04 - Ubuntu 14.04 LTS + - 16.04 - Ubuntu 16.04 LTS + diff --git a/server/docker/containers/docker-ubuntu-sshd-gpu/scripts/start b/server/docker/containers/docker-ubuntu-sshd-gpu/scripts/start new file mode 100644 index 0000000..5286196 --- /dev/null +++ b/server/docker/containers/docker-ubuntu-sshd-gpu/scripts/start @@ -0,0 +1,11 @@ +#!/bin/bash +# ----------------------------------------------------------------------------- +# docker-ubuntu-sshd /start script +# +# Authors: Art567 +# Updated: Sep 20th, 2015 +# ----------------------------------------------------------------------------- + + +# Run OpenSSH server in daemon mode +/usr/sbin/sshd -D diff --git a/server/docker/docker.go b/server/docker/docker.go index 10dd606..a23999a 100644 --- a/server/docker/docker.go +++ b/server/docker/docker.go @@ -82,6 +82,9 @@ func BuildRunContainer(NumPorts int, GPU string) (*DockerVM,error) { return nil,err } RespDocker.ImagePath = config.DockerFile + if GPU == "true" { + RespDocker.ImagePath = "/home/asleepyguy/p2p-rendering-computation/server/docker/containers/docker-ubuntu-sshd-gpu/" + } // Gets docker information from env variables cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) diff --git a/server/docker/kill-containers.sh b/server/docker/kill-containers.sh old mode 100644 new mode 100755