added python dependencies

.gitignore

@@ -7,4 +7,9 @@ server/docker/__pycache__
 p2p-rendering-computation
 p2prc
 config.json
+*.tgz
 .vscode/
+hosts
+*.retry
+**/*.swp
+**/*.retry

Makefile

@@ -1,4 +1,4 @@
-SHELL := /bin/bash
+/ https://www.youtube.com/channel/UCQdZltW7bh1ta-_nCH7LWYw/playlistSHELL := /bin/bash
 
 install:
 	go build -o p2prc

deploy/roles/packages/vars/main.yml

@@ -13,5 +13,7 @@ general:
       - software-properties-common=0.96.24.32.7 # known good with 0.96.24.32.7
       - git
       - golang
+      - g++
+      - openjdk-8-jdk
     pip:
       - docker>=3.7.0 # known good with 3.7.0

go.mod

@@ -29,7 +29,7 @@ require (
 	golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4 // indirect
 	golang.org/x/text v0.3.5 // indirect
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
-	golang.org/x/tools v0.0.0-20210106214847-113979e3529a
+	golang.org/x/tools v0.0.0-20210106214847-113979e3529a // indirect
 	google.golang.org/grpc v1.35.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gotest.tools/v3 v3.0.3 // indirect

sample-run/ML/packages.yml

@@ -5,7 +5,7 @@
 #   3. handlers - these steps run only in response to specific events. If the event does not occur, the handler does not run.
 ################################################################################
 ---
-- hosts: test # Alternatively, 'canary', 'blue'|'green', 'all' according to your deployment strategy and your /etc/ansible/hosts file
+- hosts: gpu # Alternatively, 'canary', 'blue'|'green', 'all' according to your deployment strategy and your /etc/ansible/hosts file
   become: yes # All commands must run as root
   strategy: free # Allows all hosts to run to the end of the playbook as fast as they can
 
@@ -26,8 +26,8 @@
     #   vars:
     #     users: "{{ secrets.users }}"
 
-    - role: cuda # CUDA Driver Setup
+#    - role: cuda # CUDA Driver Setup
-      tags: [cuda]
+#      tags: [cuda]
 
     # - role: nvidia # NVIDIA-Docker Runtime Setup
     #   tags: [nvidia]

sample-run/ML/requirements.yml

@@ -0,0 +1,3 @@
+---
+- name: nvidia.nvidia_driver
+  version: 2.0.0

sample-run/ML/roles/packages/vars/main.yml

@@ -11,5 +11,10 @@ general:
       - software-properties-common # known good with 0.96.24.32.7
       - git
       - golang
+      - apt-utils
+      - libopenmpi-dev
+      - python3-setuptools
+      - python3-dev
     pip:
       - docker>=3.7.0 # known good with 3.7.0
+      - tensorflow

sample-run/run.txt

@@ -0,0 +1,27 @@
+Create 3 vm using ./p2prc --CreateVM 0.0.0.0 --Ports=1 --GPU
+
+then on each run 
+
+sudo apt clean && sudo apt update && sudo apt -y install cmake ccache nano
+sudo ln -s /usr/local/cuda-11.2 /usr/local/cuda
+export PATH=${HOME}/.local/bin:$PATH && export PATH=/usr/local/cuda-11.2/bin${PATH:+:${PATH}} && export LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+wget -c https://raw.githubusercontent.com/horovod/horovod/master/examples/tensorflow2/tensorflow2_synthetic_benchmark.py
+
+pip install --no-cache-dir horovod[tensorflow]
+
+# this will test if its locally install properly
+horovodrun -np 1 python3 tensorflow2_synthetic_benchmark.py
+
+then configure machine1 ssh/config file like
+Host host1
+  HostName 0.0.0.0
+  Port 39751
+
+Host host2
+  HostName 0.0.0.0
+  Port 34779
+
+and ensure that the ssh can be connected and ssh keys are copied using ssh-copy-id command
+
+then run 
+horovodrun -np 2 -H host1:1,host2:1 python3 tensorflow2_synthetic_benchmark.py

sample-run/tensorflow/hosts

@@ -1,12 +0,0 @@
----
-all:
-  vars:
-    ansible_python_interpreter: /usr/bin/python3
-test:
-  hosts:
-    test1:
-      ansible_host: 0.0.0.0     # Replace with your remote IP
-      ansible_port: 44003          # Replace with your remote SSH port
-      ansible_user: master       # Replace wtih your username
-      ansible_ssh_pass: password
-      ansible_sudo_pass: password

sample-run/tensorflow/roles/cuda/README.md

@@ -1,23 +0,0 @@
-# The `cuda` Role
-
-The `cuda` role installs the Nvidia CUDA GPU repository and drivers.
-
-This role may be executed independently by running:
-
-```bash
-$ ansible-playbook packages.yml --tags "cuda"
-```
-
-On execution, this role installs the following directly to the remote machine's operating system:
-
-### Apt Repositories (Debian/Ubuntu)
-
-- Nvidia CUDA
-
-### Apt GPG Signing Keys (Debian/Ubuntu)
-
-- cudatools <cudatools@nvidia.com>
-
-### Operating System Packages (Debian/Ubuntu)
-
-- cuda v10.x

sample-run/tensorflow/roles/cuda/handlers/main.yml

@@ -1,5 +0,0 @@
----
-# CUDA Driver Setup
-- name: reboot the machine
-  reboot:
-  listen: "reboot the machine"

sample-run/tensorflow/roles/cuda/tasks/debian.yml

@@ -1,22 +0,0 @@
----
-# Cuda Driver Package Installation
-- name: copy the CUDA repo onto the machine
-  copy:
-    src: "{{ cuda.apt.repo }}"
-    dest: "{{ general.working_dir }}/{{ cuda.apt.repo }}"
-
-- name: get the CUDA signing key
-  apt_key:
-    state: present
-    url: "{{ cuda.apt.signing_key_url }}"
-
-- name: add the CUDA repo
-  apt:
-    deb: "{{ general.working_dir }}/{{ cuda.apt.repo }}"
-    state: present
-
-- name: install the CUDA drivers
-  apt:
-    name: "{{ cuda.apt.package }}"
-    update_cache: yes
-  notify: "reboot the machine"

sample-run/tensorflow/roles/cuda/vars/main.yml

@@ -1,10 +0,0 @@
----
-cuda:
-  apt:
-    package: cuda=11.* # known good with 10.0.130-1
-    repo: cuda-11-3_11.3.1-1_amd64.deb # Network install
-    dependency: cuda-runtime-11-3_11.3.1-1_amd64.deb
-    signing_key_url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub
-
-general:
-  working_dir: /tmp

sample-run/tensorflow/roles/packages/tasks/main.yml

@@ -1,3 +0,0 @@
----
-- import_tasks: debian.yml
-  when: ansible_facts['os_family']|lower == 'debian' # debian, ubuntu

server/docker/containers/docker-ubuntu-sshd-gpu/Dockerfile

@@ -0,0 +1,50 @@
+# -----------------------------------------------------------------------------
+# This is base image of Ubuntu LTS with SSHD service.
+#
+# Authors: Art567
+# Updated: Sep 20th, 2015
+# Require: Docker (http://www.docker.io/)
+# -----------------------------------------------------------------------------
+
+# Base system is the latest LTS version of Ubuntu.
+# from   consol/ubuntu-xfce-vnc
+
+# due to dependency issues vnc is still work in progress
+from nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04
+
+# Switch to root user to install additional software
+USER 0
+
+# Make sure we don't get notifications we can't answer during building.
+env    DEBIAN_FRONTEND noninteractive
+
+# Prepare scripts and configs
+add    ./scripts/start /start
+
+# Download and install everything from the repos.
+run    apt-get -q -y update; apt-get -q -y upgrade && \
+       apt-get -q -y install sudo openssh-server python3-pip && \
+       mkdir /var/run/sshd
+
+# Set root password
+run    echo 'root:password' >> /root/passwdfile
+
+# Create user and it's password
+run    useradd -m -G sudo master && \
+       echo 'master:password' >> /root/passwdfile
+
+# Apply root password
+run    chpasswd -c SHA512 < /root/passwdfile && \
+       rm /root/passwdfile
+
+# Port 22 is used for ssh
+expose 22
+
+# Assign /data as static volume.
+volume ["/data"]
+
+# Fix all permissions
+run    chmod +x /start
+
+# Starting sshd
+cmd    ["/start"]

server/docker/containers/docker-ubuntu-sshd-gpu/README.md

@@ -0,0 +1,72 @@
+Ubuntu LTS with SSH (Docker)
+=========
+
+A Docker file to build a [docker.io] base container with Ubuntu LTS and a sshd service
+[docker.io]: http://docker.io
+Nice and easy way to get any server up and running using docker
+
+
+Instructions
+-----------
+ - Install Docker first. 
+   Read [installation instructions] (http://docs.docker.io/en/latest/installation/).
+ 
+ 
+ - Clone this repository:
+ 
+   `$ git clone https://github.com/art567/docker-ubuntu-sshd.git`
+ 
+ 
+ - Enter local directory:
+ 
+   `$ cd docker-ubuntu-sshd`
+ 
+ - Change passwords in Dockerfile:
+ 
+   `$ vi Dockerfile`
+ 
+ - Build the container:
+ 
+   `$ sudo docker build -t art567/ubuntu .`
+ 
+ 
+ - Run the container:
+ 
+   `$ sudo docker run -d=true --name=ubuntu --restart=always -p=2222:22 -v=/opt/data:/data art567/ubuntu /start`
+ 
+ 
+ - Your container will start and bind ssh to 2222 port.
+ 
+ 
+ - Find your local IP Address:
+ 
+   `$ ifconfig`
+ 
+ 
+ - Connect to the SSH server:
+ 
+   `$ ssh root@<ip-address> -p 2222`
+ 
+ 
+ - If you don't want to deal with root:
+ 
+   `$ ssh master@<ip-address> -p 2222`
+ 
+
+**VERY IMPORTANT!!!**
+-----------
+
+ **Don't forget to change passwords in Dockerfile before building image!**
+
+
+Versions
+-----------
+Some branches represents Ubuntu versions.
+
+Branch `master` is always represented by latest Ubuntu LTS
+
+   For example:
+   - 12.04 - Ubuntu 12.04 LTS
+   - 14.04 - Ubuntu 14.04 LTS
+   - 16.04 - Ubuntu 16.04 LTS
+

server/docker/containers/docker-ubuntu-sshd-gpu/scripts/start

@@ -0,0 +1,11 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# docker-ubuntu-sshd /start script
+#
+# Authors: Art567
+# Updated: Sep 20th, 2015
+# -----------------------------------------------------------------------------
+
+
+# Run OpenSSH server in daemon mode
+/usr/sbin/sshd -D

server/docker/docker.go

@@ -82,6 +82,9 @@ func BuildRunContainer(NumPorts int, GPU string) (*DockerVM,error) {
 		return nil,err
 	}
 	RespDocker.ImagePath = config.DockerFile
+	//if GPU == "true" {
+	//	RespDocker.ImagePath = "/home/asleepyguy/p2p-rendering-computation/server/docker/containers/docker-ubuntu-sshd-gpu/"
+	//}
 
 	// Gets docker information from env variables
 	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())