diff --git a/.gitignore b/.gitignore index ff62927..c6178ed 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ p2p-rendering-computation p2prc config.json *.tgz -.vscode/ \ No newline at end of file +.vscode/ +hosts \ No newline at end of file diff --git a/sample-run/tensorflow/packages.yml b/sample-run/tensorflow/packages.yml index edabffd..9ee4a1c 100644 --- a/sample-run/tensorflow/packages.yml +++ b/sample-run/tensorflow/packages.yml @@ -32,6 +32,9 @@ - role: cudnn tags: [cudnn] + - role: ansible-role-nvidia-driver + tags: [ansible-role-nvidia-driver] + # - role: nvidia # NVIDIA-Docker Runtime Setup # tags: [nvidia] diff --git a/sample-run/tensorflow/requirements.yml b/sample-run/tensorflow/requirements.yml new file mode 100644 index 0000000..738f688 --- /dev/null +++ b/sample-run/tensorflow/requirements.yml @@ -0,0 +1,3 @@ +--- +- name: nvidia.nvidia_driver + version: 2.0.0 \ No newline at end of file diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/.gitignore b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/.gitignore new file mode 100644 index 0000000..a8b42eb --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/.gitignore @@ -0,0 +1 @@ +*.retry diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/LICENSE b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/LICENSE new file mode 100644 index 0000000..918ff18 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of NVIDIA CORPORATION nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/README.md b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/README.md new file mode 100644 index 0000000..6625ede --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/README.md @@ -0,0 +1,77 @@ +# ansible-role-nvidia-driver + +An Ansible role to install the NVIDIA driver from the NVIDIA CUDA repositories. + +## Requirements + +In the process of installing the NVIDIA driver, this role will reboot the nodes where it runs. +Because of this, we strongly recommend that you run `ansible-playbook` from a separate node than the GPU nodes where you are installing the driver. + +If you attempt to run Ansible on the same node where you are installing the driver, this role will either: + +* Refuse to proceed with an error like `Running reboot with local connection would reboot the control node` (if running with the `local` connection) +* Reboot the node you're running on, interrupting the playbook execution! (if running the an `ssh` connection against localhost) + +## Installing + +This role can be installed using [Ansible Galaxy](https://galaxy.ansible.com/nvidia/nvidia_driver): + +``` +$ ansible-galaxy install nvidia.nvidia_driver +``` + +## Role variables + + +| Variable | Default value | Description | +| -------- | ------------- | ----------- | +| `nvidia_driver_package_state` | `"present"` | Package state for NVIDIA driver packages | +| `nvidia_driver_package_version` | `""` | Package version to install. Note that this should match the actual version of the deb or RPM package to be installed. | +| `nvidia_driver_persistence_mode_on` | `yes` | Whether to enable persistence mode (boolean) | +| `nvidia_driver_skip_reboot` | `no` | Whether to skip rebooting the node during the install | +| `nvidia_driver_module_file` | `"/etc/modprobe.d/nvidia.conf"` | Filename to use for NVIDIA driver parameters | +| `nvidia_driver_module_params` | `""` | Parameters to pass to the NVIDIA driver | + +### Red Hat specific variables + + +| Variable | Default value | Description | +| -------- | ------------- | ----------- | +| `epel_package` | `"https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm"` | Package to install to enable EPEL | +| `nvidia_driver_rhel_cuda_repo_baseurl` | `"https://developer.download.nvidia.com/compute/cuda/repos/{{ _rhel_repo_dir }}/"` | Base URL to use for CUDA repo | +| `nvidia_driver_rhel_cuda_repo_gpgkey` | `"https://developer.download.nvidia.com/compute/cuda/repos/{{ _rhel_repo_dir }}/7fa2af80.pub"` | GPG key for the CUDA repo | + +### Ubuntu specific variables + +For Ubuntu installs, you have the choice of installing from the Canonical repositories and the NVIDIA CUDA repositories. + +By default, the Canonical repositories will be used, and the driver installed will be the headless server driver. + +| Variable | Default value | Description | +| -------- | ------------- | ----------- | +| `nvidia_driver_ubuntu_install_from_cuda_repo` | `no` | Flag whether to use the CUDA repo | +| `nvidia_driver_ubuntu_branch` | `450` | Driver branch to use for the install | +| `nvidia_driver_ubuntu_packages` | `["nvidia-headless-450-server", "nvidia-headless-450-utils"]` | Package names to install from Canonical repo | +| `nvidia_driver_ubuntu_cuda_repo_baseurl` | `"http://developer.download.nvidia.com/compute/cuda/repos/{{ _ubuntu_repo_dir }}"` | Base URL to use for CUDA repo | +| `nvidia_driver_ubuntu_cuda_repo_gpgkey_url` | `"https://developer.download.nvidia.com/compute/cuda/repos/{{ _ubuntu_repo_dir }}/7fa2af80.pub"` | GPG key for the CUDA repo | +| `nvidia_driver_ubuntu_cuda_repo_gpgkey_id` | `"7fa2af80"` | GPG key ID for the CUDA repo | +| `nvidia_driver_ubuntu_cuda_package` | `"cuda-drivers"` | Package name to install from CUDA repo | + +## Example playbook + +``` +- hosts: gpu_nodes + roles: + - nvidia.nvidia_driver +``` + +## Supported distributions + +Currently, this role supports the following Linux distributions: + +* NVIDIA DGX OS 4 +* NVIDIA DGX OS 5 +* Ubuntu 18.04 LTS +* Ubuntu 20.04 LTS +* CentOS 7 +* Red Hat Enterprise Linux 7 diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/defaults/main.yml b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/defaults/main.yml new file mode 100644 index 0000000..2d8b0e2 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/defaults/main.yml @@ -0,0 +1,36 @@ +nvidia_driver_package_state: present +nvidia_driver_package_version: '' +nvidia_driver_persistence_mode_on: yes +nvidia_driver_skip_reboot: no +nvidia_driver_module_file: /etc/modprobe.d/nvidia.conf +nvidia_driver_module_params: '' + + +############################################################################## +# RedHat family # +############################################################################## +epel_package: "https://dl.fedoraproject.org/pub/epel/epel-release-latest-{{ ansible_distribution_major_version }}.noarch.rpm" +nvidia_driver_rhel_cuda_repo_baseurl: "https://developer.download.nvidia.com/compute/cuda/repos/{{ _rhel_repo_dir }}/" +nvidia_driver_rhel_cuda_repo_gpgkey: "https://developer.download.nvidia.com/compute/cuda/repos/{{ _rhel_repo_dir }}/7fa2af80.pub" + + +############################################################################## +# Ubuntu # +############################################################################## + +# Determine if we should install from CUDA repo instead of Canonical repos +nvidia_driver_ubuntu_install_from_cuda_repo: no + +# Installing with Canonical repositories +nvidia_driver_ubuntu_branch: "450" +nvidia_driver_ubuntu_packages: +- "nvidia-headless-{{ nvidia_driver_ubuntu_branch }}-server" +- "nvidia-utils-{{ nvidia_driver_ubuntu_branch }}-server" +- "nvidia-headless-no-dkms-{{ nvidia_driver_ubuntu_branch }}-server" +- "nvidia-kernel-source-{{ nvidia_driver_ubuntu_branch }}-server" + +# Installing with CUDA repositories +nvidia_driver_ubuntu_cuda_repo_gpgkey_url: "https://developer.download.nvidia.com/compute/cuda/repos/{{ _ubuntu_repo_dir }}/7fa2af80.pub" +nvidia_driver_ubuntu_cuda_repo_gpgkey_id: "7fa2af80" +nvidia_driver_ubuntu_cuda_repo_baseurl: "http://developer.download.nvidia.com/compute/cuda/repos/{{ _ubuntu_repo_dir }}" +nvidia_driver_ubuntu_cuda_package: "cuda-drivers" diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/files/cuda-ubuntu.pin b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/files/cuda-ubuntu.pin new file mode 100644 index 0000000..b050308 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/files/cuda-ubuntu.pin @@ -0,0 +1,3 @@ +Package: * +Pin: release l=NVIDIA CUDA +Pin-Priority: 600 diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/files/nvidia-persistenced-override.conf b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/files/nvidia-persistenced-override.conf new file mode 100644 index 0000000..52b552d --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/files/nvidia-persistenced-override.conf @@ -0,0 +1,3 @@ +[Service] +ExecStart= +ExecStart=/usr/bin/nvidia-persistenced --user root --persistence-mode --verbose diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/meta/main.yml b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/meta/main.yml new file mode 100644 index 0000000..da7c9f0 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/meta/main.yml @@ -0,0 +1,22 @@ +galaxy_info: + role_name: nvidia_driver + author: Luke Yeager + company: NVIDIA + description: Install the NVIDIA driver + license: 3-Clause BSD + min_ansible_version: 2.7 + + platforms: + - name: Ubuntu + versions: + - 'xenial' + - 'bionic' + - 'focal' + - name: EL + versions: + - '7' + + galaxy_tags: + - 'nvidia' + - 'cuda' + - 'driver' diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/install-redhat.yml b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/install-redhat.yml new file mode 100644 index 0000000..7870178 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/install-redhat.yml @@ -0,0 +1,57 @@ +--- +# We have to do this because the CentOS mirrors don't keep kernel-headers, etc +# for older kernels. +- name: ensure we have kernel-headers installed for the current kernel + block: + - name: attempt to install kernel support packages for current version + yum: + name: + - "kernel-headers-{{ ansible_kernel }}" + - "kernel-tools-{{ ansible_kernel }}" + - "kernel-tools-libs-{{ ansible_kernel }}" + - "kernel-devel-{{ ansible_kernel }}" + - "kernel-debug-devel-{{ ansible_kernel }}" + state: present + environment: "{{proxy_env if proxy_env is defined else {}}}" + rescue: + - name: update the kernel to latest version so we have a supported version + yum: + name: + - "kernel" + - "kernel-headers" + - "kernel-tools" + - "kernel-tools-libs" + - "kernel-devel" + - "kernel-debug-devel" + state: latest + environment: "{{proxy_env if proxy_env is defined else {}}}" + - name: reboot to pick up the new kernel + reboot: + +- name: add epel repo + become: true + yum: + name: + - "{{ epel_package }}" + state: latest + environment: "{{proxy_env if proxy_env is defined else {}}}" + +- name: install dependencies + yum: name=dkms + +- name: add repo + yum_repository: + name: cuda + description: NVIDIA CUDA YUM Repo + baseurl: "{{ nvidia_driver_rhel_cuda_repo_baseurl }}" + gpgkey: "{{ nvidia_driver_rhel_cuda_repo_gpgkey }}" + environment: "{{proxy_env if proxy_env is defined else {}}}" + +- name: install driver packages + yum: + name: "{{ nvidia_driver_package_version | ternary('nvidia-driver-latest-dkms-'+nvidia_driver_package_version, 'nvidia-driver-latest-dkms') }}" + state: "{{ nvidia_driver_package_state }}" + autoremove: "{{ nvidia_driver_package_state == 'absent' }}" + register: install_driver + environment: "{{proxy_env if proxy_env is defined else {}}}" + diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/install-ubuntu-cuda-repo.yml b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/install-ubuntu-cuda-repo.yml new file mode 100644 index 0000000..7aab994 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/install-ubuntu-cuda-repo.yml @@ -0,0 +1,35 @@ +--- +- name: remove ppa + apt_repository: + repo: ppa:graphics-drivers/ppa + state: absent + +- name: add pin file + copy: + src: "cuda-ubuntu.pin" + dest: "/etc/apt/preferences.d/cuda-repository-pin-600" + owner: "root" + group: "root" + mode: "0644" + +- name: add key + apt_key: + url: "{{ nvidia_driver_ubuntu_cuda_repo_gpgkey_url }}" + id: "{{ nvidia_driver_ubuntu_cuda_repo_gpgkey_id }}" + environment: "{{proxy_env if proxy_env is defined else {}}}" + + +- name: add repo + apt_repository: + repo: "deb {{ nvidia_driver_ubuntu_cuda_repo_baseurl }} /" + update_cache: yes + environment: "{{proxy_env if proxy_env is defined else {}}}" + +- name: install driver packages + apt: + name: "{{ nvidia_driver_package_version | ternary(nvidia_driver_ubuntu_cuda_package+'='+nvidia_driver_package_version, nvidia_driver_ubuntu_cuda_package) }}" + state: "{{ nvidia_driver_package_state }}" + autoremove: "{{ nvidia_driver_package_state == 'absent' }}" + purge: "{{ nvidia_driver_package_state == 'absent' }}" + register: install_driver + environment: "{{proxy_env if proxy_env is defined else {}}}" diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/install-ubuntu.yml b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/install-ubuntu.yml new file mode 100644 index 0000000..bcd0a39 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/install-ubuntu.yml @@ -0,0 +1,15 @@ +--- +- name: remove ppa + apt_repository: + repo: ppa:graphics-drivers/ppa + state: absent + +- name: install driver packages + apt: + name: "{{ nvidia_driver_package_version | ternary(item+'='+nvidia_driver_package_version, item) }}" + state: "{{ nvidia_driver_package_state }}" + autoremove: "{{ nvidia_driver_package_state == 'absent' }}" + purge: "{{ nvidia_driver_package_state == 'absent' }}" + with_items: "{{ nvidia_driver_ubuntu_packages }}" + register: install_driver + environment: "{{proxy_env if proxy_env is defined else {}}}" diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/main.yml b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/main.yml new file mode 100644 index 0000000..d33f672 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tasks/main.yml @@ -0,0 +1,52 @@ +--- +- name: unload nouveau + modprobe: + name: nouveau + state: absent + ignore_errors: true + +- name: ubuntu install tasks (canonical repos) + include_tasks: install-ubuntu.yml + when: ansible_distribution == 'Ubuntu' and (not nvidia_driver_ubuntu_install_from_cuda_repo) + +- name: ubuntu install tasks (CUDA repo) + include_tasks: install-ubuntu-cuda-repo.yml + when: ansible_distribution == 'Ubuntu' and nvidia_driver_ubuntu_install_from_cuda_repo + +- name: redhat family install tasks + include_tasks: install-redhat.yml + when: ansible_os_family == 'RedHat' + +- name: create persistenced override dir + file: + path: /etc/systemd/system/nvidia-persistenced.service.d/ + state: directory + recurse: yes + +- name: configure persistenced service to turn on persistence mode + copy: + src: nvidia-persistenced-override.conf + dest: /etc/systemd/system/nvidia-persistenced.service.d/override.conf + when: nvidia_driver_persistence_mode_on + +- name: remove persistenced service override + file: + path: /etc/systemd/system/nvidia-persistenced.service.d/override.conf + state: absent + when: not nvidia_driver_persistence_mode_on + +- name: enable persistenced + systemd: + name: nvidia-persistenced + enabled: yes + when: nvidia_driver_package_state != 'absent' + +- name: set module parameters + template: + src: nvidia.conf.j2 + dest: "{{ nvidia_driver_module_file }}" + mode: '0644' + +- name: reboot after driver install + reboot: + when: install_driver.changed and not nvidia_driver_skip_reboot diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/templates/nvidia.conf.j2 b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/templates/nvidia.conf.j2 new file mode 100644 index 0000000..b7b6654 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/templates/nvidia.conf.j2 @@ -0,0 +1 @@ +{{ nvidia_driver_module_params }} diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tests/README.md b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tests/README.md new file mode 100644 index 0000000..4673873 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tests/README.md @@ -0,0 +1,6 @@ +Test like this: +``` +ansible-playbook --inventory tests/inventory.yml tests/playbook.yml +``` +By default, the test inventory operates on the localhost. +You may want to change this. diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tests/inventory b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tests/inventory new file mode 100644 index 0000000..2fbb50c --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tests/inventory @@ -0,0 +1 @@ +localhost diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tests/playbook.yml b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tests/playbook.yml new file mode 100644 index 0000000..b3cc2b1 --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/tests/playbook.yml @@ -0,0 +1,5 @@ +--- +- hosts: all + become: yes + roles: + - ../.. diff --git a/sample-run/tensorflow/roles/ansible-role-nvidia-driver/vars/main.yml b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/vars/main.yml new file mode 100644 index 0000000..c994f5b --- /dev/null +++ b/sample-run/tensorflow/roles/ansible-role-nvidia-driver/vars/main.yml @@ -0,0 +1,2 @@ +_ubuntu_repo_dir: "{{ ansible_distribution | lower }}{{ ansible_distribution_version | replace('.', '') }}/{{ ansible_architecture }}" +_rhel_repo_dir: "rhel{{ ansible_distribution_major_version }}/{{ ansible_architecture }}" diff --git a/server/docker/docker.go b/server/docker/docker.go index a23999a..4fbdf2e 100644 --- a/server/docker/docker.go +++ b/server/docker/docker.go @@ -82,9 +82,9 @@ func BuildRunContainer(NumPorts int, GPU string) (*DockerVM,error) { return nil,err } RespDocker.ImagePath = config.DockerFile - if GPU == "true" { - RespDocker.ImagePath = "/home/asleepyguy/p2p-rendering-computation/server/docker/containers/docker-ubuntu-sshd-gpu/" - } + //if GPU == "true" { + // RespDocker.ImagePath = "/home/asleepyguy/p2p-rendering-computation/server/docker/containers/docker-ubuntu-sshd-gpu/" + //} // Gets docker information from env variables cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())