me/p2p-rendering-computation
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added WASM support

Browse Source
This commit is contained in:
Akilan Selvacoumar
2025-05-03 01:29:11 +01:00
parent ece1e63fda
commit 034f9e2896
56 changed files with 1293 additions and 10443 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
server/docker/.DS_Store vendored
View File

Binary file not shown.

12
server/docker/Makefile
View File

@@ -1,12 +0,0 @@
SHELL := /bin/bash
.PHONY: set_virtualenv,install_docker_requirements,dockerproc
set_virtualenv:
virtualenv env
install_docker_requirements:
source env/bin/activate && pip install -r requirements.txt
dockerproc:
ADMIN_USER=admin ADMIN_PASSWORD=admin docker-compose -f dockprom/docker-compose.yml up -d

6
server/docker/README
View File

@@ -1,6 +0,0 @@
Docker Module P2P-rendering-computation
========================================
This module is incharge to spin up docker contaianers , create
SSH server and VNC server. This module is implemented using
python.

508
server/docker/docker.go
View File

@@ -1,508 +0,0 @@
package docker
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"github.com/Akilan1999/p2p-rendering-computation/config"
"github.com/docker/docker/client"
"github.com/google/uuid"
"github.com/lithammer/shortuuid"
"github.com/otiai10/copy"
"github.com/phayes/freeport"
"io"
"io/ioutil"
"os"
"os/exec"
"text/template"
)
type DockerVM struct {
SSHUsername string `json:"SSHUsername"`
SSHPublcKey string `json:"SSHPublicKey"`
ID string `json:"ID"`
TagName string `json:"TagName"`
ImagePath string `json:"ImagePath"`
Ports Ports `json:"Ports"`
GPU string `json:"GPU"`
TempPath string
BaseImage string
LogsPath string
SSHCommand string `json:"SSHCommand"`
}
type DockerContainers struct {
DockerContainer []DockerContainer `json:"DockerContainer"`
}
type DockerContainer struct {
ContainerName string `json:"DockerContainerName"`
ContainerDescription string `json:"ContainerDescription"`
}
type Ports struct {
PortSet []Port `json:"Port"`
}
type Port struct {
PortName string `json:"PortName"`
InternalPort int `json:"InternalPort"`
Type string `json:"Type"`
ExternalPort int `json:"ExternalPort"`
IsUsed bool `json:"IsUsed"`
Description string `json:"Description"`
}
type ErrorLine struct {
Error string `json:"error"`
ErrorDetail ErrorDetail `json:"errorDetail"`
}
type ErrorDetail struct {
Message string `json:"message"`
}
var dockerRegistryUserID = ""
// BuildRunContainer Function is incharge to invoke building and running contianer and also allocating external
// ports
func BuildRunContainer(NumPorts int, GPU string, ContainerName string, baseImage string, publicKey string) (*DockerVM, error) {
//Docker Struct Variable
var RespDocker *DockerVM = new(DockerVM)
// Sets if GPU is selected or not
RespDocker.GPU = GPU
// Get config informatopn
// Sets Free port to Struct
//RespDocker.SSHPort = Ports[0]
//RespDocker.VNCPort = Ports[1]
// Sets appropriate username and password to the
// variables in the struct
RespDocker.SSHUsername = "root"
//RespDocker.BaseImage = "ubuntu:20.04"
//RespDocker.VNCPassword = "vncpassword"
//Default parameters
RespDocker.TagName = "p2p-ubuntu"
// Get Path from config
config, err := config.ConfigInit(nil, nil)
if err != nil {
return nil, err
}
RespDocker.ImagePath = config.DefaultDockerFile
RespDocker.LogsPath = config.DockerRunLogs
RespDocker.SSHPublcKey = publicKey
// We are checking if the container name is not nil and not equal to the default one used
// which is docker-ubuntu-sshd
if ContainerName != "" && ContainerName != "docker-ubuntu-sshd" {
Containers, err := ViewAllContainers()
if err != nil {
return nil, err
}
for _, dockerContainer := range Containers.DockerContainer {
if dockerContainer.ContainerName == ContainerName {
RespDocker.ImagePath = config.DockerContainers + ContainerName + "/"
RespDocker.TagName = ContainerName
break
}
}
if RespDocker.ImagePath == config.DefaultDockerFile {
return nil, errors.New("Container " + ContainerName + " does not exist in the server")
}
}
// Checking if the base image is provided
if baseImage != "" {
RespDocker.BaseImage = baseImage
} else {
RespDocker.BaseImage = "ubuntu:20.04"
}
// Template docker with the base image provided
err = RespDocker.TemplateDockerContainer()
if err != nil {
return nil, err
}
// Template the DockerFile and point to the temp location
PortsInformation, err := OpenPortsFile(RespDocker.ImagePath + "/" + RespDocker.TagName + "/ports.json")
if err != nil {
return nil, err
}
// Number of perts we want to open + number of ports required inside the
// docker container
count := NumPorts + len(PortsInformation.PortSet)
// Creates number of ports
OpenPorts, err := freeport.GetFreePorts(count)
if err != nil {
return nil, err
}
// Allocate external ports to ports available in the ports.json file
for i := range PortsInformation.PortSet {
// Setting external ports
PortsInformation.PortSet[i].ExternalPort = OpenPorts[i]
PortsInformation.PortSet[i].IsUsed = true
}
//Length of Ports allocated from thr port file
portFileLength := len(PortsInformation.PortSet)
// Allocate New ports the user wants to generate
for i := 0; i < NumPorts; i++ {
var TempPort Port
TempPort.PortName = "AutoGen Port"
TempPort.Type = "tcp"
TempPort.InternalPort = OpenPorts[portFileLength+i]
TempPort.ExternalPort = OpenPorts[portFileLength+i]
TempPort.Description = "Auto generated TCP port"
TempPort.IsUsed = false
//Append temp port to port information
PortsInformation.PortSet = append(PortsInformation.PortSet, TempPort)
}
// Setting ports to the docker VM struct
RespDocker.Ports = *PortsInformation
// Gets docker information from env variables
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
return nil, err
}
// Builds docker image
err = RespDocker.imageBuild(cli)
if err != nil {
return nil, err
}
// Runs docker contianer
err = RespDocker.runContainer(cli)
if err != nil {
return nil, err
}
return RespDocker, nil
}
// Builds docker image (TODO: relative path for Dockerfile deploy)
func (d *DockerVM) imageBuild(dockerClient *client.Client) error {
//ctx, _ := context.WithTimeout(context.Background(), time.Second*2000)
//defer cancel()
var cmd bytes.Buffer
cmd.WriteString("docker build -t " + d.TagName + " " + d.ImagePath + "/" + d.TagName + ` --build-arg SSH_KEY="` + d.SSHPublcKey + `"`)
//"-v=/opt/data:/data p2p-ubuntu /start > /dev/null"
cmdStr := cmd.String()
output, err := exec.Command("/bin/sh", "-c", cmdStr).Output()
fmt.Printf("%s", output)
if err != nil {
return err
}
//tar, err := archive.TarWithOptions(d.ImagePath+"/"+d.TagName, &archive.TarOptions{})
//if err != nil {
// return err
//}
//
//opts := types.ImageBuildOptions{
// Dockerfile: "Dockerfile",
// Tags: []string{d.TagName},
// Remove: true,
//}
//res, err := dockerClient.ImageBuild(ctx, tar, opts)
//if err != nil {
// return err
//}
//
//defer res.Body.Close()
//
//err = print(res.Body)
//if err != nil {
// return err
//}
return nil
}
// Starts container and assigns port numbers
// Sample Docker run Command
// docker run -d=true --name=Test123 --restart=always --gpus all
// -p 3443:6901 -p 3453:22 -p 3434:3434 -p 3245:3245 -v=/opt/data:/data
// p2p-ubuntu /start > /dev/null
func (d *DockerVM) runContainer(dockerClient *client.Client) error {
//ctx, _ := context.WithTimeout(context.Background(), time.Second*2000)
// The first mode runs using the Docker Api. As the API supports using
// CPU and uses a shell script for GPU call because till this point of
// implementation docker api does not support the flag "--gpu all"
//if d.GPU != "true" {
// //Exposed ports for docker config file
// var ExposedPort nat.PortSet
//
// ExposedPort = nat.PortSet{
// "22/tcp": struct{}{},
// //"6901/tcp": struct{}{},
// }
//
// // Port forwarding for VNC and SSH ports
// PortForwarding := nat.PortMap{
// //"22/tcp": []nat.PortBinding{
// // {
// // HostIP: "0.0.0.0",
// // HostPort: fmt.Sprint(d.SSHPort),
// // },
// //},
// //"6901/tcp": []nat.PortBinding{
// // {
// // HostIP: "0.0.0.0",
// // HostPort: fmt.Sprint(d.VNCPort),
// // },
// //},
// }
//
// for i := range d.Ports.PortSet {
// // Parameters "tcp or udp", external port
// Port, err := nat.NewPort(d.Ports.PortSet[i].Type, fmt.Sprint(d.Ports.PortSet[i].InternalPort))
// if err != nil {
// return err
// }
//
// // Exposed Ports
// ExposedPort[Port] = struct{}{}
//
// PortForwarding[Port] = []nat.PortBinding{
// {
// HostIP: "0.0.0.0",
// HostPort: fmt.Sprint(d.Ports.PortSet[i].ExternalPort),
// },
// }
// }
//
// config := &container.Config{
// Image: d.TagName,
// Entrypoint: []string{"/start"},
// Volumes: map[string]struct{}{"/opt/data:/data": {}},
// ExposedPorts: ExposedPort,
// }
// hostConfig := &container.HostConfig{
// PortBindings: PortForwarding,
// }
//
// res, err := dockerClient.ContainerCreate(ctx, config, hostConfig,
// nil, nil, "")
//
// // Set response ID
// d.ID = res.ID
//
// if err != nil {
// return err
// }
//
// err = dockerClient.ContainerStart(ctx, res.ID, types.ContainerStartOptions{})
//
// if err != nil {
// return err
// }
//} else {
// Generate Random ID
id := shortuuid.New()
d.ID = id
var cmd bytes.Buffer
cmd.WriteString("docker run -d=true --name=" + id + " --restart=always ")
if d.GPU == "true" {
cmd.WriteString("--gpus all ")
}
for i := range d.Ports.PortSet {
cmd.WriteString("-p " + fmt.Sprint(d.Ports.PortSet[i].ExternalPort) + ":" + fmt.Sprint(d.Ports.PortSet[i].InternalPort) + " ")
}
cmd.WriteString("-v=/tmp:/data " + d.TagName + " > /dev/null")
//"-v=/opt/data:/data p2p-ubuntu /start > /dev/null"
cmdStr := cmd.String()
_, err := exec.Command("/bin/sh", "-c", cmdStr).Output()
if err != nil {
return err
}
//}
return nil
}
// StopAndRemoveContainer
// Stop and remove a container
// Reference (https://gist.github.com/frikky/e2efcea6c733ea8d8d015b7fe8a91bf6)
func StopAndRemoveContainer(containername string) error {
//ctx := context.Background()
//
//// Gets docker information from env variables
//client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
//if err != nil {
// return err
//}
//
//if err = client.ContainerStop(ctx, containername, nil); err != nil {
// return err
//}
//
//removeOptions := types.ContainerRemoveOptions{
// RemoveVolumes: true,
// Force: true,
//}
//
//if err = client.ContainerRemove(ctx, containername, removeOptions); err != nil {
// return err
//}
// stop docker container
var stop bytes.Buffer
stop.WriteString("docker stop " + containername)
cmdStr := stop.String()
_, err := exec.Command("/bin/sh", "-c", cmdStr).Output()
if err != nil {
return err
}
// remove docker container
var remove bytes.Buffer
remove.WriteString("docker remove " + containername)
cmdStr = remove.String()
_, err = exec.Command("/bin/sh", "-c", cmdStr).Output()
if err != nil {
return err
}
return nil
}
// ViewAllContainers returns all containers runnable and which can be built
func ViewAllContainers() (*DockerContainers, error) {
// Traverse the deploy path as per given in the config file
config, err := config.ConfigInit(nil, nil)
if err != nil {
return nil, err
}
folders, err := ioutil.ReadDir(config.DockerContainers)
if err != nil {
return nil, err
}
//Declare variable DockerContainers of type struct
var Containers *DockerContainers = new(DockerContainers)
for _, f := range folders {
if f.IsDir() {
//Declare variable DockerContainer of type struct
var Container DockerContainer
// Setting container name to deploy name
Container.ContainerName = f.Name()
// Getting Description from file description.txt
Description, err := ioutil.ReadFile(config.DockerContainers + "/" + Container.ContainerName + "/description.txt")
// if we os.Open returns an error then handle it
if err != nil {
return nil, err
}
// Get Description from description.txt
Container.ContainerDescription = string(Description)
Containers.DockerContainer = append(Containers.DockerContainer, Container)
}
}
return Containers, nil
}
func print(rd io.Reader) error {
var lastLine string
scanner := bufio.NewScanner(rd)
for scanner.Scan() {
lastLine = scanner.Text()
}
errLine := &ErrorLine{}
json.Unmarshal([]byte(lastLine), errLine)
if errLine.Error != "" {
return errors.New(errLine.Error)
}
if err := scanner.Err(); err != nil {
return err
}
return nil
}
func OpenPortsFile(filename string) (*Ports, error) {
buf, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
c := &Ports{}
err = json.Unmarshal(buf, c)
if err != nil {
return nil, fmt.Errorf("in file %q: %v", filename, err)
}
return c, nil
}
// TemplateDockerContainer This function templates the docker container
// with the base docker image to use
func (d *DockerVM) TemplateDockerContainer() error {
err := d.CopyToTmpContainer()
if err != nil {
return err
}
// parses the site.yml file in the tmp directory
t, err := template.ParseFiles(d.ImagePath + "/" + d.TagName + "/Dockerfile")
if err != nil {
return err
}
// opens the output file
f, err := os.Create(d.ImagePath + "/" + d.TagName + "/Dockerfile")
if err != nil {
return err
}
image := d.BaseImage
// Pass in Docker Base Image
err = t.Execute(f, image)
if err != nil {
return err
}
return nil
}
// CopyToTmpContainer Creates a copy of the docker folder
func (d *DockerVM) CopyToTmpContainer() error {
// generate rand to UUID this is debug the ansible file if needed
id := uuid.New()
// copies the plugin to the tmp directory
err := copy.Copy(d.ImagePath+"/", d.LogsPath+id.String()+"_"+d.TagName)
if err != nil {
return err
}
// Set the plugin execution to the tmp location
d.TagName = id.String() + "_" + d.TagName
// removing slash
d.ImagePath = d.LogsPath[:len(d.LogsPath)-1]
return nil
}

42
server/docker/docker_test.go
View File

@@ -1,42 +0,0 @@
package docker
// import (
// "testing"
// )
//
// func TestDockerUbuntuSSHDProvided(t *testing.T) {
// // Testing by providing default container name
// _,err := BuildRunContainer(2,"false","docker-ubuntu-sshd")
//
// if err != nil {
// t.Error(err)
// }
//
// }
//
// func TestDockerDefaultContainer(t *testing.T) {
// // Testing by providing without providing default container name
// _,err := BuildRunContainer(2,"false","")
//
// if err != nil {
// t.Error(err)
// }
// }
//
// func TestContainerHorovod(t *testing.T) {
// // Testing by providing the horovod cpu image
// _,err := BuildRunContainer(2,"false","cpuhorovod")
//
// if err != nil {
// t.Error(err)
// }
// }
//
// func TestViewAllContainers(t *testing.T) {
// _,err := ViewAllContainers()
//
// if err != nil {
// t.Error(err)
// }
//
// }

21
server/docker/dockprom/LICENSE
View File

@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2016 Stefan Prodan
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

353
server/docker/dockprom/README.md
View File

@@ -1,353 +0,0 @@
dockprom
========
A monitoring solution for Docker hosts and containers with [Prometheus](https://prometheus.io/), [Grafana](http://grafana.org/), [cAdvisor](https://github.com/google/cadvisor),
[NodeExporter](https://github.com/prometheus/node_exporter) and alerting with [AlertManager](https://github.com/prometheus/alertmanager).
***If you're looking for the Docker Swarm version please go to [stefanprodan/swarmprom](https://github.com/stefanprodan/swarmprom)***
## Install
Clone this repository on your Docker host, cd into dockprom directory and run compose up:
```bash
git clone https://github.com/stefanprodan/dockprom
cd dockprom
ADMIN_USER=admin ADMIN_PASSWORD=admin docker-compose up -d
```
Prerequisites:
* Docker Engine >= 1.13
* Docker Compose >= 1.11
Containers:
* Prometheus (metrics database) `http://<host-ip>:9090`
* Prometheus-Pushgateway (push acceptor for ephemeral and batch jobs) `http://<host-ip>:9091`
* AlertManager (alerts management) `http://<host-ip>:9093`
* Grafana (visualize metrics) `http://<host-ip>:3000`
* NodeExporter (host metrics collector)
* cAdvisor (containers metrics collector)
* Caddy (reverse proxy and basic auth provider for prometheus and alertmanager)
## Setup Grafana
Navigate to `http://<host-ip>:3000` and login with user ***admin*** password ***admin***. You can change the credentials in the compose file or by supplying the `ADMIN_USER` and `ADMIN_PASSWORD` environment variables on compose up. The config file can be added directly in grafana part like this
```
grafana:
image: grafana/grafana:7.2.0
env_file:
- config
```
and the config file format should have this content
```
GF_SECURITY_ADMIN_USER=admin
GF_SECURITY_ADMIN_PASSWORD=changeme
GF_USERS_ALLOW_SIGN_UP=false
```
If you want to change the password, you have to remove this entry, otherwise the change will not take effect
```
- grafana_data:/var/lib/grafana
```
Grafana is preconfigured with dashboards and Prometheus as the default data source:
* Name: Prometheus
* Type: Prometheus
* Url: http://prometheus:9090
* Access: proxy
***Docker Host Dashboard***
![Host](https://raw.githubusercontent.com/stefanprodan/dockprom/master/screens/Grafana_Docker_Host.png)
The Docker Host Dashboard shows key metrics for monitoring the resource usage of your server:
* Server uptime, CPU idle percent, number of CPU cores, available memory, swap and storage
* System load average graph, running and blocked by IO processes graph, interrupts graph
* CPU usage graph by mode (guest, idle, iowait, irq, nice, softirq, steal, system, user)
* Memory usage graph by distribution (used, free, buffers, cached)
* IO usage graph (read Bps, read Bps and IO time)
* Network usage graph by device (inbound Bps, Outbound Bps)
* Swap usage and activity graphs
For storage and particularly Free Storage graph, you have to specify the fstype in grafana graph request.
You can find it in `grafana/dashboards/docker_host.json`, at line 480 :
"expr": "sum(node_filesystem_free_bytes{fstype=\"btrfs\"})",
I work on BTRFS, so i need to change `aufs` to `btrfs`.
You can find right value for your system in Prometheus `http://<host-ip>:9090` launching this request :
node_filesystem_free_bytes
***Docker Containers Dashboard***
![Containers](https://raw.githubusercontent.com/stefanprodan/dockprom/master/screens/Grafana_Docker_Containers.png)
The Docker Containers Dashboard shows key metrics for monitoring running containers:
* Total containers CPU load, memory and storage usage
* Running containers graph, system load graph, IO usage graph
* Container CPU usage graph
* Container memory usage graph
* Container cached memory usage graph
* Container network inbound usage graph
* Container network outbound usage graph
> [!NOTE]
> This dashboard doesn't show the containers that are part of the monitoring stack.
***Monitor Services Dashboard***
![Monitor Services](https://raw.githubusercontent.com/stefanprodan/dockprom/master/screens/Grafana_Prometheus.png)
The Monitor Services Dashboard shows key metrics for monitoring the containers that make up the monitoring stack:
* Prometheus container uptime, monitoring stack total memory usage, Prometheus local storage memory chunks and series
* Container CPU usage graph
* Container memory usage graph
* Prometheus chunks to persist and persistence urgency graphs
* Prometheus chunks ops and checkpoint duration graphs
* Prometheus samples ingested rate, target scrapes and scrape duration graphs
* Prometheus HTTP requests graph
* Prometheus alerts graph
## Define alerts
Three alert groups have been setup within the [alert.rules](https://github.com/stefanprodan/dockprom/blob/master/prometheus/alert.rules) configuration file:
* Monitoring services alerts [targets](https://github.com/stefanprodan/dockprom/blob/master/prometheus/alert.rules#L2-L11)
* Docker Host alerts [host](https://github.com/stefanprodan/dockprom/blob/master/prometheus/alert.rules#L13-L40)
* Docker Containers alerts [containers](https://github.com/stefanprodan/dockprom/blob/master/prometheus/alert.rules#L42-L69)
You can modify the alert rules and reload them by making a HTTP POST call to Prometheus:
```
curl -X POST http://admin:admin@<host-ip>:9090/-/reload
```
***Monitoring services alerts***
Trigger an alert if any of the monitoring targets (node-exporter and cAdvisor) are down for more than 30 seconds:
```yaml
- alert: monitor_service_down
expr: up == 0
for: 30s
labels:
severity: critical
annotations:
summary: "Monitor service non-operational"
description: "Service {{ $labels.instance }} is down."
```
***Docker Host alerts***
Trigger an alert if the Docker host CPU is under high load for more than 30 seconds:
```yaml
- alert: high_cpu_load
expr: node_load1 > 1.5
for: 30s
labels:
severity: warning
annotations:
summary: "Server under high load"
description: "Docker host is under high load, the avg load 1m is at {{ $value}}. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
```
Modify the load threshold based on your CPU cores.
Trigger an alert if the Docker host memory is almost full:
```yaml
- alert: high_memory_load
expr: (sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100 > 85
for: 30s
labels:
severity: warning
annotations:
summary: "Server memory is almost full"
description: "Docker host memory usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
```
Trigger an alert if the Docker host storage is almost full:
```yaml
- alert: high_storage_load
expr: (node_filesystem_size_bytes{fstype="aufs"} - node_filesystem_free_bytes{fstype="aufs"}) / node_filesystem_size_bytes{fstype="aufs"} * 100 > 85
for: 30s
labels:
severity: warning
annotations:
summary: "Server storage is almost full"
description: "Docker host storage usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
```
***Docker Containers alerts***
Trigger an alert if a container is down for more than 30 seconds:
```yaml
- alert: jenkins_down
expr: absent(container_memory_usage_bytes{name="jenkins"})
for: 30s
labels:
severity: critical
annotations:
summary: "Jenkins down"
description: "Jenkins container is down for more than 30 seconds."
```
Trigger an alert if a container is using more than 10% of total CPU cores for more than 30 seconds:
```yaml
- alert: jenkins_high_cpu
expr: sum(rate(container_cpu_usage_seconds_total{name="jenkins"}[1m])) / count(node_cpu_seconds_total{mode="system"}) * 100 > 10
for: 30s
labels:
severity: warning
annotations:
summary: "Jenkins high CPU usage"
description: "Jenkins CPU usage is {{ humanize $value}}%."
```
Trigger an alert if a container is using more than 1.2GB of RAM for more than 30 seconds:
```yaml
- alert: jenkins_high_memory
expr: sum(container_memory_usage_bytes{name="jenkins"}) > 1200000000
for: 30s
labels:
severity: warning
annotations:
summary: "Jenkins high memory usage"
description: "Jenkins memory consumption is at {{ humanize $value}}."
```
## Setup alerting
The AlertManager service is responsible for handling alerts sent by Prometheus server.
AlertManager can send notifications via email, Pushover, Slack, HipChat or any other system that exposes a webhook interface.
A complete list of integrations can be found [here](https://prometheus.io/docs/alerting/configuration).
You can view and silence notifications by accessing `http://<host-ip>:9093`.
The notification receivers can be configured in [alertmanager/config.yml](https://github.com/stefanprodan/dockprom/blob/master/alertmanager/config.yml) file.
To receive alerts via Slack you need to make a custom integration by choose ***incoming web hooks*** in your Slack team app page.
You can find more details on setting up Slack integration [here](http://www.robustperception.io/using-slack-with-the-alertmanager/).
Copy the Slack Webhook URL into the ***api_url*** field and specify a Slack ***channel***.
```yaml
route:
receiver: 'slack'
receivers:
- name: 'slack'
slack_configs:
- send_resolved: true
text: "{{ .CommonAnnotations.description }}"
username: 'Prometheus'
channel: '#<channel>'
api_url: 'https://hooks.slack.com/services/<webhook-id>'
```
![Slack Notifications](https://raw.githubusercontent.com/stefanprodan/dockprom/master/screens/Slack_Notifications.png)
## Sending metrics to the Pushgateway
The [pushgateway](https://github.com/prometheus/pushgateway) is used to collect data from batch jobs or from services.
To push data, simply execute:
echo "some_metric 3.14" | curl --data-binary @- http://user:password@localhost:9091/metrics/job/some_job
Please replace the `user:password` part with your user and password set in the initial configuration (default: `admin:admin`).
## Updating Grafana to v5.2.2
[In Grafana versions >= 5.1 the id of the grafana user has been changed](http://docs.grafana.org/installation/docker/#migration-from-a-previous-version-of-the-docker-container-to-5-1-or-later). Unfortunately this means that files created prior to 5.1 won’t have the correct permissions for later versions.
| Version | User | User ID |
|:-------:|:-------:|:-------:|
| < 5.1 | grafana | 104 |
| \>= 5.1 | grafana | 472 |
There are two possible solutions to this problem.
- Change ownership from 104 to 472
- Start the upgraded container as user 104
##### Specifying a user in docker-compose.yml
To change ownership of the files run your grafana container as root and modify the permissions.
First perform a `docker-compose down` then modify your docker-compose.yml to include the `user: root` option:
```
grafana:
image: grafana/grafana:5.2.2
container_name: grafana
volumes:
- grafana_data:/var/lib/grafana
- ./grafana/datasources:/etc/grafana/datasources
- ./grafana/dashboards:/etc/grafana/dashboards
- ./grafana/setup.sh:/setup.sh
entrypoint: /setup.sh
user: root
environment:
- GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
- GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
- GF_USERS_ALLOW_SIGN_UP=false
restart: unless-stopped
expose:
- 3000
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
```
Perform a `docker-compose up -d` and then issue the following commands:
```
docker exec -it --user root grafana bash
# in the container you just started:
chown -R root:root /etc/grafana && \
chmod -R a+r /etc/grafana && \
chown -R grafana:grafana /var/lib/grafana && \
chown -R grafana:grafana /usr/share/grafana
```
To run the grafana container as `user: 104` change your `docker-compose.yml` like such:
```
grafana:
image: grafana/grafana:5.2.2
container_name: grafana
volumes:
- grafana_data:/var/lib/grafana
- ./grafana/datasources:/etc/grafana/datasources
- ./grafana/dashboards:/etc/grafana/dashboards
- ./grafana/setup.sh:/setup.sh
entrypoint: /setup.sh
user: "104"
environment:
- GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
- GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
- GF_USERS_ALLOW_SIGN_UP=false
restart: unless-stopped
expose:
- 3000
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
```

11
server/docker/dockprom/alertmanager/config.yml
View File

@@ -1,11 +0,0 @@
route:
receiver: 'slack'
receivers:
- name: 'slack'
slack_configs:
- send_resolved: true
text: "{{ .CommonAnnotations.description }}"
username: 'Prometheus'
channel: '#<channel-name>'
api_url: 'https://hooks.slack.com/services/<webhook-id>'

36
server/docker/dockprom/caddy/Caddyfile
View File

@@ -1,36 +0,0 @@
:9090 {
proxy / prometheus:9090 {
transparent
}
errors stderr
tls off
}
:9093 {
proxy / alertmanager:9093 {
transparent
}
errors stderr
tls off
}
:9091 {
proxy / pushgateway:9091 {
transparent
}
errors stderr
tls off
}
:3000 {
proxy / grafana:3000 {
transparent
websocket
}
errors stderr
tls off
}

3
server/docker/dockprom/config
View File

@@ -1,3 +0,0 @@
GF_SECURITY_ADMIN_USER=admin
GF_SECURITY_ADMIN_PASSWORD=changeme
GF_USERS_ALLOW_SIGN_UP=false

36
server/docker/dockprom/docker-compose.exporters.yml
View File

@@ -1,36 +0,0 @@
version: '2.1'
services:
nodeexporter:
image: prom/node-exporter:v1.0.1
container_name: nodeexporter
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.rootfs=/rootfs'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
restart: unless-stopped
network_mode: host
labels:
org.label-schema.group: "monitoring"
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.38.7
container_name: cadvisor
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker/:/var/lib/docker:ro
- /cgroup:/cgroup:ro
restart: unless-stopped
network_mode: host
labels:
org.label-schema.group: "monitoring"

146
server/docker/dockprom/docker-compose.yml
View File

@@ -1,146 +0,0 @@
version: '2.1'
networks:
monitor-net:
driver: bridge
volumes:
prometheus_data: {}
grafana_data: {}
services:
prometheus:
image: prom/prometheus:v2.24.1
container_name: prometheus
volumes:
- ./prometheus:/etc/prometheus
- prometheus_data:/prometheus
command:
- '--config.file=/etc/prometheus/prometheus.yml'
- '--storage.tsdb.path=/prometheus'
- '--web.console.libraries=/etc/prometheus/console_libraries'
- '--web.console.templates=/etc/prometheus/consoles'
- '--storage.tsdb.retention.time=200h'
- '--web.enable-lifecycle'
restart: unless-stopped
expose:
- 9090
ports:
- "9090:9090"
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
alertmanager:
image: prom/alertmanager:v0.21.0
container_name: alertmanager
volumes:
- ./alertmanager:/etc/alertmanager
command:
- '--config.file=/etc/alertmanager/config.yml'
- '--storage.path=/alertmanager'
restart: unless-stopped
expose:
- 9093
ports:
- "9093:9093"
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
nodeexporter:
image: prom/node-exporter:v1.0.1
container_name: nodeexporter
volumes:
- /proc:/host/proc:ro
- /sys:/host/sys:ro
- /:/rootfs:ro
command:
- '--path.procfs=/host/proc'
- '--path.rootfs=/rootfs'
- '--path.sysfs=/host/sys'
- '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
restart: unless-stopped
expose:
- 9100
ports:
- "9100:9100"
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
cadvisor:
image: gcr.io/cadvisor/cadvisor:v0.38.7
container_name: cadvisor
volumes:
- /:/rootfs:ro
- /var/run:/var/run:rw
- /sys:/sys:ro
- /var/lib/docker:/var/lib/docker:ro
#- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
restart: unless-stopped
expose:
- 8080
ports:
- "8080:8080"
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
grafana:
image: grafana/grafana:7.3.7
container_name: grafana
volumes:
- grafana_data:/var/lib/grafana
- ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
- ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
environment:
- GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
- GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
- GF_USERS_ALLOW_SIGN_UP=false
restart: unless-stopped
expose:
- 3000
ports:
- "3000:3000"
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
pushgateway:
image: prom/pushgateway:v1.4.0
container_name: pushgateway
restart: unless-stopped
expose:
- 9091
ports:
- "9091:9091"
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"
caddy:
image: caddy:2.3.0-alpine
container_name: caddy
ports:
- "3000:3000"
- "9090:9090"
- "9093:9093"
- "9091:9091"
volumes:
- ./caddy:/etc/caddy
environment:
- ADMIN_USER=${ADMIN_USER:-admin}
- ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
restart: unless-stopped
networks:
- monitor-net
labels:
org.label-schema.group: "monitoring"

12
server/docker/dockprom/grafana/provisioning/dashboards/dashboard.yml
View File

@@ -1,12 +0,0 @@
apiVersion: 1
providers:
- name: 'Prometheus'
orgId: 1
folder: ''
type: file
disableDeletion: false
editable: true
allowUiUpdates: true
options:
path: /etc/grafana/provisioning/dashboards

1270
server/docker/dockprom/grafana/provisioning/dashboards/docker_containers.json
View File

File diff suppressed because it is too large Load Diff

1441
server/docker/dockprom/grafana/provisioning/dashboards/docker_host.json
View File

File diff suppressed because it is too large Load Diff

3412
server/docker/dockprom/grafana/provisioning/dashboards/monitor_services.json
View File

File diff suppressed because it is too large Load Diff

398
server/docker/dockprom/grafana/provisioning/dashboards/nginx_container.json
View File

@@ -1,398 +0,0 @@
{
"id": null,
"title": "Nginx",
"description": "Nginx exporter metrics",
"tags": [
"nginx"
],
"style": "dark",
"timezone": "browser",
"editable": true,
"hideControls": false,
"sharedCrosshair": true,
"rows": [
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"aliasColors": {},
"bars": false,
"datasource": "Prometheus",
"decimals": 2,
"editable": true,
"error": false,
"fill": 1,
"grid": {
"threshold1": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2": null,
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"id": 3,
"isNew": true,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(irate(nginx_connections_processed_total{stage=\"any\"}[5m])) by (stage)",
"hide": false,
"interval": "",
"intervalFactor": 10,
"legendFormat": "requests",
"metric": "",
"refId": "B",
"step": 10
}
],
"timeFrom": null,
"timeShift": null,
"title": "Requests/sec",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"show": true
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"datasource": "Prometheus",
"decimals": 2,
"editable": true,
"error": false,
"fill": 1,
"grid": {
"threshold1": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2": null,
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"id": 2,
"isNew": true,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(nginx_connections_current) by (state)",
"interval": "",
"intervalFactor": 2,
"legendFormat": "{{state}}",
"metric": "",
"refId": "A",
"step": 2
}
],
"timeFrom": null,
"timeShift": null,
"title": "Connections",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"show": true
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": false,
"datasource": "Prometheus",
"decimals": 2,
"editable": true,
"error": false,
"fill": 1,
"grid": {
"threshold1": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2": null,
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"id": 1,
"isNew": true,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(irate(nginx_connections_processed_total{stage!=\"any\"}[5m])) by (stage)",
"hide": false,
"interval": "",
"intervalFactor": 10,
"legendFormat": "{{stage}}",
"metric": "",
"refId": "B",
"step": 10
}
],
"timeFrom": null,
"timeShift": null,
"title": "Connections rate",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"show": true
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": 0,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"title": "Nginx exporter metrics"
},
{
"collapse": false,
"editable": true,
"height": "250px",
"panels": [
{
"aliasColors": {},
"bars": false,
"datasource": null,
"editable": true,
"error": false,
"fill": 1,
"grid": {
"threshold1": null,
"threshold1Color": "rgba(216, 200, 27, 0.27)",
"threshold2": null,
"threshold2Color": "rgba(234, 112, 112, 0.22)"
},
"id": 4,
"isNew": true,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"span": 12,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"nginx\"}[5m])) / count(node_cpu_seconds_total{mode=\"system\"}) * 100",
"intervalFactor": 2,
"legendFormat": "nginx",
"refId": "A",
"step": 2
}
],
"timeFrom": null,
"timeShift": null,
"title": "CPU usage",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"show": true
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
}
],
"title": "Nginx container metrics"
}
],
"time": {
"from": "now-15m",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"templating": {
"list": []
},
"annotations": {
"list": []
},
"refresh": "10s",
"schemaVersion": 12,
"version": 9,
"links": [],
"gnetId": null
}

11
server/docker/dockprom/grafana/provisioning/datasources/datasource.yml
View File

@@ -1,11 +0,0 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
access: proxy
orgId: 1
url: http://prometheus:9090
basicAuth: false
isDefault: true
editable: true

22
server/docker/dockprom/helpers/aws/README.md
View File

@@ -1,22 +0,0 @@
# Prometheus on EC2 & ECS:
Some helpers for anyone configuring Prometheus on ECS and AWS EC2.
To get started on AWS ECS and EC2:
*For EC2/ECS nodes*:
- Import the ecs task definition and add cadvisor and node-exporter service/task definition and run them on each host you want to be monitored
- Any hosts which have "Monitoring: On" tag will be automatically added in the targets
- Expose ports 9100 and 9191 to your Prometheus host
*For Prometheus host*:
- Copy prometheus.yml configuration present here to base prometheus configuration to enable EC2 service discovery
- `docker compose up -d`
> [!NOTE]
> Set query.staleness-delta to 1m make metrics more realtime
### TODO
- [ ] Add alerting rules based on ECS

78
server/docker/dockprom/helpers/aws/cadvisor_ecs_task_definition.json
View File

@@ -1,78 +0,0 @@
{
"family": "cadvisor",
"containerDefinitions": [
{
"name": "cadvisor",
"image": "google/cadvisor",
"cpu": 10,
"memory": 300,
"portMappings": [
{
"containerPort": 9191,
"hostPort": 9191
}
],
"essential": true,
"privileged": true,
"mountPoints": [
{
"sourceVolume": "root",
"containerPath": "/rootfs",
"readOnly": true
},
{
"sourceVolume": "var_run",
"containerPath": "/var/run",
"readOnly": false
},
{
"sourceVolume": "sys",
"containerPath": "/sys",
"readOnly": true
},
{
"sourceVolume": "var_lib_docker",
"containerPath": "/var/lib/docker",
"readOnly": true
},
{
"sourceVolume": "cgroup",
"containerPath": "/cgroup",
"readOnly": true
}
]
}
],
"volumes": [
{
"name": "root",
"host": {
"sourcePath": "/"
}
},
{
"name": "var_run",
"host": {
"sourcePath": "/var/run"
}
},
{
"name": "sys",
"host": {
"sourcePath": "/sys"
}
},
{
"name": "var_lib_docker",
"host": {
"sourcePath": "/var/lib/docker/"
}
},
{
"name": "cgroup",
"host": {
"sourcePath": "/cgroup"
}
}
]
}

22
server/docker/dockprom/helpers/aws/node_exporter_task_definition.json
View File

@@ -1,22 +0,0 @@
{
"family": "prometheus",
"containerDefinitions": [
{
"portMappings": [
{
"hostPort": 9100,
"containerPort": 9100,
"protocol": "tcp"
}
],
"essential": true,
"name": "node_exporter",
"image": "prom/node-exporter",
"cpu": 0,
"privileged": null,
"memoryReservation": 150
}
],
"volumes": [],
"networkMode": "host"
}

53
server/docker/dockprom/helpers/aws/prometheus.yml
View File

@@ -1,53 +0,0 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
# Attach these labels to any time series or alerts when communicating with
# external systems (federation, remote storage, Alertmanager).
external_labels:
monitor: 'docker-host-alpha'
# Load and evaluate rules in this file every 'evaluation_interval' seconds.
rule_files:
- "targets.rules"
- "hosts.rules"
- "containers.rules"
# A scrape configuration containing exactly one endpoint to scrape.
scrape_configs:
- job_name: 'nodeexporter'
scrape_interval: 5s
static_configs:
- targets: ['nodeexporter:9100']
- job_name: 'cadvisor'
scrape_interval: 5s
static_configs:
- targets: ['cadvisor:8080']
- job_name: 'prometheus'
scrape_interval: 10s
static_configs:
- targets: ['localhost:9090']
# sample scrape configuration for AWS EC2
- job_name: 'nodeexporter'
ec2_sd_configs:
- region: us-east-1
port: 9100
relabel_configs:
# Only monitor instances which have a tag called Monitoring "Monitoring"
- source_labels: [__meta_ec2_tag_Monitoring]
regex: On
action: keep
- job_name: 'cadvisor'
ec2_sd_configs:
- region: us-east-1
port: 9010
relabel_configs:
# Only monitor instances which have a tag called Monitoring "Monitoring"
- source_labels: [__meta_ec2_tag_Monitoring]
regex: On
action: keep

70
server/docker/dockprom/prometheus/alert.rules
View File

@@ -1,70 +0,0 @@
groups:
- name: targets
rules:
- alert: monitor_service_down
expr: up == 0
for: 30s
labels:
severity: critical
annotations:
summary: "Monitor service non-operational"
description: "Service {{ $labels.instance }} is down."
- name: host
rules:
- alert: high_cpu_load
expr: node_load1 > 1.5
for: 30s
labels:
severity: warning
annotations:
summary: "Server under high load"
description: "Docker host is under high load, the avg load 1m is at {{ $value}}. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
- alert: high_memory_load
expr: (sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100 > 85
for: 30s
labels:
severity: warning
annotations:
summary: "Server memory is almost full"
description: "Docker host memory usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
- alert: high_storage_load
expr: (node_filesystem_size_bytes{fstype="aufs"} - node_filesystem_free_bytes{fstype="aufs"}) / node_filesystem_size_bytes{fstype="aufs"} * 100 > 85
for: 30s
labels:
severity: warning
annotations:
summary: "Server storage is almost full"
description: "Docker host storage usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
- name: containers
rules:
- alert: jenkins_down
expr: absent(container_memory_usage_bytes{name="jenkins"})
for: 30s
labels:
severity: critical
annotations:
summary: "Jenkins down"
description: "Jenkins container is down for more than 30 seconds."
- alert: jenkins_high_cpu
expr: sum(rate(container_cpu_usage_seconds_total{name="jenkins"}[1m])) / count(node_cpu_seconds_total{mode="system"}) * 100 > 10
for: 30s
labels:
severity: warning
annotations:
summary: "Jenkins high CPU usage"
description: "Jenkins CPU usage is {{ humanize $value}}%."
- alert: jenkins_high_memory
expr: sum(container_memory_usage_bytes{name="jenkins"}) > 1200000000
for: 30s
labels:
severity: warning
annotations:
summary: "Jenkins high memory usage"
description: "Jenkins memory consumption is at {{ humanize $value}}."

53
server/docker/dockprom/prometheus/prometheus.yml
View File

@@ -1,53 +0,0 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
# Attach these labels to any time series or alerts when communicating with
# external systems (federation, remote storage, Alertmanager).
external_labels:
monitor: 'docker-host-alpha'
# Load and evaluate rules in this file every 'evaluation_interval' seconds.
rule_files:
- "alert.rules"
# A scrape configuration containing exactly one endpoint to scrape.
scrape_configs:
- job_name: 'nodeexporter'
scrape_interval: 5s
static_configs:
- targets: ['nodeexporter:9100']
- job_name: 'cadvisor'
scrape_interval: 5s
static_configs:
- targets: ['cadvisor:8080']
- job_name: 'prometheus'
scrape_interval: 10s
static_configs:
- targets: ['localhost:9090']
- job_name: 'pushgateway'
scrape_interval: 10s
honor_labels: true
static_configs:
- targets: ['pushgateway:9091']
alerting:
alertmanagers:
- scheme: http
static_configs:
- targets:
- 'alertmanager:9093'
# - job_name: 'nginx'
# scrape_interval: 10s
# static_configs:
# - targets: ['nginxexporter:9113']
# - job_name: 'aspnetcore'
# scrape_interval: 10s
# static_configs:
# - targets: ['eventlog-proxy:5000', 'eventlog:5000']

BIN
server/docker/dockprom/screens/Grafana_Docker_Containers.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 270 KiB

BIN
server/docker/dockprom/screens/Grafana_Docker_Host.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 247 KiB

BIN
server/docker/dockprom/screens/Grafana_Prometheus.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 501 KiB

BIN
server/docker/dockprom/screens/Slack_Notifications.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 39 KiB

174
server/docker/docs.md
View File

@@ -1,174 +0,0 @@
<!-- Code generated by gomarkdoc. DO NOT EDIT -->
# docker
```go
import "github.com/Akilan1999/p2p-rendering-computation/server/docker"
```
## Index
- [func StopAndRemoveContainer\(containername string\) error](<#StopAndRemoveContainer>)
- [type DockerContainer](<#DockerContainer>)
- [type DockerContainers](<#DockerContainers>)
- [func ViewAllContainers\(\) \(\*DockerContainers, error\)](<#ViewAllContainers>)
- [type DockerVM](<#DockerVM>)
- [func BuildRunContainer\(NumPorts int, GPU string, ContainerName string, baseImage string, publicKey string\) \(\*DockerVM, error\)](<#BuildRunContainer>)
- [func \(d \*DockerVM\) CopyToTmpContainer\(\) error](<#DockerVM.CopyToTmpContainer>)
- [func \(d \*DockerVM\) TemplateDockerContainer\(\) error](<#DockerVM.TemplateDockerContainer>)
- [type ErrorDetail](<#ErrorDetail>)
- [type ErrorLine](<#ErrorLine>)
- [type Port](<#Port>)
- [type Ports](<#Ports>)
- [func OpenPortsFile\(filename string\) \(\*Ports, error\)](<#OpenPortsFile>)
<a name="StopAndRemoveContainer"></a>
## func [StopAndRemoveContainer](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L340>)
```go
func StopAndRemoveContainer(containername string) error
```
StopAndRemoveContainer Stop and remove a container Reference \(https://gist.github.com/frikky/e2efcea6c733ea8d8d015b7fe8a91bf6\)
<a name="DockerContainer"></a>
## type [DockerContainer](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L40-L43>)
```go
type DockerContainer struct {
ContainerName string `json:"DockerContainerName"`
ContainerDescription string `json:"ContainerDescription"`
}
```
<a name="DockerContainers"></a>
## type [DockerContainers](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L36-L38>)
```go
type DockerContainers struct {
DockerContainer []DockerContainer `json:"DockerContainer"`
}
```
<a name="ViewAllContainers"></a>
### func [ViewAllContainers](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L387>)
```go
func ViewAllContainers() (*DockerContainers, error)
```
ViewAllContainers returns all containers runnable and which can be built
<a name="DockerVM"></a>
## type [DockerVM](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L22-L34>)
```go
type DockerVM struct {
SSHUsername string `json:"SSHUsername"`
SSHPublcKey string `json:"SSHPublicKey"`
ID string `json:"ID"`
TagName string `json:"TagName"`
ImagePath string `json:"ImagePath"`
Ports Ports `json:"Ports"`
GPU string `json:"GPU"`
TempPath string
BaseImage string
LogsPath string
SSHCommand string `json:"SSHCommand"`
}
```
<a name="BuildRunContainer"></a>
### func [BuildRunContainer](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L70>)
```go
func BuildRunContainer(NumPorts int, GPU string, ContainerName string, baseImage string, publicKey string) (*DockerVM, error)
```
BuildRunContainer Function is incharge to invoke building and running contianer and also allocating external ports
<a name="DockerVM.CopyToTmpContainer"></a>
### func \(\*DockerVM\) [CopyToTmpContainer](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L493>)
```go
func (d *DockerVM) CopyToTmpContainer() error
```
CopyToTmpContainer Creates a copy of the docker folder
<a name="DockerVM.TemplateDockerContainer"></a>
### func \(\*DockerVM\) [TemplateDockerContainer](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L464>)
```go
func (d *DockerVM) TemplateDockerContainer() error
```
TemplateDockerContainer This function templates the docker container with the base docker image to use
<a name="ErrorDetail"></a>
## type [ErrorDetail](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L62-L64>)
```go
type ErrorDetail struct {
Message string `json:"message"`
}
```
<a name="ErrorLine"></a>
## type [ErrorLine](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L57-L60>)
```go
type ErrorLine struct {
Error string `json:"error"`
ErrorDetail ErrorDetail `json:"errorDetail"`
}
```
<a name="Port"></a>
## type [Port](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L48-L55>)
```go
type Port struct {
PortName string `json:"PortName"`
InternalPort int `json:"InternalPort"`
Type string `json:"Type"`
ExternalPort int `json:"ExternalPort"`
IsUsed bool `json:"IsUsed"`
Description string `json:"Description"`
}
```
<a name="Ports"></a>
## type [Ports](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L45-L47>)
```go
type Ports struct {
PortSet []Port `json:"Port"`
}
```
<a name="OpenPortsFile"></a>
### func [OpenPortsFile](<https://github.com/Akilan1999/p2p-rendering-computation/blob/master/server/docker/docker.go#L447>)
```go
func OpenPortsFile(filename string) (*Ports, error)
```
Generated by [gomarkdoc](<https://github.com/princjef/gomarkdoc>)

4
server/docker/kill-containers.sh
View File

@@ -1,4 +0,0 @@
docker kill $(docker ps -q)\
docker rm $(docker ps -a -q)
docker rmi $(docker images -q) --force
docker system prune --all

116
server/server.go
View File

@@ -1,7 +1,6 @@
package server
import (
b64 "encoding/base64"
"encoding/json"
"errors"
"fmt"
@@ -9,7 +8,6 @@ import (
"github.com/Akilan1999/p2p-rendering-computation/config"
"github.com/Akilan1999/p2p-rendering-computation/p2p"
"github.com/Akilan1999/p2p-rendering-computation/p2p/frp"
"github.com/Akilan1999/p2p-rendering-computation/server/docker"
"github.com/gin-gonic/gin"
"io/ioutil"
"net/http"
@@ -120,65 +118,65 @@ func Server() (*gin.Engine, error) {
})
// Starts docker container in server
r.GET("/startcontainer", func(c *gin.Context) {
// Get Number of ports to open and whether to use GPU or not
Ports := c.DefaultQuery("ports", "0")
GPU := c.DefaultQuery("GPU", "false")
ContainerName := c.DefaultQuery("ContainerName", "")
BaseImage := c.DefaultQuery("BaseImage", "")
PublicKey := c.DefaultQuery("PublicKey", "")
var PortsInt int
if PublicKey == "" {
c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", "Publickey not passed"))
}
PublicKeyDecoded, err := b64.StdEncoding.DecodeString(PublicKey)
if err != nil {
c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
}
// Convert Get Request value to int
fmt.Sscanf(Ports, "%d", &PortsInt)
fmt.Println(string(PublicKeyDecoded[:]))
// Creates container and returns-back result to
// access container
resp, err := docker.BuildRunContainer(PortsInt, GPU, ContainerName, BaseImage, string(PublicKeyDecoded[:]))
if err != nil {
c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
}
// Ensures that FRP is triggered only if a proxy address is provided
if ProxyIpAddr.Ipv4 != "" && c.Request.Host != "localhost:"+config.ServerPort && c.Request.Host != "0.0.0.0:"+config.ServerPort {
resp, err = frp.StartFRPCDockerContainer(ProxyIpAddr.Ipv4, lowestLatencyIpAddress.ServerPort, resp)
if err != nil {
c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
}
}
c.JSON(http.StatusOK, resp)
})
//r.GET("/startcontainer", func(c *gin.Context) {
// // Get Number of ports to open and whether to use GPU or not
// Ports := c.DefaultQuery("ports", "0")
// GPU := c.DefaultQuery("GPU", "false")
// ContainerName := c.DefaultQuery("ContainerName", "")
// BaseImage := c.DefaultQuery("BaseImage", "")
// PublicKey := c.DefaultQuery("PublicKey", "")
// var PortsInt int
//
// if PublicKey == "" {
// c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", "Publickey not passed"))
// }
//
// PublicKeyDecoded, err := b64.StdEncoding.DecodeString(PublicKey)
// if err != nil {
// c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
// }
//
// // Convert Get Request value to int
// fmt.Sscanf(Ports, "%d", &PortsInt)
//
// fmt.Println(string(PublicKeyDecoded[:]))
//
// // Creates container and returns-back result to
// // access container
// resp, err := docker.BuildRunContainer(PortsInt, GPU, ContainerName, BaseImage, string(PublicKeyDecoded[:]))
//
// if err != nil {
// c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
// }
//
// // Ensures that FRP is triggered only if a proxy address is provided
// if ProxyIpAddr.Ipv4 != "" && c.Request.Host != "localhost:"+config.ServerPort && c.Request.Host != "0.0.0.0:"+config.ServerPort {
// resp, err = frp.StartFRPCDockerContainer(ProxyIpAddr.Ipv4, lowestLatencyIpAddress.ServerPort, resp)
// if err != nil {
// c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
// }
// }
//
// c.JSON(http.StatusOK, resp)
//})
//Remove container
r.GET("/RemoveContainer", func(c *gin.Context) {
ID := c.DefaultQuery("id", "0")
if err := docker.StopAndRemoveContainer(ID); err != nil {
c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
}
c.String(http.StatusOK, "success")
})
//Show images available
r.GET("/ShowImages", func(c *gin.Context) {
resp, err := docker.ViewAllContainers()
if err != nil {
c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
}
c.JSON(http.StatusOK, resp)
})
//r.GET("/RemoveContainer", func(c *gin.Context) {
// ID := c.DefaultQuery("id", "0")
// if err := docker.StopAndRemoveContainer(ID); err != nil {
// c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
// }
// c.String(http.StatusOK, "success")
//})
//
////Show images available
//r.GET("/ShowImages", func(c *gin.Context) {
// resp, err := docker.ViewAllContainers()
// if err != nil {
// c.String(http.StatusInternalServerError, fmt.Sprintf("error: %s", err))
// }
// c.JSON(http.StatusOK, resp)
//})
// Request for port no from Server with address
r.GET("/FRPPort", func(c *gin.Context) {