add files

server/docker/Makefile

@@ -0,0 +1,12 @@
+SHELL := /bin/bash
+
+.PHONY: set_virtualenv,install_docker_requirements,dockerproc
+
+set_virtualenv:
+	virtualenv env
+
+install_docker_requirements:
+	source env/bin/activate && pip install -r requirements.txt
+    
+dockerproc:
+	ADMIN_USER=admin ADMIN_PASSWORD=admin docker-compose -f dockprom/docker-compose.yml up -d

server/docker/README

@@ -0,0 +1,6 @@
+Docker Module P2P-rendering-computation 
+========================================
+
+This module is incharge to spin up docker contaianers , create 
+SSH server and VNC server. This module is implemented using 
+python. 

server/docker/containers/README

@@ -0,0 +1,7 @@
+Containers
+===========
+
+Structure
+   |_ Containers 
+          |_ <name> (To ensure the name is the same as the tag name)
+                |_ DockerFile 

server/docker/containers/docker-ubuntu-sshd/Dockerfile

@@ -0,0 +1,58 @@
+# -----------------------------------------------------------------------------
+# This is base image of Ubuntu LTS with SSHD service.
+#
+# Authors: Art567
+# Updated: Sep 20th, 2015
+# Require: Docker (http://www.docker.io/)
+# -----------------------------------------------------------------------------
+
+
+# Base system is the latest LTS version of Ubuntu.
+from   consol/ubuntu-xfce-vnc
+
+# Switch to root user to install additional software
+USER 0
+
+
+# Make sure we don't get notifications we can't answer during building.
+env    DEBIAN_FRONTEND noninteractive
+
+
+# Prepare scripts and configs
+add    ./scripts/start /start
+
+
+# Download and install everything from the repos.
+run    apt-get -q -y update; apt-get -q -y upgrade && \
+       apt-get -q -y install sudo openssh-server && \
+       mkdir /var/run/sshd
+
+
+# Set root password
+run    echo 'root:password' >> /root/passwdfile
+
+
+# Create user and it's password
+run    useradd -m -G sudo master && \
+       echo 'master:password' >> /root/passwdfile
+
+
+# Apply root password
+run    chpasswd -c SHA512 < /root/passwdfile && \
+       rm /root/passwdfile
+
+
+# Port 22 is used for ssh
+expose 22
+
+
+# Assign /data as static volume.
+volume ["/data"]
+
+
+# Fix all permissions
+run    chmod +x /start
+
+
+# Starting sshd
+cmd    ["/start"]

server/docker/containers/docker-ubuntu-sshd/README.md

@@ -0,0 +1,72 @@
+Ubuntu LTS with SSH (Docker)
+=========
+
+A Docker file to build a [docker.io] base container with Ubuntu LTS and a sshd service
+[docker.io]: http://docker.io
+Nice and easy way to get any server up and running using docker
+
+
+Instructions
+-----------
+ - Install Docker first. 
+   Read [installation instructions] (http://docs.docker.io/en/latest/installation/).
+ 
+ 
+ - Clone this repository:
+ 
+   `$ git clone https://github.com/art567/docker-ubuntu-sshd.git`
+ 
+ 
+ - Enter local directory:
+ 
+   `$ cd docker-ubuntu-sshd`
+ 
+ - Change passwords in Dockerfile:
+ 
+   `$ vi Dockerfile`
+ 
+ - Build the container:
+ 
+   `$ sudo docker build -t art567/ubuntu .`
+ 
+ 
+ - Run the container:
+ 
+   `$ sudo docker run -d=true --name=ubuntu --restart=always -p=2222:22 -v=/opt/data:/data art567/ubuntu /start`
+ 
+ 
+ - Your container will start and bind ssh to 2222 port.
+ 
+ 
+ - Find your local IP Address:
+ 
+   `$ ifconfig`
+ 
+ 
+ - Connect to the SSH server:
+ 
+   `$ ssh root@<ip-address> -p 2222`
+ 
+ 
+ - If you don't want to deal with root:
+ 
+   `$ ssh master@<ip-address> -p 2222`
+ 
+
+**VERY IMPORTANT!!!**
+-----------
+
+ **Don't forget to change passwords in Dockerfile before building image!**
+
+
+Versions
+-----------
+Some branches represents Ubuntu versions.
+
+Branch `master` is always represented by latest Ubuntu LTS
+
+   For example:
+   - 12.04 - Ubuntu 12.04 LTS
+   - 14.04 - Ubuntu 14.04 LTS
+   - 16.04 - Ubuntu 16.04 LTS
+

server/docker/containers/docker-ubuntu-sshd/scripts/start

@@ -0,0 +1,11 @@
+#!/bin/bash
+# -----------------------------------------------------------------------------
+# docker-ubuntu-sshd /start script
+#
+# Authors: Art567
+# Updated: Sep 20th, 2015
+# -----------------------------------------------------------------------------
+
+
+# Run OpenSSH server in daemon mode
+/usr/sbin/sshd -D

server/docker/docker.go

@@ -0,0 +1,287 @@
+package docker
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"git.sr.ht/~akilan1999/p2p-rendering-computation/config"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/go-connections/nat"
+	"github.com/lithammer/shortuuid"
+	"github.com/phayes/freeport"
+	"io"
+	"os/exec"
+	"time"
+)
+
+type DockerVM struct {
+	 SSHPort int `json:"SSHPort"`
+	 SSHUsername string `json:"SSHUsername"`
+	 SSHPassword string `json:"SSHPassword"`
+	 VNCPort int `json:"VNCPort"`
+	 VNCPassword string `json:"VNCPassword"`
+	 ID string `json:"ID"`
+	 TagName string `json:"TagName"`
+	 ImagePath string `json:"ImagePath"`
+	 Ports []int `json:"OpenPorts"`
+	 GPU string `json:"GPU"`
+}
+
+type ErrorLine struct {
+	Error       string      `json:"error"`
+	ErrorDetail ErrorDetail `json:"errorDetail"`
+}
+
+type ErrorDetail struct {
+	Message string `json:"message"`
+}
+
+var dockerRegistryUserID = ""
+
+func BuildRunContainer(NumPorts int, GPU string) (*DockerVM,error) {
+	//Docker Struct Variable
+	var RespDocker *DockerVM = new(DockerVM)
+
+	// Count: 2 allocated ports for VNC and SSH + other ports
+	// to open
+	count := NumPorts + 2
+
+	// Gets 2 TCP ports empty
+	Ports, err := freeport.GetFreePorts(count)
+	if err != nil {
+		return nil,err
+	}
+
+	for i := 2; i < count; i++ {
+		RespDocker.Ports = append(RespDocker.Ports, Ports[i])
+	}
+
+	// Sets if GPU is selected or not
+	RespDocker.GPU =GPU
+
+	// Sets Free port to Struct
+	RespDocker.SSHPort = Ports[0]
+	RespDocker.VNCPort = Ports[1]
+	// Sets appropriate username and password to the
+	// variables in the struct
+	RespDocker.SSHUsername = "master"
+	RespDocker.SSHPassword = "password"
+	RespDocker.VNCPassword = "vncpassword"
+
+	//Default parameters
+	RespDocker.TagName = "p2p-ubuntu"
+	// Get Path from config
+	config, err := config.ConfigInit()
+	if err != nil {
+		return nil,err
+	}
+	RespDocker.ImagePath = config.DockerFile
+
+	// Gets docker information from env variables
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return nil,err
+	}
+
+	// Builds docker image
+	err = RespDocker.imageBuild(cli)
+	if err != nil {
+		return nil,err
+	}
+
+	// Runs docker contianer
+	err = RespDocker.runContainer(cli)
+
+	if err != nil {
+		return nil,err
+	}
+
+
+	return RespDocker,nil
+
+}
+
+//Builds docker image (TODO: relative path for Dockerfile folder)
+func (d *DockerVM)imageBuild(dockerClient *client.Client) error {
+	ctx, _ := context.WithTimeout(context.Background(), time.Second*2000)
+	//defer cancel()
+
+	tar, err := archive.TarWithOptions(d.ImagePath, &archive.TarOptions{})
+	if err != nil {
+		return err
+	}
+
+	opts := types.ImageBuildOptions{
+		Dockerfile: "Dockerfile",
+		Tags:       []string{d.TagName},
+		Remove:     true,
+	}
+	res, err := dockerClient.ImageBuild(ctx, tar, opts)
+	if err != nil {
+		return err
+	}
+
+	defer res.Body.Close()
+
+	err = print(res.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Starts container and assigns port numbers
+// Sample Docker run Command
+// docker run -d=true --name=Test123 --restart=always --gpus all
+//-p 3443:6901 -p 3453:22 -p 3434:3434 -p 3245:3245 -v=/opt/data:/data
+//p2p-ubuntu /start > /dev/null
+func (d *DockerVM)runContainer(dockerClient *client.Client) error{
+	ctx, _ := context.WithTimeout(context.Background(), time.Second*2000)
+
+	//Exposed ports for docker config file
+	var ExposedPort nat.PortSet
+
+	ExposedPort = nat.PortSet{
+		"22/tcp": struct{}{},
+		"6901/tcp": struct{}{},
+	}
+
+	// Port forwarding for VNC and SSH ports
+	PortForwarding := nat.PortMap{
+		"22/tcp": []nat.PortBinding{
+			{
+				HostIP: "0.0.0.0",
+				HostPort: fmt.Sprint(d.SSHPort),
+			},
+		},
+		"6901/tcp": []nat.PortBinding{
+			{
+				HostIP: "0.0.0.0",
+				HostPort: fmt.Sprint(d.VNCPort),
+			},
+		},
+	}
+
+	for i := range d.Ports {
+
+		Port, err := nat.NewPort("tcp",fmt.Sprint(d.Ports[i]))
+		if err != nil {
+			return err
+		}
+
+		// Exposed Ports
+		ExposedPort[Port] = struct{}{}
+
+		PortForwarding[Port] = []nat.PortBinding{
+			{
+				HostIP: "0.0.0.0",
+				HostPort: fmt.Sprint(d.Ports[i]),
+			},
+		}
+	}
+
+	// The first mode runs using the Docker Api. As the API supports using
+	// CPU and uses a shell script for GPU call because till this point of
+	// implementation docker api does not support the flag "--gpu all"
+    if d.GPU != "true" {
+		config := &container.Config{
+			Image:        d.TagName,
+			Entrypoint:   []string{"/dockerstartup/vnc_startup.sh", "/start"},
+			Volumes:      map[string]struct{}{"/opt/data:/data": {}},
+			ExposedPorts: ExposedPort,
+		}
+		hostConfig := &container.HostConfig{
+			PortBindings: PortForwarding,
+		}
+
+		res, err := dockerClient.ContainerCreate(ctx, config, hostConfig,
+			nil, nil, "")
+
+		// Set response ID
+		d.ID = res.ID
+
+		if err != nil {
+			return err
+		}
+
+		err = dockerClient.ContainerStart(ctx, res.ID, types.ContainerStartOptions{})
+
+		if err != nil {
+			return err
+		}
+	} else {
+		// Generate Random ID
+		id := shortuuid.New()
+		d.ID = id
+
+		var cmd bytes.Buffer
+		cmd.WriteString("docker run -d=true --name="+ id +" --restart=always --gpus all -p " + fmt.Sprint(d.VNCPort) + ":" + "6901 " + "-p " + fmt.Sprint(d.SSHPort) + ":" + "22 ")
+		for i := range d.Ports {
+			cmd.WriteString("-p " + fmt.Sprint(d.Ports[i]) + ":" + fmt.Sprint(d.Ports[i]) + " ")
+		}
+		cmd.WriteString("-v=/opt/data:/data p2p-ubuntu /start > /dev/null")
+		//"-v=/opt/data:/data p2p-ubuntu /start > /dev/null"
+		cmdStr := cmd.String()
+		_, err := exec.Command("/bin/sh", "-c", cmdStr).Output()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// StopAndRemoveContainer TODO: Implement and remove docker instance running
+// Stop and remove a container
+// Reference (https://gist.github.com/frikky/e2efcea6c733ea8d8d015b7fe8a91bf6)
+func StopAndRemoveContainer(containername string) error {
+	ctx := context.Background()
+
+	// Gets docker information from env variables
+	client, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return err
+	}
+
+	if err = client.ContainerStop(ctx, containername, nil); err != nil {
+		return err
+	}
+
+	removeOptions := types.ContainerRemoveOptions{
+		RemoveVolumes: true,
+		Force:         true,
+	}
+
+	if err = client.ContainerRemove(ctx, containername, removeOptions); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func print(rd io.Reader) error {
+	var lastLine string
+
+	scanner := bufio.NewScanner(rd)
+	for scanner.Scan() {
+		lastLine = scanner.Text()
+	}
+
+	errLine := &ErrorLine{}
+	json.Unmarshal([]byte(lastLine), errLine)
+	if errLine.Error != "" {
+		return errors.New(errLine.Error)
+	}
+
+	if err := scanner.Err(); err != nil {
+		return err
+	}
+
+	return nil
+}

server/docker/docker_test.go

@@ -0,0 +1,16 @@
+package docker
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestDocker(t *testing.T) {
+	resp,err := BuildRunContainer(2,"false")
+
+	if err != nil {
+		t.Error(err)
+	}
+
+	fmt.Print(resp.VNCPort)
+}

server/docker/docker_vm.py

@@ -0,0 +1,96 @@
+'''
+The follwing file is a docker module to interact with the p2p 
+system computation/redering system.
+
+Functionality
+1. --createvm: Creates a ubuntu virtualmachine using docker settings and takes name as the parameter
+'''
+#!/usr/bin/env python3
+
+'''
+TODO: FUTURE RELEASE TO BE CONVERTED TO GO LANG
+'''
+
+import click
+import sys
+import docker
+import socket
+import uuid 
+import os
+
+# ----------------------------------------------- CLI flags --------------------------------------------------------
+
+@click.command()
+@click.option("--createvm", help="Creates docker default VM")
+
+# -------------------------------------------------------------------------------------------------------------------
+# -------------------------------------- Actions when flags are called ----------------------------------------------
+
+def main(createvm):
+
+    # creates docker virtual machine 
+    if createvm:
+        #name = str(uuid.uuid4())
+        build_run_contianer(createvm)
+
+    else:
+        ctx = click.get_current_context()
+        click.echo(ctx.get_help())
+
+# -------------------------------------------------------------------------------------------------------------------
+# ------------------------------------------ build and run Contianer ------------------------------------------------
+
+def build_run_contianer(name):
+    # Get docker information from environment variables
+    client = docker.from_env()
+
+    image_path = "./server/docker/containers/docker-ubuntu-sshd/"
+    tag_name = "p2p-ubuntu"
+    free_port = str(get_free_tcp_port())
+
+    # Change to JSON response 
+    vnc_free_port = str(get_free_tcp_port())
+
+    # Check if image is already exists (If already exists then delete and rebuild)
+    '''
+    Note:
+      Since system dependant functionality is being used here. The 
+      Error handleing is yet to be implemented.
+    '''
+    try:
+       client.images.get(tag_name)
+       #print("------ Image exists (Running Docker container " + name + ") --------")
+
+       # Run the docker continer
+       # Running interactive version until log files for results are created
+       os.system("docker run -d=true --name="+ name +" --restart=always -p "+ vnc_free_port +":6901 -p "+ free_port +":22 -v=/opt/data:/data "+ tag_name +" /start > /dev/null")
+
+    except:
+       #print("------ Image does not exists (building " + name + " and running image) ---------")
+
+       # client.images.build(path=framework_image_path,tag=tag_name,rm=True)
+       os.system("docker build -t " + tag_name + " " + image_path)
+
+       #print("------ Running Docker contianer " + name + " ---------")
+       # Run the docker continer
+       # Running interactive version until log files for results are created
+       os.system("docker run -d=true --name="+ name +" --restart=always -p "+ vnc_free_port +":6901 -p "+ free_port +":22 -v=/opt/data:/data "+ tag_name +" /start > /dev/null")
+
+    result = {"ssh_port":free_port,"vnc_port":vnc_free_port,"id": name}
+    print(result)   
+
+# -------------------------------------------------------------------------------------------------------------------
+# ------------------------------------------ Get free TCP port ------------------------------------------------------
+
+def get_free_tcp_port():
+     tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+     tcp.bind(('', 0))
+     addr, port = tcp.getsockname()
+     tcp.close()
+     return port
+
+
+# -------------------------------------------------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    main()

server/docker/dockprom/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Stefan Prodan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

server/docker/dockprom/README.md

@@ -0,0 +1,352 @@
+dockprom
+========
+
+A monitoring solution for Docker hosts and containers with [Prometheus](https://prometheus.io/), [Grafana](http://grafana.org/), [cAdvisor](https://github.com/google/cadvisor),
+[NodeExporter](https://github.com/prometheus/node_exporter) and alerting with [AlertManager](https://github.com/prometheus/alertmanager).
+
+***If you're looking for the Docker Swarm version please go to [stefanprodan/swarmprom](https://github.com/stefanprodan/swarmprom)***
+
+## Install
+
+Clone this repository on your Docker host, cd into dockprom directory and run compose up:
+
+```bash
+git clone https://github.com/stefanprodan/dockprom
+cd dockprom
+
+ADMIN_USER=admin ADMIN_PASSWORD=admin docker-compose up -d
+```
+
+Prerequisites:
+
+* Docker Engine >= 1.13
+* Docker Compose >= 1.11
+
+Containers:
+
+* Prometheus (metrics database) `http://<host-ip>:9090`
+* Prometheus-Pushgateway (push acceptor for ephemeral and batch jobs) `http://<host-ip>:9091`
+* AlertManager (alerts management) `http://<host-ip>:9093`
+* Grafana (visualize metrics) `http://<host-ip>:3000`
+* NodeExporter (host metrics collector)
+* cAdvisor (containers metrics collector)
+* Caddy (reverse proxy and basic auth provider for prometheus and alertmanager)
+
+## Setup Grafana
+
+Navigate to `http://<host-ip>:3000` and login with user ***admin*** password ***admin***. You can change the credentials in the compose file or by supplying the `ADMIN_USER` and `ADMIN_PASSWORD` environment variables on compose up. The config file can be added directly in grafana part like this
+```
+grafana:
+  image: grafana/grafana:7.2.0
+  env_file:
+    - config
+
+```
+and the config file format should have this content
+```
+GF_SECURITY_ADMIN_USER=admin
+GF_SECURITY_ADMIN_PASSWORD=changeme
+GF_USERS_ALLOW_SIGN_UP=false
+```
+If you want to change the password, you have to remove this entry, otherwise the change will not take effect
+```
+- grafana_data:/var/lib/grafana
+```
+
+Grafana is preconfigured with dashboards and Prometheus as the default data source:
+
+* Name: Prometheus
+* Type: Prometheus
+* Url: http://prometheus:9090
+* Access: proxy
+
+***Docker Host Dashboard***
+
+![Host](https://raw.githubusercontent.com/stefanprodan/dockprom/master/screens/Grafana_Docker_Host.png)
+
+The Docker Host Dashboard shows key metrics for monitoring the resource usage of your server:
+
+* Server uptime, CPU idle percent, number of CPU cores, available memory, swap and storage
+* System load average graph, running and blocked by IO processes graph, interrupts graph
+* CPU usage graph by mode (guest, idle, iowait, irq, nice, softirq, steal, system, user)
+* Memory usage graph by distribution (used, free, buffers, cached)
+* IO usage graph (read Bps, read Bps and IO time)
+* Network usage graph by device (inbound Bps, Outbound Bps)
+* Swap usage and activity graphs
+
+For storage and particularly Free Storage graph, you have to specify the fstype in grafana graph request.
+You can find it in `grafana/dashboards/docker_host.json`, at line 480 :
+
+      "expr": "sum(node_filesystem_free_bytes{fstype=\"btrfs\"})",
+
+I work on BTRFS, so i need to change `aufs` to `btrfs`.
+
+You can find right value for your system in Prometheus `http://<host-ip>:9090` launching this request :
+
+      node_filesystem_free_bytes
+
+***Docker Containers Dashboard***
+
+![Containers](https://raw.githubusercontent.com/stefanprodan/dockprom/master/screens/Grafana_Docker_Containers.png)
+
+The Docker Containers Dashboard shows key metrics for monitoring running containers:
+
+* Total containers CPU load, memory and storage usage
+* Running containers graph, system load graph, IO usage graph
+* Container CPU usage graph
+* Container memory usage graph
+* Container cached memory usage graph
+* Container network inbound usage graph
+* Container network outbound usage graph
+
+Note that this dashboard doesn't show the containers that are part of the monitoring stack.
+
+***Monitor Services Dashboard***
+
+![Monitor Services](https://raw.githubusercontent.com/stefanprodan/dockprom/master/screens/Grafana_Prometheus.png)
+
+The Monitor Services Dashboard shows key metrics for monitoring the containers that make up the monitoring stack:
+
+* Prometheus container uptime, monitoring stack total memory usage, Prometheus local storage memory chunks and series
+* Container CPU usage graph
+* Container memory usage graph
+* Prometheus chunks to persist and persistence urgency graphs
+* Prometheus chunks ops and checkpoint duration graphs
+* Prometheus samples ingested rate, target scrapes and scrape duration graphs
+* Prometheus HTTP requests graph
+* Prometheus alerts graph
+
+## Define alerts
+
+Three alert groups have been setup within the [alert.rules](https://github.com/stefanprodan/dockprom/blob/master/prometheus/alert.rules) configuration file:
+
+* Monitoring services alerts [targets](https://github.com/stefanprodan/dockprom/blob/master/prometheus/alert.rules#L2-L11)
+* Docker Host alerts [host](https://github.com/stefanprodan/dockprom/blob/master/prometheus/alert.rules#L13-L40)
+* Docker Containers alerts [containers](https://github.com/stefanprodan/dockprom/blob/master/prometheus/alert.rules#L42-L69)
+
+You can modify the alert rules and reload them by making a HTTP POST call to Prometheus:
+
+```
+curl -X POST http://admin:admin@<host-ip>:9090/-/reload
+```
+
+***Monitoring services alerts***
+
+Trigger an alert if any of the monitoring targets (node-exporter and cAdvisor) are down for more than 30 seconds:
+
+```yaml
+- alert: monitor_service_down
+    expr: up == 0
+    for: 30s
+    labels:
+      severity: critical
+    annotations:
+      summary: "Monitor service non-operational"
+      description: "Service {{ $labels.instance }} is down."
+```
+
+***Docker Host alerts***
+
+Trigger an alert if the Docker host CPU is under high load for more than 30 seconds:
+
+```yaml
+- alert: high_cpu_load
+    expr: node_load1 > 1.5
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Server under high load"
+      description: "Docker host is under high load, the avg load 1m is at {{ $value}}. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
+```
+
+Modify the load threshold based on your CPU cores.
+
+Trigger an alert if the Docker host memory is almost full:
+
+```yaml
+- alert: high_memory_load
+    expr: (sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100 > 85
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Server memory is almost full"
+      description: "Docker host memory usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
+```
+
+Trigger an alert if the Docker host storage is almost full:
+
+```yaml
+- alert: high_storage_load
+    expr: (node_filesystem_size_bytes{fstype="aufs"} - node_filesystem_free_bytes{fstype="aufs"}) / node_filesystem_size_bytes{fstype="aufs"}  * 100 > 85
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Server storage is almost full"
+      description: "Docker host storage usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
+```
+
+***Docker Containers alerts***
+
+Trigger an alert if a container is down for more than 30 seconds:
+
+```yaml
+- alert: jenkins_down
+    expr: absent(container_memory_usage_bytes{name="jenkins"})
+    for: 30s
+    labels:
+      severity: critical
+    annotations:
+      summary: "Jenkins down"
+      description: "Jenkins container is down for more than 30 seconds."
+```
+
+Trigger an alert if a container is using more than 10% of total CPU cores for more than 30 seconds:
+
+```yaml
+- alert: jenkins_high_cpu
+    expr: sum(rate(container_cpu_usage_seconds_total{name="jenkins"}[1m])) / count(node_cpu_seconds_total{mode="system"}) * 100 > 10
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Jenkins high CPU usage"
+      description: "Jenkins CPU usage is {{ humanize $value}}%."
+```
+
+Trigger an alert if a container is using more than 1.2GB of RAM for more than 30 seconds:
+
+```yaml
+- alert: jenkins_high_memory
+    expr: sum(container_memory_usage_bytes{name="jenkins"}) > 1200000000
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Jenkins high memory usage"
+      description: "Jenkins memory consumption is at {{ humanize $value}}."
+```
+
+## Setup alerting
+
+The AlertManager service is responsible for handling alerts sent by Prometheus server.
+AlertManager can send notifications via email, Pushover, Slack, HipChat or any other system that exposes a webhook interface.
+A complete list of integrations can be found [here](https://prometheus.io/docs/alerting/configuration).
+
+You can view and silence notifications by accessing `http://<host-ip>:9093`.
+
+The notification receivers can be configured in [alertmanager/config.yml](https://github.com/stefanprodan/dockprom/blob/master/alertmanager/config.yml) file.
+
+To receive alerts via Slack you need to make a custom integration by choose ***incoming web hooks*** in your Slack team app page.
+You can find more details on setting up Slack integration [here](http://www.robustperception.io/using-slack-with-the-alertmanager/).
+
+Copy the Slack Webhook URL into the ***api_url*** field and specify a Slack ***channel***.
+
+```yaml
+route:
+    receiver: 'slack'
+
+receivers:
+    - name: 'slack'
+      slack_configs:
+          - send_resolved: true
+            text: "{{ .CommonAnnotations.description }}"
+            username: 'Prometheus'
+            channel: '#<channel>'
+            api_url: 'https://hooks.slack.com/services/<webhook-id>'
+```
+
+![Slack Notifications](https://raw.githubusercontent.com/stefanprodan/dockprom/master/screens/Slack_Notifications.png)
+
+## Sending metrics to the Pushgateway
+
+The [pushgateway](https://github.com/prometheus/pushgateway) is used to collect data from batch jobs or from services.
+
+To push data, simply execute:
+
+    echo "some_metric 3.14" | curl --data-binary @- http://user:password@localhost:9091/metrics/job/some_job
+
+Please replace the `user:password` part with your user and password set in the initial configuration (default: `admin:admin`).
+
+## Updating Grafana to v5.2.2
+
+[In Grafana versions >= 5.1 the id of the grafana user has been changed](http://docs.grafana.org/installation/docker/#migration-from-a-previous-version-of-the-docker-container-to-5-1-or-later). Unfortunately this means that files created prior to 5.1 won’t have the correct permissions for later versions.
+
+| Version |   User  | User ID |
+|:-------:|:-------:|:-------:|
+|  < 5.1  | grafana |   104   |
+|  \>= 5.1 | grafana |   472   |
+
+There are two possible solutions to this problem.
+- Change ownership from 104 to 472
+- Start the upgraded container as user 104
+
+##### Specifying a user in docker-compose.yml
+
+To change ownership of the files run your grafana container as root and modify the permissions.
+
+First perform a `docker-compose down` then modify your docker-compose.yml to include the `user: root` option:
+
+```
+  grafana:
+    image: grafana/grafana:5.2.2
+    container_name: grafana
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./grafana/datasources:/etc/grafana/datasources
+      - ./grafana/dashboards:/etc/grafana/dashboards
+      - ./grafana/setup.sh:/setup.sh
+    entrypoint: /setup.sh
+    user: root
+    environment:
+      - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
+      - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
+      - GF_USERS_ALLOW_SIGN_UP=false
+    restart: unless-stopped
+    expose:
+      - 3000
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"
+```
+
+Perform a `docker-compose up -d` and then issue the following commands:
+
+```
+docker exec -it --user root grafana bash
+
+# in the container you just started:
+chown -R root:root /etc/grafana && \
+chmod -R a+r /etc/grafana && \
+chown -R grafana:grafana /var/lib/grafana && \
+chown -R grafana:grafana /usr/share/grafana
+```
+
+To run the grafana container as `user: 104` change your `docker-compose.yml` like such:
+
+```
+  grafana:
+    image: grafana/grafana:5.2.2
+    container_name: grafana
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./grafana/datasources:/etc/grafana/datasources
+      - ./grafana/dashboards:/etc/grafana/dashboards
+      - ./grafana/setup.sh:/setup.sh
+    entrypoint: /setup.sh
+    user: "104"
+    environment:
+      - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
+      - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
+      - GF_USERS_ALLOW_SIGN_UP=false
+    restart: unless-stopped
+    expose:
+      - 3000
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"
+```

server/docker/dockprom/alertmanager/config.yml

@@ -0,0 +1,11 @@
+route:
+    receiver: 'slack'
+
+receivers:
+    - name: 'slack'
+      slack_configs:
+          - send_resolved: true
+            text: "{{ .CommonAnnotations.description }}"
+            username: 'Prometheus'
+            channel: '#<channel-name>'
+            api_url: 'https://hooks.slack.com/services/<webhook-id>'

server/docker/dockprom/caddy/Caddyfile

@@ -0,0 +1,36 @@
+:9090 {
+    proxy / prometheus:9090 {
+            transparent
+        }
+
+    errors stderr
+    tls off
+}
+
+:9093 {
+    proxy / alertmanager:9093 {
+            transparent
+        }
+
+    errors stderr
+    tls off
+}
+
+:9091 {
+    proxy / pushgateway:9091 {
+            transparent
+        }
+
+    errors stderr
+    tls off
+}
+
+:3000 {
+    proxy / grafana:3000 {
+            transparent
+            websocket
+        }
+
+    errors stderr
+    tls off
+}

server/docker/dockprom/config

@@ -0,0 +1,3 @@
+GF_SECURITY_ADMIN_USER=admin
+GF_SECURITY_ADMIN_PASSWORD=changeme
+GF_USERS_ALLOW_SIGN_UP=false

server/docker/dockprom/docker-compose.exporters.yml

@@ -0,0 +1,36 @@
+version: '2.1'
+
+services:
+
+  nodeexporter:
+    image: prom/node-exporter:v1.0.1
+    container_name: nodeexporter
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+    restart: unless-stopped
+    network_mode: host
+    labels:
+      org.label-schema.group: "monitoring"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.38.7
+    container_name: cadvisor
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+      - /cgroup:/cgroup:ro
+    restart: unless-stopped
+    network_mode: host
+    labels:
+      org.label-schema.group: "monitoring"
+  
+

server/docker/dockprom/docker-compose.yml

@@ -0,0 +1,146 @@
+version: '2.1'
+
+networks:
+  monitor-net:
+    driver: bridge
+
+volumes:
+    prometheus_data: {}
+    grafana_data: {}
+
+services:
+
+  prometheus:
+    image: prom/prometheus:v2.24.1
+    container_name: prometheus
+    volumes:
+      - ./prometheus:/etc/prometheus
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/etc/prometheus/console_libraries'
+      - '--web.console.templates=/etc/prometheus/consoles'
+      - '--storage.tsdb.retention.time=200h'
+      - '--web.enable-lifecycle'
+    restart: unless-stopped
+    expose:
+      - 9090
+    ports:
+      - "9090:9090"
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"
+
+  alertmanager:
+    image: prom/alertmanager:v0.21.0
+    container_name: alertmanager
+    volumes:
+      - ./alertmanager:/etc/alertmanager
+    command:
+      - '--config.file=/etc/alertmanager/config.yml'
+      - '--storage.path=/alertmanager'
+    restart: unless-stopped
+    expose:
+      - 9093
+    ports:
+      - "9093:9093"
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"
+
+  nodeexporter:
+    image: prom/node-exporter:v1.0.1
+    container_name: nodeexporter
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)'
+    restart: unless-stopped
+    expose:
+      - 9100
+    ports:
+      - "9100:9100"
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"
+
+  cadvisor:
+    image: gcr.io/cadvisor/cadvisor:v0.38.7
+    container_name: cadvisor
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:rw
+      - /sys:/sys:ro
+      - /var/lib/docker:/var/lib/docker:ro
+      #- /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux
+    restart: unless-stopped
+    expose:
+      - 8080
+    ports:
+      - "8080:8080"
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"
+
+  grafana:
+    image: grafana/grafana:7.3.7
+    container_name: grafana
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards
+      - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources
+    environment:
+      - GF_SECURITY_ADMIN_USER=${ADMIN_USER:-admin}
+      - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
+      - GF_USERS_ALLOW_SIGN_UP=false
+    restart: unless-stopped
+    expose:
+      - 3000
+    ports:
+      - "3000:3000"
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"
+
+  pushgateway:
+    image: prom/pushgateway:v1.4.0
+    container_name: pushgateway
+    restart: unless-stopped
+    expose:
+      - 9091
+    ports:
+      - "9091:9091"
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"
+
+  caddy:
+    image: caddy:2.3.0-alpine
+    container_name: caddy
+    ports:
+      - "3000:3000"
+      - "9090:9090"
+      - "9093:9093"
+      - "9091:9091"
+    volumes:
+      - ./caddy:/etc/caddy
+    environment:
+      - ADMIN_USER=${ADMIN_USER:-admin}
+      - ADMIN_PASSWORD=${ADMIN_PASSWORD:-admin}
+    restart: unless-stopped
+    networks:
+      - monitor-net
+    labels:
+      org.label-schema.group: "monitoring"

server/docker/dockprom/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: 'Prometheus'
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    editable: true
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/provisioning/dashboards

server/docker/dockprom/grafana/provisioning/dashboards/nginx_container.json

@@ -0,0 +1,398 @@
+{
+  "id": null,
+  "title": "Nginx",
+  "description": "Nginx exporter metrics",
+  "tags": [
+    "nginx"
+  ],
+  "style": "dark",
+  "timezone": "browser",
+  "editable": true,
+  "hideControls": false,
+  "sharedCrosshair": true,
+  "rows": [
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": "Prometheus",
+          "decimals": 2,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 3,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(irate(nginx_connections_processed_total{stage=\"any\"}[5m])) by (stage)",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 10,
+              "legendFormat": "requests",
+              "metric": "",
+              "refId": "B",
+              "step": 10
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Requests/sec",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "show": true
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": "Prometheus",
+          "decimals": 2,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 2,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(nginx_connections_current) by (state)",
+              "interval": "",
+              "intervalFactor": 2,
+              "legendFormat": "{{state}}",
+              "metric": "",
+              "refId": "A",
+              "step": 2
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Connections",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "show": true
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": "Prometheus",
+          "decimals": 2,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 1,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(irate(nginx_connections_processed_total{stage!=\"any\"}[5m])) by (stage)",
+              "hide": false,
+              "interval": "",
+              "intervalFactor": 10,
+              "legendFormat": "{{stage}}",
+              "metric": "",
+              "refId": "B",
+              "step": 10
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "Connections rate",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "show": true
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "title": "Nginx exporter metrics"
+    },
+    {
+      "collapse": false,
+      "editable": true,
+      "height": "250px",
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "datasource": null,
+          "editable": true,
+          "error": false,
+          "fill": 1,
+          "grid": {
+            "threshold1": null,
+            "threshold1Color": "rgba(216, 200, 27, 0.27)",
+            "threshold2": null,
+            "threshold2Color": "rgba(234, 112, 112, 0.22)"
+          },
+          "id": 4,
+          "isNew": true,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": true,
+            "max": true,
+            "min": true,
+            "rightSide": true,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "connected",
+          "percentage": false,
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "span": 12,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"nginx\"}[5m])) / count(node_cpu_seconds_total{mode=\"system\"}) * 100",
+              "intervalFactor": 2,
+              "legendFormat": "nginx",
+              "refId": "A",
+              "step": 2
+            }
+          ],
+          "timeFrom": null,
+          "timeShift": null,
+          "title": "CPU usage",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "show": true
+          },
+          "yaxes": [
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ]
+        }
+      ],
+      "title": "Nginx container metrics"
+    }
+  ],
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+    "templating": {
+    "list": []
+  },
+  "annotations": {
+    "list": []
+  },
+  "refresh": "10s",
+  "schemaVersion": 12,
+  "version": 9,
+  "links": [],
+  "gnetId": null
+}

server/docker/dockprom/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    orgId: 1
+    url: http://prometheus:9090
+    basicAuth: false
+    isDefault: true
+    editable: true

server/docker/dockprom/helpers/aws/README.md

@@ -0,0 +1,22 @@
+# Prometheus on EC2 & ECS:
+
+Some helpers for anyone configuring Prometheus on ECS and AWS EC2.
+
+To get started on AWS ECS and EC2:
+
+*For EC2/ECS nodes*:
+- Import the ecs task definition and add cadvisor and node-exporter service/task definition and run them on each host you want to be monitored
+- Any hosts which have "Monitoring: On" tag will be automatically added in the targets
+- Expose ports 9100 and 9191 to your Prometheus host
+
+*For Prometheus host*:
+
+- Copy prometheus.yml configuration present here to base prometheus configuration to enable EC2 service discovery
+- `docker compose up -d`
+
+**Note**:
+Set query.staleness-delta to 1m make metrics more realtime
+
+
+### TODO
+- Add alerting rules based on ECS

server/docker/dockprom/helpers/aws/cadvisor_ecs_task_definition.json

@@ -0,0 +1,78 @@
+{
+    "family": "cadvisor",
+    "containerDefinitions": [
+        {
+            "name": "cadvisor",
+            "image": "google/cadvisor",
+            "cpu": 10,
+            "memory": 300,
+            "portMappings": [
+                {
+                    "containerPort": 9191,
+                    "hostPort": 9191
+                }
+            ],
+            "essential": true,
+            "privileged": true,
+            "mountPoints": [
+                {
+                    "sourceVolume": "root",
+                    "containerPath": "/rootfs",
+                    "readOnly": true
+                },
+                {
+                    "sourceVolume": "var_run",
+                    "containerPath": "/var/run",
+                    "readOnly": false
+                },
+                {
+                    "sourceVolume": "sys",
+                    "containerPath": "/sys",
+                    "readOnly": true
+                },
+                {
+                    "sourceVolume": "var_lib_docker",
+                    "containerPath": "/var/lib/docker",
+                    "readOnly": true
+                },
+                {
+                    "sourceVolume": "cgroup",
+                    "containerPath": "/cgroup",
+                    "readOnly": true
+                }
+            ]
+        }
+    ],
+    "volumes": [
+        {
+            "name": "root",
+            "host": {
+                "sourcePath": "/"
+            }
+        },
+        {
+            "name": "var_run",
+            "host": {
+                "sourcePath": "/var/run"
+            }
+        },
+        {
+            "name": "sys",
+            "host": {
+                "sourcePath": "/sys"
+            }
+        },
+        {
+            "name": "var_lib_docker",
+            "host": {
+                "sourcePath": "/var/lib/docker/"
+            }
+        },
+        {
+            "name": "cgroup",
+            "host": {
+                "sourcePath": "/cgroup"
+            }
+        }
+    ]
+}

server/docker/dockprom/helpers/aws/node_exporter_task_definition.json

@@ -0,0 +1,22 @@
+{
+    "family": "prometheus",
+    "containerDefinitions": [
+        {
+            "portMappings": [
+                {
+                    "hostPort": 9100,
+                    "containerPort": 9100,
+                    "protocol": "tcp"
+                }
+            ],
+            "essential": true,
+            "name": "node_exporter",
+            "image": "prom/node-exporter",
+            "cpu": 0,
+            "privileged": null,
+            "memoryReservation": 150
+        }
+    ],
+    "volumes": [],
+    "networkMode": "host"
+}

server/docker/dockprom/helpers/aws/prometheus.yml

@@ -0,0 +1,53 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+      monitor: 'docker-host-alpha'
+
+# Load and evaluate rules in this file every 'evaluation_interval' seconds.
+rule_files:
+  - "targets.rules"
+  - "hosts.rules"
+  - "containers.rules"
+
+# A scrape configuration containing exactly one endpoint to scrape.
+scrape_configs:
+  - job_name: 'nodeexporter'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['nodeexporter:9100']
+
+  - job_name: 'cadvisor'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['cadvisor:8080']
+
+  - job_name: 'prometheus'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['localhost:9090']
+
+
+# sample scrape configuration for AWS EC2
+  - job_name: 'nodeexporter'
+    ec2_sd_configs:
+      - region: us-east-1
+        port: 9100
+    relabel_configs:
+        # Only monitor instances which have a tag called Monitoring "Monitoring"
+      - source_labels: [__meta_ec2_tag_Monitoring]
+        regex: On
+        action: keep
+
+  - job_name: 'cadvisor'
+    ec2_sd_configs:
+      - region: us-east-1
+        port: 9010
+    relabel_configs:
+        # Only monitor instances which have a tag called Monitoring "Monitoring"
+      - source_labels: [__meta_ec2_tag_Monitoring]
+        regex: On
+        action: keep

server/docker/dockprom/prometheus/alert.rules

@@ -0,0 +1,70 @@
+groups:
+- name: targets
+  rules:
+  - alert: monitor_service_down
+    expr: up == 0
+    for: 30s
+    labels:
+      severity: critical
+    annotations:
+      summary: "Monitor service non-operational"
+      description: "Service {{ $labels.instance }} is down."
+
+- name: host
+  rules:
+  - alert: high_cpu_load
+    expr: node_load1 > 1.5
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Server under high load"
+      description: "Docker host is under high load, the avg load 1m is at {{ $value}}. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
+
+  - alert: high_memory_load
+    expr: (sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100 > 85
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Server memory is almost full"
+      description: "Docker host memory usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
+
+  - alert: high_storage_load
+    expr: (node_filesystem_size_bytes{fstype="aufs"} - node_filesystem_free_bytes{fstype="aufs"}) / node_filesystem_size_bytes{fstype="aufs"}  * 100 > 85
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Server storage is almost full"
+      description: "Docker host storage usage is {{ humanize $value}}%. Reported by instance {{ $labels.instance }} of job {{ $labels.job }}."
+
+- name: containers
+  rules:
+  - alert: jenkins_down
+    expr: absent(container_memory_usage_bytes{name="jenkins"})
+    for: 30s
+    labels:
+      severity: critical
+    annotations:
+      summary: "Jenkins down"
+      description: "Jenkins container is down for more than 30 seconds."
+
+  - alert: jenkins_high_cpu
+    expr: sum(rate(container_cpu_usage_seconds_total{name="jenkins"}[1m])) / count(node_cpu_seconds_total{mode="system"}) * 100 > 10
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Jenkins high CPU usage"
+      description: "Jenkins CPU usage is {{ humanize $value}}%."
+
+  - alert: jenkins_high_memory
+    expr: sum(container_memory_usage_bytes{name="jenkins"}) > 1200000000
+    for: 30s
+    labels:
+      severity: warning
+    annotations:
+      summary: "Jenkins high memory usage"
+      description: "Jenkins memory consumption is at {{ humanize $value}}."
+

server/docker/dockprom/prometheus/prometheus.yml

@@ -0,0 +1,53 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+  # Attach these labels to any time series or alerts when communicating with
+  # external systems (federation, remote storage, Alertmanager).
+  external_labels:
+      monitor: 'docker-host-alpha'
+
+# Load and evaluate rules in this file every 'evaluation_interval' seconds.
+rule_files:
+  - "alert.rules"
+
+# A scrape configuration containing exactly one endpoint to scrape.
+scrape_configs:
+  - job_name: 'nodeexporter'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['nodeexporter:9100']
+
+  - job_name: 'cadvisor'
+    scrape_interval: 5s
+    static_configs:
+      - targets: ['cadvisor:8080']
+
+  - job_name: 'prometheus'
+    scrape_interval: 10s
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'pushgateway'
+    scrape_interval: 10s
+    honor_labels: true
+    static_configs:
+      - targets: ['pushgateway:9091']
+
+
+alerting:
+  alertmanagers:
+  - scheme: http
+    static_configs:
+    - targets: 
+      - 'alertmanager:9093'
+
+#  - job_name: 'nginx'
+#    scrape_interval: 10s
+#    static_configs:
+#      - targets: ['nginxexporter:9113']
+
+#  - job_name: 'aspnetcore'
+#    scrape_interval: 10s
+#    static_configs:
+#      - targets: ['eventlog-proxy:5000', 'eventlog:5000']

server/docker/kill-containers.sh

@@ -0,0 +1,4 @@
+docker kill $(docker ps -q)\
+docker rm $(docker ps -a -q)
+docker rmi $(docker images -q) --force
+docker system prune --all

server/docker/requirements.txt

@@ -0,0 +1,2 @@
+docker
+click