From 35f613b81a4245612f0134a0539e663be3f74d81 Mon Sep 17 00:00:00 2001
From: George Vlahavas <vlahavas@gmail.com>
Date: Sun, 25 Jul 2021 02:01:23 +0300
Subject: [PATCH] Add Hetzner deployment scripts
---
deploy_hetzner.sh | 75 ++++++++++++++++
docker-compose-hetzner.yml | 210 +++++++++++++++++++++++++++++++++++++++++++++
hetzner_delete.groovy | 39 +++++++++
hetzner_deploy.groovy | 86 +++++++++++++++++++
4 files changed, 410 insertions(+)
create mode 100755 deploy_hetzner.sh
create mode 100644 docker-compose-hetzner.yml
create mode 100644 hetzner_delete.groovy
create mode 100644 hetzner_deploy.groovy
diff --git a/deploy_hetzner.sh b/deploy_hetzner.sh
new file mode 100755
index 0000000..8d40d0b
--- /dev/null
+++ b/deploy_hetzner.sh
@@ -0,0 +1,75 @@
+#!/bin/sh
+
+dump_logs() {
+ CONTAINERS="suricata elasticsearch logstash kibana webserver api od mltd mqtt timescaledb grafana"
+ for CONTAINER in $CONTAINERS; do
+ echo "*** Logs for kea_$CONTAINER (start) ***"
+ docker logs kea_$CONTAINER
+ echo "*** Logs for kea_$CONTAINER (end) ***"
+ done
+}
+
+# Login into the CUREX registry
+docker login https://registry.curex-project.eu:443/artifactory/curex-local/
+
+# pull all containers
+docker-compose \
+ -f docker-compose-hetzner.yml \
+ pull
+
+sleep 1
+
+# Keep the timestamp of when starting up containers
+TIME_START=`date +%s`
+
+# Just in case something is up from previous runs
+echo "Bringing down any previous containers that are still running..."
+./delete.sh
+
+# Run containers
+echo "Bringing up containers..."
+docker-compose -f docker-compose-hetzner.yml up --detach
+
+# loop until all containers are up
+echo "Checking if all containers are up..."
+while true; do
+ # if any container other than composer has exited, bail out
+ res=`docker ps -a | \
+ grep "/curex-local/kea_" | \
+ grep "Exited" | \
+ wc -l`
+ if [ $res -gt 0 ]; then
+ echo "ERROR: Some containers have exited."
+ docker ps -a | grep "^CONTAINER\|/curex-local/kea_"
+ dump_logs
+ docker-compose -f docker-compose-hetzner.yml down
+ exit 1
+ fi
+ # count containers that are up
+ res=`docker ps -a | \
+ grep "/curex-local/kea_" | \
+ grep "Up" | \
+ wc -l`
+ if [ $res -eq 11 ]; then
+ echo "All good!"
+ break
+ fi
+ # only wait for 10 mins until everything is up. It should be more than
+ # enough.
+ TIME_NOW=`date +%s`
+ if [ $((TIME_NOW - TIME_START)) -ge 600 ]; then
+ echo "ERROR: Timeout exceeded."
+ dump_logs
+ docker-compose -f docker-compose-hetzner.yml down
+ exit 2
+ fi
+ sleep 1
+done
+
+# Wait a bit more
+sleep 60
+
+# create and populate the database
+echo "Creating and populating the database..."
+docker-compose -f docker-compose-hetzner.yml exec -T api php artisan migrate:fresh --seed --force
+
diff --git a/docker-compose-hetzner.yml b/docker-compose-hetzner.yml
new file mode 100644
index 0000000..db6b05b
--- /dev/null
+++ b/docker-compose-hetzner.yml
@@ -0,0 +1,210 @@
+version: '3.7'
+
+services:
+
+ #CEPTD
+ suricata:
+ image: registry.curex-project.eu:443/curex-local/kea_suricata:latest
+ container_name: kea_suricata
+ restart: unless-stopped
+ network_mode: "host"
+ cap_add:
+ - NET_ADMIN
+ - SYS_NICE
+ - NET_RAW
+ command: ONLINE
+ volumes:
+ - data_volume:/var/log/suricata
+ - pcap_volume:/var/pcap
+ networks:
+ - "proxy-network"
+
+ elasticsearch:
+ image: registry.curex-project.eu:443/curex-local/kea_elasticsearch:latest
+ container_name: kea_elasticsearch
+ restart: unless-stopped
+ environment:
+ - cluster.name=keacluster
+ - node.name=keacluster-node-01
+ - discovery.type=single-node
+ - bootstrap.memory_lock=true
+ - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
+ - xpack.ml.enabled=false
+ - xpack.security.enabled=false
+ - xpack.ilm.enabled=false
+ - path.logs=/data/elk/log
+ - path.data=/data/elk/data
+ - http.host=0.0.0.0
+ - http.cors.enabled=true
+ - http.cors.allow-origin="*"
+ - indices.query.bool.max_clause_count=2000
+ ulimits:
+ memlock:
+ soft: -1
+ hard: -1
+ volumes:
+ - data_volume:/data
+ networks:
+ - "proxy-network"
+
+ logstash:
+ image: registry.curex-project.eu:443/curex-local/kea_logstash:latest
+ container_name: kea_logstash
+ restart: unless-stopped
+ volumes:
+ - data_volume:/data
+ networks:
+ - "proxy-network"
+
+ kibana:
+ image: registry.curex-project.eu:443/curex-local/kea_kibana:latest
+ container_name: kea_kibana
+ restart: unless-stopped
+ expose:
+ - 5601
+ networks:
+ - "proxy-network"
+
+ #Controller
+ webserver:
+ image: registry.curex-project.eu:443/curex-local/kea_webserver:latest
+ container_name: kea_webserver
+ restart: unless-stopped
+ depends_on:
+ - api
+ tty: true
+ expose:
+ - 80
+ volumes:
+ - api_volume:/var/www
+ networks:
+ - "proxy-network"
+ environment:
+ - "VIRTUAL_HOST=kea.curex-project.eu"
+ - "VIRTUAL_PORT=80"
+ - "LETSENCRYPT_HOST=kea.curex-project.eu"
+ - "LETSENCRYPT_EMAIL=gvlahavas@csd.auth.gr"
+
+ api:
+ image: registry.curex-project.eu:443/curex-local/kea_api:latest
+ container_name: kea_api
+ restart: unless-stopped
+ tty: true
+ working_dir: /var/www
+ volumes:
+ - api_volume:/var/www
+ environment:
+ - "APP_NAME=${APP_NAME}"
+ - "APP_ENV=${APP_ENV}"
+ - "APP_DEBUG=${APP_DEBUG}"
+ - "APP_URL=${APP_URL}"
+ - "KIBANA_PORT=${KIBANA_PORT}"
+ - "GRAFANA_PORT=${GRAFANA_PORT}"
+ - "LOG_CHANNEL=${LOG_CHANNEL}"
+ - "DB_CONNECTION=${DB_CONNECTION}"
+ - "DB_HOST=${DB_HOST}"
+ - "DB_PORT=${DB_PORT}"
+ - "DB_DATABASE=${DB_DATABASE}"
+ - "DB_USERNAME=${DB_USERNAME}"
+ - "DB_PASSWORD=${DB_PASSWORD}"
+ - "BROADCAST_DRIVER=${BROADCAST_DRIVER}"
+ - "CACHE_DRIVER=${CACHE_DRIVER}"
+ - "QUEUE_CONNECTION=${QUEUE_CONNECTION}"
+ - "SESSION_DRIVER=${SESSION_DRIVER}"
+ - "SESSION_LIFETIME=${SESSION_LIFETIME}"
+ - "ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST}"
+ - "ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT}"
+ - "ELASTICSEARCH_SCHEME=${ELASTICSEARCH_SCHEME}"
+ - "MQTT_HOST=${MQTT_HOST}"
+ - "MQTT_PORT=${MQTT_PORT}"
+ - "MQTT_DEBUG=${MQTT_DEBUG}"
+ - "MQTT_QOS=${MQTT_QOS}"
+ - "MQTT_RETAIN=${MQTT_RETAIN}"
+ - "MLTD_HOST=${MLTD_HOST}"
+ - "MLTD_PORT=${MLTD_PORT}"
+ - "OD_HOST=${OD_HOST}"
+ - "OD_PORT=${OD_PORT}"
+ networks:
+ - "proxy-network"
+
+ #OD
+ od:
+ image: registry.curex-project.eu:443/curex-local/kea_od:latest
+ container_name: kea_od
+ depends_on:
+ - timescaledb
+ restart: unless-stopped
+ expose:
+ - 9091
+ networks:
+ - "proxy-network"
+
+ #MLTD
+ mltd:
+ image: registry.curex-project.eu:443/curex-local/kea_mltd:latest
+ container_name: kea_mltd
+ depends_on:
+ - timescaledb
+ - mosquitto
+ restart: unless-stopped
+ expose:
+ - 5000
+ networks:
+ - "proxy-network"
+
+ #MQTT
+ mosquitto:
+ image: registry.curex-project.eu:443/curex-local/kea_mqtt:latest
+ container_name: kea_mqtt
+ expose:
+ - 1883
+ - 9001
+ volumes:
+ - mqtt_data_volume:/mosquitto/data
+ - mqtt_log_volume:/mosquitto/log
+ networks:
+ - "proxy-network"
+
+ #Persistence
+ timescaledb:
+ image: registry.curex-project.eu:443/curex-local/kea_timescaledb:latest
+ container_name: kea_timescaledb
+ volumes:
+ - timescaledb_volume:/var/lib/postgresql/data
+ environment:
+ - POSTGRES_USER=postgres
+ - POSTGRES_PASSWORD=postgres
+ - POSTGRES_DB=kea
+ expose:
+ - 5432
+ networks:
+ - "proxy-network"
+
+ #Visualization
+ grafana:
+ image: registry.curex-project.eu:443/curex-local/kea_grafana:latest
+ container_name: kea_grafana
+ expose:
+ - 3000
+ environment:
+ - GF_SECURITY_ALLOW_EMBEDDING=true
+ - GF_SECURITY_COOKIE_SAMESITE=none
+ - GF_AUTH_ANONYMOUS_ENABLED=true
+ - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+ networks:
+ - "proxy-network"
+
+networks:
+ proxy-network:
+ external:
+ name: proxy-network
+
+
+volumes:
+ api_volume:
+ data_volume:
+ pcap_volume:
+ mqtt_data_volume:
+ mqtt_log_volume:
+ timescaledb_volume:
+
diff --git a/hetzner_delete.groovy b/hetzner_delete.groovy
new file mode 100644
index 0000000..46c8c82
--- /dev/null
+++ b/hetzner_delete.groovy
@@ -0,0 +1,39 @@
+/**
+* Jenkinsfile to deploy multiple Docker containers based on docker-compose into a DEV server and run any test.
+* This pipeline will run the Docker containers, execute the tests and then stop and remove the containers from the DEV
+* server automatically.
+**/
+
+
+pipeline {
+
+ agent any
+
+ environment {
+ // Keep as is the following two lines
+ PRIVATE_REGISTRY = "https://registry.curex-project.eu:443/curex-local/"
+ ARTIFACTORY_URL = "registry.curex-project.eu:443/curex-local/"
+
+ HOST_IP = "116.203.166.220"
+ DEPLOYMENT_HOST = "${HOST_IP}:2376"
+ DEPLOYMENT_HOST_CREDENTIALS = "vm2-creds"
+ }
+
+ stages {
+
+ stage('Stop and remove the Docker containers in DEV server') {
+ steps {
+ script {
+ docker.withServer("$DEPLOYMENT_HOST", "$DEPLOYMENT_HOST_CREDENTIALS") {
+ docker.withRegistry("$PRIVATE_REGISTRY" , 'artifactory') {
+
+ //echo 'Stop and remove the specified Docker containers from the DEV server'
+ sh 'sh delete.sh'
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
diff --git a/hetzner_deploy.groovy b/hetzner_deploy.groovy
new file mode 100644
index 0000000..93411b5
--- /dev/null
+++ b/hetzner_deploy.groovy
@@ -0,0 +1,86 @@
+/**
+* Jenkinsfile to deploy multiple Docker containers based on docker-compose into a DEV server and run any test.
+* This pipeline will run the Docker containers, execute the tests and then stop and remove the containers from the DEV
+* server automatically.
+**/
+
+
+pipeline {
+
+ agent any
+
+ environment {
+ // Keep as is the following two lines
+ PRIVATE_REGISTRY = "https://registry.curex-project.eu:443/curex-local/"
+ ARTIFACTORY_URL = "registry.curex-project.eu:443/curex-local/"
+
+ HOST_IP = "116.203.166.220"
+ DEPLOYMENT_HOST = "${HOST_IP}:2376"
+ DEPLOYMENT_HOST_CREDENTIALS = "vm2-creds"
+
+ // KEA specific
+ APP_NAME = "KEA"
+ APP_ENV = "production"
+ APP_DEBUG = "true"
+ APP_URL = "https://kea.curex-project.eu"
+
+ KIBANA_PORT = "5601"
+ GRAFANA_PORT = "3000"
+
+ LOG_CHANNEL = "stack"
+
+ DB_CONNECTION = "pgsql"
+ DB_HOST = "timescaledb"
+ DB_PORT = "5432"
+ DB_DATABASE = "kea"
+ DB_USERNAME = "postgres"
+ DB_PASSWORD = "postgres"
+
+ BROADCAST_DRIVER = "log"
+ CACHE_DRIVER = "file"
+ QUEUE_CONNECTION = "sync"
+ SESSION_DRIVER = "file"
+ SESSION_LIFETIME = "120"
+
+ JWT_TTL = "1440"
+
+ ELASTICSEARCH_HOST = "elasticsearch"
+ ELASTICSEARCH_PORT = "9200"
+ ELASTICSEARCH_SCHEME = "http"
+
+ MQTT_HOST = "mosquitto"
+ MQTT_PORT = "1883"
+ MQTT_DEBUG = "false"
+ MQTT_QOS = "0"
+ MQTT_RETAIN = "0"
+
+ MLTD_HOST = "mltd"
+ MLTD_PORT = "5000"
+ OD_HOST = "od"
+ OD_PORT = "9091"
+ }
+
+ stages {
+
+ stage('Checkout the source code') {
+ steps {
+ checkout scm
+ }
+ }
+
+
+ stage('Deploy Docker containers in DEV server') {
+ steps {
+ script {
+ docker.withServer("$DEPLOYMENT_HOST", "$DEPLOYMENT_HOST_CREDENTIALS") {
+ docker.withRegistry("$PRIVATE_REGISTRY" , 'artifactory') {
+ echo 'Deploying the specified Docker containers in DEV server'
+ sh 'sh deploy_hetzner.sh'
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
--
2.2.2