Thanasis Naskos / CUREX - Knowledge Extraction Analytics - KEA

Commit 35f613b81a4245612f0134a0539e663be3f74d81

Authored by George Vlahavas over 3 years ago

Exists in master and in 2 other branches

Add Hetzner deployment scripts

Showing 4 changed files with 410 additions and 0 deletions

deploy_hetzner.sh
docker-compose-hetzner.yml
hetzner_delete.groovy
hetzner_deploy.groovy

	1	+#!/bin/sh
	2	+
	3	+dump_logs() {
	4	+ CONTAINERS="suricata elasticsearch logstash kibana webserver api od mltd mqtt timescaledb grafana"
	5	+ for CONTAINER in $CONTAINERS; do
	6	+ echo "* Logs for kea_$CONTAINER (start) *"
	7	+ docker logs kea_$CONTAINER
	8	+ echo "* Logs for kea_$CONTAINER (end) *"
	9	+ done
	10	+}
	11	+
	12	+# Login into the CUREX registry
	13	+docker login https://registry.curex-project.eu:443/artifactory/curex-local/
	14	+
	15	+# pull all containers
	16	+docker-compose \
	17	+ -f docker-compose-hetzner.yml \
	18	+ pull
	19	+
	20	+sleep 1
	21	+
	22	+# Keep the timestamp of when starting up containers
	23	+TIME_START=`date +%s`
	24	+
	25	+# Just in case something is up from previous runs
	26	+echo "Bringing down any previous containers that are still running..."
	27	+./delete.sh
	28	+
	29	+# Run containers
	30	+echo "Bringing up containers..."
	31	+docker-compose -f docker-compose-hetzner.yml up --detach
	32	+
	33	+# loop until all containers are up
	34	+echo "Checking if all containers are up..."
	35	+while true; do
	36	+ # if any container other than composer has exited, bail out
	37	+ res=`docker ps -a \| \
	38	+ grep "/curex-local/kea_" \| \
	39	+ grep "Exited" \| \
	40	+ wc -l`
	41	+ if [ $res -gt 0 ]; then
	42	+ echo "ERROR: Some containers have exited."
	43	+ docker ps -a \| grep "^CONTAINER\\|/curex-local/kea_"
	44	+ dump_logs
	45	+ docker-compose -f docker-compose-hetzner.yml down
	46	+ exit 1
	47	+ fi
	48	+ # count containers that are up
	49	+ res=`docker ps -a \| \
	50	+ grep "/curex-local/kea_" \| \
	51	+ grep "Up" \| \
	52	+ wc -l`
	53	+ if [ $res -eq 11 ]; then
	54	+ echo "All good!"
	55	+ break
	56	+ fi
	57	+ # only wait for 10 mins until everything is up. It should be more than
	58	+ # enough.
	59	+ TIME_NOW=`date +%s`
	60	+ if [ $((TIME_NOW - TIME_START)) -ge 600 ]; then
	61	+ echo "ERROR: Timeout exceeded."
	62	+ dump_logs
	63	+ docker-compose -f docker-compose-hetzner.yml down
	64	+ exit 2
	65	+ fi
	66	+ sleep 1
	67	+done
	68	+
	69	+# Wait a bit more
	70	+sleep 60
	71	+
	72	+# create and populate the database
	73	+echo "Creating and populating the database..."
	74	+docker-compose -f docker-compose-hetzner.yml exec -T api php artisan migrate:fresh --seed --force

	1	+version: '3.7'
	2	+
	3	+services:
	4	+
	5	+ #CEPTD
	6	+ suricata:
	7	+ image: registry.curex-project.eu:443/curex-local/kea_suricata:latest
	8	+ container_name: kea_suricata
	9	+ restart: unless-stopped
	10	+ network_mode: "host"
	11	+ cap_add:
	12	+ - NET_ADMIN
	13	+ - SYS_NICE
	14	+ - NET_RAW
	15	+ command: ONLINE
	16	+ volumes:
	17	+ - data_volume:/var/log/suricata
	18	+ - pcap_volume:/var/pcap
	19	+ networks:
	20	+ - "proxy-network"
	21	+
	22	+ elasticsearch:
	23	+ image: registry.curex-project.eu:443/curex-local/kea_elasticsearch:latest
	24	+ container_name: kea_elasticsearch
	25	+ restart: unless-stopped
	26	+ environment:
	27	+ - cluster.name=keacluster
	28	+ - node.name=keacluster-node-01
	29	+ - discovery.type=single-node
	30	+ - bootstrap.memory_lock=true
	31	+ - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
	32	+ - xpack.ml.enabled=false
	33	+ - xpack.security.enabled=false
	34	+ - xpack.ilm.enabled=false
	35	+ - path.logs=/data/elk/log
	36	+ - path.data=/data/elk/data
	37	+ - http.host=0.0.0.0
	38	+ - http.cors.enabled=true
	39	+ - http.cors.allow-origin="*"
	40	+ - indices.query.bool.max_clause_count=2000
	41	+ ulimits:
	42	+ memlock:
	43	+ soft: -1
	44	+ hard: -1
	45	+ volumes:
	46	+ - data_volume:/data
	47	+ networks:
	48	+ - "proxy-network"
	49	+
	50	+ logstash:
	51	+ image: registry.curex-project.eu:443/curex-local/kea_logstash:latest
	52	+ container_name: kea_logstash
	53	+ restart: unless-stopped
	54	+ volumes:
	55	+ - data_volume:/data
	56	+ networks:
	57	+ - "proxy-network"
	58	+
	59	+ kibana:
	60	+ image: registry.curex-project.eu:443/curex-local/kea_kibana:latest
	61	+ container_name: kea_kibana
	62	+ restart: unless-stopped
	63	+ expose:
	64	+ - 5601
	65	+ networks:
	66	+ - "proxy-network"
	67	+
	68	+ #Controller
	69	+ webserver:
	70	+ image: registry.curex-project.eu:443/curex-local/kea_webserver:latest
	71	+ container_name: kea_webserver
	72	+ restart: unless-stopped
	73	+ depends_on:
	74	+ - api
	75	+ tty: true
	76	+ expose:
	77	+ - 80
	78	+ volumes:
	79	+ - api_volume:/var/www
	80	+ networks:
	81	+ - "proxy-network"
	82	+ environment:
	83	+ - "VIRTUAL_HOST=kea.curex-project.eu"
	84	+ - "VIRTUAL_PORT=80"
	85	+ - "LETSENCRYPT_HOST=kea.curex-project.eu"
	86	+ - "LETSENCRYPT_EMAIL=gvlahavas@csd.auth.gr"
	87	+
	88	+ api:
	89	+ image: registry.curex-project.eu:443/curex-local/kea_api:latest
	90	+ container_name: kea_api
	91	+ restart: unless-stopped
	92	+ tty: true
	93	+ working_dir: /var/www
	94	+ volumes:
	95	+ - api_volume:/var/www
	96	+ environment:
	97	+ - "APP_NAME=${APP_NAME}"
	98	+ - "APP_ENV=${APP_ENV}"
	99	+ - "APP_DEBUG=${APP_DEBUG}"
	100	+ - "APP_URL=${APP_URL}"
	101	+ - "KIBANA_PORT=${KIBANA_PORT}"
	102	+ - "GRAFANA_PORT=${GRAFANA_PORT}"
	103	+ - "LOG_CHANNEL=${LOG_CHANNEL}"
	104	+ - "DB_CONNECTION=${DB_CONNECTION}"
	105	+ - "DB_HOST=${DB_HOST}"
	106	+ - "DB_PORT=${DB_PORT}"
	107	+ - "DB_DATABASE=${DB_DATABASE}"
	108	+ - "DB_USERNAME=${DB_USERNAME}"
	109	+ - "DB_PASSWORD=${DB_PASSWORD}"
	110	+ - "BROADCAST_DRIVER=${BROADCAST_DRIVER}"
	111	+ - "CACHE_DRIVER=${CACHE_DRIVER}"
	112	+ - "QUEUE_CONNECTION=${QUEUE_CONNECTION}"
	113	+ - "SESSION_DRIVER=${SESSION_DRIVER}"
	114	+ - "SESSION_LIFETIME=${SESSION_LIFETIME}"
	115	+ - "ELASTICSEARCH_HOST=${ELASTICSEARCH_HOST}"
	116	+ - "ELASTICSEARCH_PORT=${ELASTICSEARCH_PORT}"
	117	+ - "ELASTICSEARCH_SCHEME=${ELASTICSEARCH_SCHEME}"
	118	+ - "MQTT_HOST=${MQTT_HOST}"
	119	+ - "MQTT_PORT=${MQTT_PORT}"
	120	+ - "MQTT_DEBUG=${MQTT_DEBUG}"
	121	+ - "MQTT_QOS=${MQTT_QOS}"
	122	+ - "MQTT_RETAIN=${MQTT_RETAIN}"
	123	+ - "MLTD_HOST=${MLTD_HOST}"
	124	+ - "MLTD_PORT=${MLTD_PORT}"
	125	+ - "OD_HOST=${OD_HOST}"
	126	+ - "OD_PORT=${OD_PORT}"
	127	+ networks:
	128	+ - "proxy-network"
	129	+
	130	+ #OD
	131	+ od:
	132	+ image: registry.curex-project.eu:443/curex-local/kea_od:latest
	133	+ container_name: kea_od
	134	+ depends_on:
	135	+ - timescaledb
	136	+ restart: unless-stopped
	137	+ expose:
	138	+ - 9091
	139	+ networks:
	140	+ - "proxy-network"
	141	+
	142	+ #MLTD
	143	+ mltd:
	144	+ image: registry.curex-project.eu:443/curex-local/kea_mltd:latest
	145	+ container_name: kea_mltd
	146	+ depends_on:
	147	+ - timescaledb
	148	+ - mosquitto
	149	+ restart: unless-stopped
	150	+ expose:
	151	+ - 5000
	152	+ networks:
	153	+ - "proxy-network"
	154	+
	155	+ #MQTT
	156	+ mosquitto:
	157	+ image: registry.curex-project.eu:443/curex-local/kea_mqtt:latest
	158	+ container_name: kea_mqtt
	159	+ expose:
	160	+ - 1883
	161	+ - 9001
	162	+ volumes:
	163	+ - mqtt_data_volume:/mosquitto/data
	164	+ - mqtt_log_volume:/mosquitto/log
	165	+ networks:
	166	+ - "proxy-network"
	167	+
	168	+ #Persistence
	169	+ timescaledb:
	170	+ image: registry.curex-project.eu:443/curex-local/kea_timescaledb:latest
	171	+ container_name: kea_timescaledb
	172	+ volumes:
	173	+ - timescaledb_volume:/var/lib/postgresql/data
	174	+ environment:
	175	+ - POSTGRES_USER=postgres
	176	+ - POSTGRES_PASSWORD=postgres
	177	+ - POSTGRES_DB=kea
	178	+ expose:
	179	+ - 5432
	180	+ networks:
	181	+ - "proxy-network"
	182	+
	183	+ #Visualization
	184	+ grafana:
	185	+ image: registry.curex-project.eu:443/curex-local/kea_grafana:latest
	186	+ container_name: kea_grafana
	187	+ expose:
	188	+ - 3000
	189	+ environment:
	190	+ - GF_SECURITY_ALLOW_EMBEDDING=true
	191	+ - GF_SECURITY_COOKIE_SAMESITE=none
	192	+ - GF_AUTH_ANONYMOUS_ENABLED=true
	193	+ - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
	194	+ networks:
	195	+ - "proxy-network"
	196	+
	197	+networks:
	198	+ proxy-network:
	199	+ external:
	200	+ name: proxy-network
	201	+
	202	+
	203	+volumes:
	204	+ api_volume:
	205	+ data_volume:
	206	+ pcap_volume:
	207	+ mqtt_data_volume:
	208	+ mqtt_log_volume:
	209	+ timescaledb_volume:

	1	+/**
	2	+* Jenkinsfile to deploy multiple Docker containers based on docker-compose into a DEV server and run any test.
	3	+* This pipeline will run the Docker containers, execute the tests and then stop and remove the containers from the DEV
	4	+* server automatically.
	5	+**/
	6	+
	7	+
	8	+pipeline {
	9	+
	10	+ agent any
	11	+
	12	+ environment {
	13	+ // Keep as is the following two lines
	14	+ PRIVATE_REGISTRY = "https://registry.curex-project.eu:443/curex-local/"
	15	+ ARTIFACTORY_URL = "registry.curex-project.eu:443/curex-local/"
	16	+
	17	+ HOST_IP = "116.203.166.220"
	18	+ DEPLOYMENT_HOST = "${HOST_IP}:2376"
	19	+ DEPLOYMENT_HOST_CREDENTIALS = "vm2-creds"
	20	+ }
	21	+
	22	+ stages {
	23	+
	24	+ stage('Stop and remove the Docker containers in DEV server') {
	25	+ steps {
	26	+ script {
	27	+ docker.withServer("$DEPLOYMENT_HOST", "$DEPLOYMENT_HOST_CREDENTIALS") {
	28	+ docker.withRegistry("$PRIVATE_REGISTRY" , 'artifactory') {
	29	+
	30	+ //echo 'Stop and remove the specified Docker containers from the DEV server'
	31	+ sh 'sh delete.sh'
	32	+ }
	33	+ }
	34	+ }
	35	+ }
	36	+ }
	37	+ }
	38	+}

	1	+/**
	2	+* Jenkinsfile to deploy multiple Docker containers based on docker-compose into a DEV server and run any test.
	3	+* This pipeline will run the Docker containers, execute the tests and then stop and remove the containers from the DEV
	4	+* server automatically.
	5	+**/
	6	+
	7	+
	8	+pipeline {
	9	+
	10	+ agent any
	11	+
	12	+ environment {
	13	+ // Keep as is the following two lines
	14	+ PRIVATE_REGISTRY = "https://registry.curex-project.eu:443/curex-local/"
	15	+ ARTIFACTORY_URL = "registry.curex-project.eu:443/curex-local/"
	16	+
	17	+ HOST_IP = "116.203.166.220"
	18	+ DEPLOYMENT_HOST = "${HOST_IP}:2376"
	19	+ DEPLOYMENT_HOST_CREDENTIALS = "vm2-creds"
	20	+
	21	+ // KEA specific
	22	+ APP_NAME = "KEA"
	23	+ APP_ENV = "production"
	24	+ APP_DEBUG = "true"
	25	+ APP_URL = "https://kea.curex-project.eu"
	26	+
	27	+ KIBANA_PORT = "5601"
	28	+ GRAFANA_PORT = "3000"
	29	+
	30	+ LOG_CHANNEL = "stack"
	31	+
	32	+ DB_CONNECTION = "pgsql"
	33	+ DB_HOST = "timescaledb"
	34	+ DB_PORT = "5432"
	35	+ DB_DATABASE = "kea"
	36	+ DB_USERNAME = "postgres"
	37	+ DB_PASSWORD = "postgres"
	38	+
	39	+ BROADCAST_DRIVER = "log"
	40	+ CACHE_DRIVER = "file"
	41	+ QUEUE_CONNECTION = "sync"
	42	+ SESSION_DRIVER = "file"
	43	+ SESSION_LIFETIME = "120"
	44	+
	45	+ JWT_TTL = "1440"
	46	+
	47	+ ELASTICSEARCH_HOST = "elasticsearch"
	48	+ ELASTICSEARCH_PORT = "9200"
	49	+ ELASTICSEARCH_SCHEME = "http"
	50	+
	51	+ MQTT_HOST = "mosquitto"
	52	+ MQTT_PORT = "1883"
	53	+ MQTT_DEBUG = "false"
	54	+ MQTT_QOS = "0"
	55	+ MQTT_RETAIN = "0"
	56	+
	57	+ MLTD_HOST = "mltd"
	58	+ MLTD_PORT = "5000"
	59	+ OD_HOST = "od"
	60	+ OD_PORT = "9091"
	61	+ }
	62	+
	63	+ stages {
	64	+
	65	+ stage('Checkout the source code') {
	66	+ steps {
	67	+ checkout scm
	68	+ }
	69	+ }
	70	+
	71	+
	72	+ stage('Deploy Docker containers in DEV server') {
	73	+ steps {
	74	+ script {
	75	+ docker.withServer("$DEPLOYMENT_HOST", "$DEPLOYMENT_HOST_CREDENTIALS") {
	76	+ docker.withRegistry("$PRIVATE_REGISTRY" , 'artifactory') {
	77	+ echo 'Deploying the specified Docker containers in DEV server'
	78	+ sh 'sh deploy_hetzner.sh'
	79	+ }
	80	+ }
	81	+ }
	82	+ }
	83	+ }
	84	+ }
	85	+}