From 44780c8f4130c830a703b4aeb5de9d434eb7c006 Mon Sep 17 00:00:00 2001 From: Gilles Mouchet Date: Sun, 15 Jun 2025 18:16:05 +0200 Subject: [PATCH] dev #20250615-01 --- LICENSE | 33 + README.md | 132 ++-- .../local-config.yaml.tmpl} | 67 +-- config/minio/policy.json.tmpl | 33 + config/{ => nginx}/default.conf | 4 +- config/{ => prometheus}/prometheus.yml | 6 +- config/{ => promtail}/config.yml | 0 manage-old.sh | 191 ++++++ manage.sh | 569 ++++++++++++++++-- 9 files changed, 827 insertions(+), 208 deletions(-) create mode 100644 LICENSE rename config/{local-config.yaml => loki/local-config.yaml.tmpl} (65%) create mode 100644 config/minio/policy.json.tmpl rename config/{ => nginx}/default.conf (95%) rename config/{ => prometheus}/prometheus.yml (68%) rename config/{ => promtail}/config.yml (100%) create mode 100755 manage-old.sh diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..49527cd --- /dev/null +++ b/LICENSE @@ -0,0 +1,33 @@ +Non-Commercial Use License – [manage.sh] + +Copyright (c) [2025] [Gilles Mouchet] + +This script is provided free of charge with its source code. You are permitted to: + +Use the script for personal, educational, or non-commercial professional purposes. +Study, modify, and share the script freely, provided this license is included. + +You are strictly prohibited from: + +- Selling this script or any modified version. +- Using it in commercial services or products. +- Distributing it in exchange for financial compensation, directly or indirectly. + +This script is provided "as is", without any warranty of any kind. +----------------------------------------------------------------------------------------- +Licence d’utilisation non commerciale – [manage.sh] + +Copyright (c) [2025] [Gilles Mouchet] + +Ce script est fourni gratuitement avec son code source. Toute personne est autorisée à : + +Utiliser le script à des fins personnelles, éducatives ou professionnelles non commerciales. +Étudier, modifier et partager le script gratuitement, à condition de conserver cette licence. + +Il est strictement interdit de : + +- Vendre ce script ou une version modifiée. +- L’utiliser dans des services ou produits commerciaux. +- Le distribuer en échange d’une contrepartie financière, directe ou indirecte. + +Ce script est fourni "tel quel", sans garantie d’aucune sorte. diff --git a/README.md b/README.md index 706cada..55b9a57 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,14 @@ This dokcer run stack grafana, loki, promtail, minio | -minio-cli-1 | - | client minio | | -prometheus-1 | `http://:9090` | prometheus | | -node-exporter-1 | - | node metrics for prometheus | +| -promtail-1 | - | promtail | +| -loki-1 | - | loki | +| -log-generator-1 | - | fake log generator | ## Requirements ### Certificats -Create a folder named `/home/docker/certs` and copy your certificates into it. +Create a folder named `/home/docker/certs` and copy your certificates into it. +IMPORTANT: Certificates must be named `tls.crt` and `tls.key` + ```bash sudo mkdir -p /home/docker/certs ``` @@ -24,69 +29,33 @@ All variables are described in the `.env.dist` file Copy the `.env.dist` file to `.env` and update it with values appropriate for your setup. -### Proxy (nginx) -Create the folder where the ngnix config will be stored and copy the `config/nginx.conf` into it -```bash -sudo mkdir -p /home/docker/nginx -sudo cp ./config/default.conf /home/docker/nginx/. -``` - -### Minio -Create the folder where the minio data will be stored -```bash -sudo mkdir -p /home/docker/minio -``` - -### Minio client -Create the folder where the minio client comfig will be stored -```bash -sudo mkdir -p /home/docker/minio-cli -``` - -### Grafana -Create the folder where the grafana data will be stored -```bash -sudo mkdir -p /home/docker/grafana -``` - -### Prometheus -Create the folder where the grafana data will be stored -```bash -sudo mkdir -p /home/docker/prometheus -``` -Create the folder where the prometheus data will be stored -```bash -sudo mkdir -p /home/docker/prometheus-data -``` - -### Loki -Create the folder where the loki config will be stored -```bash -sudo mkdir -p /home/docker/loki -``` - -Create the folder where the loki data will be stored -```bash -sudo mkdir -p /home/docker/loki-data -``` - -Create the folder where the loki log generated will be stored -```bash -sudo mkdir -p /home/docker/loki-gen-log -``` - -### Promtail -Create the folder where the promtail config will be stored -```bash -sudo mkdir -p /home/docker/promtail -``` - ## Run + +Create folder for persistents volumes and copy config files +To be executed each time the configuration files are modified ```bash -docker compose up -d +./manage.sh config ``` +Create container +```bash +./manage.sh create +``` + +Start the stack +```bash +./manage.sh start +``` + +For more information about `manage.sh` script execute `./manage.sh --help` + ## Config +### Prometheus +#### grafana source +`http://prometheus:9090` +#### grafana dashboard +3662 - Prometheus 2.0 Overview + ### Minio client Create an alias ```bash @@ -98,9 +67,8 @@ Create prometheus metrics job docker exec -it -minio-cli-1 /bin/sh -c "mc admin prometheus generate myminio" ``` -Copy the output into prometheus.yml +Copy the output into `/prometheus.yml ```yaml -scrape_configs: - job_name: minio-job bearer_token: eyJhbGciOiJIUzUxMiIs..InR5cCI6IkpXVCJ9 metrics_path: /minio/v2/metrics/cluster @@ -113,20 +81,33 @@ Restart prometheus ```bash docker compose restart prometheus ``` - -Add dashboard ***13502 - MinIO Dashboard*** to grafana - -### Prometheus -#### grafana source -`http://prometheus:9090` - #### grafana dashboard -3662 - Prometheus 2.0 Overview +13502 - MinIO Dashboard + +### Loki +Create bulk loki +```bash +docker exec -it -minio-cli-1 /bin/sh -c "mc mb myminio/loki" +``` +#### grafana source +`http://loki:3100` +#### grafana dashboard +13639 - Logs / App ### Node exporter #### grafana dashboard 1860 - Node Exporter Full +## Fake log generator +Use for testing only +```bash +docker compose up log-generator -d +``` +On grafana select - Explore - Loki (to the right of Outline) - Label filters job=generated-logs +```bash +docker compose stop log-generator +``` + ## Access **dns-tools** @@ -166,7 +147,7 @@ docker exec -it -proxy-1 /bin/sh -c "curl http://prometheu TOKEN=$(echo "Authorization: Bearer $(grep "bearer_token" config/prometheus.yml | cut -d':' -f2 |cut -d' ' -f2)") ``` ```bash -docker exec -it gmo-loki-proxy-1 /bin/bash -c "curl -H \"Accept: application/json\" -H \"${TOKEN}\" http://minio:9000/minio/v2/metrics/cluster" +docker exec -it -proxy-1 /bin/bash -c "curl -H \"Accept: application/json\" -H \"${TOKEN}\" http://minio:9000/minio/v2/metrics/cluster" ``` ## Sources @@ -187,6 +168,15 @@ docker exec -it gmo-loki-proxy-1 /bin/bash -c "curl -H \"Accept: application/jso * https://medium.com/@netopschic/implementing-the-log-monitoring-stack-using-promtail-loki-and-grafana-using-docker-compose-bcb07d1a51aa ## Changelog +### [1.2.0] - 2025-06-14 +#### Added +- promtail container +- loki container +- log generator container +#### Created +- script manage.sh +#### Fixed +- certificates names ### [1.1.0] - 2025-06-09 #### Added - prometheus container @@ -195,7 +185,7 @@ docker exec -it gmo-loki-proxy-1 /bin/bash -c "curl -H \"Accept: application/jso - 1860 - Node Exporter Full - 3662 - Prometheus 2.0 Overview - 13502 - MinIO Dashboard - - 13639 Loki - Log analysis + - 13639 Logs / APP ### [1.0.0] - 2025-06-08 #### Added diff --git a/config/local-config.yaml b/config/loki/local-config.yaml.tmpl similarity index 65% rename from config/local-config.yaml rename to config/loki/local-config.yaml.tmpl index 270edc0..10593ba 100644 --- a/config/local-config.yaml +++ b/config/loki/local-config.yaml.tmpl @@ -1,66 +1,3 @@ -# -#auth_enabled: false -# -#server: -# http_listen_port: 3100 -# -#common: -# ring: -# instance_addr: 127.0.0.1 -# kvstore: -# store: inmemory -# path_prefix: /loki -# compactor_address: http://loki:3100 -# replication_factor: 1 -# -#schema_config: -# configs: -# - from: 2020-08-01 -# store: boltdb-shipper -# object_store: s3 -# schema: v11 -# index: -# prefix: index_ -# period: 24h -# - from: 2023-07-11 -# store: tsdb -# object_store: s3 -# schema: v12 -# index: -# prefix: index_ -# period: 24h -# - from: 2024-01-10 -# store: tsdb -# object_store: s3 -# schema: v12 -# index: -# prefix: index_ -# period: 24h -# - from: 2024-03-29 -# store: tsdb -# object_store: s3 -# schema: v13 -# index: -# prefix: index_ -# period: 24h -# -# -#storage_config: -# boltdb_shipper: -# active_index_directory: /loki/index -# cache_location: /loki/index_cache -# resync_interval: 5s -# shared_store: s3 -# aws: -# s3: http://minio:9000/loki -# bucketnames: loki -# endpoint: minio:9000 -# access_key_id: admin -# secret_access_key: pa55w0rd -# insecure: true # Important si MinIO ne supporte pas HTTPS -# s3forcepathstyle: true - -# FROM AI auth_enabled: false server: @@ -123,8 +60,8 @@ storage_config: aws: bucketnames: loki endpoint: http://minio:9000 - access_key_id: admin - secret_access_key: pa55w0rd + access_key_id: ${MINIO_BUCKET_USER} + secret_access_key: ${MINIO_BUCKET_PASS} insecure: true s3forcepathstyle: true diff --git a/config/minio/policy.json.tmpl b/config/minio/policy.json.tmpl new file mode 100644 index 0000000..fdac0ff --- /dev/null +++ b/config/minio/policy.json.tmpl @@ -0,0 +1,33 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:PutBucketPolicy", + "s3:GetBucketPolicy", + "s3:DeleteBucketPolicy", + "s3:ListAllMyBuckets", + "s3:ListBucket" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::${MINIO_BUCKET_NAME}" + ], + "Sid": "" + }, + { + "Action": [ + "s3:AbortMultipartUpload", + "s3:DeleteObject", + "s3:GetObject", + "s3:ListMultipartUploadParts", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::${MINIO_BUCKET_NAME}/*" + ], + "Sid": "" + } + ] +} \ No newline at end of file diff --git a/config/default.conf b/config/nginx/default.conf similarity index 95% rename from config/default.conf rename to config/nginx/default.conf index 558ae13..49fc06b 100644 --- a/config/default.conf +++ b/config/nginx/default.conf @@ -13,8 +13,8 @@ server { listen 8443 ssl; #listen [::]:8443; server_name _; - ssl_certificate /etc/nginx/certs/vdglab.net.crt; - ssl_certificate_key /etc/nginx/certs/vdglab.net.key; + ssl_certificate /etc/nginx/certs/tls.crt; + ssl_certificate_key /etc/nginx/certs/tls.key; # allow special characters in headers ignore_invalid_headers off; # allow any size file to be uploaded. diff --git a/config/prometheus.yml b/config/prometheus/prometheus.yml similarity index 68% rename from config/prometheus.yml rename to config/prometheus/prometheus.yml index 34b302f..a34483a 100644 --- a/config/prometheus.yml +++ b/config/prometheus/prometheus.yml @@ -11,11 +11,9 @@ scrape_configs: - targets: - node-exporter:9100 - job_name: minio-job - bearer_token: eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJwcm9tZXRoZXVzIiwic3ViIjoiYWRtaW4iLCJleHAiOjQ5MDI5NzQ1Mzh9.BuzZdu0W1TBfolWIMUPYhuRHuAFfrHuLw0CYMCK3NhqWzmx9u_P6uu3rJl4R71JlUhTDU31U3q4-mh9Ev7aOjQ + bearer_token: eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJwcm9tZXRoZXVzIiwic3ViIjoibWluaW8iLCJleHAiOjQ5MDM0OTIwNjN9.ailN0Xo5ZXOGDUIe_jrqHpNP2JlQKhi7Ss4XIPIa5M8Hjm7UGJuOKLRnpdysHCYo1cPwlWzMCaGpkfE0KbHmqA metrics_path: /minio/v2/metrics/cluster scheme: http static_configs: - - targets: ['minio:9000'] - - + - targets: ['minio:9000'] \ No newline at end of file diff --git a/config/config.yml b/config/promtail/config.yml similarity index 100% rename from config/config.yml rename to config/promtail/config.yml diff --git a/manage-old.sh b/manage-old.sh new file mode 100755 index 0000000..e8cf2fa --- /dev/null +++ b/manage-old.sh @@ -0,0 +1,191 @@ +#!/bin/bash +############################################################ +# Decription: Script to manage stack docker +# +# Author: Gilles Mouchet (gilles.mouchet@gmail.com) +# Creation Date: 12-06-2025 +# Version: 1.0 +# Usage: ./manage.sh +# Changelog: +# V1.0 - dd-Mmm-2025 - GMo +# Added +# - Creation of script from scratch +# +############################################################ + +#----------------------------------------------------------------- +# Doesn't change anything from here +#----------------------------------------------------------------- +version="1.0-rc1" + +prog_name="./$(/bin/basename $0)" + +containers_to_start=(init minio minio-cli grafana loki promtail prometheus node-exporter proxy) +containers_all=("${containers_to_start[@]}" log-generator dns-tools) + +# Couleurs +VERT="\e[32m" +ROUGE="\e[31m" +NORMAL="\e[0m" +#----------------------------------------------------------------- +# Functions +#----------------------------------------------------------------- +# display help +function print_usage { + /bin/cat << EOF + +Usage: $prog_name [options] + +Options: + config Create folder for persit. vol. and copy conf files + param Display all vars defined in .env file + list Display container (servie) list define in docker compose file + create Create container + start Start stack '${COMPOSE_PROJECT_NAME}' + stop Stop stack '${COMPOSE_PROJECT_NAME}' + down Stop stack '${COMPOSE_PROJECT_NAME}' and delete containers + restart Restart stack '${COMPOSE_PROJECT_NAME}' + version,-v,--version Display script version + help,-h,--help Display this help + +Examples: + To start stack '${COMPOSE_PROJECT_NAME}' + $prog_name start + To stop stack '${COMPOSE_PROJECT_NAME}' + $prog_name stop + To start a specific container and his depend + docker compose up -d + Ex: + docker compose up minio-cli -d + To create a new stack + $prog_name config + $prog_name create + $prog_name start +EOF +} +# create all containers +function create_container { + + for cont_item in "${containers_all[@]}"; do + # check if container exist. If not create it + if [ $(docker ps -a | grep ${COMPOSE_PROJECT_NAME}-$cont_item | wc -l) -eq 0 ]; then + echo -n "Create container $cont_item: " + if docker compose create $cont_item > /dev/null 2>&1; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR${NORMAL}" + fi + else + echo "Container $cont_item already exist" + fi + done +} +# start container +function start_container { + for cont_item in "${containers_to_start[@]}"; do + echo -n "Start container $cont_item: " + if docker compose up -d $cont_item > /dev/null 2>&1; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR${NORMAL}" + fi + done +} + +#----------------------------------------------------------------- +# MAIN +#----------------------------------------------------------------- + +# read .env if exist +if [ ! -f .env ]; then + echo "file '.env' doesn't exist!" + echo "See README.md" + exit +fi +. .env + +#if [ ! -z "$1" ]; then +#print_usage +# exit 0 +#fi + +case "$1" in + config) + echo "Create folder for persistent volume" + sudo mkdir -p ${PRX_CERTS_DIR} + sudo mkdir -p ${PRX_NGINX_CONF_DIR} + sudo mkdir -p ${MINIO_DATA_ROOT_DIR} + sudo mkdir -p ${MINIO_CONF_CLI_DIR} + sudo mkdir -p ${GF_DATA_DIR} + sudo mkdir -p ${PROM_CONF_DIR} + sudo mkdir -p ${PROM_DATA_DIR} + sudo mkdir -p ${LOKI_CONF_DIR} + sudo mkdir -p ${LOKI_DATA_DIR} + sudo mkdir -p ${LOKI_GEN_LOG_DIR} + sudo mkdir -p ${PROMTAIL_CONF_DIR} + echo "Copy configs files" + sudo cp ./config/nginx/default.conf ${PRX_NGINX_CONF_DIR}/. + sudo cp ./config/prometheus/prometheus.yml ${PROM_CONF_DIR}/. + sudo cp ./config/loki/local-config.yaml ${LOKI_CONF_DIR}/. + sudo cp ./config/promtail/config.yml ${PROMTAIL_CONF_DIR}/. + ;; + param) + cat << EOF +COMPOSE_PROJECT_NAME = ${COMPOSE_PROJECT_NAME} + PRX_CERTS_DIR = ${PRX_CERTS_DIR} + PRX_NGINX_CONF_DIR = ${PRX_NGINX_CONF_DIR} + MINIO_DATA_ROOT_DIR = ${MINIO_DATA_ROOT_DIR} + MINIO_CONF_CLI_DIR = ${MINIO_CONF_CLI_DIR} + MINIO_ACCESS_KEY = ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY = ${MINIO_SECRET_KEY} + MINIO_REDIRECT_URL = ${MINIO_REDIRECT_URL} + GF_ADMIN_USER = ${GF_ADMIN_USER} + GF_ADMIN_PASS = ${GF_ADMIN_PASS} + GF_DATA_DIR = ${GF_DATA_DIR} + GF_ROOT_URL = ${GF_ROOT_URL} + PROM_CONF_DIR = ${PROM_CONF_DIR} + PROM_DATA_DIR = ${PROM_DATA_DIR} + LOKI_CONF_DIR = ${LOKI_CONF_DIR} + LOKI_DATA_DIR = ${LOKI_DATA_DIR} + LOKI_GEN_LOG_DIR = ${LOKI_GEN_LOG_DIR} + PROMTAIL_CONF_DIR = ${PROMTAIL_CONF_DIR} +EOF + ;; + create) + create_container + ;; + start) + create_container + start_container + ;; + stop) + docker compose stop + ;; + down) + docker compose down + ;; + list) + echo "-------------------------------------" + echo " Container list" + echo "-------------------------------------" + docker compose config --services | sort + #for cont_item in "${containers_all[@]}"; do + # echo "- $cont_item" + #done + ;; + -h|--help|help) + print_usage + exit 0 + ;; + -v|--version|version) + cat << EOF +$(basename "$0") v$version (c) 1990 - $(date +%Y) by Gilles Mouchet +Non-Commercial Use License – See LICENSE for details +EOF + exit 0 + ;; + *) + print_usage + exit 0 + ;; + esac diff --git a/manage.sh b/manage.sh index 42115e7..aa2c130 100755 --- a/manage.sh +++ b/manage.sh @@ -5,9 +5,7 @@ # Author: Gilles Mouchet (gilles.mouchet@gmail.com) # Creation Date: 12-06-2025 # Version: 1.0 -# Install: If applicable - Install instruction or -# see README.md -# Usage: If applicable - usage or see README.md +# Usage: ./manage.sh # Changelog: # V1.0 - dd-Mmm-2025 - GMo # Added @@ -18,24 +16,32 @@ #----------------------------------------------------------------- # Doesn't change anything from here #----------------------------------------------------------------- -version="v1.0-rc1" +version="1.0-rc1" -progName="./$(/bin/basename $0)" +prog_name="./$(/bin/basename $0)" -containers=(minio grafana proxy titi) +containers_to_start=(init minio minio-cli grafana loki promtail prometheus node-exporter proxy) +containers_all=("${containers_to_start[@]}" log-generator dns-tools) + +# Couleurs +VERT="\e[32m" +ROUGE="\e[31m" +NORMAL="\e[0m" #----------------------------------------------------------------- # Functions #----------------------------------------------------------------- # display help function print_usage { -# print -/bin/cat << EOF + /bin/cat << EOF -Usage: $progName [options] +Usage: $prog_name [options] Options: + config Create folder for persit. vol. and copy conf files + param Display all vars defined in .env file + list Display container (servie) list define in docker compose file create Create container - start Start stack '${COMPOSE_PROJECT_NAME}' + start-stack Start stack '${COMPOSE_PROJECT_NAME}' stop Stop stack '${COMPOSE_PROJECT_NAME}' down Stop stack '${COMPOSE_PROJECT_NAME}' and delete containers restart Restart stack '${COMPOSE_PROJECT_NAME}' @@ -43,71 +49,502 @@ Options: help,-h,--help Display this help Examples: - To start stack '${COMPOSE_PROJECT_NAME}' - $progName start - To stop stack '${COMPOSE_PROJECT_NAME}' - $progName stop - + To start stack '${COMPOSE_PROJECT_NAME}' + $prog_name start + To stop stack '${COMPOSE_PROJECT_NAME}' + $prog_name stop + To start a specific container and his depend + docker compose up -d + Ex: + docker compose up minio-cli -d To create a new stack - $progName create - $progName start + $prog_name config + $prog_name create + $prog_name start EOF } +#----------------------------------------------------------------- +# create folders for persistent volume and config file +function create_folder { + for folder_item in "${folder_to_create[@]}"; do + echo -n "Create folder '${folder_item}': " + sudo mkdir -p ${folder_item} + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR${NORMAL}" + exit $rc + fi + done +} +#----------------------------------------------------------------- +# remove folders for persistent volume and config file +function remove_folder { + for folder_item in "${folder_to_create[@]}"; do + # we doesn't delete the certs folder. + # this folder is maybe shared with other container + if [ "${folder_item}" != "${PRX_CERTS_DIR}" ]; then + echo -n "Remove folder '${folder_item}': " + sudo rm -rf ${folder_item} + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR${NORMAL}" + exit $rc + fi + fi + done +} +#----------------------------------------------------------------- +# copy config file +function copy_conf_file { + echo -n "Copy loki config file into ${LOKI_CONF_DIR}/: " + envsubst < ./config/loki/local-config.yaml.tmpl | \ + sudo tee ${LOKI_CONF_DIR}/local-config.yaml > /dev/null + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + + echo -n "Copy proxy (nginx) config file into ${PRX_NGINX_CONF_DIR}/: " + sudo cp ./config/nginx/default.conf ${PRX_NGINX_CONF_DIR}/. > /dev/null 2>&1 + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + + echo -n "Copy the Prometheus configuration file into ${PROM_CONF_DIR}/: " + sudo cp ./config/prometheus/prometheus.yml ${PROM_CONF_DIR}/. > /dev/null 2>&1 + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + + echo -n "Copy promtail config file into ${PROMTAIL_CONF_DIR}/: " + sudo cp ./config/promtail/config.yml ${PROMTAIL_CONF_DIR}/. > /dev/null 2>&1 + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + +} +#----------------------------------------------------------------- +# create all containers +function create_container { + for cont_item in "${containers_all[@]}"; do + # check if container exist. If not create it + if [ $(docker ps -a | grep ${COMPOSE_PROJECT_NAME}-$cont_item | wc -l) -eq 0 ]; then + echo -n "Create container $cont_item: " + if docker compose create $cont_item > /dev/null 2>&1; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR${NORMAL}" + fi +# else +# echo "Container $cont_item already exist" + fi + done +} +#----------------------------------------------------------------- +# start container +function start_container { + for cont_item in "${containers_to_start[@]}"; do + echo -n "Start container $cont_item: " + if docker compose up -d $cont_item > /dev/null 2>&1; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR${NORMAL}" + fi + done +} +#----------------------------------------------------------------- +# create grafana ds +# $1 = ds name +# $2 = url +function add_gf_data_source { + curl -k -X POST "${GF_ROOT_URL}/api/datasources" \ + -u "${GF_ADMIN_USER}:${GF_ADMIN_PASS}" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "'"${1^}"'", + "type": "'"$1"'", + "access": "proxy", + "url": "'"$2"'", + "basicAuth": false, + "isDefault": false, + "jsonData": { + "maxLines": 1000 + } + }' > /dev/null 2>&1 +} +#----------------------------------------------------------------- +# import grafana dashboard +# $1 = dashboard ID +# $2 = source +# $3 = dashboard title. Needed for check if already impoerted +function import_gf_dashboard { + db_file=/tmp/dashboard-$1.json + wr_file=/tmp/wrapped-dashboard.json + + #download json + curl -s -k -o "$db_file" "https://grafana.com/api/dashboards/$1/revisions/latest/download" + if [ ! -s "$db_file" ]; then + exit 1 + fi + + # stage 2 - clean up and add datadource + sed -i 's/"id":[ ]*[0-9]\+/"id": null/g' "$db_file" + sed -i 's/"uid":[ ]*"[^"]*"/"uid": null/g' "$db_file" + sed -i 's/"datasource": ".*"/"datasource": "'"${2^}"'"/g' "$db_file" + + # stage 3 - creating the final JSON file expected by the Grafana API + echo '{' > "$wr_file" + echo '"dashboard":' >> "$wr_file" + cat "$db_file" >> "$wr_file" + echo ',' >> "$wr_file" + echo '"overwrite": true' >> "$wr_file" + echo '}' >> "$wr_file" + + # stage 4 - import dashboard into grafana + curl -s -k -X POST "$GF_ROOT_URL/api/dashboards/db" \ + -u "$GF_ADMIN_USER:$GF_ADMIN_PASS" \ + -H "Content-Type: application/json" \ + --data-binary "@$wr_file" > /dev/null 2>&1 + + #rm -f $db_file $wr_file +} +#----------------------------------------------------------------- +# MAIN +#----------------------------------------------------------------- + # read .env if exist if [ ! -f .env ]; then echo "file '.env' doesn't exist!" echo "See README.md" exit fi -. .env +set -a +source .env +set +a -#if [ ! -z "$1" ]; then -#print_usage -# exit 0 -#fi +# create array folder to create +folder_to_create=(${PRX_CERTS_DIR} + ${PRX_NGINX_CONF_DIR} + ${MINIO_DATA_ROOT_DIR} + ${MINIO_CONF_CLI_DIR} + ${GF_DATA_DIR} + ${PROM_CONF_DIR} + ${PROM_DATA_DIR} + ${LOKI_CONF_DIR} + ${LOKI_DATA_DIR} + ${LOKI_GEN_LOG_DIR} + ${PROMTAIL_CONF_DIR}) - case "$1" in - create) - docker compose create - ;; - start) - for cont_item in "${containers[@]}"; do - echo "Start container $cont_item" - docker compose up -d $cont_item --remove-orphans - echo $? - done - ;; - down) - docker compose down - ;; - -h|--help|help) - print_usage - exit 0 - ;; - -v|--version|version) - echo $version - ;; - *) - #echo "${progName}: invalid option -- '$1'!" - #echo -e "Try '$progName help' for more information.\n" - print_usage - exit 0 - ;; +case "$1" in + param) + cat << EOF + COMPOSE_PROJECT_NAME = ${COMPOSE_PROJECT_NAME} + PRX_CERTS_DIR = ${PRX_CERTS_DIR} + PRX_NGINX_CONF_DIR = ${PRX_NGINX_CONF_DIR} + MINIO_DATA_ROOT_DIR = ${MINIO_DATA_ROOT_DIR} + MINIO_CONF_CLI_DIR = ${MINIO_CONF_CLI_DIR} + MINIO_ACCESS_KEY = ${MINIO_ACCESS_KEY} + MINIO_SECRET_KEY = ${MINIO_SECRET_KEY} + MINIO_BUCKET_NAME = ${MINIO_BUCKET_NAME} + MINIO_BUCKET_USER = ${MINIO_BUCKET_USER} + MINIO_BUCKET_PASS = ${MINIO_BUCKET_PASS} +MINIO_BUCKET_POL_NAME = ${MINIO_BUCKET_POL_NAME} + MINIO_REDIRECT_URL = ${MINIO_REDIRECT_URL} + MINIO_ALIAS = ${MINIO_ALIAS} + GF_ADMIN_USER = ${GF_ADMIN_USER} + GF_ADMIN_PASS = ${GF_ADMIN_PASS} + GF_DATA_DIR = ${GF_DATA_DIR} + GF_ROOT_URL = ${GF_ROOT_URL} + PROM_CONF_DIR = ${PROM_CONF_DIR} + PROM_DATA_DIR = ${PROM_DATA_DIR} + LOKI_CONF_DIR = ${LOKI_CONF_DIR} + LOKI_DATA_DIR = ${LOKI_DATA_DIR} + LOKI_GEN_LOG_DIR = ${LOKI_GEN_LOG_DIR} + PROMTAIL_CONF_DIR = ${PROMTAIL_CONF_DIR} +EOF + ;; + start-stack) + echo -e "---- Create Folders ----" + create_folder + echo -e "\n---- Copy config file ----" + copy_conf_file + echo -e "\n---- Config minio ----" + #----- + echo -n "Start minio container: " + if docker compose up -d minio > /dev/null 2>&1; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR${NORMAL}" + fi + #---- + echo -n "Wait until minio container is started: " + while ! curl -s "http://$HOSTNAME:9000/minio/health/live" >/dev/null; do + sleep 1 + done + echo -e "${VERT}STARTED${NORMAL}" + #----- + echo -n "Start minio-cli container: " + docker compose up -d minio-cli > /dev/null 2>&1 + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + #----- + echo -n "Create minios alias: " + docker exec ${COMPOSE_PROJECT_NAME}-minio-cli-1 mc alias set ${MINIO_ALIAS} http://minio:9000 \ + ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY} > /dev/null 2>&1 + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + #----- + echo -n "Copy mino policy config file into ${MINIO_CONF_CLI_DIR}/: " + envsubst < ./config/minio/policy.json.tmpl | + sudo tee ${MINIO_CONF_CLI_DIR}/.mc/policy.json > /dev/null + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + #----- + echo -n "Create user for bucket ${MINIO_BUCKET_NAME}: " + docker exec ${COMPOSE_PROJECT_NAME}-minio-cli-1 mc admin user add \ + ${MINIO_ALIAS} ${MINIO_BUCKET_USER} ${MINIO_BUCKET_PASS} >/dev/null 2>&1 + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + #----- + echo -n "Create policy for access to bucket ${MINIO_BUCKET_NAME}: " + docker exec ${COMPOSE_PROJECT_NAME}-minio-cli-1 mc admin policy create \ + ${MINIO_ALIAS} ${MINIO_BUCKET_POL_NAME} /root/.mc/policy.json >/dev/null 2>&1 + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + #----- + echo -n "Attach policy to user ${MINIO_BUCKET_USER}: " + docker exec ${COMPOSE_PROJECT_NAME}-minio-cli-1 mc admin policy attach \ + ${MINIO_ALIAS} ${MINIO_BUCKET_POL_NAME} --user=${MINIO_BUCKET_USER} >/dev/null 2>&1 + rc=$? + if [ $rc -eq 0 ]; then + echo -e "${VERT}SUCCESS${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc)${NORMAL}" + exit $rc + fi + #----- + echo -n "Create bucket if not exist: " + docker exec ${COMPOSE_PROJECT_NAME}-minio-cli-1 mc ls "${MINIO_ALIAS}/$MINIO_BUCKET_NAME" >/dev/null 2>&1 + rc=$? + if [ $rc -ne 0 ]; then + docker exec -it ${COMPOSE_PROJECT_NAME}-minio-cli-1 /bin/sh -c \ + "mc mb ${MINIO_ALIAS}/${MINIO_BUCKET_NAME}" >/dev/null 2>&1 + rc1=$? + if [ $rc1 -eq 0 ]; then + echo -e "${VERT}SUCCESSFULLY CREATED${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc1)${NORMAL}" + exit $rc1 + fi + else + echo -e "${VERT}ALREADY EXISTING${NORMAL}" + fi + # ---- + echo -e "\n---- Create container ----" + create_container + echo -e "\n---- Start container ----" + start_container + echo -e "\n---- Create Grafana DS ----" + # ---- + # wait until grafan is up + echo -n "Wait until grafana is starting: " + timeout=60 # Max time to wait in seconds + interval=2 # Wait time between checks + elapsed=0 + while true; do + status_code=$(curl -k -s -o /dev/null -w "%{http_code}" \ + -u "$GF_ADMIN_USER:$GF_ADMIN_PASS" \ + "$GF_ROOT_URL/api/health") + + if [ "$status_code" -eq 200 ]; then + echo -e "${VERT}STARTED${NORMAL}" + break + fi + + if [ "$elapsed" -ge "$timeout" ]; then + echo -e "${ROUGE}ERROR ($status_code)${NORMAL}" + exit 1 + fi + + sleep "$interval" + elapsed=$((elapsed + interval)) + done + # ---- + echo -n "Create Loki DS: " + add_gf_data_source loki http://loki:3100 + rc1=$? + if [ $rc1 -eq 0 ]; then + echo -e "${VERT}SUCCESSFULLY CREATED${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc1)${NORMAL}" + exit $rc1 + fi + # ---- + echo -n "Create Prometheus DS: " + add_gf_data_source prometheus http://prometheus:9090 "" + rc1=$? + if [ $rc1 -eq 0 ]; then + echo -e "${VERT}SUCCESSFULLY CREATED${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc1)${NORMAL}" + exit $rc1 + fi + echo -e "\n---- Install Grafana dashboard ----" + # ---- + db_name="Node Exporter Full" + echo -n "Import '$db_name' dashboard: " + # check if already imported + response=$(curl -k -s -u "$GF_ADMIN_USER:$GF_ADMIN_PASS" "$GF_ROOT_URL/api/search?query=") + if echo "$response" | grep -iq "\"title\":\"$db_name\""; then + echo -e "${VERT}ALREADY IMPORTED${NORMAL}" + else + import_gf_dashboard 1860 prometheus + rc1=$? + if [ $rc1 -eq 0 ]; then + echo -e "${VERT}SUCCESSFULLY IMPORTED${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc1)${NORMAL}" + exit $rc1 + fi + fi + # ---- + db_name="Logs / App" + echo -n "Import '$db_name' dashboard: " + # check if already imported + response=$(curl -k -s -u "$GF_ADMIN_USER:$GF_ADMIN_PASS" "$GF_ROOT_URL/api/search?query=") + if echo "$response" | grep -iq "\"title\":\"$db_name\""; then + echo -e "${VERT}ALREADY IMPORTED${NORMAL}" + else + import_gf_dashboard 13639 loki + rc1=$? + if [ $rc1 -eq 0 ]; then + echo -e "${VERT}SUCCESSFULLY IMPORTED${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc1)${NORMAL}" + exit $rc1 + fi + fi + # ---- + db_name="Prometheus 2.0 Overview" + echo -n "Import '$db_name' dashboard: " + # check if already imported + response=$(curl -k -s -u "$GF_ADMIN_USER:$GF_ADMIN_PASS" "$GF_ROOT_URL/api/search?query=") + if echo "$response" | grep -iq "\"title\":\"$db_name\""; then + echo -e "${VERT}ALREADY IMPORTED${NORMAL}" + else + import_gf_dashboard 3662 prometheus + rc1=$? + if [ $rc1 -eq 0 ]; then + echo -e "${VERT}SUCCESSFULLY IMPORTED${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc1)${NORMAL}" + exit $rc1 + fi + fi + # ---- + db_name="MinIO Dashboard" + echo -n "Import $db_name dashboard: " + # check if already imported + response=$(curl -k -s -u "$GF_ADMIN_USER:$GF_ADMIN_PASS" "$GF_ROOT_URL/api/search?query=") + if echo "$response" | grep -iq "\"title\":\"$db_name\""; then + echo -e "${VERT}ALREADY IMPORTED${NORMAL}" + else + import_gf_dashboard 13502 prometheus + rc1=$? + if [ $rc1 -eq 0 ]; then + echo -e "${VERT}SUCCESSFULLY IMPORTED${NORMAL}" + else + echo -e "${ROUGE}ERROR ($rc1)${NORMAL}" + exit $rc1 + fi + fi + ;; + stop-stack) + docker compose stop + ;; + down-down) + docker compose down + ;; + del-stack) + # confirm installation + while true; do + read -p "Are you sure you want to permanently delete the stack '${COMPOSE_PROJECT_NAME}'? [y/n] ? " yn + case $yn in + [Yy]* ) break;; + * ) exit; #echo "Please answer yes or no.";; + esac + done + echo -e "\n---- Stop and remove all container ----" + docker compose down + echo -e "\n---- Delete all folder ----" + remove_folder + ;; + list) + echo "-------------------------------------" + echo " Container list" + echo "-------------------------------------" + docker compose config --services | sort + #for cont_item in "${containers_all[@]}"; do + # echo "- $cont_item" + #done + ;; + -h|--help|help) + print_usage + exit 0 + ;; + -v|--version|version) + cat << EOF +$(basename "$0") v$version (c) 1990 - $(date +%Y) by Gilles Mouchet +Non-Commercial Use License – See LICENSE for details +EOF + exit 0 + ;; + *) + print_usage + exit 0 + ;; esac - -#!/bin/bash - -# Couleurs -VERT="\e[32m" -ROUGE="\e[31m" -NORMAL="\e[0m" - -## Message initial -#echo "Lancement du conteneur..." -# -## Exécution de la commande -#if docker compose start toto > /dev/null 2>&1; then - #echo -e "Lancement du conteneur : ${VERT}OK${NORMAL}" -#else - #echo -e "Lancement du conteneur : ${ROUGE}notOK${NORMAL}" -#fi \ No newline at end of file