From 6269e85e88f1f2d5f560b315ed720399ead3a76e Mon Sep 17 00:00:00 2001 From: staffadmin Date: Sat, 23 Sep 2023 19:11:31 +0200 Subject: [PATCH] add all monitoring tools --- all.yml | 2 ++ group_vars/perso.yml | 13 ++++++++ host_vars/scaleway_fr.yml | 28 ++++++++++++++++- templates/alerts.sh.j2 | 63 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 1 deletion(-) diff --git a/all.yml b/all.yml index 1c25ca6..65f0d5e 100644 --- a/all.yml +++ b/all.yml @@ -65,6 +65,8 @@ job: "{{ item.job }}" cron_file: "{{ item.cron_file }}" loop: "{{ alerts_cron }}" + tags: + - alerts # - name: Show ssh backup # debug: diff --git a/group_vars/perso.yml b/group_vars/perso.yml index 72280ad..2f0fab2 100644 --- a/group_vars/perso.yml +++ b/group_vars/perso.yml @@ -98,3 +98,16 @@ apt_ppas: [] # #codename: trusty apt_packages: - name: openssh-server + + + ######### + # ALERT # + ######### + +alert_username: jingohalert +alert_password: jMVmbM2VQ5gEiV +alert_vault: "Jingoh0947;" +alert_list_server: + - 163.172.84.28 + - 37.187.127.90 +alert_server_ssl: gitea.jingoh.fr diff --git a/host_vars/scaleway_fr.yml b/host_vars/scaleway_fr.yml index 5f9f57e..b557d0a 100644 --- a/host_vars/scaleway_fr.yml +++ b/host_vars/scaleway_fr.yml @@ -84,4 +84,30 @@ logrotate_scripts: - compress - missingok - notifempty - - create 0644 root root \ No newline at end of file + - create 0644 root root + - name: dockerapps + path: /opt/dockerapps/logs/homeserver/*.log + options: + - rotate 12 + - monthly + - compress + - missingok + - delaycompress + scripts: + - name: restart gitea + script: docker-compose restart gitea + - name: restart vaultwarden + script: docker-compose restart vaultwarden + - name: restart grafana + script: docker-compose restart grafana + - name: restart traefrik + script: docker-compose restart traefrik + +alerts_storage: scaleway +alerts_load: scaleway +alerts_ping: ovh +alerts_health: scaleway +alerts_backup_git: scaleway +alerts_backup_vault: scaleway +alerts_cpu: scaleway +alerts_ssl: scaleway diff --git a/templates/alerts.sh.j2 b/templates/alerts.sh.j2 index e69de29..daafb94 100644 --- a/templates/alerts.sh.j2 +++ b/templates/alerts.sh.j2 @@ -0,0 +1,63 @@ +#!/bin/bash + +# Monitoring script + +# Secrets +username="{{ alert_username }}" +password="{{ alert_password }}" +VAULT="{{ alert_vault }}" + +# Servers +servers=("{{ alert_list_server | join (' ') }}") +local_ip=$(hostname -I | awk '{print $1}') + +# SSL +site="{{ alert_server_ssl }}" + +# Vérifier s'il y a un paramètre +if [ $# -ne 1 ]; then + echo "Usage : $0 [storage|load|ping|health|ssl|backup_git|backup_vault|cpu]" + exit 1 +fi + +# Récupérer le paramètre +parametre="$1" + +# Vérifier la valeur du paramètre et afficher le résultat correspondant +if [ "$parametre" = "storage" ]; then + [ $(df -h / | awk 'NR==2 {sub(/%/, "", $(NF-1)); print $(NF-1)}') -gt 80 ] && curl -u "$username:$password" -H "Title: Full Storage" -H "ta:card_index_dividers" -d "90% used on `hostname`" https://alert.jingoh.fr/{{ alerts_storage }} +elif [ "$parametre" = "load" ]; then + [ $(uptime | awk -F'load average: ' '{print $2}' | awk '{print $1}' | cut -d , -f1) '>' $(nproc) ] && curl -u "$username:$password" -H "Title: Load" -H "ta:battery" -d "`hostname` Load with `uptime`" https://alert.jingoh.fr/{{ alerts_load }} +elif [ "$parametre" = "ping" ]; then + for ip in "${servers[@]}" + do + if [ "$ip" != "$local_ip" ]; then + ping -c 1 "$ip" || curl -u "$username:$password" -H "Title: Ping Server" -H "ta:sos" -d "Server ping failed from `hostname` to $ip" https://alert.jingoh.fr/{{ alerts_ping }} + fi + done +elif [ "$parametre" = "health" ]; then + [ $(curl -s -o /dev/null -w "%{http_code}" https://gitea.jingoh.fr) -gt 400 ] && curl -u "$username:$password" -H "Title: Service gitea" -H "ta:bangbang" -d "No response From gitea.jingoh.fr" https://alert.jingoh.fr/{{ alerts_health }} + [ $(curl -s -o /dev/null -w "%{http_code}" https://vault.jingoh.fr) -gt 400 ] && curl -u "$username:$password" -H "Title: Service vault" -H "ta:bangbang" -d "No response From vault.jingoh.fr" https://alert.jingoh.fr/{{ alerts_health }} + [ $(curl -s -o /dev/null -w "%{http_code}" https://homepage.jingoh.fr) -gt 400 ] && curl -u "$username:$password" -H "Title: Service homepage" -H "ta:bangbang" -d "No response From homepage.jingoh.fr" https://alert.jingoh.fr/{{ alerts_health }} +elif [ "$parametre" = "ssl" ]; then + expiration_timestamp=$(date -d "$(echo | openssl s_client -servername $site -connect $site:443 2>/dev/null | openssl x509 -noout -enddate | cut -d "=" -f 2)" +%s) + current_timestamp=$(date +%s) + difference=$((expiration_timestamp - current_timestamp)) + threshold=$((20 * 24 * 3600)) # 20 jours en secondes + if [ $difference -lt $threshold ]; then + curl -u "$username:$password" -H "Title: HTTPS Certificats" -H "ta:closed_lock_with_key" -d "*.jingoh.fr Less than 20 days" https://alert.jingoh.fr/{{ alerts_ssl }} + fi +elif [ "$parametre" = "backup_git" ]; then + docker exec -u git -w /data/ gitea gitea dump -c /data/gitea/conf/app.ini + mv /opt/dockerapps/appdata/gitea/gitea/gitea-dump-*.zip /opt/dockerapps/backup/ + docker exec gitea-db pg_dump -U root gitea > gitea-db-pg.sql + mv ./gitea-db-pg.sql /opt/dockerapps/backup/ + curl -u "$username:$password" -H "Title: Backup gitea" -H "ta:inbox_tray" -d "Local Backup gitea done !" https://alert.jingoh.fr/{{ alerts_backup_gitea }} +elif [ "$parametre" = "backup_vault" ]; then + docker run --rm --volumes-from=vault -e UID=0 -e BACKUP_DIR=/data/backup -e TIMESTAMP=true -e ENCRYPTION_PASSWORD="$VAULT" bruceforce/vaultwarden-backup manual + curl -u "$username:$password" -H "Title: Backup vault" -H "ta:inbox_tray" -d "Local Backup vault done !" https://alert.jingoh.fr/{{ alerts_backup_vault }} +elif [ "$parametre" = "cpu" ]; then + [ "$(echo "$(ps -eo %cpu --sort=-%cpu | awk 'NR>1 { sum += $1 } END { print sum }') > $(nproc) * 50" | bc)" -eq 1 ] && curl -u "$username:$password" -H "Title: CPU `nproc` cores" -H "ta:warning" -d "High usage `ps -eo %cpu --sort=-%cpu | awk 'NR>1 { sum += $1 } END { print sum }'`" https://alert.jingoh.fr/{{ alerts_cpu }} +else + echo "Paramètre invalide : Utilisez [storage|load|ping|health|ssl|backup_git|backup_vault|cpu]" +fi \ No newline at end of file