From 41ad1b36c59e2c3a4667280ebe3023115694bc9b Mon Sep 17 00:00:00 2001 From: DoTheEvo Date: Sat, 4 Mar 2023 15:44:27 +0100 Subject: [PATCH] update --- prometheus_grafana/readme.md | 177 +++++++++++++++++------------------ 1 file changed, 86 insertions(+), 91 deletions(-) diff --git a/prometheus_grafana/readme.md b/prometheus_grafana/readme.md index d592122..f9e5e97 100644 --- a/prometheus_grafana/readme.md +++ b/prometheus_grafana/readme.md @@ -290,14 +290,88 @@ the default time interval is set to 1h instead of 15m # Pushgateway -Gives freedom to push information in to prometheus from anywhere. No need -to be on the same network. -Just a html post request to pushgateway url.
+Gives freedom to push information in to prometheus from anywhere. -The setup and the real world use of pushgateway, along with small steps -when learning it are in the repo - -[Veeam Prometheus Grafana](https://github.com/DoTheEvo/veeam-prometheus-grafana)
-Including pushing information from windows powershell. +To add pushgateway functionality to the current stack. + +- New container `alertmanager` added to the compose file.
+
+ docker-compose.yml + + ```yml + services: + + # PUSHGATEWAY FOR PROMETHEUS + pushgateway: + image: prom/pushgateway:v1.5.1 + container_name: pushgateway + hostname: pushgateway + restart: unless-stopped + command: + - '--web.enable-admin-api' + expose: + - "9091" + + networks: + default: + name: $DOCKER_MY_NETWORK + external: true + ``` +
+ +- Adding pushgateway to the Caddyfile of the reverse proxy so that it can + be reached at `https://push.example.com`
+
+ .env + + `Caddyfile` + ```php + push.{$MY_DOMAIN} { + reverse_proxy pushgateway:9091 + } + ``` +
+ +- Adding pushgateway's scrape point to `prometheus.yml`
+
+ prometheus.yml + + ```yml + global: + scrape_interval: 15s + evaluation_interval: 15s + + scrape_configs: + - job_name: 'pushgateway-scrape' + scrape_interval: 60s + honor_labels: true + static_configs: + - targets: ['pushgateway:9091'] + ``` +
+ +![veeam-dash](https://i.imgur.com/TOuv9bM.png) + +To **test pushing** some metric, execute in linux:
+`echo "some_metric 3.14" | curl --data-binary @- https://push.example.com/metrics/job/blabla/instance/whatever` + +You see **labels** being applied to the pushed metric in the path after metrics.
+Label `job` is required, but after that it's whatever you want, +though use of `instance` label is customary.
+Now in grafana, in **Explore** section you should see some results +when quering for `some_metric`. + +The metrics sit on the pushgateway **forever** unless overwritten/deleted/shutdown +of the container. And if the values are there, prometheus will consider +it the true value for every scrape of the pushgateway, not just +at the moment it was first scraped. + +To wipe the pushgateway clean
+`curl -X PUT https://push.example.com/api/v1/admin/wipe` + +More on pushgateway setup, with the real world use to monitor backups, +along with pushing metrics from windows in powershell - +[**Veeam Prometheus Grafana**](https://github.com/DoTheEvo/veeam-prometheus-grafana)
![veeam-dash](https://i.imgur.com/dUyzuyl.png) @@ -325,87 +399,7 @@ Several changes are needed: ```yml services: - # MONITORING SYSTEM AND THE METRICS DATABASE - prometheus: - image: prom/prometheus:v2.42.0 - container_name: prometheus - hostname: prometheus - user: root - restart: unless-stopped - depends_on: - - cadvisor - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--storage.tsdb.path=/prometheus' - - '--web.console.libraries=/etc/prometheus/console_libraries' - - '--web.console.templates=/etc/prometheus/consoles' - - '--storage.tsdb.retention.time=240h' - - '--web.enable-lifecycle' - volumes: - - ./prometheus_data:/prometheus - - ./prometheus.yml:/etc/prometheus/prometheus.yml - - ./alert.rules:/etc/prometheus/rules/alert.rules - expose: - - "9090" - labels: - org.label-schema.group: "monitoring" - - # WEB BASED UI VISUALISATION OF METRICS - grafana: - image: grafana/grafana:9.4.3 - container_name: grafana - hostname: grafana - user: root - restart: unless-stopped - env_file: .env - volumes: - - ./grafana_data:/var/lib/grafana - expose: - - "3000" - labels: - org.label-schema.group: "monitoring" - - # HOST LINUX MACHINE METRICS EXPORTER - nodeexporter: - image: prom/node-exporter:v1.5.0 - container_name: nodeexporter - hostname: nodeexporter - restart: unless-stopped - command: - - '--path.procfs=/host/proc' - - '--path.rootfs=/rootfs' - - '--path.sysfs=/host/sys' - - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - expose: - - "9100" - labels: - org.label-schema.group: "monitoring" - - # DOCKER CONTAINERS METRICS EXPORTER - cadvisor: - image: gcr.io/cadvisor/cadvisor:v0.47.1 - container_name: cadvisor - hostname: cadvisor - restart: unless-stopped - privileged: true - devices: - - /dev/kmsg:/dev/kmsg - volumes: - - /:/rootfs:ro - - /var/run:/var/run:ro - - /sys:/sys:ro - - /var/lib/docker:/var/lib/docker:ro - - /cgroup:/cgroup:ro #doesn't work on MacOS only for Linux - expose: - - "3000" - labels: - org.label-schema.group: "monitoring" - - # ALERT MANAGMENT BY PROMETHEUS + # ALERT MANAGMENT FOR PROMETHEUS alertmanager: image: prom/alertmanager:v0.25.0 container_name: alertmanager @@ -538,11 +532,12 @@ But first to add Loki to the current grafana, prometheus stack: but url is changed, and compactor section is added to allow for data retention. * install [loki-docker-driver](https://grafana.com/docs/loki/latest/clients/docker-driver/)
`docker plugin install grafana/loki-docker-driver:latest --alias loki --grant-all-permissions`
+ check if it's installed: `docker plugin ls` * adding logging section to compose files of a containers that should be monitored.
-**example compose with logging enabled through loki driver** +example compose with logging enabled through loki driver ```yml services: @@ -559,7 +554,7 @@ services:
- **loki container to be added to grafana/prometheus stack** + loki container to be added to grafana/prometheus stack ```yml services: @@ -589,7 +584,7 @@ networks:
- **loki-docker-config.yml** + loki-docker-config.yml ```yml auth_enabled: false