Deployed prometheus/grafana
This commit is contained in:
26
ansible/playbooks/day1_deploy_monitoring.yml
Normal file
26
ansible/playbooks/day1_deploy_monitoring.yml
Normal file
@@ -0,0 +1,26 @@
|
||||
---
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: playbooks/day1_deploy_monitoring.yml
|
||||
# DESCRIPTION: Deploys Prometheus + Grafana + Node Exporter + Alertmanager
|
||||
# on cinderella-castle (10.1.71.31)
|
||||
# Runs: common → docker-host → monitoring
|
||||
#
|
||||
# USAGE:
|
||||
# ansible-playbook -i inventory.yml playbooks/day1_deploy_monitoring.yml
|
||||
# ansible-playbook -i inventory.yml playbooks/day1_deploy_monitoring.yml --limit cinderella_castle
|
||||
#
|
||||
# SECRETS REQUIRED IN VAULT (group_vars/all/vault):
|
||||
# vault_grafana_admin_password
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
- name: Deploy monitoring stack on cinderella-castle
|
||||
hosts: cinderella-castle
|
||||
become: true
|
||||
|
||||
vars:
|
||||
grafana_admin_password: "{{ vault_grafana_admin_password }}"
|
||||
|
||||
roles:
|
||||
- role: common
|
||||
- role: docker-host
|
||||
- role: monitoring
|
||||
38
ansible/playbooks/roles/monitoring/defaults/main.yml
Normal file
38
ansible/playbooks/roles/monitoring/defaults/main.yml
Normal file
@@ -0,0 +1,38 @@
|
||||
---
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/monitoring/defaults/main.yml
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Directory layout on cinderella-castle
|
||||
monitoring_base_dir: /opt/docker/monitoring
|
||||
|
||||
# Data disk — LVM on /dev/sdb, mounted at this path for Prometheus TSDB
|
||||
prometheus_data_dir: /var/lib/prometheus
|
||||
prometheus_vg: prometheus-vg
|
||||
prometheus_lv: prometheus-lv
|
||||
|
||||
# Prometheus retention
|
||||
prometheus_retention_time: 15d
|
||||
prometheus_retention_size: 58GB # 64GB disk, leaving headroom
|
||||
|
||||
# Scrape interval
|
||||
prometheus_scrape_interval: 30s
|
||||
|
||||
# Grafana
|
||||
grafana_port: 3000
|
||||
grafana_admin_user: admin
|
||||
|
||||
# Service DNS (via Traefik)
|
||||
grafana_hostname: grafana.local.mk-labs.cloud
|
||||
|
||||
# Trusted proxy (lightning-lane)
|
||||
trusted_proxy_ip: 10.1.71.35
|
||||
|
||||
# Prometheus port (internal, not exposed via Traefik)
|
||||
prometheus_port: 9090
|
||||
|
||||
# Alertmanager port (internal)
|
||||
alertmanager_port: 9093
|
||||
|
||||
# Node exporter port
|
||||
node_exporter_port: 9100
|
||||
10
ansible/playbooks/roles/monitoring/handlers/main.yml
Normal file
10
ansible/playbooks/roles/monitoring/handlers/main.yml
Normal file
@@ -0,0 +1,10 @@
|
||||
---
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/monitoring/handlers/main.yml
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
- name: restart monitoring
|
||||
community.docker.docker_compose_v2:
|
||||
project_src: "{{ monitoring_base_dir }}"
|
||||
state: present
|
||||
recreate: always
|
||||
152
ansible/playbooks/roles/monitoring/tasks/main.yml
Normal file
152
ansible/playbooks/roles/monitoring/tasks/main.yml
Normal file
@@ -0,0 +1,152 @@
|
||||
---
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/monitoring/tasks/main.yml
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# ── Data disk (LVM) ───────────────────────────────────────────────────────────
|
||||
|
||||
- name: Install LVM2
|
||||
ansible.builtin.apt:
|
||||
name: lvm2
|
||||
state: present
|
||||
update_cache: true
|
||||
|
||||
- name: Check if PV already exists on /dev/sdb
|
||||
ansible.builtin.command: pvs /dev/sdb
|
||||
register: pvs_result
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Check if disk has a partition table signature
|
||||
ansible.builtin.command: wipefs /dev/sdb
|
||||
register: wipefs_check
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
when: pvs_result.rc != 0
|
||||
|
||||
- name: Wipe GPT protective partition signature from /dev/sdb
|
||||
ansible.builtin.command: wipefs -a /dev/sdb
|
||||
when:
|
||||
- pvs_result.rc != 0
|
||||
- wipefs_check.stdout | length > 0
|
||||
|
||||
- name: Create LVM volume group on /dev/sdb
|
||||
community.general.lvg:
|
||||
vg: prometheus-vg
|
||||
pvs: /dev/sdb
|
||||
state: present
|
||||
when: pvs_result.rc != 0
|
||||
|
||||
- name: Check if logical volume already exists
|
||||
ansible.builtin.command: lvs /dev/prometheus-vg/prometheus-lv
|
||||
register: lvs_result
|
||||
changed_when: false
|
||||
failed_when: false
|
||||
|
||||
- name: Create logical volume for Prometheus TSDB
|
||||
community.general.lvol:
|
||||
vg: prometheus-vg
|
||||
lv: prometheus-lv
|
||||
size: 100%FREE
|
||||
state: present
|
||||
shrink: false
|
||||
when: lvs_result.rc != 0
|
||||
|
||||
- name: Format logical volume as ext4
|
||||
community.general.filesystem:
|
||||
fstype: ext4
|
||||
dev: /dev/prometheus-vg/prometheus-lv
|
||||
|
||||
- name: Create Prometheus data directory mount point
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_data_dir }}"
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Mount Prometheus logical volume
|
||||
ansible.posix.mount:
|
||||
path: "{{ prometheus_data_dir }}"
|
||||
src: /dev/prometheus-vg/prometheus-lv
|
||||
fstype: ext4
|
||||
opts: defaults
|
||||
state: mounted
|
||||
|
||||
- name: Set ownership of Prometheus data directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ prometheus_data_dir }}"
|
||||
state: directory
|
||||
owner: "65534" # nobody — Prometheus container runs as this UID
|
||||
group: "65534"
|
||||
mode: "0755"
|
||||
|
||||
# ── Directory layout ───────────────────────────────────────────────────────────
|
||||
|
||||
- name: Create monitoring directory structure
|
||||
ansible.builtin.file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ ansible_user }}"
|
||||
group: docker
|
||||
mode: "0775"
|
||||
loop:
|
||||
- "{{ monitoring_base_dir }}"
|
||||
- "{{ monitoring_base_dir }}/prometheus"
|
||||
- "{{ monitoring_base_dir }}/grafana"
|
||||
- "{{ monitoring_base_dir }}/alertmanager"
|
||||
|
||||
# ── Config files ───────────────────────────────────────────────────────────────
|
||||
|
||||
- name: Deploy Prometheus configuration
|
||||
ansible.builtin.template:
|
||||
src: prometheus.yaml.j2
|
||||
dest: "{{ monitoring_base_dir }}/prometheus/prometheus.yaml"
|
||||
owner: "{{ ansible_user }}"
|
||||
group: docker
|
||||
mode: "0644"
|
||||
notify: restart monitoring
|
||||
|
||||
- name: Deploy Alertmanager configuration
|
||||
ansible.builtin.template:
|
||||
src: alertmanager.yml.j2
|
||||
dest: "{{ monitoring_base_dir }}/alertmanager/alertmanager.yml"
|
||||
owner: "{{ ansible_user }}"
|
||||
group: docker
|
||||
mode: "0644"
|
||||
notify: restart monitoring
|
||||
|
||||
- name: Deploy Grafana provisioning datasource
|
||||
ansible.builtin.template:
|
||||
src: grafana-datasource.yml.j2
|
||||
dest: "{{ monitoring_base_dir }}/grafana/datasource.yml"
|
||||
owner: "{{ ansible_user }}"
|
||||
group: docker
|
||||
mode: "0644"
|
||||
notify: restart monitoring
|
||||
|
||||
# ── Compose stack ──────────────────────────────────────────────────────────────
|
||||
|
||||
- name: Deploy monitoring compose file from boilerplate
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/../../boilerplates/monitoring/compose.yaml"
|
||||
dest: "{{ monitoring_base_dir }}/compose.yaml"
|
||||
owner: "{{ ansible_user }}"
|
||||
group: docker
|
||||
mode: "0644"
|
||||
notify: restart monitoring
|
||||
|
||||
- name: Deploy .env with secrets from vault
|
||||
ansible.builtin.template:
|
||||
src: env.j2
|
||||
dest: "{{ monitoring_base_dir }}/.env"
|
||||
owner: "{{ ansible_user }}"
|
||||
group: docker
|
||||
mode: "0600"
|
||||
notify: restart monitoring
|
||||
|
||||
- name: Start monitoring stack
|
||||
community.docker.docker_compose_v2:
|
||||
project_src: "{{ monitoring_base_dir }}"
|
||||
state: present
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/monitoring/templates/alertmanager.yml.j2
|
||||
# DESCRIPTION: Alertmanager configuration — routing deferred (P.XX)
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
route:
|
||||
receiver: blackhole
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 4h
|
||||
|
||||
receivers:
|
||||
- name: blackhole
|
||||
11
ansible/playbooks/roles/monitoring/templates/env.j2
Normal file
11
ansible/playbooks/roles/monitoring/templates/env.j2
Normal file
@@ -0,0 +1,11 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/monitoring/templates/env.j2
|
||||
# DESCRIPTION: Environment file for monitoring compose stack
|
||||
# Deployed to {{ monitoring_base_dir }}/.env
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
GRAFANA_ADMIN_USER={{ grafana_admin_user }}
|
||||
GRAFANA_ADMIN_PASSWORD={{ grafana_admin_password }}
|
||||
PROMETHEUS_RETENTION_TIME={{ prometheus_retention_time }}
|
||||
PROMETHEUS_RETENTION_SIZE={{ prometheus_retention_size }}
|
||||
TRUSTED_PROXIES={{ trusted_proxy_ip }}
|
||||
@@ -0,0 +1,14 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/monitoring/templates/grafana-datasource.yml.j2
|
||||
# DESCRIPTION: Auto-provisions Prometheus as Grafana default datasource
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
editable: false
|
||||
@@ -0,0 +1,50 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/monitoring/templates/prometheus.yml.j2
|
||||
# DESCRIPTION: Prometheus scrape configuration
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
global:
|
||||
scrape_interval: {{ prometheus_scrape_interval }}
|
||||
evaluation_interval: {{ prometheus_scrape_interval }}
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
rule_files: []
|
||||
|
||||
scrape_configs:
|
||||
|
||||
# Prometheus self-monitoring
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost:9090
|
||||
|
||||
# cinderella-castle node exporter (this host)
|
||||
- job_name: node-cinderella-castle
|
||||
static_configs:
|
||||
- targets:
|
||||
- cinderella-castle:9100
|
||||
labels:
|
||||
hostname: cinderella-castle
|
||||
|
||||
# Proxmox nodes
|
||||
- job_name: proxmox-nodes
|
||||
static_configs:
|
||||
- targets:
|
||||
- 10.1.71.11:9100
|
||||
- 10.1.71.12:9100
|
||||
- 10.1.71.13:9100
|
||||
labels:
|
||||
job: proxmox
|
||||
|
||||
# Traefik metrics
|
||||
- job_name: traefik
|
||||
static_configs:
|
||||
- targets:
|
||||
- 10.1.71.35:8080
|
||||
labels:
|
||||
hostname: lightning-lane
|
||||
Reference in New Issue
Block a user