feat(llm-inference): Phase 7 — Prometheus monitoring + Grafana dashboard
- Phase 7 task file: monitoring.yml
- node_exporter (port 9100) via apt, systemd managed
- nvidia_gpu_exporter v1.2.2 (port 9835) — GPU util, VRAM, temp, power
- Patches kube-prometheus additionalScrapeConfigs secret with 3 new jobs:
node-astro-orbiter, gpu-astro-orbiter, vllm-astro-orbiter
- Deploys Grafana dashboard ConfigMap via kubectl apply
- Grafana dashboard (11 panels):
- Row 1: GPU util %, VRAM used, GPU temp gauge
- Row 2: GPU power draw, vLLM token throughput, request queue depth
- Row 3: vLLM e2e latency p50/p95/p99, KV cache utilization %
- Row 4: System CPU %, memory, root disk gauge
- defaults/main.yml: llm_gpu_exporter_version, llm_gpu_exporter_port
- handlers/main.yml: restart nvidia-gpu-exporter
This commit is contained in:
Hermes Agent service account
198
ansible/roles/llm-inference/tasks/monitoring.yml
Normal file
198
ansible/roles/llm-inference/tasks/monitoring.yml
Normal file
@@ -0,0 +1,198 @@
|
||||
---
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/llm-inference/tasks/monitoring.yml
|
||||
# DESCRIPTION: Phase 7 — Prometheus monitoring for the LLM inference stack.
|
||||
# Deploys three metric sources on astro-orbiter:
|
||||
#
|
||||
# 1. node_exporter (port 9100) — system: CPU, RAM, disk, network
|
||||
# 2. nvidia_gpu_exporter (port 9835) — GPU: VRAM, temp, util, power
|
||||
# 3. vLLM built-in metrics (port 8000/metrics) — already exposed
|
||||
# by vLLM; just needs a scrape job (no extra process)
|
||||
#
|
||||
# Wires all three into Prometheus via additionalScrapeConfigs on
|
||||
# the kube-prometheus-stack secret on carousel-of-progress.
|
||||
# Deploys a Grafana dashboard ConfigMap in the monitoring namespace.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# 1. node_exporter
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
- name: Install prometheus-node-exporter
|
||||
ansible.builtin.apt:
|
||||
name: prometheus-node-exporter
|
||||
state: present
|
||||
update_cache: false
|
||||
|
||||
- name: Enable and start node_exporter
|
||||
ansible.builtin.systemd:
|
||||
name: prometheus-node-exporter
|
||||
state: started
|
||||
enabled: true
|
||||
|
||||
- name: Verify node_exporter is reachable
|
||||
ansible.builtin.uri:
|
||||
url: "http://localhost:9100/metrics"
|
||||
status_code: 200
|
||||
register: node_exporter_health
|
||||
retries: 6
|
||||
delay: 5
|
||||
until: node_exporter_health.status == 200
|
||||
changed_when: false
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# 2. nvidia_gpu_exporter (utkuozdemir/nvidia_gpu_exporter)
|
||||
# Lightweight single-binary exporter — no CUDA dependency, uses nvidia-smi.
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
- name: Create nvidia_gpu_exporter install directory
|
||||
ansible.builtin.file:
|
||||
path: /opt/nvidia_gpu_exporter
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0755"
|
||||
|
||||
- name: Download nvidia_gpu_exporter binary
|
||||
ansible.builtin.get_url:
|
||||
url: "https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v{{ llm_gpu_exporter_version }}/nvidia_gpu_exporter_{{ llm_gpu_exporter_version }}_linux_x86_64.tar.gz"
|
||||
dest: "/tmp/nvidia_gpu_exporter.tar.gz"
|
||||
mode: "0644"
|
||||
register: gpu_exporter_download
|
||||
|
||||
- name: Extract nvidia_gpu_exporter binary
|
||||
ansible.builtin.unarchive:
|
||||
src: /tmp/nvidia_gpu_exporter.tar.gz
|
||||
dest: /opt/nvidia_gpu_exporter
|
||||
remote_src: true
|
||||
creates: /opt/nvidia_gpu_exporter/nvidia_gpu_exporter
|
||||
|
||||
- name: Deploy nvidia_gpu_exporter systemd service
|
||||
ansible.builtin.copy:
|
||||
dest: /etc/systemd/system/nvidia-gpu-exporter.service
|
||||
mode: "0644"
|
||||
content: |
|
||||
[Unit]
|
||||
Description=NVIDIA GPU Prometheus Exporter
|
||||
After=network.target nvidia-persistenced.service
|
||||
Wants=nvidia-persistenced.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/opt/nvidia_gpu_exporter/nvidia_gpu_exporter \
|
||||
--web.listen-address=:{{ llm_gpu_exporter_port }}
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=nvidia-gpu-exporter
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
notify:
|
||||
- reload systemd
|
||||
- restart nvidia-gpu-exporter
|
||||
|
||||
- name: Flush handlers before starting gpu exporter
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
- name: Enable and start nvidia-gpu-exporter
|
||||
ansible.builtin.systemd:
|
||||
name: nvidia-gpu-exporter
|
||||
state: started
|
||||
enabled: true
|
||||
daemon_reload: true
|
||||
|
||||
- name: Verify nvidia_gpu_exporter is reachable
|
||||
ansible.builtin.uri:
|
||||
url: "http://localhost:{{ llm_gpu_exporter_port }}/metrics"
|
||||
status_code: 200
|
||||
register: gpu_exporter_health
|
||||
retries: 6
|
||||
delay: 5
|
||||
until: gpu_exporter_health.status == 200
|
||||
changed_when: false
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# 3. Patch additionalScrapeConfigs secret on carousel-of-progress
|
||||
# Adds three new scrape jobs: node, gpu, vllm
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
- name: Read current additionalScrapeConfigs from Prometheus secret
|
||||
ansible.builtin.command:
|
||||
cmd: >
|
||||
kubectl get secret monitoring-kube-prometheus-prometheus-scrape-confg
|
||||
-n monitoring
|
||||
-o jsonpath='{.data.additional-scrape-configs\.yaml}'
|
||||
register: current_scrape_b64
|
||||
changed_when: false
|
||||
delegate_to: carousel-of-progress
|
||||
become: false
|
||||
|
||||
- name: Decode current scrape configs
|
||||
ansible.builtin.set_fact:
|
||||
current_scrape_yaml: "{{ current_scrape_b64.stdout | b64decode }}"
|
||||
|
||||
- name: Check if astro-orbiter scrape jobs already present
|
||||
ansible.builtin.set_fact:
|
||||
scrape_already_patched: "{{ 'astro-orbiter' in current_scrape_yaml }}"
|
||||
|
||||
- name: Append astro-orbiter scrape jobs to additionalScrapeConfigs
|
||||
when: not scrape_already_patched
|
||||
block:
|
||||
- name: Build new scrape config with astro-orbiter jobs appended
|
||||
ansible.builtin.set_fact:
|
||||
new_scrape_yaml: |
|
||||
{{ current_scrape_yaml }}
|
||||
- job_name: node-astro-orbiter
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- labels:
|
||||
hostname: astro-orbiter
|
||||
targets:
|
||||
- {{ hostvars['astro-orbiter']['ansible_host'] }}:9100
|
||||
- job_name: gpu-astro-orbiter
|
||||
scrape_interval: 15s
|
||||
static_configs:
|
||||
- labels:
|
||||
hostname: astro-orbiter
|
||||
gpu: rtx3090
|
||||
targets:
|
||||
- {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_gpu_exporter_port }}
|
||||
- job_name: vllm-astro-orbiter
|
||||
scrape_interval: 15s
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- labels:
|
||||
hostname: astro-orbiter
|
||||
model: "{{ llm_hf_model }}"
|
||||
targets:
|
||||
- {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_serve_port }}
|
||||
|
||||
- name: Patch Prometheus additionalScrapeConfigs secret
|
||||
ansible.builtin.command:
|
||||
cmd: >
|
||||
kubectl patch secret monitoring-kube-prometheus-prometheus-scrape-confg
|
||||
-n monitoring
|
||||
--type=json
|
||||
-p='[{"op":"replace","path":"/data/additional-scrape-configs.yaml","value":"{{ new_scrape_yaml | b64encode }}"}]'
|
||||
delegate_to: carousel-of-progress
|
||||
become: false
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# 4. Deploy Grafana dashboard
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
- name: Deploy LLM inference Grafana dashboard ConfigMap
|
||||
ansible.builtin.template:
|
||||
src: grafana-llm-dashboard.json.j2
|
||||
dest: /tmp/grafana-llm-dashboard-cm.yaml
|
||||
delegate_to: carousel-of-progress
|
||||
become: false
|
||||
|
||||
- name: Apply Grafana dashboard ConfigMap to cluster
|
||||
ansible.builtin.command:
|
||||
cmd: kubectl apply -f /tmp/grafana-llm-dashboard-cm.yaml
|
||||
delegate_to: carousel-of-progress
|
||||
become: false
|
||||
changed_when: true
|
||||
Reference in New Issue
Block a user