aff792a061
- Prometheus scrape configs for node/gpu/llama-server exporters on astro-orbiter now declared in cluster/applications/monitoring/values.yaml (additionalScrapeConfigs), applied via ArgoCD sync instead of an imperative kubectl secret patch from the Ansible role. - Grafana dashboard for astro-orbiter LLM inference added as a ConfigMap in cluster/applications/monitoring/dashboards.yaml (grafana_dashboard=1 sidecar label), replacing the role's ad-hoc kubectl apply of a rendered Jinja template. - ansible/roles/llm-inference/tasks/monitoring.yml: removed the kubectl get/patch/apply tasks and orphaned grafana-llm-dashboard.json.j2 template; role now only stands up node_exporter + nvidia_gpu_exporter and verifies they're reachable — cluster-facing config lives in Git. - host_vars/vars.yml + inventory.yml: finalize astro-orbiter as the llama.cpp/RTX 3090 host (jarvis user, ssh key), drop stale Ollama/AMD vars and ollama_server inventory group superseded by the ATX rebuild.
132 lines
4.9 KiB
YAML
132 lines
4.9 KiB
YAML
---
|
|
# ------------------------------------------------------------------------------
|
|
# FILE: roles/llm-inference/tasks/monitoring.yml
|
|
# DESCRIPTION: Phase 7 — Prometheus monitoring for the LLM inference stack.
|
|
# Deploys two metric-producing exporters on astro-orbiter:
|
|
#
|
|
# 1. node_exporter (port 9100) — system: CPU, RAM, disk, network
|
|
# 2. nvidia_gpu_exporter (port 9835) — GPU: VRAM, temp, util, power
|
|
# 3. llama-server built-in metrics (port 8000/metrics, enabled via
|
|
# --metrics) — just needs a scrape job (no extra process)
|
|
#
|
|
# GitOps note: the Prometheus scrape jobs for all three targets and
|
|
# the Grafana dashboard are declared in the homelab Git repo and
|
|
# applied by ArgoCD — NOT by this role:
|
|
# - cluster/applications/monitoring/values.yaml
|
|
# (prometheus.prometheusSpec.additionalScrapeConfigs)
|
|
# - cluster/applications/monitoring/dashboards.yaml
|
|
# (grafana-llm-inference-dashboard ConfigMap)
|
|
# This role's job is only to stand up the two exporters + verify
|
|
# they're reachable. Do NOT reintroduce kubectl patch/apply tasks
|
|
# here — cluster-facing changes go through Git commit + ArgoCD
|
|
# sync so state stays reproducible and self-healing.
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# -----------------------------------------------------------------------
|
|
# 1. node_exporter
|
|
# -----------------------------------------------------------------------
|
|
|
|
- name: Install prometheus-node-exporter
|
|
ansible.builtin.apt:
|
|
name: prometheus-node-exporter
|
|
state: present
|
|
update_cache: false
|
|
|
|
- name: Enable and start node_exporter
|
|
ansible.builtin.systemd:
|
|
name: prometheus-node-exporter
|
|
state: started
|
|
enabled: true
|
|
|
|
- name: Verify node_exporter is reachable
|
|
ansible.builtin.uri:
|
|
url: "http://localhost:9100/metrics"
|
|
status_code: 200
|
|
register: node_exporter_health
|
|
retries: 6
|
|
delay: 5
|
|
until: node_exporter_health.status == 200
|
|
changed_when: false
|
|
|
|
# -----------------------------------------------------------------------
|
|
# 2. nvidia_gpu_exporter (utkuozdemir/nvidia_gpu_exporter)
|
|
# Lightweight single-binary exporter — no CUDA dependency, uses nvidia-smi.
|
|
# -----------------------------------------------------------------------
|
|
|
|
- name: Create nvidia_gpu_exporter install directory
|
|
ansible.builtin.file:
|
|
path: /opt/nvidia_gpu_exporter
|
|
state: directory
|
|
owner: root
|
|
group: root
|
|
mode: "0755"
|
|
|
|
- name: Download nvidia_gpu_exporter binary
|
|
ansible.builtin.get_url:
|
|
url: "https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v{{ llm_gpu_exporter_version }}/nvidia_gpu_exporter_{{ llm_gpu_exporter_version }}_linux_x86_64.tar.gz"
|
|
dest: "/tmp/nvidia_gpu_exporter.tar.gz"
|
|
mode: "0644"
|
|
register: gpu_exporter_download
|
|
|
|
- name: Extract nvidia_gpu_exporter binary
|
|
ansible.builtin.unarchive:
|
|
src: /tmp/nvidia_gpu_exporter.tar.gz
|
|
dest: /opt/nvidia_gpu_exporter
|
|
remote_src: true
|
|
creates: /opt/nvidia_gpu_exporter/nvidia_gpu_exporter
|
|
|
|
- name: Deploy nvidia_gpu_exporter systemd service
|
|
ansible.builtin.copy:
|
|
dest: /etc/systemd/system/nvidia-gpu-exporter.service
|
|
mode: "0644"
|
|
content: |
|
|
[Unit]
|
|
Description=NVIDIA GPU Prometheus Exporter
|
|
After=network.target nvidia-persistenced.service
|
|
Wants=nvidia-persistenced.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
ExecStart=/opt/nvidia_gpu_exporter/nvidia_gpu_exporter \
|
|
--web.listen-address=:{{ llm_gpu_exporter_port }}
|
|
Restart=on-failure
|
|
RestartSec=5
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
SyslogIdentifier=nvidia-gpu-exporter
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
notify:
|
|
- reload systemd
|
|
- restart nvidia-gpu-exporter
|
|
|
|
- name: Flush handlers before starting gpu exporter
|
|
ansible.builtin.meta: flush_handlers
|
|
|
|
- name: Enable and start nvidia-gpu-exporter
|
|
ansible.builtin.systemd:
|
|
name: nvidia-gpu-exporter
|
|
state: started
|
|
enabled: true
|
|
daemon_reload: true
|
|
|
|
- name: Verify nvidia_gpu_exporter is reachable
|
|
ansible.builtin.uri:
|
|
url: "http://localhost:{{ llm_gpu_exporter_port }}/metrics"
|
|
status_code: 200
|
|
register: gpu_exporter_health
|
|
retries: 6
|
|
delay: 5
|
|
until: gpu_exporter_health.status == 200
|
|
changed_when: false
|
|
|
|
# -----------------------------------------------------------------------
|
|
# 3. Prometheus scrape configs + Grafana dashboard
|
|
#
|
|
# Intentionally NOT managed here. See file header: these are declared
|
|
# in cluster/applications/monitoring/{values.yaml,dashboards.yaml} in
|
|
# the homelab Git repo and rolled out by ArgoCD sync, keeping cluster
|
|
# state in Git rather than mutated imperatively from the control node.
|
|
# -----------------------------------------------------------------------
|