diff --git a/ansible/roles/llm-inference/defaults/main.yml b/ansible/roles/llm-inference/defaults/main.yml index cde1985..8988ca0 100644 --- a/ansible/roles/llm-inference/defaults/main.yml +++ b/ansible/roles/llm-inference/defaults/main.yml @@ -22,3 +22,7 @@ llm_serve_port: 8000 llm_serve_host: "0.0.0.0" llm_gpu_memory_utilization: "0.90" llm_max_model_len: 8192 + +# Monitoring +llm_gpu_exporter_version: "1.2.2" +llm_gpu_exporter_port: 9835 diff --git a/ansible/roles/llm-inference/handlers/main.yml b/ansible/roles/llm-inference/handlers/main.yml index 61225c4..14fa54e 100644 --- a/ansible/roles/llm-inference/handlers/main.yml +++ b/ansible/roles/llm-inference/handlers/main.yml @@ -27,3 +27,9 @@ delegate_to: carousel-of-progress listen: "restart hermes" ignore_errors: true + +- name: Restart nvidia-gpu-exporter + ansible.builtin.systemd: + name: nvidia-gpu-exporter + state: restarted + listen: "restart nvidia-gpu-exporter" diff --git a/ansible/roles/llm-inference/tasks/main.yml b/ansible/roles/llm-inference/tasks/main.yml index 9eca43a..5e41f8d 100644 --- a/ansible/roles/llm-inference/tasks/main.yml +++ b/ansible/roles/llm-inference/tasks/main.yml @@ -22,3 +22,6 @@ # Phase 6 — Integration - import_tasks: integration.yml + +# Phase 7 — Monitoring +- import_tasks: monitoring.yml diff --git a/ansible/roles/llm-inference/tasks/monitoring.yml b/ansible/roles/llm-inference/tasks/monitoring.yml new file mode 100644 index 0000000..9eac145 --- /dev/null +++ b/ansible/roles/llm-inference/tasks/monitoring.yml @@ -0,0 +1,198 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference/tasks/monitoring.yml +# DESCRIPTION: Phase 7 — Prometheus monitoring for the LLM inference stack. +# Deploys three metric sources on astro-orbiter: +# +# 1. node_exporter (port 9100) — system: CPU, RAM, disk, network +# 2. nvidia_gpu_exporter (port 9835) — GPU: VRAM, temp, util, power +# 3. vLLM built-in metrics (port 8000/metrics) — already exposed +# by vLLM; just needs a scrape job (no extra process) +# +# Wires all three into Prometheus via additionalScrapeConfigs on +# the kube-prometheus-stack secret on carousel-of-progress. +# Deploys a Grafana dashboard ConfigMap in the monitoring namespace. +# ------------------------------------------------------------------------------ + +# ----------------------------------------------------------------------- +# 1. node_exporter +# ----------------------------------------------------------------------- + +- name: Install prometheus-node-exporter + ansible.builtin.apt: + name: prometheus-node-exporter + state: present + update_cache: false + +- name: Enable and start node_exporter + ansible.builtin.systemd: + name: prometheus-node-exporter + state: started + enabled: true + +- name: Verify node_exporter is reachable + ansible.builtin.uri: + url: "http://localhost:9100/metrics" + status_code: 200 + register: node_exporter_health + retries: 6 + delay: 5 + until: node_exporter_health.status == 200 + changed_when: false + +# ----------------------------------------------------------------------- +# 2. nvidia_gpu_exporter (utkuozdemir/nvidia_gpu_exporter) +# Lightweight single-binary exporter — no CUDA dependency, uses nvidia-smi. +# ----------------------------------------------------------------------- + +- name: Create nvidia_gpu_exporter install directory + ansible.builtin.file: + path: /opt/nvidia_gpu_exporter + state: directory + owner: root + group: root + mode: "0755" + +- name: Download nvidia_gpu_exporter binary + ansible.builtin.get_url: + url: "https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v{{ llm_gpu_exporter_version }}/nvidia_gpu_exporter_{{ llm_gpu_exporter_version }}_linux_x86_64.tar.gz" + dest: "/tmp/nvidia_gpu_exporter.tar.gz" + mode: "0644" + register: gpu_exporter_download + +- name: Extract nvidia_gpu_exporter binary + ansible.builtin.unarchive: + src: /tmp/nvidia_gpu_exporter.tar.gz + dest: /opt/nvidia_gpu_exporter + remote_src: true + creates: /opt/nvidia_gpu_exporter/nvidia_gpu_exporter + +- name: Deploy nvidia_gpu_exporter systemd service + ansible.builtin.copy: + dest: /etc/systemd/system/nvidia-gpu-exporter.service + mode: "0644" + content: | + [Unit] + Description=NVIDIA GPU Prometheus Exporter + After=network.target nvidia-persistenced.service + Wants=nvidia-persistenced.service + + [Service] + Type=simple + ExecStart=/opt/nvidia_gpu_exporter/nvidia_gpu_exporter \ + --web.listen-address=:{{ llm_gpu_exporter_port }} + Restart=on-failure + RestartSec=5 + StandardOutput=journal + StandardError=journal + SyslogIdentifier=nvidia-gpu-exporter + + [Install] + WantedBy=multi-user.target + notify: + - reload systemd + - restart nvidia-gpu-exporter + +- name: Flush handlers before starting gpu exporter + ansible.builtin.meta: flush_handlers + +- name: Enable and start nvidia-gpu-exporter + ansible.builtin.systemd: + name: nvidia-gpu-exporter + state: started + enabled: true + daemon_reload: true + +- name: Verify nvidia_gpu_exporter is reachable + ansible.builtin.uri: + url: "http://localhost:{{ llm_gpu_exporter_port }}/metrics" + status_code: 200 + register: gpu_exporter_health + retries: 6 + delay: 5 + until: gpu_exporter_health.status == 200 + changed_when: false + +# ----------------------------------------------------------------------- +# 3. Patch additionalScrapeConfigs secret on carousel-of-progress +# Adds three new scrape jobs: node, gpu, vllm +# ----------------------------------------------------------------------- + +- name: Read current additionalScrapeConfigs from Prometheus secret + ansible.builtin.command: + cmd: > + kubectl get secret monitoring-kube-prometheus-prometheus-scrape-confg + -n monitoring + -o jsonpath='{.data.additional-scrape-configs\.yaml}' + register: current_scrape_b64 + changed_when: false + delegate_to: carousel-of-progress + become: false + +- name: Decode current scrape configs + ansible.builtin.set_fact: + current_scrape_yaml: "{{ current_scrape_b64.stdout | b64decode }}" + +- name: Check if astro-orbiter scrape jobs already present + ansible.builtin.set_fact: + scrape_already_patched: "{{ 'astro-orbiter' in current_scrape_yaml }}" + +- name: Append astro-orbiter scrape jobs to additionalScrapeConfigs + when: not scrape_already_patched + block: + - name: Build new scrape config with astro-orbiter jobs appended + ansible.builtin.set_fact: + new_scrape_yaml: | + {{ current_scrape_yaml }} + - job_name: node-astro-orbiter + scrape_interval: 30s + static_configs: + - labels: + hostname: astro-orbiter + targets: + - {{ hostvars['astro-orbiter']['ansible_host'] }}:9100 + - job_name: gpu-astro-orbiter + scrape_interval: 15s + static_configs: + - labels: + hostname: astro-orbiter + gpu: rtx3090 + targets: + - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_gpu_exporter_port }} + - job_name: vllm-astro-orbiter + scrape_interval: 15s + metrics_path: /metrics + static_configs: + - labels: + hostname: astro-orbiter + model: "{{ llm_hf_model }}" + targets: + - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_serve_port }} + + - name: Patch Prometheus additionalScrapeConfigs secret + ansible.builtin.command: + cmd: > + kubectl patch secret monitoring-kube-prometheus-prometheus-scrape-confg + -n monitoring + --type=json + -p='[{"op":"replace","path":"/data/additional-scrape-configs.yaml","value":"{{ new_scrape_yaml | b64encode }}"}]' + delegate_to: carousel-of-progress + become: false + +# ----------------------------------------------------------------------- +# 4. Deploy Grafana dashboard +# ----------------------------------------------------------------------- + +- name: Deploy LLM inference Grafana dashboard ConfigMap + ansible.builtin.template: + src: grafana-llm-dashboard.json.j2 + dest: /tmp/grafana-llm-dashboard-cm.yaml + delegate_to: carousel-of-progress + become: false + +- name: Apply Grafana dashboard ConfigMap to cluster + ansible.builtin.command: + cmd: kubectl apply -f /tmp/grafana-llm-dashboard-cm.yaml + delegate_to: carousel-of-progress + become: false + changed_when: true diff --git a/ansible/roles/llm-inference/templates/grafana-llm-dashboard.json.j2 b/ansible/roles/llm-inference/templates/grafana-llm-dashboard.json.j2 new file mode 100644 index 0000000..a9598ef --- /dev/null +++ b/ansible/roles/llm-inference/templates/grafana-llm-dashboard.json.j2 @@ -0,0 +1,293 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-llm-inference-dashboard + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + llm-inference.json: | + { + "title": "LLM Inference — astro-orbiter", + "uid": "llm-astro-orbiter", + "timezone": "browser", + "refresh": "30s", + "schemaVersion": 38, + "tags": ["llm", "gpu", "vllm", "astro-orbiter"], + "time": { "from": "now-1h", "to": "now" }, + "templating": { + "list": [ + { + "name": "instance", + "type": "constant", + "label": "Host", + "query": "{{ hostvars['astro-orbiter']['ansible_host'] }}", + "hide": 0 + } + ] + }, + "panels": [ + { + "id": 1, + "title": "GPU Utilization %", + "type": "timeseries", + "gridPos": { "x": 0, "y": 0, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, "max": 100, + "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + } + } + }, + "targets": [ + { + "expr": "nvidia_smi_utilization_gpu_ratio{hostname=\"astro-orbiter\"} * 100", + "legendFormat": "GPU Util" + } + ] + }, + { + "id": 2, + "title": "GPU VRAM Used", + "type": "timeseries", + "gridPos": { "x": 8, "y": 0, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { "mode": "palette-classic" } + } + }, + "targets": [ + { + "expr": "nvidia_smi_memory_used_bytes{hostname=\"astro-orbiter\"}", + "legendFormat": "VRAM Used" + }, + { + "expr": "nvidia_smi_memory_total_bytes{hostname=\"astro-orbiter\"}", + "legendFormat": "VRAM Total" + } + ] + }, + { + "id": 3, + "title": "GPU Temperature", + "type": "gauge", + "gridPos": { "x": 16, "y": 0, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 0, "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 85 } + ] + } + } + }, + "targets": [ + { + "expr": "nvidia_smi_temperature_gpu{hostname=\"astro-orbiter\"}", + "legendFormat": "GPU Temp" + } + ] + }, + { + "id": 4, + "title": "GPU Power Draw", + "type": "timeseries", + "gridPos": { "x": 0, "y": 8, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "color": { "mode": "palette-classic" } + } + }, + "targets": [ + { + "expr": "nvidia_smi_power_draw_watts{hostname=\"astro-orbiter\"}", + "legendFormat": "Power Draw" + }, + { + "expr": "nvidia_smi_power_limit_watts{hostname=\"astro-orbiter\"}", + "legendFormat": "Power Limit" + } + ] + }, + { + "id": 5, + "title": "vLLM — Token Throughput", + "type": "timeseries", + "gridPos": { "x": 8, "y": 8, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { "mode": "palette-classic" } + } + }, + "targets": [ + { + "expr": "rate(vllm:generation_tokens_total{hostname=\"astro-orbiter\"}[1m])", + "legendFormat": "Tokens/s (gen)" + }, + { + "expr": "rate(vllm:prompt_tokens_total{hostname=\"astro-orbiter\"}[1m])", + "legendFormat": "Tokens/s (prompt)" + } + ] + }, + { + "id": 6, + "title": "vLLM — Request Queue Depth", + "type": "timeseries", + "gridPos": { "x": 16, "y": 8, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "palette-classic" } + } + }, + "targets": [ + { + "expr": "vllm:num_requests_running{hostname=\"astro-orbiter\"}", + "legendFormat": "Running" + }, + { + "expr": "vllm:num_requests_waiting{hostname=\"astro-orbiter\"}", + "legendFormat": "Waiting" + }, + { + "expr": "vllm:num_requests_swapped{hostname=\"astro-orbiter\"}", + "legendFormat": "Swapped" + } + ] + }, + { + "id": 7, + "title": "vLLM — E2E Request Latency (p50/p95/p99)", + "type": "timeseries", + "gridPos": { "x": 0, "y": 16, "w": 12, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { "mode": "palette-classic" } + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, rate(vllm:e2e_request_latency_seconds_bucket{hostname=\"astro-orbiter\"}[5m]))", + "legendFormat": "p50" + }, + { + "expr": "histogram_quantile(0.95, rate(vllm:e2e_request_latency_seconds_bucket{hostname=\"astro-orbiter\"}[5m]))", + "legendFormat": "p95" + }, + { + "expr": "histogram_quantile(0.99, rate(vllm:e2e_request_latency_seconds_bucket{hostname=\"astro-orbiter\"}[5m]))", + "legendFormat": "p99" + } + ] + }, + { + "id": 8, + "title": "vLLM — KV Cache Utilization %", + "type": "timeseries", + "gridPos": { "x": 12, "y": 16, "w": 12, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 75 }, + { "color": "red", "value": 90 } + ] + } + } + }, + "targets": [ + { + "expr": "vllm:gpu_cache_usage_perc{hostname=\"astro-orbiter\"} * 100", + "legendFormat": "KV Cache %" + } + ] + }, + { + "id": 9, + "title": "System — CPU Usage %", + "type": "timeseries", + "gridPos": { "x": 0, "y": 24, "w": 8, "h": 7 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, "max": 100 + } + }, + "targets": [ + { + "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\",instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"}[1m])) * 100)", + "legendFormat": "CPU Used" + } + ] + }, + { + "id": 10, + "title": "System — Memory Usage", + "type": "timeseries", + "gridPos": { "x": 8, "y": 24, "w": 8, "h": 7 }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"} - node_memory_MemAvailable_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"}", + "legendFormat": "Used" + }, + { + "expr": "node_memory_MemTotal_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"}", + "legendFormat": "Total" + } + ] + }, + { + "id": 11, + "title": "System — Disk Usage (root)", + "type": "gauge", + "gridPos": { "x": 16, "y": 24, "w": 8, "h": 7 }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.75 }, + { "color": "red", "value": 0.90 } + ] + } + } + }, + "targets": [ + { + "expr": "1 - (node_filesystem_avail_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\",mountpoint=\"/\"} / node_filesystem_size_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\",mountpoint=\"/\"})", + "legendFormat": "Root disk" + } + ] + } + ] + }