feat(llm-inference): Phase 7 — Prometheus monitoring + Grafana dashboard

- Phase 7 task file: monitoring.yml
  - node_exporter (port 9100) via apt, systemd managed
  - nvidia_gpu_exporter v1.2.2 (port 9835) — GPU util, VRAM, temp, power
  - Patches kube-prometheus additionalScrapeConfigs secret with 3 new jobs:
    node-astro-orbiter, gpu-astro-orbiter, vllm-astro-orbiter
  - Deploys Grafana dashboard ConfigMap via kubectl apply

- Grafana dashboard (11 panels):
  - Row 1: GPU util %, VRAM used, GPU temp gauge
  - Row 2: GPU power draw, vLLM token throughput, request queue depth
  - Row 3: vLLM e2e latency p50/p95/p99, KV cache utilization %
  - Row 4: System CPU %, memory, root disk gauge

- defaults/main.yml: llm_gpu_exporter_version, llm_gpu_exporter_port
- handlers/main.yml: restart nvidia-gpu-exporter

ansible/roles/llm-inference/defaults/main.yml

@@ -22,3 +22,7 @@ llm_serve_port: 8000
 llm_serve_host: "0.0.0.0"
 llm_gpu_memory_utilization: "0.90"
 llm_max_model_len: 8192
+
+# Monitoring
+llm_gpu_exporter_version: "1.2.2"
+llm_gpu_exporter_port: 9835

ansible/roles/llm-inference/handlers/main.yml

@@ -27,3 +27,9 @@
   delegate_to: carousel-of-progress
   listen: "restart hermes"
   ignore_errors: true
+
+- name: Restart nvidia-gpu-exporter
+  ansible.builtin.systemd:
+    name: nvidia-gpu-exporter
+    state: restarted
+  listen: "restart nvidia-gpu-exporter"

ansible/roles/llm-inference/tasks/main.yml

@@ -22,3 +22,6 @@
 
 # Phase 6 — Integration
 - import_tasks: integration.yml
+
+# Phase 7 — Monitoring
+- import_tasks: monitoring.yml

ansible/roles/llm-inference/tasks/monitoring.yml

@@ -0,0 +1,198 @@
+---
+# ------------------------------------------------------------------------------
+# FILE: roles/llm-inference/tasks/monitoring.yml
+# DESCRIPTION: Phase 7 — Prometheus monitoring for the LLM inference stack.
+#              Deploys three metric sources on astro-orbiter:
+#
+#              1. node_exporter (port 9100) — system: CPU, RAM, disk, network
+#              2. nvidia_gpu_exporter (port 9835) — GPU: VRAM, temp, util, power
+#              3. vLLM built-in metrics (port 8000/metrics) — already exposed
+#                 by vLLM; just needs a scrape job (no extra process)
+#
+#              Wires all three into Prometheus via additionalScrapeConfigs on
+#              the kube-prometheus-stack secret on carousel-of-progress.
+#              Deploys a Grafana dashboard ConfigMap in the monitoring namespace.
+# ------------------------------------------------------------------------------
+
+# -----------------------------------------------------------------------
+# 1. node_exporter
+# -----------------------------------------------------------------------
+
+- name: Install prometheus-node-exporter
+  ansible.builtin.apt:
+    name: prometheus-node-exporter
+    state: present
+    update_cache: false
+
+- name: Enable and start node_exporter
+  ansible.builtin.systemd:
+    name: prometheus-node-exporter
+    state: started
+    enabled: true
+
+- name: Verify node_exporter is reachable
+  ansible.builtin.uri:
+    url: "http://localhost:9100/metrics"
+    status_code: 200
+  register: node_exporter_health
+  retries: 6
+  delay: 5
+  until: node_exporter_health.status == 200
+  changed_when: false
+
+# -----------------------------------------------------------------------
+# 2. nvidia_gpu_exporter (utkuozdemir/nvidia_gpu_exporter)
+#    Lightweight single-binary exporter — no CUDA dependency, uses nvidia-smi.
+# -----------------------------------------------------------------------
+
+- name: Create nvidia_gpu_exporter install directory
+  ansible.builtin.file:
+    path: /opt/nvidia_gpu_exporter
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Download nvidia_gpu_exporter binary
+  ansible.builtin.get_url:
+    url: "https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v{{ llm_gpu_exporter_version }}/nvidia_gpu_exporter_{{ llm_gpu_exporter_version }}_linux_x86_64.tar.gz"
+    dest: "/tmp/nvidia_gpu_exporter.tar.gz"
+    mode: "0644"
+  register: gpu_exporter_download
+
+- name: Extract nvidia_gpu_exporter binary
+  ansible.builtin.unarchive:
+    src: /tmp/nvidia_gpu_exporter.tar.gz
+    dest: /opt/nvidia_gpu_exporter
+    remote_src: true
+    creates: /opt/nvidia_gpu_exporter/nvidia_gpu_exporter
+
+- name: Deploy nvidia_gpu_exporter systemd service
+  ansible.builtin.copy:
+    dest: /etc/systemd/system/nvidia-gpu-exporter.service
+    mode: "0644"
+    content: |
+      [Unit]
+      Description=NVIDIA GPU Prometheus Exporter
+      After=network.target nvidia-persistenced.service
+      Wants=nvidia-persistenced.service
+
+      [Service]
+      Type=simple
+      ExecStart=/opt/nvidia_gpu_exporter/nvidia_gpu_exporter \
+          --web.listen-address=:{{ llm_gpu_exporter_port }}
+      Restart=on-failure
+      RestartSec=5
+      StandardOutput=journal
+      StandardError=journal
+      SyslogIdentifier=nvidia-gpu-exporter
+
+      [Install]
+      WantedBy=multi-user.target
+  notify:
+    - reload systemd
+    - restart nvidia-gpu-exporter
+
+- name: Flush handlers before starting gpu exporter
+  ansible.builtin.meta: flush_handlers
+
+- name: Enable and start nvidia-gpu-exporter
+  ansible.builtin.systemd:
+    name: nvidia-gpu-exporter
+    state: started
+    enabled: true
+    daemon_reload: true
+
+- name: Verify nvidia_gpu_exporter is reachable
+  ansible.builtin.uri:
+    url: "http://localhost:{{ llm_gpu_exporter_port }}/metrics"
+    status_code: 200
+  register: gpu_exporter_health
+  retries: 6
+  delay: 5
+  until: gpu_exporter_health.status == 200
+  changed_when: false
+
+# -----------------------------------------------------------------------
+# 3. Patch additionalScrapeConfigs secret on carousel-of-progress
+#    Adds three new scrape jobs: node, gpu, vllm
+# -----------------------------------------------------------------------
+
+- name: Read current additionalScrapeConfigs from Prometheus secret
+  ansible.builtin.command:
+    cmd: >
+      kubectl get secret monitoring-kube-prometheus-prometheus-scrape-confg
+      -n monitoring
+      -o jsonpath='{.data.additional-scrape-configs\.yaml}'
+  register: current_scrape_b64
+  changed_when: false
+  delegate_to: carousel-of-progress
+  become: false
+
+- name: Decode current scrape configs
+  ansible.builtin.set_fact:
+    current_scrape_yaml: "{{ current_scrape_b64.stdout | b64decode }}"
+
+- name: Check if astro-orbiter scrape jobs already present
+  ansible.builtin.set_fact:
+    scrape_already_patched: "{{ 'astro-orbiter' in current_scrape_yaml }}"
+
+- name: Append astro-orbiter scrape jobs to additionalScrapeConfigs
+  when: not scrape_already_patched
+  block:
+    - name: Build new scrape config with astro-orbiter jobs appended
+      ansible.builtin.set_fact:
+        new_scrape_yaml: |
+          {{ current_scrape_yaml }}
+          - job_name: node-astro-orbiter
+            scrape_interval: 30s
+            static_configs:
+            - labels:
+                hostname: astro-orbiter
+              targets:
+              - {{ hostvars['astro-orbiter']['ansible_host'] }}:9100
+          - job_name: gpu-astro-orbiter
+            scrape_interval: 15s
+            static_configs:
+            - labels:
+                hostname: astro-orbiter
+                gpu: rtx3090
+              targets:
+              - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_gpu_exporter_port }}
+          - job_name: vllm-astro-orbiter
+            scrape_interval: 15s
+            metrics_path: /metrics
+            static_configs:
+            - labels:
+                hostname: astro-orbiter
+                model: "{{ llm_hf_model }}"
+              targets:
+              - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_serve_port }}
+
+    - name: Patch Prometheus additionalScrapeConfigs secret
+      ansible.builtin.command:
+        cmd: >
+          kubectl patch secret monitoring-kube-prometheus-prometheus-scrape-confg
+          -n monitoring
+          --type=json
+          -p='[{"op":"replace","path":"/data/additional-scrape-configs.yaml","value":"{{ new_scrape_yaml | b64encode }}"}]'
+      delegate_to: carousel-of-progress
+      become: false
+
+# -----------------------------------------------------------------------
+# 4. Deploy Grafana dashboard
+# -----------------------------------------------------------------------
+
+- name: Deploy LLM inference Grafana dashboard ConfigMap
+  ansible.builtin.template:
+    src: grafana-llm-dashboard.json.j2
+    dest: /tmp/grafana-llm-dashboard-cm.yaml
+  delegate_to: carousel-of-progress
+  become: false
+
+- name: Apply Grafana dashboard ConfigMap to cluster
+  ansible.builtin.command:
+    cmd: kubectl apply -f /tmp/grafana-llm-dashboard-cm.yaml
+  delegate_to: carousel-of-progress
+  become: false
+  changed_when: true

ansible/roles/llm-inference/templates/grafana-llm-dashboard.json.j2

@@ -0,0 +1,293 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-llm-inference-dashboard
+  namespace: monitoring
+  labels:
+    grafana_dashboard: "1"
+data:
+  llm-inference.json: |
+    {
+      "title": "LLM Inference — astro-orbiter",
+      "uid": "llm-astro-orbiter",
+      "timezone": "browser",
+      "refresh": "30s",
+      "schemaVersion": 38,
+      "tags": ["llm", "gpu", "vllm", "astro-orbiter"],
+      "time": { "from": "now-1h", "to": "now" },
+      "templating": {
+        "list": [
+          {
+            "name": "instance",
+            "type": "constant",
+            "label": "Host",
+            "query": "{{ hostvars['astro-orbiter']['ansible_host'] }}",
+            "hide": 0
+          }
+        ]
+      },
+      "panels": [
+        {
+          "id": 1,
+          "title": "GPU Utilization %",
+          "type": "timeseries",
+          "gridPos": { "x": 0, "y": 0, "w": 8, "h": 8 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "percent",
+              "min": 0, "max": 100,
+              "color": { "mode": "palette-classic" },
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "yellow", "value": 70 },
+                  { "color": "red", "value": 90 }
+                ]
+              }
+            }
+          },
+          "targets": [
+            {
+              "expr": "nvidia_smi_utilization_gpu_ratio{hostname=\"astro-orbiter\"} * 100",
+              "legendFormat": "GPU Util"
+            }
+          ]
+        },
+        {
+          "id": 2,
+          "title": "GPU VRAM Used",
+          "type": "timeseries",
+          "gridPos": { "x": 8, "y": 0, "w": 8, "h": 8 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "bytes",
+              "color": { "mode": "palette-classic" }
+            }
+          },
+          "targets": [
+            {
+              "expr": "nvidia_smi_memory_used_bytes{hostname=\"astro-orbiter\"}",
+              "legendFormat": "VRAM Used"
+            },
+            {
+              "expr": "nvidia_smi_memory_total_bytes{hostname=\"astro-orbiter\"}",
+              "legendFormat": "VRAM Total"
+            }
+          ]
+        },
+        {
+          "id": 3,
+          "title": "GPU Temperature",
+          "type": "gauge",
+          "gridPos": { "x": 16, "y": 0, "w": 8, "h": 8 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "celsius",
+              "min": 0, "max": 100,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "yellow", "value": 70 },
+                  { "color": "red", "value": 85 }
+                ]
+              }
+            }
+          },
+          "targets": [
+            {
+              "expr": "nvidia_smi_temperature_gpu{hostname=\"astro-orbiter\"}",
+              "legendFormat": "GPU Temp"
+            }
+          ]
+        },
+        {
+          "id": 4,
+          "title": "GPU Power Draw",
+          "type": "timeseries",
+          "gridPos": { "x": 0, "y": 8, "w": 8, "h": 8 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "watt",
+              "color": { "mode": "palette-classic" }
+            }
+          },
+          "targets": [
+            {
+              "expr": "nvidia_smi_power_draw_watts{hostname=\"astro-orbiter\"}",
+              "legendFormat": "Power Draw"
+            },
+            {
+              "expr": "nvidia_smi_power_limit_watts{hostname=\"astro-orbiter\"}",
+              "legendFormat": "Power Limit"
+            }
+          ]
+        },
+        {
+          "id": 5,
+          "title": "vLLM — Token Throughput",
+          "type": "timeseries",
+          "gridPos": { "x": 8, "y": 8, "w": 8, "h": 8 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "reqps",
+              "color": { "mode": "palette-classic" }
+            }
+          },
+          "targets": [
+            {
+              "expr": "rate(vllm:generation_tokens_total{hostname=\"astro-orbiter\"}[1m])",
+              "legendFormat": "Tokens/s (gen)"
+            },
+            {
+              "expr": "rate(vllm:prompt_tokens_total{hostname=\"astro-orbiter\"}[1m])",
+              "legendFormat": "Tokens/s (prompt)"
+            }
+          ]
+        },
+        {
+          "id": 6,
+          "title": "vLLM — Request Queue Depth",
+          "type": "timeseries",
+          "gridPos": { "x": 16, "y": 8, "w": 8, "h": 8 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "short",
+              "color": { "mode": "palette-classic" }
+            }
+          },
+          "targets": [
+            {
+              "expr": "vllm:num_requests_running{hostname=\"astro-orbiter\"}",
+              "legendFormat": "Running"
+            },
+            {
+              "expr": "vllm:num_requests_waiting{hostname=\"astro-orbiter\"}",
+              "legendFormat": "Waiting"
+            },
+            {
+              "expr": "vllm:num_requests_swapped{hostname=\"astro-orbiter\"}",
+              "legendFormat": "Swapped"
+            }
+          ]
+        },
+        {
+          "id": 7,
+          "title": "vLLM — E2E Request Latency (p50/p95/p99)",
+          "type": "timeseries",
+          "gridPos": { "x": 0, "y": 16, "w": 12, "h": 8 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "s",
+              "color": { "mode": "palette-classic" }
+            }
+          },
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.50, rate(vllm:e2e_request_latency_seconds_bucket{hostname=\"astro-orbiter\"}[5m]))",
+              "legendFormat": "p50"
+            },
+            {
+              "expr": "histogram_quantile(0.95, rate(vllm:e2e_request_latency_seconds_bucket{hostname=\"astro-orbiter\"}[5m]))",
+              "legendFormat": "p95"
+            },
+            {
+              "expr": "histogram_quantile(0.99, rate(vllm:e2e_request_latency_seconds_bucket{hostname=\"astro-orbiter\"}[5m]))",
+              "legendFormat": "p99"
+            }
+          ]
+        },
+        {
+          "id": 8,
+          "title": "vLLM — KV Cache Utilization %",
+          "type": "timeseries",
+          "gridPos": { "x": 12, "y": 16, "w": 12, "h": 8 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "percent",
+              "min": 0, "max": 100,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "yellow", "value": 75 },
+                  { "color": "red", "value": 90 }
+                ]
+              }
+            }
+          },
+          "targets": [
+            {
+              "expr": "vllm:gpu_cache_usage_perc{hostname=\"astro-orbiter\"} * 100",
+              "legendFormat": "KV Cache %"
+            }
+          ]
+        },
+        {
+          "id": 9,
+          "title": "System — CPU Usage %",
+          "type": "timeseries",
+          "gridPos": { "x": 0, "y": 24, "w": 8, "h": 7 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "percent",
+              "min": 0, "max": 100
+            }
+          },
+          "targets": [
+            {
+              "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\",instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"}[1m])) * 100)",
+              "legendFormat": "CPU Used"
+            }
+          ]
+        },
+        {
+          "id": 10,
+          "title": "System — Memory Usage",
+          "type": "timeseries",
+          "gridPos": { "x": 8, "y": 24, "w": 8, "h": 7 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "bytes"
+            }
+          },
+          "targets": [
+            {
+              "expr": "node_memory_MemTotal_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"} - node_memory_MemAvailable_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"}",
+              "legendFormat": "Used"
+            },
+            {
+              "expr": "node_memory_MemTotal_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"}",
+              "legendFormat": "Total"
+            }
+          ]
+        },
+        {
+          "id": 11,
+          "title": "System — Disk Usage (root)",
+          "type": "gauge",
+          "gridPos": { "x": 16, "y": 24, "w": 8, "h": 7 },
+          "fieldConfig": {
+            "defaults": {
+              "unit": "percentunit",
+              "min": 0, "max": 1,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "yellow", "value": 0.75 },
+                  { "color": "red", "value": 0.90 }
+                ]
+              }
+            }
+          },
+          "targets": [
+            {
+              "expr": "1 - (node_filesystem_avail_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\",mountpoint=\"/\"} / node_filesystem_size_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\",mountpoint=\"/\"})",
+              "legendFormat": "Root disk"
+            }
+          ]
+        }
+      ]
+    }