homelab/ansible/roles/llm-inference/tasks/monitoring.yml

---
# ------------------------------------------------------------------------------
# FILE: roles/llm-inference/tasks/monitoring.yml
# DESCRIPTION: Phase 7 — Prometheus monitoring for the LLM inference stack.
#              Deploys three metric sources on astro-orbiter:
#
#              1. node_exporter (port 9100) — system: CPU, RAM, disk, network
#              2. nvidia_gpu_exporter (port 9835) — GPU: VRAM, temp, util, power
#              3. vLLM built-in metrics (port 8000/metrics) — already exposed
#                 by vLLM; just needs a scrape job (no extra process)
#
#              Wires all three into Prometheus via additionalScrapeConfigs on
#              the kube-prometheus-stack secret on carousel-of-progress.
#              Deploys a Grafana dashboard ConfigMap in the monitoring namespace.
# ------------------------------------------------------------------------------

# -----------------------------------------------------------------------
# 1. node_exporter
# -----------------------------------------------------------------------

- name: Install prometheus-node-exporter
  ansible.builtin.apt:
    name: prometheus-node-exporter
    state: present
    update_cache: false

- name: Enable and start node_exporter
  ansible.builtin.systemd:
    name: prometheus-node-exporter
    state: started
    enabled: true

- name: Verify node_exporter is reachable
  ansible.builtin.uri:
    url: "http://localhost:9100/metrics"
    status_code: 200
  register: node_exporter_health
  retries: 6
  delay: 5
  until: node_exporter_health.status == 200
  changed_when: false

# -----------------------------------------------------------------------
# 2. nvidia_gpu_exporter (utkuozdemir/nvidia_gpu_exporter)
#    Lightweight single-binary exporter — no CUDA dependency, uses nvidia-smi.
# -----------------------------------------------------------------------

- name: Create nvidia_gpu_exporter install directory
  ansible.builtin.file:
    path: /opt/nvidia_gpu_exporter
    state: directory
    owner: root
    group: root
    mode: "0755"

- name: Download nvidia_gpu_exporter binary
  ansible.builtin.get_url:
    url: "https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v{{ llm_gpu_exporter_version }}/nvidia_gpu_exporter_{{ llm_gpu_exporter_version }}_linux_x86_64.tar.gz"
    dest: "/tmp/nvidia_gpu_exporter.tar.gz"
    mode: "0644"
  register: gpu_exporter_download

- name: Extract nvidia_gpu_exporter binary
  ansible.builtin.unarchive:
    src: /tmp/nvidia_gpu_exporter.tar.gz
    dest: /opt/nvidia_gpu_exporter
    remote_src: true
    creates: /opt/nvidia_gpu_exporter/nvidia_gpu_exporter

- name: Deploy nvidia_gpu_exporter systemd service
  ansible.builtin.copy:
    dest: /etc/systemd/system/nvidia-gpu-exporter.service
    mode: "0644"
    content: |
      [Unit]
      Description=NVIDIA GPU Prometheus Exporter
      After=network.target nvidia-persistenced.service
      Wants=nvidia-persistenced.service

      [Service]
      Type=simple
      ExecStart=/opt/nvidia_gpu_exporter/nvidia_gpu_exporter \
          --web.listen-address=:{{ llm_gpu_exporter_port }}
      Restart=on-failure
      RestartSec=5
      StandardOutput=journal
      StandardError=journal
      SyslogIdentifier=nvidia-gpu-exporter

      [Install]
      WantedBy=multi-user.target
  notify:
    - reload systemd
    - restart nvidia-gpu-exporter

- name: Flush handlers before starting gpu exporter
  ansible.builtin.meta: flush_handlers

- name: Enable and start nvidia-gpu-exporter
  ansible.builtin.systemd:
    name: nvidia-gpu-exporter
    state: started
    enabled: true
    daemon_reload: true

- name: Verify nvidia_gpu_exporter is reachable
  ansible.builtin.uri:
    url: "http://localhost:{{ llm_gpu_exporter_port }}/metrics"
    status_code: 200
  register: gpu_exporter_health
  retries: 6
  delay: 5
  until: gpu_exporter_health.status == 200
  changed_when: false

# -----------------------------------------------------------------------
# 3. Patch additionalScrapeConfigs secret on carousel-of-progress
#    Adds three new scrape jobs: node, gpu, vllm
# -----------------------------------------------------------------------

- name: Read current additionalScrapeConfigs from Prometheus secret
  ansible.builtin.command:
    cmd: >
      kubectl get secret monitoring-kube-prometheus-prometheus-scrape-confg
      -n monitoring
      -o jsonpath='{.data.additional-scrape-configs\.yaml}'
  register: current_scrape_b64
  changed_when: false
  delegate_to: carousel-of-progress
  become: false

- name: Decode current scrape configs
  ansible.builtin.set_fact:
    current_scrape_yaml: "{{ current_scrape_b64.stdout | b64decode }}"

- name: Check if astro-orbiter scrape jobs already present
  ansible.builtin.set_fact:
    scrape_already_patched: "{{ 'astro-orbiter' in current_scrape_yaml }}"

- name: Append astro-orbiter scrape jobs to additionalScrapeConfigs
  when: not scrape_already_patched
  block:
    - name: Build new scrape config with astro-orbiter jobs appended
      ansible.builtin.set_fact:
        new_scrape_yaml: |
          {{ current_scrape_yaml }}
          - job_name: node-astro-orbiter
            scrape_interval: 30s
            static_configs:
            - labels:
                hostname: astro-orbiter
              targets:
              - {{ hostvars['astro-orbiter']['ansible_host'] }}:9100
          - job_name: gpu-astro-orbiter
            scrape_interval: 15s
            static_configs:
            - labels:
                hostname: astro-orbiter
                gpu: rtx3090
              targets:
              - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_gpu_exporter_port }}
          - job_name: vllm-astro-orbiter
            scrape_interval: 15s
            metrics_path: /metrics
            static_configs:
            - labels:
                hostname: astro-orbiter
                model: "{{ llm_hf_model }}"
              targets:
              - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_serve_port }}

    - name: Patch Prometheus additionalScrapeConfigs secret
      ansible.builtin.command:
        cmd: >
          kubectl patch secret monitoring-kube-prometheus-prometheus-scrape-confg
          -n monitoring
          --type=json
          -p='[{"op":"replace","path":"/data/additional-scrape-configs.yaml","value":"{{ new_scrape_yaml | b64encode }}"}]'
      delegate_to: carousel-of-progress
      become: false

# -----------------------------------------------------------------------
# 4. Deploy Grafana dashboard
# -----------------------------------------------------------------------

- name: Deploy LLM inference Grafana dashboard ConfigMap
  ansible.builtin.template:
    src: grafana-llm-dashboard.json.j2
    dest: /tmp/grafana-llm-dashboard-cm.yaml
  delegate_to: carousel-of-progress
  become: false

- name: Apply Grafana dashboard ConfigMap to cluster
  ansible.builtin.command:
    cmd: kubectl apply -f /tmp/grafana-llm-dashboard-cm.yaml
  delegate_to: carousel-of-progress
  become: false
  changed_when: true