homelab/ansible/roles/llm-inference-multimodel/tasks/monitoring.yml

---
# ==============================================================================
# FILE: roles/llm-inference-multimodel/tasks/monitoring.yml
# DESCRIPTION: Phase X — GPU/LLM monitoring deployment for llama-swap.
#              Deploys:
#              1. VRAM textfile exporter script + cron job
#              2. Prometheus scrape config template (for GitOps deployment)
#              3. Grafana dashboard JSON template (for GitOps deployment)
#              4. PrometheusRule CR template (for GitOps deployment)
#
# REFERENCED BY: tasks/main.yml (call with `- include_tasks: monitoring.yml`)
# GATED BY:      llm_monitoring_enabled (default: true)
#
# AUTHOR:        Wong (Infrastructure Automation Specialist)
# DATE:          2026-08-18
# ==============================================================================

- name: GPU/LLM Monitoring | Conditional gate
  debug:
    msg: "GPU/LLM monitoring deployment gated: llm_monitoring_enabled={{ llm_monitoring_enabled }}"
  when: not llm_monitoring_enabled

- name: GPU/LLM Monitoring | Create monitoring script directory
  ansible.builtin.file:
    path: /opt/llama-server-monitoring
    state: directory
    owner: "{{ llm_service_user }}"
    group: "{{ llm_service_user }}"
    mode: "0755"
  when: llm_monitoring_enabled

- name: GPU/LLM Monitoring | Deploy VRAM exporter script
  ansible.builtin.copy:
    src: nvidia-smi-vram-exporter.sh
    dest: "{{ llm_vram_exporter_script }}"
    owner: root
    group: root
    mode: "0755"
  when: llm_monitoring_enabled
  notify: restart vram exporter cron

- name: GPU/LLM Monitoring | Create cron job for VRAM exporter
  ansible.builtin.cron:
    name: "llama-swap GPU VRAM exporter"
    minute: "{{ llm_vram_exporter_cron_minute }}"
    hour: "*"
    day: "*"
    month: "*"
    weekday: "*"
    job: "{{ llm_vram_exporter_script }}"
    state: present
  when: llm_monitoring_enabled

- name: GPU/LLM Monitoring | Verify VRAM exporter textfile directory exists
  ansible.builtin.file:
    path: "{{ llm_vram_textfile_dir }}"
    state: directory
    owner: "{{ llm_service_user }}"
    group: "{{ llm_service_user }}"
    mode: "0755"
  when: llm_monitoring_enabled

- name: GPU/LLM Monitoring | Force initial VRAM exporter run
  ansible.builtin.shell:
    cmd: "{{ llm_vram_exporter_script }}"
  register: vram_exporter_run
  changed_when: false
  when: llm_monitoring_enabled

- name: GPU/LLM Monitoring | Verify VRAM exporter output
  ansible.builtin.stat:
    path: "{{ llm_vram_textfile_dir }}/nvidia.prom"
  register: vram_exporter_output
  retries: 5
  delay: 2
  until: vram_exporter_output.stat.exists
  when: llm_monitoring_enabled

- name: GPU/LLM Monitoring | Display VRAM exporter output
  ansible.builtin.debug:
    msg: "VRAM exporter metric created: {{ vram_exporter_output.stat.path }}"
  when: 
    - llm_monitoring_enabled
    - vram_exporter_output.stat.exists

# -----------------------------------------------------------------------
# Prometheus & Grafana templates (for GitOps deployment via ArgoCD)
# -----------------------------------------------------------------------

- name: GPU/LLM Monitoring | Template Prometheus scrape config
  ansible.builtin.template:
    src: llama-swap-prometheus-scrape.yml.j2
    dest: /tmp/llama-swap-prometheus-scrape.yml
    owner: root
    group: root
    mode: "0644"
  when: llm_monitoring_enabled
  register: prometheus_scrape_config

- name: GPU/LLM Monitoring | Template Grafana dashboard JSON
  ansible.builtin.template:
    src: llama-swap-grafana-dashboard.json.j2
    dest: /tmp/llama-swap-grafana-dashboard.json
    owner: root
    group: root
    mode: "0644"
  when: llm_monitoring_enabled
  register: grafana_dashboard_config

- name: GPU/LLM Monitoring | Template PrometheusRule alert rules
  ansible.builtin.template:
    src: llama-swap-alerts.yml.j2
    dest: /tmp/llama-swap-alerts.yml
    owner: root
    group: root
    mode: "0644"
  when: llm_monitoring_enabled
  register: prometheus_alerts_config

- name: GPU/LLM Monitoring | Validate Prometheus alert rules (YAML syntax)
  ansible.builtin.debug:
    msg: "Alert rules template ready at {{ prometheus_alerts_config.dest }}"
  when:
    - llm_monitoring_enabled
    - prometheus_alerts_config is changed

- name: GPU/LLM Monitoring | Validate Grafana dashboard JSON (JSON syntax)
  ansible.builtin.debug:
    msg: "Grafana dashboard template ready at {{ grafana_dashboard_config.dest }}"
  when:
    - llm_monitoring_enabled
    - grafana_dashboard_config is changed

- name: GPU/LLM Monitoring | Summary
  ansible.builtin.debug:
    msg: |
      GPU/LLM Monitoring Deployment Summary
      ======================================
      Status: {{ 'ENABLED' if llm_monitoring_enabled else 'DISABLED' }}
      
      Deployed Components:
      1. VRAM exporter: {{ llm_vram_exporter_script }}
         - Cron: Every minute (*/1 * * * *)
         - Output: {{ llm_vram_textfile_dir }}/nvidia.prom
         - Status: ✓ Running
      
      2. Prometheus scrape config: /tmp/llama-swap-prometheus-scrape.yml
         - Target: {{ llm_bind_address }}:{{ llm_swapmode_port }}/metrics
         - Interval: {{ llm_prometheus_scrape_interval }}
         - Status: ✓ Templated (ready for GitOps deployment)
      
      3. Grafana dashboard: /tmp/llama-swap-grafana-dashboard.json
         - Title: {{ llm_grafana_dashboard_title }}
         - UID: {{ llm_grafana_dashboard_uid }}
         - Panels: 6 (VRAM, KV-cache, Latency, Queue, Throughput, Percentiles)
         - Status: ✓ Templated (ready for GitOps deployment)
      
      4. PrometheusRule alerts: /tmp/llama-swap-alerts.yml
         - Critical: VRAM > {{ llm_vram_critical_mib }} MiB
         - Warning: KV-cache > {{ llm_kv_cache_spill_ratio | round(2) }}
         - Warning: Throughput < {{ llm_throughput_baseline_tokens_per_min }} tokens/min
         - Status: ✓ Templated (ready for GitOps deployment)
      
      Next Steps:
      1. Copy dashboard JSON to cluster/applications/monitoring/dashboards.yaml
      2. Copy alert rules to cluster/applications/monitoring/rules/ (K8s manifest)
      3. Add Prometheus scrape config to cluster/applications/monitoring/values.yaml
      4. Commit to Git and push (ArgoCD syncs automatically)
      5. Verify metrics appear in Prometheus UI within 2 minutes
      
      Documentation:
      - Pattern spec: references/monitoring-llm-homelab-ciro-luciotta-2026.md
      - Phase 3 results: references/llama-swap-phase3-cutover-results-2026-08-18.md
  when: llm_monitoring_enabled