---
# ------------------------------------------------------------------------------
# FILE: roles/llm-inference/tasks/monitoring.yml
# DESCRIPTION: Phase 7 — Prometheus monitoring for the LLM inference stack.
#              Deploys two metric-producing exporters on astro-orbiter:
#
#              1. node_exporter (port 9100) — system: CPU, RAM, disk, network
#              2. nvidia_gpu_exporter (port 9835) — GPU: VRAM, temp, util, power
#              3. llama-server built-in metrics (port 8000/metrics, enabled via
#                 --metrics) — just needs a scrape job (no extra process)
#
#              GitOps note: the Prometheus scrape jobs for all three targets and
#              the Grafana dashboard are declared in the homelab Git repo and
#              applied by ArgoCD — NOT by this role:
#                - cluster/applications/monitoring/values.yaml
#                  (prometheus.prometheusSpec.additionalScrapeConfigs)
#                - cluster/applications/monitoring/dashboards.yaml
#                  (grafana-llm-inference-dashboard ConfigMap)
#              This role's job is only to stand up the two exporters + verify
#              they're reachable. Do NOT reintroduce kubectl patch/apply tasks
#              here — cluster-facing changes go through Git commit + ArgoCD
#              sync so state stays reproducible and self-healing.
# ------------------------------------------------------------------------------

# -----------------------------------------------------------------------
# 1. node_exporter
# -----------------------------------------------------------------------

- name: Install prometheus-node-exporter
  ansible.builtin.apt:
    name: prometheus-node-exporter
    state: present
    update_cache: false

- name: Enable and start node_exporter
  ansible.builtin.systemd:
    name: prometheus-node-exporter
    state: started
    enabled: true

- name: Verify node_exporter is reachable
  ansible.builtin.uri:
    url: "http://localhost:9100/metrics"
    status_code: 200
  register: node_exporter_health
  retries: 6
  delay: 5
  until: node_exporter_health.status == 200
  changed_when: false

# -----------------------------------------------------------------------
# 2. nvidia_gpu_exporter (utkuozdemir/nvidia_gpu_exporter)
#    Lightweight single-binary exporter — no CUDA dependency, uses nvidia-smi.
# -----------------------------------------------------------------------

- name: Create nvidia_gpu_exporter install directory
  ansible.builtin.file:
    path: /opt/nvidia_gpu_exporter
    state: directory
    owner: root
    group: root
    mode: "0755"

- name: Download nvidia_gpu_exporter binary
  ansible.builtin.get_url:
    url: "https://github.com/utkuozdemir/nvidia_gpu_exporter/releases/download/v{{ llm_gpu_exporter_version }}/nvidia_gpu_exporter_{{ llm_gpu_exporter_version }}_linux_x86_64.tar.gz"
    dest: "/tmp/nvidia_gpu_exporter.tar.gz"
    mode: "0644"
  register: gpu_exporter_download

- name: Extract nvidia_gpu_exporter binary
  ansible.builtin.unarchive:
    src: /tmp/nvidia_gpu_exporter.tar.gz
    dest: /opt/nvidia_gpu_exporter
    remote_src: true
    creates: /opt/nvidia_gpu_exporter/nvidia_gpu_exporter

- name: Deploy nvidia_gpu_exporter systemd service
  ansible.builtin.copy:
    dest: /etc/systemd/system/nvidia-gpu-exporter.service
    mode: "0644"
    content: |
      [Unit]
      Description=NVIDIA GPU Prometheus Exporter
      After=network.target nvidia-persistenced.service
      Wants=nvidia-persistenced.service

      [Service]
      Type=simple
      ExecStart=/opt/nvidia_gpu_exporter/nvidia_gpu_exporter \
          --web.listen-address=:{{ llm_gpu_exporter_port }}
      Restart=on-failure
      RestartSec=5
      StandardOutput=journal
      StandardError=journal
      SyslogIdentifier=nvidia-gpu-exporter

      [Install]
      WantedBy=multi-user.target
  notify:
    - reload systemd
    - restart nvidia-gpu-exporter

- name: Flush handlers before starting gpu exporter
  ansible.builtin.meta: flush_handlers

- name: Enable and start nvidia-gpu-exporter
  ansible.builtin.systemd:
    name: nvidia-gpu-exporter
    state: started
    enabled: true
    daemon_reload: true

- name: Verify nvidia_gpu_exporter is reachable
  ansible.builtin.uri:
    url: "http://localhost:{{ llm_gpu_exporter_port }}/metrics"
    status_code: 200
  register: gpu_exporter_health
  retries: 6
  delay: 5
  until: gpu_exporter_health.status == 200
  changed_when: false

# -----------------------------------------------------------------------
# 3. Prometheus scrape configs + Grafana dashboard
#
#    Intentionally NOT managed here. See file header: these are declared
#    in cluster/applications/monitoring/{values.yaml,dashboards.yaml} in
#    the homelab Git repo and rolled out by ArgoCD sync, keeping cluster
#    state in Git rather than mutated imperatively from the control node.
# -----------------------------------------------------------------------