feat(llm-inference): move astro-orbiter monitoring to GitOps (values.yaml + dashboards.yaml)

- Prometheus scrape configs for node/gpu/llama-server exporters on
  astro-orbiter now declared in cluster/applications/monitoring/values.yaml
  (additionalScrapeConfigs), applied via ArgoCD sync instead of an
  imperative kubectl secret patch from the Ansible role.
- Grafana dashboard for astro-orbiter LLM inference added as a ConfigMap
  in cluster/applications/monitoring/dashboards.yaml (grafana_dashboard=1
  sidecar label), replacing the role's ad-hoc kubectl apply of a rendered
  Jinja template.
- ansible/roles/llm-inference/tasks/monitoring.yml: removed the kubectl
  get/patch/apply tasks and orphaned grafana-llm-dashboard.json.j2
  template; role now only stands up node_exporter + nvidia_gpu_exporter
  and verifies they're reachable — cluster-facing config lives in Git.
- host_vars/vars.yml + inventory.yml: finalize astro-orbiter as the
  llama.cpp/RTX 3090 host (jarvis user, ssh key), drop stale Ollama/AMD
  vars and ollama_server inventory group superseded by the ATX rebuild.

ansible/roles/llm-inference/tasks/monitoring.yml

@@ -2,16 +2,24 @@
 # ------------------------------------------------------------------------------
 # FILE: roles/llm-inference/tasks/monitoring.yml
 # DESCRIPTION: Phase 7 — Prometheus monitoring for the LLM inference stack.
-#              Deploys three metric sources on astro-orbiter:
+#              Deploys two metric-producing exporters on astro-orbiter:
 #
 #              1. node_exporter (port 9100) — system: CPU, RAM, disk, network
 #              2. nvidia_gpu_exporter (port 9835) — GPU: VRAM, temp, util, power
-#              3. vLLM built-in metrics (port 8000/metrics) — already exposed
-#                 by vLLM; just needs a scrape job (no extra process)
+#              3. llama-server built-in metrics (port 8000/metrics, enabled via
+#                 --metrics) — just needs a scrape job (no extra process)
 #
-#              Wires all three into Prometheus via additionalScrapeConfigs on
-#              the kube-prometheus-stack secret on carousel-of-progress.
-#              Deploys a Grafana dashboard ConfigMap in the monitoring namespace.
+#              GitOps note: the Prometheus scrape jobs for all three targets and
+#              the Grafana dashboard are declared in the homelab Git repo and
+#              applied by ArgoCD — NOT by this role:
+#                - cluster/applications/monitoring/values.yaml
+#                  (prometheus.prometheusSpec.additionalScrapeConfigs)
+#                - cluster/applications/monitoring/dashboards.yaml
+#                  (grafana-llm-inference-dashboard ConfigMap)
+#              This role's job is only to stand up the two exporters + verify
+#              they're reachable. Do NOT reintroduce kubectl patch/apply tasks
+#              here — cluster-facing changes go through Git commit + ArgoCD
+#              sync so state stays reproducible and self-healing.
 # ------------------------------------------------------------------------------
 
 # -----------------------------------------------------------------------
@@ -114,85 +122,10 @@
   changed_when: false
 
 # -----------------------------------------------------------------------
-# 3. Patch additionalScrapeConfigs secret on carousel-of-progress
-#    Adds three new scrape jobs: node, gpu, vllm
+# 3. Prometheus scrape configs + Grafana dashboard
+#
+#    Intentionally NOT managed here. See file header: these are declared
+#    in cluster/applications/monitoring/{values.yaml,dashboards.yaml} in
+#    the homelab Git repo and rolled out by ArgoCD sync, keeping cluster
+#    state in Git rather than mutated imperatively from the control node.
 # -----------------------------------------------------------------------
-
-- name: Read current additionalScrapeConfigs from Prometheus secret
-  ansible.builtin.command:
-    cmd: >
-      kubectl get secret monitoring-kube-prometheus-prometheus-scrape-confg
-      -n monitoring
-      -o jsonpath='{.data.additional-scrape-configs\.yaml}'
-  register: current_scrape_b64
-  changed_when: false
-  delegate_to: carousel-of-progress
-  become: false
-
-- name: Decode current scrape configs
-  ansible.builtin.set_fact:
-    current_scrape_yaml: "{{ current_scrape_b64.stdout | b64decode }}"
-
-- name: Check if astro-orbiter scrape jobs already present
-  ansible.builtin.set_fact:
-    scrape_already_patched: "{{ 'astro-orbiter' in current_scrape_yaml }}"
-
-- name: Append astro-orbiter scrape jobs to additionalScrapeConfigs
-  when: not scrape_already_patched
-  block:
-    - name: Build new scrape config with astro-orbiter jobs appended
-      ansible.builtin.set_fact:
-        new_scrape_yaml: |
-          {{ current_scrape_yaml }}
-          - job_name: node-astro-orbiter
-            scrape_interval: 30s
-            static_configs:
-            - labels:
-                hostname: astro-orbiter
-              targets:
-              - {{ hostvars['astro-orbiter']['ansible_host'] }}:9100
-          - job_name: gpu-astro-orbiter
-            scrape_interval: 15s
-            static_configs:
-            - labels:
-                hostname: astro-orbiter
-                gpu: rtx3090
-              targets:
-              - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_gpu_exporter_port }}
-          - job_name: vllm-astro-orbiter
-            scrape_interval: 15s
-            metrics_path: /metrics
-            static_configs:
-            - labels:
-                hostname: astro-orbiter
-                model: "{{ llm_hf_model }}"
-              targets:
-              - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_serve_port }}
-
-    - name: Patch Prometheus additionalScrapeConfigs secret
-      ansible.builtin.command:
-        cmd: >
-          kubectl patch secret monitoring-kube-prometheus-prometheus-scrape-confg
-          -n monitoring
-          --type=json
-          -p='[{"op":"replace","path":"/data/additional-scrape-configs.yaml","value":"{{ new_scrape_yaml | b64encode }}"}]'
-      delegate_to: carousel-of-progress
-      become: false
-
-# -----------------------------------------------------------------------
-# 4. Deploy Grafana dashboard
-# -----------------------------------------------------------------------
-
-- name: Deploy LLM inference Grafana dashboard ConfigMap
-  ansible.builtin.template:
-    src: grafana-llm-dashboard.json.j2
-    dest: /tmp/grafana-llm-dashboard-cm.yaml
-  delegate_to: carousel-of-progress
-  become: false
-
-- name: Apply Grafana dashboard ConfigMap to cluster
-  ansible.builtin.command:
-    cmd: kubectl apply -f /tmp/grafana-llm-dashboard-cm.yaml
-  delegate_to: carousel-of-progress
-  become: false
-  changed_when: true

ansible/roles/llm-inference/tasks/serve.yml

@@ -22,6 +22,17 @@
     state: present
     update_cache: false
 
+# NOTE: nvidia-driver-595-open provides the runtime driver only (nvidia-smi,
+# libcuda.so) — it does NOT ship nvcc/CUDA headers needed to build GGML_CUDA=ON.
+# Ubuntu 24.04's nvidia-cuda-toolkit (12.0.x) is sufficient to build llama.cpp
+# against; it does not need to match the 595 driver's CUDA 13.2 runtime version.
+- name: Install NVIDIA CUDA toolkit (nvcc) for building llama.cpp with CUDA support
+  ansible.builtin.apt:
+    name: nvidia-cuda-toolkit
+    state: present
+    update_cache: false
+  become: true
+
 - name: Clone llama.cpp repository
   ansible.builtin.git:
     repo: https://github.com/ggml-org/llama.cpp.git
@@ -30,6 +41,20 @@
     update: false
   become: true
 
+- name: Check for incomplete/stale llama.cpp CMake configuration
+  ansible.builtin.stat:
+    path: /opt/llama.cpp/build/Makefile
+  register: llama_cmake_generated
+
+- name: Remove stale llama.cpp build dir if CMake configure never completed
+  ansible.builtin.file:
+    path: /opt/llama.cpp/build
+    state: absent
+  become: true
+  when:
+    - not llama_cmake_generated.stat.exists
+    - not (ansible_check_mode | default(false))
+
 - name: Build llama.cpp with CUDA support
   ansible.builtin.command:
     cmd: cmake -B build -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Release
@@ -53,6 +78,11 @@
     group: "{{ llm_venv_owner }}"
     mode: "0755"
 
+- name: Check whether GGUF already exists (avoid re-downloading 16.6GB on every run)
+  ansible.builtin.stat:
+    path: "{{ llm_gguf_path }}"
+  register: llm_gguf_stat
+
 - name: Download Gemma 2 27B Q4_K_M GGUF from HuggingFace
   ansible.builtin.get_url:
     url: "https://huggingface.co/bartowski/gemma-2-27b-it-GGUF/resolve/main/gemma-2-27b-it-Q4_K_M.gguf"
@@ -63,8 +93,13 @@
     group: "{{ llm_venv_owner }}"
     mode: "0644"
     timeout: 7200
+    force: false
   become: true
   no_log: true
+  # Idempotency: skip entirely once the file exists and is reasonably sized
+  # (the finished GGUF is ~16.6GB; guard against a truncated partial download
+  # being mistaken for complete by only trusting files > 15GB).
+  when: not llm_gguf_stat.stat.exists or (llm_gguf_stat.stat.size | int) < 15000000000
 
 - name: Disable and stop vllm-serve if present
   ansible.builtin.systemd: