monitoring: llama-swap GPU/LLM stack (v250) — PrometheusRule, Grafana dashboard, scrape config, VRAM exporter

cluster/applications/monitoring/values.yaml

@@ -125,7 +125,7 @@ prometheus:
           - target_label: __address__
             replacement: snmp-exporter.monitoring.svc.cluster.local:9116
 
-      # usw-pro-aggregation
+      # SNMP – usw-pro-aggregation
       - job_name: snmp-usw-pro-aggregation
         scrape_interval: 60s
         scrape_timeout: 55s
@@ -266,31 +266,56 @@ prometheus:
       #         endpoint: astro-orbiter-router
       #         model: Qwen3.6-35B-A3B-UD-Q4_K_S
 
-      - job_name: llama-server-astro-orbiter-llama3
-        scrape_interval: 90s
-        metrics_path: /metrics
-        params:
-          model: ["Meta-Llama-3.1-8B-Instruct-Q4_K_M"]
-        static_configs:
-          - targets:
-              - 10.1.71.130:8002
-            labels:
-              hostname: astro-orbiter
-              endpoint: astro-orbiter-router
-              model: Meta-Llama-3.1-8B-Instruct-Q4_K_M
+      # llama-server-astro-orbiter-llama3 — DEPRECATED (2026-08-18):
+      # Router mode on :8002 replaced by llama-swap on :8001. llama-swap exposes
+      # single /metrics endpoint (not per-model). See llama-swap job below.
+      # - job_name: llama-server-astro-orbiter-llama3
+      #   scrape_interval: 90s
+      #   metrics_path: /metrics
+      #   params:
+      #     model: ["Meta-Llama-3.1-8B-Instruct-Q4_K_M"]
+      #   static_configs:
+      #     - targets:
+      #         - 10.1.71.130:8002
+      #       labels:
+      #         hostname: astro-orbiter
+      #         endpoint: astro-orbiter-router
+      #         model: Meta-Llama-3.1-8B-Instruct-Q4_K_M
 
-      - job_name: llama-server-astro-orbiter-phi35
-        scrape_interval: 90s
-        metrics_path: /metrics
-        params:
-          model: ["Phi-3.5-mini-instruct-Q8_0"]
+      # llama-server-astro-orbiter-phi35 — DEPRECATED (2026-08-18):
+      # Same as above — router replaced by llama-swap. Use llama-swap /metrics.
+      # - job_name: llama-server-astro-orbiter-phi35
+      #   scrape_interval: 90s
+      #   metrics_path: /metrics
+      #   params:
+      #     model: ["Phi-3.5-mini-instruct-Q8_0"]
+      #   static_configs:
+      #     - targets:
+      #         - 10.1.71.130:8002
+      #       labels:
+      #         hostname: astro-orbiter
+      #         endpoint: astro-orbiter-router
+      #         model: Phi-3.5-mini-instruct-Q8_0
+
+      # llama-swap (production, since 2026-08-18)
+      # Replaces the per-model /metrics?model=<id> jobs above (all targeting now-deprecated :8002).
+      # llama-swap natively exposes /metrics on its own endpoint with model-labeled metrics.
+      - job_name: llama-swap
+        scrape_interval: 30s
+        scrape_timeout: 10s
         static_configs:
           - targets:
-              - 10.1.71.130:8002
+              - 10.1.71.130:8001
             labels:
               hostname: astro-orbiter
-              endpoint: astro-orbiter-router
-              model: Phi-3.5-mini-instruct-Q8_0
+              service: llama-swap
+              environment: homelab
+        metrics_path: /metrics
+        honor_labels: true
+        metric_relabel_configs:
+          - source_labels: [__name__]
+            regex: 'llamacpp_.*'
+            action: keep
 
 # ─── Grafana ──────────────────────────────────────────────────────────────────
 grafana: