monitoring: llama-swap GPU/LLM stack (v250) — PrometheusRule, Grafana dashboard, scrape config, VRAM exporter

cluster/applications/monitoring/llama-swap-alerts.yaml

@@ -0,0 +1,73 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: llama-swap-alerts
+  namespace: monitoring
+  labels:
+    prometheus: kube-prometheus
+    app.kubernetes.io/part-of: monitoring
+spec:
+  groups:
+    - name: llama-swap.rules
+      interval: 30s
+      rules:
+        # ====================================================================
+        # CRITICAL: GPU VRAM saturation (OOM risk)
+        # ====================================================================
+        - alert: LlamaSwapVramSaturation
+          expr: llamacpp_vram_used_mib > 24000
+          for: 1m
+          labels:
+            severity: critical
+            component: llm-inference
+          annotations:
+            summary: "GPU VRAM saturation on {{ $labels.instance }}"
+            description: |
+              GPU VRAM usage is {{ $value | humanize }}MiB (critical threshold: 24000MiB).
+              The system is at risk of out-of-memory (OOM) kernel-kill events.
+
+        # ====================================================================
+        # WARNING: KV-cache spill risk (context cache pressure)
+        # ====================================================================
+        - alert: LlamaSwapKvCacheSpill
+          expr: llamacpp_kv_cache_usage_ratio > 0.92
+          for: 2m
+          labels:
+            severity: warning
+            component: llm-inference
+          annotations:
+            summary: "KV-cache spill risk on model {{ $labels.model }}"
+            description: |
+              KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
+              (warning threshold: 92%).
+
+        # ====================================================================
+        # WARNING: Throughput degradation (possible throttling)
+        # ====================================================================
+        - alert: LlamaSwapThroughputDegradation
+          expr: |
+            (rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40
+          for: 5m
+          labels:
+            severity: warning
+            component: llm-inference
+          annotations:
+            summary: "Token generation throughput low on {{ $labels.model }}"
+            description: |
+              Token generation rate is {{ $value | humanize }} tokens/min on {{ $labels.model }}
+              (baseline threshold: ~50+ tokens/min).
+
+        # ====================================================================
+        # WARNING: Scrape failures (monitoring health)
+        # ====================================================================
+        - alert: LlamaSwapScrapeFailed
+          expr: up{job="llama-swap"} == 0
+          for: 2m
+          labels:
+            severity: warning
+            component: monitoring
+          annotations:
+            summary: "llama-swap Prometheus scrape failed"
+            description: |
+              Prometheus cannot scrape llama-swap's /metrics endpoint.
+              Check: systemctl status llama-swap, curl http://{{ $labels.instance }}/metrics