diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-swap-alerts.yml.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-swap-alerts.yml.j2 index e714af4..e5baf1a 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-swap-alerts.yml.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-swap-alerts.yml.j2 @@ -33,7 +33,7 @@ spec: # CRITICAL: GPU VRAM saturation (OOM risk) # ==================================================================== - alert: LlamaSwapVramSaturation - expr: llamacpp_vram_used_mib > {{ llm_swapmode_vram_max_mib | int }} + expr: (llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576) > {{ llm_swapmode_vram_max_mib | int }} for: 1m labels: severity: critical @@ -56,14 +56,14 @@ spec: # ==================================================================== # WARNING: KV-cache spill risk (context cache pressure) # ==================================================================== - - alert: LlamaSwapKvCacheSpill - expr: llamacpp_kv_cache_usage_ratio > 0.92 + - alert: LlamaSwapVramPressure + expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92 for: 2m labels: severity: warning component: llm-inference annotations: - summary: "KV-cache spill risk on model {{ $labels.model }}" + summary: "GPU memory utilization high (possible VRAM pressure)" description: | KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }} (warning threshold: 92%). @@ -81,18 +81,18 @@ spec: # ==================================================================== # WARNING: Throughput degradation (possible throttling) # ==================================================================== - - alert: LlamaSwapThroughputDegradation + - alert: LlamaSwapInferenceStall expr: | - (rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40 + (llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50) for: 5m labels: severity: warning component: llm-inference annotations: - summary: "Token generation throughput low on {{ $labels.model }}" + summary: "GPU compute stall detected (memory loaded but no utilization)" description: | - Token generation rate is {{ $value | humanize }}tokens/min on {{ $labels.model }} - (baseline threshold: ~50+ tokens/min). + The RTX 3090 has >50% memory utilization but 0% compute utilization + for more than 5 minutes. This may indicate: This may indicate: 1. Thermal throttling (GPU temperature limiting frequency) diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-swap-grafana-dashboard.json.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-swap-grafana-dashboard.json.j2 index 9cb829e..f1894c1 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-swap-grafana-dashboard.json.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-swap-grafana-dashboard.json.j2 @@ -100,13 +100,13 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "llamacpp_vram_used_mib{job=\"node\"}", + "expr": "llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576", "interval": "", "legendFormat": "VRAM Used", "refId": "A" } ], - "title": "GPU VRAM Usage", + "title": "GPU VRAM Usage (MiB)", "type": "timeseries" }, { @@ -166,13 +166,13 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "llamacpp_kv_cache_usage_ratio", + "expr": "llamaswap_gpu_memory_util_percent{job=\"llama-swap\"}", "interval": "", "legendFormat": "{{ model }}", "refId": "A" } ], - "title": "KV-Cache Utilization (Gauge)", + "title": "GPU Memory Utilization %", "type": "gauge" }, { @@ -183,7 +183,7 @@ "mode": "palette-classic" }, "custom": { - "axisLabel": "ms/token", + "axisLabel": "%", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -220,7 +220,7 @@ } ] }, - "unit": "ms" + "unit": "percent" }, "overrides": [] }, @@ -247,13 +247,13 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "rate(llamacpp_time_predict_ms_sum[5m]) / rate(llamacpp_time_predict_ms_count[5m])", + "expr": "llamaswap_gpu_util_percent{job=\"llama-swap\"}", "interval": "", "legendFormat": "{{ model }}", "refId": "A" } ], - "title": "Prediction Latency by Model", + "title": "GPU Utilization %", "type": "timeseries" }, { @@ -264,7 +264,7 @@ "mode": "palette-classic" }, "custom": { - "axisLabel": "Queue Size", + "axisLabel": "%", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -309,7 +309,7 @@ } ] }, - "unit": "short" + "unit": "percent" }, "overrides": [] }, @@ -336,13 +336,13 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "llamacpp_queue_size", + "expr": "avg(llamaswap_cpu_util_percent{job=\"llama-swap\"})", "interval": "", "legendFormat": "{{ model }}", "refId": "A" } ], - "title": "Request Queue Depth", + "title": "CPU Utilization %", "type": "timeseries" }, { @@ -353,7 +353,7 @@ "mode": "palette-classic" }, "custom": { - "axisLabel": "tokens/min", + "axisLabel": "W", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -390,7 +390,7 @@ } ] }, - "unit": "short" + "unit": "watt" }, "overrides": [] }, @@ -416,13 +416,13 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "rate(llamacpp_tokens_predicted_total[1m]) * 60", + "expr": "llamaswap_gpu_power_draw_watts{job=\"llama-swap\"}", "interval": "", "legendFormat": "{{ model }} (tokens/min)", "refId": "A" } ], - "title": "Token Generation Throughput", + "title": "GPU Power Draw (W)", "type": "timeseries" }, { @@ -433,7 +433,7 @@ "mode": "palette-classic" }, "custom": { - "axisLabel": "Tokens", + "axisLabel": "load", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", @@ -494,19 +494,19 @@ "pluginVersion": "8.0.0", "targets": [ { - "expr": "histogram_quantile(0.95, rate(llamacpp_time_predict_ms_bucket[5m]))", + "expr": "llamaswap_load_average{interval=\"5m\"}", "interval": "", "legendFormat": "p95 latency", "refId": "A" }, { - "expr": "histogram_quantile(0.99, rate(llamacpp_time_predict_ms_bucket[5m]))", + "expr": "llamaswap_load_average{interval=\"5m\"}", "interval": "", "legendFormat": "p99 latency", "refId": "B" } ], - "title": "Latency Percentiles (p95, p99)", + "title": "System Load Average (5m)", "type": "timeseries" } ], @@ -528,7 +528,7 @@ }, "timepicker": {}, "timezone": "", - "title": "llama-swap GPU/LLM Monitoring", - "uid": "llama-swap-monitor", + "title": {{ llm_grafana_dashboard_title }}, + "uid": {{ llm_grafana_dashboard_uid }}, "version": 1 -} +} \ No newline at end of file diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-swap-prometheus-scrape.yml.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-swap-prometheus-scrape.yml.j2 index 081c085..be5bf5e 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-swap-prometheus-scrape.yml.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-swap-prometheus-scrape.yml.j2 @@ -30,7 +30,10 @@ metrics_path: /metrics # Relabeling: extract model name from metric labels for dashboard grouping - metric_relabel_configs: - - source_labels: [__name__] - regex: 'llamacpp_.*' - action: keep +# llama-swap exposes llamaswap_* metrics (GPU VRAM, utilization, power, CPU, +# network, load average). Per-model inference metrics are not available at the +# proxy level. Filter to keep only llamaswap_* metrics to reduce cardinality. + metric_relabel_configs: + - source_labels: [__name__] + regex: 'llamaswap_.*' + action: keep diff --git a/cluster/applications/monitoring/llama-swap-alerts.yaml b/cluster/applications/monitoring/llama-swap-alerts.yaml index bb1107d..df328fe 100644 --- a/cluster/applications/monitoring/llama-swap-alerts.yaml +++ b/cluster/applications/monitoring/llama-swap-alerts.yaml @@ -15,7 +15,7 @@ spec: # CRITICAL: GPU VRAM saturation (OOM risk) # ==================================================================== - alert: LlamaSwapVramSaturation - expr: llamacpp_vram_used_mib > 24000 + expr: (llamaswap_gpu_memory_used_bytes{job="llama-swap"} / 1048576) > 24000 for: 1m labels: severity: critical @@ -27,35 +27,34 @@ spec: The system is at risk of out-of-memory (OOM) kernel-kill events. # ==================================================================== - # WARNING: KV-cache spill risk (context cache pressure) + # WARNING: GPU memory utilization (VRAM pressure proxy) # ==================================================================== - - alert: LlamaSwapKvCacheSpill - expr: llamacpp_kv_cache_usage_ratio > 0.92 + - alert: LlamaSwapVramPressure + expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92 for: 2m labels: severity: warning component: llm-inference annotations: - summary: "KV-cache spill risk on model {{ $labels.model }}" + summary: "GPU memory utilization high (possible VRAM pressure)" description: | - KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }} - (warning threshold: 92%). + GPU memory utilization is {{ $value | humanize }}% (warning threshold: 92%). # ==================================================================== - # WARNING: Throughput degradation (possible throttling) + # WARNING: Inference stall (GPU compute idle while VRAM loaded) # ==================================================================== - - alert: LlamaSwapThroughputDegradation + - alert: LlamaSwapInferenceStall expr: | - (rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40 + (llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50) for: 5m labels: severity: warning component: llm-inference annotations: - summary: "Token generation throughput low on {{ $labels.model }}" + summary: "GPU compute stall detected (memory loaded but no utilization)" description: | - Token generation rate is {{ $value | humanize }} tokens/min on {{ $labels.model }} - (baseline threshold: ~50+ tokens/min). + The RTX 3090 has >50% memory utilization but 0% compute utilization + for more than 5 minutes. This may indicate: # ==================================================================== # WARNING: Scrape failures (monitoring health) diff --git a/cluster/applications/monitoring/llama-swap-dashboard.yaml b/cluster/applications/monitoring/llama-swap-dashboard.yaml index 03285d2..941cde6 100644 --- a/cluster/applications/monitoring/llama-swap-dashboard.yaml +++ b/cluster/applications/monitoring/llama-swap-dashboard.yaml @@ -2,12 +2,12 @@ # ------------------------------------------------------------------------------ # FILE: cluster/applications/monitoring/llama-swap-dashboard.yaml # DESCRIPTION: Custom Grafana dashboard for llama-swap GPU/LLM monitoring. -# Picked up automatically by the Grafana sidecar via label: -# grafana_dashboard: "1" -# Based on the Ciro Luciotta homelab LLM monitoring pattern. +# Uses llama-swap native metrics (llamaswap_* prefix). # -# USAGE: This ConfigMap is reconciled by ArgoCD. The dashboard JSON is -# embedded inline (data key ends in .json). +# USAGE: Reconciled by ArgoCD. Picked up by Grafana sidecar via label: +# grafana_dashboard: "1" +# Reference: Ciro Luciotta homelab monitoring pattern (adapted) +# Updated: 2026-08-18 — metric names corrected for llama-swap v250 # ------------------------------------------------------------------------------ apiVersion: v1 @@ -122,13 +122,13 @@ data: "pluginVersion": "8.0.0", "targets": [ { - "expr": "llamacpp_vram_used_mib{job=\"node\"}", + "expr": "llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576", "interval": "", "legendFormat": "VRAM Used", "refId": "A" } ], - "title": "GPU VRAM Usage", + "title": "GPU VRAM Usage (MiB)", "type": "timeseries" }, { @@ -188,13 +188,13 @@ data: "pluginVersion": "8.0.0", "targets": [ { - "expr": "llamacpp_kv_cache_usage_ratio", + "expr": "llamaswap_gpu_memory_util_percent{job=\"llama-swap\"}", "interval": "", "legendFormat": "{{ model }}", "refId": "A" } ], - "title": "KV-Cache Utilization (Gauge)", + "title": "GPU Memory Utilization %", "type": "gauge" }, { @@ -205,7 +205,7 @@ data: "mode": "palette-classic" }, "custom": { - "axisLabel": "ms/token", + "axisLabel": "%", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -242,7 +242,7 @@ data: } ] }, - "unit": "ms" + "unit": "percent" }, "overrides": [] }, @@ -269,13 +269,13 @@ data: "pluginVersion": "8.0.0", "targets": [ { - "expr": "rate(llamacpp_time_predict_ms_sum[5m]) / rate(llamacpp_time_predict_ms_count[5m])", + "expr": "llamaswap_gpu_util_percent{job=\"llama-swap\"}", "interval": "", "legendFormat": "{{ model }}", "refId": "A" } ], - "title": "Prediction Latency by Model", + "title": "GPU Utilization %", "type": "timeseries" }, { @@ -286,7 +286,7 @@ data: "mode": "palette-classic" }, "custom": { - "axisLabel": "Queue Size", + "axisLabel": "%", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -331,7 +331,7 @@ data: } ] }, - "unit": "short" + "unit": "percent" }, "overrides": [] }, @@ -358,13 +358,13 @@ data: "pluginVersion": "8.0.0", "targets": [ { - "expr": "llamacpp_queue_size", + "expr": "avg(llamaswap_cpu_util_percent{job=\"llama-swap\"})", "interval": "", "legendFormat": "{{ model }}", "refId": "A" } ], - "title": "Request Queue Depth", + "title": "CPU Utilization %", "type": "timeseries" }, { @@ -375,7 +375,7 @@ data: "mode": "palette-classic" }, "custom": { - "axisLabel": "tokens/min", + "axisLabel": "W", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -412,7 +412,7 @@ data: } ] }, - "unit": "short" + "unit": "watt" }, "overrides": [] }, @@ -438,13 +438,13 @@ data: "pluginVersion": "8.0.0", "targets": [ { - "expr": "rate(llamacpp_tokens_predicted_total[1m]) * 60", + "expr": "llamaswap_gpu_power_draw_watts{job=\"llama-swap\"}", "interval": "", "legendFormat": "{{ model }} (tokens/min)", "refId": "A" } ], - "title": "Token Generation Throughput", + "title": "GPU Power Draw (W)", "type": "timeseries" }, { @@ -455,7 +455,7 @@ data: "mode": "palette-classic" }, "custom": { - "axisLabel": "Tokens", + "axisLabel": "load", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", @@ -516,19 +516,19 @@ data: "pluginVersion": "8.0.0", "targets": [ { - "expr": "histogram_quantile(0.95, rate(llamacpp_time_predict_ms_bucket[5m]))", + "expr": "llamaswap_load_average{interval=\"5m\"}", "interval": "", "legendFormat": "p95 latency", "refId": "A" }, { - "expr": "histogram_quantile(0.99, rate(llamacpp_time_predict_ms_bucket[5m]))", + "expr": "llamaswap_load_average{interval=\"5m\"}", "interval": "", "legendFormat": "p99 latency", "refId": "B" } ], - "title": "Latency Percentiles (p95, p99)", + "title": "System Load Average (5m)", "type": "timeseries" } ], diff --git a/cluster/applications/monitoring/values.yaml b/cluster/applications/monitoring/values.yaml index 96dd4db..1da5a50 100644 --- a/cluster/applications/monitoring/values.yaml +++ b/cluster/applications/monitoring/values.yaml @@ -299,7 +299,10 @@ prometheus: # llama-swap (production, since 2026-08-18) # Replaces the per-model /metrics?model= jobs above (all targeting now-deprecated :8002). - # llama-swap natively exposes /metrics on its own endpoint with model-labeled metrics. + # llama-swap exposes system-level metrics (llamaswap_*) — VRAM, GPU util, power, CPU, network. + # Per-model inference metrics (tokens/sec, latency, KV-cache) are NOT exposed at the proxy level; + # they remain on the individual llama-server child instances, scraped via node_exporter textfile + # collector for VRAM, and via the GPU exporter (:9835) for GPU-level telemetry. - job_name: llama-swap scrape_interval: 30s scrape_timeout: 10s @@ -314,7 +317,7 @@ prometheus: honor_labels: true metric_relabel_configs: - source_labels: [__name__] - regex: 'llamacpp_.*' + regex: 'llamaswap_.*' action: keep # ─── Grafana ──────────────────────────────────────────────────────────────────