rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: correct metric names in llama-swap monitoring (llamacpp_* -> llamaswap_*), update alerts + dashboard + scrape config

Browse Source
This commit is contained in:
Hermes Agent service account
2026-08-18 23:18:22 -05:00
parent 7867be688a
commit 24735f7e5c
6 changed files with 81 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
ansible/roles/llm-inference-multimodel/templates/llama-swap-alerts.yml.j2
View File

@@ -33,7 +33,7 @@ spec:
# CRITICAL: GPU VRAM saturation (OOM risk)
# ====================================================================
- alert: LlamaSwapVramSaturation
expr: llamacpp_vram_used_mib > {{ llm_swapmode_vram_max_mib | int }}
expr: (llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576) > {{ llm_swapmode_vram_max_mib | int }}
for: 1m
labels:
severity: critical
@@ -56,14 +56,14 @@ spec:
# ====================================================================
# WARNING: KV-cache spill risk (context cache pressure)
# ====================================================================
- alert: LlamaSwapKvCacheSpill
expr: llamacpp_kv_cache_usage_ratio > 0.92
- alert: LlamaSwapVramPressure
expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92
for: 2m
labels:
severity: warning
component: llm-inference
annotations:
summary: "KV-cache spill risk on model {{ $labels.model }}"
summary: "GPU memory utilization high (possible VRAM pressure)"
description: |
KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
(warning threshold: 92%).
@@ -81,18 +81,18 @@ spec:
# ====================================================================
# WARNING: Throughput degradation (possible throttling)
# ====================================================================
- alert: LlamaSwapThroughputDegradation
- alert: LlamaSwapInferenceStall
expr: |
(rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40
(llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50)
for: 5m
labels:
severity: warning
component: llm-inference
annotations:
summary: "Token generation throughput low on {{ $labels.model }}"
summary: "GPU compute stall detected (memory loaded but no utilization)"
description: |
Token generation rate is {{ $value | humanize }}tokens/min on {{ $labels.model }}
(baseline threshold: ~50+ tokens/min).
The RTX 3090 has >50% memory utilization but 0% compute utilization
for more than 5 minutes. This may indicate:
This may indicate:
1. Thermal throttling (GPU temperature limiting frequency)

44
ansible/roles/llm-inference-multimodel/templates/llama-swap-grafana-dashboard.json.j2
View File

@@ -100,13 +100,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_vram_used_mib{job=\"node\"}",
"expr": "llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576",
"interval": "",
"legendFormat": "VRAM Used",
"refId": "A"
}
],
"title": "GPU VRAM Usage",
"title": "GPU VRAM Usage (MiB)",
"type": "timeseries"
},
{
@@ -166,13 +166,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_kv_cache_usage_ratio",
"expr": "llamaswap_gpu_memory_util_percent{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "KV-Cache Utilization (Gauge)",
"title": "GPU Memory Utilization %",
"type": "gauge"
},
{
@@ -183,7 +183,7 @@
"mode": "palette-classic"
},
"custom": {
"axisLabel": "ms/token",
"axisLabel": "%",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -220,7 +220,7 @@
}
]
},
"unit": "ms"
"unit": "percent"
},
"overrides": []
},
@@ -247,13 +247,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "rate(llamacpp_time_predict_ms_sum[5m]) / rate(llamacpp_time_predict_ms_count[5m])",
"expr": "llamaswap_gpu_util_percent{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "Prediction Latency by Model",
"title": "GPU Utilization %",
"type": "timeseries"
},
{
@@ -264,7 +264,7 @@
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Queue Size",
"axisLabel": "%",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -309,7 +309,7 @@
}
]
},
"unit": "short"
"unit": "percent"
},
"overrides": []
},
@@ -336,13 +336,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_queue_size",
"expr": "avg(llamaswap_cpu_util_percent{job=\"llama-swap\"})",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "Request Queue Depth",
"title": "CPU Utilization %",
"type": "timeseries"
},
{
@@ -353,7 +353,7 @@
"mode": "palette-classic"
},
"custom": {
"axisLabel": "tokens/min",
"axisLabel": "W",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -390,7 +390,7 @@
}
]
},
"unit": "short"
"unit": "watt"
},
"overrides": []
},
@@ -416,13 +416,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "rate(llamacpp_tokens_predicted_total[1m]) * 60",
"expr": "llamaswap_gpu_power_draw_watts{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }} (tokens/min)",
"refId": "A"
}
],
"title": "Token Generation Throughput",
"title": "GPU Power Draw (W)",
"type": "timeseries"
},
{
@@ -433,7 +433,7 @@
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Tokens",
"axisLabel": "load",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
@@ -494,19 +494,19 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, rate(llamacpp_time_predict_ms_bucket[5m]))",
"expr": "llamaswap_load_average{interval=\"5m\"}",
"interval": "",
"legendFormat": "p95 latency",
"refId": "A"
},
{
"expr": "histogram_quantile(0.99, rate(llamacpp_time_predict_ms_bucket[5m]))",
"expr": "llamaswap_load_average{interval=\"5m\"}",
"interval": "",
"legendFormat": "p99 latency",
"refId": "B"
}
],
"title": "Latency Percentiles (p95, p99)",
"title": "System Load Average (5m)",
"type": "timeseries"
}
],
@@ -528,7 +528,7 @@
},
"timepicker": {},
"timezone": "",
"title": "llama-swap GPU/LLM Monitoring",
"uid": "llama-swap-monitor",
"title": {{ llm_grafana_dashboard_title }},
"uid": {{ llm_grafana_dashboard_uid }},
"version": 1
}

11
ansible/roles/llm-inference-multimodel/templates/llama-swap-prometheus-scrape.yml.j2
View File

@@ -30,7 +30,10 @@
metrics_path: /metrics
# Relabeling: extract model name from metric labels for dashboard grouping
metric_relabel_configs:
- source_labels: [__name__]
regex: 'llamacpp_.*'
action: keep
# llama-swap exposes llamaswap_* metrics (GPU VRAM, utilization, power, CPU,
# network, load average). Per-model inference metrics are not available at the
# proxy level. Filter to keep only llamaswap_* metrics to reduce cardinality.
metric_relabel_configs:
- source_labels: [__name__]
regex: 'llamaswap_.*'
action: keep

25
cluster/applications/monitoring/llama-swap-alerts.yaml
View File

@@ -15,7 +15,7 @@ spec:
# CRITICAL: GPU VRAM saturation (OOM risk)
# ====================================================================
- alert: LlamaSwapVramSaturation
expr: llamacpp_vram_used_mib > 24000
expr: (llamaswap_gpu_memory_used_bytes{job="llama-swap"} / 1048576) > 24000
for: 1m
labels:
severity: critical
@@ -27,35 +27,34 @@ spec:
The system is at risk of out-of-memory (OOM) kernel-kill events.
# ====================================================================
# WARNING: KV-cache spill risk (context cache pressure)
# WARNING: GPU memory utilization (VRAM pressure proxy)
# ====================================================================
- alert: LlamaSwapKvCacheSpill
expr: llamacpp_kv_cache_usage_ratio > 0.92
- alert: LlamaSwapVramPressure
expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92
for: 2m
labels:
severity: warning
component: llm-inference
annotations:
summary: "KV-cache spill risk on model {{ $labels.model }}"
summary: "GPU memory utilization high (possible VRAM pressure)"
description: |
KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
(warning threshold: 92%).
GPU memory utilization is {{ $value | humanize }}% (warning threshold: 92%).
# ====================================================================
# WARNING: Throughput degradation (possible throttling)
# WARNING: Inference stall (GPU compute idle while VRAM loaded)
# ====================================================================
- alert: LlamaSwapThroughputDegradation
- alert: LlamaSwapInferenceStall
expr: |
(rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40
(llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50)
for: 5m
labels:
severity: warning
component: llm-inference
annotations:
summary: "Token generation throughput low on {{ $labels.model }}"
summary: "GPU compute stall detected (memory loaded but no utilization)"
description: |
Token generation rate is {{ $value | humanize }} tokens/min on {{ $labels.model }}
(baseline threshold: ~50+ tokens/min).
The RTX 3090 has >50% memory utilization but 0% compute utilization
for more than 5 minutes. This may indicate:
# ====================================================================
# WARNING: Scrape failures (monitoring health)

50
cluster/applications/monitoring/llama-swap-dashboard.yaml
View File

@@ -2,12 +2,12 @@
# ------------------------------------------------------------------------------
# FILE: cluster/applications/monitoring/llama-swap-dashboard.yaml
# DESCRIPTION: Custom Grafana dashboard for llama-swap GPU/LLM monitoring.
# Picked up automatically by the Grafana sidecar via label:
# grafana_dashboard: "1"
# Based on the Ciro Luciotta homelab LLM monitoring pattern.
# Uses llama-swap native metrics (llamaswap_* prefix).
#
# USAGE: This ConfigMap is reconciled by ArgoCD. The dashboard JSON is
# embedded inline (data key ends in .json).
# USAGE: Reconciled by ArgoCD. Picked up by Grafana sidecar via label:
# grafana_dashboard: "1"
# Reference: Ciro Luciotta homelab monitoring pattern (adapted)
# Updated: 2026-08-18 — metric names corrected for llama-swap v250
# ------------------------------------------------------------------------------
apiVersion: v1
@@ -122,13 +122,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_vram_used_mib{job=\"node\"}",
"expr": "llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576",
"interval": "",
"legendFormat": "VRAM Used",
"refId": "A"
}
],
"title": "GPU VRAM Usage",
"title": "GPU VRAM Usage (MiB)",
"type": "timeseries"
},
{
@@ -188,13 +188,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_kv_cache_usage_ratio",
"expr": "llamaswap_gpu_memory_util_percent{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "KV-Cache Utilization (Gauge)",
"title": "GPU Memory Utilization %",
"type": "gauge"
},
{
@@ -205,7 +205,7 @@ data:
"mode": "palette-classic"
},
"custom": {
"axisLabel": "ms/token",
"axisLabel": "%",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -242,7 +242,7 @@ data:
}
]
},
"unit": "ms"
"unit": "percent"
},
"overrides": []
},
@@ -269,13 +269,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "rate(llamacpp_time_predict_ms_sum[5m]) / rate(llamacpp_time_predict_ms_count[5m])",
"expr": "llamaswap_gpu_util_percent{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "Prediction Latency by Model",
"title": "GPU Utilization %",
"type": "timeseries"
},
{
@@ -286,7 +286,7 @@ data:
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Queue Size",
"axisLabel": "%",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -331,7 +331,7 @@ data:
}
]
},
"unit": "short"
"unit": "percent"
},
"overrides": []
},
@@ -358,13 +358,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_queue_size",
"expr": "avg(llamaswap_cpu_util_percent{job=\"llama-swap\"})",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "Request Queue Depth",
"title": "CPU Utilization %",
"type": "timeseries"
},
{
@@ -375,7 +375,7 @@ data:
"mode": "palette-classic"
},
"custom": {
"axisLabel": "tokens/min",
"axisLabel": "W",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -412,7 +412,7 @@ data:
}
]
},
"unit": "short"
"unit": "watt"
},
"overrides": []
},
@@ -438,13 +438,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "rate(llamacpp_tokens_predicted_total[1m]) * 60",
"expr": "llamaswap_gpu_power_draw_watts{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }} (tokens/min)",
"refId": "A"
}
],
"title": "Token Generation Throughput",
"title": "GPU Power Draw (W)",
"type": "timeseries"
},
{
@@ -455,7 +455,7 @@ data:
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Tokens",
"axisLabel": "load",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
@@ -516,19 +516,19 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, rate(llamacpp_time_predict_ms_bucket[5m]))",
"expr": "llamaswap_load_average{interval=\"5m\"}",
"interval": "",
"legendFormat": "p95 latency",
"refId": "A"
},
{
"expr": "histogram_quantile(0.99, rate(llamacpp_time_predict_ms_bucket[5m]))",
"expr": "llamaswap_load_average{interval=\"5m\"}",
"interval": "",
"legendFormat": "p99 latency",
"refId": "B"
}
],
"title": "Latency Percentiles (p95, p99)",
"title": "System Load Average (5m)",
"type": "timeseries"
}
],

7
cluster/applications/monitoring/values.yaml
View File

@@ -299,7 +299,10 @@ prometheus:
# llama-swap (production, since 2026-08-18)
# Replaces the per-model /metrics?model=<id> jobs above (all targeting now-deprecated :8002).
# llama-swap natively exposes /metrics on its own endpoint with model-labeled metrics.
# llama-swap exposes system-level metrics (llamaswap_*) — VRAM, GPU util, power, CPU, network.
# Per-model inference metrics (tokens/sec, latency, KV-cache) are NOT exposed at the proxy level;
# they remain on the individual llama-server child instances, scraped via node_exporter textfile
# collector for VRAM, and via the GPU exporter (:9835) for GPU-level telemetry.
- job_name: llama-swap
scrape_interval: 30s
scrape_timeout: 10s
@@ -314,7 +317,7 @@ prometheus:
honor_labels: true
metric_relabel_configs:
- source_labels: [__name__]
regex: 'llamacpp_.*'
regex: 'llamaswap_.*'
action: keep
# ─── Grafana ──────────────────────────────────────────────────────────────────