rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: correct metric names in llama-swap monitoring (llamacpp_* -> llamaswap_*), update alerts + dashboard + scrape config

Browse Source
This commit is contained in:
Hermes Agent service account
2026-08-18 23:18:22 -05:00
parent 7867be688a
commit 24735f7e5c
6 changed files with 81 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
cluster/applications/monitoring/llama-swap-dashboard.yaml
View File

@@ -2,12 +2,12 @@
# ------------------------------------------------------------------------------
# FILE: cluster/applications/monitoring/llama-swap-dashboard.yaml
# DESCRIPTION: Custom Grafana dashboard for llama-swap GPU/LLM monitoring.
# Picked up automatically by the Grafana sidecar via label:
# grafana_dashboard: "1"
# Based on the Ciro Luciotta homelab LLM monitoring pattern.
# Uses llama-swap native metrics (llamaswap_* prefix).
#
# USAGE: This ConfigMap is reconciled by ArgoCD. The dashboard JSON is
# embedded inline (data key ends in .json).
# USAGE: Reconciled by ArgoCD. Picked up by Grafana sidecar via label:
# grafana_dashboard: "1"
# Reference: Ciro Luciotta homelab monitoring pattern (adapted)
# Updated: 2026-08-18 — metric names corrected for llama-swap v250
# ------------------------------------------------------------------------------
apiVersion: v1
@@ -122,13 +122,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_vram_used_mib{job=\"node\"}",
"expr": "llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576",
"interval": "",
"legendFormat": "VRAM Used",
"refId": "A"
}
],
"title": "GPU VRAM Usage",
"title": "GPU VRAM Usage (MiB)",
"type": "timeseries"
},
{
@@ -188,13 +188,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_kv_cache_usage_ratio",
"expr": "llamaswap_gpu_memory_util_percent{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "KV-Cache Utilization (Gauge)",
"title": "GPU Memory Utilization %",
"type": "gauge"
},
{
@@ -205,7 +205,7 @@ data:
"mode": "palette-classic"
},
"custom": {
"axisLabel": "ms/token",
"axisLabel": "%",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -242,7 +242,7 @@ data:
}
]
},
"unit": "ms"
"unit": "percent"
},
"overrides": []
},
@@ -269,13 +269,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "rate(llamacpp_time_predict_ms_sum[5m]) / rate(llamacpp_time_predict_ms_count[5m])",
"expr": "llamaswap_gpu_util_percent{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "Prediction Latency by Model",
"title": "GPU Utilization %",
"type": "timeseries"
},
{
@@ -286,7 +286,7 @@ data:
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Queue Size",
"axisLabel": "%",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -331,7 +331,7 @@ data:
}
]
},
"unit": "short"
"unit": "percent"
},
"overrides": []
},
@@ -358,13 +358,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_queue_size",
"expr": "avg(llamaswap_cpu_util_percent{job=\"llama-swap\"})",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "Request Queue Depth",
"title": "CPU Utilization %",
"type": "timeseries"
},
{
@@ -375,7 +375,7 @@ data:
"mode": "palette-classic"
},
"custom": {
"axisLabel": "tokens/min",
"axisLabel": "W",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -412,7 +412,7 @@ data:
}
]
},
"unit": "short"
"unit": "watt"
},
"overrides": []
},
@@ -438,13 +438,13 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "rate(llamacpp_tokens_predicted_total[1m]) * 60",
"expr": "llamaswap_gpu_power_draw_watts{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }} (tokens/min)",
"refId": "A"
}
],
"title": "Token Generation Throughput",
"title": "GPU Power Draw (W)",
"type": "timeseries"
},
{
@@ -455,7 +455,7 @@ data:
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Tokens",
"axisLabel": "load",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
@@ -516,19 +516,19 @@ data:
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, rate(llamacpp_time_predict_ms_bucket[5m]))",
"expr": "llamaswap_load_average{interval=\"5m\"}",
"interval": "",
"legendFormat": "p95 latency",
"refId": "A"
},
{
"expr": "histogram_quantile(0.99, rate(llamacpp_time_predict_ms_bucket[5m]))",
"expr": "llamaswap_load_average{interval=\"5m\"}",
"interval": "",
"legendFormat": "p99 latency",
"refId": "B"
}
],
"title": "Latency Percentiles (p95, p99)",
"title": "System Load Average (5m)",
"type": "timeseries"
}
],