rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: correct metric names in llama-swap monitoring (llamacpp_* -> llamaswap_*), update alerts + dashboard + scrape config

Browse Source
This commit is contained in:
Hermes Agent service account
2026-08-18 23:18:22 -05:00
parent 7867be688a
commit 24735f7e5c
6 changed files with 81 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
ansible/roles/llm-inference-multimodel/templates/llama-swap-grafana-dashboard.json.j2
View File

@@ -100,13 +100,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_vram_used_mib{job=\"node\"}",
"expr": "llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576",
"interval": "",
"legendFormat": "VRAM Used",
"refId": "A"
}
],
"title": "GPU VRAM Usage",
"title": "GPU VRAM Usage (MiB)",
"type": "timeseries"
},
{
@@ -166,13 +166,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_kv_cache_usage_ratio",
"expr": "llamaswap_gpu_memory_util_percent{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "KV-Cache Utilization (Gauge)",
"title": "GPU Memory Utilization %",
"type": "gauge"
},
{
@@ -183,7 +183,7 @@
"mode": "palette-classic"
},
"custom": {
"axisLabel": "ms/token",
"axisLabel": "%",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -220,7 +220,7 @@
}
]
},
"unit": "ms"
"unit": "percent"
},
"overrides": []
},
@@ -247,13 +247,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "rate(llamacpp_time_predict_ms_sum[5m]) / rate(llamacpp_time_predict_ms_count[5m])",
"expr": "llamaswap_gpu_util_percent{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "Prediction Latency by Model",
"title": "GPU Utilization %",
"type": "timeseries"
},
{
@@ -264,7 +264,7 @@
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Queue Size",
"axisLabel": "%",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -309,7 +309,7 @@
}
]
},
"unit": "short"
"unit": "percent"
},
"overrides": []
},
@@ -336,13 +336,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "llamacpp_queue_size",
"expr": "avg(llamaswap_cpu_util_percent{job=\"llama-swap\"})",
"interval": "",
"legendFormat": "{{ model }}",
"refId": "A"
}
],
"title": "Request Queue Depth",
"title": "CPU Utilization %",
"type": "timeseries"
},
{
@@ -353,7 +353,7 @@
"mode": "palette-classic"
},
"custom": {
"axisLabel": "tokens/min",
"axisLabel": "W",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
@@ -390,7 +390,7 @@
}
]
},
"unit": "short"
"unit": "watt"
},
"overrides": []
},
@@ -416,13 +416,13 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "rate(llamacpp_tokens_predicted_total[1m]) * 60",
"expr": "llamaswap_gpu_power_draw_watts{job=\"llama-swap\"}",
"interval": "",
"legendFormat": "{{ model }} (tokens/min)",
"refId": "A"
}
],
"title": "Token Generation Throughput",
"title": "GPU Power Draw (W)",
"type": "timeseries"
},
{
@@ -433,7 +433,7 @@
"mode": "palette-classic"
},
"custom": {
"axisLabel": "Tokens",
"axisLabel": "load",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "bars",
@@ -494,19 +494,19 @@
"pluginVersion": "8.0.0",
"targets": [
{
"expr": "histogram_quantile(0.95, rate(llamacpp_time_predict_ms_bucket[5m]))",
"expr": "llamaswap_load_average{interval=\"5m\"}",
"interval": "",
"legendFormat": "p95 latency",
"refId": "A"
},
{
"expr": "histogram_quantile(0.99, rate(llamacpp_time_predict_ms_bucket[5m]))",
"expr": "llamaswap_load_average{interval=\"5m\"}",
"interval": "",
"legendFormat": "p99 latency",
"refId": "B"
}
],
"title": "Latency Percentiles (p95, p99)",
"title": "System Load Average (5m)",
"type": "timeseries"
}
],
@@ -528,7 +528,7 @@
},
"timepicker": {},
"timezone": "",
"title": "llama-swap GPU/LLM Monitoring",
"uid": "llama-swap-monitor",
"title": {{ llm_grafana_dashboard_title }},
"uid": {{ llm_grafana_dashboard_uid }},
"version": 1
}
}