fix: correct metric names in llama-swap monitoring (llamacpp_* -> llamaswap_*), update alerts + dashboard + scrape config
This commit is contained in:
Hermes Agent service account
@@ -33,7 +33,7 @@ spec:
|
||||
# CRITICAL: GPU VRAM saturation (OOM risk)
|
||||
# ====================================================================
|
||||
- alert: LlamaSwapVramSaturation
|
||||
expr: llamacpp_vram_used_mib > {{ llm_swapmode_vram_max_mib | int }}
|
||||
expr: (llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576) > {{ llm_swapmode_vram_max_mib | int }}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
@@ -56,14 +56,14 @@ spec:
|
||||
# ====================================================================
|
||||
# WARNING: KV-cache spill risk (context cache pressure)
|
||||
# ====================================================================
|
||||
- alert: LlamaSwapKvCacheSpill
|
||||
expr: llamacpp_kv_cache_usage_ratio > 0.92
|
||||
- alert: LlamaSwapVramPressure
|
||||
expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
component: llm-inference
|
||||
annotations:
|
||||
summary: "KV-cache spill risk on model {{ $labels.model }}"
|
||||
summary: "GPU memory utilization high (possible VRAM pressure)"
|
||||
description: |
|
||||
KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
|
||||
(warning threshold: 92%).
|
||||
@@ -81,18 +81,18 @@ spec:
|
||||
# ====================================================================
|
||||
# WARNING: Throughput degradation (possible throttling)
|
||||
# ====================================================================
|
||||
- alert: LlamaSwapThroughputDegradation
|
||||
- alert: LlamaSwapInferenceStall
|
||||
expr: |
|
||||
(rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40
|
||||
(llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
component: llm-inference
|
||||
annotations:
|
||||
summary: "Token generation throughput low on {{ $labels.model }}"
|
||||
summary: "GPU compute stall detected (memory loaded but no utilization)"
|
||||
description: |
|
||||
Token generation rate is {{ $value | humanize }}tokens/min on {{ $labels.model }}
|
||||
(baseline threshold: ~50+ tokens/min).
|
||||
The RTX 3090 has >50% memory utilization but 0% compute utilization
|
||||
for more than 5 minutes. This may indicate:
|
||||
|
||||
This may indicate:
|
||||
1. Thermal throttling (GPU temperature limiting frequency)
|
||||
|
||||
@@ -100,13 +100,13 @@
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "llamacpp_vram_used_mib{job=\"node\"}",
|
||||
"expr": "llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576",
|
||||
"interval": "",
|
||||
"legendFormat": "VRAM Used",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "GPU VRAM Usage",
|
||||
"title": "GPU VRAM Usage (MiB)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
@@ -166,13 +166,13 @@
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "llamacpp_kv_cache_usage_ratio",
|
||||
"expr": "llamaswap_gpu_memory_util_percent{job=\"llama-swap\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ model }}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "KV-Cache Utilization (Gauge)",
|
||||
"title": "GPU Memory Utilization %",
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
@@ -183,7 +183,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "ms/token",
|
||||
"axisLabel": "%",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
@@ -220,7 +220,7 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ms"
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -247,13 +247,13 @@
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(llamacpp_time_predict_ms_sum[5m]) / rate(llamacpp_time_predict_ms_count[5m])",
|
||||
"expr": "llamaswap_gpu_util_percent{job=\"llama-swap\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ model }}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Prediction Latency by Model",
|
||||
"title": "GPU Utilization %",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
@@ -264,7 +264,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "Queue Size",
|
||||
"axisLabel": "%",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
@@ -309,7 +309,7 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -336,13 +336,13 @@
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "llamacpp_queue_size",
|
||||
"expr": "avg(llamaswap_cpu_util_percent{job=\"llama-swap\"})",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ model }}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Request Queue Depth",
|
||||
"title": "CPU Utilization %",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
@@ -353,7 +353,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "tokens/min",
|
||||
"axisLabel": "W",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
@@ -390,7 +390,7 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
"unit": "watt"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
@@ -416,13 +416,13 @@
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(llamacpp_tokens_predicted_total[1m]) * 60",
|
||||
"expr": "llamaswap_gpu_power_draw_watts{job=\"llama-swap\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ model }} (tokens/min)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Token Generation Throughput",
|
||||
"title": "GPU Power Draw (W)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
@@ -433,7 +433,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "Tokens",
|
||||
"axisLabel": "load",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "bars",
|
||||
@@ -494,19 +494,19 @@
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, rate(llamacpp_time_predict_ms_bucket[5m]))",
|
||||
"expr": "llamaswap_load_average{interval=\"5m\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "p95 latency",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.99, rate(llamacpp_time_predict_ms_bucket[5m]))",
|
||||
"expr": "llamaswap_load_average{interval=\"5m\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "p99 latency",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Latency Percentiles (p95, p99)",
|
||||
"title": "System Load Average (5m)",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
@@ -528,7 +528,7 @@
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": "llama-swap GPU/LLM Monitoring",
|
||||
"uid": "llama-swap-monitor",
|
||||
"title": {{ llm_grafana_dashboard_title }},
|
||||
"uid": {{ llm_grafana_dashboard_uid }},
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
@@ -30,7 +30,10 @@
|
||||
metrics_path: /metrics
|
||||
|
||||
# Relabeling: extract model name from metric labels for dashboard grouping
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: 'llamacpp_.*'
|
||||
action: keep
|
||||
# llama-swap exposes llamaswap_* metrics (GPU VRAM, utilization, power, CPU,
|
||||
# network, load average). Per-model inference metrics are not available at the
|
||||
# proxy level. Filter to keep only llamaswap_* metrics to reduce cardinality.
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: 'llamaswap_.*'
|
||||
action: keep
|
||||
|
||||
Reference in New Issue
Block a user