rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: correct metric names in llama-swap monitoring (llamacpp_* -> llamaswap_*), update alerts + dashboard + scrape config

Browse Source
This commit is contained in:
Hermes Agent service account
2026-08-18 23:18:22 -05:00
parent 7867be688a
commit 24735f7e5c
6 changed files with 81 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
cluster/applications/monitoring/llama-swap-alerts.yaml
View File

@@ -15,7 +15,7 @@ spec:
# CRITICAL: GPU VRAM saturation (OOM risk)
# ====================================================================
- alert: LlamaSwapVramSaturation
expr: llamacpp_vram_used_mib > 24000
expr: (llamaswap_gpu_memory_used_bytes{job="llama-swap"} / 1048576) > 24000
for: 1m
labels:
severity: critical
@@ -27,35 +27,34 @@ spec:
The system is at risk of out-of-memory (OOM) kernel-kill events.
# ====================================================================
# WARNING: KV-cache spill risk (context cache pressure)
# WARNING: GPU memory utilization (VRAM pressure proxy)
# ====================================================================
- alert: LlamaSwapKvCacheSpill
expr: llamacpp_kv_cache_usage_ratio > 0.92
- alert: LlamaSwapVramPressure
expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92
for: 2m
labels:
severity: warning
component: llm-inference
annotations:
summary: "KV-cache spill risk on model {{ $labels.model }}"
summary: "GPU memory utilization high (possible VRAM pressure)"
description: |
KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
(warning threshold: 92%).
GPU memory utilization is {{ $value | humanize }}% (warning threshold: 92%).
# ====================================================================
# WARNING: Throughput degradation (possible throttling)
# WARNING: Inference stall (GPU compute idle while VRAM loaded)
# ====================================================================
- alert: LlamaSwapThroughputDegradation
- alert: LlamaSwapInferenceStall
expr: |
(rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40
(llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50)
for: 5m
labels:
severity: warning
component: llm-inference
annotations:
summary: "Token generation throughput low on {{ $labels.model }}"
summary: "GPU compute stall detected (memory loaded but no utilization)"
description: |
Token generation rate is {{ $value | humanize }} tokens/min on {{ $labels.model }}
(baseline threshold: ~50+ tokens/min).
The RTX 3090 has >50% memory utilization but 0% compute utilization
for more than 5 minutes. This may indicate:
# ====================================================================
# WARNING: Scrape failures (monitoring health)