rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: correct metric names in llama-swap monitoring (llamacpp_* -> llamaswap_*), update alerts + dashboard + scrape config

Browse Source
This commit is contained in:
Hermes Agent service account
2026-08-18 23:18:22 -05:00
parent 7867be688a
commit 24735f7e5c
6 changed files with 81 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
ansible/roles/llm-inference-multimodel/templates/llama-swap-alerts.yml.j2
View File

@@ -33,7 +33,7 @@ spec:
# CRITICAL: GPU VRAM saturation (OOM risk)
# ====================================================================
- alert: LlamaSwapVramSaturation
expr: llamacpp_vram_used_mib > {{ llm_swapmode_vram_max_mib | int }}
expr: (llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576) > {{ llm_swapmode_vram_max_mib | int }}
for: 1m
labels:
severity: critical
@@ -56,14 +56,14 @@ spec:
# ====================================================================
# WARNING: KV-cache spill risk (context cache pressure)
# ====================================================================
- alert: LlamaSwapKvCacheSpill
expr: llamacpp_kv_cache_usage_ratio > 0.92
- alert: LlamaSwapVramPressure
expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92
for: 2m
labels:
severity: warning
component: llm-inference
annotations:
summary: "KV-cache spill risk on model {{ $labels.model }}"
summary: "GPU memory utilization high (possible VRAM pressure)"
description: |
KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
(warning threshold: 92%).
@@ -81,18 +81,18 @@ spec:
# ====================================================================
# WARNING: Throughput degradation (possible throttling)
# ====================================================================
- alert: LlamaSwapThroughputDegradation
- alert: LlamaSwapInferenceStall
expr: |
(rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40
(llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50)
for: 5m
labels:
severity: warning
component: llm-inference
annotations:
summary: "Token generation throughput low on {{ $labels.model }}"
summary: "GPU compute stall detected (memory loaded but no utilization)"
description: |
Token generation rate is {{ $value | humanize }}tokens/min on {{ $labels.model }}
(baseline threshold: ~50+ tokens/min).
The RTX 3090 has >50% memory utilization but 0% compute utilization
for more than 5 minutes. This may indicate:
This may indicate:
1. Thermal throttling (GPU temperature limiting frequency)