133 lines
6.3 KiB
Django/Jinja
133 lines
6.3 KiB
Django/Jinja
# ==============================================================================
|
|
# FILE: roles/llm-inference-multimodel/templates/llama-swap-alerts.yml.j2
|
|
# DESCRIPTION: PrometheusRule CustomResource for llama-swap alert rules.
|
|
# Defines CRITICAL, WARNING, and INFO alerts per the Ciro Luciotta
|
|
# monitoring pattern (references/monitoring-llm-homelab-ciro-luciotta-2026.md).
|
|
#
|
|
# Deployed by ArgoCD as a K8s resource in the monitoring namespace.
|
|
# Prometheus loads these rules automatically on sync.
|
|
#
|
|
# SCOPE: Alerts fire when:
|
|
# - VRAM exceeds physical limit (24GB) — pending OOM-kill
|
|
# - KV-cache spills to CPU (>92% utilization) — requests may drop
|
|
# - Throughput degrades below baseline — model may be throttled
|
|
#
|
|
# AUTHOR: Wong (Infrastructure Automation Specialist)
|
|
# DATE: 2026-08-18
|
|
# ==============================================================================
|
|
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: llama-swap-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
prometheus: kube-prometheus
|
|
spec:
|
|
groups:
|
|
- name: llama-swap.rules
|
|
interval: 30s
|
|
rules:
|
|
|
|
# ====================================================================
|
|
# CRITICAL: GPU VRAM saturation (OOM risk)
|
|
# ====================================================================
|
|
- alert: LlamaSwapVramSaturation
|
|
expr: (llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576) > {{ llm_swapmode_vram_max_mib | int }}
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
component: llm-inference
|
|
annotations:
|
|
summary: "GPU VRAM saturation on {{ $labels.instance }}"
|
|
description: |
|
|
GPU VRAM usage is {{ $value | humanize }}MiB (critical threshold: {{ llm_swapmode_vram_max_mib }}MiB).
|
|
|
|
The system is at risk of out-of-memory (OOM) kernel-kill events.
|
|
Immediate action required:
|
|
1. Check Prometheus dashboard for request queue depth and active models
|
|
2. Identify which model(s) are consuming VRAM
|
|
3. If queue depth is high, consider rate-limiting or routing requests
|
|
4. If a single request caused the spike, investigate context-window size
|
|
|
|
Instance: {{ $labels.instance }}
|
|
Time: {{ $value | humanizeDuration }}
|
|
|
|
# ====================================================================
|
|
# WARNING: KV-cache spill risk (context cache pressure)
|
|
# ====================================================================
|
|
- alert: LlamaSwapVramPressure
|
|
expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
component: llm-inference
|
|
annotations:
|
|
summary: "GPU memory utilization high (possible VRAM pressure)"
|
|
description: |
|
|
KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
|
|
(warning threshold: 92%).
|
|
|
|
The model's context cache is nearly full. Requests with large context windows
|
|
may not fit and could be dropped from the queue. Consider:
|
|
1. Reviewing incoming request context-window distribution
|
|
2. Reducing n_ctx for non-critical models (if router mode is active)
|
|
3. Routing long-context requests to a different model with more capacity
|
|
4. Investigating whether concurrent requests are competing for KV space
|
|
|
|
Model: {{ $labels.model }}
|
|
Instance: {{ $labels.instance }}
|
|
|
|
# ====================================================================
|
|
# WARNING: Throughput degradation (possible throttling)
|
|
# ====================================================================
|
|
- alert: LlamaSwapInferenceStall
|
|
expr: |
|
|
(llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50)
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
component: llm-inference
|
|
annotations:
|
|
summary: "GPU compute stall detected (memory loaded but no utilization)"
|
|
description: |
|
|
The RTX 3090 has >50% memory utilization but 0% compute utilization
|
|
for more than 5 minutes. This may indicate:
|
|
|
|
This may indicate:
|
|
1. Thermal throttling (GPU temperature limiting frequency)
|
|
2. Memory pressure (even if VRAM not full, latency can increase)
|
|
3. CPU contention (if models are CPU-offloaded)
|
|
4. Incoming request rate exceeds model capacity (check queue depth)
|
|
|
|
Recommended actions:
|
|
- Check nvidia-smi output for GPU temperature and throttle flags
|
|
- Compare queue depth to baseline (alert if >5 sustained)
|
|
- Check CPU usage and interrupt frequency (vmstat 1 1)
|
|
- Review log tail for errors or warnings from llama-swap
|
|
|
|
Model: {{ $labels.model }}
|
|
Instance: {{ $labels.instance }}
|
|
|
|
# ====================================================================
|
|
# INFO: Scrape failures (monitoring health)
|
|
# ====================================================================
|
|
- alert: LlamaSwapScrapeFailed
|
|
expr: up{job="llama-swap"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
component: monitoring
|
|
annotations:
|
|
summary: "llama-swap Prometheus scrape failed"
|
|
description: |
|
|
Prometheus cannot scrape llama-swap's /metrics endpoint at
|
|
http://{{ $labels.instance }}/metrics (HTTP {{ $value }} or timeout).
|
|
|
|
The monitoring pipeline is degraded. Check:
|
|
1. llama-swap service status: systemctl status llama-swap
|
|
2. Network reachability: curl http://{{ $labels.instance }}/metrics
|
|
3. Prometheus scrape logs in Prometheus UI (Alerts -> llama-swap)
|
|
|
|
Instance: {{ $labels.instance }}
|