73 lines
3.1 KiB
YAML
73 lines
3.1 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: llama-swap-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
prometheus: kube-prometheus
|
|
app.kubernetes.io/part-of: monitoring
|
|
spec:
|
|
groups:
|
|
- name: llama-swap.rules
|
|
interval: 30s
|
|
rules:
|
|
# ====================================================================
|
|
# CRITICAL: GPU VRAM saturation (OOM risk)
|
|
# ====================================================================
|
|
- alert: LlamaSwapVramSaturation
|
|
expr: (llamaswap_gpu_memory_used_bytes{job="llama-swap"} / 1048576) > 24000
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
component: llm-inference
|
|
annotations:
|
|
summary: "GPU VRAM saturation on {{ $labels.instance }}"
|
|
description: |
|
|
GPU VRAM usage is {{ $value | humanize }}MiB (critical threshold: 24000MiB).
|
|
The system is at risk of out-of-memory (OOM) kernel-kill events.
|
|
|
|
# ====================================================================
|
|
# WARNING: GPU memory utilization (VRAM pressure proxy)
|
|
# ====================================================================
|
|
- alert: LlamaSwapVramPressure
|
|
expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
component: llm-inference
|
|
annotations:
|
|
summary: "GPU memory utilization high (possible VRAM pressure)"
|
|
description: |
|
|
GPU memory utilization is {{ $value | humanize }}% (warning threshold: 92%).
|
|
|
|
# ====================================================================
|
|
# WARNING: Inference stall (GPU compute idle while VRAM loaded)
|
|
# ====================================================================
|
|
- alert: LlamaSwapInferenceStall
|
|
expr: |
|
|
(llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50)
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
component: llm-inference
|
|
annotations:
|
|
summary: "GPU compute stall detected (memory loaded but no utilization)"
|
|
description: |
|
|
The RTX 3090 has >50% memory utilization but 0% compute utilization
|
|
for more than 5 minutes. This may indicate:
|
|
|
|
# ====================================================================
|
|
# WARNING: Scrape failures (monitoring health)
|
|
# ====================================================================
|
|
- alert: LlamaSwapScrapeFailed
|
|
expr: up{job="llama-swap"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
component: monitoring
|
|
annotations:
|
|
summary: "llama-swap Prometheus scrape failed"
|
|
description: |
|
|
Prometheus cannot scrape llama-swap's /metrics endpoint.
|
|
Check: systemctl status llama-swap, curl http://{{ $labels.instance }}/metrics
|