74 lines
3.0 KiB
YAML
74 lines
3.0 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: llama-swap-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
prometheus: kube-prometheus
|
|
app.kubernetes.io/part-of: monitoring
|
|
spec:
|
|
groups:
|
|
- name: llama-swap.rules
|
|
interval: 30s
|
|
rules:
|
|
# ====================================================================
|
|
# CRITICAL: GPU VRAM saturation (OOM risk)
|
|
# ====================================================================
|
|
- alert: LlamaSwapVramSaturation
|
|
expr: llamacpp_vram_used_mib > 24000
|
|
for: 1m
|
|
labels:
|
|
severity: critical
|
|
component: llm-inference
|
|
annotations:
|
|
summary: "GPU VRAM saturation on {{ $labels.instance }}"
|
|
description: |
|
|
GPU VRAM usage is {{ $value | humanize }}MiB (critical threshold: 24000MiB).
|
|
The system is at risk of out-of-memory (OOM) kernel-kill events.
|
|
|
|
# ====================================================================
|
|
# WARNING: KV-cache spill risk (context cache pressure)
|
|
# ====================================================================
|
|
- alert: LlamaSwapKvCacheSpill
|
|
expr: llamacpp_kv_cache_usage_ratio > 0.92
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
component: llm-inference
|
|
annotations:
|
|
summary: "KV-cache spill risk on model {{ $labels.model }}"
|
|
description: |
|
|
KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
|
|
(warning threshold: 92%).
|
|
|
|
# ====================================================================
|
|
# WARNING: Throughput degradation (possible throttling)
|
|
# ====================================================================
|
|
- alert: LlamaSwapThroughputDegradation
|
|
expr: |
|
|
(rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
component: llm-inference
|
|
annotations:
|
|
summary: "Token generation throughput low on {{ $labels.model }}"
|
|
description: |
|
|
Token generation rate is {{ $value | humanize }} tokens/min on {{ $labels.model }}
|
|
(baseline threshold: ~50+ tokens/min).
|
|
|
|
# ====================================================================
|
|
# WARNING: Scrape failures (monitoring health)
|
|
# ====================================================================
|
|
- alert: LlamaSwapScrapeFailed
|
|
expr: up{job="llama-swap"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
component: monitoring
|
|
annotations:
|
|
summary: "llama-swap Prometheus scrape failed"
|
|
description: |
|
|
Prometheus cannot scrape llama-swap's /metrics endpoint.
|
|
Check: systemctl status llama-swap, curl http://{{ $labels.instance }}/metrics
|