# ==============================================================================
# FILE: roles/llm-inference-multimodel/templates/llama-swap-alerts.yml.j2
# DESCRIPTION: PrometheusRule CustomResource for llama-swap alert rules.
#              Defines CRITICAL, WARNING, and INFO alerts per the Ciro Luciotta
#              monitoring pattern (references/monitoring-llm-homelab-ciro-luciotta-2026.md).
#
#              Deployed by ArgoCD as a K8s resource in the monitoring namespace.
#              Prometheus loads these rules automatically on sync.
#
# SCOPE:       Alerts fire when:
#              - VRAM exceeds physical limit (24GB) — pending OOM-kill
#              - KV-cache spills to CPU (>92% utilization) — requests may drop
#              - Throughput degrades below baseline — model may be throttled
#
# AUTHOR:      Wong (Infrastructure Automation Specialist)
# DATE:        2026-08-18
# ==============================================================================

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: llama-swap-alerts
  namespace: monitoring
  labels:
    prometheus: kube-prometheus
spec:
  groups:
    - name: llama-swap.rules
      interval: 30s
      rules:

        # ====================================================================
        # CRITICAL: GPU VRAM saturation (OOM risk)
        # ====================================================================
        - alert: LlamaSwapVramSaturation
          expr: llamacpp_vram_used_mib > {{ llm_swapmode_vram_max_mib | int }}
          for: 1m
          labels:
            severity: critical
            component: llm-inference
          annotations:
            summary: "GPU VRAM saturation on {{ $labels.instance }}"
            description: |
              GPU VRAM usage is {{ $value | humanize }}MiB (critical threshold: {{ llm_swapmode_vram_max_mib }}MiB).
              
              The system is at risk of out-of-memory (OOM) kernel-kill events. 
              Immediate action required:
              1. Check Prometheus dashboard for request queue depth and active models
              2. Identify which model(s) are consuming VRAM
              3. If queue depth is high, consider rate-limiting or routing requests
              4. If a single request caused the spike, investigate context-window size
              
              Instance: {{ $labels.instance }}
              Time: {{ $value | humanizeDuration }}

        # ====================================================================
        # WARNING: KV-cache spill risk (context cache pressure)
        # ====================================================================
        - alert: LlamaSwapKvCacheSpill
          expr: llamacpp_kv_cache_usage_ratio > 0.92
          for: 2m
          labels:
            severity: warning
            component: llm-inference
          annotations:
            summary: "KV-cache spill risk on model {{ $labels.model }}"
            description: |
              KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
              (warning threshold: 92%).
              
              The model's context cache is nearly full. Requests with large context windows
              may not fit and could be dropped from the queue. Consider:
              1. Reviewing incoming request context-window distribution
              2. Reducing n_ctx for non-critical models (if router mode is active)
              3. Routing long-context requests to a different model with more capacity
              4. Investigating whether concurrent requests are competing for KV space
              
              Model: {{ $labels.model }}
              Instance: {{ $labels.instance }}

        # ====================================================================
        # WARNING: Throughput degradation (possible throttling)
        # ====================================================================
        - alert: LlamaSwapThroughputDegradation
          expr: |
            (rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40
          for: 5m
          labels:
            severity: warning
            component: llm-inference
          annotations:
            summary: "Token generation throughput low on {{ $labels.model }}"
            description: |
              Token generation rate is {{ $value | humanize }}tokens/min on {{ $labels.model }}
              (baseline threshold: ~50+ tokens/min).
              
              This may indicate:
              1. Thermal throttling (GPU temperature limiting frequency)
              2. Memory pressure (even if VRAM not full, latency can increase)
              3. CPU contention (if models are CPU-offloaded)
              4. Incoming request rate exceeds model capacity (check queue depth)
              
              Recommended actions:
              - Check nvidia-smi output for GPU temperature and throttle flags
              - Compare queue depth to baseline (alert if >5 sustained)
              - Check CPU usage and interrupt frequency (vmstat 1 1)
              - Review log tail for errors or warnings from llama-swap
              
              Model: {{ $labels.model }}
              Instance: {{ $labels.instance }}

        # ====================================================================
        # INFO: Scrape failures (monitoring health)
        # ====================================================================
        - alert: LlamaSwapScrapeFailed
          expr: up{job="llama-swap"} == 0
          for: 2m
          labels:
            severity: warning
            component: monitoring
          annotations:
            summary: "llama-swap Prometheus scrape failed"
            description: |
              Prometheus cannot scrape llama-swap's /metrics endpoint at
              http://{{ $labels.instance }}/metrics (HTTP {{ $value }} or timeout).
              
              The monitoring pipeline is degraded. Check:
              1. llama-swap service status: systemctl status llama-swap
              2. Network reachability: curl http://{{ $labels.instance }}/metrics
              3. Prometheus scrape logs in Prometheus UI (Alerts -> llama-swap)
              
              Instance: {{ $labels.instance }}