apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: llama-swap-alerts
  namespace: monitoring
  labels:
    prometheus: kube-prometheus
    app.kubernetes.io/part-of: monitoring
spec:
  groups:
    - name: llama-swap.rules
      interval: 30s
      rules:
        # ====================================================================
        # CRITICAL: GPU VRAM saturation (OOM risk)
        # ====================================================================
        - alert: LlamaSwapVramSaturation
          expr: llamacpp_vram_used_mib > 24000
          for: 1m
          labels:
            severity: critical
            component: llm-inference
          annotations:
            summary: "GPU VRAM saturation on {{ $labels.instance }}"
            description: |
              GPU VRAM usage is {{ $value | humanize }}MiB (critical threshold: 24000MiB).
              The system is at risk of out-of-memory (OOM) kernel-kill events.

        # ====================================================================
        # WARNING: KV-cache spill risk (context cache pressure)
        # ====================================================================
        - alert: LlamaSwapKvCacheSpill
          expr: llamacpp_kv_cache_usage_ratio > 0.92
          for: 2m
          labels:
            severity: warning
            component: llm-inference
          annotations:
            summary: "KV-cache spill risk on model {{ $labels.model }}"
            description: |
              KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
              (warning threshold: 92%).

        # ====================================================================
        # WARNING: Throughput degradation (possible throttling)
        # ====================================================================
        - alert: LlamaSwapThroughputDegradation
          expr: |
            (rate(llamacpp_tokens_predicted_total[5m]) * 60) < 40
          for: 5m
          labels:
            severity: warning
            component: llm-inference
          annotations:
            summary: "Token generation throughput low on {{ $labels.model }}"
            description: |
              Token generation rate is {{ $value | humanize }} tokens/min on {{ $labels.model }}
              (baseline threshold: ~50+ tokens/min).

        # ====================================================================
        # WARNING: Scrape failures (monitoring health)
        # ====================================================================
        - alert: LlamaSwapScrapeFailed
          expr: up{job="llama-swap"} == 0
          for: 2m
          labels:
            severity: warning
            component: monitoring
          annotations:
            summary: "llama-swap Prometheus scrape failed"
            description: |
              Prometheus cannot scrape llama-swap's /metrics endpoint.
              Check: systemctl status llama-swap, curl http://{{ $labels.instance }}/metrics