apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: llama-swap-alerts
  namespace: monitoring
  labels:
    prometheus: kube-prometheus
    app.kubernetes.io/part-of: monitoring
spec:
  groups:
    - name: llama-swap.rules
      interval: 30s
      rules:
        # ====================================================================
        # CRITICAL: GPU VRAM saturation (OOM risk)
        # ====================================================================
        - alert: LlamaSwapVramSaturation
          expr: (llamaswap_gpu_memory_used_bytes{job="llama-swap"} / 1048576) > 24000
          for: 1m
          labels:
            severity: critical
            component: llm-inference
          annotations:
            summary: "GPU VRAM saturation on {{ $labels.instance }}"
            description: |
              GPU VRAM usage is {{ $value | humanize }}MiB (critical threshold: 24000MiB).
              The system is at risk of out-of-memory (OOM) kernel-kill events.

        # ====================================================================
        # WARNING: GPU memory utilization (VRAM pressure proxy)
        # ====================================================================
        - alert: LlamaSwapVramPressure
          expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92
          for: 2m
          labels:
            severity: warning
            component: llm-inference
          annotations:
            summary: "GPU memory utilization high (possible VRAM pressure)"
            description: |
              GPU memory utilization is {{ $value | humanize }}% (warning threshold: 92%).

        # ====================================================================
        # WARNING: Inference stall (GPU compute idle while VRAM loaded)
        # ====================================================================
        - alert: LlamaSwapInferenceStall
          expr: |
            (llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50)
          for: 5m
          labels:
            severity: warning
            component: llm-inference
          annotations:
            summary: "GPU compute stall detected (memory loaded but no utilization)"
            description: |
              The RTX 3090 has >50% memory utilization but 0% compute utilization
              for more than 5 minutes. This may indicate:

        # ====================================================================
        # WARNING: Scrape failures (monitoring health)
        # ====================================================================
        - alert: LlamaSwapScrapeFailed
          expr: up{job="llama-swap"} == 0
          for: 2m
          labels:
            severity: warning
            component: monitoring
          annotations:
            summary: "llama-swap Prometheus scrape failed"
            description: |
              Prometheus cannot scrape llama-swap's /metrics endpoint.
              Check: systemctl status llama-swap, curl http://{{ $labels.instance }}/metrics