# ------------------------------------------------------------------------------
# kube-prometheus-stack — Helm Values
# Chart: https://prometheus-community.github.io/helm-charts
# Version: 70.4.2
# ------------------------------------------------------------------------------

# ─── Alertmanager ─────────────────────────────────────────────────────────────
alertmanager:
  enabled: true
  alertmanagerSpec:
    storage:
      volumeClaimTemplate:
        spec:
          storageClassName: nfs-emporium
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 5Gi

# ─── Prometheus ───────────────────────────────────────────────────────────────
prometheus:
  prometheusSpec:
    retention: 30d
    retentionSize: "45GB"

    storageSpec:
      volumeClaimTemplate:
        spec:
          storageClassName: nfs-emporium
          accessModes:
            - ReadWriteOnce
          resources:
            requests:
              storage: 50Gi

    # Scrape all ServiceMonitors/PodMonitors regardless of namespace
    serviceMonitorSelectorNilUsesHelmValues: false
    podMonitorSelectorNilUsesHelmValues: false
    ruleSelectorNilUsesHelmValues: false

    resources:
      requests:
        cpu: 200m
        memory: 512Mi
      limits:
        cpu: 1000m
        memory: 2Gi

    # ─── Additional scrape configs — external targets ──────────────────────────
    additionalScrapeConfigs:

      # Proxmox nodes — Node Exporter on each hypervisor
      - job_name: proxmox-nodes
        scrape_interval: 30s
        static_configs:
          - targets:
              - 10.1.71.11:9100
              - 10.1.71.12:9100
              - 10.1.71.13:9100
            labels:
              job: proxmox

      # cinderella-castle — existing VM-side Prometheus host
      # Remove this once cinderella-castle is decommissioned
      - job_name: node-cinderella-castle
        scrape_interval: 30s
        static_configs:
          - targets:
              - 10.1.71.31:9100
            labels:
              hostname: cinderella-castle

      # Traefik metrics
      - job_name: traefik
        scrape_interval: 30s
        static_configs:
          - targets:
              - 10.1.71.35:8080
            labels:
              hostname: lightning-lane

      # Proxmox VE Exporter — cluster-level metrics (VM states, storage, CPU)
      # Multi-target pattern: Prometheus passes the cluster address as a param
      - job_name: proxmox-cluster
        scrape_interval: 30s
        metrics_path: /pve
        params:
          module: [default]
          cluster: ["1"]
          node: [""]
        static_configs:
          - targets:
              - 10.1.71.11  # main-street-usa — cluster API endpoint
            labels:
              cluster: magic-kingdom
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: pve-exporter.monitoring.svc.cluster.local:9221

      # SNMP — UniFi devices (multi-target pattern)
      # world-drive (UDM Pro)
      - job_name: snmp-udm-pro
        scrape_interval: 60s
        scrape_timeout: 55s
        metrics_path: /snmp
        params:
          auth: [mk_labs_v2c]
          module: [if_mib]
        static_configs:
          - targets:
              - 192.168.1.1
            labels:
              device: world-drive
              model: UDM-Pro
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: snmp-exporter.monitoring.svc.cluster.local:9116

      # SNMP – usw-pro-aggregation
      - job_name: snmp-usw-pro-aggregation
        scrape_interval: 60s
        scrape_timeout: 55s
        metrics_path: /snmp
        params:
          auth: [mk_labs_v2c]
          module: [if_mib]
        static_configs:
          - targets:
              - 192.168.1.123
            labels:
              device: usw-pro-aggregation
              model: USW-Pro-Aggregation
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: snmp-exporter.monitoring.svc.cluster.local:9116

      # usw-pro-max-24
      - job_name: snmp-usw-pro-max-24
        scrape_interval: 60s
        scrape_timeout: 55s
        metrics_path: /snmp
        params:
          auth: [mk_labs_v2c]
          module: [if_mib]
        static_configs:
          - targets:
              - 192.168.1.226
            labels:
              device: usw-pro-max-24
              model: USW-Pro-Max-24
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: snmp-exporter.monitoring.svc.cluster.local:9116

      # usw-lite-16-poe
      - job_name: snmp-usw-lite-16-poe
        scrape_interval: 60s
        scrape_timeout: 55s
        metrics_path: /snmp
        params:
          auth: [mk_labs_v2c]
          module: [if_mib]
        static_configs:
          - targets:
              - 192.168.1.136
            labels:
              device: usw-lite-16-poe
              model: USW-Lite-16-PoE
        relabel_configs:
          - source_labels: [__address__]
            target_label: __param_target
          - source_labels: [__param_target]
            target_label: instance
          - target_label: __address__
            replacement: snmp-exporter.monitoring.svc.cluster.local:9116

      # Pure FlashArray — DEFERRED until Purity 6.7+
      # Uncomment when utilidor is upgraded. Native OpenMetrics endpoint,
      # no exporter container needed. Add API token to 1Password as
      # item "pure-flasharray-utilidor" field "api-token".
      # - job_name: pure-flasharray-utilidor
      #   scrape_interval: 30s
      #   metrics_path: /metrics/array
      #   scheme: https
      #   tls_config:
      #     insecure_skip_verify: true
      #   authorization:
      #     credentials_file: /etc/prometheus/secrets/pure-api-token/api-token
      #   params:
      #     namespace: ["purefa"]
      #   static_configs:
      #     - targets:
      #         - utilidor.local.mk-labs.cloud
      #       labels:
      #         array: utilidor
      #         array_type: physical

      # astro-orbiter — LLM inference host (Ryzen 7 5800XT / RTX 3090)
      # Managed by roles/llm-inference (Phase monitoring). Three targets:
      # node (system), gpu (nvidia_gpu_exporter), llama-server (inference metrics)
      - job_name: node-astro-orbiter
        scrape_interval: 30s
        static_configs:
          - targets:
              - 10.1.71.130:9100
            labels:
              hostname: astro-orbiter

      - job_name: gpu-astro-orbiter
        scrape_interval: 15s
        static_configs:
          - targets:
              - 10.1.71.130:9835
            labels:
              hostname: astro-orbiter
              gpu: rtx3090

      # Router exposes per-model Prometheus metrics via /metrics?model=<id>.
      # The old :8000 target (gemma-2-27b-it-GGUF) was removed after the day2
      # cutover (2026-08-12) to llama-server-router on :8002. Three separate
      # jobs — one per model registered in the router — are used so each gets
      # its own model label. The static `model` label is set to the canonical
      # llama.cpp model id (the `id` field from /v1/models, not the alias).
      # scrape_interval: 90s (reduced from 15s) -- each /metrics?model= request
      # wakes the GPU sub-server to P2 (~110W); at 15s, continuous spikes kept
      # the GPU drawing ~110W despite zero inference load. At 90s, the GPU gets
      # ~80s of genuine P8 idle (~20W) between wake-ups. (t_e7d547ea, 2026-08-13)

      # llama-server-astro-orbiter-qwen3 REMOVED (t_02c15dae, 2026-08-13):
      # /metrics?model=Qwen3.6 forces the router to attempt loading Qwen3.6 each
      # scrape cycle, which fails with CUDA OOM (VRAM already consumed by resident
      # Llama3+Phi3.5). This generated real GPU power spikes, not just a benign
      # counter read. nvidia_gpu_exporter (:9835) already covers GPU power/VRAM/
      # utilization at zero wake cost. No Grafana dashboard panel depends on
      # Qwen3.6-specific llama-server metrics. Ryan approved full removal.
      # Commented out rather than deleted for easy revert if Qwen3.6 is ever
      # re-added as a resident model.
      #
      # - job_name: llama-server-astro-orbiter-qwen3
      #   scrape_interval: 90s
      #   metrics_path: /metrics
      #   params:
      #     model: ["Qwen3.6-35B-A3B-UD-Q4_K_S"]
      #   static_configs:
      #     - targets:
      #         - 10.1.71.130:8002
      #       labels:
      #         hostname: astro-orbiter
      #         endpoint: astro-orbiter-router
      #         model: Qwen3.6-35B-A3B-UD-Q4_K_S

      # llama-server-astro-orbiter-llama3 — DEPRECATED (2026-08-18):
      # Router mode on :8002 replaced by llama-swap on :8001. llama-swap exposes
      # single /metrics endpoint (not per-model). See llama-swap job below.
      # - job_name: llama-server-astro-orbiter-llama3
      #   scrape_interval: 90s
      #   metrics_path: /metrics
      #   params:
      #     model: ["Meta-Llama-3.1-8B-Instruct-Q4_K_M"]
      #   static_configs:
      #     - targets:
      #         - 10.1.71.130:8002
      #       labels:
      #         hostname: astro-orbiter
      #         endpoint: astro-orbiter-router
      #         model: Meta-Llama-3.1-8B-Instruct-Q4_K_M

      # llama-server-astro-orbiter-phi35 — DEPRECATED (2026-08-18):
      # Same as above — router replaced by llama-swap. Use llama-swap /metrics.
      # - job_name: llama-server-astro-orbiter-phi35
      #   scrape_interval: 90s
      #   metrics_path: /metrics
      #   params:
      #     model: ["Phi-3.5-mini-instruct-Q8_0"]
      #   static_configs:
      #     - targets:
      #         - 10.1.71.130:8002
      #       labels:
      #         hostname: astro-orbiter
      #         endpoint: astro-orbiter-router
      #         model: Phi-3.5-mini-instruct-Q8_0

      # llama-swap (production, since 2026-08-18)
      # Replaces the per-model /metrics?model=<id> jobs above (all targeting now-deprecated :8002).
      # llama-swap exposes system-level metrics (llamaswap_*) — VRAM, GPU util, power, CPU, network.
      # Per-model inference metrics (tokens/sec, latency, KV-cache) are NOT exposed at the proxy level;
      # they remain on the individual llama-server child instances, scraped via node_exporter textfile
      # collector for VRAM, and via the GPU exporter (:9835) for GPU-level telemetry.
      - job_name: llama-swap
        scrape_interval: 30s
        scrape_timeout: 10s
        static_configs:
          - targets:
              - 10.1.71.130:8001
            labels:
              hostname: astro-orbiter
              service: llama-swap
              environment: homelab
        metrics_path: /metrics
        honor_labels: true
        metric_relabel_configs:
          - source_labels: [__name__]
            regex: 'llamaswap_.*'
            action: keep

# ─── Grafana ──────────────────────────────────────────────────────────────────
grafana:
  enabled: true
  persistence:
    enabled: false

  admin:
    existingSecret: grafana-admin-secret
    userKey: admin-user
    passwordKey: admin-password

  grafana.ini:
    server:
      root_url: https://grafana.local.mk-labs.cloud
    auth:
      disable_login_form: false

  defaultDashboardsEnabled: true
  defaultDashboardsTimezone: browser

  dashboardProviders:
    dashboardproviders.yaml:
      apiVersion: 1
      providers:
        - name: default
          orgId: 1
          folder: ""
          type: file
          disableDeletion: false
          editable: true
          options:
            path: /var/lib/grafana/dashboards/default

  dashboards:
    default:
      # ── Already present ──────────────────────────────────────────────────────
      node-exporter-full:
        gnetId: 1860
        revision: 37
        datasource: Prometheus
      proxmox-cluster:
        gnetId: 10347
        revision: 5
        datasource: Prometheus
      unifi-poller:
        gnetId: 11315
        revision: 9
        datasource: Prometheus

      # ── Kubernetes ───────────────────────────────────────────────────────────
      kubernetes-global:
        gnetId: 15757
        revision: 37
        datasource: Prometheus
      kubernetes-namespace:
        gnetId: 15758
        revision: 34
        datasource: Prometheus
      kubernetes-pods:
        gnetId: 6336
        revision: 1
        datasource: Prometheus

      # ── Platform services ─────────────────────────────────────────────────────
      argocd:
        gnetId: 19974
        revision: 3
        datasource: Prometheus
      cert-manager:
        gnetId: 20842
        revision: 3
        datasource: Prometheus
      cilium-dashboard:
        gnetId: 16611
        revision: 1
        datasource: Prometheus

      # ── Infrastructure ────────────────────────────────────────────────────────
      # Traefik — uncomment when P.02 is done (--metrics.prometheus=true)
      # traefik:
      #   gnetId: 17346
      #   revision: 9
      #   datasource: Prometheus

  sidecar:
    dashboards:
      enabled: true
      searchNamespace: ALL
      label: grafana_dashboard
      labelValue: "1"

  resources:
    requests:
      cpu: 100m
      memory: 256Mi
    limits:
      cpu: 500m
      memory: 512Mi

# ─── Node Exporter ────────────────────────────────────────────────────────────
nodeExporter:
  enabled: true

prometheus-node-exporter:
  tolerations:
    - key: node-role.kubernetes.io/control-plane
      operator: Exists
      effect: NoSchedule

# ─── kube-state-metrics ───────────────────────────────────────────────────────
kubeStateMetrics:
  enabled: true

# ─── Prometheus Operator ──────────────────────────────────────────────────────
prometheusOperator:
  enabled: true

# ─── Component scraping ───────────────────────────────────────────────────────
kubeApiServer:
  enabled: true

kubelet:
  enabled: true

kubeControllerManager:
  enabled: true

kubeScheduler:
  enabled: true

kubeProxy:
  enabled: false  # Cilium replaces kube-proxy

kubeEtcd:
  enabled: true
  endpoints:
    - 10.1.71.66
    - 10.1.71.67
    - 10.1.71.68
  service:
    enabled: true
    port: 2381
    targetPort: 2381