466 lines
16 KiB
YAML
466 lines
16 KiB
YAML
# ------------------------------------------------------------------------------
|
||
# kube-prometheus-stack — Helm Values
|
||
# Chart: https://prometheus-community.github.io/helm-charts
|
||
# Version: 70.4.2
|
||
# ------------------------------------------------------------------------------
|
||
|
||
# ─── Alertmanager ─────────────────────────────────────────────────────────────
|
||
alertmanager:
|
||
enabled: true
|
||
alertmanagerSpec:
|
||
storage:
|
||
volumeClaimTemplate:
|
||
spec:
|
||
storageClassName: nfs-emporium
|
||
accessModes:
|
||
- ReadWriteOnce
|
||
resources:
|
||
requests:
|
||
storage: 5Gi
|
||
|
||
# ─── Prometheus ───────────────────────────────────────────────────────────────
|
||
prometheus:
|
||
prometheusSpec:
|
||
retention: 30d
|
||
retentionSize: "45GB"
|
||
|
||
storageSpec:
|
||
volumeClaimTemplate:
|
||
spec:
|
||
storageClassName: nfs-emporium
|
||
accessModes:
|
||
- ReadWriteOnce
|
||
resources:
|
||
requests:
|
||
storage: 50Gi
|
||
|
||
# Scrape all ServiceMonitors/PodMonitors regardless of namespace
|
||
serviceMonitorSelectorNilUsesHelmValues: false
|
||
podMonitorSelectorNilUsesHelmValues: false
|
||
ruleSelectorNilUsesHelmValues: false
|
||
|
||
resources:
|
||
requests:
|
||
cpu: 200m
|
||
memory: 512Mi
|
||
limits:
|
||
cpu: 1000m
|
||
memory: 2Gi
|
||
|
||
# ─── Additional scrape configs — external targets ──────────────────────────
|
||
additionalScrapeConfigs:
|
||
|
||
# Proxmox nodes — Node Exporter on each hypervisor
|
||
- job_name: proxmox-nodes
|
||
scrape_interval: 30s
|
||
static_configs:
|
||
- targets:
|
||
- 10.1.71.11:9100
|
||
- 10.1.71.12:9100
|
||
- 10.1.71.13:9100
|
||
labels:
|
||
job: proxmox
|
||
|
||
# cinderella-castle — existing VM-side Prometheus host
|
||
# Remove this once cinderella-castle is decommissioned
|
||
- job_name: node-cinderella-castle
|
||
scrape_interval: 30s
|
||
static_configs:
|
||
- targets:
|
||
- 10.1.71.31:9100
|
||
labels:
|
||
hostname: cinderella-castle
|
||
|
||
# Traefik metrics
|
||
- job_name: traefik
|
||
scrape_interval: 30s
|
||
static_configs:
|
||
- targets:
|
||
- 10.1.71.35:8080
|
||
labels:
|
||
hostname: lightning-lane
|
||
|
||
# Proxmox VE Exporter — cluster-level metrics (VM states, storage, CPU)
|
||
# Multi-target pattern: Prometheus passes the cluster address as a param
|
||
- job_name: proxmox-cluster
|
||
scrape_interval: 30s
|
||
metrics_path: /pve
|
||
params:
|
||
module: [default]
|
||
cluster: ["1"]
|
||
node: [""]
|
||
static_configs:
|
||
- targets:
|
||
- 10.1.71.11 # main-street-usa — cluster API endpoint
|
||
labels:
|
||
cluster: magic-kingdom
|
||
relabel_configs:
|
||
- source_labels: [__address__]
|
||
target_label: __param_target
|
||
- source_labels: [__param_target]
|
||
target_label: instance
|
||
- target_label: __address__
|
||
replacement: pve-exporter.monitoring.svc.cluster.local:9221
|
||
|
||
# SNMP — UniFi devices (multi-target pattern)
|
||
# world-drive (UDM Pro)
|
||
- job_name: snmp-udm-pro
|
||
scrape_interval: 60s
|
||
scrape_timeout: 55s
|
||
metrics_path: /snmp
|
||
params:
|
||
auth: [mk_labs_v2c]
|
||
module: [if_mib]
|
||
static_configs:
|
||
- targets:
|
||
- 192.168.1.1
|
||
labels:
|
||
device: world-drive
|
||
model: UDM-Pro
|
||
relabel_configs:
|
||
- source_labels: [__address__]
|
||
target_label: __param_target
|
||
- source_labels: [__param_target]
|
||
target_label: instance
|
||
- target_label: __address__
|
||
replacement: snmp-exporter.monitoring.svc.cluster.local:9116
|
||
|
||
# SNMP – usw-pro-aggregation
|
||
- job_name: snmp-usw-pro-aggregation
|
||
scrape_interval: 60s
|
||
scrape_timeout: 55s
|
||
metrics_path: /snmp
|
||
params:
|
||
auth: [mk_labs_v2c]
|
||
module: [if_mib]
|
||
static_configs:
|
||
- targets:
|
||
- 192.168.1.123
|
||
labels:
|
||
device: usw-pro-aggregation
|
||
model: USW-Pro-Aggregation
|
||
relabel_configs:
|
||
- source_labels: [__address__]
|
||
target_label: __param_target
|
||
- source_labels: [__param_target]
|
||
target_label: instance
|
||
- target_label: __address__
|
||
replacement: snmp-exporter.monitoring.svc.cluster.local:9116
|
||
|
||
# usw-pro-max-24
|
||
- job_name: snmp-usw-pro-max-24
|
||
scrape_interval: 60s
|
||
scrape_timeout: 55s
|
||
metrics_path: /snmp
|
||
params:
|
||
auth: [mk_labs_v2c]
|
||
module: [if_mib]
|
||
static_configs:
|
||
- targets:
|
||
- 192.168.1.226
|
||
labels:
|
||
device: usw-pro-max-24
|
||
model: USW-Pro-Max-24
|
||
relabel_configs:
|
||
- source_labels: [__address__]
|
||
target_label: __param_target
|
||
- source_labels: [__param_target]
|
||
target_label: instance
|
||
- target_label: __address__
|
||
replacement: snmp-exporter.monitoring.svc.cluster.local:9116
|
||
|
||
# usw-lite-16-poe
|
||
- job_name: snmp-usw-lite-16-poe
|
||
scrape_interval: 60s
|
||
scrape_timeout: 55s
|
||
metrics_path: /snmp
|
||
params:
|
||
auth: [mk_labs_v2c]
|
||
module: [if_mib]
|
||
static_configs:
|
||
- targets:
|
||
- 192.168.1.136
|
||
labels:
|
||
device: usw-lite-16-poe
|
||
model: USW-Lite-16-PoE
|
||
relabel_configs:
|
||
- source_labels: [__address__]
|
||
target_label: __param_target
|
||
- source_labels: [__param_target]
|
||
target_label: instance
|
||
- target_label: __address__
|
||
replacement: snmp-exporter.monitoring.svc.cluster.local:9116
|
||
|
||
# Pure FlashArray — DEFERRED until Purity 6.7+
|
||
# Uncomment when utilidor is upgraded. Native OpenMetrics endpoint,
|
||
# no exporter container needed. Add API token to 1Password as
|
||
# item "pure-flasharray-utilidor" field "api-token".
|
||
# - job_name: pure-flasharray-utilidor
|
||
# scrape_interval: 30s
|
||
# metrics_path: /metrics/array
|
||
# scheme: https
|
||
# tls_config:
|
||
# insecure_skip_verify: true
|
||
# authorization:
|
||
# credentials_file: /etc/prometheus/secrets/pure-api-token/api-token
|
||
# params:
|
||
# namespace: ["purefa"]
|
||
# static_configs:
|
||
# - targets:
|
||
# - utilidor.local.mk-labs.cloud
|
||
# labels:
|
||
# array: utilidor
|
||
# array_type: physical
|
||
|
||
# astro-orbiter — LLM inference host (Ryzen 7 5800XT / RTX 3090)
|
||
# Managed by roles/llm-inference (Phase monitoring). Three targets:
|
||
# node (system), gpu (nvidia_gpu_exporter), llama-server (inference metrics)
|
||
- job_name: node-astro-orbiter
|
||
scrape_interval: 30s
|
||
static_configs:
|
||
- targets:
|
||
- 10.1.71.130:9100
|
||
labels:
|
||
hostname: astro-orbiter
|
||
|
||
- job_name: gpu-astro-orbiter
|
||
scrape_interval: 15s
|
||
static_configs:
|
||
- targets:
|
||
- 10.1.71.130:9835
|
||
labels:
|
||
hostname: astro-orbiter
|
||
gpu: rtx3090
|
||
|
||
# Router exposes per-model Prometheus metrics via /metrics?model=<id>.
|
||
# The old :8000 target (gemma-2-27b-it-GGUF) was removed after the day2
|
||
# cutover (2026-08-12) to llama-server-router on :8002. Three separate
|
||
# jobs — one per model registered in the router — are used so each gets
|
||
# its own model label. The static `model` label is set to the canonical
|
||
# llama.cpp model id (the `id` field from /v1/models, not the alias).
|
||
# scrape_interval: 90s (reduced from 15s) -- each /metrics?model= request
|
||
# wakes the GPU sub-server to P2 (~110W); at 15s, continuous spikes kept
|
||
# the GPU drawing ~110W despite zero inference load. At 90s, the GPU gets
|
||
# ~80s of genuine P8 idle (~20W) between wake-ups. (t_e7d547ea, 2026-08-13)
|
||
|
||
# llama-server-astro-orbiter-qwen3 REMOVED (t_02c15dae, 2026-08-13):
|
||
# /metrics?model=Qwen3.6 forces the router to attempt loading Qwen3.6 each
|
||
# scrape cycle, which fails with CUDA OOM (VRAM already consumed by resident
|
||
# Llama3+Phi3.5). This generated real GPU power spikes, not just a benign
|
||
# counter read. nvidia_gpu_exporter (:9835) already covers GPU power/VRAM/
|
||
# utilization at zero wake cost. No Grafana dashboard panel depends on
|
||
# Qwen3.6-specific llama-server metrics. Ryan approved full removal.
|
||
# Commented out rather than deleted for easy revert if Qwen3.6 is ever
|
||
# re-added as a resident model.
|
||
#
|
||
# - job_name: llama-server-astro-orbiter-qwen3
|
||
# scrape_interval: 90s
|
||
# metrics_path: /metrics
|
||
# params:
|
||
# model: ["Qwen3.6-35B-A3B-UD-Q4_K_S"]
|
||
# static_configs:
|
||
# - targets:
|
||
# - 10.1.71.130:8002
|
||
# labels:
|
||
# hostname: astro-orbiter
|
||
# endpoint: astro-orbiter-router
|
||
# model: Qwen3.6-35B-A3B-UD-Q4_K_S
|
||
|
||
# llama-server-astro-orbiter-llama3 — DEPRECATED (2026-08-18):
|
||
# Router mode on :8002 replaced by llama-swap on :8001. llama-swap exposes
|
||
# single /metrics endpoint (not per-model). See llama-swap job below.
|
||
# - job_name: llama-server-astro-orbiter-llama3
|
||
# scrape_interval: 90s
|
||
# metrics_path: /metrics
|
||
# params:
|
||
# model: ["Meta-Llama-3.1-8B-Instruct-Q4_K_M"]
|
||
# static_configs:
|
||
# - targets:
|
||
# - 10.1.71.130:8002
|
||
# labels:
|
||
# hostname: astro-orbiter
|
||
# endpoint: astro-orbiter-router
|
||
# model: Meta-Llama-3.1-8B-Instruct-Q4_K_M
|
||
|
||
# llama-server-astro-orbiter-phi35 — DEPRECATED (2026-08-18):
|
||
# Same as above — router replaced by llama-swap. Use llama-swap /metrics.
|
||
# - job_name: llama-server-astro-orbiter-phi35
|
||
# scrape_interval: 90s
|
||
# metrics_path: /metrics
|
||
# params:
|
||
# model: ["Phi-3.5-mini-instruct-Q8_0"]
|
||
# static_configs:
|
||
# - targets:
|
||
# - 10.1.71.130:8002
|
||
# labels:
|
||
# hostname: astro-orbiter
|
||
# endpoint: astro-orbiter-router
|
||
# model: Phi-3.5-mini-instruct-Q8_0
|
||
|
||
# llama-swap (production, since 2026-08-18)
|
||
# Replaces the per-model /metrics?model=<id> jobs above (all targeting now-deprecated :8002).
|
||
# llama-swap exposes system-level metrics (llamaswap_*) — VRAM, GPU util, power, CPU, network.
|
||
# Per-model inference metrics (tokens/sec, latency, KV-cache) are NOT exposed at the proxy level;
|
||
# they remain on the individual llama-server child instances, scraped via node_exporter textfile
|
||
# collector for VRAM, and via the GPU exporter (:9835) for GPU-level telemetry.
|
||
- job_name: llama-swap
|
||
scrape_interval: 30s
|
||
scrape_timeout: 10s
|
||
static_configs:
|
||
- targets:
|
||
- 10.1.71.130:8001
|
||
labels:
|
||
hostname: astro-orbiter
|
||
service: llama-swap
|
||
environment: homelab
|
||
metrics_path: /metrics
|
||
honor_labels: true
|
||
metric_relabel_configs:
|
||
- source_labels: [__name__]
|
||
regex: 'llamaswap_.*'
|
||
action: keep
|
||
|
||
# ─── Grafana ──────────────────────────────────────────────────────────────────
|
||
grafana:
|
||
enabled: true
|
||
persistence:
|
||
enabled: false
|
||
|
||
admin:
|
||
existingSecret: grafana-admin-secret
|
||
userKey: admin-user
|
||
passwordKey: admin-password
|
||
|
||
grafana.ini:
|
||
server:
|
||
root_url: https://grafana.local.mk-labs.cloud
|
||
auth:
|
||
disable_login_form: false
|
||
|
||
defaultDashboardsEnabled: true
|
||
defaultDashboardsTimezone: browser
|
||
|
||
dashboardProviders:
|
||
dashboardproviders.yaml:
|
||
apiVersion: 1
|
||
providers:
|
||
- name: default
|
||
orgId: 1
|
||
folder: ""
|
||
type: file
|
||
disableDeletion: false
|
||
editable: true
|
||
options:
|
||
path: /var/lib/grafana/dashboards/default
|
||
|
||
dashboards:
|
||
default:
|
||
# ── Already present ──────────────────────────────────────────────────────
|
||
node-exporter-full:
|
||
gnetId: 1860
|
||
revision: 37
|
||
datasource: Prometheus
|
||
proxmox-cluster:
|
||
gnetId: 10347
|
||
revision: 5
|
||
datasource: Prometheus
|
||
unifi-poller:
|
||
gnetId: 11315
|
||
revision: 9
|
||
datasource: Prometheus
|
||
|
||
# ── Kubernetes ───────────────────────────────────────────────────────────
|
||
kubernetes-global:
|
||
gnetId: 15757
|
||
revision: 37
|
||
datasource: Prometheus
|
||
kubernetes-namespace:
|
||
gnetId: 15758
|
||
revision: 34
|
||
datasource: Prometheus
|
||
kubernetes-pods:
|
||
gnetId: 6336
|
||
revision: 1
|
||
datasource: Prometheus
|
||
|
||
# ── Platform services ─────────────────────────────────────────────────────
|
||
argocd:
|
||
gnetId: 19974
|
||
revision: 3
|
||
datasource: Prometheus
|
||
cert-manager:
|
||
gnetId: 20842
|
||
revision: 3
|
||
datasource: Prometheus
|
||
cilium-dashboard:
|
||
gnetId: 16611
|
||
revision: 1
|
||
datasource: Prometheus
|
||
|
||
# ── Infrastructure ────────────────────────────────────────────────────────
|
||
# Traefik — uncomment when P.02 is done (--metrics.prometheus=true)
|
||
# traefik:
|
||
# gnetId: 17346
|
||
# revision: 9
|
||
# datasource: Prometheus
|
||
|
||
sidecar:
|
||
dashboards:
|
||
enabled: true
|
||
searchNamespace: ALL
|
||
label: grafana_dashboard
|
||
labelValue: "1"
|
||
|
||
resources:
|
||
requests:
|
||
cpu: 100m
|
||
memory: 256Mi
|
||
limits:
|
||
cpu: 500m
|
||
memory: 512Mi
|
||
|
||
# ─── Node Exporter ────────────────────────────────────────────────────────────
|
||
nodeExporter:
|
||
enabled: true
|
||
|
||
prometheus-node-exporter:
|
||
tolerations:
|
||
- key: node-role.kubernetes.io/control-plane
|
||
operator: Exists
|
||
effect: NoSchedule
|
||
|
||
# ─── kube-state-metrics ───────────────────────────────────────────────────────
|
||
kubeStateMetrics:
|
||
enabled: true
|
||
|
||
# ─── Prometheus Operator ──────────────────────────────────────────────────────
|
||
prometheusOperator:
|
||
enabled: true
|
||
|
||
# ─── Component scraping ───────────────────────────────────────────────────────
|
||
kubeApiServer:
|
||
enabled: true
|
||
|
||
kubelet:
|
||
enabled: true
|
||
|
||
kubeControllerManager:
|
||
enabled: true
|
||
|
||
kubeScheduler:
|
||
enabled: true
|
||
|
||
kubeProxy:
|
||
enabled: false # Cilium replaces kube-proxy
|
||
|
||
kubeEtcd:
|
||
enabled: true
|
||
endpoints:
|
||
- 10.1.71.66
|
||
- 10.1.71.67
|
||
- 10.1.71.68
|
||
service:
|
||
enabled: true
|
||
port: 2381
|
||
targetPort: 2381
|