rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

additional monitoring scrapes

Browse Source
This commit is contained in:
Ryan Blundon
2026-05-18 22:28:14 -05:00
parent b45dcea4c3
commit a7f77514d6
23 changed files with 837 additions and 26235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
cluster/applications/monitoring/externalsecret.yaml
View File

@@ -1,27 +0,0 @@
# ------------------------------------------------------------------------------
# ExternalSecret — Grafana admin credentials
# 1Password item: "grafana-admin" in the "mk-labs" vault
# Fields: admin-user, admin-password
# ------------------------------------------------------------------------------
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: grafana-admin-secret
namespace: monitoring
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-connect
refreshInterval: "1h"
target:
name: grafana-admin-secret
creationPolicy: Owner
data:
- secretKey: admin-user
remoteRef:
key: grafana-admin
property: admin-user
- secretKey: admin-password
remoteRef:
key: grafana-admin
property: admin-password

67
cluster/applications/monitoring/externalsecrets.yaml Normal file
View File

@@ -0,0 +1,67 @@
# ------------------------------------------------------------------------------
# ExternalSecrets — monitoring namespace secrets from 1Password
# ------------------------------------------------------------------------------
# Grafana admin credentials
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: grafana-admin-secret
namespace: monitoring
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-connect
refreshInterval: "1h"
target:
name: grafana-admin-secret
creationPolicy: Owner
data:
- secretKey: admin-user
remoteRef:
key: grafana-admin
property: admin-user
- secretKey: admin-password
remoteRef:
key: grafana-admin
property: admin-password
---
# Proxmox VE Exporter API token
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: pve-exporter-secret
namespace: monitoring
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-connect
refreshInterval: "1h"
target:
name: pve-exporter-secret
creationPolicy: Owner
data:
- secretKey: token-value
remoteRef:
key: proxmox-prometheus
property: token-value
---
# SNMP community string
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
name: snmp-community-secret
namespace: monitoring
spec:
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-connect
refreshInterval: "1h"
target:
name: snmp-community-secret
creationPolicy: Owner
data:
- secretKey: community
remoteRef:
key: snmp-community
property: community

117
cluster/applications/monitoring/pve-exporter.yaml Normal file
View File

@@ -0,0 +1,117 @@
# ------------------------------------------------------------------------------
# Proxmox VE Exporter
# Scrapes the Proxmox cluster API for VM states, storage, CPU metrics.
# Uses multi-target pattern — Prometheus passes the target node as a param.
#
# Credentials: prometheus@pve!prometheus-token
# Config file mounted from Secret (populated by ESO from 1Password)
# ------------------------------------------------------------------------------
# ConfigMap for pve.yml — credentials file for the exporter
# Token value injected at runtime from the Secret via init container
apiVersion: v1
kind: ConfigMap
metadata:
name: pve-exporter-config
namespace: monitoring
data:
pve.yml: |
default:
user: prometheus@pve
token_name: prometheus-token
verify_ssl: false
---
# Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
name: pve-exporter
namespace: monitoring
labels:
app: pve-exporter
spec:
replicas: 1
selector:
matchLabels:
app: pve-exporter
template:
metadata:
labels:
app: pve-exporter
spec:
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
seccompProfile:
type: RuntimeDefault
initContainers:
# Merge the token value from the secret into the config file
- name: config-init
image: busybox:1.36
command:
- sh
- -c
- |
cp /config-ro/pve.yml /config/pve.yml
echo " token_value: $(cat /secrets/token-value)" >> /config/pve.yml
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
volumeMounts:
- name: config-ro
mountPath: /config-ro
- name: config-rw
mountPath: /config
- name: secret
mountPath: /secrets
containers:
- name: pve-exporter
image: prompve/prometheus-pve-exporter:3.5.1
args:
- --config.file=/config/pve.yml
ports:
- containerPort: 9221
name: metrics
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
volumeMounts:
- name: config-rw
mountPath: /config
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
volumes:
- name: config-ro
configMap:
name: pve-exporter-config
- name: config-rw
emptyDir: {}
- name: secret
secret:
secretName: pve-exporter-secret
---
# Service
apiVersion: v1
kind: Service
metadata:
name: pve-exporter
namespace: monitoring
labels:
app: pve-exporter
spec:
selector:
app: pve-exporter
ports:
- name: metrics
port: 9221
targetPort: 9221

152
cluster/applications/monitoring/snmp-exporter.yaml Normal file
View File

@@ -0,0 +1,152 @@
# ------------------------------------------------------------------------------
# SNMP Exporter
# Scrapes UniFi network devices via SNMP v2c.
# Uses multi-target pattern — Prometheus passes device IP as a param.
#
# Devices:
# - world-drive (UDM Pro) 192.168.1.1
# - usw-pro-agg (USW-Pro-Agg) 192.168.1.123
# - usw-pro-max-24 (USW-Pro-Max-24) 192.168.1.226
# - usw-lite-16-poe (USW-Lite-16) 192.168.1.136
# - ups-tower (UPS-Tower) 192.168.1.233
# ------------------------------------------------------------------------------
# SNMP exporter config — if_mib module with community from secret
# Uses an init container to inject the community string at runtime
apiVersion: v1
kind: ConfigMap
metadata:
name: snmp-exporter-config
namespace: monitoring
data:
snmp.yml: |
auths:
mk_labs_v2c:
community: COMMUNITY_PLACEHOLDER
version: 2
modules:
if_mib:
walk:
- interfaces
auth: mk_labs_v2c
metrics:
- name: ifIndex
oid: 1.3.6.1.2.1.2.2.1.1
type: gauge
help: A unique value for each interface
- name: ifDescr
oid: 1.3.6.1.2.1.2.2.1.2
type: DisplayString
help: Interface description
- name: ifOperStatus
oid: 1.3.6.1.2.1.2.2.1.8
type: gauge
help: Interface operational status
- name: ifInOctets
oid: 1.3.6.1.2.1.2.2.1.10
type: counter
help: Bytes received
- name: ifOutOctets
oid: 1.3.6.1.2.1.2.2.1.16
type: counter
help: Bytes sent
- name: ifInErrors
oid: 1.3.6.1.2.1.2.2.1.14
type: counter
help: Inbound errors
- name: ifOutErrors
oid: 1.3.6.1.2.1.2.2.1.20
type: counter
help: Outbound errors
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: snmp-exporter
namespace: monitoring
labels:
app: snmp-exporter
spec:
replicas: 1
selector:
matchLabels:
app: snmp-exporter
template:
metadata:
labels:
app: snmp-exporter
spec:
securityContext:
runAsNonRoot: true
runAsUser: 1000
runAsGroup: 1000
seccompProfile:
type: RuntimeDefault
initContainers:
- name: config-init
image: busybox:1.36
command:
- sh
- -c
- |
COMMUNITY=$(cat /secrets/community)
sed "s/COMMUNITY_PLACEHOLDER/$COMMUNITY/" /config-ro/snmp.yml > /config/snmp.yml
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
volumeMounts:
- name: config-ro
mountPath: /config-ro
- name: config-rw
mountPath: /config
- name: secret
mountPath: /secrets
containers:
- name: snmp-exporter
image: prom/snmp-exporter:v0.26.0
args:
- --config.file=/config/snmp.yml
ports:
- containerPort: 9116
name: metrics
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
volumeMounts:
- name: config-rw
mountPath: /config
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 200m
memory: 128Mi
volumes:
- name: config-ro
configMap:
name: snmp-exporter-config
- name: config-rw
emptyDir: {}
- name: secret
secret:
secretName: snmp-community-secret
---
apiVersion: v1
kind: Service
metadata:
name: snmp-exporter
namespace: monitoring
labels:
app: snmp-exporter
spec:
selector:
app: snmp-exporter
ports:
- name: metrics
port: 9116
targetPort: 9116

211
cluster/applications/monitoring/values.yaml
View File

@@ -4,11 +4,7 @@
# Version: 70.4.2
# ------------------------------------------------------------------------------
# ─── Global ───────────────────────────────────────────────────────────────────
fullnameOverride: ""
# ─── Alertmanager ─────────────────────────────────────────────────────────────
# Deployed but unconfigured — wire into n8n later
alertmanager:
enabled: true
alertmanagerSpec:
@@ -25,11 +21,9 @@ alertmanager:
# ─── Prometheus ───────────────────────────────────────────────────────────────
prometheus:
prometheusSpec:
# Retention
retention: 30d
retentionSize: "45GB"
# Storage — NFS via csi-driver-nfs
storageSpec:
volumeClaimTemplate:
spec:
@@ -40,12 +34,11 @@ prometheus:
requests:
storage: 50Gi
# Scrape all ServiceMonitors and PodMonitors regardless of namespace
# Scrape all ServiceMonitors/PodMonitors regardless of namespace
serviceMonitorSelectorNilUsesHelmValues: false
podMonitorSelectorNilUsesHelmValues: false
ruleSelectorNilUsesHelmValues: false
# Resource limits — fastpass workers have reasonable headroom
resources:
requests:
cpu: 200m
@@ -54,32 +47,213 @@ prometheus:
cpu: 1000m
memory: 2Gi
# ─── Additional scrape configs — external targets ──────────────────────────
additionalScrapeConfigs:
# Proxmox nodes — Node Exporter on each hypervisor
- job_name: proxmox-nodes
scrape_interval: 30s
static_configs:
- targets:
- 10.1.71.11:9100
- 10.1.71.12:9100
- 10.1.71.13:9100
labels:
job: proxmox
# cinderella-castle — existing VM-side Prometheus host
# Remove this once cinderella-castle is decommissioned
- job_name: node-cinderella-castle
scrape_interval: 30s
static_configs:
- targets:
- 10.1.71.31:9100
labels:
hostname: cinderella-castle
# Traefik metrics
- job_name: traefik
scrape_interval: 30s
static_configs:
- targets:
- 10.1.71.35:8080
labels:
hostname: lightning-lane
# Proxmox VE Exporter — cluster-level metrics (VM states, storage, CPU)
# Multi-target pattern: Prometheus passes the cluster address as a param
- job_name: proxmox-cluster
scrape_interval: 30s
metrics_path: /pve
params:
module: [default]
cluster: ["1"]
node: [""]
static_configs:
- targets:
- 10.1.71.11 # main-street-usa — cluster API endpoint
labels:
cluster: magic-kingdom
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: pve-exporter.monitoring.svc.cluster.local:9221
# SNMP — UniFi devices (multi-target pattern)
# world-drive (UDM Pro)
- job_name: snmp-udm-pro
scrape_interval: 60s
scrape_timeout: 55s
metrics_path: /snmp
params:
auth: [mk_labs_v2c]
module: [if_mib]
static_configs:
- targets:
- 192.168.1.1
labels:
device: world-drive
model: UDM-Pro
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: snmp-exporter.monitoring.svc.cluster.local:9116
# usw-pro-aggregation
- job_name: snmp-usw-pro-aggregation
scrape_interval: 60s
scrape_timeout: 55s
metrics_path: /snmp
params:
auth: [mk_labs_v2c]
module: [if_mib]
static_configs:
- targets:
- 192.168.1.123
labels:
device: usw-pro-aggregation
model: USW-Pro-Aggregation
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: snmp-exporter.monitoring.svc.cluster.local:9116
# usw-pro-max-24
- job_name: snmp-usw-pro-max-24
scrape_interval: 60s
scrape_timeout: 55s
metrics_path: /snmp
params:
auth: [mk_labs_v2c]
module: [if_mib]
static_configs:
- targets:
- 192.168.1.226
labels:
device: usw-pro-max-24
model: USW-Pro-Max-24
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: snmp-exporter.monitoring.svc.cluster.local:9116
# usw-lite-16-poe
- job_name: snmp-usw-lite-16-poe
scrape_interval: 60s
scrape_timeout: 55s
metrics_path: /snmp
params:
auth: [mk_labs_v2c]
module: [if_mib]
static_configs:
- targets:
- 192.168.1.136
labels:
device: usw-lite-16-poe
model: USW-Lite-16-PoE
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: snmp-exporter.monitoring.svc.cluster.local:9116
# ups-tower
- job_name: snmp-ups-tower
scrape_interval: 60s
scrape_timeout: 55s
metrics_path: /snmp
params:
auth: [mk_labs_v2c]
module: [if_mib]
static_configs:
- targets:
- 192.168.1.233
labels:
device: ups-tower
model: UPS-Tower
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: snmp-exporter.monitoring.svc.cluster.local:9116
# Pure FlashArray — DEFERRED until Purity 6.7+
# Uncomment when utilidor is upgraded. Native OpenMetrics endpoint,
# no exporter container needed. Add API token to 1Password as
# item "pure-flasharray-utilidor" field "api-token".
# - job_name: pure-flasharray-utilidor
# scrape_interval: 30s
# metrics_path: /metrics/array
# scheme: https
# tls_config:
# insecure_skip_verify: true
# authorization:
# credentials_file: /etc/prometheus/secrets/pure-api-token/api-token
# params:
# namespace: ["purefa"]
# static_configs:
# - targets:
# - utilidor.local.mk-labs.cloud
# labels:
# array: utilidor
# array_type: physical
# ─── Grafana ──────────────────────────────────────────────────────────────────
grafana:
enabled: true
# Stateless — dashboards managed as ConfigMaps, no PVC needed
persistence:
enabled: false
# Admin credentials via ESO from 1Password
admin:
existingSecret: grafana-admin-secret
userKey: admin-user
passwordKey: admin-password
# Grafana URL — used for redirects and OAuth callbacks
grafana.ini:
server:
root_url: https://grafana.local.mk-labs.cloud
auth:
disable_login_form: false
# Default dashboards
defaultDashboardsEnabled: true
defaultDashboardsTimezone: browser
# Node Exporter Full — dashboard 1860
dashboardProviders:
dashboardproviders.yaml:
apiVersion: 1
@@ -107,8 +281,15 @@ grafana:
gnetId: 6336
revision: 1
datasource: Prometheus
unifi-poller:
gnetId: 11315
revision: 9
datasource: Prometheus
proxmox-cluster:
gnetId: 10347
revision: 5
datasource: Prometheus
# Sidecar for ConfigMap-based dashboards
sidecar:
dashboards:
enabled: true