Deploy monitoring stack

2026-05-18 21:42:32 -05:00
parent dbf0b18f07
commit 2ed70b4964
8 changed files with 386 additions and 25 deletions
--- a/cluster/applications/monitoring/application.yaml
+++ b/cluster/applications/monitoring/application.yaml
@@ -0,0 +1,39 @@
+# ------------------------------------------------------------------------------
+# Application: monitoring (kube-prometheus-stack)
+# Wave 10 — first application workload, after all platform services
+# Multi-source: Helm chart from upstream + values + HTTPRoute from repo
+# ------------------------------------------------------------------------------
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: monitoring
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "10"
+spec:
+  project: default
+  sources:
+    # Source 1: Helm chart from upstream
+    - repoURL: https://prometheus-community.github.io/helm-charts
+      chart: kube-prometheus-stack
+      targetRevision: 70.4.2
+      helm:
+        valueFiles:
+          - $values/cluster/applications/monitoring/values.yaml
+    # Source 2: Repo — values ref + HTTPRoute
+    - repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git
+      targetRevision: main
+      path: cluster/applications/monitoring
+      ref: values
+      directory:
+        exclude: "application.yaml"
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: monitoring
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
--- a/cluster/applications/monitoring/externalsecret.yaml
+++ b/cluster/applications/monitoring/externalsecret.yaml
@@ -0,0 +1,27 @@
+# ------------------------------------------------------------------------------
+# ExternalSecret — Grafana admin credentials
+# 1Password item: "grafana-admin" in the "mk-labs" vault
+# Fields: admin-user, admin-password
+# ------------------------------------------------------------------------------
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: grafana-admin-secret
+  namespace: monitoring
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  refreshInterval: "1h"
+  target:
+    name: grafana-admin-secret
+    creationPolicy: Owner
+  data:
+    - secretKey: admin-user
+      remoteRef:
+        key: grafana-admin
+        property: admin-user
+    - secretKey: admin-password
+      remoteRef:
+        key: grafana-admin
+        property: admin-password
--- a/cluster/applications/monitoring/httproute.yaml
+++ b/cluster/applications/monitoring/httproute.yaml
@@ -0,0 +1,36 @@
+# ------------------------------------------------------------------------------
+# HTTPRoute — Grafana via Cilium Gateway
+# Cert issued per-service via cert-manager annotation on the Certificate
+# ------------------------------------------------------------------------------
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: grafana-tls
+  namespace: monitoring
+spec:
+  secretName: grafana-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  dnsNames:
+    - grafana.local.mk-labs.cloud
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: grafana
+  namespace: monitoring
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: grafana.local.mk-labs.cloud
+    external-dns.alpha.kubernetes.io/target: "10.1.71.90"
+spec:
+  parentRefs:
+    - name: fastpass-gateway
+      namespace: gateway
+      sectionName: https
+  hostnames:
+    - grafana.local.mk-labs.cloud
+  rules:
+    - backendRefs:
+        - name: monitoring-grafana
+          port: 80
--- a/cluster/applications/monitoring/values.yaml
+++ b/cluster/applications/monitoring/values.yaml
@@ -0,0 +1,170 @@
+# ------------------------------------------------------------------------------
+# kube-prometheus-stack — Helm Values
+# Chart: https://prometheus-community.github.io/helm-charts
+# Version: 70.4.2
+# ------------------------------------------------------------------------------
+
+# ─── Global ───────────────────────────────────────────────────────────────────
+fullnameOverride: ""
+
+# ─── Alertmanager ─────────────────────────────────────────────────────────────
+# Deployed but unconfigured — wire into n8n later
+alertmanager:
+  enabled: true
+  alertmanagerSpec:
+    storage:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: nfs-emporium
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 5Gi
+
+# ─── Prometheus ───────────────────────────────────────────────────────────────
+prometheus:
+  prometheusSpec:
+    # Retention
+    retention: 30d
+    retentionSize: "45GB"
+
+    # Storage — NFS via csi-driver-nfs
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: nfs-emporium
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: 50Gi
+
+    # Scrape all ServiceMonitors and PodMonitors regardless of namespace
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    ruleSelectorNilUsesHelmValues: false
+
+    # Resource limits — fastpass workers have reasonable headroom
+    resources:
+      requests:
+        cpu: 200m
+        memory: 512Mi
+      limits:
+        cpu: 1000m
+        memory: 2Gi
+
+# ─── Grafana ──────────────────────────────────────────────────────────────────
+grafana:
+  enabled: true
+
+  # Stateless — dashboards managed as ConfigMaps, no PVC needed
+  persistence:
+    enabled: false
+
+  # Admin credentials via ESO from 1Password
+  admin:
+    existingSecret: grafana-admin-secret
+    userKey: admin-user
+    passwordKey: admin-password
+
+  # Grafana URL — used for redirects and OAuth callbacks
+  grafana.ini:
+    server:
+      root_url: https://grafana.local.mk-labs.cloud
+    auth:
+      disable_login_form: false
+
+  # Default dashboards
+  defaultDashboardsEnabled: true
+  defaultDashboardsTimezone: browser
+
+  # Node Exporter Full — dashboard 1860
+  dashboardProviders:
+    dashboardproviders.yaml:
+      apiVersion: 1
+      providers:
+        - name: default
+          orgId: 1
+          folder: ""
+          type: file
+          disableDeletion: false
+          editable: true
+          options:
+            path: /var/lib/grafana/dashboards/default
+
+  dashboards:
+    default:
+      node-exporter-full:
+        gnetId: 1860
+        revision: 37
+        datasource: Prometheus
+      kubernetes-cluster:
+        gnetId: 7249
+        revision: 1
+        datasource: Prometheus
+      kubernetes-pods:
+        gnetId: 6336
+        revision: 1
+        datasource: Prometheus
+
+  # Sidecar for ConfigMap-based dashboards
+  sidecar:
+    dashboards:
+      enabled: true
+      searchNamespace: ALL
+      label: grafana_dashboard
+      labelValue: "1"
+
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
+# ─── Node Exporter ────────────────────────────────────────────────────────────
+nodeExporter:
+  enabled: true
+
+prometheus-node-exporter:
+  tolerations:
+    - key: node-role.kubernetes.io/control-plane
+      operator: Exists
+      effect: NoSchedule
+
+# ─── kube-state-metrics ───────────────────────────────────────────────────────
+kubeStateMetrics:
+  enabled: true
+
+# ─── Prometheus Operator ──────────────────────────────────────────────────────
+prometheusOperator:
+  enabled: true
+
+# ─── Component scraping ───────────────────────────────────────────────────────
+kubeApiServer:
+  enabled: true
+
+kubelet:
+  enabled: true
+
+kubeControllerManager:
+  enabled: true
+
+kubeScheduler:
+  enabled: true
+
+kubeProxy:
+  enabled: false  # Cilium replaces kube-proxy
+
+kubeEtcd:
+  enabled: true
+  endpoints:
+    - 10.1.71.66
+    - 10.1.71.67
+    - 10.1.71.68
+  service:
+    enabled: true
+    port: 2381
+    targetPort: 2381
--- a/cluster/argocd/externalsecret.yaml
+++ b/cluster/argocd/externalsecret.yaml
@@ -1,10 +1,9 @@
 # ------------------------------------------------------------------------------
 # ExternalSecret — Authentik OIDC credentials for ArgoCD
-# Pulled from 1Password via Connect Server, materialized as argocd-oidc-secret
-# in the argocd namespace.
 #
-# 1Password item: "argocd-oidc" in the "mk-labs" vault
-# Fields: client-id, client-secret
+# Targets argocd-secret directly — ArgoCD reads oidc.authentik.* keys
+# from its own secret. ESO merges these fields into the existing secret
+# without overwriting admin.password etc. (creationPolicy: Merge)
 # ------------------------------------------------------------------------------
 apiVersion: external-secrets.io/v1beta1
 kind: ExternalSecret
@@ -17,14 +16,14 @@ spec:
    name: onepassword-connect
  refreshInterval: "1h"
  target:
-    name: argocd-oidc-secret
-    creationPolicy: Owner
+    name: argocd-secret
+    creationPolicy: Merge
  data:
-    - secretKey: client-id
+    - secretKey: oidc.authentik.clientID
      remoteRef:
        key: argocd-oidc
        property: client-id
-    - secretKey: client-secret
+    - secretKey: oidc.authentik.clientSecret
      remoteRef:
        key: argocd-oidc
        property: client-secret
--- a/cluster/argocd/values.yaml
+++ b/cluster/argocd/values.yaml
@@ -83,14 +83,6 @@ configs:
      g, argocd-admins, role:admin
      g, argocd-viewers, role:readonly

-  # Secret reference for OIDC credentials
-  # ArgoCD reads $oidc.authentik.clientID and $oidc.authentik.clientSecret
-  # from the argocd-secret Kubernetes secret. We patch it via the ExternalSecret.
-  secret:
-    extra:
-      oidc.authentik.clientID: $argocd-oidc-secret:client-id
-      oidc.authentik.clientSecret: $argocd-oidc-secret:client-secret
-
  # Repository credentials — Gitea SSH
  repositories:
    homelab-repo:
@@ -116,3 +108,7 @@ applicationSet:
 # Disabled — using Authentik as the native OIDC provider
 dex:
  enabled: false
+
+# Disable default admin user
+admin:
+  enabled: false
--- a/cluster/bootstrap/README.md
+++ b/cluster/bootstrap/README.md
@@ -273,3 +273,89 @@ Once ingress-nginx and cert-manager are synced:
 - Authentik OIDC: uncomment `oidc.config` in `cluster/argocd/values.yaml` once
  the ArgoCD provider is configured in Authentik
 - Migrate `gitea-repo` secret to ESO once 1Password Connect is healthy
+
+---
+
+## Operational Runbook — Adding a New Application
+
+When deploying a new application that needs HTTPS access via the Cilium Gateway,
+follow this checklist:
+
+### 1. Add the namespace to the ReferenceGrant
+
+Edit `cluster/platform/gateway/referencegrant.yaml` and add a new `from` entry:
+
+```yaml
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      namespace: <your-app-namespace>
+```
+
+This is **required** by the Gateway API spec — without it, the HTTPRoute will
+fail to attach to the Gateway and traffic won't route. The ReferenceGrant must
+live in the `gateway` namespace, so this file is the one place to update.
+
+### 2. Create the application directory
+
+```
+cluster/applications/<app-name>/
+├── application.yaml    ← ArgoCD Application, wave 10+
+├── values.yaml         ← Helm values (if Helm-based)
+├── externalsecret.yaml ← ESO resource if secrets needed
+└── httproute.yaml      ← Certificate + HTTPRoute for Gateway access
+```
+
+### 3. HTTPRoute pattern
+
+Every HTTPRoute needs two annotations for DNS and routing to work:
+
+```yaml
+annotations:
+  external-dns.alpha.kubernetes.io/hostname: <service>.local.mk-labs.cloud
+  external-dns.alpha.kubernetes.io/target: "10.1.71.90"
+```
+
+And a parentRef pointing at the platform Gateway:
+
+```yaml
+spec:
+  parentRefs:
+    - name: fastpass-gateway
+      namespace: gateway
+      sectionName: https
+```
+
+### 4. Per-service certificate
+
+Each service gets its own cert via cert-manager. Add to `httproute.yaml`:
+
+```yaml
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: <service>-tls
+  namespace: <app-namespace>
+spec:
+  secretName: <service>-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  dnsNames:
+    - <service>.local.mk-labs.cloud
+```
+
+### 5. Push and verify
+
+```bash
+# Watch ArgoCD sync
+kubectl -n argocd get applications --watch
+
+# Verify cert issued
+kubectl get certificate -n <app-namespace> -w
+
+# Verify DNS record created
+nslookup <service>.local.mk-labs.cloud 10.1.71.32
+
+# Test
+curl https://<service>.local.mk-labs.cloud
+```
--- a/cluster/platform/gateway/referencegrant.yaml
+++ b/cluster/platform/gateway/referencegrant.yaml
@@ -1,13 +1,21 @@
 # ------------------------------------------------------------------------------
-# ReferenceGrant — allow HTTPRoutes from any namespace to attach to the Gateway
+# ReferenceGrant — allow HTTPRoutes from app namespaces to attach to the Gateway
 #
-# Required by Gateway API spec when parentRef crosses namespace boundaries.
-# `namespace` is required in the from stanza — use a specific namespace or
-# create one ReferenceGrant per namespace that needs access.
+# IMPORTANT: Every new application namespace that needs to route traffic through
+# the Cilium Gateway MUST have an entry added here. This is a Gateway API
+# security requirement — the grant must live in the gateway namespace.
 #
-# For a homelab cluster where all namespaces are trusted, we create a single
-# catch-all by listing the namespaces we expect to use. Add entries as new
-# app namespaces are created.
+# When adding a new application:
+#   1. Add a new `from` entry below with the app namespace
+#   2. Push to main
+#   3. ArgoCD will sync the updated ReferenceGrant automatically
+#
+# Namespaces currently permitted:
+#   - default       (test workloads)
+#   - argocd        (ArgoCD UI)
+#   - monitoring    (Grafana)
+#   - authentik     (Authentik SSO — future)
+#   - gitea         (Gitea — future)
 # ------------------------------------------------------------------------------
 apiVersion: gateway.networking.k8s.io/v1beta1
 kind: ReferenceGrant
@@ -21,10 +29,10 @@ spec:
      namespace: default
    - group: gateway.networking.k8s.io
      kind: HTTPRoute
-      namespace: monitoring
+      namespace: argocd
    - group: gateway.networking.k8s.io
      kind: HTTPRoute
-      namespace: argocd
+      namespace: monitoring
    - group: gateway.networking.k8s.io
      kind: HTTPRoute
      namespace: authentik