From 2ed70b4964679d8d82ee95c46f1fb2f5d679f8f4 Mon Sep 17 00:00:00 2001 From: Ryan Blundon Date: Mon, 18 May 2026 21:42:32 -0500 Subject: [PATCH] Deploy monitoring stack --- .../applications/monitoring/application.yaml | 39 ++++ .../monitoring/externalsecret.yaml | 27 +++ .../applications/monitoring/httproute.yaml | 36 ++++ cluster/applications/monitoring/values.yaml | 170 ++++++++++++++++++ cluster/argocd/externalsecret.yaml | 15 +- cluster/argocd/values.yaml | 12 +- cluster/bootstrap/README.md | 86 +++++++++ cluster/platform/gateway/referencegrant.yaml | 26 ++- 8 files changed, 386 insertions(+), 25 deletions(-) create mode 100644 cluster/applications/monitoring/application.yaml create mode 100644 cluster/applications/monitoring/externalsecret.yaml create mode 100644 cluster/applications/monitoring/httproute.yaml create mode 100644 cluster/applications/monitoring/values.yaml diff --git a/cluster/applications/monitoring/application.yaml b/cluster/applications/monitoring/application.yaml new file mode 100644 index 0000000..497ee09 --- /dev/null +++ b/cluster/applications/monitoring/application.yaml @@ -0,0 +1,39 @@ +# ------------------------------------------------------------------------------ +# Application: monitoring (kube-prometheus-stack) +# Wave 10 — first application workload, after all platform services +# Multi-source: Helm chart from upstream + values + HTTPRoute from repo +# ------------------------------------------------------------------------------ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: monitoring + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "10" +spec: + project: default + sources: + # Source 1: Helm chart from upstream + - repoURL: https://prometheus-community.github.io/helm-charts + chart: kube-prometheus-stack + targetRevision: 70.4.2 + helm: + valueFiles: + - $values/cluster/applications/monitoring/values.yaml + # Source 2: Repo — values ref + HTTPRoute + - repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git + targetRevision: main + path: cluster/applications/monitoring + ref: values + directory: + exclude: "application.yaml" + destination: + server: https://kubernetes.default.svc + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/cluster/applications/monitoring/externalsecret.yaml b/cluster/applications/monitoring/externalsecret.yaml new file mode 100644 index 0000000..7d97dc7 --- /dev/null +++ b/cluster/applications/monitoring/externalsecret.yaml @@ -0,0 +1,27 @@ +# ------------------------------------------------------------------------------ +# ExternalSecret — Grafana admin credentials +# 1Password item: "grafana-admin" in the "mk-labs" vault +# Fields: admin-user, admin-password +# ------------------------------------------------------------------------------ +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: grafana-admin-secret + namespace: monitoring +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-connect + refreshInterval: "1h" + target: + name: grafana-admin-secret + creationPolicy: Owner + data: + - secretKey: admin-user + remoteRef: + key: grafana-admin + property: admin-user + - secretKey: admin-password + remoteRef: + key: grafana-admin + property: admin-password diff --git a/cluster/applications/monitoring/httproute.yaml b/cluster/applications/monitoring/httproute.yaml new file mode 100644 index 0000000..6a64e36 --- /dev/null +++ b/cluster/applications/monitoring/httproute.yaml @@ -0,0 +1,36 @@ +# ------------------------------------------------------------------------------ +# HTTPRoute — Grafana via Cilium Gateway +# Cert issued per-service via cert-manager annotation on the Certificate +# ------------------------------------------------------------------------------ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: grafana-tls + namespace: monitoring +spec: + secretName: grafana-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - grafana.local.mk-labs.cloud +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: grafana + namespace: monitoring + annotations: + external-dns.alpha.kubernetes.io/hostname: grafana.local.mk-labs.cloud + external-dns.alpha.kubernetes.io/target: "10.1.71.90" +spec: + parentRefs: + - name: fastpass-gateway + namespace: gateway + sectionName: https + hostnames: + - grafana.local.mk-labs.cloud + rules: + - backendRefs: + - name: monitoring-grafana + port: 80 diff --git a/cluster/applications/monitoring/values.yaml b/cluster/applications/monitoring/values.yaml new file mode 100644 index 0000000..f115600 --- /dev/null +++ b/cluster/applications/monitoring/values.yaml @@ -0,0 +1,170 @@ +# ------------------------------------------------------------------------------ +# kube-prometheus-stack — Helm Values +# Chart: https://prometheus-community.github.io/helm-charts +# Version: 70.4.2 +# ------------------------------------------------------------------------------ + +# ─── Global ─────────────────────────────────────────────────────────────────── +fullnameOverride: "" + +# ─── Alertmanager ───────────────────────────────────────────────────────────── +# Deployed but unconfigured — wire into n8n later +alertmanager: + enabled: true + alertmanagerSpec: + storage: + volumeClaimTemplate: + spec: + storageClassName: nfs-emporium + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + +# ─── Prometheus ─────────────────────────────────────────────────────────────── +prometheus: + prometheusSpec: + # Retention + retention: 30d + retentionSize: "45GB" + + # Storage — NFS via csi-driver-nfs + storageSpec: + volumeClaimTemplate: + spec: + storageClassName: nfs-emporium + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + + # Scrape all ServiceMonitors and PodMonitors regardless of namespace + serviceMonitorSelectorNilUsesHelmValues: false + podMonitorSelectorNilUsesHelmValues: false + ruleSelectorNilUsesHelmValues: false + + # Resource limits — fastpass workers have reasonable headroom + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi + +# ─── Grafana ────────────────────────────────────────────────────────────────── +grafana: + enabled: true + + # Stateless — dashboards managed as ConfigMaps, no PVC needed + persistence: + enabled: false + + # Admin credentials via ESO from 1Password + admin: + existingSecret: grafana-admin-secret + userKey: admin-user + passwordKey: admin-password + + # Grafana URL — used for redirects and OAuth callbacks + grafana.ini: + server: + root_url: https://grafana.local.mk-labs.cloud + auth: + disable_login_form: false + + # Default dashboards + defaultDashboardsEnabled: true + defaultDashboardsTimezone: browser + + # Node Exporter Full — dashboard 1860 + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + + dashboards: + default: + node-exporter-full: + gnetId: 1860 + revision: 37 + datasource: Prometheus + kubernetes-cluster: + gnetId: 7249 + revision: 1 + datasource: Prometheus + kubernetes-pods: + gnetId: 6336 + revision: 1 + datasource: Prometheus + + # Sidecar for ConfigMap-based dashboards + sidecar: + dashboards: + enabled: true + searchNamespace: ALL + label: grafana_dashboard + labelValue: "1" + + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +# ─── Node Exporter ──────────────────────────────────────────────────────────── +nodeExporter: + enabled: true + +prometheus-node-exporter: + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + +# ─── kube-state-metrics ─────────────────────────────────────────────────────── +kubeStateMetrics: + enabled: true + +# ─── Prometheus Operator ────────────────────────────────────────────────────── +prometheusOperator: + enabled: true + +# ─── Component scraping ─────────────────────────────────────────────────────── +kubeApiServer: + enabled: true + +kubelet: + enabled: true + +kubeControllerManager: + enabled: true + +kubeScheduler: + enabled: true + +kubeProxy: + enabled: false # Cilium replaces kube-proxy + +kubeEtcd: + enabled: true + endpoints: + - 10.1.71.66 + - 10.1.71.67 + - 10.1.71.68 + service: + enabled: true + port: 2381 + targetPort: 2381 diff --git a/cluster/argocd/externalsecret.yaml b/cluster/argocd/externalsecret.yaml index 6f522ec..1ff02be 100644 --- a/cluster/argocd/externalsecret.yaml +++ b/cluster/argocd/externalsecret.yaml @@ -1,10 +1,9 @@ # ------------------------------------------------------------------------------ # ExternalSecret — Authentik OIDC credentials for ArgoCD -# Pulled from 1Password via Connect Server, materialized as argocd-oidc-secret -# in the argocd namespace. # -# 1Password item: "argocd-oidc" in the "mk-labs" vault -# Fields: client-id, client-secret +# Targets argocd-secret directly — ArgoCD reads oidc.authentik.* keys +# from its own secret. ESO merges these fields into the existing secret +# without overwriting admin.password etc. (creationPolicy: Merge) # ------------------------------------------------------------------------------ apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -17,14 +16,14 @@ spec: name: onepassword-connect refreshInterval: "1h" target: - name: argocd-oidc-secret - creationPolicy: Owner + name: argocd-secret + creationPolicy: Merge data: - - secretKey: client-id + - secretKey: oidc.authentik.clientID remoteRef: key: argocd-oidc property: client-id - - secretKey: client-secret + - secretKey: oidc.authentik.clientSecret remoteRef: key: argocd-oidc property: client-secret diff --git a/cluster/argocd/values.yaml b/cluster/argocd/values.yaml index fc4c9ec..c518e9e 100644 --- a/cluster/argocd/values.yaml +++ b/cluster/argocd/values.yaml @@ -83,14 +83,6 @@ configs: g, argocd-admins, role:admin g, argocd-viewers, role:readonly - # Secret reference for OIDC credentials - # ArgoCD reads $oidc.authentik.clientID and $oidc.authentik.clientSecret - # from the argocd-secret Kubernetes secret. We patch it via the ExternalSecret. - secret: - extra: - oidc.authentik.clientID: $argocd-oidc-secret:client-id - oidc.authentik.clientSecret: $argocd-oidc-secret:client-secret - # Repository credentials — Gitea SSH repositories: homelab-repo: @@ -116,3 +108,7 @@ applicationSet: # Disabled — using Authentik as the native OIDC provider dex: enabled: false + +# Disable default admin user +admin: + enabled: false diff --git a/cluster/bootstrap/README.md b/cluster/bootstrap/README.md index 4a5c911..0b9f001 100644 --- a/cluster/bootstrap/README.md +++ b/cluster/bootstrap/README.md @@ -273,3 +273,89 @@ Once ingress-nginx and cert-manager are synced: - Authentik OIDC: uncomment `oidc.config` in `cluster/argocd/values.yaml` once the ArgoCD provider is configured in Authentik - Migrate `gitea-repo` secret to ESO once 1Password Connect is healthy + +--- + +## Operational Runbook — Adding a New Application + +When deploying a new application that needs HTTPS access via the Cilium Gateway, +follow this checklist: + +### 1. Add the namespace to the ReferenceGrant + +Edit `cluster/platform/gateway/referencegrant.yaml` and add a new `from` entry: + +```yaml + - group: gateway.networking.k8s.io + kind: HTTPRoute + namespace: +``` + +This is **required** by the Gateway API spec — without it, the HTTPRoute will +fail to attach to the Gateway and traffic won't route. The ReferenceGrant must +live in the `gateway` namespace, so this file is the one place to update. + +### 2. Create the application directory + +``` +cluster/applications// +├── application.yaml ← ArgoCD Application, wave 10+ +├── values.yaml ← Helm values (if Helm-based) +├── externalsecret.yaml ← ESO resource if secrets needed +└── httproute.yaml ← Certificate + HTTPRoute for Gateway access +``` + +### 3. HTTPRoute pattern + +Every HTTPRoute needs two annotations for DNS and routing to work: + +```yaml +annotations: + external-dns.alpha.kubernetes.io/hostname: .local.mk-labs.cloud + external-dns.alpha.kubernetes.io/target: "10.1.71.90" +``` + +And a parentRef pointing at the platform Gateway: + +```yaml +spec: + parentRefs: + - name: fastpass-gateway + namespace: gateway + sectionName: https +``` + +### 4. Per-service certificate + +Each service gets its own cert via cert-manager. Add to `httproute.yaml`: + +```yaml +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: -tls + namespace: +spec: + secretName: -tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - .local.mk-labs.cloud +``` + +### 5. Push and verify + +```bash +# Watch ArgoCD sync +kubectl -n argocd get applications --watch + +# Verify cert issued +kubectl get certificate -n -w + +# Verify DNS record created +nslookup .local.mk-labs.cloud 10.1.71.32 + +# Test +curl https://.local.mk-labs.cloud +``` diff --git a/cluster/platform/gateway/referencegrant.yaml b/cluster/platform/gateway/referencegrant.yaml index 1ca62f2..1393578 100644 --- a/cluster/platform/gateway/referencegrant.yaml +++ b/cluster/platform/gateway/referencegrant.yaml @@ -1,13 +1,21 @@ # ------------------------------------------------------------------------------ -# ReferenceGrant — allow HTTPRoutes from any namespace to attach to the Gateway +# ReferenceGrant — allow HTTPRoutes from app namespaces to attach to the Gateway # -# Required by Gateway API spec when parentRef crosses namespace boundaries. -# `namespace` is required in the from stanza — use a specific namespace or -# create one ReferenceGrant per namespace that needs access. +# IMPORTANT: Every new application namespace that needs to route traffic through +# the Cilium Gateway MUST have an entry added here. This is a Gateway API +# security requirement — the grant must live in the gateway namespace. # -# For a homelab cluster where all namespaces are trusted, we create a single -# catch-all by listing the namespaces we expect to use. Add entries as new -# app namespaces are created. +# When adding a new application: +# 1. Add a new `from` entry below with the app namespace +# 2. Push to main +# 3. ArgoCD will sync the updated ReferenceGrant automatically +# +# Namespaces currently permitted: +# - default (test workloads) +# - argocd (ArgoCD UI) +# - monitoring (Grafana) +# - authentik (Authentik SSO — future) +# - gitea (Gitea — future) # ------------------------------------------------------------------------------ apiVersion: gateway.networking.k8s.io/v1beta1 kind: ReferenceGrant @@ -21,10 +29,10 @@ spec: namespace: default - group: gateway.networking.k8s.io kind: HTTPRoute - namespace: monitoring + namespace: argocd - group: gateway.networking.k8s.io kind: HTTPRoute - namespace: argocd + namespace: monitoring - group: gateway.networking.k8s.io kind: HTTPRoute namespace: authentik