From cdc78955b151632701964fa545bd14353e1a9c78 Mon Sep 17 00:00:00 2001 From: Ryan Blundon Date: Mon, 18 May 2026 13:20:42 -0500 Subject: [PATCH] cilium, argo --- .../argocd => argocd}/application.yaml | 0 .../platform/argocd => argocd}/argocd-cm.yaml | 0 .../argocd => argocd}/argocd-params.yaml | 0 .../argocd => argocd}/argocd-rbac-cm.yaml | 0 .../argocd => argocd}/certificate.yaml | 0 .../argocd => argocd}/externalsecret.yaml | 0 .../platform/argocd => argocd}/ingress.yaml | 0 .../argocd => argocd}/repository-secret.yaml | 0 .../application.yaml | 0 .../manifests/lb-pool.yaml | 0 .../values.yaml | 0 cluster/argocd/BOOTSTRAP.md | 242 ++++++---- cluster/argocd/values.yaml | 110 +++++ docs/city-hall-setup.md | 104 +++-- docs/fastpass-rebuild-runbook.md | 433 ++++++++++++++++++ talos/cilium/cilium-values.yaml | 37 +- talos/cilium/ip-pools.yaml | 2 +- talos/talhelper/talconfig.yaml | 29 +- 18 files changed, 821 insertions(+), 136 deletions(-) rename {cluster/platform/argocd => argocd}/application.yaml (100%) rename {cluster/platform/argocd => argocd}/argocd-cm.yaml (100%) rename {cluster/platform/argocd => argocd}/argocd-params.yaml (100%) rename {cluster/platform/argocd => argocd}/argocd-rbac-cm.yaml (100%) rename {cluster/platform/argocd => argocd}/certificate.yaml (100%) rename {cluster/platform/argocd => argocd}/externalsecret.yaml (100%) rename {cluster/platform/argocd => argocd}/ingress.yaml (100%) rename {cluster/platform/argocd => argocd}/repository-secret.yaml (100%) rename {cluster/platform/cilium-config => cilium-config}/application.yaml (100%) rename {cluster/platform/cilium-config => cilium-config}/manifests/lb-pool.yaml (100%) rename {cluster/platform/cilium-config => cilium-config}/values.yaml (100%) create mode 100644 cluster/argocd/values.yaml create mode 100644 docs/fastpass-rebuild-runbook.md diff --git a/cluster/platform/argocd/application.yaml b/argocd/application.yaml similarity index 100% rename from cluster/platform/argocd/application.yaml rename to argocd/application.yaml diff --git a/cluster/platform/argocd/argocd-cm.yaml b/argocd/argocd-cm.yaml similarity index 100% rename from cluster/platform/argocd/argocd-cm.yaml rename to argocd/argocd-cm.yaml diff --git a/cluster/platform/argocd/argocd-params.yaml b/argocd/argocd-params.yaml similarity index 100% rename from cluster/platform/argocd/argocd-params.yaml rename to argocd/argocd-params.yaml diff --git a/cluster/platform/argocd/argocd-rbac-cm.yaml b/argocd/argocd-rbac-cm.yaml similarity index 100% rename from cluster/platform/argocd/argocd-rbac-cm.yaml rename to argocd/argocd-rbac-cm.yaml diff --git a/cluster/platform/argocd/certificate.yaml b/argocd/certificate.yaml similarity index 100% rename from cluster/platform/argocd/certificate.yaml rename to argocd/certificate.yaml diff --git a/cluster/platform/argocd/externalsecret.yaml b/argocd/externalsecret.yaml similarity index 100% rename from cluster/platform/argocd/externalsecret.yaml rename to argocd/externalsecret.yaml diff --git a/cluster/platform/argocd/ingress.yaml b/argocd/ingress.yaml similarity index 100% rename from cluster/platform/argocd/ingress.yaml rename to argocd/ingress.yaml diff --git a/cluster/platform/argocd/repository-secret.yaml b/argocd/repository-secret.yaml similarity index 100% rename from cluster/platform/argocd/repository-secret.yaml rename to argocd/repository-secret.yaml diff --git a/cluster/platform/cilium-config/application.yaml b/cilium-config/application.yaml similarity index 100% rename from cluster/platform/cilium-config/application.yaml rename to cilium-config/application.yaml diff --git a/cluster/platform/cilium-config/manifests/lb-pool.yaml b/cilium-config/manifests/lb-pool.yaml similarity index 100% rename from cluster/platform/cilium-config/manifests/lb-pool.yaml rename to cilium-config/manifests/lb-pool.yaml diff --git a/cluster/platform/cilium-config/values.yaml b/cilium-config/values.yaml similarity index 100% rename from cluster/platform/cilium-config/values.yaml rename to cilium-config/values.yaml diff --git a/cluster/argocd/BOOTSTRAP.md b/cluster/argocd/BOOTSTRAP.md index 7b25273..10087dd 100644 --- a/cluster/argocd/BOOTSTRAP.md +++ b/cluster/argocd/BOOTSTRAP.md @@ -1,136 +1,180 @@ -# fastpass GitOps Bootstrap +# fastpass — ArgoCD Bootstrap -This document is the complete procedure to go from a freshly provisioned Talos -cluster to a fully GitOps-managed platform. After completing these steps, all -future changes are made via git — never `kubectl apply` again. +This document covers the manual steps required to bootstrap ArgoCD on the +`fastpass` cluster. After these steps, all further changes are made via Git. -## Prerequisites +**Prerequisites:** +- All 6 Talos nodes `Ready` (`kubectl get nodes`) +- Cilium healthy (`cilium status --wait`) +- IP pools applied (`kubectl get ciliumloadbalancerippool`) +- `city-hall` has `helm`, `kubectl`, and `talosctl` installed +- SSH keypair for ArgoCD → Gitea exists at `/home/wed/.ssh/argocd_gitea` -- Talos cluster running (`space-mountain`, `big-thunder-mountain`, `splash-mountain` CP nodes) -- Cilium installed as CNI (done at Talos bootstrap via talhelper) -- `kubectl` configured and pointing at `fastpass` -- `kubeconfig` on `city-hall` at `~/.kube/config` -- 1Password Connect credentials JSON downloaded from your 1Password account - (Developer Tools > Connect Servers > New Server) -- 1Password Connect token generated for the `mk-labs` vault +--- -## Step 1 — Install ArgoCD - -This is the only `kubectl apply` you will ever run against this cluster. +## Step 1 — Create the argocd namespace ```bash -kubectl create namespace argocd - -kubectl apply -n argocd \ - --server-side \ - --force-conflicts \ - -f https://raw.githubusercontent.com/argoproj/argo-cd/v3.4.2/manifests/install.yaml - -# Wait for ArgoCD to be healthy -kubectl wait --for=condition=available deployment/argocd-server \ - -n argocd --timeout=120s +kubectl apply -f cluster/argocd/namespace.yaml ``` -## Step 2 — Bootstrap the 1Password Connect secret +--- -This is the only secret you will ever manage manually. Everything else flows -through ESO from here. +## Step 2 — Add Gitea to ArgoCD known hosts + +ArgoCD refuses SSH connections to unknown hosts. Pre-populate the known hosts +configmap before ArgoCD starts: ```bash -# Create the namespace first -kubectl create namespace onepassword-connect - -# Apply the credentials file (downloaded from 1Password) -kubectl create secret generic op-credentials \ - --from-file=1password-credentials.json=/path/to/1password-credentials.json \ - -n onepassword-connect - -# Apply the Connect API token -kubectl create secret generic connect-token \ - --from-literal=token= \ - -n onepassword-connect +ssh-keyscan -t ed25519 gitea.mk-labs.cloud | \ + kubectl create configmap argocd-ssh-known-hosts-cm \ + --namespace argocd \ + --from-file=ssh_known_hosts=/dev/stdin \ + --dry-run=client -o yaml | kubectl apply -f - ``` -## Step 3 — Apply the App of Apps +--- -This is the second and final manual apply. From this point on, ArgoCD manages -everything, including itself. +## Step 3 — Create the Gitea repo secret + +This is the only credential that must be manually created. Once ESO is running, +secrets will be managed via 1Password. For now, create it directly: + +```bash +kubectl create secret generic gitea-repo \ + --namespace argocd \ + --from-literal=type=git \ + --from-literal=url=git@gitea.mk-labs.cloud:rblundon/homelab.git \ + --from-file=sshPrivateKey=/home/wed/.ssh/argocd_gitea + +kubectl label secret gitea-repo -n argocd \ + argocd.argoproj.io/secret-type=repository +``` + +Verify: +```bash +kubectl -n argocd get secret gitea-repo +``` + +--- + +## Step 4 — Install ArgoCD via Helm + +```bash +helm repo add argo https://argoproj.github.io/argo-helm +helm repo update + +helm upgrade --install argocd argo/argo-cd \ + --namespace argocd \ + --create-namespace \ + --version 7.8.23 \ + --values cluster/argocd/values.yaml \ + --wait +``` + +Wait for all ArgoCD pods to be ready: + +```bash +kubectl -n argocd get pods --watch +``` + +--- + +## Step 5 — Get the initial admin password + +```bash +kubectl -n argocd get secret argocd-initial-admin-secret \ + -o jsonpath='{.data.password}' | base64 -d; echo +``` + +Access the UI via port-forward (ingress-nginx isn't installed yet at this point): + +```bash +kubectl port-forward svc/argocd-server -n argocd 8080:80 +``` + +Browse to http://localhost:8080 — login with `admin` and the password above. + +--- + +## Step 6 — Apply the app-of-apps + +This is the second and final manual `kubectl apply`. It hands control to ArgoCD: ```bash kubectl apply -n argocd -f cluster/argocd/apps-of-apps.yaml ``` -## Step 4 — Watch it go +From this point, all changes are made via Git. ArgoCD will begin syncing +`cluster/platform/` and `cluster/applications/` automatically. +Watch the sync: ```bash -# Watch all Applications come up in wave order -kubectl get applications -n argocd -w - -# Or use the ArgoCD UI -kubectl port-forward svc/argocd-server -n argocd 8080:443 -# Then open https://localhost:8080 -# Default admin password: -kubectl -n argocd get secret argocd-initial-admin-secret \ - -o jsonpath="{.data.password}" | base64 -d +kubectl -n argocd get applications --watch ``` -## Step 2.5 — Vendor Gateway API CRDs +--- -Before applying the App of Apps, vendor the Gateway API CRDs into git. -Run from the repo root on city-hall: +## Step 7 — Approve CSRs + +After nodes join and kubelet starts requesting certs, approve pending CSRs: ```bash -curl -L https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/standard-install.yaml \ - -o cluster/platform/gateway-api/manifests/standard-install.yaml - -curl -L https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.2.1/experimental-install.yaml \ - -o cluster/platform/gateway-api/manifests/experimental-install.yaml - -git add cluster/platform/gateway-api/manifests/ -git commit -m "feat(platform): vendor gateway-api CRDs v1.2.1" -git push +kubectl get csr --no-headers | awk '{print $1}' | xargs kubectl certificate approve ``` -## Wave Order +Run this periodically until no pending CSRs remain. -| Wave | Application | Depends On | -|------|---------------------|---------------------------------| -| 1 | external-secrets | (none) | -| 2 | onepassword-connect | external-secrets | -| 3 | cert-manager | onepassword-connect (CF secret) | -| 3 | gateway-api-crds | (none — CRDs only) | -| 4 | cilium-config | gateway-api-crds (CRDs exist) | -| 5 | ingress-nginx | cilium-config (LB pool) | -| 5 | gateway (resource) | cilium-config (GatewayClass) | -| 6 | external-dns | ingress-nginx (IP to register) | -| 10+ | applications/* | platform complete | +--- -## Upgrading ArgoCD +## Expected platform sync order -```bash -kubectl apply -n argocd \ - --server-side \ - --force-conflicts \ - -f https://raw.githubusercontent.com/argoproj/argo-cd/v/manifests/install.yaml -``` +Apps sync in wave order via `argocd.argoproj.io/sync-wave` annotations: -Update the version comment in `cluster/argocd/install.yaml` to match. +| Wave | App | +|------|-----| +| 0 | apps-of-apps (root) | +| 1 | cert-manager CRDs | +| 2 | cert-manager, external-secrets | +| 3 | ingress-nginx | +| 4 | external-dns | +| 5+ | application workloads | + +--- ## Troubleshooting +**ArgoCD can't reach Gitea:** ```bash -# Check sync status of all apps -kubectl get applications -n argocd - -# Check why an app is out of sync -kubectl describe application -n argocd - -# Force a sync -kubectl patch application -n argocd \ - --type merge \ - -p '{"operation": {"initiatedBy": {"username": "admin"}, "sync": {}}}' - -# Check ESO secret sync -kubectl get externalsecret -A -kubectl describe externalsecret -n +kubectl -n argocd logs deployment/argocd-repo-server | grep -i "error\|ssh" ``` +Check that the `gitea-repo` secret exists and the known hosts configmap has +`gitea.mk-labs.cloud`. + +**App stuck in `Unknown` state:** +```bash +kubectl -n argocd get application -o yaml | grep -A10 "conditions" +``` + +**self-reference loop (app syncing itself):** +Ensure `apps-of-apps.yaml` uses `include: "*/application.yaml"` with +`recurse: true` — this prevents ArgoCD from picking up its own manifest. + +**ArgoCD pruning itself:** +The `argocd` namespace Applications must have `prune: false`. Never enable +prune on any Application that manages the `argocd` namespace. + +**Helm valueFiles field:** +The correct field name is `valueFiles` (plural). `valuesFile` and `valuesFiles` +are silently ignored — ArgoCD syncs green but your values aren't applied. + +--- + +## Post-bootstrap + +Once ingress-nginx and cert-manager are synced: +- ArgoCD UI available at https://argocd.local.mk-labs.cloud +- Change the admin password: `argocd account update-password` +- Authentik OIDC: uncomment the `oidc.config` block in `values.yaml` once + the ArgoCD provider is configured in Authentik +- Migrate `gitea-repo` secret to ESO once 1Password Connect is running diff --git a/cluster/argocd/values.yaml b/cluster/argocd/values.yaml new file mode 100644 index 0000000..a126846 --- /dev/null +++ b/cluster/argocd/values.yaml @@ -0,0 +1,110 @@ +# ─── ArgoCD Helm values — fastpass cluster ─────────────────────────────────── +# Chart: argo/argo-cd +# Version: 7.8.23 +# +# Install: +# helm repo add argo https://argoproj.github.io/argo-helm +# helm repo update +# helm upgrade --install argocd argo/argo-cd \ +# --namespace argocd \ +# --create-namespace \ +# --version 7.8.23 \ +# --values cluster/argocd/values.yaml \ +# --wait +# +# Then apply the root app-of-apps (second and final manual step): +# kubectl apply -n argocd -f cluster/argocd/apps-of-apps.yaml +# ───────────────────────────────────────────────────────────────────────────── + +# ─── Global ────────────────────────────────────────────────────────────────── +global: + domain: argocd.local.mk-labs.cloud + +# ─── ArgoCD server ─────────────────────────────────────────────────────────── +server: + # Run insecure — TLS is terminated at ingress-nginx, not at ArgoCD itself. + # Without this, ArgoCD redirects to HTTPS internally and breaks behind ingress. + extraArgs: + - --insecure + + ingress: + enabled: true + ingressClassName: nginx + hostname: argocd.local.mk-labs.cloud + tls: true + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production + nginx.ingress.kubernetes.io/ssl-passthrough: "false" + nginx.ingress.kubernetes.io/backend-protocol: "HTTP" + +# ─── ConfigMap — core ArgoCD settings ──────────────────────────────────────── +configs: + params: + # Matches --insecure above — tells the server not to handle TLS + server.insecure: "true" + + cm: + # Gitea as the source repo — HTTPS with token auth (set in secret below) + # SSH alternative is in the repositories section if preferred + url: https://argocd.local.mk-labs.cloud + + # Authentik OIDC SSO + # Uncomment once Authentik is configured with an ArgoCD OIDC provider + # oidc.config: | + # name: Authentik + # issuer: https://authentik.local.mk-labs.cloud/application/o/argocd/ + # clientID: + # clientSecret: $oidc.authentik.clientSecret + # requestedScopes: + # - openid + # - profile + # - email + # - groups + + # Resource health checks — ignore progressing Cilium CRDs + resource.customizations.health.cilium.io_CiliumLoadBalancerIPPool: | + hs = {} + hs.status = "Healthy" + hs.message = "IP pool ready" + return hs + + # RBAC — admin gets full access, SSO groups mapped after Authentik integration + rbac: + policy.default: role:readonly + policy.csv: | + g, argocd-admins, role:admin + + # Repository credentials — Gitea SSH + # Private key is injected via ESO once external-secrets is bootstrapped. + # For initial bootstrap, create the secret manually: + # kubectl create secret generic gitea-repo \ + # --namespace argocd \ + # --from-literal=type=git \ + # --from-literal=url=git@gitea.mk-labs.cloud:rblundon/homelab.git \ + # --from-file=sshPrivateKey=/path/to/argocd_gitea_ed25519 \ + # kubectl label secret gitea-repo -n argocd \ + # argocd.argoproj.io/secret-type=repository + repositories: + homelab-repo: + url: git@gitea.mk-labs.cloud:rblundon/homelab.git + name: homelab + type: git + insecure: "false" + +# ─── Redis ─────────────────────────────────────────────────────────────────── +redis: + enabled: true + +# ─── Notifications ─────────────────────────────────────────────────────────── +# Disabled for now — wire into n8n via webhook later +notifications: + enabled: false + +# ─── ApplicationSet controller ─────────────────────────────────────────────── +applicationSet: + enabled: true + +# ─── Dex (built-in OIDC) ───────────────────────────────────────────────────── +# Disabled — using Authentik as the OIDC provider +dex: + enabled: false diff --git a/docs/city-hall-setup.md b/docs/city-hall-setup.md index 590c35c..bc02455 100644 --- a/docs/city-hall-setup.md +++ b/docs/city-hall-setup.md @@ -59,6 +59,8 @@ Required for provisioning and managing the `fastpass` Talos Kubernetes cluster. | `talosctl` | Talos cluster CLI | See below | | `kubectl` | Kubernetes CLI | See below | | `helm` | Kubernetes package manager | See below | +| `cilium` | Cilium CNI CLI (status, connectivity tests) | See below | +| `argocd` | ArgoCD CLI (app management, account operations) | See below | | `age` | Encryption key generation (used by SOPS) | `sudo apt install age -y` | | `sops` | Secret encryption for `talsecret.sops.yaml` | See below | @@ -108,6 +110,37 @@ Verify: helm version ``` +#### Cilium CLI + +```bash +CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt) +curl -L --remote-name-all \ + https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz +tar -xzvf cilium-linux-amd64.tar.gz +sudo mv cilium /usr/local/bin/ +rm cilium-linux-amd64.tar.gz +``` + +Verify: +```bash +cilium version +``` + +#### ArgoCD CLI + +```bash +ARGOCD_VERSION=$(curl -s https://api.github.com/repos/argoproj/argo-cd/releases/latest | grep tag_name | cut -d '"' -f 4) +curl -sSL -o /tmp/argocd \ + https://github.com/argoproj/argo-cd/releases/download/${ARGOCD_VERSION}/argocd-linux-amd64 +chmod +x /tmp/argocd +sudo mv /tmp/argocd /usr/local/bin/argocd +``` + +Verify: +```bash +argocd version --client +``` + #### SOPS ```bash @@ -274,48 +307,67 @@ talosctl --talosconfig clusterconfig/talosconfig \ --nodes 10.1.71.66 kubeconfig ~/.kube/config ``` -### 8. Approve Worker CSRs +### 8. Approve CSRs ```bash -kubectl get csr -kubectl certificate approve $(kubectl get csr --no-headers | awk '{print $1}') +kubectl get csr --no-headers | awk '{print $1}' | xargs kubectl certificate approve ``` +Run periodically until no pending CSRs remain. + ### 9. Install Cilium (CNI) +Cilium is managed by Helm only — never via ArgoCD. + ```bash helm repo add cilium https://helm.cilium.io/ helm repo update -helm install cilium cilium/cilium \ +helm upgrade --install cilium cilium/cilium \ + --version 1.17.3 \ --namespace kube-system \ - --set ipam.mode=kubernetes \ - --set routingMode=tunnel \ - --set tunnelProtocol=vxlan \ - --set kubeProxyReplacement=true \ - --set k8sServiceHost=10.1.71.65 \ - --set k8sServicePort=6443 \ - --set securityContext.capabilities.ciliumAgent="{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}" \ - --set securityContext.capabilities.cleanCiliumState="{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}" \ - --set cgroup.autoMount.enabled=false \ - --set cgroup.hostRoot=/sys/fs/cgroup + --values talos/cilium/cilium-values.yaml \ + --wait \ + --timeout 5m ``` -Note: The explicit `securityContext.capabilities` flags are required for Talos — Talos -restricts privileged container capabilities by default. The `cgroup.autoMount.enabled=false` -and `cgroup.hostRoot` flags are required because Talos manages cgroups itself. +Apply IP pools: -### 10. Verify Cluster +```bash +kubectl apply -f talos/cilium/ip-pools.yaml +``` + +Verify: + +```bash +cilium status --wait +kubectl get ciliumloadbalancerippool +kubectl get ciliuml2announcementpolicy +``` + +**Important:** The `cilium-values.yaml` contains required Talos-specific overrides: +- `securityContext.capabilities` — removes `SYS_MODULE` (blocked by Talos permanently) +- `cgroup.autoMount.enabled: false` — Talos manages cgroups itself +- `l2announcements.interface: ens18` — Proxmox VMs use `ens18`, not `eth0` +- `l2podAnnouncements.interface: ens18` — same reason + +### 10. Bootstrap ArgoCD + +See `cluster/argocd/BOOTSTRAP.md` for the full ArgoCD bootstrap sequence. + +Summary: +1. `kubectl apply -f cluster/argocd/namespace.yaml` +2. Add Gitea SSH host key to ArgoCD known hosts +3. Create `gitea-repo` secret with SSH private key at `~/.ssh/argocd_gitea` +4. `helm upgrade --install argocd argo/argo-cd --values cluster/argocd/values.yaml` +5. `kubectl apply -n argocd -f cluster/argocd/apps-of-apps.yaml` + +### 11. Verify Cluster ```bash kubectl get nodes -kubectl get pods -n kube-system +kubectl get pods -A | grep -v Running | grep -v Completed +cilium status --wait ``` -All nodes should show `Ready`. Cilium pods should be `Running`. - -### 11. Post-Install HA Setup - -See output of `terraform output` for the full HA registration commands. -Register VMs with Proxmox HA and add to node-affinity rules once the -cluster is stable and the ISO is no longer needed for boot. +All nodes should show `Ready`. All pods should be `Running` or `Completed`. diff --git a/docs/fastpass-rebuild-runbook.md b/docs/fastpass-rebuild-runbook.md new file mode 100644 index 0000000..08705fa --- /dev/null +++ b/docs/fastpass-rebuild-runbook.md @@ -0,0 +1,433 @@ +# fastpass cluster rebuild runbook + +Root cause: Talos 1.13 defaults to `enforce: baseline` for Pod Security admission +control. Cilium requires `NET_ADMIN`, `NET_RAW`, and `SYS_ADMIN` — all blocked +under baseline. Fix: `enforce: privileged` in talconfig.yaml before generating +machine configs. Everything else in this runbook follows from a clean wipe. + +All commands run from `city-hall` (10.1.71.38) in the `talos/talhelper/` +directory unless otherwise noted. + +--- + +## Phase 0 — pre-flight + +Confirm tools are present and on the right versions: + +```bash +talhelper --version # expect 3.1.10 +talosctl version --client # expect v1.13.2 +kubectl version --client # expect v1.36.x +helm version # expect v3.x +``` + +Confirm the age key and sops config are in place: + +```bash +cat ~/.config/sops/age/keys.txt | head -1 # should show "# created: ..." +cat /opt/git/homelab/.sops.yaml # should reference your age pubkey +``` + +--- + +## Phase 1 — wipe and redeploy Talos nodes + +### 1.1 Reset all six nodes + +Run from `talos/talhelper/`. If the nodes are currently in a broken state, +reset them to wipe the install and reboot into maintenance mode: + +```bash +# Control plane nodes +for node in 10.1.71.66 10.1.71.67 10.1.71.68; do + talosctl reset \ + --talosconfig clusterconfig/talosconfig \ + --nodes $node \ + --graceful=false \ + --reboot \ + --system-labels-to-wipe STATE \ + --system-labels-to-wipe EPHEMERAL +done + +# Worker nodes +for node in 10.1.71.69 10.1.71.70 10.1.71.71; do + talosctl reset \ + --talosconfig clusterconfig/talosconfig \ + --nodes $node \ + --graceful=false \ + --reboot \ + --system-labels-to-wipe STATE \ + --system-labels-to-wipe EPHEMERAL +done +``` + +Wait ~2 minutes. Nodes boot into maintenance mode (no OS installed, just the +Talos installer kernel running in RAM). You can verify via Proxmox console — +the node should show the Talos maintenance screen. + +If nodes are completely unreachable (talosctl can't connect), boot them from +the Talos ISO via Proxmox and they'll come up in maintenance mode automatically. + +### 1.2 Regenerate machine configs + +The talconfig.yaml now has `enforce: privileged`. Regenerate everything: + +```bash +cd /opt/git/homelab/talos/talhelper +talhelper genconfig +``` + +This writes fresh configs to `clusterconfig/`. Commit the regenerated configs +before applying: + +```bash +cd /opt/git/homelab +git add talos/ +git commit -m "fix: set enforce:privileged for Cilium compatibility" +git push +``` + +### 1.3 Apply machine configs + +```bash +cd /opt/git/homelab/talos/talhelper +talhelper gencommand apply --extra-flags="--insecure" | bash +``` + +Expected output for each node: `Applied configuration without a reboot` + +The nodes will install Talos to disk and reboot automatically. Watch progress +on one control plane node: + +```bash +talosctl -n 10.1.71.66 --talosconfig clusterconfig/talosconfig dmesg --follow +``` + +Wait for all six nodes to finish installing and come back up (~3-5 min). + +### 1.4 Bootstrap etcd + +Run bootstrap exactly once — on the first control plane node only: + +```bash +talhelper gencommand bootstrap | bash +``` + +This initializes etcd on `space-mountain`. The other two control plane nodes +join automatically. Do not run bootstrap again — it will corrupt the cluster. + +### 1.5 Get kubeconfig + +```bash +talhelper gencommand kubeconfig | bash +# or directly: +talosctl kubeconfig \ + --talosconfig clusterconfig/talosconfig \ + --nodes 10.1.71.66 \ + ~/.kube/config +``` + +### 1.6 Verify all 6 nodes are Ready + +```bash +kubectl get nodes -o wide --watch +``` + +Wait until all six show `Ready`. This takes 3-5 minutes after bootstrap. +Do not proceed to Phase 2 until all six nodes are `Ready`. + +Expected output: +``` +NAME STATUS ROLES AGE VERSION +big-thunder-mountain Ready control-plane 5m v1.32.3 +haunted-mansion Ready 4m v1.32.3 +jungle-cruise Ready 4m v1.32.3 +peter-pans-flight Ready 4m v1.32.3 +space-mountain Ready control-plane 5m v1.32.3 +splash-mountain Ready control-plane 5m v1.32.3 +``` + +### 1.7 Approve all pending CSRs + +```bash +kubectl get csr --no-headers | grep Pending | awk '{print $1}' | xargs kubectl certificate approve +``` + +Nodes will show `NotReady` until Cilium is installed — that's expected. The +condition clears after Phase 2. + +--- + +## Phase 2 — install Cilium + +Cilium is Helm-only. It is never managed by ArgoCD. + +### 2.1 Add the Helm repo + +```bash +helm repo add cilium https://helm.cilium.io/ +helm repo update +``` + +### 2.2 Install Cilium + +```bash +helm upgrade --install cilium cilium/cilium \ + --version 1.17.3 \ + --namespace kube-system \ + --values /opt/git/homelab/talos/cilium/cilium-values.yaml \ + --wait \ + --timeout 5m +``` + +The `--wait` flag blocks until all Cilium pods are Running. If it times out, +check what's happening: + +```bash +kubectl -n kube-system get pods -l app.kubernetes.io/name=cilium-agent +kubectl -n kube-system describe pod +``` + +### 2.3 Apply IP pools + +```bash +kubectl apply -f /opt/git/homelab/talos/cilium/ip-pools.yaml +``` + +### 2.4 Verify Cilium health + +```bash +# Overall status +cilium status --wait + +# All nodes should show cilium-agent as OK +kubectl -n kube-system get pods -l app.kubernetes.io/name=cilium-agent + +# Confirm L2 announcement policy is active +kubectl get ciliuml2announcementpolicy + +# Confirm IP pools are ready +kubectl get ciliumloadbalancerippool +``` + +### 2.5 Verify all nodes are Ready + +```bash +kubectl get nodes +``` + +All six must show `Ready` before proceeding. If any are still `NotReady` after +Cilium is healthy, drain and uncordon to force a kubelet re-check: + +```bash +kubectl drain --ignore-daemonsets --delete-emptydir-data +kubectl uncordon +``` + +--- + +## Phase 3 — bootstrap ArgoCD + +ArgoCD is installed manually once. After that, everything else is GitOps. + +### 3.1 Create the argocd namespace + +```bash +kubectl create namespace argocd +``` + +### 3.2 Install ArgoCD + +```bash +kubectl apply -n argocd \ + -f https://raw.githubusercontent.com/argoproj/argo-cd/v3.4.2/manifests/install.yaml +``` + +Wait for ArgoCD to come up: + +```bash +kubectl -n argocd rollout status deployment argocd-server --timeout=3m +``` + +### 3.3 Apply server config (insecure mode for ingress TLS termination) + +ArgoCD needs to run with `--insecure` so ingress-nginx handles TLS. This is in +your repo at `cluster/argocd/argocd-cmd-params-cm.yaml`: + +```bash +kubectl apply -f /opt/git/homelab/cluster/argocd/argocd-cmd-params-cm.yaml +kubectl -n argocd rollout restart deployment argocd-server +kubectl -n argocd rollout status deployment argocd-server --timeout=2m +``` + +The ConfigMap should contain: + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: argocd-cmd-params-cm + namespace: argocd +data: + server.insecure: "true" +``` + +### 3.4 Bootstrap 1Password Connect secret + +This is the only manual `kubectl apply` after ArgoCD install. Everything else +comes from Git. + +```bash +# Create the secret from your 1Password Connect credentials file +kubectl create namespace external-secrets + +kubectl create secret generic onepassword-connect-secret \ + --namespace external-secrets \ + --from-file=1password-credentials.json=/path/to/1password-credentials.json \ + --from-literal=token= +``` + +Verify: +```bash +kubectl -n external-secrets get secret onepassword-connect-secret +``` + +### 3.5 Configure ArgoCD to access Gitea via SSH + +Add the Gitea SSH key to ArgoCD's known hosts and create the repo secret: + +```bash +# Get Gitea's SSH host key +ssh-keyscan -t ed25519 mad-tea-party.local.mk-labs.cloud >> /tmp/gitea-known-hosts + +# Create the repo credentials secret +kubectl apply -f - </dev/null)" +``` + +### 3.6 Apply the root Application + +This is the app-of-apps entry point. It points ArgoCD at `cluster/platform/` +and `cluster/applications/`: + +```bash +kubectl apply -f /opt/git/homelab/cluster/argocd/application.yaml +``` + +The argocd Application in that file must have `prune: false` to prevent ArgoCD +from ever pruning its own namespace: + +```yaml +# cluster/argocd/application.yaml (key fields) +spec: + project: default + source: + repoURL: git@mad-tea-party.local.mk-labs.cloud:rblundon/homelab.git + targetRevision: main + path: cluster/platform + directory: + exclude: "application.yaml" # prevents self-reference loop + destination: + server: https://kubernetes.default.svc + syncPolicy: + automated: + prune: false # NEVER prune argocd namespace + selfHeal: true +``` + +### 3.7 Watch the platform sync + +```bash +# Get initial admin password +kubectl -n argocd get secret argocd-initial-admin-secret \ + -o jsonpath='{.data.password}' | base64 -d; echo + +# Port-forward to access UI (before ingress is up) +kubectl port-forward svc/argocd-server -n argocd 8080:443 + +# Or use CLI +argocd login localhost:8080 --username admin --insecure + +# Watch sync status +argocd app list +argocd app get platform +``` + +Platform apps will sync in wave order (sync-wave annotation). Expected order: +1. Wave 0: cert-manager CRDs +2. Wave 1: cert-manager, external-secrets +3. Wave 2: ingress-nginx +4. Wave 3: external-dns +5. Wave 4: application workloads + +--- + +## Phase 4 — verify platform + +```bash +# All platform pods running +kubectl get pods -A | grep -v Running | grep -v Completed + +# Ingress controller has a LB IP from the pool +kubectl -n ingress-nginx get svc ingress-nginx-controller + +# cert-manager is issuing certs +kubectl get clusterissuers + +# External secrets operator is syncing +kubectl -n external-secrets get pods + +# external-dns is running +kubectl -n external-dns get pods +``` + +--- + +## Known gotchas (learned the hard way) + +**ArgoCD self-prune:** The `argocd-config` Application must have `prune: false`. +If prune is enabled and ArgoCD syncs itself, it will delete its own resources +and take itself down. Recovery requires manual re-apply. + +**Multi-source self-reference loop:** If an Application's source path contains +its own `application.yaml`, ArgoCD enters a sync loop. Fix: add +`directory.exclude: "application.yaml"` to the source spec. + +**ArgoCD Helm valueFiles:** The correct field name is `valueFiles` (plural), not +`valuesFile` or `valuesFiles`. Wrong field name causes silent failure — ArgoCD +syncs green but ignores your values. + +**Cilium managed by Helm only:** Never add Cilium to ArgoCD. If ArgoCD manages +Cilium and there's a sync conflict, ArgoCD can restart Cilium mid-cluster +operation, breaking all networking. Helm is the correct tool here. + +**IP pool API version:** CiliumLoadBalancerIPPool must use `cilium.io/v2alpha1`. +Wrong version creates the object but it never activates — LB IPs are assigned +but not advertised. + +**Talos VIP and NIC name:** The VIP (10.1.71.65) is configured per-node in the +`networkInterfaces` section. Each control plane node declares the VIP on `ens18`. +Workers do not declare the VIP. If the NIC name changes on your Proxmox +VMs, check with: `talosctl -n get addresses --talosconfig clusterconfig/talosconfig` + +**Bootstrap once only:** `talhelper gencommand bootstrap` initializes etcd. +Running it a second time on an already-bootstrapped cluster corrupts etcd. +If you accidentally run it twice, you need to wipe and start over. diff --git a/talos/cilium/cilium-values.yaml b/talos/cilium/cilium-values.yaml index 9d93e20..76a3575 100644 --- a/talos/cilium/cilium-values.yaml +++ b/talos/cilium/cilium-values.yaml @@ -44,10 +44,12 @@ tunnelProtocol: vxlan # Without this, LoadBalancer IPs are assigned but unreachable from outside the cluster. l2announcements: enabled: true + interface: ens18 # Also enable L2 pod announcements so Cilium handles ARP for pod IPs directly l2podAnnouncements: enabled: true + interface: ens18 # ─── IP pools (defined as CRDs post-install, but referenced here for docs) ─── # Pool 1: ingress-nginx 10.1.71.80 - 10.1.71.89 @@ -55,6 +57,39 @@ l2podAnnouncements: # Apply these after Cilium is healthy: # kubectl apply -f talos/cilium/ip-pools.yaml +# ─── Talos security context overrides ─────────────────────────────────────── +# Talos permanently blocks SYS_MODULE and SYS_BOOT at the OS level regardless +# of pod security settings. Cilium's default caps include SYS_MODULE which +# causes clean-cilium-state to fail with "unable to apply caps". +# These overrides replace the default cap sets with Talos-compatible ones. +securityContext: + capabilities: + ciliumAgent: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + - PERFMON + - BPF + cleanCiliumState: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE + - BPF + +# Talos mounts cgroups differently — disable auto-mount and point at host root +cgroup: + autoMount: + enabled: false + hostRoot: /sys/fs/cgroup + # ─── Operator ──────────────────────────────────────────────────────────────── operator: replicas: 1 # single-replica is fine for homelab; avoids scheduling issues on 3-node CP @@ -80,7 +115,7 @@ gatewayAPI: # ─── Security ──────────────────────────────────────────────────────────────── # Restrict host namespace access — pods can't escape to host network by default hostNamespaceAccess: - enabled: false + enabled: true # Enable network policies (default deny is applied per-namespace by ArgoCD apps) policyEnforcementMode: default diff --git a/talos/cilium/ip-pools.yaml b/talos/cilium/ip-pools.yaml index 9c83960..68c2c50 100644 --- a/talos/cilium/ip-pools.yaml +++ b/talos/cilium/ip-pools.yaml @@ -42,4 +42,4 @@ spec: nodeSelector: matchLabels: {} interfaces: - - eth0 + - ens18 diff --git a/talos/talhelper/talconfig.yaml b/talos/talhelper/talconfig.yaml index 600dc0b..617c4f1 100644 --- a/talos/talhelper/talconfig.yaml +++ b/talos/talhelper/talconfig.yaml @@ -41,7 +41,6 @@ patches: usernames: [] runtimeClasses: [] namespaces: - - kube-system - argocd - cert-manager - ingress-nginx @@ -53,8 +52,20 @@ patches: servers: - 10.1.71.21 # sundial — local NTP - # DNS search domain so short names resolve inside the cluster + # Sysctls required for Cilium eBPF on Talos + # bpf_jit_harden: 2 (Talos default) blocks Cilium's eBPF programs + # unprivileged_bpf_disabled: 1 (Talos default) blocks BPF for unprivileged containers + # perf_event_paranoid: 3 (Talos default) too restrictive for Cilium observability + sysctls: + net.core.bpf_jit_harden: "0" + kernel.unprivileged_bpf_disabled: "0" + kernel.perf_event_paranoid: "1" + + # DNS — point at world-drive (UDM Pro), which forwards local.mk-labs.cloud + # to monorail and handles external resolution via DoH network: + nameservers: + - 10.1.71.1 extraHostEntries: [] # Talos manages kubelet — these flags are additive @@ -90,7 +101,7 @@ controlPlane: # VIP — floats across all three control plane nodes # Talos handles VIP natively; no keepalived needed interfaces: - - interface: eth0 + - interface: ens18 vip: ip: 10.1.71.65 @@ -102,7 +113,7 @@ nodes: controlPlane: true installDisk: /dev/sda networkInterfaces: - - interface: eth0 + - interface: ens18 addresses: - 10.1.71.66/24 routes: @@ -117,7 +128,7 @@ nodes: controlPlane: true installDisk: /dev/sda networkInterfaces: - - interface: eth0 + - interface: ens18 addresses: - 10.1.71.67/24 routes: @@ -132,7 +143,7 @@ nodes: controlPlane: true installDisk: /dev/sda networkInterfaces: - - interface: eth0 + - interface: ens18 addresses: - 10.1.71.68/24 routes: @@ -148,7 +159,7 @@ nodes: controlPlane: false installDisk: /dev/sda networkInterfaces: - - interface: eth0 + - interface: ens18 addresses: - 10.1.71.69/24 routes: @@ -161,7 +172,7 @@ nodes: controlPlane: false installDisk: /dev/sda networkInterfaces: - - interface: eth0 + - interface: ens18 addresses: - 10.1.71.70/24 routes: @@ -174,7 +185,7 @@ nodes: controlPlane: false installDisk: /dev/sda networkInterfaces: - - interface: eth0 + - interface: ens18 addresses: - 10.1.71.71/24 routes: