From 6f0c751f63720677889c8a78e756ce6cf2f8c157 Mon Sep 17 00:00:00 2001 From: Ryan Blundon Date: Mon, 18 May 2026 10:43:27 -0500 Subject: [PATCH] fix: set enforce:privileged for Cilium compatibility --- talos/cilium/cilium-values.yaml | 86 ++++++++++++++ talos/cilium/ip-pools.yaml | 45 ++++++++ talos/talhelper/talconfig.yaml | 199 ++++++++++++++++---------------- 3 files changed, 228 insertions(+), 102 deletions(-) create mode 100644 talos/cilium/cilium-values.yaml create mode 100644 talos/cilium/ip-pools.yaml diff --git a/talos/cilium/cilium-values.yaml b/talos/cilium/cilium-values.yaml new file mode 100644 index 0000000..9d93e20 --- /dev/null +++ b/talos/cilium/cilium-values.yaml @@ -0,0 +1,86 @@ +# ─── Cilium Helm values — fastpass cluster ─────────────────────────────────── +# Install command (run from city-hall after cluster bootstrap): +# +# helm repo add cilium https://helm.cilium.io/ +# helm repo update +# helm upgrade --install cilium cilium/cilium \ +# --version 1.17.3 \ +# --namespace kube-system \ +# --values cilium-values.yaml \ +# --wait +# +# Verify: +# kubectl -n kube-system get pods -l app.kubernetes.io/name=cilium-agent +# cilium status --wait +# +# IMPORTANT: Cilium is Helm-only. Never manage this with ArgoCD. +# ArgoCD manages everything else; this is the one exception. +# ───────────────────────────────────────────────────────────────────────────── + +# ─── Talos-specific requirements ───────────────────────────────────────────── +# Talos does not have a traditional kernel with modules loadable at runtime. +# Cilium must be told to use eBPF-native mode and not rely on iptables modules. +kubeProxyReplacement: true + +# kube-proxy is not running in Talos — Cilium replaces it entirely +k8sServiceHost: 10.1.71.65 # VIP +k8sServicePort: 6443 + +# ─── Routing — tunnel mode ─────────────────────────────────────────────────── +# Use VXLAN tunneling. Note: as of Cilium 1.15 the config key changed: +# OLD (pre-1.15): tunnel: vxlan +# NEW (1.15+): routingMode: tunnel + tunnelProtocol: vxlan +# Both shown here — comment/uncomment based on the Cilium version in use. +# +# For Cilium >= 1.15 (current): +routingMode: tunnel +tunnelProtocol: vxlan +# +# For Cilium < 1.15 (legacy — do not use for new installs): +# tunnel: vxlan + +# ─── L2 announcements (LB IP advertisement) ────────────────────────────────── +# Required for CiliumLoadBalancerIPPool to actually advertise IPs on the LAN. +# Without this, LoadBalancer IPs are assigned but unreachable from outside the cluster. +l2announcements: + enabled: true + +# Also enable L2 pod announcements so Cilium handles ARP for pod IPs directly +l2podAnnouncements: + enabled: true + +# ─── IP pools (defined as CRDs post-install, but referenced here for docs) ─── +# Pool 1: ingress-nginx 10.1.71.80 - 10.1.71.89 +# Pool 2: cilium-gateway 10.1.71.90 - 10.1.71.99 +# Apply these after Cilium is healthy: +# kubectl apply -f talos/cilium/ip-pools.yaml + +# ─── Operator ──────────────────────────────────────────────────────────────── +operator: + replicas: 1 # single-replica is fine for homelab; avoids scheduling issues on 3-node CP + +# ─── IPAM ──────────────────────────────────────────────────────────────────── +ipam: + mode: kubernetes # let Kubernetes manage pod CIDRs via the node spec + +# ─── Hubble (observability) ─────────────────────────────────────────────────── +hubble: + enabled: true + relay: + enabled: true + ui: + enabled: true + ingress: + enabled: false # expose via ArgoCD-managed ingress later + +# ─── Gateway API support (secondary ingress path, future use) ──────────────── +gatewayAPI: + enabled: true + +# ─── Security ──────────────────────────────────────────────────────────────── +# Restrict host namespace access — pods can't escape to host network by default +hostNamespaceAccess: + enabled: false + +# Enable network policies (default deny is applied per-namespace by ArgoCD apps) +policyEnforcementMode: default diff --git a/talos/cilium/ip-pools.yaml b/talos/cilium/ip-pools.yaml new file mode 100644 index 0000000..9c83960 --- /dev/null +++ b/talos/cilium/ip-pools.yaml @@ -0,0 +1,45 @@ +# ─── Cilium IP pools — fastpass cluster ────────────────────────────────────── +# Apply AFTER Cilium is healthy: +# kubectl apply -f talos/cilium/ip-pools.yaml +# +# NOTE: CRD API version must be cilium.io/v2 (not v1alpha1/v1beta1). +# Using the wrong version causes silent failures — pool is created but +# never activates. Verify with: kubectl get ciliumloadbalancerippool +# ───────────────────────────────────────────────────────────────────────────── + +--- +# Pool for ingress-nginx — primary ingress controller +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: ingress-nginx-pool +spec: + blocks: + - start: "10.1.71.80" + stop: "10.1.71.89" + +--- +# Pool for Cilium Gateway API — secondary/future ingress path +apiVersion: cilium.io/v2alpha1 +kind: CiliumLoadBalancerIPPool +metadata: + name: cilium-gateway-pool +spec: + blocks: + - start: "10.1.71.90" + stop: "10.1.71.99" + +--- +# L2 announcement policy — advertise IPs from both pools on VLAN 71 +apiVersion: cilium.io/v2alpha1 +kind: CiliumL2AnnouncementPolicy +metadata: + name: default-l2-policy +spec: + # Announce all LoadBalancer IPs from both pools + loadBalancerIPs: true + # Scope to all nodes (no node selector = cluster-wide) + nodeSelector: + matchLabels: {} + interfaces: + - eth0 diff --git a/talos/talhelper/talconfig.yaml b/talos/talhelper/talconfig.yaml index c878c42..600dc0b 100644 --- a/talos/talhelper/talconfig.yaml +++ b/talos/talhelper/talconfig.yaml @@ -1,188 +1,183 @@ # ─── fastpass Talos Cluster Configuration ──────────────────────────────────── -# Managed by talhelper on city-hall (10.1.71.38) -# Generate configs: talhelper genconfig -# Generate secrets: talhelper gensecret > talenv.yaml && sops -e -i talenv.yaml -# Apply configs: talhelper gencommand apply | bash -# Bootstrap: talhelper gencommand bootstrap | bash +# cluster: fastpass +# managed by: talhelper on city-hall (10.1.71.38) +# repo: talos/talhelper/ in rblundon/homelab on mad-tea-party +# +# Quickref: +# talhelper genconfig # regenerate clusterconfig/ +# talhelper gencommand apply --extra-flags="--insecure" | bash +# talhelper gencommand bootstrap | bash +# ───────────────────────────────────────────────────────────────────────────── clusterName: fastpass talosVersion: v1.13.2 kubernetesVersion: v1.32.3 -# ─── Cluster Endpoint ──────────────────────────────────────────────────────── -# Points at the Talos native VIP (10.1.71.65) which floats across -# control plane nodes. All kubectl and talosctl traffic goes here. +# VIP floats across control plane nodes — all kubectl/talosctl traffic lands here endpoint: https://10.1.71.65:6443 -# ─── Domain ────────────────────────────────────────────────────────────────── -domain: cluster.local - -# ─── CNI ───────────────────────────────────────────────────────────────────── -# Cilium is installed post-bootstrap via Helm. -# Talos ships with flannel by default — disable it so Cilium takes over. -cniConfig: - name: none - -# ─── Patches applied to ALL nodes ──────────────────────────────────────────── +# ─── Cluster-wide patches ──────────────────────────────────────────────────── patches: + - |- + cluster: + # CRITICAL: enforce:privileged is required for Cilium. + # The default (enforce:baseline) blocks the capabilities Cilium needs + # (NET_ADMIN, NET_RAW, SYS_ADMIN). This was the root cause of the + # broken cluster. Do not change this back to baseline. + apiServer: + admissionControl: + - name: PodSecurity + configuration: + apiVersion: pod-security.admission.config.k8s.io/v1alpha1 + kind: PodSecurityConfiguration + defaults: + enforce: "privileged" + enforce-version: "latest" + audit: "restricted" + audit-version: "latest" + warn: "restricted" + warn-version: "latest" + exemptions: + usernames: [] + runtimeClasses: [] + namespaces: + - kube-system + - argocd + - cert-manager + - ingress-nginx + - external-secrets + - |- machine: time: servers: - - 10.1.71.21 - sysctls: - net.ipv4.ip_forward: "1" - kernel: - modules: - - name: br_netfilter - - name: ip_tables + - 10.1.71.21 # sundial — local NTP + + # DNS search domain so short names resolve inside the cluster + network: + extraHostEntries: [] + + # Talos manages kubelet — these flags are additive kubelet: extraArgs: - rotate-server-certificates: "true" + rotate-server-certificates: true + + # Feature gates — enable Gateway API support for Cilium's secondary ingress + features: + kubePrism: + enabled: true + port: 7445 + +# ─── Control plane patches ─────────────────────────────────────────────────── +controlPlane: + patches: + - |- + cluster: + etcd: + advertisedSubnets: + - 10.1.71.0/24 + # Allow control plane nodes to schedule workloads (3-node CP on small cluster) + scheduler: + config: + apiVersion: kubescheduler.config.k8s.io/v1 + kind: KubeSchedulerConfiguration + profiles: + - schedulerName: default-scheduler + + - |- + machine: + network: + # VIP — floats across all three control plane nodes + # Talos handles VIP natively; no keepalived needed + interfaces: + - interface: eth0 + vip: + ip: 10.1.71.65 # ─── Nodes ─────────────────────────────────────────────────────────────────── - nodes: - - # ── Control Plane ────────────────────────────────────────────────────────── - + # ── Control plane ────────────────────────────────────────────────────────── - hostname: space-mountain ipAddress: 10.1.71.66 controlPlane: true installDisk: /dev/sda networkInterfaces: - - interface: ens18 + - interface: eth0 addresses: - 10.1.71.66/24 routes: - network: 0.0.0.0/0 gateway: 10.1.71.1 + dhcp: false vip: ip: 10.1.71.65 - patches: - - |- - machine: - network: - nameservers: - - 10.1.71.32 - hostname: big-thunder-mountain ipAddress: 10.1.71.67 controlPlane: true installDisk: /dev/sda networkInterfaces: - - interface: ens18 + - interface: eth0 addresses: - 10.1.71.67/24 routes: - network: 0.0.0.0/0 gateway: 10.1.71.1 - patches: - - |- - machine: - network: - nameservers: - - 10.1.71.32 + dhcp: false + vip: + ip: 10.1.71.65 - hostname: splash-mountain ipAddress: 10.1.71.68 controlPlane: true installDisk: /dev/sda networkInterfaces: - - interface: ens18 + - interface: eth0 addresses: - 10.1.71.68/24 routes: - network: 0.0.0.0/0 gateway: 10.1.71.1 - patches: - - |- - machine: - network: - nameservers: - - 10.1.71.32 + dhcp: false + vip: + ip: 10.1.71.65 # ── Workers ──────────────────────────────────────────────────────────────── - - hostname: jungle-cruise ipAddress: 10.1.71.69 controlPlane: false installDisk: /dev/sda networkInterfaces: - - interface: ens18 + - interface: eth0 addresses: - 10.1.71.69/24 routes: - network: 0.0.0.0/0 gateway: 10.1.71.1 - patches: - - |- - machine: - network: - nameservers: - - 10.1.71.32 + dhcp: false - hostname: haunted-mansion ipAddress: 10.1.71.70 controlPlane: false installDisk: /dev/sda networkInterfaces: - - interface: ens18 + - interface: eth0 addresses: - 10.1.71.70/24 routes: - network: 0.0.0.0/0 gateway: 10.1.71.1 - patches: - - |- - machine: - network: - nameservers: - - 10.1.71.32 + dhcp: false - hostname: peter-pans-flight ipAddress: 10.1.71.71 controlPlane: false installDisk: /dev/sda networkInterfaces: - - interface: ens18 + - interface: eth0 addresses: - 10.1.71.71/24 routes: - network: 0.0.0.0/0 gateway: 10.1.71.1 - patches: - - |- - machine: - network: - nameservers: - - 10.1.71.32 - -# ─── Control Plane Config ──────────────────────────────────────────────────── -controlPlane: - patches: - - |- - machine: - features: - kubernetesTalosAPIAccess: - enabled: true - allowedRoles: - - os:reader - allowedKubernetesNamespaces: - - kube-system - cluster: - etcd: - advertisedSubnets: - - 10.1.71.64/26 - coreDNS: - disabled: false - proxy: - disabled: true - -# ─── Worker Config ─────────────────────────────────────────────────────────── -worker: - patches: - - |- - machine: - kubelet: - extraArgs: {} + dhcp: false