fix: set enforce:privileged for Cilium compatibility

talos/cilium/cilium-values.yaml

@@ -0,0 +1,86 @@
+# ─── Cilium Helm values — fastpass cluster ───────────────────────────────────
+# Install command (run from city-hall after cluster bootstrap):
+#
+#   helm repo add cilium https://helm.cilium.io/
+#   helm repo update
+#   helm upgrade --install cilium cilium/cilium \
+#     --version 1.17.3 \
+#     --namespace kube-system \
+#     --values cilium-values.yaml \
+#     --wait
+#
+# Verify:
+#   kubectl -n kube-system get pods -l app.kubernetes.io/name=cilium-agent
+#   cilium status --wait
+#
+# IMPORTANT: Cilium is Helm-only. Never manage this with ArgoCD.
+# ArgoCD manages everything else; this is the one exception.
+# ─────────────────────────────────────────────────────────────────────────────
+
+# ─── Talos-specific requirements ─────────────────────────────────────────────
+# Talos does not have a traditional kernel with modules loadable at runtime.
+# Cilium must be told to use eBPF-native mode and not rely on iptables modules.
+kubeProxyReplacement: true
+
+# kube-proxy is not running in Talos — Cilium replaces it entirely
+k8sServiceHost: 10.1.71.65   # VIP
+k8sServicePort: 6443
+
+# ─── Routing — tunnel mode ───────────────────────────────────────────────────
+# Use VXLAN tunneling. Note: as of Cilium 1.15 the config key changed:
+#   OLD (pre-1.15):  tunnel: vxlan
+#   NEW (1.15+):     routingMode: tunnel + tunnelProtocol: vxlan
+# Both shown here — comment/uncomment based on the Cilium version in use.
+#
+# For Cilium >= 1.15 (current):
+routingMode: tunnel
+tunnelProtocol: vxlan
+#
+# For Cilium < 1.15 (legacy — do not use for new installs):
+# tunnel: vxlan
+
+# ─── L2 announcements (LB IP advertisement) ──────────────────────────────────
+# Required for CiliumLoadBalancerIPPool to actually advertise IPs on the LAN.
+# Without this, LoadBalancer IPs are assigned but unreachable from outside the cluster.
+l2announcements:
+  enabled: true
+
+# Also enable L2 pod announcements so Cilium handles ARP for pod IPs directly
+l2podAnnouncements:
+  enabled: true
+
+# ─── IP pools (defined as CRDs post-install, but referenced here for docs) ───
+# Pool 1: ingress-nginx   10.1.71.80 - 10.1.71.89
+# Pool 2: cilium-gateway  10.1.71.90 - 10.1.71.99
+# Apply these after Cilium is healthy:
+#   kubectl apply -f talos/cilium/ip-pools.yaml
+
+# ─── Operator ────────────────────────────────────────────────────────────────
+operator:
+  replicas: 1   # single-replica is fine for homelab; avoids scheduling issues on 3-node CP
+
+# ─── IPAM ────────────────────────────────────────────────────────────────────
+ipam:
+  mode: kubernetes   # let Kubernetes manage pod CIDRs via the node spec
+
+# ─── Hubble (observability) ───────────────────────────────────────────────────
+hubble:
+  enabled: true
+  relay:
+    enabled: true
+  ui:
+    enabled: true
+    ingress:
+      enabled: false   # expose via ArgoCD-managed ingress later
+
+# ─── Gateway API support (secondary ingress path, future use) ────────────────
+gatewayAPI:
+  enabled: true
+
+# ─── Security ────────────────────────────────────────────────────────────────
+# Restrict host namespace access — pods can't escape to host network by default
+hostNamespaceAccess:
+  enabled: false
+
+# Enable network policies (default deny is applied per-namespace by ArgoCD apps)
+policyEnforcementMode: default

talos/cilium/ip-pools.yaml

@@ -0,0 +1,45 @@
+# ─── Cilium IP pools — fastpass cluster ──────────────────────────────────────
+# Apply AFTER Cilium is healthy:
+#   kubectl apply -f talos/cilium/ip-pools.yaml
+#
+# NOTE: CRD API version must be cilium.io/v2 (not v1alpha1/v1beta1).
+# Using the wrong version causes silent failures — pool is created but
+# never activates. Verify with: kubectl get ciliumloadbalancerippool
+# ─────────────────────────────────────────────────────────────────────────────
+
+---
+# Pool for ingress-nginx — primary ingress controller
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: ingress-nginx-pool
+spec:
+  blocks:
+    - start: "10.1.71.80"
+      stop:  "10.1.71.89"
+
+---
+# Pool for Cilium Gateway API — secondary/future ingress path
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: cilium-gateway-pool
+spec:
+  blocks:
+    - start: "10.1.71.90"
+      stop:  "10.1.71.99"
+
+---
+# L2 announcement policy — advertise IPs from both pools on VLAN 71
+apiVersion: cilium.io/v2alpha1
+kind: CiliumL2AnnouncementPolicy
+metadata:
+  name: default-l2-policy
+spec:
+  # Announce all LoadBalancer IPs from both pools
+  loadBalancerIPs: true
+  # Scope to all nodes (no node selector = cluster-wide)
+  nodeSelector:
+    matchLabels: {}
+  interfaces:
+    - eth0

talos/talhelper/talconfig.yaml

@@ -1,188 +1,183 @@
 # ─── fastpass Talos Cluster Configuration ────────────────────────────────────
-# Managed by talhelper on city-hall (10.1.71.38)
+# cluster:     fastpass
-# Generate configs:  talhelper genconfig
+# managed by:  talhelper on city-hall (10.1.71.38)
-# Generate secrets:  talhelper gensecret > talenv.yaml && sops -e -i talenv.yaml
+# repo:        talos/talhelper/ in rblundon/homelab on mad-tea-party
-# Apply configs:     talhelper gencommand apply | bash
+#
-# Bootstrap:         talhelper gencommand bootstrap | bash
+# Quickref:
+#   talhelper genconfig                              # regenerate clusterconfig/
+#   talhelper gencommand apply --extra-flags="--insecure" | bash
+#   talhelper gencommand bootstrap | bash
+# ─────────────────────────────────────────────────────────────────────────────
 
 clusterName: fastpass
 talosVersion: v1.13.2
 kubernetesVersion: v1.32.3
 
-# ─── Cluster Endpoint ────────────────────────────────────────────────────────
+# VIP floats across control plane nodes — all kubectl/talosctl traffic lands here
-# Points at the Talos native VIP (10.1.71.65) which floats across
-# control plane nodes. All kubectl and talosctl traffic goes here.
 endpoint: https://10.1.71.65:6443
 
-# ─── Domain ──────────────────────────────────────────────────────────────────
+# ─── Cluster-wide patches ────────────────────────────────────────────────────
-domain: cluster.local
-
-# ─── CNI ─────────────────────────────────────────────────────────────────────
-# Cilium is installed post-bootstrap via Helm.
-# Talos ships with flannel by default — disable it so Cilium takes over.
-cniConfig:
-  name: none
-
-# ─── Patches applied to ALL nodes ────────────────────────────────────────────
 patches:
+  - |-
+    cluster:
+      # CRITICAL: enforce:privileged is required for Cilium.
+      # The default (enforce:baseline) blocks the capabilities Cilium needs
+      # (NET_ADMIN, NET_RAW, SYS_ADMIN). This was the root cause of the
+      # broken cluster. Do not change this back to baseline.
+      apiServer:
+        admissionControl:
+          - name: PodSecurity
+            configuration:
+              apiVersion: pod-security.admission.config.k8s.io/v1alpha1
+              kind: PodSecurityConfiguration
+              defaults:
+                enforce: "privileged"
+                enforce-version: "latest"
+                audit: "restricted"
+                audit-version: "latest"
+                warn: "restricted"
+                warn-version: "latest"
+              exemptions:
+                usernames: []
+                runtimeClasses: []
+                namespaces:
+                  - kube-system
+                  - argocd
+                  - cert-manager
+                  - ingress-nginx
+                  - external-secrets
+
   - |-
     machine:
       time:
         servers:
-          - 10.1.71.21
+          - 10.1.71.21   # sundial — local NTP
-      sysctls:
+
-        net.ipv4.ip_forward: "1"
+      # DNS search domain so short names resolve inside the cluster
-      kernel:
+      network:
-        modules:
+        extraHostEntries: []
-          - name: br_netfilter
+
-          - name: ip_tables
+      # Talos manages kubelet — these flags are additive
       kubelet:
         extraArgs:
-          rotate-server-certificates: "true"
+          rotate-server-certificates: true
+
+      # Feature gates — enable Gateway API support for Cilium's secondary ingress
+      features:
+        kubePrism:
+          enabled: true
+          port: 7445
+
+# ─── Control plane patches ───────────────────────────────────────────────────
+controlPlane:
+  patches:
+    - |-
+      cluster:
+        etcd:
+          advertisedSubnets:
+            - 10.1.71.0/24
+        # Allow control plane nodes to schedule workloads (3-node CP on small cluster)
+        scheduler:
+          config:
+            apiVersion: kubescheduler.config.k8s.io/v1
+            kind: KubeSchedulerConfiguration
+            profiles:
+              - schedulerName: default-scheduler
+
+    - |-
+      machine:
+        network:
+          # VIP — floats across all three control plane nodes
+          # Talos handles VIP natively; no keepalived needed
+          interfaces:
+            - interface: eth0
+              vip:
+                ip: 10.1.71.65
 
 # ─── Nodes ───────────────────────────────────────────────────────────────────
-
 nodes:
-
+  # ── Control plane ──────────────────────────────────────────────────────────
-  # ── Control Plane ──────────────────────────────────────────────────────────
-
   - hostname: space-mountain
     ipAddress: 10.1.71.66
     controlPlane: true
     installDisk: /dev/sda
     networkInterfaces:
-      - interface: ens18
+      - interface: eth0
         addresses:
           - 10.1.71.66/24
         routes:
           - network: 0.0.0.0/0
             gateway: 10.1.71.1
+        dhcp: false
         vip:
           ip: 10.1.71.65
-    patches:
-      - |-
-        machine:
-          network:
-            nameservers:
-              - 10.1.71.32
 
   - hostname: big-thunder-mountain
     ipAddress: 10.1.71.67
     controlPlane: true
     installDisk: /dev/sda
     networkInterfaces:
-      - interface: ens18
+      - interface: eth0
         addresses:
           - 10.1.71.67/24
         routes:
           - network: 0.0.0.0/0
             gateway: 10.1.71.1
-    patches:
+        dhcp: false
-      - |-
+        vip:
-        machine:
+          ip: 10.1.71.65
-          network:
-            nameservers:
-              - 10.1.71.32
 
   - hostname: splash-mountain
     ipAddress: 10.1.71.68
     controlPlane: true
     installDisk: /dev/sda
     networkInterfaces:
-      - interface: ens18
+      - interface: eth0
         addresses:
           - 10.1.71.68/24
         routes:
           - network: 0.0.0.0/0
             gateway: 10.1.71.1
-    patches:
+        dhcp: false
-      - |-
+        vip:
-        machine:
+          ip: 10.1.71.65
-          network:
-            nameservers:
-              - 10.1.71.32
 
   # ── Workers ────────────────────────────────────────────────────────────────
-
   - hostname: jungle-cruise
     ipAddress: 10.1.71.69
     controlPlane: false
     installDisk: /dev/sda
     networkInterfaces:
-      - interface: ens18
+      - interface: eth0
         addresses:
           - 10.1.71.69/24
         routes:
           - network: 0.0.0.0/0
             gateway: 10.1.71.1
-    patches:
+        dhcp: false
-      - |-
-        machine:
-          network:
-            nameservers:
-              - 10.1.71.32
 
   - hostname: haunted-mansion
     ipAddress: 10.1.71.70
     controlPlane: false
     installDisk: /dev/sda
     networkInterfaces:
-      - interface: ens18
+      - interface: eth0
         addresses:
           - 10.1.71.70/24
         routes:
           - network: 0.0.0.0/0
             gateway: 10.1.71.1
-    patches:
+        dhcp: false
-      - |-
-        machine:
-          network:
-            nameservers:
-              - 10.1.71.32
 
   - hostname: peter-pans-flight
     ipAddress: 10.1.71.71
     controlPlane: false
     installDisk: /dev/sda
     networkInterfaces:
-      - interface: ens18
+      - interface: eth0
         addresses:
           - 10.1.71.71/24
         routes:
           - network: 0.0.0.0/0
             gateway: 10.1.71.1
-    patches:
+        dhcp: false
-      - |-
-        machine:
-          network:
-            nameservers:
-              - 10.1.71.32
-
-# ─── Control Plane Config ────────────────────────────────────────────────────
-controlPlane:
-  patches:
-    - |-
-      machine:
-        features:
-          kubernetesTalosAPIAccess:
-            enabled: true
-            allowedRoles:
-              - os:reader
-            allowedKubernetesNamespaces:
-              - kube-system
-      cluster:
-        etcd:
-          advertisedSubnets:
-            - 10.1.71.64/26
-        coreDNS:
-          disabled: false
-        proxy:
-          disabled: true
-
-# ─── Worker Config ───────────────────────────────────────────────────────────
-worker:
-  patches:
-    - |-
-      machine:
-        kubelet:
-          extraArgs: {}