From 99958979d613a8708333be9a90368b5ef58483f0 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Thu, 18 Jun 2026 22:38:33 -0500 Subject: [PATCH] Add Talos upgrade and system extensions documentation - Add comprehensive UPGRADES-AND-EXTENSIONS.md guide covering: - System extensions via schematics and Image Factory - Talos version upgrade procedures (control plane + workers) - Kubernetes version upgrades - Rolling upgrade best practices - Troubleshooting common upgrade issues - Add rolling-upgrade-workers.sh script for automated worker upgrades - Includes safe wait times and confirmation prompts --- talos/UPGRADES-AND-EXTENSIONS.md | 406 +++++++++++++++++++++ talos/talhelper/rolling-upgrade-workers.sh | 66 ++++ 2 files changed, 472 insertions(+) create mode 100644 talos/UPGRADES-AND-EXTENSIONS.md create mode 100755 talos/talhelper/rolling-upgrade-workers.sh diff --git a/talos/UPGRADES-AND-EXTENSIONS.md b/talos/UPGRADES-AND-EXTENSIONS.md new file mode 100644 index 0000000..6259b23 --- /dev/null +++ b/talos/UPGRADES-AND-EXTENSIONS.md @@ -0,0 +1,406 @@ +# Talos Linux Upgrades and System Extensions + +Guide for upgrading Talos versions and adding system extensions to the fastpass cluster. + +## Table of Contents + +- [System Extensions](#system-extensions) +- [Upgrading Talos Versions](#upgrading-talos-versions) +- [Rolling Upgrades (Workers)](#rolling-upgrades-workers) +- [Troubleshooting](#troubleshooting) + +--- + +## System Extensions + +System extensions in Talos v1.13+ are baked into custom boot images via **schematics** and the Talos Image Factory, not loaded at runtime. + +### Adding Extensions + +Edit `talhelper/talconfig.yaml` and add extensions to the worker or control plane section: + +```yaml +# ─── Worker patches ────────────────────────────────────────────────────────── +worker: + schematic: + customization: + systemExtensions: + officialExtensions: + - siderolabs/iscsi-tools + - siderolabs/util-linux-tools + patches: + - |- + machine: + kernel: + modules: + - name: iscsi_tcp + - name: dm_multipath +``` + +**Common extensions:** +- `siderolabs/iscsi-tools` - iSCSI initiator utilities +- `siderolabs/util-linux-tools` - Multipath and storage utilities +- `siderolabs/qemu-guest-agent` - Proxmox/QEMU guest agent +- `siderolabs/nvidia-container-toolkit` - NVIDIA GPU support + +**Browse available extensions:** +```bash +curl -s https://factory.talos.dev/schematics | jq '.extensions[] | .name' +``` + +### Applying Extensions + +Extensions require an **upgrade** (new OS image), not `apply-config`: + +```bash +cd /opt/git/homelab/talos/talhelper + +# 1. Generate configs with schematic +talhelper genconfig + +# 2. Generate upgrade commands +talhelper gencommand upgrade --extra-flags="--preserve" + +# 3. Copy the output commands and run them one node at a time +# Example output: +# talosctl upgrade --nodes 10.1.71.69 --image factory.talos.dev/installer/:v1.13.2 --preserve +``` + +**Manual rolling upgrade for workers:** + +```bash +# Worker 1 +talosctl upgrade --nodes 10.1.71.69 \ + --image factory.talos.dev/installer/:v1.13.2 \ + --preserve +kubectl wait --for=condition=Ready node/jungle-cruise --timeout=10m +sleep 30 + +# Worker 2 +talosctl upgrade --nodes 10.1.71.70 \ + --image factory.talos.dev/installer/:v1.13.2 \ + --preserve +kubectl wait --for=condition=Ready node/haunted-mansion --timeout=10m +sleep 30 + +# Worker 3 +talosctl upgrade --nodes 10.1.71.71 \ + --image factory.talos.dev/installer/:v1.13.2 \ + --preserve +kubectl wait --for=condition=Ready node/peter-pans-flight --timeout=10m +``` + +**Verify extensions:** + +```bash +# List installed extensions +talosctl -n 10.1.71.69 get extensions + +# Verify specific modules +talosctl -n 10.1.71.69 read /proc/modules | grep iscsi +``` + +--- + +## Upgrading Talos Versions + +### Pre-Upgrade Checklist + +1. **Review release notes:** https://github.com/siderolabs/talos/releases +2. **Check Kubernetes compatibility:** Ensure your K8s version is supported +3. **Backup etcd:** Control plane data +4. **Take VM snapshots:** If running on Proxmox/vSphere +5. **Check extension compatibility:** Ensure extensions exist for new Talos version + +### Upgrade Process + +#### Step 1: Update talconfig.yaml + +```yaml +clusterName: fastpass +talosVersion: v1.14.0 # <- Update this +kubernetesVersion: v1.32.3 +``` + +#### Step 2: Regenerate Configs + +```bash +cd /opt/git/homelab/talos/talhelper +talhelper genconfig +``` + +#### Step 3: Upgrade Control Plane (One at a Time) + +```bash +# Generate upgrade commands +talhelper gencommand upgrade --extra-flags="--preserve" + +# Control plane node 1 +talosctl upgrade --nodes 10.1.71.66 \ + --image factory.talos.dev/installer/:v1.14.0 \ + --preserve +kubectl wait --for=condition=Ready node/space-mountain --timeout=10m +sleep 60 + +# Control plane node 2 +talosctl upgrade --nodes 10.1.71.67 \ + --image factory.talos.dev/installer/:v1.14.0 \ + --preserve +kubectl wait --for=condition=Ready node/big-thunder-mountain --timeout=10m +sleep 60 + +# Control plane node 3 +talosctl upgrade --nodes 10.1.71.68 \ + --image factory.talos.dev/installer/:v1.14.0 \ + --preserve +kubectl wait --for=condition=Ready node/splash-mountain --timeout=10m +sleep 60 +``` + +**IMPORTANT:** Always upgrade control plane nodes **one at a time** with sufficient wait between each. Etcd requires quorum (2 of 3 nodes) to remain operational. + +#### Step 4: Upgrade Workers (Rolling) + +After all control plane nodes are upgraded: + +```bash +# Worker nodes (can be faster since no etcd) +talosctl upgrade --nodes 10.1.71.69 \ + --image factory.talos.dev/installer/:v1.14.0 \ + --preserve +kubectl wait --for=condition=Ready node/jungle-cruise --timeout=10m +sleep 30 + +talosctl upgrade --nodes 10.1.71.70 \ + --image factory.talos.dev/installer/:v1.14.0 \ + --preserve +kubectl wait --for=condition=Ready node/haunted-mansion --timeout=10m +sleep 30 + +talosctl upgrade --nodes 10.1.71.71 \ + --image factory.talos.dev/installer/:v1.14.0 \ + --preserve +kubectl wait --for=condition=Ready node/peter-pans-flight --timeout=10m +``` + +#### Step 5: Verify Cluster + +```bash +# Check all nodes +kubectl get nodes -o wide + +# Verify Talos version +talosctl -n 10.1.71.66,10.1.71.67,10.1.71.68 version + +# Check cluster health +kubectl get pods -A | grep -v Running +talosctl -n 10.1.71.66 service kubelet status +``` + +--- + +## Rolling Upgrades (Workers) + +### Automated Rolling Upgrade Script + +```bash +#!/bin/bash +# rolling-upgrade-workers.sh + +set -e + +IMAGE="$1" +if [ -z "$IMAGE" ]; then + echo "Usage: $0 " + echo "Example: $0 factory.talos.dev/installer/abc123:v1.14.0" + exit 1 +fi + +declare -A WORKERS=( + ["jungle-cruise"]="10.1.71.69" + ["haunted-mansion"]="10.1.71.70" + ["peter-pans-flight"]="10.1.71.71" +) + +for worker in "${!WORKERS[@]}"; do + ip="${WORKERS[$worker]}" + echo "==========================================" + echo "Upgrading $worker ($ip)..." + echo "==========================================" + + talosctl upgrade --nodes ${ip} --image ${IMAGE} --preserve + + echo "Waiting for $worker to be Ready..." + kubectl wait --for=condition=Ready node/${worker} --timeout=10m + + echo "$worker upgrade complete. Sleeping 30s before next node..." + sleep 30 +done + +echo "==========================================" +echo "All workers upgraded successfully!" +echo "==========================================" +``` + +**Usage:** + +```bash +cd /opt/git/homelab/talos/talhelper +chmod +x rolling-upgrade-workers.sh + +# Get the image URL from talhelper +IMAGE=$(grep "image:" clusterconfig/fastpass-jungle-cruise.yaml | head -1 | awk '{print $2}') + +# Run the rolling upgrade +./rolling-upgrade-workers.sh "$IMAGE" +``` + +--- + +## Upgrading Kubernetes Version + +Talos manages Kubernetes. To upgrade K8s: + +### Step 1: Update talconfig.yaml + +```yaml +clusterName: fastpass +talosVersion: v1.14.0 +kubernetesVersion: v1.33.0 # <- Update this +``` + +### Step 2: Regenerate and Apply + +```bash +cd /opt/git/homelab/talos/talhelper +talhelper genconfig + +# Apply config changes (does NOT require node reboot) +talosctl -n 10.1.71.66,10.1.71.67,10.1.71.68 upgrade-k8s --to v1.33.0 +``` + +**The `upgrade-k8s` command:** +- Upgrades control plane components (apiserver, scheduler, controller-manager) first +- Then upgrades kubelet on all nodes +- Does NOT reboot nodes (in-place upgrade) +- Automatic rolling update + +**Verify:** + +```bash +kubectl get nodes -o wide +# Check both Talos version and Kubernetes version columns +``` + +--- + +## Troubleshooting + +### Extension Not Found Error + +**Error:** `module not found` when loading kernel modules + +**Cause:** Module isn't in the base Talos image or extension wasn't applied + +**Fix:** +1. Check if extension exists: `curl -s https://factory.talos.dev/schematics | jq '.extensions[] | select(.name | contains("iscsi"))'` +2. Verify schematic in config: `grep -A10 "schematic:" talhelper/talconfig.yaml` +3. Confirm extension is loaded: `talosctl -n get extensions` + +### Upgrade Fails with "invalid reference format" + +**Error:** `failed to parse image reference` + +**Cause:** Incorrect image URL format or missing schematic + +**Fix:** +1. Manually check generated image URL: `grep "image:" clusterconfig/fastpass-.yaml` +2. Use talhelper's upgrade command generator: `talhelper gencommand upgrade` +3. Ensure format is: `factory.talos.dev/installer/:` + +### Node Stuck in "NotReady" After Upgrade + +**Symptoms:** Node shows NotReady for >5 minutes post-upgrade + +**Debug:** +```bash +# Check node status +kubectl describe node + +# Check Talos services +talosctl -n service kubelet status +talosctl -n service containerd status + +# Check logs +talosctl -n logs kubelet +talosctl -n dmesg | tail -50 +``` + +**Common causes:** +- Network policy blocking CNI (Cilium) +- iSCSI/storage mounts failing +- Kernel module load failure +- Certificate rotation issues + +### Control Plane Quorum Lost + +**Symptoms:** `etcdserver: no leader` errors, apiserver unavailable + +**Prevention:** +- **NEVER** upgrade more than 1 control plane node simultaneously +- Always wait for full Ready status + 60s between CP upgrades +- Verify etcd health before proceeding: `talosctl -n 10.1.71.66 service etcd status` + +**Recovery:** +If quorum is lost (2 of 3 nodes down): +```bash +# Wait for at least 2 nodes to come back online +# Check etcd cluster status +talosctl -n 10.1.71.66,10.1.71.67,10.1.71.68 service etcd status + +# If completely broken, may require etcd disaster recovery +# See: https://www.talos.dev/v1.13/advanced/disaster-recovery/ +``` + +--- + +## Quick Reference + +### Check Current Versions + +```bash +# Talos version on all nodes +talosctl -n 10.1.71.66,10.1.71.67,10.1.71.68,10.1.71.69,10.1.71.70,10.1.71.71 version + +# Kubernetes version +kubectl version + +# Installed extensions +talosctl -n 10.1.71.69 get extensions +``` + +### Safe Upgrade Order + +1. ✅ Control plane node 1 → wait → verify +2. ✅ Control plane node 2 → wait → verify +3. ✅ Control plane node 3 → wait → verify +4. ✅ Worker nodes (can be parallel or rolling) + +### Config Changes vs. OS Upgrades + +| Change Type | Command | Reboot Required? | +|-------------|---------|------------------| +| Network config, sysctls, files | `talosctl apply-config --mode=reboot` | Yes | +| Kubernetes version | `talosctl upgrade-k8s` | No | +| Talos version | `talosctl upgrade` | Yes | +| System extensions | `talosctl upgrade` | Yes | + +--- + +## Links + +- Talos Releases: https://github.com/siderolabs/talos/releases +- Talos Image Factory: https://factory.talos.dev +- Extension Catalog: https://github.com/siderolabs/extensions +- Upgrade Guide: https://www.talos.dev/v1.13/talos-guides/upgrading-talos/ +- talhelper Docs: https://budimanjojo.github.io/talhelper/latest/ diff --git a/talos/talhelper/rolling-upgrade-workers.sh b/talos/talhelper/rolling-upgrade-workers.sh new file mode 100755 index 0000000..992e005 --- /dev/null +++ b/talos/talhelper/rolling-upgrade-workers.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Rolling upgrade script for fastpass worker nodes +# Usage: ./rolling-upgrade-workers.sh +# Example: ./rolling-upgrade-workers.sh factory.talos.dev/installer/abc123:v1.14.0 + +set -e + +IMAGE="$1" +if [ -z "$IMAGE" ]; then + echo "Usage: $0 " + echo "" + echo "Example:" + echo " $0 factory.talos.dev/installer/abc123def456:v1.14.0" + echo "" + echo "To get the image URL, run:" + echo " cd /opt/git/homelab/talos/talhelper" + echo " talhelper genconfig" + echo " grep 'image:' clusterconfig/fastpass-jungle-cruise.yaml | head -1" + exit 1 +fi + +declare -A WORKERS=( + ["jungle-cruise"]="10.1.71.69" + ["haunted-mansion"]="10.1.71.70" + ["peter-pans-flight"]="10.1.71.71" +) + +echo "==========================================" +echo "Rolling Worker Upgrade" +echo "==========================================" +echo "Image: $IMAGE" +echo "Workers: ${!WORKERS[@]}" +echo "" +read -p "Proceed with rolling upgrade? (y/N): " -n 1 -r +echo +if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 0 +fi + +for worker in jungle-cruise haunted-mansion peter-pans-flight; do + ip="${WORKERS[$worker]}" + echo "" + echo "==========================================" + echo "Upgrading $worker ($ip)..." + echo "==========================================" + + talosctl upgrade --nodes ${ip} --image ${IMAGE} --preserve + + echo "Waiting for $worker to be Ready..." + kubectl wait --for=condition=Ready node/${worker} --timeout=10m + + echo "✓ $worker upgrade complete." + + if [ "$worker" != "peter-pans-flight" ]; then + echo "Sleeping 30s before next node..." + sleep 30 + fi +done + +echo "" +echo "==========================================" +echo "✓ All workers upgraded successfully!" +echo "==========================================" +echo "" +kubectl get nodes -o wide