Fix Talos iSCSI configuration for Portworx CSI
Root cause: Previous config violated boot-time security model - Removed /etc/iscsi mount (iscsi-tools extension manages it) - Moved multipath.conf to post-boot DaemonSet - Added explicit kubelet nodeIP for dual-NIC workers Deliverables: - Fixed talconfig.yaml with working worker patch - iscsi-multipath-init.yaml DaemonSet for multipath config - Automated deployment and verification scripts - Complete documentation suite Ready for production deployment to fastpass worker nodes. Co-authored-by: Talos Specialist <subagent@hermes>
This commit is contained in:
Hermes Agent service account
230
talos/talhelper/apply-iscsi-fix.sh
Executable file
230
talos/talhelper/apply-iscsi-fix.sh
Executable file
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env bash
|
||||
# apply-iscsi-fix.sh
|
||||
#
|
||||
# Applies the fixed iSCSI configuration to Talos worker nodes one at a time.
|
||||
# This script prevents the boot failure that occurred with the previous config.
|
||||
#
|
||||
# Usage:
|
||||
# ./apply-iscsi-fix.sh [--dry-run]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
DRY_RUN=false
|
||||
if [[ "${1:-}" == "--dry-run" ]]; then
|
||||
DRY_RUN=true
|
||||
echo -e "${YELLOW}DRY RUN MODE - no changes will be applied${NC}\n"
|
||||
fi
|
||||
|
||||
WORKER_NODES=(
|
||||
"jungle-cruise:10.1.71.69"
|
||||
"haunted-mansion:10.1.71.70"
|
||||
"peter-pans-flight:10.1.71.71"
|
||||
)
|
||||
|
||||
echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ Talos iSCSI Configuration Fix - Worker Node Update ║${NC}"
|
||||
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
|
||||
echo
|
||||
|
||||
echo -e "${YELLOW}This script will:${NC}"
|
||||
echo " 1. Regenerate Talos config with fixed iSCSI settings"
|
||||
echo " 2. Apply config to each worker node sequentially"
|
||||
echo " 3. Wait for each node to reboot and become Ready"
|
||||
echo " 4. Verify iSCSI functionality on each node"
|
||||
echo " 5. Deploy multipath configuration DaemonSet"
|
||||
echo
|
||||
|
||||
echo -e "${YELLOW}Key fixes in this update:${NC}"
|
||||
echo " ✓ Removed /etc/iscsi mount (iscsi-tools extension manages it)"
|
||||
echo " ✓ Added nodeIP.validSubnets to fix dual-NIC node IP selection"
|
||||
echo " ✓ Removed multipath.conf file writing at boot time"
|
||||
echo " ✓ Added 'rw' option to /var/lib/iscsi mount"
|
||||
echo
|
||||
|
||||
if [[ "$DRY_RUN" == "false" ]]; then
|
||||
read -p "Continue with worker node updates? (yes/no): " -r
|
||||
if [[ ! $REPLY =~ ^[Yy]es$ ]]; then
|
||||
echo "Aborted."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${GREEN}Step 1: Regenerating Talos configuration...${NC}"
|
||||
if [[ "$DRY_RUN" == "false" ]]; then
|
||||
talhelper genconfig
|
||||
echo "✓ Configuration generated in clusterconfig/"
|
||||
else
|
||||
echo "[DRY RUN] Would run: talhelper genconfig"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${GREEN}Step 2: Applying configuration to worker nodes...${NC}"
|
||||
for node_spec in "${WORKER_NODES[@]}"; do
|
||||
IFS=':' read -r hostname ip <<< "$node_spec"
|
||||
|
||||
echo
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${BLUE}Processing: $hostname ($ip)${NC}"
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
|
||||
config_file="clusterconfig/fastpass-${hostname}.yaml"
|
||||
|
||||
if [[ ! -f "$config_file" ]]; then
|
||||
echo -e "${RED}✗ Config file not found: $config_file${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Applying configuration..."
|
||||
if [[ "$DRY_RUN" == "false" ]]; then
|
||||
talosctl apply-config --file "$config_file" --nodes "$ip"
|
||||
echo "✓ Configuration applied, node will reboot"
|
||||
else
|
||||
echo "[DRY RUN] Would run: talosctl apply-config --file $config_file --nodes $ip"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Waiting for node to reboot and become Ready..."
|
||||
if [[ "$DRY_RUN" == "false" ]]; then
|
||||
# Give node time to start rebooting
|
||||
sleep 30
|
||||
|
||||
# Wait up to 10 minutes for node to be Ready
|
||||
timeout=600
|
||||
elapsed=0
|
||||
while [[ $elapsed -lt $timeout ]]; do
|
||||
if kubectl wait --for=condition=Ready "node/$hostname" --timeout=10s 2>/dev/null; then
|
||||
echo -e "${GREEN}✓ Node $hostname is Ready${NC}"
|
||||
break
|
||||
fi
|
||||
elapsed=$((elapsed + 10))
|
||||
echo -n "."
|
||||
done
|
||||
|
||||
if [[ $elapsed -ge $timeout ]]; then
|
||||
echo
|
||||
echo -e "${RED}✗ Node $hostname did not become Ready within 10 minutes${NC}"
|
||||
echo "Check node status with: talosctl -n $ip dmesg | tail -100"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "[DRY RUN] Would wait for node/$hostname to become Ready"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "Verifying iSCSI configuration..."
|
||||
if [[ "$DRY_RUN" == "false" ]]; then
|
||||
echo -n " Checking iscsid service... "
|
||||
if talosctl -n "$ip" service iscsid 2>/dev/null | grep -q "STATE.*Running"; then
|
||||
echo -e "${GREEN}✓${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ not running (will start when needed)${NC}"
|
||||
fi
|
||||
|
||||
echo -n " Checking kernel modules... "
|
||||
if talosctl -n "$ip" read /proc/modules 2>/dev/null | grep -q "iscsi_tcp"; then
|
||||
echo -e "${GREEN}✓${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ iscsi_tcp not loaded${NC}"
|
||||
fi
|
||||
|
||||
echo -n " Checking initiator name... "
|
||||
if talosctl -n "$ip" read /etc/iscsi/initiatorname.iscsi 2>/dev/null | grep -q "InitiatorName="; then
|
||||
echo -e "${GREEN}✓${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ initiator name not set${NC}"
|
||||
fi
|
||||
|
||||
echo -n " Checking /var/lib/iscsi... "
|
||||
if talosctl -n "$ip" ls /var/lib/iscsi >/dev/null 2>&1; then
|
||||
echo -e "${GREEN}✓${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ directory not accessible${NC}"
|
||||
fi
|
||||
|
||||
echo -n " Checking node IP... "
|
||||
node_ip=$(kubectl get node "$hostname" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
|
||||
if [[ "$node_ip" == "10.1.71."* ]]; then
|
||||
echo -e "${GREEN}✓ $node_ip (correct network)${NC}"
|
||||
else
|
||||
echo -e "${RED}✗ $node_ip (should be 10.1.71.x)${NC}"
|
||||
fi
|
||||
else
|
||||
echo "[DRY RUN] Would verify iSCSI on $hostname"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${GREEN}✓ Node $hostname update complete${NC}"
|
||||
|
||||
# Brief pause before next node
|
||||
if [[ "$DRY_RUN" == "false" ]]; then
|
||||
sleep 10
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo -e "${GREEN}Step 3: Deploying multipath configuration DaemonSet...${NC}"
|
||||
if [[ "$DRY_RUN" == "false" ]]; then
|
||||
kubectl apply -f iscsi-multipath-init.yaml
|
||||
echo "Waiting for DaemonSet to run on all workers..."
|
||||
kubectl rollout status daemonset/iscsi-multipath-init -n kube-system --timeout=5m
|
||||
echo "✓ Multipath configuration applied to all nodes"
|
||||
else
|
||||
echo "[DRY RUN] Would run: kubectl apply -f iscsi-multipath-init.yaml"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${GREEN}Step 4: Final verification...${NC}"
|
||||
if [[ "$DRY_RUN" == "false" ]]; then
|
||||
echo
|
||||
echo "Worker node status:"
|
||||
kubectl get nodes -l node-role.kubernetes.io/worker --show-labels | grep -E "NAME|jungle-cruise|haunted-mansion|peter-pans-flight"
|
||||
|
||||
echo
|
||||
echo "Multipath DaemonSet status:"
|
||||
kubectl get pods -n kube-system -l app=iscsi-multipath-init -o wide
|
||||
|
||||
echo
|
||||
echo -e "${YELLOW}Multipath configuration on each node:${NC}"
|
||||
for node_spec in "${WORKER_NODES[@]}"; do
|
||||
IFS=':' read -r hostname ip <<< "$node_spec"
|
||||
echo
|
||||
echo "=== $hostname ($ip) ==="
|
||||
talosctl -n "$ip" read /etc/multipath.conf 2>/dev/null || echo " multipath.conf not found (will be created by DaemonSet)"
|
||||
echo
|
||||
echo "Multipath devices:"
|
||||
talosctl -n "$ip" exec -- multipath -ll 2>/dev/null || echo " (no multipath devices yet)"
|
||||
done
|
||||
else
|
||||
echo "[DRY RUN] Would show final verification output"
|
||||
fi
|
||||
|
||||
echo
|
||||
echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${BLUE}║ Update Complete! ✓ ║${NC}"
|
||||
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
|
||||
echo
|
||||
|
||||
echo -e "${GREEN}All worker nodes have been updated with fixed iSCSI configuration.${NC}"
|
||||
echo
|
||||
echo -e "${YELLOW}Next steps:${NC}"
|
||||
echo " 1. Deploy Portworx CSI driver (if not already deployed)"
|
||||
echo " 2. Verify iSCSI discovery to FlashArray:"
|
||||
echo " talosctl -n 10.1.71.69 exec -- iscsiadm -m discovery -t st -p <flasharray-iscsi-ip>"
|
||||
echo " 3. Create test PVC using pure-block StorageClass"
|
||||
echo " 4. Monitor Portworx pod logs for iSCSI session establishment"
|
||||
echo
|
||||
|
||||
if [[ "$DRY_RUN" == "false" ]]; then
|
||||
echo "Configuration details documented in: ISCSI_CONFIG_FIX.md"
|
||||
fi
|
||||
Reference in New Issue
Block a user