e8303d5129
Root cause: Previous config violated boot-time security model - Removed /etc/iscsi mount (iscsi-tools extension manages it) - Moved multipath.conf to post-boot DaemonSet - Added explicit kubelet nodeIP for dual-NIC workers Deliverables: - Fixed talconfig.yaml with working worker patch - iscsi-multipath-init.yaml DaemonSet for multipath config - Automated deployment and verification scripts - Complete documentation suite Ready for production deployment to fastpass worker nodes. Co-authored-by: Talos Specialist <subagent@hermes>
231 lines
8.1 KiB
Bash
Executable File
231 lines
8.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# apply-iscsi-fix.sh
|
|
#
|
|
# Applies the fixed iSCSI configuration to Talos worker nodes one at a time.
|
|
# This script prevents the boot failure that occurred with the previous config.
|
|
#
|
|
# Usage:
|
|
# ./apply-iscsi-fix.sh [--dry-run]
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$SCRIPT_DIR"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
DRY_RUN=false
|
|
if [[ "${1:-}" == "--dry-run" ]]; then
|
|
DRY_RUN=true
|
|
echo -e "${YELLOW}DRY RUN MODE - no changes will be applied${NC}\n"
|
|
fi
|
|
|
|
WORKER_NODES=(
|
|
"jungle-cruise:10.1.71.69"
|
|
"haunted-mansion:10.1.71.70"
|
|
"peter-pans-flight:10.1.71.71"
|
|
)
|
|
|
|
echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
|
|
echo -e "${BLUE}║ Talos iSCSI Configuration Fix - Worker Node Update ║${NC}"
|
|
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
|
|
echo
|
|
|
|
echo -e "${YELLOW}This script will:${NC}"
|
|
echo " 1. Regenerate Talos config with fixed iSCSI settings"
|
|
echo " 2. Apply config to each worker node sequentially"
|
|
echo " 3. Wait for each node to reboot and become Ready"
|
|
echo " 4. Verify iSCSI functionality on each node"
|
|
echo " 5. Deploy multipath configuration DaemonSet"
|
|
echo
|
|
|
|
echo -e "${YELLOW}Key fixes in this update:${NC}"
|
|
echo " ✓ Removed /etc/iscsi mount (iscsi-tools extension manages it)"
|
|
echo " ✓ Added nodeIP.validSubnets to fix dual-NIC node IP selection"
|
|
echo " ✓ Removed multipath.conf file writing at boot time"
|
|
echo " ✓ Added 'rw' option to /var/lib/iscsi mount"
|
|
echo
|
|
|
|
if [[ "$DRY_RUN" == "false" ]]; then
|
|
read -p "Continue with worker node updates? (yes/no): " -r
|
|
if [[ ! $REPLY =~ ^[Yy]es$ ]]; then
|
|
echo "Aborted."
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
echo -e "${GREEN}Step 1: Regenerating Talos configuration...${NC}"
|
|
if [[ "$DRY_RUN" == "false" ]]; then
|
|
talhelper genconfig
|
|
echo "✓ Configuration generated in clusterconfig/"
|
|
else
|
|
echo "[DRY RUN] Would run: talhelper genconfig"
|
|
fi
|
|
|
|
echo
|
|
echo -e "${GREEN}Step 2: Applying configuration to worker nodes...${NC}"
|
|
for node_spec in "${WORKER_NODES[@]}"; do
|
|
IFS=':' read -r hostname ip <<< "$node_spec"
|
|
|
|
echo
|
|
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
|
echo -e "${BLUE}Processing: $hostname ($ip)${NC}"
|
|
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
|
|
|
config_file="clusterconfig/fastpass-${hostname}.yaml"
|
|
|
|
if [[ ! -f "$config_file" ]]; then
|
|
echo -e "${RED}✗ Config file not found: $config_file${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
echo "Applying configuration..."
|
|
if [[ "$DRY_RUN" == "false" ]]; then
|
|
talosctl apply-config --file "$config_file" --nodes "$ip"
|
|
echo "✓ Configuration applied, node will reboot"
|
|
else
|
|
echo "[DRY RUN] Would run: talosctl apply-config --file $config_file --nodes $ip"
|
|
fi
|
|
|
|
echo
|
|
echo "Waiting for node to reboot and become Ready..."
|
|
if [[ "$DRY_RUN" == "false" ]]; then
|
|
# Give node time to start rebooting
|
|
sleep 30
|
|
|
|
# Wait up to 10 minutes for node to be Ready
|
|
timeout=600
|
|
elapsed=0
|
|
while [[ $elapsed -lt $timeout ]]; do
|
|
if kubectl wait --for=condition=Ready "node/$hostname" --timeout=10s 2>/dev/null; then
|
|
echo -e "${GREEN}✓ Node $hostname is Ready${NC}"
|
|
break
|
|
fi
|
|
elapsed=$((elapsed + 10))
|
|
echo -n "."
|
|
done
|
|
|
|
if [[ $elapsed -ge $timeout ]]; then
|
|
echo
|
|
echo -e "${RED}✗ Node $hostname did not become Ready within 10 minutes${NC}"
|
|
echo "Check node status with: talosctl -n $ip dmesg | tail -100"
|
|
exit 1
|
|
fi
|
|
else
|
|
echo "[DRY RUN] Would wait for node/$hostname to become Ready"
|
|
fi
|
|
|
|
echo
|
|
echo "Verifying iSCSI configuration..."
|
|
if [[ "$DRY_RUN" == "false" ]]; then
|
|
echo -n " Checking iscsid service... "
|
|
if talosctl -n "$ip" service iscsid 2>/dev/null | grep -q "STATE.*Running"; then
|
|
echo -e "${GREEN}✓${NC}"
|
|
else
|
|
echo -e "${YELLOW}⚠ not running (will start when needed)${NC}"
|
|
fi
|
|
|
|
echo -n " Checking kernel modules... "
|
|
if talosctl -n "$ip" read /proc/modules 2>/dev/null | grep -q "iscsi_tcp"; then
|
|
echo -e "${GREEN}✓${NC}"
|
|
else
|
|
echo -e "${RED}✗ iscsi_tcp not loaded${NC}"
|
|
fi
|
|
|
|
echo -n " Checking initiator name... "
|
|
if talosctl -n "$ip" read /etc/iscsi/initiatorname.iscsi 2>/dev/null | grep -q "InitiatorName="; then
|
|
echo -e "${GREEN}✓${NC}"
|
|
else
|
|
echo -e "${RED}✗ initiator name not set${NC}"
|
|
fi
|
|
|
|
echo -n " Checking /var/lib/iscsi... "
|
|
if talosctl -n "$ip" ls /var/lib/iscsi >/dev/null 2>&1; then
|
|
echo -e "${GREEN}✓${NC}"
|
|
else
|
|
echo -e "${YELLOW}⚠ directory not accessible${NC}"
|
|
fi
|
|
|
|
echo -n " Checking node IP... "
|
|
node_ip=$(kubectl get node "$hostname" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
|
|
if [[ "$node_ip" == "10.1.71."* ]]; then
|
|
echo -e "${GREEN}✓ $node_ip (correct network)${NC}"
|
|
else
|
|
echo -e "${RED}✗ $node_ip (should be 10.1.71.x)${NC}"
|
|
fi
|
|
else
|
|
echo "[DRY RUN] Would verify iSCSI on $hostname"
|
|
fi
|
|
|
|
echo
|
|
echo -e "${GREEN}✓ Node $hostname update complete${NC}"
|
|
|
|
# Brief pause before next node
|
|
if [[ "$DRY_RUN" == "false" ]]; then
|
|
sleep 10
|
|
fi
|
|
done
|
|
|
|
echo
|
|
echo -e "${GREEN}Step 3: Deploying multipath configuration DaemonSet...${NC}"
|
|
if [[ "$DRY_RUN" == "false" ]]; then
|
|
kubectl apply -f iscsi-multipath-init.yaml
|
|
echo "Waiting for DaemonSet to run on all workers..."
|
|
kubectl rollout status daemonset/iscsi-multipath-init -n kube-system --timeout=5m
|
|
echo "✓ Multipath configuration applied to all nodes"
|
|
else
|
|
echo "[DRY RUN] Would run: kubectl apply -f iscsi-multipath-init.yaml"
|
|
fi
|
|
|
|
echo
|
|
echo -e "${GREEN}Step 4: Final verification...${NC}"
|
|
if [[ "$DRY_RUN" == "false" ]]; then
|
|
echo
|
|
echo "Worker node status:"
|
|
kubectl get nodes -l node-role.kubernetes.io/worker --show-labels | grep -E "NAME|jungle-cruise|haunted-mansion|peter-pans-flight"
|
|
|
|
echo
|
|
echo "Multipath DaemonSet status:"
|
|
kubectl get pods -n kube-system -l app=iscsi-multipath-init -o wide
|
|
|
|
echo
|
|
echo -e "${YELLOW}Multipath configuration on each node:${NC}"
|
|
for node_spec in "${WORKER_NODES[@]}"; do
|
|
IFS=':' read -r hostname ip <<< "$node_spec"
|
|
echo
|
|
echo "=== $hostname ($ip) ==="
|
|
talosctl -n "$ip" read /etc/multipath.conf 2>/dev/null || echo " multipath.conf not found (will be created by DaemonSet)"
|
|
echo
|
|
echo "Multipath devices:"
|
|
talosctl -n "$ip" exec -- multipath -ll 2>/dev/null || echo " (no multipath devices yet)"
|
|
done
|
|
else
|
|
echo "[DRY RUN] Would show final verification output"
|
|
fi
|
|
|
|
echo
|
|
echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
|
|
echo -e "${BLUE}║ Update Complete! ✓ ║${NC}"
|
|
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
|
|
echo
|
|
|
|
echo -e "${GREEN}All worker nodes have been updated with fixed iSCSI configuration.${NC}"
|
|
echo
|
|
echo -e "${YELLOW}Next steps:${NC}"
|
|
echo " 1. Deploy Portworx CSI driver (if not already deployed)"
|
|
echo " 2. Verify iSCSI discovery to FlashArray:"
|
|
echo " talosctl -n 10.1.71.69 exec -- iscsiadm -m discovery -t st -p <flasharray-iscsi-ip>"
|
|
echo " 3. Create test PVC using pure-block StorageClass"
|
|
echo " 4. Monitor Portworx pod logs for iSCSI session establishment"
|
|
echo
|
|
|
|
if [[ "$DRY_RUN" == "false" ]]; then
|
|
echo "Configuration details documented in: ISCSI_CONFIG_FIX.md"
|
|
fi
|