#!/usr/bin/env bash
# apply-iscsi-fix.sh
# 
# Applies the fixed iSCSI configuration to Talos worker nodes one at a time.
# This script prevents the boot failure that occurred with the previous config.
#
# Usage:
#   ./apply-iscsi-fix.sh [--dry-run]

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$SCRIPT_DIR"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

DRY_RUN=false
if [[ "${1:-}" == "--dry-run" ]]; then
  DRY_RUN=true
  echo -e "${YELLOW}DRY RUN MODE - no changes will be applied${NC}\n"
fi

WORKER_NODES=(
  "jungle-cruise:10.1.71.69"
  "haunted-mansion:10.1.71.70"
  "peter-pans-flight:10.1.71.71"
)

echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║  Talos iSCSI Configuration Fix - Worker Node Update       ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
echo

echo -e "${YELLOW}This script will:${NC}"
echo "  1. Regenerate Talos config with fixed iSCSI settings"
echo "  2. Apply config to each worker node sequentially"
echo "  3. Wait for each node to reboot and become Ready"
echo "  4. Verify iSCSI functionality on each node"
echo "  5. Deploy multipath configuration DaemonSet"
echo

echo -e "${YELLOW}Key fixes in this update:${NC}"
echo "  ✓ Removed /etc/iscsi mount (iscsi-tools extension manages it)"
echo "  ✓ Added nodeIP.validSubnets to fix dual-NIC node IP selection"
echo "  ✓ Removed multipath.conf file writing at boot time"
echo "  ✓ Added 'rw' option to /var/lib/iscsi mount"
echo

if [[ "$DRY_RUN" == "false" ]]; then
  read -p "Continue with worker node updates? (yes/no): " -r
  if [[ ! $REPLY =~ ^[Yy]es$ ]]; then
    echo "Aborted."
    exit 0
  fi
fi

echo
echo -e "${GREEN}Step 1: Regenerating Talos configuration...${NC}"
if [[ "$DRY_RUN" == "false" ]]; then
  talhelper genconfig
  echo "✓ Configuration generated in clusterconfig/"
else
  echo "[DRY RUN] Would run: talhelper genconfig"
fi

echo
echo -e "${GREEN}Step 2: Applying configuration to worker nodes...${NC}"
for node_spec in "${WORKER_NODES[@]}"; do
  IFS=':' read -r hostname ip <<< "$node_spec"
  
  echo
  echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
  echo -e "${BLUE}Processing: $hostname ($ip)${NC}"
  echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
  
  config_file="clusterconfig/fastpass-${hostname}.yaml"
  
  if [[ ! -f "$config_file" ]]; then
    echo -e "${RED}✗ Config file not found: $config_file${NC}"
    exit 1
  fi
  
  echo "Applying configuration..."
  if [[ "$DRY_RUN" == "false" ]]; then
    talosctl apply-config --file "$config_file" --nodes "$ip"
    echo "✓ Configuration applied, node will reboot"
  else
    echo "[DRY RUN] Would run: talosctl apply-config --file $config_file --nodes $ip"
  fi
  
  echo
  echo "Waiting for node to reboot and become Ready..."
  if [[ "$DRY_RUN" == "false" ]]; then
    # Give node time to start rebooting
    sleep 30
    
    # Wait up to 10 minutes for node to be Ready
    timeout=600
    elapsed=0
    while [[ $elapsed -lt $timeout ]]; do
      if kubectl wait --for=condition=Ready "node/$hostname" --timeout=10s 2>/dev/null; then
        echo -e "${GREEN}✓ Node $hostname is Ready${NC}"
        break
      fi
      elapsed=$((elapsed + 10))
      echo -n "."
    done
    
    if [[ $elapsed -ge $timeout ]]; then
      echo
      echo -e "${RED}✗ Node $hostname did not become Ready within 10 minutes${NC}"
      echo "Check node status with: talosctl -n $ip dmesg | tail -100"
      exit 1
    fi
  else
    echo "[DRY RUN] Would wait for node/$hostname to become Ready"
  fi
  
  echo
  echo "Verifying iSCSI configuration..."
  if [[ "$DRY_RUN" == "false" ]]; then
    echo -n "  Checking iscsid service... "
    if talosctl -n "$ip" service iscsid 2>/dev/null | grep -q "STATE.*Running"; then
      echo -e "${GREEN}✓${NC}"
    else
      echo -e "${YELLOW}⚠ not running (will start when needed)${NC}"
    fi
    
    echo -n "  Checking kernel modules... "
    if talosctl -n "$ip" read /proc/modules 2>/dev/null | grep -q "iscsi_tcp"; then
      echo -e "${GREEN}✓${NC}"
    else
      echo -e "${RED}✗ iscsi_tcp not loaded${NC}"
    fi
    
    echo -n "  Checking initiator name... "
    if talosctl -n "$ip" read /etc/iscsi/initiatorname.iscsi 2>/dev/null | grep -q "InitiatorName="; then
      echo -e "${GREEN}✓${NC}"
    else
      echo -e "${RED}✗ initiator name not set${NC}"
    fi
    
    echo -n "  Checking /var/lib/iscsi... "
    if talosctl -n "$ip" ls /var/lib/iscsi >/dev/null 2>&1; then
      echo -e "${GREEN}✓${NC}"
    else
      echo -e "${YELLOW}⚠ directory not accessible${NC}"
    fi
    
    echo -n "  Checking node IP... "
    node_ip=$(kubectl get node "$hostname" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
    if [[ "$node_ip" == "10.1.71."* ]]; then
      echo -e "${GREEN}✓ $node_ip (correct network)${NC}"
    else
      echo -e "${RED}✗ $node_ip (should be 10.1.71.x)${NC}"
    fi
  else
    echo "[DRY RUN] Would verify iSCSI on $hostname"
  fi
  
  echo
  echo -e "${GREEN}✓ Node $hostname update complete${NC}"
  
  # Brief pause before next node
  if [[ "$DRY_RUN" == "false" ]]; then
    sleep 10
  fi
done

echo
echo -e "${GREEN}Step 3: Deploying multipath configuration DaemonSet...${NC}"
if [[ "$DRY_RUN" == "false" ]]; then
  kubectl apply -f iscsi-multipath-init.yaml
  echo "Waiting for DaemonSet to run on all workers..."
  kubectl rollout status daemonset/iscsi-multipath-init -n kube-system --timeout=5m
  echo "✓ Multipath configuration applied to all nodes"
else
  echo "[DRY RUN] Would run: kubectl apply -f iscsi-multipath-init.yaml"
fi

echo
echo -e "${GREEN}Step 4: Final verification...${NC}"
if [[ "$DRY_RUN" == "false" ]]; then
  echo
  echo "Worker node status:"
  kubectl get nodes -l node-role.kubernetes.io/worker --show-labels | grep -E "NAME|jungle-cruise|haunted-mansion|peter-pans-flight"
  
  echo
  echo "Multipath DaemonSet status:"
  kubectl get pods -n kube-system -l app=iscsi-multipath-init -o wide
  
  echo
  echo -e "${YELLOW}Multipath configuration on each node:${NC}"
  for node_spec in "${WORKER_NODES[@]}"; do
    IFS=':' read -r hostname ip <<< "$node_spec"
    echo
    echo "=== $hostname ($ip) ==="
    talosctl -n "$ip" read /etc/multipath.conf 2>/dev/null || echo "  multipath.conf not found (will be created by DaemonSet)"
    echo
    echo "Multipath devices:"
    talosctl -n "$ip" exec -- multipath -ll 2>/dev/null || echo "  (no multipath devices yet)"
  done
else
  echo "[DRY RUN] Would show final verification output"
fi

echo
echo -e "${BLUE}╔════════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║                    Update Complete! ✓                      ║${NC}"
echo -e "${BLUE}╚════════════════════════════════════════════════════════════╝${NC}"
echo

echo -e "${GREEN}All worker nodes have been updated with fixed iSCSI configuration.${NC}"
echo
echo -e "${YELLOW}Next steps:${NC}"
echo "  1. Deploy Portworx CSI driver (if not already deployed)"
echo "  2. Verify iSCSI discovery to FlashArray:"
echo "     talosctl -n 10.1.71.69 exec -- iscsiadm -m discovery -t st -p <flasharray-iscsi-ip>"
echo "  3. Create test PVC using pure-block StorageClass"
echo "  4. Monitor Portworx pod logs for iSCSI session establishment"
echo

if [[ "$DRY_RUN" == "false" ]]; then
  echo "Configuration details documented in: ISCSI_CONFIG_FIX.md"
fi