Add Portworx CSI driver for Pure Storage FlashArray
- Deploy Portworx Operator + CSI driver via ArgoCD - Support both iSCSI block and NFS file storage from FlashArray - Integrate with 1Password External Secrets for FlashArray credentials - Include comprehensive deployment documentation and validation script - Storage classes: pure-block (iSCSI) and pure-file (NFS) - Talos Linux compatible with iSCSI/multipath configuration
This commit is contained in:
Hermes Agent service account
438
cluster/platform/portworx-csi/QUICKREF.md
Normal file
438
cluster/platform/portworx-csi/QUICKREF.md
Normal file
@@ -0,0 +1,438 @@
|
||||
# Portworx CSI Quick Reference
|
||||
|
||||
Common commands and operations for Portworx CSI with Pure Storage FlashArray.
|
||||
|
||||
## Check Status
|
||||
|
||||
```bash
|
||||
# ArgoCD application
|
||||
kubectl get application -n argocd portworx-csi
|
||||
|
||||
# All pods in portworx namespace
|
||||
kubectl get pods -n portworx
|
||||
|
||||
# StorageCluster status
|
||||
kubectl get storagecluster -n portworx px-cluster-fastpass
|
||||
kubectl describe storagecluster -n portworx px-cluster-fastpass
|
||||
|
||||
# CSI driver registration
|
||||
kubectl get csidrivers pxd.portworx.com
|
||||
kubectl get csinode
|
||||
|
||||
# StorageClasses
|
||||
kubectl get storageclass pure-block pure-file
|
||||
```
|
||||
|
||||
## Volume Operations
|
||||
|
||||
### Create PVC (Block Storage)
|
||||
```bash
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: my-app-data
|
||||
namespace: default
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: pure-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
EOF
|
||||
```
|
||||
|
||||
### Create PVC (File Storage - NFS)
|
||||
```bash
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: shared-data
|
||||
namespace: default
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
storageClassName: pure-file
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
EOF
|
||||
```
|
||||
|
||||
### Expand Volume
|
||||
```bash
|
||||
# Edit PVC to increase size
|
||||
kubectl patch pvc my-app-data -p '{"spec":{"resources":{"requests":{"storage":"20Gi"}}}}'
|
||||
|
||||
# Verify expansion
|
||||
kubectl get pvc my-app-data
|
||||
```
|
||||
|
||||
### Create Snapshot
|
||||
```bash
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: snapshot.storage.k8s.io/v1
|
||||
kind: VolumeSnapshot
|
||||
metadata:
|
||||
name: my-app-snapshot
|
||||
namespace: default
|
||||
spec:
|
||||
volumeSnapshotClassName: px-csi-snapshot-class
|
||||
source:
|
||||
persistentVolumeClaimName: my-app-data
|
||||
EOF
|
||||
```
|
||||
|
||||
### Restore from Snapshot
|
||||
```bash
|
||||
kubectl apply -f - <<EOF
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: my-app-data-restored
|
||||
namespace: default
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: pure-block
|
||||
dataSource:
|
||||
name: my-app-snapshot
|
||||
kind: VolumeSnapshot
|
||||
apiGroup: snapshot.storage.k8s.io
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
EOF
|
||||
```
|
||||
|
||||
## Logs
|
||||
|
||||
```bash
|
||||
# Portworx Operator
|
||||
kubectl logs -n portworx -l app=portworx-operator --tail=100
|
||||
|
||||
# PX-CSI Controller
|
||||
kubectl logs -n portworx -l app=px-csi-controller --tail=100
|
||||
|
||||
# PX-CSI Node driver (specific node)
|
||||
kubectl logs -n portworx -l app=px-csi-node --tail=100
|
||||
|
||||
# Follow logs in real-time
|
||||
kubectl logs -n portworx -l app=px-csi-controller -f
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Check iSCSI on Talos Nodes
|
||||
```bash
|
||||
# List iSCSI sessions
|
||||
talosctl -n <node-ip> exec -- iscsiadm -m session
|
||||
|
||||
# Discover FlashArray targets
|
||||
talosctl -n <node-ip> exec -- iscsiadm -m discovery -t st -p <flasharray-ip>
|
||||
|
||||
# Check initiator name (must be unique)
|
||||
talosctl -n <node-ip> read /etc/iscsi/initiatorname.iscsi
|
||||
```
|
||||
|
||||
### Check Multipath
|
||||
```bash
|
||||
# Multipath status
|
||||
talosctl -n <node-ip> exec -- multipath -ll
|
||||
|
||||
# Reload multipath config
|
||||
talosctl -n <node-ip> exec -- systemctl reload multipathd
|
||||
|
||||
# Multipath devices
|
||||
talosctl -n <node-ip> exec -- ls -l /dev/mapper/
|
||||
```
|
||||
|
||||
### Verify FlashArray Connectivity
|
||||
```bash
|
||||
# From PX-CSI pod
|
||||
kubectl exec -n portworx -it <px-pod> -- ping <flasharray-ip>
|
||||
|
||||
# Test API endpoint
|
||||
kubectl exec -n portworx -it <px-pod> -- \
|
||||
curl -k https://<flasharray-mgmt-ip>/api/2.0/arrays
|
||||
```
|
||||
|
||||
### Check Secret
|
||||
```bash
|
||||
# Verify secret exists
|
||||
kubectl get secret -n portworx px-pure-secret
|
||||
|
||||
# View secret content (pure.json)
|
||||
kubectl get secret -n portworx px-pure-secret \
|
||||
-o jsonpath='{.data.pure\.json}' | base64 -d | jq .
|
||||
|
||||
# Check ExternalSecret status
|
||||
kubectl get externalsecret -n portworx px-pure-secret
|
||||
kubectl describe externalsecret -n portworx px-pure-secret
|
||||
```
|
||||
|
||||
### Volume Not Attaching
|
||||
```bash
|
||||
# Check PVC events
|
||||
kubectl describe pvc <pvc-name>
|
||||
|
||||
# Check pod events
|
||||
kubectl describe pod <pod-name>
|
||||
|
||||
# Check PV
|
||||
kubectl get pv
|
||||
kubectl describe pv <pv-name>
|
||||
|
||||
# Check volume attachment
|
||||
kubectl get volumeattachment
|
||||
```
|
||||
|
||||
### Pod Stuck in ContainerCreating
|
||||
```bash
|
||||
# Check events
|
||||
kubectl describe pod <pod-name>
|
||||
|
||||
# Check PVC status
|
||||
kubectl get pvc -n <namespace>
|
||||
|
||||
# Check node where pod is scheduled
|
||||
POD_NODE=$(kubectl get pod <pod-name> -o jsonpath='{.spec.nodeName}')
|
||||
echo "Pod scheduled on: $POD_NODE"
|
||||
|
||||
# Check iSCSI on that node
|
||||
talosctl -n $POD_NODE exec -- iscsiadm -m session
|
||||
```
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Prometheus Queries
|
||||
```promql
|
||||
# Cluster status
|
||||
portworx_cluster_status
|
||||
|
||||
# Volume capacity
|
||||
portworx_volume_capacity_bytes
|
||||
|
||||
# Volume usage
|
||||
portworx_volume_usage_bytes
|
||||
|
||||
# I/O metrics
|
||||
rate(portworx_volume_read_bytes[5m])
|
||||
rate(portworx_volume_write_bytes[5m])
|
||||
```
|
||||
|
||||
### List All PVCs Using Pure Storage
|
||||
```bash
|
||||
# Block storage
|
||||
kubectl get pvc --all-namespaces -o json | \
|
||||
jq -r '.items[] | select(.spec.storageClassName == "pure-block") |
|
||||
"\(.metadata.namespace)/\(.metadata.name) - \(.status.phase)"'
|
||||
|
||||
# File storage
|
||||
kubectl get pvc --all-namespaces -o json | \
|
||||
jq -r '.items[] | select(.spec.storageClassName == "pure-file") |
|
||||
"\(.metadata.namespace)/\(.metadata.name) - \(.status.phase)"'
|
||||
```
|
||||
|
||||
### Storage Usage by Namespace
|
||||
```bash
|
||||
kubectl get pvc --all-namespaces -o json | \
|
||||
jq -r '.items[] | select(.spec.storageClassName | startswith("pure-")) |
|
||||
"\(.metadata.namespace)\t\(.spec.resources.requests.storage)"' | \
|
||||
awk '{arr[$1]+=$2} END {for (i in arr) print i, arr[i]}'
|
||||
```
|
||||
|
||||
## Maintenance
|
||||
|
||||
### Update FlashArray Credentials
|
||||
```bash
|
||||
# Update in 1Password:
|
||||
# Vault: homelab
|
||||
# Item: pure-flasharray-fastpass
|
||||
# Field: pure.json
|
||||
|
||||
# Force ExternalSecret refresh
|
||||
kubectl delete secret -n portworx px-pure-secret
|
||||
|
||||
# Wait for ExternalSecret to recreate it (check status)
|
||||
kubectl get externalsecret -n portworx px-pure-secret -w
|
||||
```
|
||||
|
||||
### Restart PX-CSI Components
|
||||
```bash
|
||||
# Restart controller
|
||||
kubectl rollout restart deployment -n portworx px-csi-controller
|
||||
|
||||
# Restart node drivers (DaemonSet)
|
||||
kubectl rollout restart daemonset -n portworx px-csi-node
|
||||
|
||||
# Check rollout status
|
||||
kubectl rollout status deployment -n portworx px-csi-controller
|
||||
kubectl rollout status daemonset -n portworx px-csi-node
|
||||
```
|
||||
|
||||
### Drain Node for Maintenance
|
||||
```bash
|
||||
# Cordon node
|
||||
kubectl cordon <node-name>
|
||||
|
||||
# Drain node (evict pods)
|
||||
kubectl drain <node-name> --ignore-daemonsets --delete-emptydir-data
|
||||
|
||||
# Perform maintenance...
|
||||
|
||||
# Uncordon node
|
||||
kubectl uncordon <node-name>
|
||||
```
|
||||
|
||||
## ArgoCD Operations
|
||||
|
||||
### Sync Application
|
||||
```bash
|
||||
# Sync via CLI
|
||||
argocd app sync portworx-csi
|
||||
|
||||
# Force sync (ignore differences)
|
||||
argocd app sync portworx-csi --force
|
||||
|
||||
# Sync specific resource
|
||||
argocd app sync portworx-csi --resource storagecluster:px-cluster-fastpass
|
||||
```
|
||||
|
||||
### View Application Details
|
||||
```bash
|
||||
# App status
|
||||
argocd app get portworx-csi
|
||||
|
||||
# App history
|
||||
argocd app history portworx-csi
|
||||
|
||||
# App diff (compare git vs cluster)
|
||||
argocd app diff portworx-csi
|
||||
```
|
||||
|
||||
### Rollback via ArgoCD
|
||||
```bash
|
||||
# List history
|
||||
argocd app history portworx-csi
|
||||
|
||||
# Rollback to previous version
|
||||
argocd app rollback portworx-csi <revision-number>
|
||||
```
|
||||
|
||||
## Configuration Updates
|
||||
|
||||
### Edit StorageCluster
|
||||
```bash
|
||||
# Edit directly (will be overwritten by ArgoCD sync)
|
||||
kubectl edit storagecluster -n portworx px-cluster-fastpass
|
||||
|
||||
# Permanent change: update git
|
||||
cd ~/git/homelab
|
||||
vi cluster/platform/portworx-csi/storagecluster.yaml
|
||||
git commit -am "Update Portworx StorageCluster config"
|
||||
git push
|
||||
```
|
||||
|
||||
### Add Environment Variable
|
||||
```yaml
|
||||
# In storagecluster.yaml, add to spec.env:
|
||||
env:
|
||||
- name: NEW_ENV_VAR
|
||||
value: "value"
|
||||
```
|
||||
|
||||
### Change Log Level
|
||||
```yaml
|
||||
# In storagecluster.yaml:
|
||||
env:
|
||||
- name: LOG_LEVEL
|
||||
value: "debug" # or: info, warn, error
|
||||
```
|
||||
|
||||
## Talos Node Configuration
|
||||
|
||||
### View Current Config
|
||||
```bash
|
||||
# Read multipath.conf
|
||||
talosctl -n <node-ip> read /etc/multipath.conf
|
||||
|
||||
# Read udev rules
|
||||
talosctl -n <node-ip> read /etc/udev/rules.d/99-pure-storage.rules
|
||||
|
||||
# Check loaded modules
|
||||
talosctl -n <node-ip> exec -- lsmod | grep -E "iscsi|multipath"
|
||||
```
|
||||
|
||||
### Apply Updated Talos Config
|
||||
```bash
|
||||
cd ~/git/homelab/talos/talhelper
|
||||
|
||||
# Regenerate configs after updating talconfig.yaml
|
||||
talhelper genconfig
|
||||
|
||||
# Apply to specific node
|
||||
talosctl apply-config \
|
||||
--file clusterconfig/<node-name>.yaml \
|
||||
--nodes <node-ip>
|
||||
|
||||
# Reboot if kernel modules changed
|
||||
talosctl reboot --nodes <node-ip>
|
||||
```
|
||||
|
||||
## Emergency Procedures
|
||||
|
||||
### Complete Reinstall
|
||||
```bash
|
||||
# 1. Delete ArgoCD application
|
||||
kubectl delete application -n argocd portworx-csi
|
||||
|
||||
# 2. Delete namespace
|
||||
kubectl delete namespace portworx
|
||||
|
||||
# 3. Delete CRDs
|
||||
kubectl delete crd storageclusters.core.libopenstorage.org
|
||||
|
||||
# 4. Delete StorageClasses
|
||||
kubectl delete storageclass pure-block pure-file
|
||||
|
||||
# 5. Re-apply via ArgoCD
|
||||
argocd app sync portworx-csi
|
||||
```
|
||||
|
||||
### Recover from Failed State
|
||||
```bash
|
||||
# 1. Check StorageCluster status
|
||||
kubectl get storagecluster -n portworx -o yaml
|
||||
|
||||
# 2. Delete and recreate StorageCluster
|
||||
kubectl delete storagecluster -n portworx px-cluster-fastpass
|
||||
|
||||
# 3. Let ArgoCD recreate it
|
||||
argocd app sync portworx-csi --force
|
||||
|
||||
# 4. Monitor deployment
|
||||
kubectl get pods -n portworx -w
|
||||
```
|
||||
|
||||
## Useful Aliases
|
||||
|
||||
Add to `~/.bashrc` or `~/.zshrc`:
|
||||
|
||||
```bash
|
||||
# Portworx aliases
|
||||
alias pxpods='kubectl get pods -n portworx'
|
||||
alias pxlogs='kubectl logs -n portworx'
|
||||
alias pxstc='kubectl get storagecluster -n portworx'
|
||||
alias pxpvc='kubectl get pvc --all-namespaces -o wide | grep pure'
|
||||
alias pxsc='kubectl get storageclass | grep pure'
|
||||
```
|
||||
|
||||
## References
|
||||
|
||||
- Full Documentation: `cluster/platform/portworx-csi/README.md`
|
||||
- Deployment Checklist: `cluster/platform/portworx-csi/DEPLOYMENT-CHECKLIST.md`
|
||||
- Portworx Docs: https://docs.portworx.com/portworx-csi/
|
||||
- FlashArray Docs: https://support.purestorage.com/
|
||||
Reference in New Issue
Block a user