459dbc5d18
- Deploy Portworx Operator + CSI driver via ArgoCD - Support both iSCSI block and NFS file storage from FlashArray - Integrate with 1Password External Secrets for FlashArray credentials - Include comprehensive deployment documentation and validation script - Storage classes: pure-block (iSCSI) and pure-file (NFS) - Talos Linux compatible with iSCSI/multipath configuration
439 lines
9.2 KiB
Markdown
439 lines
9.2 KiB
Markdown
# Portworx CSI Quick Reference
|
|
|
|
Common commands and operations for Portworx CSI with Pure Storage FlashArray.
|
|
|
|
## Check Status
|
|
|
|
```bash
|
|
# ArgoCD application
|
|
kubectl get application -n argocd portworx-csi
|
|
|
|
# All pods in portworx namespace
|
|
kubectl get pods -n portworx
|
|
|
|
# StorageCluster status
|
|
kubectl get storagecluster -n portworx px-cluster-fastpass
|
|
kubectl describe storagecluster -n portworx px-cluster-fastpass
|
|
|
|
# CSI driver registration
|
|
kubectl get csidrivers pxd.portworx.com
|
|
kubectl get csinode
|
|
|
|
# StorageClasses
|
|
kubectl get storageclass pure-block pure-file
|
|
```
|
|
|
|
## Volume Operations
|
|
|
|
### Create PVC (Block Storage)
|
|
```bash
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: my-app-data
|
|
namespace: default
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: pure-block
|
|
resources:
|
|
requests:
|
|
storage: 10Gi
|
|
EOF
|
|
```
|
|
|
|
### Create PVC (File Storage - NFS)
|
|
```bash
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: shared-data
|
|
namespace: default
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteMany
|
|
storageClassName: pure-file
|
|
resources:
|
|
requests:
|
|
storage: 50Gi
|
|
EOF
|
|
```
|
|
|
|
### Expand Volume
|
|
```bash
|
|
# Edit PVC to increase size
|
|
kubectl patch pvc my-app-data -p '{"spec":{"resources":{"requests":{"storage":"20Gi"}}}}'
|
|
|
|
# Verify expansion
|
|
kubectl get pvc my-app-data
|
|
```
|
|
|
|
### Create Snapshot
|
|
```bash
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: snapshot.storage.k8s.io/v1
|
|
kind: VolumeSnapshot
|
|
metadata:
|
|
name: my-app-snapshot
|
|
namespace: default
|
|
spec:
|
|
volumeSnapshotClassName: px-csi-snapshot-class
|
|
source:
|
|
persistentVolumeClaimName: my-app-data
|
|
EOF
|
|
```
|
|
|
|
### Restore from Snapshot
|
|
```bash
|
|
kubectl apply -f - <<EOF
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: my-app-data-restored
|
|
namespace: default
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
storageClassName: pure-block
|
|
dataSource:
|
|
name: my-app-snapshot
|
|
kind: VolumeSnapshot
|
|
apiGroup: snapshot.storage.k8s.io
|
|
resources:
|
|
requests:
|
|
storage: 20Gi
|
|
EOF
|
|
```
|
|
|
|
## Logs
|
|
|
|
```bash
|
|
# Portworx Operator
|
|
kubectl logs -n portworx -l app=portworx-operator --tail=100
|
|
|
|
# PX-CSI Controller
|
|
kubectl logs -n portworx -l app=px-csi-controller --tail=100
|
|
|
|
# PX-CSI Node driver (specific node)
|
|
kubectl logs -n portworx -l app=px-csi-node --tail=100
|
|
|
|
# Follow logs in real-time
|
|
kubectl logs -n portworx -l app=px-csi-controller -f
|
|
```
|
|
|
|
## Troubleshooting
|
|
|
|
### Check iSCSI on Talos Nodes
|
|
```bash
|
|
# List iSCSI sessions
|
|
talosctl -n <node-ip> exec -- iscsiadm -m session
|
|
|
|
# Discover FlashArray targets
|
|
talosctl -n <node-ip> exec -- iscsiadm -m discovery -t st -p <flasharray-ip>
|
|
|
|
# Check initiator name (must be unique)
|
|
talosctl -n <node-ip> read /etc/iscsi/initiatorname.iscsi
|
|
```
|
|
|
|
### Check Multipath
|
|
```bash
|
|
# Multipath status
|
|
talosctl -n <node-ip> exec -- multipath -ll
|
|
|
|
# Reload multipath config
|
|
talosctl -n <node-ip> exec -- systemctl reload multipathd
|
|
|
|
# Multipath devices
|
|
talosctl -n <node-ip> exec -- ls -l /dev/mapper/
|
|
```
|
|
|
|
### Verify FlashArray Connectivity
|
|
```bash
|
|
# From PX-CSI pod
|
|
kubectl exec -n portworx -it <px-pod> -- ping <flasharray-ip>
|
|
|
|
# Test API endpoint
|
|
kubectl exec -n portworx -it <px-pod> -- \
|
|
curl -k https://<flasharray-mgmt-ip>/api/2.0/arrays
|
|
```
|
|
|
|
### Check Secret
|
|
```bash
|
|
# Verify secret exists
|
|
kubectl get secret -n portworx px-pure-secret
|
|
|
|
# View secret content (pure.json)
|
|
kubectl get secret -n portworx px-pure-secret \
|
|
-o jsonpath='{.data.pure\.json}' | base64 -d | jq .
|
|
|
|
# Check ExternalSecret status
|
|
kubectl get externalsecret -n portworx px-pure-secret
|
|
kubectl describe externalsecret -n portworx px-pure-secret
|
|
```
|
|
|
|
### Volume Not Attaching
|
|
```bash
|
|
# Check PVC events
|
|
kubectl describe pvc <pvc-name>
|
|
|
|
# Check pod events
|
|
kubectl describe pod <pod-name>
|
|
|
|
# Check PV
|
|
kubectl get pv
|
|
kubectl describe pv <pv-name>
|
|
|
|
# Check volume attachment
|
|
kubectl get volumeattachment
|
|
```
|
|
|
|
### Pod Stuck in ContainerCreating
|
|
```bash
|
|
# Check events
|
|
kubectl describe pod <pod-name>
|
|
|
|
# Check PVC status
|
|
kubectl get pvc -n <namespace>
|
|
|
|
# Check node where pod is scheduled
|
|
POD_NODE=$(kubectl get pod <pod-name> -o jsonpath='{.spec.nodeName}')
|
|
echo "Pod scheduled on: $POD_NODE"
|
|
|
|
# Check iSCSI on that node
|
|
talosctl -n $POD_NODE exec -- iscsiadm -m session
|
|
```
|
|
|
|
## Monitoring
|
|
|
|
### Prometheus Queries
|
|
```promql
|
|
# Cluster status
|
|
portworx_cluster_status
|
|
|
|
# Volume capacity
|
|
portworx_volume_capacity_bytes
|
|
|
|
# Volume usage
|
|
portworx_volume_usage_bytes
|
|
|
|
# I/O metrics
|
|
rate(portworx_volume_read_bytes[5m])
|
|
rate(portworx_volume_write_bytes[5m])
|
|
```
|
|
|
|
### List All PVCs Using Pure Storage
|
|
```bash
|
|
# Block storage
|
|
kubectl get pvc --all-namespaces -o json | \
|
|
jq -r '.items[] | select(.spec.storageClassName == "pure-block") |
|
|
"\(.metadata.namespace)/\(.metadata.name) - \(.status.phase)"'
|
|
|
|
# File storage
|
|
kubectl get pvc --all-namespaces -o json | \
|
|
jq -r '.items[] | select(.spec.storageClassName == "pure-file") |
|
|
"\(.metadata.namespace)/\(.metadata.name) - \(.status.phase)"'
|
|
```
|
|
|
|
### Storage Usage by Namespace
|
|
```bash
|
|
kubectl get pvc --all-namespaces -o json | \
|
|
jq -r '.items[] | select(.spec.storageClassName | startswith("pure-")) |
|
|
"\(.metadata.namespace)\t\(.spec.resources.requests.storage)"' | \
|
|
awk '{arr[$1]+=$2} END {for (i in arr) print i, arr[i]}'
|
|
```
|
|
|
|
## Maintenance
|
|
|
|
### Update FlashArray Credentials
|
|
```bash
|
|
# Update in 1Password:
|
|
# Vault: homelab
|
|
# Item: pure-flasharray-fastpass
|
|
# Field: pure.json
|
|
|
|
# Force ExternalSecret refresh
|
|
kubectl delete secret -n portworx px-pure-secret
|
|
|
|
# Wait for ExternalSecret to recreate it (check status)
|
|
kubectl get externalsecret -n portworx px-pure-secret -w
|
|
```
|
|
|
|
### Restart PX-CSI Components
|
|
```bash
|
|
# Restart controller
|
|
kubectl rollout restart deployment -n portworx px-csi-controller
|
|
|
|
# Restart node drivers (DaemonSet)
|
|
kubectl rollout restart daemonset -n portworx px-csi-node
|
|
|
|
# Check rollout status
|
|
kubectl rollout status deployment -n portworx px-csi-controller
|
|
kubectl rollout status daemonset -n portworx px-csi-node
|
|
```
|
|
|
|
### Drain Node for Maintenance
|
|
```bash
|
|
# Cordon node
|
|
kubectl cordon <node-name>
|
|
|
|
# Drain node (evict pods)
|
|
kubectl drain <node-name> --ignore-daemonsets --delete-emptydir-data
|
|
|
|
# Perform maintenance...
|
|
|
|
# Uncordon node
|
|
kubectl uncordon <node-name>
|
|
```
|
|
|
|
## ArgoCD Operations
|
|
|
|
### Sync Application
|
|
```bash
|
|
# Sync via CLI
|
|
argocd app sync portworx-csi
|
|
|
|
# Force sync (ignore differences)
|
|
argocd app sync portworx-csi --force
|
|
|
|
# Sync specific resource
|
|
argocd app sync portworx-csi --resource storagecluster:px-cluster-fastpass
|
|
```
|
|
|
|
### View Application Details
|
|
```bash
|
|
# App status
|
|
argocd app get portworx-csi
|
|
|
|
# App history
|
|
argocd app history portworx-csi
|
|
|
|
# App diff (compare git vs cluster)
|
|
argocd app diff portworx-csi
|
|
```
|
|
|
|
### Rollback via ArgoCD
|
|
```bash
|
|
# List history
|
|
argocd app history portworx-csi
|
|
|
|
# Rollback to previous version
|
|
argocd app rollback portworx-csi <revision-number>
|
|
```
|
|
|
|
## Configuration Updates
|
|
|
|
### Edit StorageCluster
|
|
```bash
|
|
# Edit directly (will be overwritten by ArgoCD sync)
|
|
kubectl edit storagecluster -n portworx px-cluster-fastpass
|
|
|
|
# Permanent change: update git
|
|
cd ~/git/homelab
|
|
vi cluster/platform/portworx-csi/storagecluster.yaml
|
|
git commit -am "Update Portworx StorageCluster config"
|
|
git push
|
|
```
|
|
|
|
### Add Environment Variable
|
|
```yaml
|
|
# In storagecluster.yaml, add to spec.env:
|
|
env:
|
|
- name: NEW_ENV_VAR
|
|
value: "value"
|
|
```
|
|
|
|
### Change Log Level
|
|
```yaml
|
|
# In storagecluster.yaml:
|
|
env:
|
|
- name: LOG_LEVEL
|
|
value: "debug" # or: info, warn, error
|
|
```
|
|
|
|
## Talos Node Configuration
|
|
|
|
### View Current Config
|
|
```bash
|
|
# Read multipath.conf
|
|
talosctl -n <node-ip> read /etc/multipath.conf
|
|
|
|
# Read udev rules
|
|
talosctl -n <node-ip> read /etc/udev/rules.d/99-pure-storage.rules
|
|
|
|
# Check loaded modules
|
|
talosctl -n <node-ip> exec -- lsmod | grep -E "iscsi|multipath"
|
|
```
|
|
|
|
### Apply Updated Talos Config
|
|
```bash
|
|
cd ~/git/homelab/talos/talhelper
|
|
|
|
# Regenerate configs after updating talconfig.yaml
|
|
talhelper genconfig
|
|
|
|
# Apply to specific node
|
|
talosctl apply-config \
|
|
--file clusterconfig/<node-name>.yaml \
|
|
--nodes <node-ip>
|
|
|
|
# Reboot if kernel modules changed
|
|
talosctl reboot --nodes <node-ip>
|
|
```
|
|
|
|
## Emergency Procedures
|
|
|
|
### Complete Reinstall
|
|
```bash
|
|
# 1. Delete ArgoCD application
|
|
kubectl delete application -n argocd portworx-csi
|
|
|
|
# 2. Delete namespace
|
|
kubectl delete namespace portworx
|
|
|
|
# 3. Delete CRDs
|
|
kubectl delete crd storageclusters.core.libopenstorage.org
|
|
|
|
# 4. Delete StorageClasses
|
|
kubectl delete storageclass pure-block pure-file
|
|
|
|
# 5. Re-apply via ArgoCD
|
|
argocd app sync portworx-csi
|
|
```
|
|
|
|
### Recover from Failed State
|
|
```bash
|
|
# 1. Check StorageCluster status
|
|
kubectl get storagecluster -n portworx -o yaml
|
|
|
|
# 2. Delete and recreate StorageCluster
|
|
kubectl delete storagecluster -n portworx px-cluster-fastpass
|
|
|
|
# 3. Let ArgoCD recreate it
|
|
argocd app sync portworx-csi --force
|
|
|
|
# 4. Monitor deployment
|
|
kubectl get pods -n portworx -w
|
|
```
|
|
|
|
## Useful Aliases
|
|
|
|
Add to `~/.bashrc` or `~/.zshrc`:
|
|
|
|
```bash
|
|
# Portworx aliases
|
|
alias pxpods='kubectl get pods -n portworx'
|
|
alias pxlogs='kubectl logs -n portworx'
|
|
alias pxstc='kubectl get storagecluster -n portworx'
|
|
alias pxpvc='kubectl get pvc --all-namespaces -o wide | grep pure'
|
|
alias pxsc='kubectl get storageclass | grep pure'
|
|
```
|
|
|
|
## References
|
|
|
|
- Full Documentation: `cluster/platform/portworx-csi/README.md`
|
|
- Deployment Checklist: `cluster/platform/portworx-csi/DEPLOYMENT-CHECKLIST.md`
|
|
- Portworx Docs: https://docs.portworx.com/portworx-csi/
|
|
- FlashArray Docs: https://support.purestorage.com/
|