12 Commits

Author SHA1 Message Date
Hermes Agent service account
bc34a1f915 couchdb: increase nginx proxy body size to 100m
Fixes 413 Entity Too Large error in Obsidian LiveSync sync operations.
Applies to both internal (communicore.local) and public (communicore.mk-labs.cloud) ingress routes.
2026-06-30 23:09:56 -05:00
Hermes Agent service account
64e690737e fix(traefik): remove WebSocket middleware - Traefik v3 handles WS natively 2026-06-30 17:19:29 -05:00
Hermes Agent service account
0693fdcd26 fix(traefik): add WebSocket middleware for Hermes dashboard Chat tab
- Add websocket-headers middleware to jarvis router
- Set Connection: Upgrade and Upgrade: websocket headers
- Fixes 'Chat unavailable:1' WebSocket connection failures through Traefik reverse proxy
2026-06-30 17:12:59 -05:00
19a807899f reference file 2026-06-29 20:55:25 -05:00
5bacf9fbca talos multipath patch 2026-06-29 20:49:57 -05:00
1da9bfd43c talos 2026-06-25 10:02:16 -05:00
0bc9b2e788 Continued talos multipath troubleshooting. 2026-06-22 22:30:35 -05:00
dcfb6825e8 Talos multipath. 2026-06-22 20:40:06 -05:00
0b9ac4dc74 Pre-upgrade snapshot: Talos v1.13.2, before multipath implementation 2026-06-22 18:39:35 -05:00
Hermes Agent service account
aabf758c91 Add multipath.conf for Pure FlashArray to Talos worker nodes
- Add /etc/multipath.conf file creation in worker patches
- Configuration optimized for Pure FlashArray iSCSI
- Required for PX-CSI node pods to start successfully
- Blacklists Portworx virtual devices (pxd*)

Ref: Portworx → democratic-csi migration Phase 3
2026-06-22 16:53:38 -05:00
Hermes Agent service account
489b8aeb35 WIP: iscsi-multipath-init DaemonSet attempts
Successfully writes /system/etc/multipath.conf but cannot write to /etc
due to Talos read-only filesystem restrictions.

Attempts made:
- nsenter with sh/cat/ln - commands don't exist in Talos minimal env
- Mount /proc/1/root/etc - still read-only
- Bind mount - invalid argument

Blocker: Talos /etc is truly read-only post-boot. PX-CSI also fails
with same nsenter/command issues when trying to validate multipath.conf.

Next: Investigate PX-CSI configuration options or Talos machine config alternatives.
2026-06-21 00:11:18 -05:00
Hermes Agent service account
002d6799b1 Fix iscsi-multipath-init DaemonSet for Talos read-only filesystem
Use nsenter to write multipath.conf in host's mount namespace instead
of trying to write to /host/etc which is read-only in containers.

Talos mounts /etc as read-only in container namespaces but allows writes
in the host mount namespace. This fix uses nsenter to access PID 1's
mount namespace where /etc is writable.

Also removed unnecessary volumeMounts and volumes since we're using
nsenter instead of hostPath mounts.

Fixes: Init:Error - 'can't create /host/etc/multipath.conf: Read-only file system'
2026-06-20 23:56:20 -05:00
12 changed files with 522 additions and 30 deletions

View File

@@ -88,6 +88,10 @@ papermc_server:
dev_servers:
hosts:
scrim:
backstage:
ansible_host: 10.1.71.133
ansible_user: wed
ansible_become: true
# dhcp_server:
# hosts:

View File

@@ -15,6 +15,7 @@ metadata:
external-dns.alpha.kubernetes.io/hostname: "communicore.mk-labs.cloud"
external-dns.alpha.kubernetes.io/target: "ingress.mk-labs.cloud"
external-dns.alpha.kubernetes.io/public: "true"
nginx.ingress.kubernetes.io/proxy-body-size: "100m"
spec:
ingressClassName: nginx
rules:

View File

@@ -62,6 +62,7 @@ ingress:
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
external-dns.alpha.kubernetes.io/hostname: "communicore.local.mk-labs.cloud"
nginx.ingress.kubernetes.io/proxy-body-size: "100m"
hosts:
- communicore.local.mk-labs.cloud
tls:

156
fastpass-jungle-cruise.yaml Normal file
View File

@@ -0,0 +1,156 @@
version: v1alpha1
debug: false
persist: true
machine:
type: worker
token: i1qaz0.1elrf4bh8hyvo09e
ca:
crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJQekNCOHFBREFnRUNBaEVBcFV2UmZVSVJVTWVoSTN2elNGTXIyakFGQmdNclpYQXdFREVPTUF3R0ExVUUKQ2hNRmRHRnNiM013SGhjTk1qWXdOVEUzTWpBd05qRXpXaGNOTXpZd05URTBNakF3TmpFeldqQVFNUTR3REFZRApWUVFLRXdWMFlXeHZjekFxTUFVR0F5dGxjQU1oQVBzeXFTVUpnSUhvQjMxZWd1OXpGVStPcnpvb1NJOC9FNkkzCmJvOG1GaVIrbzJFd1h6QU9CZ05WSFE4QkFmOEVCQU1DQW9Rd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSEF3RUcKQ0NzR0FRVUZCd01DTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRFZSME9CQllFRkEvUm02ZHdwaGpCS3gxUQoraitrb1h1ZU9NbStNQVVHQXl0bGNBTkJBQTRwZmtaU0VlQkFpN0ZseDg1a1lDb2Q4aUpScEFBUi9RZkNDZ2hCCjBGMmtPOU5wZVhRb3BRL3V0VnJCdlkrNzJKRFFLRWdGTTc5MncwVkFaQzVmb2drPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
key: ""
certSANs: []
kubelet:
image: ghcr.io/siderolabs/kubelet:v1.32.3
extraArgs:
rotate-server-certificates: "true"
defaultRuntimeSeccompProfileEnabled: true
nodeIP:
validSubnets:
- 10.1.71.0/24
disableManifestsDirectory: true
network:
nameservers:
- 10.1.71.1
install:
disk: /dev/sda
image: factory.talos.dev/metal-installer/10326733f72d0b39b6750c291fc499359135c5486d4066efad56fcf4f3af6923:v1.13.5
wipe: false
grubUseUKICmdline: true
time:
servers:
- 10.1.71.21
sysctls:
kernel.perf_event_paranoid: "1"
kernel.unprivileged_bpf_disabled: "0"
net.core.bpf_jit_harden: "0"
net.ipv4.conf.all.arp_announce: "2"
net.ipv4.conf.all.arp_ignore: "1"
features:
diskQuotaSupport: true
kubePrism:
enabled: true
port: 7445
hostDNS:
enabled: true
forwardKubeDNSToHost: true
kernel:
modules:
- name: iscsi_tcp
- name: dm_multipath
- name: dm_round_robin
cluster:
id: 6FRjhITdPHrGGI35sp0DwqaZP9INupBQIBVn9AZb6NA=
secret: P2qqriHtWSm/8iGQMA4vnanm8Mh1f3nuhEFYH1yBPQM=
controlPlane:
endpoint: https://10.1.71.65:6443
clusterName: fastpass
network:
dnsDomain: cluster.local
podSubnets:
- 10.244.0.0/16
serviceSubnets:
- 10.96.0.0/12
token: 9w22i0.383cxk2u6ybraufx
ca:
crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVMrZ0F3SUJBZ0lRWGZvbjBrR2hZUHNTRXcrOUo3VjdCakFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTJNRFV4TnpJd01EWXhNbG9YRFRNMk1EVXhOREl3TURZeApNbG93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCT1g5aDA5eFFDMGFVY3JRRDVGT2lVVzlHOG5abUhCcWc2dHRCVjhNdXBKSjNNMjdreFBGR2VEM05MQkYKVEFVbnBFRE1Jc2RLY09TZFNudWk0ZjJNY0hDallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVVtbXp5UGV4Qk1VRWk2V3Y3MXBwK3V6SjRUWDR3Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loQVBpelUyM1EKMDVMV0xNSG5lMWo0SWF5dVRrU1pjeXRiZThBN2Y3VDlkb3J4QWlFQWwzSTJhTDhpSHczd2JSVjREQ3NPczZ3WAo5akxLa0prN09pWnhMNlJLR2RNPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
key: ""
apiServer:
admissionControl:
- name: PodSecurity
configuration:
apiVersion: pod-security.admission.config.k8s.io/v1alpha1
defaults:
audit: restricted
audit-version: latest
enforce: privileged
enforce-version: latest
warn: restricted
warn-version: latest
exemptions:
namespaces:
- argocd
- cert-manager
- ingress-nginx
- external-secrets
runtimeClasses: []
usernames: []
kind: PodSecurityConfiguration
discovery:
enabled: true
registries:
kubernetes:
disabled: true
service: {}
---
apiVersion: v1alpha1
kind: HostnameConfig
auto: "off"
hostname: jungle-cruise
---
apiVersion: v1alpha1
kind: LinkConfig
name: ens18
addresses:
- address: 10.1.71.69/24
routes:
- gateway: 10.1.71.1
---
apiVersion: v1alpha1
kind: LinkConfig
name: ens19
addresses:
- address: 10.1.75.69/24
---
apiVersion: v1alpha1
kind: ExtensionServiceConfig
name: multipathd
configFiles:
- content: |-
# Pure Storage FlashArray multipath configuration
# Configured via ExtensionServiceConfig for multipathd system service
defaults {
polling_interval 10
path_selector "round-robin 0"
path_grouping_policy group_by_prio
path_checker tur
prio alua
failback immediate
user_friendly_names no
find_multipaths yes
fast_io_fail_tmo 10
dev_loss_tmo 600
no_path_retry 0
}
devices {
device {
vendor "PURE"
product "FlashArray"
path_selector "round-robin 0"
path_grouping_policy group_by_prio
prio alua
path_checker tur
fast_io_fail_tmo 10
user_friendly_names no
no_path_retry 0
hardware_handler "1 alua"
dev_loss_tmo 600
failback immediate
}
}
blacklist {
# Exclude Portworx PXD devices from multipathing
devnode "^pxd[0-9]*"
devnode "^pxd.*"
}
mountPath: /etc/multipath.conf

46
multipath-patch.yaml Normal file
View File

@@ -0,0 +1,46 @@
---
apiVersion: v1alpha1
kind: ExtensionServiceConfig
name: multipathd
configFiles:
- content: |-
# Pure Storage FlashArray multipath configuration
# Configured via ExtensionServiceConfig for multipathd system service
defaults {
polling_interval 10
path_selector "round-robin 0"
path_grouping_policy group_by_prio
path_checker tur
prio alua
failback immediate
user_friendly_names no
find_multipaths yes
fast_io_fail_tmo 10
dev_loss_tmo 600
no_path_retry 0
}
devices {
device {
vendor "PURE"
product "FlashArray"
path_selector "round-robin 0"
path_grouping_policy group_by_prio
prio alua
path_checker tur
fast_io_fail_tmo 10
user_friendly_names no
no_path_retry 0
hardware_handler "1 alua"
dev_loss_tmo 600
failback immediate
}
}
blacklist {
# Exclude Portworx PXD devices from multipathing
devnode "^pxd[0-9]*"
devnode "^pxd.*"
}
mountPath: /etc/multipath.conf

View File

@@ -0,0 +1,7 @@
fastpass-space-mountain.yaml
fastpass-big-thunder-mountain.yaml
fastpass-splash-mountain.yaml
fastpass-jungle-cruise.yaml
fastpass-haunted-mansion.yaml
fastpass-peter-pans-flight.yaml
talosconfig

View File

@@ -2,12 +2,10 @@
# iSCSI Multipath Initialization DaemonSet
#
# Configures multipath.conf for Pure Storage FlashArray on Talos worker nodes.
# Runs as a DaemonSet with an initContainer that writes configuration, then
# sleeps indefinitely to mark the node as configured.
# Runs as a DaemonSet with an initContainer that writes configuration.
#
# This approach avoids Talos boot-time file writing issues while ensuring
# multipath is properly configured before Portworx CSI driver starts using
# iSCSI volumes.
# IMPORTANT: On Talos, /etc is an overlayfs with writable upper layer in /system/etc
# We write directly to /system/etc which then appears in /etc
apiVersion: apps/v1
kind: DaemonSet
@@ -43,6 +41,9 @@ spec:
image: alpine:3.19
securityContext:
privileged: true
volumeMounts:
- name: system-etc
mountPath: /system-etc
command:
- sh
- -c
@@ -51,8 +52,8 @@ spec:
echo "Configuring multipath for Pure Storage FlashArray..."
# Write multipath.conf
cat > /host/etc/multipath.conf <<'MPCONF'
# Write to /system/etc
cat > /system-etc/multipath.conf <<'MPCONF'
# Multipath configuration for Pure Storage FlashArray
# Managed by iscsi-multipath-init DaemonSet
@@ -86,24 +87,24 @@ spec:
}
MPCONF
echo "✓ Multipath configuration written to /etc/multipath.conf"
echo "✓ Multipath configuration written to /system/etc/multipath.conf"
# Reload multipathd if running (Talos manages the service)
# Copy file to /etc using direct file copy (no ln/cp command needed)
# Mount PID 1's /etc as writable and copy the file
cat /system-etc/multipath.conf > /proc/1/root/etc/multipath.conf || \
echo "Warning: Could not copy to /etc directly, file available at /system/etc/multipath.conf"
echo "✓ Configuration deployed to /etc/multipath.conf"
# Reload multipathd if running
if nsenter -t 1 -m -u -i -n -- multipathd show status >/dev/null 2>&1; then
echo "Reloading multipathd..."
nsenter -t 1 -m -u -i -n -- multipathd reconfigure || true
fi
echo "✓ Multipath configuration complete"
echo "Current multipath devices:"
nsenter -t 1 -m -u -i -n -- multipath -ll || echo " (no multipath devices yet)"
volumeMounts:
- name: host-etc
mountPath: /host/etc
echo "✓ Configuration complete"
containers:
# Pause container keeps the pod running to indicate configuration is applied
- name: pause
image: registry.k8s.io/pause:3.9
resources:
@@ -115,7 +116,7 @@ spec:
memory: 16Mi
volumes:
- name: host-etc
- name: system-etc
hostPath:
path: /etc
type: Directory
path: /system/etc
type: DirectoryOrCreate

View File

@@ -0,0 +1,28 @@
apiVersion: v1alpha1
kind: ExtensionServiceConfig
name: multipathd
configFiles:
- content: |
# Your multipathd configuration content here
defaults {
user_friendly_names yes
find_multipaths yes
}
blacklist {
devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
devnode "^(hd|sda|sd[a-z])[0-9]*" # Adjust to blacklist local disks
devnode "^cciss!.*"
}
devices {
device {
vendor "PURE"
product "FlashArray"
path_grouping_policy multibus
path_selector "queue-length 0"
path_checker tur
no_path_retry 0
rr_min_io 1
dev_loss_tmo 30
}
}
mountPath: /etc/multipath.conf

View File

@@ -0,0 +1,118 @@
---
# Patch for px-pure-csi-node DaemonSet to work on Talos Linux
# This version DISABLES multipath to avoid multipath.conf issues on Talos
# Apply with: kubectl patch daemonset px-pure-csi-node -n portworx --patch-file px-csi-talos-patch-v2.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: px-pure-csi-node
namespace: portworx
spec:
template:
spec:
containers:
- name: node-plugin
env:
# CRITICAL: Disable multipath to avoid /etc/multipath.conf requirement
- name: DISABLE_MULTIPATH
value: "true"
# Keep existing env vars (these will merge with existing)
- name: PX_LOGLEVEL
value: INFO
- name: LOG_FILE
value: /var/log/px-pure-csi/node.log
- name: PURE_DISCOVERY_CONF
value: /config/pure.json
- name: CSI_ENDPOINT
value: unix:/csi/csi.sock
- name: CSI_NODE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
- name: NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: CLUSTER_UUID
valueFrom:
configMapKeyRef:
key: ClusterUUID
name: pure-storage-cluster-cm
- name: CLUSTER_NAME
valueFrom:
configMapKeyRef:
key: ClusterName
name: pure-storage-cluster-cm
- name: NODE_STAGE_CONCURRENCY
value: "25"
- name: PURE_FLASHARRAY_SAN_TYPE
value: ISCSI
volumeMounts:
# Add Talos-specific mounts
- name: usr-local-sbin
mountPath: /usr/local/sbin
mountPropagation: Bidirectional
- name: var-lib-iscsi
mountPath: /var/lib/iscsi
mountPropagation: Bidirectional
# Fix existing mounts (keep originals, add propagation)
- name: log-dir
mountPath: /var/log/px-pure-csi
- name: sys
mountPath: /sys
- name: kubelet-dir
mountPath: /var/lib/kubelet
mountPropagation: Bidirectional
- name: kubelet-dir
mountPath: /csi
subPath: plugins/pxd.portworx.com
- name: device-dir
mountPath: /dev
- name: iscsi
mountPath: /etc/iscsi
- name: host-root
mountPath: /host
mountPropagation: Bidirectional
volumes:
# Add Talos-specific volumes
- name: usr-local-sbin
hostPath:
path: /usr/local/sbin
type: Directory
- name: var-lib-iscsi
hostPath:
path: /var/lib/iscsi
type: DirectoryOrCreate
- name: host-root
hostPath:
path: /
type: Directory
# Keep existing volumes
- name: kubelet-dir
hostPath:
path: /var/lib/kubelet
- name: log-dir
hostPath:
path: /var/log/px-pure-csi
- name: device-dir
hostPath:
path: /dev
type: Directory
- name: probe-dir
emptyDir: {}
- name: iscsi
hostPath:
path: /etc/iscsi
- name: pure-config
secret:
secretName: px-pure-secret
items:
- key: pure.json
path: pure.json
- name: sys
hostPath:
path: /sys

View File

@@ -0,0 +1,44 @@
---
# Patch for px-pure-csi-node DaemonSet to work on Talos Linux
# Apply with: kubectl patch daemonset px-pure-csi-node -n portworx --patch-file px-csi-talos-patch.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: px-pure-csi-node
namespace: portworx
spec:
template:
spec:
containers:
- name: node-plugin
volumeMounts:
# Add Talos-specific mounts
- name: usr-local-sbin
mountPath: /usr/local/sbin
mountPropagation: Bidirectional
- name: var-lib-iscsi
mountPath: /var/lib/iscsi
mountPropagation: Bidirectional
# Fix existing kubelet-dir mount propagation
- name: kubelet-dir
mountPath: /var/lib/kubelet
mountPropagation: Bidirectional
# Mount host root for nsenter access
- name: host-root
mountPath: /host
mountPropagation: Bidirectional
volumes:
# Add Talos-specific volumes
- name: usr-local-sbin
hostPath:
path: /usr/local/sbin
type: Directory
- name: var-lib-iscsi
hostPath:
path: /var/lib/iscsi
type: DirectoryOrCreate
- name: host-root
hostPath:
path: /
type: Directory

View File

@@ -8,9 +8,9 @@
# talhelper gencommand apply --extra-flags="--insecure" | bash
# talhelper gencommand bootstrap | bash
# ─────────────────────────────────────────────────────────────────────────────
---
clusterName: fastpass
talosVersion: v1.13.2
talosVersion: v1.13.5
kubernetesVersion: v1.32.3
# VIP floats across control plane nodes — all kubectl/talosctl traffic lands here
@@ -120,6 +120,7 @@ worker:
- siderolabs/qemu-guest-agent
- siderolabs/util-linux-tools
- siderolabs/iscsi-tools
- siderolabs/multipath-tools
patches:
- |-
@@ -142,13 +143,13 @@ worker:
# Only mount /var/lib/iscsi (NOT /etc/iscsi)
# The iscsi-tools extension manages /etc/iscsi automatically
# /var/lib/iscsi contains iSCSI session and node data that must persist
- destination: /var/lib/iscsi
type: bind
source: /var/lib/iscsi
options:
- bind
- rshared
- rw
# - destination: /var/lib/iscsi
# type: bind
# source: /var/lib/iscsi
# options:
# - bind
# - rshared
# - rw
# ARP tuning for dual-NIC setup
sysctls:

View File

@@ -0,0 +1,85 @@
---
# Talos machine config patch for Pure FlashArray multipath support
# Add this to your Talos machine config under worker nodes
# Apply with: talosctl patch machineconfig -n <worker-ip> --patch @talos-multipath-patch.yaml
machine:
files:
- path: /var/etc/multipath.conf
permissions: 0644
op: create
# content: |-
# # Pure Storage FlashArray multipath configuration
# defaults {
# polling_interval 10
# path_selector "round-robin 0"
# path_grouping_policy group_by_prio
# path_checker tur
# prio alua
# failback immediate
# user_friendly_names no
# find_multipaths yes
# fast_io_fail_tmo 10
# dev_loss_tmo 600
# no_path_retry 0
# }
# devices {
# device {
# vendor "PURE"
# product "FlashArray"
# path_selector "round-robin 0"
# path_grouping_policy group_by_prio
# prio alua
# path_checker tur
# fast_io_fail_tmo 10
# user_friendly_names no
# no_path_retry 0
# hardware_handler "1 alua"
# dev_loss_tmo 600
# failback immediate
# }
# }
# blacklist {
# devnode "^pxd[0-9]*"
# devnode "^pxd.*"
# }
# Symlink for PX-CSI to find multipath.conf
- path: /etc/multipath.conf
permissions: 0644
op: create
# content: |-
# # Symlink to persistent location
# # Note: Talos /etc is read-only, this gets regenerated on boot
# defaults {
# polling_interval 10
# path_selector "round-robin 0"
# path_grouping_policy group_by_prio
# path_checker tur
# prio alua
# failback immediate
# user_friendly_names no
# find_multipaths yes
# fast_io_fail_tmo 10
# dev_loss_tmo 600
# no_path_retry 0
# }
# devices {
# device {
# vendor "PURE"
# product "FlashArray"
# path_selector "round-robin 0"
# path_grouping_policy group_by_prio
# prio alua
# path_checker tur
# fast_io_fail_tmo 10
# user_friendly_names no
# no_path_retry 0
# hardware_handler "1 alua"
# dev_loss_tmo 600
# failback immediate
# }
# }
# blacklist {
# devnode "^pxd[0-9]*"
# devnode "^pxd.*"
# }