2 Commits

Author SHA1 Message Date
Hermes Agent service account
f37021346b Enable iSCSI support for Portworx CSI
- Add dm_round_robin kernel module for Pure Storage multipath
- Uncomment and enable /etc/multipath.conf with Pure-specific settings
- Add apply-iscsi-config.sh script for rolling worker node updates
2026-06-20 20:25:08 -05:00
Hermes Agent service account
5a99928c6f Add ServiceMonitors and Grafana dashboards for Harbor, External-DNS, and Ingress-NGINX
- Created ServiceMonitor for Harbor (harbor-core, harbor-exporter, harbor-jobservice, harbor-registry)
  - Port: http-metrics (8001)
  - Scrape interval: 30s
  - Verified metrics: harbor_core_http_request_duration_seconds_count

- Created ServiceMonitor for External-DNS
  - Port: http (7979)
  - Scrape interval: 30s
  - Verified metrics: external_dns_registry_endpoints_total

- Created ServiceMonitor for Ingress-NGINX
  - Port: metrics (10254)
  - Scrape interval: 30s
  - Verified metrics: nginx_ingress_controller_requests

- Added Grafana dashboards:
  - Harbor Overview (dashboard 16366)
  - External-DNS (dashboard 15038)
  - Ingress-NGINX Controller (dashboard 9614)

All ServiceMonitors deployed and actively scraping. Prometheus targets confirmed UP.
2026-06-19 18:26:54 -05:00
6 changed files with 6014 additions and 23 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,36 @@
# ------------------------------------------------------------------------------
# External-DNS Metrics - ServiceMonitor
#
# Configures Prometheus Operator to scrape External-DNS metrics
#
# Scrape interval: 30s
# Endpoint: /metrics on port 7979
# Metrics namespace: externaldns_* (externaldns_registry_endpoints_total, etc.)
# ------------------------------------------------------------------------------
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: external-dns-metrics
namespace: external-dns
labels:
app.kubernetes.io/name: external-dns
app.kubernetes.io/component: metrics
# Label selector for kube-prometheus-stack discovery
release: monitoring
spec:
# Select the external-dns service
selector:
matchLabels:
app.kubernetes.io/name: external-dns
# Namespace selector
namespaceSelector:
matchNames:
- external-dns
# Scrape endpoint configuration
endpoints:
- port: http
interval: 30s
scrapeTimeout: 10s
path: /metrics

View File

@@ -0,0 +1,49 @@
# ------------------------------------------------------------------------------
# Harbor Metrics - ServiceMonitor
#
# Configures Prometheus Operator to scrape Harbor metrics
#
# Scrape interval: 30s
# Components scraped:
# - harbor-core: Core service HTTP metrics
# - harbor-registry: Registry-specific metrics
# - harbor-jobservice: Job and vulnerability scan metrics
# - harbor-exporter: Harbor metrics exporter
# Metrics namespace: harbor_* (harbor_core_*, harbor_project_*, etc.)
# ------------------------------------------------------------------------------
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: harbor-metrics
namespace: harbor
labels:
app: harbor
app.kubernetes.io/name: harbor
app.kubernetes.io/component: metrics
# Label selector for kube-prometheus-stack discovery
release: monitoring
spec:
# Select Harbor services with metrics endpoints
selector:
matchLabels:
app: harbor
# Namespace selector
namespaceSelector:
matchNames:
- harbor
# Scrape endpoint configuration
endpoints:
# Harbor Core - main API and web UI metrics
- port: http-metrics
interval: 30s
scrapeTimeout: 10s
path: /metrics
relabelings:
- sourceLabels: [__meta_kubernetes_service_label_component]
targetLabel: component
action: replace
- sourceLabels: [__meta_kubernetes_service_name]
targetLabel: service
action: replace

View File

@@ -0,0 +1,37 @@
# ------------------------------------------------------------------------------
# Ingress-NGINX Controller Metrics - ServiceMonitor
#
# Configures Prometheus Operator to scrape Ingress-NGINX Controller metrics
#
# Scrape interval: 30s
# Endpoint: /metrics on port 10254
# Metrics namespace: nginx_ingress_* (nginx_ingress_controller_requests, etc.)
# ------------------------------------------------------------------------------
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
name: ingress-nginx-metrics
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/component: metrics
# Label selector for kube-prometheus-stack discovery
release: monitoring
spec:
# Select the ingress-nginx-controller-metrics service
selector:
matchLabels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/component: controller
# Namespace selector
namespaceSelector:
matchNames:
- ingress-nginx
# Scrape endpoint configuration
endpoints:
- port: metrics
interval: 30s
scrapeTimeout: 10s
path: /metrics

View File

@@ -0,0 +1,69 @@
#!/bin/bash
# Apply iSCSI configuration to worker nodes
# This regenerates configs with the iscsi-tools extension and applies them
set -e
echo "=========================================="
echo "Talos iSCSI Configuration Update"
echo "=========================================="
echo ""
echo "This will:"
echo " 1. Generate new Talos configs with iscsi-tools extension"
echo " 2. Apply configs to each worker node (one at a time)"
echo " 3. Reboot each node to load iSCSI modules"
echo " 4. Wait for each node to become Ready before continuing"
echo ""
read -p "Proceed? (y/N): " -n 1 -r
echo
if [[ ! $REPLY =~ ^[Yy]$ ]]; then
echo "Aborted."
exit 0
fi
echo ""
echo "Step 1: Generating Talos configurations..."
talhelper genconfig
declare -A WORKERS=(
["jungle-cruise"]="10.1.71.69"
["haunted-mansion"]="10.1.71.70"
["peter-pans-flight"]="10.1.71.71"
)
for worker in jungle-cruise haunted-mansion peter-pans-flight; do
ip="${WORKERS[$worker]}"
echo ""
echo "=========================================="
echo "Updating $worker ($ip)..."
echo "=========================================="
echo "Applying configuration (will reboot)..."
talosctl apply-config --nodes ${ip} \
--file clusterconfig/fastpass-${worker}.yaml \
--mode=reboot
echo "Waiting for $worker to reboot and become Ready..."
sleep 30 # Give it time to start rebooting
kubectl wait --for=condition=Ready node/${worker} --timeout=10m
echo "$worker is back up."
if [ "$worker" != "peter-pans-flight" ]; then
echo "Sleeping 30s before next node..."
sleep 30
fi
done
echo ""
echo "=========================================="
echo "✓ All workers updated!"
echo "=========================================="
echo ""
echo "Verifying iSCSI modules on jungle-cruise..."
kubectl debug node/jungle-cruise -it --image=busybox -- cat /proc/modules | grep -E "iscsi|multipath" || echo "Modules check failed"
echo ""
echo "Next steps:"
echo " 1. Verify iSCSI modules loaded on all nodes"
echo " 2. Proceed with Portworx CSI installation"

View File

@@ -128,6 +128,7 @@ worker:
modules: modules:
- name: iscsi_tcp - name: iscsi_tcp
- name: dm_multipath - name: dm_multipath
- name: dm_round_robin
kubelet: kubelet:
extraMounts: extraMounts:
@@ -148,29 +149,30 @@ worker:
sysctls: sysctls:
net.ipv4.conf.all.arp_announce: "2" net.ipv4.conf.all.arp_announce: "2"
net.ipv4.conf.all.arp_ignore: "1" net.ipv4.conf.all.arp_ignore: "1"
# files:
# - content: | files:
# defaults { - content: |
# polling_interval 10 defaults {
# } polling_interval 10
# devices { }
# device { devices {
# vendor "PURE" device {
# product "FlashArray" vendor "PURE"
# path_selector "queue-length 0" product "FlashArray"
# path_grouping_policy group_by_prio path_selector "queue-length 0"
# prio alua path_grouping_policy group_by_prio
# path_checker tur prio alua
# fast_io_fail_tmo 10 path_checker tur
# user_friendly_names no fast_io_fail_tmo 10
# no_path_retry 0 user_friendly_names no
# hardware_handler "1 alua" no_path_retry 0
# dev_loss_tmo 600 hardware_handler "1 alua"
# failback immediate dev_loss_tmo 600
# } failback immediate
# } }
# path: /etc/multipath.conf }
# op: create path: /etc/multipath.conf
op: create
# ─── Nodes ─────────────────────────────────────────────────────────────────── # ─── Nodes ───────────────────────────────────────────────────────────────────
nodes: nodes: