monitoring: llama-swap GPU/LLM stack (v250) — PrometheusRule, Grafana dashboard, scrape config, VRAM exporter
This commit is contained in:
Hermes Agent service account
248
ansible/roles/llm-inference-multimodel/verify-monitoring-deployment.sh
Executable file
248
ansible/roles/llm-inference-multimodel/verify-monitoring-deployment.sh
Executable file
@@ -0,0 +1,248 @@
|
||||
#!/bin/bash
|
||||
# ==============================================================================
|
||||
# VERIFICATION SCRIPT: GPU/LLM Monitoring Deployment (Task t_57a9f82f)
|
||||
# ==============================================================================
|
||||
# Run this script AFTER Ansible role deployment to verify all monitoring
|
||||
# components are installed and functional.
|
||||
#
|
||||
# Usage:
|
||||
# bash verify-monitoring-deployment.sh
|
||||
#
|
||||
# Expected output: All checks ✓ (green)
|
||||
# ==============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROLE_DIR="/home/hermes/git/homelab/ansible/roles/llm-inference-multimodel"
|
||||
VRAM_EXPORTER_SCRIPT="/opt/llama-server-monitoring/nvidia-smi-vram-exporter.sh"
|
||||
VRAM_EXPORTER_OUTPUT="/var/lib/node_exporter/textfile_collector/nvidia.prom"
|
||||
|
||||
CHECKS_PASSED=0
|
||||
CHECKS_FAILED=0
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Helper function for check results
|
||||
check_pass() {
|
||||
local desc="$1"
|
||||
echo -e "${GREEN}✓${NC} $desc"
|
||||
((CHECKS_PASSED++))
|
||||
}
|
||||
|
||||
check_fail() {
|
||||
local desc="$1"
|
||||
local reason="${2:-Unknown reason}"
|
||||
echo -e "${RED}✗${NC} $desc"
|
||||
echo " Reason: $reason"
|
||||
((CHECKS_FAILED++))
|
||||
}
|
||||
|
||||
echo "================================================================================"
|
||||
echo "GPU/LLM Monitoring Deployment Verification"
|
||||
echo "================================================================================"
|
||||
echo ""
|
||||
|
||||
# 1. Check role structure
|
||||
echo "1. Role Structure & Deliverables"
|
||||
echo "=================================="
|
||||
|
||||
if [ -f "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md" ]; then
|
||||
check_pass "Reference docs: monitoring-llm-homelab-ciro-luciotta-2026.md exists"
|
||||
else
|
||||
check_fail "Reference docs: monitoring-llm-homelab-ciro-luciotta-2026.md NOT FOUND"
|
||||
fi
|
||||
|
||||
if [ -f "$ROLE_DIR/references/llama-swap-phase3-cutover-results-2026-08-18.md" ]; then
|
||||
check_pass "Phase 3 results: llama-swap-phase3-cutover-results-2026-08-18.md exists"
|
||||
else
|
||||
check_fail "Phase 3 results: llama-swap-phase3-cutover-results-2026-08-18.md NOT FOUND"
|
||||
fi
|
||||
|
||||
if [ -f "$ROLE_DIR/scripts/nvidia-smi-vram-exporter.sh" ]; then
|
||||
check_pass "VRAM exporter script: nvidia-smi-vram-exporter.sh exists"
|
||||
else
|
||||
check_fail "VRAM exporter script: nvidia-smi-vram-exporter.sh NOT FOUND"
|
||||
fi
|
||||
|
||||
if [ -x "$ROLE_DIR/scripts/nvidia-smi-vram-exporter.sh" ]; then
|
||||
check_pass "VRAM exporter script: executable"
|
||||
else
|
||||
check_fail "VRAM exporter script: not executable"
|
||||
fi
|
||||
|
||||
if [ -f "$ROLE_DIR/templates/llama-swap-prometheus-scrape.yml.j2" ]; then
|
||||
check_pass "Prometheus scrape config template exists"
|
||||
else
|
||||
check_fail "Prometheus scrape config template NOT FOUND"
|
||||
fi
|
||||
|
||||
if [ -f "$ROLE_DIR/templates/llama-swap-grafana-dashboard.json.j2" ]; then
|
||||
check_pass "Grafana dashboard template exists"
|
||||
else
|
||||
check_fail "Grafana dashboard template NOT FOUND"
|
||||
fi
|
||||
|
||||
if [ -f "$ROLE_DIR/templates/llama-swap-alerts.yml.j2" ]; then
|
||||
check_pass "Alert rules template exists"
|
||||
else
|
||||
check_fail "Alert rules template NOT FOUND"
|
||||
fi
|
||||
|
||||
if [ -f "$ROLE_DIR/tasks/monitoring.yml" ]; then
|
||||
check_pass "Monitoring tasks file exists"
|
||||
else
|
||||
check_fail "Monitoring tasks file NOT FOUND"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# 2. Check runtime deployment (if on astro-orbiter)
|
||||
echo "2. Runtime Deployment Status (astro-orbiter)"
|
||||
echo "=============================================="
|
||||
|
||||
if [ -x "$VRAM_EXPORTER_SCRIPT" ]; then
|
||||
check_pass "VRAM exporter script deployed at $VRAM_EXPORTER_SCRIPT"
|
||||
|
||||
# Try to run it
|
||||
if output=$($VRAM_EXPORTER_SCRIPT 2>&1) && [ -f "$VRAM_EXPORTER_OUTPUT" ]; then
|
||||
check_pass "VRAM exporter runs successfully"
|
||||
|
||||
# Check metric format
|
||||
if grep -q "llamacpp_vram_used_mib" "$VRAM_EXPORTER_OUTPUT"; then
|
||||
check_pass "VRAM metric format is correct"
|
||||
|
||||
# Extract and display the value
|
||||
vram_value=$(grep "llamacpp_vram_used_mib " "$VRAM_EXPORTER_OUTPUT" | awk '{print $NF}')
|
||||
echo " Current VRAM usage: ${vram_value} MiB"
|
||||
else
|
||||
check_fail "VRAM metric format incorrect" "Expected 'llamacpp_vram_used_mib' in output"
|
||||
fi
|
||||
else
|
||||
check_fail "VRAM exporter failed to run" "$output"
|
||||
fi
|
||||
else
|
||||
echo -e "${YELLOW}⊘${NC} VRAM exporter not deployed yet (expected if running on non-astro-orbiter)"
|
||||
fi
|
||||
|
||||
if crontab -l 2>/dev/null | grep -q "nvidia-smi-vram-exporter"; then
|
||||
check_pass "VRAM exporter cron job is installed"
|
||||
else
|
||||
echo -e "${YELLOW}⊘${NC} VRAM exporter cron job not installed (expected if not on astro-orbiter)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# 3. Check Ansible variables
|
||||
echo "3. Ansible Configuration Variables"
|
||||
echo "===================================="
|
||||
|
||||
if grep -q "llm_monitoring_enabled" "$ROLE_DIR/defaults/main.yml"; then
|
||||
check_pass "llm_monitoring_enabled variable defined"
|
||||
else
|
||||
check_fail "llm_monitoring_enabled variable NOT FOUND"
|
||||
fi
|
||||
|
||||
if grep -q "llm_vram_critical_mib" "$ROLE_DIR/defaults/main.yml"; then
|
||||
check_pass "Alert threshold variables defined"
|
||||
else
|
||||
check_fail "Alert threshold variables NOT FOUND"
|
||||
fi
|
||||
|
||||
if grep -q "llm_grafana_dashboard_uid" "$ROLE_DIR/defaults/main.yml"; then
|
||||
check_pass "Grafana dashboard variables defined"
|
||||
else
|
||||
check_fail "Grafana dashboard variables NOT FOUND"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# 4. Syntax validation
|
||||
echo "4. Template & Configuration Syntax"
|
||||
echo "===================================="
|
||||
|
||||
# Validate shell script
|
||||
if bash -n "$ROLE_DIR/scripts/nvidia-smi-vram-exporter.sh" 2>/dev/null; then
|
||||
check_pass "VRAM exporter script syntax (bash)"
|
||||
else
|
||||
check_fail "VRAM exporter script syntax error"
|
||||
fi
|
||||
|
||||
# Validate JSON dashboard (without Jinja2 rendering)
|
||||
if python3 -m json.tool "$ROLE_DIR/templates/llama-swap-grafana-dashboard.json.j2" > /dev/null 2>&1; then
|
||||
check_pass "Grafana dashboard template syntax (JSON)"
|
||||
else
|
||||
check_fail "Grafana dashboard template syntax error"
|
||||
fi
|
||||
|
||||
# Validate YAML structure (basic check)
|
||||
if grep -q "^- job_name:" "$ROLE_DIR/templates/llama-swap-prometheus-scrape.yml.j2"; then
|
||||
check_pass "Prometheus scrape template structure (YAML)"
|
||||
else
|
||||
check_fail "Prometheus scrape template structure error"
|
||||
fi
|
||||
|
||||
if grep -q "^kind: PrometheusRule" "$ROLE_DIR/templates/llama-swap-alerts.yml.j2"; then
|
||||
check_pass "Alert rules template structure (YAML)"
|
||||
else
|
||||
check_fail "Alert rules template structure error"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# 5. Documentation completeness
|
||||
echo "5. Documentation Completeness"
|
||||
echo "=============================="
|
||||
|
||||
if grep -q "VRAM textfile exporter" "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md"; then
|
||||
check_pass "Monitoring pattern docs include VRAM exporter section"
|
||||
else
|
||||
check_fail "Monitoring pattern docs incomplete: missing VRAM exporter section"
|
||||
fi
|
||||
|
||||
if grep -q "Grafana Dashboard Panels" "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md"; then
|
||||
check_pass "Monitoring pattern docs include dashboard panels section"
|
||||
else
|
||||
check_fail "Monitoring pattern docs incomplete: missing dashboard panels section"
|
||||
fi
|
||||
|
||||
if grep -q "Alert Rules" "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md"; then
|
||||
check_pass "Monitoring pattern docs include alert rules section"
|
||||
else
|
||||
check_fail "Monitoring pattern docs incomplete: missing alert rules section"
|
||||
fi
|
||||
|
||||
if grep -q "18560" "$ROLE_DIR/references/llama-swap-phase3-cutover-results-2026-08-18.md"; then
|
||||
check_pass "Phase 3 results include VRAM baseline figures"
|
||||
else
|
||||
check_fail "Phase 3 results incomplete: missing VRAM baseline"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
|
||||
# 6. Summary
|
||||
echo "================================================================================"
|
||||
echo "Summary"
|
||||
echo "================================================================================"
|
||||
echo "Checks passed: ${GREEN}${CHECKS_PASSED}${NC}"
|
||||
echo "Checks failed: ${RED}${CHECKS_FAILED}${NC}"
|
||||
echo ""
|
||||
|
||||
if [ $CHECKS_FAILED -eq 0 ]; then
|
||||
echo -e "${GREEN}All checks passed! ✓${NC}"
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " 1. Copy Grafana dashboard JSON to cluster/applications/monitoring/"
|
||||
echo " 2. Add Prometheus scrape config to cluster/applications/monitoring/values.yaml"
|
||||
echo " 3. Deploy PrometheusRule CR to cluster/applications/monitoring/"
|
||||
echo " 4. Commit to Git and push (ArgoCD syncs automatically)"
|
||||
echo " 5. Verify metrics in Prometheus UI: http://imagineering.local.mk-labs.cloud/prometheus"
|
||||
echo " 6. Verify dashboard in Grafana UI: http://imagineering.local.mk-labs.cloud/grafana"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}Some checks failed. See above for details.${NC}"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user