249 lines
8.4 KiB
Bash
Executable File
249 lines
8.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# ==============================================================================
|
|
# VERIFICATION SCRIPT: GPU/LLM Monitoring Deployment (Task t_57a9f82f)
|
|
# ==============================================================================
|
|
# Run this script AFTER Ansible role deployment to verify all monitoring
|
|
# components are installed and functional.
|
|
#
|
|
# Usage:
|
|
# bash verify-monitoring-deployment.sh
|
|
#
|
|
# Expected output: All checks ✓ (green)
|
|
# ==============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
ROLE_DIR="/home/hermes/git/homelab/ansible/roles/llm-inference-multimodel"
|
|
VRAM_EXPORTER_SCRIPT="/opt/llama-server-monitoring/nvidia-smi-vram-exporter.sh"
|
|
VRAM_EXPORTER_OUTPUT="/var/lib/node_exporter/textfile_collector/nvidia.prom"
|
|
|
|
CHECKS_PASSED=0
|
|
CHECKS_FAILED=0
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Helper function for check results
|
|
check_pass() {
|
|
local desc="$1"
|
|
echo -e "${GREEN}✓${NC} $desc"
|
|
((CHECKS_PASSED++))
|
|
}
|
|
|
|
check_fail() {
|
|
local desc="$1"
|
|
local reason="${2:-Unknown reason}"
|
|
echo -e "${RED}✗${NC} $desc"
|
|
echo " Reason: $reason"
|
|
((CHECKS_FAILED++))
|
|
}
|
|
|
|
echo "================================================================================"
|
|
echo "GPU/LLM Monitoring Deployment Verification"
|
|
echo "================================================================================"
|
|
echo ""
|
|
|
|
# 1. Check role structure
|
|
echo "1. Role Structure & Deliverables"
|
|
echo "=================================="
|
|
|
|
if [ -f "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md" ]; then
|
|
check_pass "Reference docs: monitoring-llm-homelab-ciro-luciotta-2026.md exists"
|
|
else
|
|
check_fail "Reference docs: monitoring-llm-homelab-ciro-luciotta-2026.md NOT FOUND"
|
|
fi
|
|
|
|
if [ -f "$ROLE_DIR/references/llama-swap-phase3-cutover-results-2026-08-18.md" ]; then
|
|
check_pass "Phase 3 results: llama-swap-phase3-cutover-results-2026-08-18.md exists"
|
|
else
|
|
check_fail "Phase 3 results: llama-swap-phase3-cutover-results-2026-08-18.md NOT FOUND"
|
|
fi
|
|
|
|
if [ -f "$ROLE_DIR/scripts/nvidia-smi-vram-exporter.sh" ]; then
|
|
check_pass "VRAM exporter script: nvidia-smi-vram-exporter.sh exists"
|
|
else
|
|
check_fail "VRAM exporter script: nvidia-smi-vram-exporter.sh NOT FOUND"
|
|
fi
|
|
|
|
if [ -x "$ROLE_DIR/scripts/nvidia-smi-vram-exporter.sh" ]; then
|
|
check_pass "VRAM exporter script: executable"
|
|
else
|
|
check_fail "VRAM exporter script: not executable"
|
|
fi
|
|
|
|
if [ -f "$ROLE_DIR/templates/llama-swap-prometheus-scrape.yml.j2" ]; then
|
|
check_pass "Prometheus scrape config template exists"
|
|
else
|
|
check_fail "Prometheus scrape config template NOT FOUND"
|
|
fi
|
|
|
|
if [ -f "$ROLE_DIR/templates/llama-swap-grafana-dashboard.json.j2" ]; then
|
|
check_pass "Grafana dashboard template exists"
|
|
else
|
|
check_fail "Grafana dashboard template NOT FOUND"
|
|
fi
|
|
|
|
if [ -f "$ROLE_DIR/templates/llama-swap-alerts.yml.j2" ]; then
|
|
check_pass "Alert rules template exists"
|
|
else
|
|
check_fail "Alert rules template NOT FOUND"
|
|
fi
|
|
|
|
if [ -f "$ROLE_DIR/tasks/monitoring.yml" ]; then
|
|
check_pass "Monitoring tasks file exists"
|
|
else
|
|
check_fail "Monitoring tasks file NOT FOUND"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# 2. Check runtime deployment (if on astro-orbiter)
|
|
echo "2. Runtime Deployment Status (astro-orbiter)"
|
|
echo "=============================================="
|
|
|
|
if [ -x "$VRAM_EXPORTER_SCRIPT" ]; then
|
|
check_pass "VRAM exporter script deployed at $VRAM_EXPORTER_SCRIPT"
|
|
|
|
# Try to run it
|
|
if output=$($VRAM_EXPORTER_SCRIPT 2>&1) && [ -f "$VRAM_EXPORTER_OUTPUT" ]; then
|
|
check_pass "VRAM exporter runs successfully"
|
|
|
|
# Check metric format
|
|
if grep -q "llamacpp_vram_used_mib" "$VRAM_EXPORTER_OUTPUT"; then
|
|
check_pass "VRAM metric format is correct"
|
|
|
|
# Extract and display the value
|
|
vram_value=$(grep "llamacpp_vram_used_mib " "$VRAM_EXPORTER_OUTPUT" | awk '{print $NF}')
|
|
echo " Current VRAM usage: ${vram_value} MiB"
|
|
else
|
|
check_fail "VRAM metric format incorrect" "Expected 'llamacpp_vram_used_mib' in output"
|
|
fi
|
|
else
|
|
check_fail "VRAM exporter failed to run" "$output"
|
|
fi
|
|
else
|
|
echo -e "${YELLOW}⊘${NC} VRAM exporter not deployed yet (expected if running on non-astro-orbiter)"
|
|
fi
|
|
|
|
if crontab -l 2>/dev/null | grep -q "nvidia-smi-vram-exporter"; then
|
|
check_pass "VRAM exporter cron job is installed"
|
|
else
|
|
echo -e "${YELLOW}⊘${NC} VRAM exporter cron job not installed (expected if not on astro-orbiter)"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# 3. Check Ansible variables
|
|
echo "3. Ansible Configuration Variables"
|
|
echo "===================================="
|
|
|
|
if grep -q "llm_monitoring_enabled" "$ROLE_DIR/defaults/main.yml"; then
|
|
check_pass "llm_monitoring_enabled variable defined"
|
|
else
|
|
check_fail "llm_monitoring_enabled variable NOT FOUND"
|
|
fi
|
|
|
|
if grep -q "llm_vram_critical_mib" "$ROLE_DIR/defaults/main.yml"; then
|
|
check_pass "Alert threshold variables defined"
|
|
else
|
|
check_fail "Alert threshold variables NOT FOUND"
|
|
fi
|
|
|
|
if grep -q "llm_grafana_dashboard_uid" "$ROLE_DIR/defaults/main.yml"; then
|
|
check_pass "Grafana dashboard variables defined"
|
|
else
|
|
check_fail "Grafana dashboard variables NOT FOUND"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# 4. Syntax validation
|
|
echo "4. Template & Configuration Syntax"
|
|
echo "===================================="
|
|
|
|
# Validate shell script
|
|
if bash -n "$ROLE_DIR/scripts/nvidia-smi-vram-exporter.sh" 2>/dev/null; then
|
|
check_pass "VRAM exporter script syntax (bash)"
|
|
else
|
|
check_fail "VRAM exporter script syntax error"
|
|
fi
|
|
|
|
# Validate JSON dashboard (without Jinja2 rendering)
|
|
if python3 -m json.tool "$ROLE_DIR/templates/llama-swap-grafana-dashboard.json.j2" > /dev/null 2>&1; then
|
|
check_pass "Grafana dashboard template syntax (JSON)"
|
|
else
|
|
check_fail "Grafana dashboard template syntax error"
|
|
fi
|
|
|
|
# Validate YAML structure (basic check)
|
|
if grep -q "^- job_name:" "$ROLE_DIR/templates/llama-swap-prometheus-scrape.yml.j2"; then
|
|
check_pass "Prometheus scrape template structure (YAML)"
|
|
else
|
|
check_fail "Prometheus scrape template structure error"
|
|
fi
|
|
|
|
if grep -q "^kind: PrometheusRule" "$ROLE_DIR/templates/llama-swap-alerts.yml.j2"; then
|
|
check_pass "Alert rules template structure (YAML)"
|
|
else
|
|
check_fail "Alert rules template structure error"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# 5. Documentation completeness
|
|
echo "5. Documentation Completeness"
|
|
echo "=============================="
|
|
|
|
if grep -q "VRAM textfile exporter" "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md"; then
|
|
check_pass "Monitoring pattern docs include VRAM exporter section"
|
|
else
|
|
check_fail "Monitoring pattern docs incomplete: missing VRAM exporter section"
|
|
fi
|
|
|
|
if grep -q "Grafana Dashboard Panels" "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md"; then
|
|
check_pass "Monitoring pattern docs include dashboard panels section"
|
|
else
|
|
check_fail "Monitoring pattern docs incomplete: missing dashboard panels section"
|
|
fi
|
|
|
|
if grep -q "Alert Rules" "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md"; then
|
|
check_pass "Monitoring pattern docs include alert rules section"
|
|
else
|
|
check_fail "Monitoring pattern docs incomplete: missing alert rules section"
|
|
fi
|
|
|
|
if grep -q "18560" "$ROLE_DIR/references/llama-swap-phase3-cutover-results-2026-08-18.md"; then
|
|
check_pass "Phase 3 results include VRAM baseline figures"
|
|
else
|
|
check_fail "Phase 3 results incomplete: missing VRAM baseline"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# 6. Summary
|
|
echo "================================================================================"
|
|
echo "Summary"
|
|
echo "================================================================================"
|
|
echo "Checks passed: ${GREEN}${CHECKS_PASSED}${NC}"
|
|
echo "Checks failed: ${RED}${CHECKS_FAILED}${NC}"
|
|
echo ""
|
|
|
|
if [ $CHECKS_FAILED -eq 0 ]; then
|
|
echo -e "${GREEN}All checks passed! ✓${NC}"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Copy Grafana dashboard JSON to cluster/applications/monitoring/"
|
|
echo " 2. Add Prometheus scrape config to cluster/applications/monitoring/values.yaml"
|
|
echo " 3. Deploy PrometheusRule CR to cluster/applications/monitoring/"
|
|
echo " 4. Commit to Git and push (ArgoCD syncs automatically)"
|
|
echo " 5. Verify metrics in Prometheus UI: http://imagineering.local.mk-labs.cloud/prometheus"
|
|
echo " 6. Verify dashboard in Grafana UI: http://imagineering.local.mk-labs.cloud/grafana"
|
|
exit 0
|
|
else
|
|
echo -e "${RED}Some checks failed. See above for details.${NC}"
|
|
exit 1
|
|
fi
|