#!/bin/bash
# ==============================================================================
# VERIFICATION SCRIPT: GPU/LLM Monitoring Deployment (Task t_57a9f82f)
# ==============================================================================
# Run this script AFTER Ansible role deployment to verify all monitoring
# components are installed and functional.
#
# Usage:
#   bash verify-monitoring-deployment.sh
#
# Expected output: All checks ✓ (green)
# ==============================================================================

set -euo pipefail

ROLE_DIR="/home/hermes/git/homelab/ansible/roles/llm-inference-multimodel"
VRAM_EXPORTER_SCRIPT="/opt/llama-server-monitoring/nvidia-smi-vram-exporter.sh"
VRAM_EXPORTER_OUTPUT="/var/lib/node_exporter/textfile_collector/nvidia.prom"

CHECKS_PASSED=0
CHECKS_FAILED=0

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Helper function for check results
check_pass() {
    local desc="$1"
    echo -e "${GREEN}✓${NC} $desc"
    ((CHECKS_PASSED++))
}

check_fail() {
    local desc="$1"
    local reason="${2:-Unknown reason}"
    echo -e "${RED}✗${NC} $desc"
    echo "  Reason: $reason"
    ((CHECKS_FAILED++))
}

echo "================================================================================"
echo "GPU/LLM Monitoring Deployment Verification"
echo "================================================================================"
echo ""

# 1. Check role structure
echo "1. Role Structure & Deliverables"
echo "=================================="

if [ -f "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md" ]; then
    check_pass "Reference docs: monitoring-llm-homelab-ciro-luciotta-2026.md exists"
else
    check_fail "Reference docs: monitoring-llm-homelab-ciro-luciotta-2026.md NOT FOUND"
fi

if [ -f "$ROLE_DIR/references/llama-swap-phase3-cutover-results-2026-08-18.md" ]; then
    check_pass "Phase 3 results: llama-swap-phase3-cutover-results-2026-08-18.md exists"
else
    check_fail "Phase 3 results: llama-swap-phase3-cutover-results-2026-08-18.md NOT FOUND"
fi

if [ -f "$ROLE_DIR/scripts/nvidia-smi-vram-exporter.sh" ]; then
    check_pass "VRAM exporter script: nvidia-smi-vram-exporter.sh exists"
else
    check_fail "VRAM exporter script: nvidia-smi-vram-exporter.sh NOT FOUND"
fi

if [ -x "$ROLE_DIR/scripts/nvidia-smi-vram-exporter.sh" ]; then
    check_pass "VRAM exporter script: executable"
else
    check_fail "VRAM exporter script: not executable"
fi

if [ -f "$ROLE_DIR/templates/llama-swap-prometheus-scrape.yml.j2" ]; then
    check_pass "Prometheus scrape config template exists"
else
    check_fail "Prometheus scrape config template NOT FOUND"
fi

if [ -f "$ROLE_DIR/templates/llama-swap-grafana-dashboard.json.j2" ]; then
    check_pass "Grafana dashboard template exists"
else
    check_fail "Grafana dashboard template NOT FOUND"
fi

if [ -f "$ROLE_DIR/templates/llama-swap-alerts.yml.j2" ]; then
    check_pass "Alert rules template exists"
else
    check_fail "Alert rules template NOT FOUND"
fi

if [ -f "$ROLE_DIR/tasks/monitoring.yml" ]; then
    check_pass "Monitoring tasks file exists"
else
    check_fail "Monitoring tasks file NOT FOUND"
fi

echo ""

# 2. Check runtime deployment (if on astro-orbiter)
echo "2. Runtime Deployment Status (astro-orbiter)"
echo "=============================================="

if [ -x "$VRAM_EXPORTER_SCRIPT" ]; then
    check_pass "VRAM exporter script deployed at $VRAM_EXPORTER_SCRIPT"
    
    # Try to run it
    if output=$($VRAM_EXPORTER_SCRIPT 2>&1) && [ -f "$VRAM_EXPORTER_OUTPUT" ]; then
        check_pass "VRAM exporter runs successfully"
        
        # Check metric format
        if grep -q "llamacpp_vram_used_mib" "$VRAM_EXPORTER_OUTPUT"; then
            check_pass "VRAM metric format is correct"
            
            # Extract and display the value
            vram_value=$(grep "llamacpp_vram_used_mib " "$VRAM_EXPORTER_OUTPUT" | awk '{print $NF}')
            echo "  Current VRAM usage: ${vram_value} MiB"
        else
            check_fail "VRAM metric format incorrect" "Expected 'llamacpp_vram_used_mib' in output"
        fi
    else
        check_fail "VRAM exporter failed to run" "$output"
    fi
else
    echo -e "${YELLOW}⊘${NC} VRAM exporter not deployed yet (expected if running on non-astro-orbiter)"
fi

if crontab -l 2>/dev/null | grep -q "nvidia-smi-vram-exporter"; then
    check_pass "VRAM exporter cron job is installed"
else
    echo -e "${YELLOW}⊘${NC} VRAM exporter cron job not installed (expected if not on astro-orbiter)"
fi

echo ""

# 3. Check Ansible variables
echo "3. Ansible Configuration Variables"
echo "===================================="

if grep -q "llm_monitoring_enabled" "$ROLE_DIR/defaults/main.yml"; then
    check_pass "llm_monitoring_enabled variable defined"
else
    check_fail "llm_monitoring_enabled variable NOT FOUND"
fi

if grep -q "llm_vram_critical_mib" "$ROLE_DIR/defaults/main.yml"; then
    check_pass "Alert threshold variables defined"
else
    check_fail "Alert threshold variables NOT FOUND"
fi

if grep -q "llm_grafana_dashboard_uid" "$ROLE_DIR/defaults/main.yml"; then
    check_pass "Grafana dashboard variables defined"
else
    check_fail "Grafana dashboard variables NOT FOUND"
fi

echo ""

# 4. Syntax validation
echo "4. Template & Configuration Syntax"
echo "===================================="

# Validate shell script
if bash -n "$ROLE_DIR/scripts/nvidia-smi-vram-exporter.sh" 2>/dev/null; then
    check_pass "VRAM exporter script syntax (bash)"
else
    check_fail "VRAM exporter script syntax error"
fi

# Validate JSON dashboard (without Jinja2 rendering)
if python3 -m json.tool "$ROLE_DIR/templates/llama-swap-grafana-dashboard.json.j2" > /dev/null 2>&1; then
    check_pass "Grafana dashboard template syntax (JSON)"
else
    check_fail "Grafana dashboard template syntax error"
fi

# Validate YAML structure (basic check)
if grep -q "^- job_name:" "$ROLE_DIR/templates/llama-swap-prometheus-scrape.yml.j2"; then
    check_pass "Prometheus scrape template structure (YAML)"
else
    check_fail "Prometheus scrape template structure error"
fi

if grep -q "^kind: PrometheusRule" "$ROLE_DIR/templates/llama-swap-alerts.yml.j2"; then
    check_pass "Alert rules template structure (YAML)"
else
    check_fail "Alert rules template structure error"
fi

echo ""

# 5. Documentation completeness
echo "5. Documentation Completeness"
echo "=============================="

if grep -q "VRAM textfile exporter" "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md"; then
    check_pass "Monitoring pattern docs include VRAM exporter section"
else
    check_fail "Monitoring pattern docs incomplete: missing VRAM exporter section"
fi

if grep -q "Grafana Dashboard Panels" "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md"; then
    check_pass "Monitoring pattern docs include dashboard panels section"
else
    check_fail "Monitoring pattern docs incomplete: missing dashboard panels section"
fi

if grep -q "Alert Rules" "$ROLE_DIR/references/monitoring-llm-homelab-ciro-luciotta-2026.md"; then
    check_pass "Monitoring pattern docs include alert rules section"
else
    check_fail "Monitoring pattern docs incomplete: missing alert rules section"
fi

if grep -q "18560" "$ROLE_DIR/references/llama-swap-phase3-cutover-results-2026-08-18.md"; then
    check_pass "Phase 3 results include VRAM baseline figures"
else
    check_fail "Phase 3 results incomplete: missing VRAM baseline"
fi

echo ""

# 6. Summary
echo "================================================================================"
echo "Summary"
echo "================================================================================"
echo "Checks passed: ${GREEN}${CHECKS_PASSED}${NC}"
echo "Checks failed: ${RED}${CHECKS_FAILED}${NC}"
echo ""

if [ $CHECKS_FAILED -eq 0 ]; then
    echo -e "${GREEN}All checks passed! ✓${NC}"
    echo ""
    echo "Next steps:"
    echo "  1. Copy Grafana dashboard JSON to cluster/applications/monitoring/"
    echo "  2. Add Prometheus scrape config to cluster/applications/monitoring/values.yaml"
    echo "  3. Deploy PrometheusRule CR to cluster/applications/monitoring/"
    echo "  4. Commit to Git and push (ArgoCD syncs automatically)"
    echo "  5. Verify metrics in Prometheus UI: http://imagineering.local.mk-labs.cloud/prometheus"
    echo "  6. Verify dashboard in Grafana UI: http://imagineering.local.mk-labs.cloud/grafana"
    exit 0
else
    echo -e "${RED}Some checks failed. See above for details.${NC}"
    exit 1
fi