423891001c
- Phase 7 task file: monitoring.yml
- node_exporter (port 9100) via apt, systemd managed
- nvidia_gpu_exporter v1.2.2 (port 9835) — GPU util, VRAM, temp, power
- Patches kube-prometheus additionalScrapeConfigs secret with 3 new jobs:
node-astro-orbiter, gpu-astro-orbiter, vllm-astro-orbiter
- Deploys Grafana dashboard ConfigMap via kubectl apply
- Grafana dashboard (11 panels):
- Row 1: GPU util %, VRAM used, GPU temp gauge
- Row 2: GPU power draw, vLLM token throughput, request queue depth
- Row 3: vLLM e2e latency p50/p95/p99, KV cache utilization %
- Row 4: System CPU %, memory, root disk gauge
- defaults/main.yml: llm_gpu_exporter_version, llm_gpu_exporter_port
- handlers/main.yml: restart nvidia-gpu-exporter
36 lines
891 B
YAML
36 lines
891 B
YAML
---
|
|
# ------------------------------------------------------------------------------
|
|
# FILE: roles/llm-inference/handlers/main.yml
|
|
# ------------------------------------------------------------------------------
|
|
|
|
- name: Reload systemd
|
|
ansible.builtin.systemd:
|
|
daemon_reload: true
|
|
listen: "reload systemd"
|
|
|
|
- name: Restart vllm-serve
|
|
ansible.builtin.systemd:
|
|
name: vllm-serve
|
|
state: restarted
|
|
listen: "restart vllm-serve"
|
|
|
|
- name: Restart Hermes on carousel
|
|
ansible.builtin.systemd:
|
|
name: "{{ item }}"
|
|
state: restarted
|
|
scope: user
|
|
loop:
|
|
- hermes-gateway
|
|
- hermes-dashboard
|
|
become: true
|
|
become_user: wed
|
|
delegate_to: carousel-of-progress
|
|
listen: "restart hermes"
|
|
ignore_errors: true
|
|
|
|
- name: Restart nvidia-gpu-exporter
|
|
ansible.builtin.systemd:
|
|
name: nvidia-gpu-exporter
|
|
state: restarted
|
|
listen: "restart nvidia-gpu-exporter"
|