feat(llm-inference): move astro-orbiter monitoring to GitOps (values.yaml + dashboards.yaml)
- Prometheus scrape configs for node/gpu/llama-server exporters on astro-orbiter now declared in cluster/applications/monitoring/values.yaml (additionalScrapeConfigs), applied via ArgoCD sync instead of an imperative kubectl secret patch from the Ansible role. - Grafana dashboard for astro-orbiter LLM inference added as a ConfigMap in cluster/applications/monitoring/dashboards.yaml (grafana_dashboard=1 sidecar label), replacing the role's ad-hoc kubectl apply of a rendered Jinja template. - ansible/roles/llm-inference/tasks/monitoring.yml: removed the kubectl get/patch/apply tasks and orphaned grafana-llm-dashboard.json.j2 template; role now only stands up node_exporter + nvidia_gpu_exporter and verifies they're reachable — cluster-facing config lives in Git. - host_vars/vars.yml + inventory.yml: finalize astro-orbiter as the llama.cpp/RTX 3090 host (jarvis user, ssh key), drop stale Ollama/AMD vars and ollama_server inventory group superseded by the ATX rebuild.
This commit is contained in:
Hermes Agent service account
@@ -2,16 +2,24 @@
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/llm-inference/tasks/monitoring.yml
|
||||
# DESCRIPTION: Phase 7 — Prometheus monitoring for the LLM inference stack.
|
||||
# Deploys three metric sources on astro-orbiter:
|
||||
# Deploys two metric-producing exporters on astro-orbiter:
|
||||
#
|
||||
# 1. node_exporter (port 9100) — system: CPU, RAM, disk, network
|
||||
# 2. nvidia_gpu_exporter (port 9835) — GPU: VRAM, temp, util, power
|
||||
# 3. vLLM built-in metrics (port 8000/metrics) — already exposed
|
||||
# by vLLM; just needs a scrape job (no extra process)
|
||||
# 3. llama-server built-in metrics (port 8000/metrics, enabled via
|
||||
# --metrics) — just needs a scrape job (no extra process)
|
||||
#
|
||||
# Wires all three into Prometheus via additionalScrapeConfigs on
|
||||
# the kube-prometheus-stack secret on carousel-of-progress.
|
||||
# Deploys a Grafana dashboard ConfigMap in the monitoring namespace.
|
||||
# GitOps note: the Prometheus scrape jobs for all three targets and
|
||||
# the Grafana dashboard are declared in the homelab Git repo and
|
||||
# applied by ArgoCD — NOT by this role:
|
||||
# - cluster/applications/monitoring/values.yaml
|
||||
# (prometheus.prometheusSpec.additionalScrapeConfigs)
|
||||
# - cluster/applications/monitoring/dashboards.yaml
|
||||
# (grafana-llm-inference-dashboard ConfigMap)
|
||||
# This role's job is only to stand up the two exporters + verify
|
||||
# they're reachable. Do NOT reintroduce kubectl patch/apply tasks
|
||||
# here — cluster-facing changes go through Git commit + ArgoCD
|
||||
# sync so state stays reproducible and self-healing.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
@@ -114,85 +122,10 @@
|
||||
changed_when: false
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# 3. Patch additionalScrapeConfigs secret on carousel-of-progress
|
||||
# Adds three new scrape jobs: node, gpu, vllm
|
||||
# 3. Prometheus scrape configs + Grafana dashboard
|
||||
#
|
||||
# Intentionally NOT managed here. See file header: these are declared
|
||||
# in cluster/applications/monitoring/{values.yaml,dashboards.yaml} in
|
||||
# the homelab Git repo and rolled out by ArgoCD sync, keeping cluster
|
||||
# state in Git rather than mutated imperatively from the control node.
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
- name: Read current additionalScrapeConfigs from Prometheus secret
|
||||
ansible.builtin.command:
|
||||
cmd: >
|
||||
kubectl get secret monitoring-kube-prometheus-prometheus-scrape-confg
|
||||
-n monitoring
|
||||
-o jsonpath='{.data.additional-scrape-configs\.yaml}'
|
||||
register: current_scrape_b64
|
||||
changed_when: false
|
||||
delegate_to: carousel-of-progress
|
||||
become: false
|
||||
|
||||
- name: Decode current scrape configs
|
||||
ansible.builtin.set_fact:
|
||||
current_scrape_yaml: "{{ current_scrape_b64.stdout | b64decode }}"
|
||||
|
||||
- name: Check if astro-orbiter scrape jobs already present
|
||||
ansible.builtin.set_fact:
|
||||
scrape_already_patched: "{{ 'astro-orbiter' in current_scrape_yaml }}"
|
||||
|
||||
- name: Append astro-orbiter scrape jobs to additionalScrapeConfigs
|
||||
when: not scrape_already_patched
|
||||
block:
|
||||
- name: Build new scrape config with astro-orbiter jobs appended
|
||||
ansible.builtin.set_fact:
|
||||
new_scrape_yaml: |
|
||||
{{ current_scrape_yaml }}
|
||||
- job_name: node-astro-orbiter
|
||||
scrape_interval: 30s
|
||||
static_configs:
|
||||
- labels:
|
||||
hostname: astro-orbiter
|
||||
targets:
|
||||
- {{ hostvars['astro-orbiter']['ansible_host'] }}:9100
|
||||
- job_name: gpu-astro-orbiter
|
||||
scrape_interval: 15s
|
||||
static_configs:
|
||||
- labels:
|
||||
hostname: astro-orbiter
|
||||
gpu: rtx3090
|
||||
targets:
|
||||
- {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_gpu_exporter_port }}
|
||||
- job_name: vllm-astro-orbiter
|
||||
scrape_interval: 15s
|
||||
metrics_path: /metrics
|
||||
static_configs:
|
||||
- labels:
|
||||
hostname: astro-orbiter
|
||||
model: "{{ llm_hf_model }}"
|
||||
targets:
|
||||
- {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_serve_port }}
|
||||
|
||||
- name: Patch Prometheus additionalScrapeConfigs secret
|
||||
ansible.builtin.command:
|
||||
cmd: >
|
||||
kubectl patch secret monitoring-kube-prometheus-prometheus-scrape-confg
|
||||
-n monitoring
|
||||
--type=json
|
||||
-p='[{"op":"replace","path":"/data/additional-scrape-configs.yaml","value":"{{ new_scrape_yaml | b64encode }}"}]'
|
||||
delegate_to: carousel-of-progress
|
||||
become: false
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# 4. Deploy Grafana dashboard
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
- name: Deploy LLM inference Grafana dashboard ConfigMap
|
||||
ansible.builtin.template:
|
||||
src: grafana-llm-dashboard.json.j2
|
||||
dest: /tmp/grafana-llm-dashboard-cm.yaml
|
||||
delegate_to: carousel-of-progress
|
||||
become: false
|
||||
|
||||
- name: Apply Grafana dashboard ConfigMap to cluster
|
||||
ansible.builtin.command:
|
||||
cmd: kubectl apply -f /tmp/grafana-llm-dashboard-cm.yaml
|
||||
delegate_to: carousel-of-progress
|
||||
become: false
|
||||
changed_when: true
|
||||
|
||||
Reference in New Issue
Block a user