feat(llm-inference): move astro-orbiter monitoring to GitOps (values.yaml + dashboards.yaml)
- Prometheus scrape configs for node/gpu/llama-server exporters on astro-orbiter now declared in cluster/applications/monitoring/values.yaml (additionalScrapeConfigs), applied via ArgoCD sync instead of an imperative kubectl secret patch from the Ansible role. - Grafana dashboard for astro-orbiter LLM inference added as a ConfigMap in cluster/applications/monitoring/dashboards.yaml (grafana_dashboard=1 sidecar label), replacing the role's ad-hoc kubectl apply of a rendered Jinja template. - ansible/roles/llm-inference/tasks/monitoring.yml: removed the kubectl get/patch/apply tasks and orphaned grafana-llm-dashboard.json.j2 template; role now only stands up node_exporter + nvidia_gpu_exporter and verifies they're reachable — cluster-facing config lives in Git. - host_vars/vars.yml + inventory.yml: finalize astro-orbiter as the llama.cpp/RTX 3090 host (jarvis user, ssh key), drop stale Ollama/AMD vars and ollama_server inventory group superseded by the ATX rebuild.
This commit is contained in:
Hermes Agent service account
@@ -9198,4 +9198,231 @@ data:
|
||||
"uid": "nginx",
|
||||
"version": 1,
|
||||
"description": "Ingress-nginx supports a rich collection of prometheus metrics. If you have prometheus and grafana installed on your cluster then prometheus will already be scraping this data due to the scrape annotation on the deployment."
|
||||
}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-llm-inference-dashboard
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
llm-inference-astro-orbiter.json: |
|
||||
{
|
||||
"title": "LLM Inference — astro-orbiter",
|
||||
"uid": "llm-astro-orbiter",
|
||||
"timezone": "browser",
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 38,
|
||||
"tags": ["llm", "gpu", "llama-server", "astro-orbiter"],
|
||||
"time": { "from": "now-1h", "to": "now" },
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "GPU Utilization %",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "x": 0, "y": 0, "w": 8, "h": 8 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0, "max": 100,
|
||||
"color": { "mode": "palette-classic" },
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 70 },
|
||||
{ "color": "red", "value": 90 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "nvidia_smi_utilization_gpu_ratio{hostname=\"astro-orbiter\"} * 100",
|
||||
"legendFormat": "GPU Util"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "GPU VRAM Used",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "x": 8, "y": 0, "w": 8, "h": 8 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes",
|
||||
"color": { "mode": "palette-classic" }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "nvidia_smi_memory_used_bytes{hostname=\"astro-orbiter\"}",
|
||||
"legendFormat": "VRAM Used"
|
||||
},
|
||||
{
|
||||
"expr": "nvidia_smi_memory_total_bytes{hostname=\"astro-orbiter\"}",
|
||||
"legendFormat": "VRAM Total"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "GPU Temperature",
|
||||
"type": "gauge",
|
||||
"gridPos": { "x": 16, "y": 0, "w": 8, "h": 8 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "celsius",
|
||||
"min": 0, "max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 70 },
|
||||
{ "color": "red", "value": 85 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "nvidia_smi_temperature_gpu{hostname=\"astro-orbiter\"}",
|
||||
"legendFormat": "GPU Temp"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "GPU Power Draw",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "x": 0, "y": 8, "w": 8, "h": 8 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "watt",
|
||||
"color": { "mode": "palette-classic" }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "nvidia_smi_power_draw_watts{hostname=\"astro-orbiter\"}",
|
||||
"legendFormat": "Power Draw"
|
||||
},
|
||||
{
|
||||
"expr": "nvidia_smi_power_limit_watts{hostname=\"astro-orbiter\"}",
|
||||
"legendFormat": "Power Limit"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "llama-server — Requests per Second",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "x": 8, "y": 8, "w": 8, "h": 8 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps",
|
||||
"color": { "mode": "palette-classic" }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(llamacpp:tokens_predicted_total{hostname=\"astro-orbiter\"}[1m])",
|
||||
"legendFormat": "Tokens/s (predicted)"
|
||||
},
|
||||
{
|
||||
"expr": "rate(llamacpp:prompt_tokens_total{hostname=\"astro-orbiter\"}[1m])",
|
||||
"legendFormat": "Tokens/s (prompt)"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "llama-server — Requests Processing/Deferred",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "x": 16, "y": 8, "w": 8, "h": 8 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"color": { "mode": "palette-classic" }
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "llamacpp:requests_processing{hostname=\"astro-orbiter\"}",
|
||||
"legendFormat": "Processing"
|
||||
},
|
||||
{
|
||||
"expr": "llamacpp:requests_deferred{hostname=\"astro-orbiter\"}",
|
||||
"legendFormat": "Deferred"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "System — CPU Usage %",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "x": 0, "y": 16, "w": 8, "h": 7 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percent",
|
||||
"min": 0, "max": 100
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\",instance=~\"10.1.71.130:.*\"}[1m])) * 100)",
|
||||
"legendFormat": "CPU Used"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "System — Memory Usage",
|
||||
"type": "timeseries",
|
||||
"gridPos": { "x": 8, "y": 16, "w": 8, "h": 7 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_memory_MemTotal_bytes{instance=~\"10.1.71.130:.*\"} - node_memory_MemAvailable_bytes{instance=~\"10.1.71.130:.*\"}",
|
||||
"legendFormat": "Used"
|
||||
},
|
||||
{
|
||||
"expr": "node_memory_MemTotal_bytes{instance=~\"10.1.71.130:.*\"}",
|
||||
"legendFormat": "Total"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "System — Disk Usage (root)",
|
||||
"type": "gauge",
|
||||
"gridPos": { "x": 16, "y": 16, "w": 8, "h": 7 },
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit",
|
||||
"min": 0, "max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 0.75 },
|
||||
{ "color": "red", "value": 0.90 }
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "1 - (node_filesystem_avail_bytes{instance=~\"10.1.71.130:.*\",mountpoint=\"/\"} / node_filesystem_size_bytes{instance=~\"10.1.71.130:.*\",mountpoint=\"/\"})",
|
||||
"legendFormat": "Root disk"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user