diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml index 7d982b6..1727646 100644 --- a/ansible/host_vars/astro-orbiter/vars.yml +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -2,11 +2,14 @@ # ------------------------------------------------------------------------------ # FILE: ansible/host_vars/astro_orbiter/vars.yml # HOST: astro-orbiter (10.1.71.130) -# ROLE: Ollama inference host with AMD RX 5700 GPU passthrough +# ROLE: llama.cpp LLM inference host — Ryzen 7 5800XT / RTX 3090 (ATX rebuild, +# 2026-08-04). Superseded the prior AMD RX 5700 / Ollama config below; +# drive was transplanted into new hardware, not reinstalled. # ------------------------------------------------------------------------------ ansible_host: 10.1.71.130 -ansible_user: wed +ansible_user: jarvis +ansible_ssh_private_key_file: ~/.ssh/id_jarvis ansible_become: true # LVM root expansion — xlarge template uses sda3 partition, standard VG/LV names @@ -15,11 +18,3 @@ common_root_pv: /dev/sda3 common_root_vg: ubuntu-vg common_root_lv: ubuntu-lv -# Ollama — all defaults apply; explicitly documented here for visibility -ollama_rocm_version: "6.2" -ollama_default_model: "qwen3:8b" -ollama_hsa_override_gfx_version: "10.1.0" -ollama_data_disk: /dev/sdb -ollama_data_vg: ollama-vg -ollama_data_lv: ollama-lv -ollama_data_dir: /var/lib/ollama diff --git a/ansible/inventory.yml b/ansible/inventory.yml index 50b4f96..a90f53c 100755 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -59,13 +59,6 @@ n8n_server: hosts: tiki-room: -ollama_server: - hosts: - astro-orbiter: - ansible_host: 10.1.71.130 - ansible_user: wed - ansible_become: true - astro_orbiter: hosts: astro-orbiter: @@ -76,6 +69,7 @@ hermes_server: ansible_host: 10.1.71.131 ansible_user: wed ansible_become: true + ansible_ssh_private_key_file: ~/.ssh/ansible honcho_server: hosts: diff --git a/ansible/roles/llm-inference/tasks/monitoring.yml b/ansible/roles/llm-inference/tasks/monitoring.yml index 9eac145..fcc1ddf 100644 --- a/ansible/roles/llm-inference/tasks/monitoring.yml +++ b/ansible/roles/llm-inference/tasks/monitoring.yml @@ -2,16 +2,24 @@ # ------------------------------------------------------------------------------ # FILE: roles/llm-inference/tasks/monitoring.yml # DESCRIPTION: Phase 7 — Prometheus monitoring for the LLM inference stack. -# Deploys three metric sources on astro-orbiter: +# Deploys two metric-producing exporters on astro-orbiter: # # 1. node_exporter (port 9100) — system: CPU, RAM, disk, network # 2. nvidia_gpu_exporter (port 9835) — GPU: VRAM, temp, util, power -# 3. vLLM built-in metrics (port 8000/metrics) — already exposed -# by vLLM; just needs a scrape job (no extra process) +# 3. llama-server built-in metrics (port 8000/metrics, enabled via +# --metrics) — just needs a scrape job (no extra process) # -# Wires all three into Prometheus via additionalScrapeConfigs on -# the kube-prometheus-stack secret on carousel-of-progress. -# Deploys a Grafana dashboard ConfigMap in the monitoring namespace. +# GitOps note: the Prometheus scrape jobs for all three targets and +# the Grafana dashboard are declared in the homelab Git repo and +# applied by ArgoCD — NOT by this role: +# - cluster/applications/monitoring/values.yaml +# (prometheus.prometheusSpec.additionalScrapeConfigs) +# - cluster/applications/monitoring/dashboards.yaml +# (grafana-llm-inference-dashboard ConfigMap) +# This role's job is only to stand up the two exporters + verify +# they're reachable. Do NOT reintroduce kubectl patch/apply tasks +# here — cluster-facing changes go through Git commit + ArgoCD +# sync so state stays reproducible and self-healing. # ------------------------------------------------------------------------------ # ----------------------------------------------------------------------- @@ -114,85 +122,10 @@ changed_when: false # ----------------------------------------------------------------------- -# 3. Patch additionalScrapeConfigs secret on carousel-of-progress -# Adds three new scrape jobs: node, gpu, vllm +# 3. Prometheus scrape configs + Grafana dashboard +# +# Intentionally NOT managed here. See file header: these are declared +# in cluster/applications/monitoring/{values.yaml,dashboards.yaml} in +# the homelab Git repo and rolled out by ArgoCD sync, keeping cluster +# state in Git rather than mutated imperatively from the control node. # ----------------------------------------------------------------------- - -- name: Read current additionalScrapeConfigs from Prometheus secret - ansible.builtin.command: - cmd: > - kubectl get secret monitoring-kube-prometheus-prometheus-scrape-confg - -n monitoring - -o jsonpath='{.data.additional-scrape-configs\.yaml}' - register: current_scrape_b64 - changed_when: false - delegate_to: carousel-of-progress - become: false - -- name: Decode current scrape configs - ansible.builtin.set_fact: - current_scrape_yaml: "{{ current_scrape_b64.stdout | b64decode }}" - -- name: Check if astro-orbiter scrape jobs already present - ansible.builtin.set_fact: - scrape_already_patched: "{{ 'astro-orbiter' in current_scrape_yaml }}" - -- name: Append astro-orbiter scrape jobs to additionalScrapeConfigs - when: not scrape_already_patched - block: - - name: Build new scrape config with astro-orbiter jobs appended - ansible.builtin.set_fact: - new_scrape_yaml: | - {{ current_scrape_yaml }} - - job_name: node-astro-orbiter - scrape_interval: 30s - static_configs: - - labels: - hostname: astro-orbiter - targets: - - {{ hostvars['astro-orbiter']['ansible_host'] }}:9100 - - job_name: gpu-astro-orbiter - scrape_interval: 15s - static_configs: - - labels: - hostname: astro-orbiter - gpu: rtx3090 - targets: - - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_gpu_exporter_port }} - - job_name: vllm-astro-orbiter - scrape_interval: 15s - metrics_path: /metrics - static_configs: - - labels: - hostname: astro-orbiter - model: "{{ llm_hf_model }}" - targets: - - {{ hostvars['astro-orbiter']['ansible_host'] }}:{{ llm_serve_port }} - - - name: Patch Prometheus additionalScrapeConfigs secret - ansible.builtin.command: - cmd: > - kubectl patch secret monitoring-kube-prometheus-prometheus-scrape-confg - -n monitoring - --type=json - -p='[{"op":"replace","path":"/data/additional-scrape-configs.yaml","value":"{{ new_scrape_yaml | b64encode }}"}]' - delegate_to: carousel-of-progress - become: false - -# ----------------------------------------------------------------------- -# 4. Deploy Grafana dashboard -# ----------------------------------------------------------------------- - -- name: Deploy LLM inference Grafana dashboard ConfigMap - ansible.builtin.template: - src: grafana-llm-dashboard.json.j2 - dest: /tmp/grafana-llm-dashboard-cm.yaml - delegate_to: carousel-of-progress - become: false - -- name: Apply Grafana dashboard ConfigMap to cluster - ansible.builtin.command: - cmd: kubectl apply -f /tmp/grafana-llm-dashboard-cm.yaml - delegate_to: carousel-of-progress - become: false - changed_when: true diff --git a/ansible/roles/llm-inference/tasks/serve.yml b/ansible/roles/llm-inference/tasks/serve.yml index 96feceb..650a76b 100644 --- a/ansible/roles/llm-inference/tasks/serve.yml +++ b/ansible/roles/llm-inference/tasks/serve.yml @@ -22,6 +22,17 @@ state: present update_cache: false +# NOTE: nvidia-driver-595-open provides the runtime driver only (nvidia-smi, +# libcuda.so) — it does NOT ship nvcc/CUDA headers needed to build GGML_CUDA=ON. +# Ubuntu 24.04's nvidia-cuda-toolkit (12.0.x) is sufficient to build llama.cpp +# against; it does not need to match the 595 driver's CUDA 13.2 runtime version. +- name: Install NVIDIA CUDA toolkit (nvcc) for building llama.cpp with CUDA support + ansible.builtin.apt: + name: nvidia-cuda-toolkit + state: present + update_cache: false + become: true + - name: Clone llama.cpp repository ansible.builtin.git: repo: https://github.com/ggml-org/llama.cpp.git @@ -30,6 +41,20 @@ update: false become: true +- name: Check for incomplete/stale llama.cpp CMake configuration + ansible.builtin.stat: + path: /opt/llama.cpp/build/Makefile + register: llama_cmake_generated + +- name: Remove stale llama.cpp build dir if CMake configure never completed + ansible.builtin.file: + path: /opt/llama.cpp/build + state: absent + become: true + when: + - not llama_cmake_generated.stat.exists + - not (ansible_check_mode | default(false)) + - name: Build llama.cpp with CUDA support ansible.builtin.command: cmd: cmake -B build -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Release @@ -53,6 +78,11 @@ group: "{{ llm_venv_owner }}" mode: "0755" +- name: Check whether GGUF already exists (avoid re-downloading 16.6GB on every run) + ansible.builtin.stat: + path: "{{ llm_gguf_path }}" + register: llm_gguf_stat + - name: Download Gemma 2 27B Q4_K_M GGUF from HuggingFace ansible.builtin.get_url: url: "https://huggingface.co/bartowski/gemma-2-27b-it-GGUF/resolve/main/gemma-2-27b-it-Q4_K_M.gguf" @@ -63,8 +93,13 @@ group: "{{ llm_venv_owner }}" mode: "0644" timeout: 7200 + force: false become: true no_log: true + # Idempotency: skip entirely once the file exists and is reasonably sized + # (the finished GGUF is ~16.6GB; guard against a truncated partial download + # being mistaken for complete by only trusting files > 15GB). + when: not llm_gguf_stat.stat.exists or (llm_gguf_stat.stat.size | int) < 15000000000 - name: Disable and stop vllm-serve if present ansible.builtin.systemd: diff --git a/ansible/roles/llm-inference/templates/grafana-llm-dashboard.json.j2 b/ansible/roles/llm-inference/templates/grafana-llm-dashboard.json.j2 deleted file mode 100644 index a9598ef..0000000 --- a/ansible/roles/llm-inference/templates/grafana-llm-dashboard.json.j2 +++ /dev/null @@ -1,293 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-llm-inference-dashboard - namespace: monitoring - labels: - grafana_dashboard: "1" -data: - llm-inference.json: | - { - "title": "LLM Inference — astro-orbiter", - "uid": "llm-astro-orbiter", - "timezone": "browser", - "refresh": "30s", - "schemaVersion": 38, - "tags": ["llm", "gpu", "vllm", "astro-orbiter"], - "time": { "from": "now-1h", "to": "now" }, - "templating": { - "list": [ - { - "name": "instance", - "type": "constant", - "label": "Host", - "query": "{{ hostvars['astro-orbiter']['ansible_host'] }}", - "hide": 0 - } - ] - }, - "panels": [ - { - "id": 1, - "title": "GPU Utilization %", - "type": "timeseries", - "gridPos": { "x": 0, "y": 0, "w": 8, "h": 8 }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, "max": 100, - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 70 }, - { "color": "red", "value": 90 } - ] - } - } - }, - "targets": [ - { - "expr": "nvidia_smi_utilization_gpu_ratio{hostname=\"astro-orbiter\"} * 100", - "legendFormat": "GPU Util" - } - ] - }, - { - "id": 2, - "title": "GPU VRAM Used", - "type": "timeseries", - "gridPos": { "x": 8, "y": 0, "w": 8, "h": 8 }, - "fieldConfig": { - "defaults": { - "unit": "bytes", - "color": { "mode": "palette-classic" } - } - }, - "targets": [ - { - "expr": "nvidia_smi_memory_used_bytes{hostname=\"astro-orbiter\"}", - "legendFormat": "VRAM Used" - }, - { - "expr": "nvidia_smi_memory_total_bytes{hostname=\"astro-orbiter\"}", - "legendFormat": "VRAM Total" - } - ] - }, - { - "id": 3, - "title": "GPU Temperature", - "type": "gauge", - "gridPos": { "x": 16, "y": 0, "w": 8, "h": 8 }, - "fieldConfig": { - "defaults": { - "unit": "celsius", - "min": 0, "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 70 }, - { "color": "red", "value": 85 } - ] - } - } - }, - "targets": [ - { - "expr": "nvidia_smi_temperature_gpu{hostname=\"astro-orbiter\"}", - "legendFormat": "GPU Temp" - } - ] - }, - { - "id": 4, - "title": "GPU Power Draw", - "type": "timeseries", - "gridPos": { "x": 0, "y": 8, "w": 8, "h": 8 }, - "fieldConfig": { - "defaults": { - "unit": "watt", - "color": { "mode": "palette-classic" } - } - }, - "targets": [ - { - "expr": "nvidia_smi_power_draw_watts{hostname=\"astro-orbiter\"}", - "legendFormat": "Power Draw" - }, - { - "expr": "nvidia_smi_power_limit_watts{hostname=\"astro-orbiter\"}", - "legendFormat": "Power Limit" - } - ] - }, - { - "id": 5, - "title": "vLLM — Token Throughput", - "type": "timeseries", - "gridPos": { "x": 8, "y": 8, "w": 8, "h": 8 }, - "fieldConfig": { - "defaults": { - "unit": "reqps", - "color": { "mode": "palette-classic" } - } - }, - "targets": [ - { - "expr": "rate(vllm:generation_tokens_total{hostname=\"astro-orbiter\"}[1m])", - "legendFormat": "Tokens/s (gen)" - }, - { - "expr": "rate(vllm:prompt_tokens_total{hostname=\"astro-orbiter\"}[1m])", - "legendFormat": "Tokens/s (prompt)" - } - ] - }, - { - "id": 6, - "title": "vLLM — Request Queue Depth", - "type": "timeseries", - "gridPos": { "x": 16, "y": 8, "w": 8, "h": 8 }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "palette-classic" } - } - }, - "targets": [ - { - "expr": "vllm:num_requests_running{hostname=\"astro-orbiter\"}", - "legendFormat": "Running" - }, - { - "expr": "vllm:num_requests_waiting{hostname=\"astro-orbiter\"}", - "legendFormat": "Waiting" - }, - { - "expr": "vllm:num_requests_swapped{hostname=\"astro-orbiter\"}", - "legendFormat": "Swapped" - } - ] - }, - { - "id": 7, - "title": "vLLM — E2E Request Latency (p50/p95/p99)", - "type": "timeseries", - "gridPos": { "x": 0, "y": 16, "w": 12, "h": 8 }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { "mode": "palette-classic" } - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.50, rate(vllm:e2e_request_latency_seconds_bucket{hostname=\"astro-orbiter\"}[5m]))", - "legendFormat": "p50" - }, - { - "expr": "histogram_quantile(0.95, rate(vllm:e2e_request_latency_seconds_bucket{hostname=\"astro-orbiter\"}[5m]))", - "legendFormat": "p95" - }, - { - "expr": "histogram_quantile(0.99, rate(vllm:e2e_request_latency_seconds_bucket{hostname=\"astro-orbiter\"}[5m]))", - "legendFormat": "p99" - } - ] - }, - { - "id": 8, - "title": "vLLM — KV Cache Utilization %", - "type": "timeseries", - "gridPos": { "x": 12, "y": 16, "w": 12, "h": 8 }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, "max": 100, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 75 }, - { "color": "red", "value": 90 } - ] - } - } - }, - "targets": [ - { - "expr": "vllm:gpu_cache_usage_perc{hostname=\"astro-orbiter\"} * 100", - "legendFormat": "KV Cache %" - } - ] - }, - { - "id": 9, - "title": "System — CPU Usage %", - "type": "timeseries", - "gridPos": { "x": 0, "y": 24, "w": 8, "h": 7 }, - "fieldConfig": { - "defaults": { - "unit": "percent", - "min": 0, "max": 100 - } - }, - "targets": [ - { - "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\",instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"}[1m])) * 100)", - "legendFormat": "CPU Used" - } - ] - }, - { - "id": 10, - "title": "System — Memory Usage", - "type": "timeseries", - "gridPos": { "x": 8, "y": 24, "w": 8, "h": 7 }, - "fieldConfig": { - "defaults": { - "unit": "bytes" - } - }, - "targets": [ - { - "expr": "node_memory_MemTotal_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"} - node_memory_MemAvailable_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"}", - "legendFormat": "Used" - }, - { - "expr": "node_memory_MemTotal_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\"}", - "legendFormat": "Total" - } - ] - }, - { - "id": 11, - "title": "System — Disk Usage (root)", - "type": "gauge", - "gridPos": { "x": 16, "y": 24, "w": 8, "h": 7 }, - "fieldConfig": { - "defaults": { - "unit": "percentunit", - "min": 0, "max": 1, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.75 }, - { "color": "red", "value": 0.90 } - ] - } - } - }, - "targets": [ - { - "expr": "1 - (node_filesystem_avail_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\",mountpoint=\"/\"} / node_filesystem_size_bytes{instance=~\"{{ hostvars['astro-orbiter']['ansible_host'] }}:.*\",mountpoint=\"/\"})", - "legendFormat": "Root disk" - } - ] - } - ] - } diff --git a/ansible/roles/llm-inference/templates/llama-server.service.j2 b/ansible/roles/llm-inference/templates/llama-server.service.j2 index 7554f71..1c64ab3 100644 --- a/ansible/roles/llm-inference/templates/llama-server.service.j2 +++ b/ansible/roles/llm-inference/templates/llama-server.service.j2 @@ -15,7 +15,12 @@ ExecStart=/opt/llama.cpp/build/bin/llama-server \ --ctx-size {{ llm_max_model_len }} \ --n-gpu-layers {{ llm_gpu_layers }} \ --parallel {{ llm_parallel_slots }} \ - --chat-template gemma + --metrics +# NOTE: no --chat-template flag — llama-server auto-detects and uses the +# GGUF's own embedded Jinja chat template (verified correct Gemma-2 +# start_of_turn/end_of_turn format for bartowski's gemma-2-27b-it-Q4_K_M). +# The built-in "--chat-template gemma" name does NOT match this model's +# expected format on this llama.cpp build and produced garbled completions. Restart=on-failure RestartSec=10 TimeoutStartSec=120 diff --git a/cluster/applications/monitoring/dashboards.yaml b/cluster/applications/monitoring/dashboards.yaml index fd75c43..424004d 100644 --- a/cluster/applications/monitoring/dashboards.yaml +++ b/cluster/applications/monitoring/dashboards.yaml @@ -9198,4 +9198,231 @@ data: "uid": "nginx", "version": 1, "description": "Ingress-nginx supports a rich collection of prometheus metrics. If you have prometheus and grafana installed on your cluster then prometheus will already be scraping this data due to the scrape annotation on the deployment." + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-llm-inference-dashboard + namespace: monitoring + labels: + grafana_dashboard: "1" +data: + llm-inference-astro-orbiter.json: | + { + "title": "LLM Inference — astro-orbiter", + "uid": "llm-astro-orbiter", + "timezone": "browser", + "refresh": "30s", + "schemaVersion": 38, + "tags": ["llm", "gpu", "llama-server", "astro-orbiter"], + "time": { "from": "now-1h", "to": "now" }, + "panels": [ + { + "id": 1, + "title": "GPU Utilization %", + "type": "timeseries", + "gridPos": { "x": 0, "y": 0, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, "max": 100, + "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 90 } + ] + } + } + }, + "targets": [ + { + "expr": "nvidia_smi_utilization_gpu_ratio{hostname=\"astro-orbiter\"} * 100", + "legendFormat": "GPU Util" + } + ] + }, + { + "id": 2, + "title": "GPU VRAM Used", + "type": "timeseries", + "gridPos": { "x": 8, "y": 0, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { "mode": "palette-classic" } + } + }, + "targets": [ + { + "expr": "nvidia_smi_memory_used_bytes{hostname=\"astro-orbiter\"}", + "legendFormat": "VRAM Used" + }, + { + "expr": "nvidia_smi_memory_total_bytes{hostname=\"astro-orbiter\"}", + "legendFormat": "VRAM Total" + } + ] + }, + { + "id": 3, + "title": "GPU Temperature", + "type": "gauge", + "gridPos": { "x": 16, "y": 0, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "celsius", + "min": 0, "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "red", "value": 85 } + ] + } + } + }, + "targets": [ + { + "expr": "nvidia_smi_temperature_gpu{hostname=\"astro-orbiter\"}", + "legendFormat": "GPU Temp" + } + ] + }, + { + "id": 4, + "title": "GPU Power Draw", + "type": "timeseries", + "gridPos": { "x": 0, "y": 8, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "watt", + "color": { "mode": "palette-classic" } + } + }, + "targets": [ + { + "expr": "nvidia_smi_power_draw_watts{hostname=\"astro-orbiter\"}", + "legendFormat": "Power Draw" + }, + { + "expr": "nvidia_smi_power_limit_watts{hostname=\"astro-orbiter\"}", + "legendFormat": "Power Limit" + } + ] + }, + { + "id": 5, + "title": "llama-server — Requests per Second", + "type": "timeseries", + "gridPos": { "x": 8, "y": 8, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "color": { "mode": "palette-classic" } + } + }, + "targets": [ + { + "expr": "rate(llamacpp:tokens_predicted_total{hostname=\"astro-orbiter\"}[1m])", + "legendFormat": "Tokens/s (predicted)" + }, + { + "expr": "rate(llamacpp:prompt_tokens_total{hostname=\"astro-orbiter\"}[1m])", + "legendFormat": "Tokens/s (prompt)" + } + ] + }, + { + "id": 6, + "title": "llama-server — Requests Processing/Deferred", + "type": "timeseries", + "gridPos": { "x": 16, "y": 8, "w": 8, "h": 8 }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "mode": "palette-classic" } + } + }, + "targets": [ + { + "expr": "llamacpp:requests_processing{hostname=\"astro-orbiter\"}", + "legendFormat": "Processing" + }, + { + "expr": "llamacpp:requests_deferred{hostname=\"astro-orbiter\"}", + "legendFormat": "Deferred" + } + ] + }, + { + "id": 9, + "title": "System — CPU Usage %", + "type": "timeseries", + "gridPos": { "x": 0, "y": 16, "w": 8, "h": 7 }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "min": 0, "max": 100 + } + }, + "targets": [ + { + "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\",instance=~\"10.1.71.130:.*\"}[1m])) * 100)", + "legendFormat": "CPU Used" + } + ] + }, + { + "id": 10, + "title": "System — Memory Usage", + "type": "timeseries", + "gridPos": { "x": 8, "y": 16, "w": 8, "h": 7 }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "targets": [ + { + "expr": "node_memory_MemTotal_bytes{instance=~\"10.1.71.130:.*\"} - node_memory_MemAvailable_bytes{instance=~\"10.1.71.130:.*\"}", + "legendFormat": "Used" + }, + { + "expr": "node_memory_MemTotal_bytes{instance=~\"10.1.71.130:.*\"}", + "legendFormat": "Total" + } + ] + }, + { + "id": 11, + "title": "System — Disk Usage (root)", + "type": "gauge", + "gridPos": { "x": 16, "y": 16, "w": 8, "h": 7 }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "min": 0, "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 0.75 }, + { "color": "red", "value": 0.90 } + ] + } + } + }, + "targets": [ + { + "expr": "1 - (node_filesystem_avail_bytes{instance=~\"10.1.71.130:.*\",mountpoint=\"/\"} / node_filesystem_size_bytes{instance=~\"10.1.71.130:.*\",mountpoint=\"/\"})", + "legendFormat": "Root disk" + } + ] + } + ] } \ No newline at end of file diff --git a/cluster/applications/monitoring/values.yaml b/cluster/applications/monitoring/values.yaml index 52a48e7..fcf46f7 100644 --- a/cluster/applications/monitoring/values.yaml +++ b/cluster/applications/monitoring/values.yaml @@ -212,6 +212,36 @@ prometheus: # array: utilidor # array_type: physical + # astro-orbiter — LLM inference host (Ryzen 7 5800XT / RTX 3090) + # Managed by roles/llm-inference (Phase monitoring). Three targets: + # node (system), gpu (nvidia_gpu_exporter), llama-server (inference metrics) + - job_name: node-astro-orbiter + scrape_interval: 30s + static_configs: + - targets: + - 10.1.71.130:9100 + labels: + hostname: astro-orbiter + + - job_name: gpu-astro-orbiter + scrape_interval: 15s + static_configs: + - targets: + - 10.1.71.130:9835 + labels: + hostname: astro-orbiter + gpu: rtx3090 + + - job_name: llama-server-astro-orbiter + scrape_interval: 15s + metrics_path: /metrics + static_configs: + - targets: + - 10.1.71.130:8000 + labels: + hostname: astro-orbiter + model: bartowski/gemma-2-27b-it-GGUF + # ─── Grafana ────────────────────────────────────────────────────────────────── grafana: enabled: true