From a04435ee9bd5db072d965c90b86ac10ebcc9ae2d Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Thu, 13 Aug 2026 18:46:54 -0500 Subject: [PATCH] =?UTF-8?q?fix(monitoring):=20drop=20Qwen3.6=20scrape=20jo?= =?UTF-8?q?b=20=E2=80=94=20causes=20CUDA=20OOM=20on=20each=20scrape=20(t?= =?UTF-8?q?=5F02c15dae)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /metrics?model=Qwen3.6 forces the router to attempt loading Qwen3.6 every 90s scrape cycle, triggering a CUDA OOM error since VRAM is already consumed by the resident Llama3+Phi3.5 models. This produces real GPU power spikes (~110W load-attempt), not a benign counter read like the Llama3/Phi3.5 jobs. nvidia_gpu_exporter (:9835) already provides GPU power/VRAM/utilization at zero wake cost. No Grafana dashboard panel references Qwen3.6 model-specific llama-server metrics. Ryan approved full removal. Job is commented out (not deleted) for easy revert if Qwen3.6 is ever re-added as a resident model. Llama3/Phi3.5 scrape jobs untouched. --- cluster/applications/monitoring/values.yaml | 35 ++++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/cluster/applications/monitoring/values.yaml b/cluster/applications/monitoring/values.yaml index 790e079..cfa4b1c 100644 --- a/cluster/applications/monitoring/values.yaml +++ b/cluster/applications/monitoring/values.yaml @@ -242,18 +242,29 @@ prometheus: # wakes the GPU sub-server to P2 (~110W); at 15s, continuous spikes kept # the GPU drawing ~110W despite zero inference load. At 90s, the GPU gets # ~80s of genuine P8 idle (~20W) between wake-ups. (t_e7d547ea, 2026-08-13) - - job_name: llama-server-astro-orbiter-qwen3 - scrape_interval: 90s - metrics_path: /metrics - params: - model: ["Qwen3.6-35B-A3B-UD-Q4_K_S"] - static_configs: - - targets: - - 10.1.71.130:8002 - labels: - hostname: astro-orbiter - endpoint: astro-orbiter-router - model: Qwen3.6-35B-A3B-UD-Q4_K_S + + # llama-server-astro-orbiter-qwen3 REMOVED (t_02c15dae, 2026-08-13): + # /metrics?model=Qwen3.6 forces the router to attempt loading Qwen3.6 each + # scrape cycle, which fails with CUDA OOM (VRAM already consumed by resident + # Llama3+Phi3.5). This generated real GPU power spikes, not just a benign + # counter read. nvidia_gpu_exporter (:9835) already covers GPU power/VRAM/ + # utilization at zero wake cost. No Grafana dashboard panel depends on + # Qwen3.6-specific llama-server metrics. Ryan approved full removal. + # Commented out rather than deleted for easy revert if Qwen3.6 is ever + # re-added as a resident model. + # + # - job_name: llama-server-astro-orbiter-qwen3 + # scrape_interval: 90s + # metrics_path: /metrics + # params: + # model: ["Qwen3.6-35B-A3B-UD-Q4_K_S"] + # static_configs: + # - targets: + # - 10.1.71.130:8002 + # labels: + # hostname: astro-orbiter + # endpoint: astro-orbiter-router + # model: Qwen3.6-35B-A3B-UD-Q4_K_S - job_name: llama-server-astro-orbiter-llama3 scrape_interval: 90s