diff --git a/cluster/applications/monitoring/values.yaml b/cluster/applications/monitoring/values.yaml index 790e079..cfa4b1c 100644 --- a/cluster/applications/monitoring/values.yaml +++ b/cluster/applications/monitoring/values.yaml @@ -242,18 +242,29 @@ prometheus: # wakes the GPU sub-server to P2 (~110W); at 15s, continuous spikes kept # the GPU drawing ~110W despite zero inference load. At 90s, the GPU gets # ~80s of genuine P8 idle (~20W) between wake-ups. (t_e7d547ea, 2026-08-13) - - job_name: llama-server-astro-orbiter-qwen3 - scrape_interval: 90s - metrics_path: /metrics - params: - model: ["Qwen3.6-35B-A3B-UD-Q4_K_S"] - static_configs: - - targets: - - 10.1.71.130:8002 - labels: - hostname: astro-orbiter - endpoint: astro-orbiter-router - model: Qwen3.6-35B-A3B-UD-Q4_K_S + + # llama-server-astro-orbiter-qwen3 REMOVED (t_02c15dae, 2026-08-13): + # /metrics?model=Qwen3.6 forces the router to attempt loading Qwen3.6 each + # scrape cycle, which fails with CUDA OOM (VRAM already consumed by resident + # Llama3+Phi3.5). This generated real GPU power spikes, not just a benign + # counter read. nvidia_gpu_exporter (:9835) already covers GPU power/VRAM/ + # utilization at zero wake cost. No Grafana dashboard panel depends on + # Qwen3.6-specific llama-server metrics. Ryan approved full removal. + # Commented out rather than deleted for easy revert if Qwen3.6 is ever + # re-added as a resident model. + # + # - job_name: llama-server-astro-orbiter-qwen3 + # scrape_interval: 90s + # metrics_path: /metrics + # params: + # model: ["Qwen3.6-35B-A3B-UD-Q4_K_S"] + # static_configs: + # - targets: + # - 10.1.71.130:8002 + # labels: + # hostname: astro-orbiter + # endpoint: astro-orbiter-router + # model: Qwen3.6-35B-A3B-UD-Q4_K_S - job_name: llama-server-astro-orbiter-llama3 scrape_interval: 90s