From a2ddb6542522e03f822a1e791a47fc0fe9c893a8 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Thu, 13 Aug 2026 15:43:43 -0500 Subject: [PATCH] fix(monitoring): reduce llama-server scrape_interval 15s -> 90s to allow GPU P8 idle Each /metrics?model= request on the llama.cpp router wakes the GPU sub-server to P2 (~110W). At 15s with two active jobs (llama3 + phi35), combined scrape frequency (~7-8s effective) keeps the GPU continuously at P2 despite zero real inference requests. At 90s: each scrape wakes GPU for ~5-10s then it drops to P8 (~20W) for ~80s. Verified via iptables block test on 2026-08-13 (t_e7d547ea): Before block: 110-115W P2 continuously After block: 19-21W P8 consistently After unblock: returned to 110W P2 within seconds qwen3 interval also set to 90s (model not loaded so moot, but consistent). Ref: t_e7d547ea --- cluster/applications/monitoring/values.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cluster/applications/monitoring/values.yaml b/cluster/applications/monitoring/values.yaml index 4b4536e..790e079 100644 --- a/cluster/applications/monitoring/values.yaml +++ b/cluster/applications/monitoring/values.yaml @@ -238,8 +238,12 @@ prometheus: # jobs — one per model registered in the router — are used so each gets # its own model label. The static `model` label is set to the canonical # llama.cpp model id (the `id` field from /v1/models, not the alias). + # scrape_interval: 90s (reduced from 15s) -- each /metrics?model= request + # wakes the GPU sub-server to P2 (~110W); at 15s, continuous spikes kept + # the GPU drawing ~110W despite zero inference load. At 90s, the GPU gets + # ~80s of genuine P8 idle (~20W) between wake-ups. (t_e7d547ea, 2026-08-13) - job_name: llama-server-astro-orbiter-qwen3 - scrape_interval: 15s + scrape_interval: 90s metrics_path: /metrics params: model: ["Qwen3.6-35B-A3B-UD-Q4_K_S"] @@ -252,7 +256,7 @@ prometheus: model: Qwen3.6-35B-A3B-UD-Q4_K_S - job_name: llama-server-astro-orbiter-llama3 - scrape_interval: 15s + scrape_interval: 90s metrics_path: /metrics params: model: ["Meta-Llama-3.1-8B-Instruct-Q4_K_M"] @@ -265,7 +269,7 @@ prometheus: model: Meta-Llama-3.1-8B-Instruct-Q4_K_M - job_name: llama-server-astro-orbiter-phi35 - scrape_interval: 15s + scrape_interval: 90s metrics_path: /metrics params: model: ["Phi-3.5-mini-instruct-Q8_0"]