monitoring: llama-swap GPU/LLM stack (v250) — PrometheusRule, Grafana dashboard, scrape config, VRAM exporter
This commit is contained in:
Hermes Agent service account
@@ -165,12 +165,14 @@ llm_router_coder_flash_attn: "true"
|
||||
llm_router_coder_gpu_layers: 0
|
||||
llm_router_llama_gpu_layers: 0
|
||||
llm_router_preset_path: /opt/llama-server-router-preset.ini
|
||||
# Qwen3.8-27B: ctx=131072 (128K). Bumped from 32768 -> 131072 per Ryan approval (t_441470b9, 2026-08-16).
|
||||
# Measured VRAM: 20,282 MiB at 131072 ctx (empirically tested in t_4455a44c); nomic-embed 558 MiB
|
||||
# always resident -> ~20.8GB total, ~3.2GB headroom on 24GB RTX 3090. Comfortably safe.
|
||||
# Prior value was 32768 (17,068 MiB) — bumping 4x for genuine 128K context.
|
||||
# Native context of Qwen3.8-27B is 262,144 tokens; 128K is a practical production ceiling.
|
||||
llm_router_qwen38_ctx_size: 131072
|
||||
# Qwen3.8-27B: ctx=65536 (64K). Bumped 32768 -> 131072 (t_441470b9, 2026-08-16);
|
||||
# rolled back to 65536 (t_c9fed26c follow-up, 2026-08-18) after t_72646029 CPU-offload
|
||||
# deployment moved Phi-3.5mini back to GPU, exceeding RTX 3090 24,576 MiB ceiling.
|
||||
# At 131072 ctx + all 5 models resident, Qwen3.8 fails to load (HTTP 500 OOM).
|
||||
# 64K satisfies the 2026-08-12 cutover validation Gate 1 (n_ctx >= 64000).
|
||||
# Full VRAM analysis and Phase 2 options documented in
|
||||
# playbooks/day2_qwen38_ctx128k_rollback.yml.
|
||||
llm_router_qwen38_ctx_size: 65536
|
||||
# nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83
|
||||
# No flash_attn or KV cache params - embedding models use bidirectional forward pass,
|
||||
# not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it
|
||||
@@ -187,3 +189,149 @@ llm_router_nomic_ctx_size: 8192
|
||||
# comfortably while staying under ctx-size=8192.
|
||||
llm_router_nomic_batch_size: 4096
|
||||
llm_router_nomic_ubatch_size: 4096
|
||||
|
||||
# --- Monitoring: VRAM exporter + Prometheus scrape + Grafana dashboard -------
|
||||
# Phase 3: GPU/LLM monitoring deployment (Wong, 2026-08-18)
|
||||
# Provides: VRAM textfile exporter, Prometheus scrape config for llama-swap
|
||||
# /metrics endpoint, Grafana 6-panel dashboard, PrometheusRule alert rules.
|
||||
#
|
||||
# Ref: roles/llm-inference-multimodel/references/monitoring-llm-homelab-ciro-luciotta-2026.md
|
||||
llm_monitoring_enabled: true # gate for monitoring tasks
|
||||
llm_vram_exporter_script: /opt/llama-server-monitoring/nvidia-smi-vram-exporter.sh
|
||||
llm_vram_exporter_cron_minute: "*" # run every minute
|
||||
llm_vram_exporter_gpu_index: 0 # GPU 0 (RTX 3090 on astro-orbiter)
|
||||
llm_vram_textfile_dir: /var/lib/node_exporter/textfile_collector
|
||||
|
||||
# Alert thresholds (per Ciro Luciotta pattern)
|
||||
llm_vram_critical_mib: 24000 # ~90% of 24GB RTX 3090
|
||||
llm_kv_cache_spill_ratio: 0.92 # KV-cache spill threshold
|
||||
llm_throughput_baseline_tokens_per_min: 50 # baseline for degradation alert
|
||||
|
||||
# Grafana dashboard
|
||||
llm_grafana_dashboard_uid: llama-swap-monitor
|
||||
llm_grafana_dashboard_title: "llama-swap GPU/LLM Monitoring"
|
||||
llm_grafana_dashboard_tags:
|
||||
- llm
|
||||
- llama-swap
|
||||
- gpu-monitoring
|
||||
- ciro-luciotta
|
||||
llm_grafana_dashboard_refresh: "30s"
|
||||
llm_grafana_dashboard_time_from: "now-24h"
|
||||
|
||||
# Prometheus scrape job
|
||||
llm_prometheus_scrape_interval: "30s"
|
||||
llm_prometheus_scrape_timeout: "10s"
|
||||
|
||||
# --- llama-swap mode (port 8001) -----------------------------------------------
|
||||
# Deploy llama-swap — Go-based hot-swap proxy (v250+) for model orchestration.
|
||||
# Replaces router mode entirely: single binary + YAML config.json, no --models-preset INI.
|
||||
# Additive deployment (non-invasive); production router (port 8002) stays running during Phase 1 shadow.
|
||||
#
|
||||
# Default: llm_swapmode_enabled: false — all llama-swap tasks are no-ops until flipped to true.
|
||||
# Gated by Phase 3 go/no-go once War Machine Phase 1-2 validation completes.
|
||||
#
|
||||
# NOTE: llama-swap v250 config format differs from evaluation docs (§4b).
|
||||
# Uses routing.router DSL with expression-based matrix, not old list-of-arrays syntax.
|
||||
# See /etc/llama-swap/config.yaml on astro-orbiter (Phase 1 artifact) for reference.
|
||||
#
|
||||
# Added 2026-08-18 (t_c1e44190): llama-swap Phase 3 Ansible integration — Wong.
|
||||
llm_swapmode_enabled: false # Gate for llama-swap tasks (Phase 3)
|
||||
llm_swapmode_port: 8001 # Shadow port (Phase 1), becomes production in Phase 3
|
||||
llm_swapmode_bind_address: "{{ llm_bind_address }}" # 10.1.71.130
|
||||
llm_swapmode_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24
|
||||
|
||||
# Binary installation
|
||||
llm_swapmode_binary_url: "https://github.com/mostlygeek/llama-swap/releases/download/v250/llama-swap-linux-amd64.tar.gz"
|
||||
llm_swapmode_binary_version: "v250"
|
||||
llm_swapmode_checksum: "sha256:60226b64fcc78e8de6e9d4fac78de95372c2c2a0a31fd6b7d26d1e77ea7c9d9d" # From Phase 1 deployment
|
||||
|
||||
# Directories
|
||||
llm_swapmode_config_dir: /etc/llama-swap
|
||||
llm_swapmode_config_file: "{{ llm_swapmode_config_dir }}/config.yaml"
|
||||
llm_swapmode_models_dir: "{{ llm_models_dir }}" # /opt/models — same as production
|
||||
|
||||
# Service
|
||||
llm_swapmode_service_name: llama-swap
|
||||
llm_swapmode_service_user: "{{ llm_service_user }}" # jarvis
|
||||
llm_swapmode_vram_max_mib: 23000 # Gate 3: fail if exceeded under load
|
||||
|
||||
# Consolidated model list for llama-swap config.yaml
|
||||
# Each model specifies full per-model config (ctx_size, n_gpu_layers, cmd args)
|
||||
# Instead of scattered llm_router_* variables, this is the structure llama-swap expects
|
||||
# (matches the v250 config.yaml YAML structure, not the router's INI/per-model variables)
|
||||
llm_swapmode_models:
|
||||
- id: Qwen3.8-27B-Q4_K_M
|
||||
gguf_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf"
|
||||
port: 8105
|
||||
n_gpu_layers: -1 # -1 = auto-detect / all layers to GPU
|
||||
ctx_size: 65536
|
||||
batch_size: 4096
|
||||
ubatch_size: 4096
|
||||
parallel: 1
|
||||
cache_type: q8_0
|
||||
flash_attn: true
|
||||
sleep_idle_seconds: -1 # never idle (primary model — always ready)
|
||||
load_on_startup: true
|
||||
|
||||
- id: Qwen2.5-Coder-14B-Instruct-Q4_K_M
|
||||
gguf_path: "{{ llm_models_dir }}/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf"
|
||||
port: 8101
|
||||
n_gpu_layers: 0 # CPU-offload (aux model)
|
||||
ctx_size: 16384
|
||||
batch_size: 4096
|
||||
ubatch_size: 4096
|
||||
parallel: 1
|
||||
flash_attn: "true"
|
||||
sleep_idle_seconds: 60 # idle after 60s no requests
|
||||
|
||||
- id: Meta-Llama-3.1-8B-Instruct-Q4_K_M
|
||||
gguf_path: "{{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"
|
||||
port: 8102
|
||||
n_gpu_layers: 0 # CPU-offload (aux model)
|
||||
ctx_size: 8192
|
||||
batch_size: 4096
|
||||
ubatch_size: 4096
|
||||
parallel: 1
|
||||
flash_attn: "true"
|
||||
sleep_idle_seconds: 60
|
||||
|
||||
- id: Phi-3.5-mini-instruct-Q8_0
|
||||
gguf_path: "{{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf"
|
||||
port: 8104
|
||||
n_gpu_layers: 0 # CPU-offload (aux model)
|
||||
ctx_size: 32768
|
||||
batch_size: 4096
|
||||
ubatch_size: 4096
|
||||
parallel: 1
|
||||
flash_attn: "true"
|
||||
sleep_idle_seconds: 60
|
||||
|
||||
- id: nomic-embed-text-v1.5
|
||||
gguf_path: "{{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf"
|
||||
port: 8103
|
||||
n_gpu_layers: 0 # CPU-offload (embedding model — always on)
|
||||
ctx_size: 8192
|
||||
batch_size: 4096
|
||||
ubatch_size: 4096
|
||||
parallel: 1
|
||||
sleep_idle_seconds: -1 # never idle (always ready for embeddings)
|
||||
load_on_startup: true
|
||||
|
||||
# llama-swap matrix routing configuration
|
||||
# Each row defines a set of models that can be co-resident and hot-swappable
|
||||
# Syntax: "model1 & model2" = both models in same row (via v250 expression DSL)
|
||||
llm_swapmode_matrix_rows:
|
||||
- row: row0
|
||||
expr: "nomic-embed-text-v1.5" # Embedding-only row
|
||||
|
||||
- row: row1
|
||||
expr: "Qwen3.8-27B-Q4_K_M & nomic-embed-text-v1.5" # Primary + embed
|
||||
|
||||
- row: row2
|
||||
expr: "Meta-Llama-3.1-8B-Instruct-Q4_K_M & nomic-embed-text-v1.5" # Aux LLM + embed
|
||||
|
||||
- row: row3
|
||||
expr: "Qwen2.5-Coder-14B-Instruct-Q4_K_M & nomic-embed-text-v1.5" # Coder + embed
|
||||
|
||||
- row: row4
|
||||
expr: "Phi-3.5-mini-instruct-Q8_0 & nomic-embed-text-v1.5" # Mini + embed
|
||||
|
||||
Reference in New Issue
Block a user