diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml index b31ee1e..91a2d7f 100644 --- a/ansible/host_vars/astro-orbiter/vars.yml +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -65,10 +65,16 @@ common_root_lv: ubuntu-lv # auxiliary models evict it on demand, and vice versa). # NOTE: Qwen3.8 does NOT have load-on-startup — it loads on first request. # This avoids an LRU eviction race with nomic-embed at startup. -# UPDATED (t_441470b9, 2026-08-16): ctx bumped to 131072 (128K). Measured -# VRAM: 20,282 MiB at 131072 ctx. nomic-embed 558 MiB always resident -> -# ~20.8GB total, ~3.2GB headroom. models-max=2 unchanged (same constraint). -llm_router_models_max: 2 +# UPDATED (t_72646029, 2026-08-17): CPU offload for Coder + Llama changes the +# constraint. Coder and Llama now use CPU inference (n-gpu-layers=0). GPU-resident +# VRAM: Qwen3.8 (~20,302 MiB at 128K ctx) + nomic-embed (558 MiB, pinned) plus the +# CUDA-context buffers llama.cpp 6ea215d allocates for the CPU models (~1.4-1.7GB +# each) = ~24,004 MiB steady-state, below the 24,576 MiB physical limit. +# models-max raised to 4: nomic (slot 1, pinned) + Qwen3.8 (slot 2, GPU) + +# Llama (slot 3, CPU) + Coder (slot 4, CPU). Phi (GPU, ~8.3GB) can still be +# requested but evicts Qwen3.8 due to VRAM constraint. models-max=4 +# is required so CPU-offloaded models count as loaded without evicting Qwen3.8. +llm_router_models_max: 4 llm_staged_models: - filename: "Phi-3.5-mini-instruct-Q8_0.gguf" diff --git a/ansible/playbooks/day2_cpu_offload_aux_models.yml b/ansible/playbooks/day2_cpu_offload_aux_models.yml new file mode 100644 index 0000000..2b54b29 --- /dev/null +++ b/ansible/playbooks/day2_cpu_offload_aux_models.yml @@ -0,0 +1,59 @@ +--- +# ------------------------------------------------------------------------------ +# Playbook: day2_cpu_offload_aux_models.yml +# Purpose: CPU-offload Qwen2.5-Coder-14B and Meta-Llama-3.1-8B on +# astro-orbiter's production router (port 8002). +# +# What this playbook does: +# 1. Re-renders llama-server-router-preset.ini (Coder + Llama sections now +# use per-model n-gpu-layers vars = 0 -> full CPU inference). +# 2. Re-renders the router unit (--models-max now 4 via host_vars, global +# --n-gpu-layers removed per t_72646029 unit template fix) and restarts +# llama-server-router so both changes take effect. +# 3. Verifies per the role's router_preset phase. +# +# Context (2026-08-17): +# - RAM/model-swap audit, TIER 1 (Coder-14B CPU offload) + TIER 2 +# (Llama-3.1-8B CPU offload) — Ryan approved 1 & 2 on 2026-08-17. +# See inbox/ryan/2026-08-17-llm-system-ram-model-swap.md. +# - Unit template fix (t_72646029): global --n-gpu-layers removed from +# ExecStart in preset mode. Each INI section now sets n-gpu-layers +# explicitly (Qwen3.8=99, Phi=99, nomic=99, Coder=0, Llama=0). +# - Concurrent residency after change: Qwen3.8-27B (20,302 MiB @ 128K ctx) +# + nomic-embed (558 MiB, pinned) + Coder (CPU, ~1,390 MiB CUDA ctx) + +# Llama (CPU, ~1,706 MiB CUDA ctx) = ~24,004 MiB. NOTE: llama.cpp 6ea215d +# allocates CUDA-context VRAM even at n-gpu-layers=0, so CPU models are not +# 0-VRAM; total sits at the 24,576 MiB physical limit (headroom ~572 MiB). +# Qwen3.8 is never evicted for a CPU aux model; Phi-3.5-mini (GPU, 8.3GB) +# still evicts as before. +# - CPU speed (8-core Ryzen 7 5800XT): ~5-10 tok/s (14B), ~10-20 tok/s (8B). +# - Semaphore SSH gap for astro-orbiter still applies (t_730f9584 / +# t_33acbb2e); running direct CLI Ansible per standing exception. +# +# Run: +# cd /home/hermes/git/homelab/ansible +# env -u ANSIBLE_VAULT_PASSWORD_FILE ansible-playbook \ +# -i inventory.yml \ +# playbooks/day2_cpu_offload_aux_models.yml +# +# Rollback: +# git checkout -- \ +# roles/llm-inference-multimodel/templates/llama-server-router.service.j2 \ +# roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 \ +# roles/llm-inference-multimodel/defaults/main.yml \ +# host_vars/astro-orbiter/vars.yml +# (restores n-gpu-layers=99 global flag, models-max=2, all GPU) +# then re-run this playbook to redeploy rollback state. +# Note: playbooks/day2_cpu_offload_aux_models.yml is untracked — left on disk. +# ------------------------------------------------------------------------------ +- name: CPU-offload Coder-14B and Llama-3.1-8B on astro-orbiter + hosts: astro-orbiter + become: true + vars: + llm_router_preset_enabled: true + llm_router_enabled: true + llm_router_port: 8002 + + roles: + - role: llm-inference-multimodel + tags: [always] diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 4e85e4e..48218ad 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -158,6 +158,12 @@ llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}" # Qwen2.5-Coder-14B: ctx_size=16384, flash_attn=true per task t_55c164f5 llm_router_coder_ctx_size: 16384 llm_router_coder_flash_attn: "true" +# CPU offload vars (t_72646029, 2026-08-17): n-gpu-layers=0 moves Coder and Llama to +# full CPU inference. Allows concurrent residency with Qwen3.8-27B. NOTE: llama.cpp +# 6ea215d still allocates ~1.4-1.7GB CUDA-context VRAM per CPU model, so steady-state +# is ~24,004 MiB (at the 24,576 MiB physical limit), not the 0-VRAM the spec assumed. +llm_router_coder_gpu_layers: 0 +llm_router_llama_gpu_layers: 0 llm_router_preset_path: /opt/llama-server-router-preset.ini # Qwen3.8-27B: ctx=131072 (128K). Bumped from 32768 -> 131072 per Ryan approval (t_441470b9, 2026-08-16). # Measured VRAM: 20,282 MiB at 131072 ctx (empirically tested in t_4455a44c); nomic-embed 558 MiB diff --git a/ansible/roles/llm-inference-multimodel/tasks/verify.yml b/ansible/roles/llm-inference-multimodel/tasks/verify.yml index 396900b..27c4384 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/verify.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/verify.yml @@ -65,7 +65,7 @@ ansible.builtin.debug: msg: - "Qwen (:{{ llm_qwen_port }}) serving: {{ llm_qwen_models.json.data | map(attribute='id') | list }}" - - "Verified n_ctx (must be >= 64000, not just requested): {{ llm_qwen_models.json.data | map(attribute='meta') | map(attribute='n_ctx') | list }}" + - "Verified n_ctx (must be >= 64000, not just requested): {{ llm_qwen_models.json.data | map(attribute='meta', default={}) | map(attribute='n_ctx', default=0) | list }}" when: - llm_qwen_service_enabled | default(false) - llm_qwen_models is defined diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 index dc2d354..5ca4d35 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 @@ -68,17 +68,12 @@ ; back to non-fused implementation. Inference works correctly but may be ; slower on the GDN layers. An updated llama.cpp may improve throughput. ; load-on-startup NOT set (loads on first request, ~30-60s cold load). -; With models-max=2 in host_vars, nomic-embed occupies slot 1 (pinned), -; and the generative slot (slot 2) is Qwen3.8 on first request. Auxiliary -; models (Phi, Llama, Coder) evict Qwen3.8 when requested; Qwen3.8 evicts -; them in turn. One cold-load (~30-60s) per switch between Qwen3.8 and -; auxiliary models is expected and acceptable. In practice, once Hermes -; config.yaml references Qwen3.8 as primary, it stays resident. -; ctx-size raised to 131072 (128K) per Ryan approval (t_441470b9, 2026-08-16). +; n-gpu-layers=99: GPU (all layers). Explicit here so global CLI flag removal +; (t_72646029, 2026-08-17) does not change Qwen3.8 behavior. ; Primary model ID: Qwen3.8-27B-Q4_K_M [Qwen3.8-27B-Q4_K_M] model = {{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf -n-gpu-layers = {{ llm_router_gpu_layers }} +n-gpu-layers = 99 ctx-size = {{ llm_router_qwen38_ctx_size }} cache-type-k = {{ llm_router_cache_type_k }} cache-type-v = {{ llm_router_cache_type_v }} @@ -110,7 +105,7 @@ parallel = {{ llm_router_parallel }} [Phi-3.5-mini-instruct-Q8_0] model = {{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf alias = Phi-3.5-mini-instruct-8bit -n-gpu-layers = {{ llm_router_gpu_layers }} +n-gpu-layers = 99 ctx-size = {{ llm_router_phi_ctx_size }} flash-attn = {{ llm_router_phi_flash_attn }} cache-type-k = {{ llm_router_cache_type_k }} @@ -124,10 +119,15 @@ parallel = {{ llm_router_parallel }} ; Alias: Meta-Llama-3.1-8B-Instruct-4bit (NEW — friendlier name) ; Both names resolve to this GGUF child process. ; ~4.6GB, general-purpose small model. Works with json_schema structured output. +; n-gpu-layers=0 (CPU offload, t_72646029 2026-08-17): Llama moves to full CPU +; inference to allow concurrent residency with Qwen3.8-27B (which uses ~20.8GB +; VRAM including nomic-embed). At models-max=4, Llama and Coder run on CPU — +; llama.cpp 6ea215d still holds ~1.4-1.7GB CUDA-context VRAM per CPU model, so +; steady-state is ~24,004 MiB (below the 24,576 MiB physical limit). [Meta-Llama-3.1-8B-Instruct-Q4_K_M] model = {{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf alias = Meta-Llama-3.1-8B-Instruct-4bit -n-gpu-layers = {{ llm_router_gpu_layers }} +n-gpu-layers = {{ llm_router_llama_gpu_layers }} ctx-size = {{ llm_router_llama_ctx_size }} flash-attn = {{ llm_router_llama_flash_attn }} cache-type-k = {{ llm_router_cache_type_k }} @@ -140,14 +140,19 @@ parallel = {{ llm_router_parallel }} ; Primary model ID: Qwen2.5-Coder-14B-Instruct-Q4_K_M (filename-derived) ; Alias: Qwen2.5-Coder-14B-Instruct-4bit (friendlier name) ; Both names resolve to this GGUF child process. -; ~8.4GB weights + ~0.6GB KV @ 16K ctx = ~9.0GB VRAM. +; ~8.4GB weights + ~0.6GB KV @ 16K ctx = ~9.0GB VRAM (GPU); ~1,390 MiB CUDA ctx (CPU). ; ctx-size=16384, flash-attn=true per task t_55c164f5 / Ryan's request. ; Source: bartowski/Qwen2.5-Coder-14B-Instruct-GGUF (public, no auth) ; Added 2026-08-13 (t_55c164f5) — War Machine. +; n-gpu-layers=0 (CPU offload, t_72646029 2026-08-17): Coder moves to full CPU +; inference to allow concurrent residency with Qwen3.8-27B (which uses ~20.8GB +; VRAM including nomic-embed). At models-max=4, Coder and Llama run on CPU — +; llama.cpp 6ea215d still holds ~1.4-1.7GB CUDA-context VRAM per CPU model, so +; steady-state is ~24,004 MiB (below the 24,576 MiB physical limit). [Qwen2.5-Coder-14B-Instruct-Q4_K_M] model = {{ llm_models_dir }}/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf alias = Qwen2.5-Coder-14B-Instruct-4bit -n-gpu-layers = {{ llm_router_gpu_layers }} +n-gpu-layers = {{ llm_router_coder_gpu_layers }} ctx-size = {{ llm_router_coder_ctx_size }} flash-attn = {{ llm_router_coder_flash_attn }} cache-type-k = {{ llm_router_cache_type_k }} @@ -194,7 +199,7 @@ parallel = {{ llm_router_parallel }} [nomic-embed-text-v1.5] model = {{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf embedding = true -n-gpu-layers = {{ llm_router_gpu_layers }} +n-gpu-layers = 99 ctx-size = {{ llm_router_nomic_ctx_size }} batch-size = {{ llm_router_nomic_batch_size }} ubatch-size = {{ llm_router_nomic_ubatch_size }} diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router.service.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router.service.j2 index 1f08ba4..54f64ef 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router.service.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router.service.j2 @@ -18,8 +18,8 @@ ExecStart={{ llm_binary_path }} \ --models-max {{ llm_router_models_max }} \ --host {{ llm_router_bind_address }} \ --port {{ llm_router_port }} \ - --n-gpu-layers {{ llm_router_gpu_layers }} \ {% if not (llm_router_preset_enabled | default(false)) %} + --n-gpu-layers {{ llm_router_gpu_layers }} \ --ctx-size {{ llm_router_ctx_size }} \ --flash-attn {{ llm_router_flash_attn }} \ {% endif %} @@ -30,7 +30,7 @@ ExecStart={{ llm_binary_path }} \ --parallel {{ llm_router_parallel }} \ --metrics -# ROUTER MODE NOTES (2026-08-12, t_0cca74a2 / updated t_9adf0889): +# ROUTER MODE NOTES (2026-08-12, t_0cca74a2 / updated t_9adf0889 / updated t_72646029): # - NO -m/--model flag: this is what enables llama-server router/supervisor mode. # Without -m, llama-server discovers all .gguf files in --models-dir, or uses # the per-model definitions in a --models-preset INI file. @@ -40,12 +40,16 @@ ExecStart={{ llm_binary_path }} \ # The preset INI is at {{ llm_router_preset_path | default('/opt/llama-server-router-preset.ini') }}. # Both the section name and the alias field in the INI work as model IDs. # GH #22364 (extra "default" entry in /v1/models) is expected in preset mode — cosmetic. +# - --n-gpu-layers is INTENTIONALLY OMITTED from preset mode (t_72646029, 2026-08-17): +# In --models-preset mode every model section in the INI sets n-gpu-layers explicitly. +# A global CLI --n-gpu-layers has HIGHEST precedence in llama.cpp (CLI > model-section > global-INI) +# and would override per-model INI values (e.g. n-gpu-layers=0 for CPU offload). +# When preset mode is disabled (--models-dir), --n-gpu-layers is emitted normally. # - --models-max {{ llm_router_models_max }} is driven by llm_router_models_max # (default 1 in defaults/main.yml; overridden to 4 in host_vars/astro-orbiter -# as of t_33acbb2e after VRAM budget review — see host_vars for OOM risk note). -# Default llama-server cap is 4 simultaneous — OOM on 24GB if all 3 current -# GGUFs load at once. LRU eviction mitigates in practice but review before adding -# models. See host_vars/astro-orbiter/vars.yml for full VRAM breakdown. +# as of t_72646029 after CPU-offload enabling — CPU models count against models-max +# and hold ~1.4-1.7GB CUDA-context VRAM each (llama.cpp 6ea215d allocates it even at +# n-gpu-layers=0); steady-state ~24,004 MiB, below the 24,576 MiB physical limit). # - Clients select a model via "model": "" in their # chat completion request. Hermes sends model: "" on every request already. # - Cold model load on first request: ~30-60s for Qwen3.6-35B. First response