03b3ce9dee
- Remove global --n-gpu-layers from router unit ExecStart in preset mode (llama.cpp CLI arg outranked per-model INI n-gpu-layers=0; root cause from War Machine's run 1). Flag now emitted only in --models-dir mode. - All 5 preset INI sections carry explicit n-gpu-layers: Qwen3.8=99, Phi=99, nomic=99, Coder=0, Llama=0. - host_vars/astro-orbiter: llm_router_models_max 2 -> 4 so CPU-offloaded models count as loaded without LRU-evicting Qwen3.8. - defaults: llm_router_coder_gpu_layers / llm_router_llama_gpu_layers = 0. - verify.yml: fix pre-existing .meta attribute crash in router mode. - New playbook day2_cpu_offload_aux_models.yml. Deployed + verified on astro-orbiter (gates A-E PASS): concurrent residency achieved, Qwen3.8 stays GPU-resident. Measured CPU throughput Llama 9.0 / Coder 4.7 tok/s. VRAM note: llama.cpp 6ea215d allocates ~1.4-1.7GB CUDA-context per CPU model even at n-gpu-layers=0 -> ~24,004 MiB steady-state, below the 24,576 MiB physical limit. Comments corrected to match the measurement. Report: friday/inbox/ryan/2026-08-17-llm-cpu-offload-coder-llama-deployed.md
75 lines
3.8 KiB
Django/Jinja
75 lines
3.8 KiB
Django/Jinja
[Unit]
|
|
Description=llama-server router — {{ llm_router_models_dir }} (OpenAI-compatible, port {{ llm_router_port }})
|
|
Documentation=https://github.com/ggml-org/llama.cpp
|
|
After=network.target nvidia-persistenced.service
|
|
Wants=nvidia-persistenced.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
User={{ llm_service_user }}
|
|
Group={{ llm_service_user }}
|
|
Environment="HOME=/home/{{ llm_service_user }}"
|
|
ExecStart={{ llm_binary_path }} \
|
|
{% if llm_router_preset_enabled | default(false) %}
|
|
--models-preset {{ llm_router_preset_path }} \
|
|
{% else %}
|
|
--models-dir {{ llm_router_models_dir }} \
|
|
{% endif %}
|
|
--models-max {{ llm_router_models_max }} \
|
|
--host {{ llm_router_bind_address }} \
|
|
--port {{ llm_router_port }} \
|
|
{% if not (llm_router_preset_enabled | default(false)) %}
|
|
--n-gpu-layers {{ llm_router_gpu_layers }} \
|
|
--ctx-size {{ llm_router_ctx_size }} \
|
|
--flash-attn {{ llm_router_flash_attn }} \
|
|
{% endif %}
|
|
--cache-type-k {{ llm_router_cache_type_k }} \
|
|
--cache-type-v {{ llm_router_cache_type_v }} \
|
|
--batch-size {{ llm_router_batch_size }} \
|
|
--ubatch-size {{ llm_router_ubatch_size }} \
|
|
--parallel {{ llm_router_parallel }} \
|
|
--metrics
|
|
|
|
# ROUTER MODE NOTES (2026-08-12, t_0cca74a2 / updated t_9adf0889 / updated t_72646029):
|
|
# - NO -m/--model flag: this is what enables llama-server router/supervisor mode.
|
|
# Without -m, llama-server discovers all .gguf files in --models-dir, or uses
|
|
# the per-model definitions in a --models-preset INI file.
|
|
# - PRESET MODE (t_9adf0889, 2026-08-12):
|
|
# llm_router_preset_enabled=true switches from --models-dir to --models-preset.
|
|
# Preset mode adds alias support (--models-dir cannot assign aliases).
|
|
# The preset INI is at {{ llm_router_preset_path | default('/opt/llama-server-router-preset.ini') }}.
|
|
# Both the section name and the alias field in the INI work as model IDs.
|
|
# GH #22364 (extra "default" entry in /v1/models) is expected in preset mode — cosmetic.
|
|
# - --n-gpu-layers is INTENTIONALLY OMITTED from preset mode (t_72646029, 2026-08-17):
|
|
# In --models-preset mode every model section in the INI sets n-gpu-layers explicitly.
|
|
# A global CLI --n-gpu-layers has HIGHEST precedence in llama.cpp (CLI > model-section > global-INI)
|
|
# and would override per-model INI values (e.g. n-gpu-layers=0 for CPU offload).
|
|
# When preset mode is disabled (--models-dir), --n-gpu-layers is emitted normally.
|
|
# - --models-max {{ llm_router_models_max }} is driven by llm_router_models_max
|
|
# (default 1 in defaults/main.yml; overridden to 4 in host_vars/astro-orbiter
|
|
# as of t_72646029 after CPU-offload enabling — CPU models count against models-max
|
|
# and hold ~1.4-1.7GB CUDA-context VRAM each (llama.cpp 6ea215d allocates it even at
|
|
# n-gpu-layers=0); steady-state ~24,004 MiB, below the 24,576 MiB physical limit).
|
|
# - Clients select a model via "model": "<section-name-or-alias>" in their
|
|
# chat completion request. Hermes sends model: "<id>" on every request already.
|
|
# - Cold model load on first request: ~30-60s for Qwen3.6-35B. First response
|
|
# will be slow. This is expected. Document in runbook.
|
|
# - No --jinja flag: Qwen3.6-35B uses its own embedded chat template correctly.
|
|
# If per-model template overrides are ever needed, use --models-preset INI
|
|
# (but note GH #23460: sampler params in presets may not work in router mode).
|
|
#
|
|
# SHADOW DEPLOYMENT NOTE (historical — 2026-08-12, t_0cca74a2):
|
|
# This unit was originally deployed on port 8003 as a shadow. After validation,
|
|
# it was promoted to production on port 8002 (t_cd0d5388). The --port value
|
|
# above is the authoritative value; the port 8003 references below are historical.
|
|
# Production is now llama-server-router (this unit); llama-server-qwen is the rollback target.
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
TimeoutStartSec=600
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
SyslogIdentifier=llama-server-router
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|