9c969f783d
Changes: - host_vars/astro-orbiter/vars.yml: add llm_router_models_max: 4 (overrides conservative default of 1). Detailed VRAM OOM risk note included inline: worst-case 3-model co-residency ~31GB > 24GB RTX 3090. LRU eviction mitigates in single-user operation; flagged for Ryan's review. - playbooks/day2_bump_router_models_max.yml: new targeted playbook; deploys updated router unit, restarts the live service, verifies /health 200 and /v1/models lists all 3 GGUFs post-restart. - group_vars/all/semaphore.yml: add llm_router_update_unit template pointing at the new playbook. - roles/llm-inference-multimodel/defaults/main.yml: update comment to reflect the var is now overridden in host_vars rather than 'hardcoded to 1'. - roles/llm-inference-multimodel/templates/llama-server-router.service.j2: correct stale 'HARDCODED TO 1' comment — value is variable-driven. Constraints honored: - --parallel 1 left untouched (not in scope, not modified anywhere) - No ad-hoc SSH/systemctl/curl state mutation; all execution via Semaphore - No installed/vendored code patched
61 lines
2.9 KiB
Django/Jinja
61 lines
2.9 KiB
Django/Jinja
[Unit]
|
|
Description=llama-server router — {{ llm_router_models_dir }} (OpenAI-compatible, port {{ llm_router_port }})
|
|
Documentation=https://github.com/ggml-org/llama.cpp
|
|
After=network.target nvidia-persistenced.service
|
|
Wants=nvidia-persistenced.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
User={{ llm_service_user }}
|
|
Group={{ llm_service_user }}
|
|
Environment="HOME=/home/{{ llm_service_user }}"
|
|
ExecStart={{ llm_binary_path }} \
|
|
--models-dir {{ llm_router_models_dir }} \
|
|
--models-max {{ llm_router_models_max }} \
|
|
--host {{ llm_router_bind_address }} \
|
|
--port {{ llm_router_port }} \
|
|
--n-gpu-layers {{ llm_router_gpu_layers }} \
|
|
--ctx-size {{ llm_router_ctx_size }} \
|
|
--flash-attn {{ llm_router_flash_attn }} \
|
|
--cache-type-k {{ llm_router_cache_type_k }} \
|
|
--cache-type-v {{ llm_router_cache_type_v }} \
|
|
--batch-size {{ llm_router_batch_size }} \
|
|
--ubatch-size {{ llm_router_ubatch_size }} \
|
|
--parallel {{ llm_router_parallel }} \
|
|
--metrics
|
|
|
|
# ROUTER MODE NOTES (2026-08-12, t_0cca74a2):
|
|
# - NO -m/--model flag: this is what enables llama-server router/supervisor mode.
|
|
# Without -m, llama-server discovers all .gguf files in --models-dir, spawning
|
|
# each as its own child process on demand (LRU-eviction when over models-max).
|
|
# - --models-max {{ llm_router_models_max }} is driven by llm_router_models_max
|
|
# (default 1 in defaults/main.yml; overridden to 4 in host_vars/astro-orbiter
|
|
# as of t_33acbb2e after VRAM budget review — see host_vars for OOM risk note).
|
|
# Default llama-server cap is 4 simultaneous — OOM on 24GB if all 3 current
|
|
# GGUFs load at once. LRU eviction mitigates in practice but review before adding
|
|
# models. See host_vars/astro-orbiter/vars.yml for full VRAM breakdown.
|
|
# - --models-dir /opt/models: auto-discovers all .gguf files. Keep that directory
|
|
# clean (Qwen-only) to avoid spurious extra entries in /v1/models.
|
|
# - Clients select a model via "model": "<gguf-basename-without-.gguf>" in their
|
|
# chat completion request. Hermes sends model: "<id>" on every request already.
|
|
# - Cold model load on first request: ~30-60s for Qwen3.6-35B. First response
|
|
# will be slow. This is expected. Document in runbook.
|
|
# - No --jinja flag: Qwen3.6-35B uses its own embedded chat template correctly.
|
|
# If per-model template overrides are ever needed, use --models-preset INI
|
|
# (but note GH #23460: sampler params in presets may not work in router mode).
|
|
#
|
|
# SHADOW DEPLOYMENT NOTE (historical — 2026-08-12, t_0cca74a2):
|
|
# This unit was originally deployed on port 8003 as a shadow. After validation,
|
|
# it was promoted to production on port 8002 (t_cd0d5388). The --port value
|
|
# above is the authoritative value; the port 8003 references below are historical.
|
|
# Production is now llama-server-router (this unit); llama-server-qwen is the rollback target.
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
TimeoutStartSec=600
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
SyslogIdentifier=llama-server-router
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|