a47b29d49f
- Add templates/llama-server-router-preset.ini.j2: defines all 3 router GGUFs (Qwen3.6-35B, Phi-3.5-mini-Q8_0, Meta-Llama-3.1-8B) with explicit ctx-size, gpu-layers, cache settings carried over from --models-dir baseline. The Phi entry adds alias=Phi-3.5-mini-instruct-8bit (Ryan's aux.title_generation target). - Update templates/llama-server-router.service.j2: Jinja2 conditional emits --models-preset <path> when llm_router_preset_enabled=true, otherwise --models-dir (backward compat, default unchanged). - Add tasks/preset.yml: deploy preset INI, restart router on change, verify both Phi-3.5-mini-instruct-Q8_0 (primary) and Phi-3.5-mini-instruct-8bit (alias) appear in /v1/models, plus Qwen and Llama IDs unchanged. - Update defaults/main.yml: add llm_router_preset_enabled (default false) and llm_router_preset_path=/opt/llama-server-router-preset.ini. - Update tasks/main.yml: import preset.yml as Phase P (gated, no-op by default). - Update handlers/main.yml: add 'restart router' handler for preset changes. - Add playbooks/day2_add_phi_alias.yml: single-command deployment. GH #22364 note: --models-preset mode creates an extra 'default' entry in /v1/models — cosmetic, does not affect model selection by name.
69 lines
3.4 KiB
Django/Jinja
69 lines
3.4 KiB
Django/Jinja
[Unit]
|
|
Description=llama-server router — {{ llm_router_models_dir }} (OpenAI-compatible, port {{ llm_router_port }})
|
|
Documentation=https://github.com/ggml-org/llama.cpp
|
|
After=network.target nvidia-persistenced.service
|
|
Wants=nvidia-persistenced.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
User={{ llm_service_user }}
|
|
Group={{ llm_service_user }}
|
|
Environment="HOME=/home/{{ llm_service_user }}"
|
|
ExecStart={{ llm_binary_path }} \
|
|
{% if llm_router_preset_enabled | default(false) %}
|
|
--models-preset {{ llm_router_preset_path }} \
|
|
{% else %}
|
|
--models-dir {{ llm_router_models_dir }} \
|
|
{% endif %}
|
|
--models-max {{ llm_router_models_max }} \
|
|
--host {{ llm_router_bind_address }} \
|
|
--port {{ llm_router_port }} \
|
|
--n-gpu-layers {{ llm_router_gpu_layers }} \
|
|
--ctx-size {{ llm_router_ctx_size }} \
|
|
--flash-attn {{ llm_router_flash_attn }} \
|
|
--cache-type-k {{ llm_router_cache_type_k }} \
|
|
--cache-type-v {{ llm_router_cache_type_v }} \
|
|
--batch-size {{ llm_router_batch_size }} \
|
|
--ubatch-size {{ llm_router_ubatch_size }} \
|
|
--parallel {{ llm_router_parallel }} \
|
|
--metrics
|
|
|
|
# ROUTER MODE NOTES (2026-08-12, t_0cca74a2 / updated t_9adf0889):
|
|
# - NO -m/--model flag: this is what enables llama-server router/supervisor mode.
|
|
# Without -m, llama-server discovers all .gguf files in --models-dir, or uses
|
|
# the per-model definitions in a --models-preset INI file.
|
|
# - PRESET MODE (t_9adf0889, 2026-08-12):
|
|
# llm_router_preset_enabled=true switches from --models-dir to --models-preset.
|
|
# Preset mode adds alias support (--models-dir cannot assign aliases).
|
|
# The preset INI is at {{ llm_router_preset_path | default('/opt/llama-server-router-preset.ini') }}.
|
|
# Both the section name and the alias field in the INI work as model IDs.
|
|
# GH #22364 (extra "default" entry in /v1/models) is expected in preset mode — cosmetic.
|
|
# - --models-max {{ llm_router_models_max }} is driven by llm_router_models_max
|
|
# (default 1 in defaults/main.yml; overridden to 4 in host_vars/astro-orbiter
|
|
# as of t_33acbb2e after VRAM budget review — see host_vars for OOM risk note).
|
|
# Default llama-server cap is 4 simultaneous — OOM on 24GB if all 3 current
|
|
# GGUFs load at once. LRU eviction mitigates in practice but review before adding
|
|
# models. See host_vars/astro-orbiter/vars.yml for full VRAM breakdown.
|
|
# - Clients select a model via "model": "<section-name-or-alias>" in their
|
|
# chat completion request. Hermes sends model: "<id>" on every request already.
|
|
# - Cold model load on first request: ~30-60s for Qwen3.6-35B. First response
|
|
# will be slow. This is expected. Document in runbook.
|
|
# - No --jinja flag: Qwen3.6-35B uses its own embedded chat template correctly.
|
|
# If per-model template overrides are ever needed, use --models-preset INI
|
|
# (but note GH #23460: sampler params in presets may not work in router mode).
|
|
#
|
|
# SHADOW DEPLOYMENT NOTE (historical — 2026-08-12, t_0cca74a2):
|
|
# This unit was originally deployed on port 8003 as a shadow. After validation,
|
|
# it was promoted to production on port 8002 (t_cd0d5388). The --port value
|
|
# above is the authoritative value; the port 8003 references below are historical.
|
|
# Production is now llama-server-router (this unit); llama-server-qwen is the rollback target.
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
TimeoutStartSec=600
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
SyslogIdentifier=llama-server-router
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|