7b44a41da3
Ryan-directed model swap (kanban t_f5f7e9ad, 2026-08-16). Changes: - Replace [Qwen3.6-35B-A3B-UD-Q4_K_S] with [Qwen3.8-27B-Q4_K_M] in llama-server-router-preset.ini.j2 (production model slot). - Qwen3.8-27B: dense 27B VLM, Apache-2.0, Alibaba Aug 2026. Unsloth Dynamic V3.0 GGUF quantization. Q4_K_M chosen: 17,106,775,008 bytes, 17.1GB. Measured VRAM: 17,068 MiB at ctx=32768 (q4_0 KV cache). - ctx-size set to 32768 (32K) via new variable llm_router_qwen38_ctx_size. Native context is 262K; 32K chosen to maintain eviction headroom on 24GB RTX 3090. - models-max reduced 4 -> 2 in host_vars. Qwen3.8 (17.6GB) + nomic-embed (558MB) exhaust the 24GB card; no auxiliary model can co-reside with Qwen3.8. LRU eviction handles model switching with ~30-60s cold-load latency. - llm_router_expected_model_id updated to Qwen3.8-27B-Q4_K_M. - Qwen3.6 GGUF retained at /opt/models/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf (not deleted — pending stable period and explicit cleanup task). - day2_swap_qwen38.yml playbook added for Ansible idempotent redeployment. Architecture note: Qwen3.8 uses Gated DeltaNet; llama.cpp 6ea215d logs 'fused Gated Delta Net (chunked) not supported, set to disabled'. Inference works correctly on the non-fused fallback. A llama.cpp update may improve throughput on the GDN layers. Smoke test passed: model responded via router endpoint (http://10.1.71.130:8002). VRAM: 17,630 MiB (Qwen3.8) + 5,928 MiB (Llama-8B concurrent) = 23,558 MiB. Also commits accumulated but unpushed changes: - nomic-embed batch-size/rope-scaling fix (t_openviking_embed_batch) - per-model ctx-size day2 playbook (day2_per_model_ctx_size.yml) - llama-server-router.service.j2 minor update
71 lines
3.4 KiB
Django/Jinja
71 lines
3.4 KiB
Django/Jinja
[Unit]
|
|
Description=llama-server router — {{ llm_router_models_dir }} (OpenAI-compatible, port {{ llm_router_port }})
|
|
Documentation=https://github.com/ggml-org/llama.cpp
|
|
After=network.target nvidia-persistenced.service
|
|
Wants=nvidia-persistenced.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
User={{ llm_service_user }}
|
|
Group={{ llm_service_user }}
|
|
Environment="HOME=/home/{{ llm_service_user }}"
|
|
ExecStart={{ llm_binary_path }} \
|
|
{% if llm_router_preset_enabled | default(false) %}
|
|
--models-preset {{ llm_router_preset_path }} \
|
|
{% else %}
|
|
--models-dir {{ llm_router_models_dir }} \
|
|
{% endif %}
|
|
--models-max {{ llm_router_models_max }} \
|
|
--host {{ llm_router_bind_address }} \
|
|
--port {{ llm_router_port }} \
|
|
--n-gpu-layers {{ llm_router_gpu_layers }} \
|
|
{% if not (llm_router_preset_enabled | default(false)) %}
|
|
--ctx-size {{ llm_router_ctx_size }} \
|
|
--flash-attn {{ llm_router_flash_attn }} \
|
|
{% endif %}
|
|
--cache-type-k {{ llm_router_cache_type_k }} \
|
|
--cache-type-v {{ llm_router_cache_type_v }} \
|
|
--batch-size {{ llm_router_batch_size }} \
|
|
--ubatch-size {{ llm_router_ubatch_size }} \
|
|
--parallel {{ llm_router_parallel }} \
|
|
--metrics
|
|
|
|
# ROUTER MODE NOTES (2026-08-12, t_0cca74a2 / updated t_9adf0889):
|
|
# - NO -m/--model flag: this is what enables llama-server router/supervisor mode.
|
|
# Without -m, llama-server discovers all .gguf files in --models-dir, or uses
|
|
# the per-model definitions in a --models-preset INI file.
|
|
# - PRESET MODE (t_9adf0889, 2026-08-12):
|
|
# llm_router_preset_enabled=true switches from --models-dir to --models-preset.
|
|
# Preset mode adds alias support (--models-dir cannot assign aliases).
|
|
# The preset INI is at {{ llm_router_preset_path | default('/opt/llama-server-router-preset.ini') }}.
|
|
# Both the section name and the alias field in the INI work as model IDs.
|
|
# GH #22364 (extra "default" entry in /v1/models) is expected in preset mode — cosmetic.
|
|
# - --models-max {{ llm_router_models_max }} is driven by llm_router_models_max
|
|
# (default 1 in defaults/main.yml; overridden to 4 in host_vars/astro-orbiter
|
|
# as of t_33acbb2e after VRAM budget review — see host_vars for OOM risk note).
|
|
# Default llama-server cap is 4 simultaneous — OOM on 24GB if all 3 current
|
|
# GGUFs load at once. LRU eviction mitigates in practice but review before adding
|
|
# models. See host_vars/astro-orbiter/vars.yml for full VRAM breakdown.
|
|
# - Clients select a model via "model": "<section-name-or-alias>" in their
|
|
# chat completion request. Hermes sends model: "<id>" on every request already.
|
|
# - Cold model load on first request: ~30-60s for Qwen3.6-35B. First response
|
|
# will be slow. This is expected. Document in runbook.
|
|
# - No --jinja flag: Qwen3.6-35B uses its own embedded chat template correctly.
|
|
# If per-model template overrides are ever needed, use --models-preset INI
|
|
# (but note GH #23460: sampler params in presets may not work in router mode).
|
|
#
|
|
# SHADOW DEPLOYMENT NOTE (historical — 2026-08-12, t_0cca74a2):
|
|
# This unit was originally deployed on port 8003 as a shadow. After validation,
|
|
# it was promoted to production on port 8002 (t_cd0d5388). The --port value
|
|
# above is the authoritative value; the port 8003 references below are historical.
|
|
# Production is now llama-server-router (this unit); llama-server-qwen is the rollback target.
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
TimeoutStartSec=600
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
SyslogIdentifier=llama-server-router
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|