--- # ------------------------------------------------------------------------------ # FILE: roles/llm-inference-multimodel/defaults/main.yml # DESCRIPTION: Overridable defaults for the llm-inference-multimodel role. # Deploy target: astro-orbiter (10.1.71.130, RTX 3090 24GB). # Built ALONGSIDE roles/llm-inference (not a replacement) — that # role's CUDA/build/driver phases are the prerequisite; this role # assumes /opt/llama.cpp/build/bin/llama-server already exists. # # See /home/hermes/astro-orbiter-multi-model-plan.md for the full # approved design (VRAM math, rationale, rollback story). # ------------------------------------------------------------------------------ # Shared llm_service_user: jarvis llm_binary_path: /opt/llama.cpp/build/bin/llama-server llm_models_dir: /opt/models # Bind address — deliberately NOT 0.0.0.0 (see plan §5). Default to the private # LAN interface so both instances are reachable from Hermes but not the world. # Override to 127.0.0.1 if even LAN-wide reachability is unwanted and a reverse # proxy/localhost-only tunnel is used instead. llm_bind_address: "10.1.71.130" # Firewall scoping (Phase 3) — subnet/hosts allowed to reach the ports above. # Override per-environment; default assumes Hermes runs somewhere on this /24. llm_allowed_source_cidr: "10.1.70.0/24" # --- RETIRED (2026-08-06): Aux / classification instance (port 8000, Phi-4-14B) # and Tool-calling instance (port 8001, Mistral-Small-24B) -------------------- # Consolidated down to a single production model (Qwen2.5-14B-Instruct-1M, # port 8002) serving BOTH the friday and war-machine Hermes profiles. Ryan # explicitly accepted the tradeoffs (single model for chat + tool-calling + # aux duties) over keeping the aux/toolcall split running. # Both llama-server-aux and llama-server-toolcall services were stopped, # disabled, and had their unit files removed from astro-orbiter; their GGUF # weights (phi-4-14b-instruct-Q4_K_M.gguf, mistral-small-24b-instruct-2501- # Q3_K_M.gguf) were deleted from /opt/models (~45GB reclaimed). The # templates/tasks that deployed them have been removed from this role — see # git log for the prior variable definitions and unit templates if a future # rollback needs them restored. # --- Production instance (port 8002, Qwen2.5-14B-Instruct-1M) ---------------- # History (2026-08-06): Qwen2.5-14B-Instruct (base) was deployed to this slot # and DISQUALIFIED — live /v1/models meta reported n_ctx_train=32768, well # under the 64K Hermes floor (the model card's "128K" figure conflated # YaRN-extended inference-time scaling with actual trained context; disabled # by default, not baked in). Llama-3.1-8B-Instruct was tried next — cleared # the context gate (verified live n_ctx_train=131072) but failed the # tool-calling validation harness badly (8/10 hallucination-stress prompts # triggered spurious tool_calls even at temp=0.1 with the correct official # chat template) — purged from disk and Ansible entirely, see git log. # Current model: Qwen2.5-14B-Instruct-1M (bartowski GGUF) — distinct # checkpoint with genuine additional long-context pretraining, NOT the same # weights as the disqualified base model above. Live-verified 2026-08-06: # /v1/models reports n_ctx=65536, n_ctx_train=1010000 (well over the 64K # floor). Tool-calling verified live via a /v1/chat/completions probe with a # tools= payload — returned a well-formed tool_calls response (finish_reason # "tool_calls", valid JSON arguments), no hallucinated calls observed. # PROMOTED TO PRODUCTION (2026-08-06): llm_qwen_service_enabled now defaults # to true — this is the sole model serving both Hermes profiles. Ports # 8000/8001 are permanently freed; no co-residency VRAM gate applies anymore. llm_qwen_service_enabled: true llm_qwen_port: 8002 llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf" llm_qwen_model_min_bytes: 17000000000 # guard threshold; complete file ~17.1GB llm_qwen_ctx_size: 65536 llm_qwen_parallel: 1 llm_qwen_gpu_layers: 99 llm_qwen_batch_size: 4096 llm_qwen_ubatch_size: 4096 llm_qwen_service_name: llama-server-qwen llm_qwen_model_id: Qwen3.8-27B-Q4_K_M llm_qwen_expected_vram_gb: 17 # Q4_K_M = 17.1GB weights + ~6GB KV @ 65536 ctx = ~23GB max # NOTE (2026-08-16 t_f5f7e9ad): Qwen3.6-35B-A3B-UD-Q4_K_S superseded by # Qwen3.8-27B-Q4_K_M per Ryan's direction. Qwen3.8-27B is a dense 27B VLM # (Apache-2.0, Alibaba, Aug 2026) quantized by Unsloth Dynamic V3.0. # Q4_K_M: 17,106,775,008 bytes. Downloaded out-of-band via wget. # llm_qwen_model_url: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf # --- Staged GGUF models (data-driven, idempotent staging) -------------------- # Additional GGUFs to ensure are present in llm_models_dir, alongside the # production Qwen3.6-35B. Consumed by tasks/models.yml (loop over # tasks/stage_model.yml). Each entry: # filename: target filename in llm_models_dir # url: HuggingFace resolve URL (public repos; no auth needed) # size_bytes: EXACT expected byte size (HF manifest) — guard: download only # if the file is missing OR its size != this value (idempotent; # never re-pulls a correct file, never needlessly restarts). # source_repo: upstream HF repo (audit/lineage) # The REAL list is defined per-host in host_vars/astro-orbiter/vars.yml (NOT # hardcoded here) so the role stays generic and reusable for future model adds. # Empty default = nothing staged (safe no-op). llm_staged_models: [] # --- Existing Gemma baseline (rollback target — never modified by this role) - # Populated by Phase 0 discovery (tasks/discover.yml) if not already known. # Set here only as a fallback name to search for; discovery is authoritative. llm_existing_gemma_service_name_guess: llama-server # --- Router mode shadow deployment (port 8003) -------------------------------- # Deploy llama-server in router/supervisor mode (no -m flag) on a shadow port. # Production unit (llama-server-qwen, port 8002) is UNCHANGED until validation # gates pass and Ryan explicitly approves cutover. # # Default: llm_router_enabled: false — all router tasks are no-ops until you # flip this to true (either in host_vars, extra-vars, or the shadow playbook). # # CRITICAL: llm_router_models_max default is 1 here for safety. It is # overridden to 4 in host_vars/astro-orbiter/vars.yml (t_33acbb2e) with # a full VRAM budget note. DO NOT raise it without a VRAM budget review. # Default llama-server cap is 4 simultaneous — that would OOM a 24GB card # immediately when Qwen3.6-35B (20GB) is the resident model. # # Added 2026-08-12 (t_0cca74a2): router mode migration — War Machine. llm_router_enabled: false llm_router_port: 8003 llm_router_service_name: llama-server-router llm_router_models_dir: "{{ llm_models_dir }}" # /opt/models — same dir as production llm_router_models_max: 1 # CRITICAL: RTX 3090 24GB, single model only llm_router_ctx_size: 65536 # 64K — must match production (Hermes floor) llm_router_parallel: 1 llm_router_gpu_layers: 99 llm_router_batch_size: 4096 llm_router_ubatch_size: 4096 llm_router_cache_type_k: q4_0 # required to fit 64K KV in 24GB llm_router_cache_type_v: q4_0 llm_router_flash_attn: "auto" llm_router_bind_address: "{{ llm_bind_address }}" # 10.1.71.130 llm_router_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24 llm_router_expected_model_id: "Qwen3.8-27B-Q4_K_M" # verified at Gate 1 llm_router_vram_max_mib: 23000 # Gate 3: fail if exceeded under load # --- Router preset mode (--models-preset INI) --------------------------------- # Set llm_router_preset_enabled: true to switch from --models-dir to # --models-preset. Preset mode is REQUIRED to support model aliases. # The template at llama-server-router-preset.ini.j2 defines all 3 router models: # - Qwen3.6-35B-A3B-UD-Q4_K_S (no alias — primary ID unchanged) # - Phi-3.5-mini-instruct-Q8_0 (alias: Phi-3.5-mini-instruct-8bit) <-- t_9adf0889 # - Meta-Llama-3.1-8B-Instruct-Q4_K_M (no alias — primary ID unchanged) # # llm_router_preset_path: on-disk path where the rendered INI is deployed. # Default: /opt/llama-server-router-preset.ini (owned by root, readable by all). # # GH #22364 note: --models-preset causes an extra "default" entry in /v1/models. # This is cosmetic and does not affect model selection by name. Accept it. # # Added 2026-08-12 (t_9adf0889) — War Machine. llm_router_preset_enabled: false # flip true to activate preset mode # Per-model ctx-size / flash-attn overrides for preset mode (t_ryan_per_model_ctx). # Defaults mirror the prior uniform 65536/auto behavior; host_vars or the # deploy playbook override these to the values Ryan requested per workload. llm_router_llama_ctx_size: "{{ llm_router_ctx_size }}" llm_router_llama_flash_attn: "{{ llm_router_flash_attn }}" llm_router_phi_ctx_size: "{{ llm_router_ctx_size }}" llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}" # Qwen2.5-Coder-14B: ctx_size=16384, flash_attn=true per task t_55c164f5 llm_router_coder_ctx_size: 16384 llm_router_coder_flash_attn: "true" # CPU offload vars (t_72646029, 2026-08-17): n-gpu-layers=0 moves Coder and Llama to # full CPU inference. Allows concurrent residency with Qwen3.8-27B. NOTE: llama.cpp # 6ea215d still allocates ~1.4-1.7GB CUDA-context VRAM per CPU model, so steady-state # is ~24,004 MiB (at the 24,576 MiB physical limit), not the 0-VRAM the spec assumed. llm_router_coder_gpu_layers: 0 llm_router_llama_gpu_layers: 0 llm_router_preset_path: /opt/llama-server-router-preset.ini # Qwen3.8-27B: ctx=65536 (64K). Bumped 32768 -> 131072 (t_441470b9, 2026-08-16); # rolled back to 65536 (t_c9fed26c follow-up, 2026-08-18) after t_72646029 CPU-offload # deployment moved Phi-3.5mini back to GPU, exceeding RTX 3090 24,576 MiB ceiling. # At 131072 ctx + all 5 models resident, Qwen3.8 fails to load (HTTP 500 OOM). # 64K satisfies the 2026-08-12 cutover validation Gate 1 (n_ctx >= 64000). # Full VRAM analysis and Phase 2 options documented in # playbooks/day2_qwen38_ctx128k_rollback.yml. llm_router_qwen38_ctx_size: 65536 # nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83 # No flash_attn or KV cache params - embedding models use bidirectional forward pass, # not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it # always warm at negligible VRAM cost (~84MB). llm_router_nomic_ctx_size: 8192 # FIX (2026-08-14, t_openviking_embed_batch): batch-size/ubatch-size were # previously omitted from this section entirely, so llama-server silently # defaulted the physical batch (ubatch-size) to 512 tokens. Embedding requests # cannot be split across ubatches in llama.cpp, so any OpenViking chunk over # ~512 tokens (observed 2000-3400 tokens/chunk from openviking-config's # embedding.dense chunking) hard-failed with "input (N tokens) is too large to # process. increase the physical batch size" - this fed OpenViking's circuit # breaker into a permanent fail/re-enqueue loop. 4096 covers the observed max # comfortably while staying under ctx-size=8192. llm_router_nomic_batch_size: 4096 llm_router_nomic_ubatch_size: 4096 # --- Monitoring: VRAM exporter + Prometheus scrape + Grafana dashboard ------- # Phase 3: GPU/LLM monitoring deployment (Wong, 2026-08-18) # Provides: VRAM textfile exporter, Prometheus scrape config for llama-swap # /metrics endpoint, Grafana 6-panel dashboard, PrometheusRule alert rules. # # Ref: roles/llm-inference-multimodel/references/monitoring-llm-homelab-ciro-luciotta-2026.md llm_monitoring_enabled: true # gate for monitoring tasks llm_vram_exporter_script: /opt/llama-server-monitoring/nvidia-smi-vram-exporter.sh llm_vram_exporter_cron_minute: "*" # run every minute llm_vram_exporter_gpu_index: 0 # GPU 0 (RTX 3090 on astro-orbiter) llm_vram_textfile_dir: /var/lib/node_exporter/textfile_collector # Alert thresholds (per Ciro Luciotta pattern) llm_vram_critical_mib: 24000 # ~90% of 24GB RTX 3090 llm_kv_cache_spill_ratio: 0.92 # KV-cache spill threshold llm_throughput_baseline_tokens_per_min: 50 # baseline for degradation alert # Grafana dashboard llm_grafana_dashboard_uid: llama-swap-monitor llm_grafana_dashboard_title: "llama-swap GPU/LLM Monitoring" llm_grafana_dashboard_tags: - llm - llama-swap - gpu-monitoring - ciro-luciotta llm_grafana_dashboard_refresh: "30s" llm_grafana_dashboard_time_from: "now-24h" # Prometheus scrape job llm_prometheus_scrape_interval: "30s" llm_prometheus_scrape_timeout: "10s" # --- llama-swap mode (port 8001) ----------------------------------------------- # Deploy llama-swap — Go-based hot-swap proxy (v250+) for model orchestration. # Replaces router mode entirely: single binary + YAML config.json, no --models-preset INI. # Additive deployment (non-invasive); production router (port 8002) stays running during Phase 1 shadow. # # Default: llm_swapmode_enabled: false — all llama-swap tasks are no-ops until flipped to true. # Gated by Phase 3 go/no-go once War Machine Phase 1-2 validation completes. # # NOTE: llama-swap v250 config format differs from evaluation docs (§4b). # Uses routing.router DSL with expression-based matrix, not old list-of-arrays syntax. # See /etc/llama-swap/config.yaml on astro-orbiter (Phase 1 artifact) for reference. # # Added 2026-08-18 (t_c1e44190): llama-swap Phase 3 Ansible integration — Wong. llm_swapmode_enabled: false # Gate for llama-swap tasks (Phase 3) llm_swapmode_port: 8001 # Shadow port (Phase 1), becomes production in Phase 3 llm_swapmode_bind_address: "{{ llm_bind_address }}" # 10.1.71.130 llm_swapmode_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24 # Binary installation llm_swapmode_binary_url: "https://github.com/mostlygeek/llama-swap/releases/download/v250/llama-swap-linux-amd64.tar.gz" llm_swapmode_binary_version: "v250" llm_swapmode_checksum: "sha256:60226b64fcc78e8de6e9d4fac78de95372c2c2a0a31fd6b7d26d1e77ea7c9d9d" # From Phase 1 deployment # Directories llm_swapmode_config_dir: /etc/llama-swap llm_swapmode_config_file: "{{ llm_swapmode_config_dir }}/config.yaml" llm_swapmode_models_dir: "{{ llm_models_dir }}" # /opt/models — same as production # Service llm_swapmode_service_name: llama-swap llm_swapmode_service_user: "{{ llm_service_user }}" # jarvis llm_swapmode_vram_max_mib: 23000 # Gate 3: fail if exceeded under load # Consolidated model list for llama-swap config.yaml # Each model specifies full per-model config (ctx_size, n_gpu_layers, cmd args) # Instead of scattered llm_router_* variables, this is the structure llama-swap expects # (matches the v250 config.yaml YAML structure, not the router's INI/per-model variables) llm_swapmode_models: - id: Qwen3.8-27B-Q4_K_M gguf_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf" port: 8105 n_gpu_layers: -1 # -1 = auto-detect / all layers to GPU ctx_size: 65536 batch_size: 4096 ubatch_size: 4096 parallel: 1 cache_type: q8_0 flash_attn: true sleep_idle_seconds: -1 # never idle (primary model — always ready) load_on_startup: true - id: Qwen2.5-Coder-14B-Instruct-Q4_K_M gguf_path: "{{ llm_models_dir }}/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf" port: 8101 n_gpu_layers: 0 # CPU-offload (aux model) ctx_size: 16384 batch_size: 4096 ubatch_size: 4096 parallel: 1 flash_attn: "true" sleep_idle_seconds: 60 # idle after 60s no requests - id: Meta-Llama-3.1-8B-Instruct-Q4_K_M gguf_path: "{{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf" port: 8102 n_gpu_layers: 0 # CPU-offload (aux model) ctx_size: 8192 batch_size: 4096 ubatch_size: 4096 parallel: 1 flash_attn: "true" sleep_idle_seconds: 60 - id: Phi-3.5-mini-instruct-Q8_0 gguf_path: "{{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf" port: 8104 n_gpu_layers: 0 # CPU-offload (aux model) ctx_size: 32768 batch_size: 4096 ubatch_size: 4096 parallel: 1 flash_attn: "true" sleep_idle_seconds: 60 - id: nomic-embed-text-v1.5 gguf_path: "{{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf" port: 8103 n_gpu_layers: 0 # CPU-offload (embedding model — always on) ctx_size: 8192 batch_size: 4096 ubatch_size: 4096 parallel: 1 sleep_idle_seconds: -1 # never idle (always ready for embeddings) load_on_startup: true # llama-swap matrix routing configuration # Each row defines a set of models that can be co-resident and hot-swappable # Syntax: "model1 & model2" = both models in same row (via v250 expression DSL) llm_swapmode_matrix_rows: - row: row0 expr: "nomic-embed-text-v1.5" # Embedding-only row - row: row1 expr: "Qwen3.8-27B-Q4_K_M & nomic-embed-text-v1.5" # Primary + embed - row: row2 expr: "Meta-Llama-3.1-8B-Instruct-Q4_K_M & nomic-embed-text-v1.5" # Aux LLM + embed - row: row3 expr: "Qwen2.5-Coder-14B-Instruct-Q4_K_M & nomic-embed-text-v1.5" # Coder + embed - row: row4 expr: "Phi-3.5-mini-instruct-Q8_0 & nomic-embed-text-v1.5" # Mini + embed