--- # ------------------------------------------------------------------------------ # FILE: roles/llm-inference-multimodel/defaults/main.yml # DESCRIPTION: Overridable defaults for the llm-inference-multimodel role. # Deploy target: astro-orbiter (10.1.71.130, RTX 3090 24GB). # Built ALONGSIDE roles/llm-inference (not a replacement) — that # role's CUDA/build/driver phases are the prerequisite; this role # assumes /opt/llama.cpp/build/bin/llama-server already exists. # # See /home/hermes/astro-orbiter-multi-model-plan.md for the full # approved design (VRAM math, rationale, rollback story). # ------------------------------------------------------------------------------ # Shared llm_service_user: jarvis llm_binary_path: /opt/llama.cpp/build/bin/llama-server llm_models_dir: /opt/models # Bind address — deliberately NOT 0.0.0.0 (see plan §5). Default to the private # LAN interface so both instances are reachable from Hermes but not the world. # Override to 127.0.0.1 if even LAN-wide reachability is unwanted and a reverse # proxy/localhost-only tunnel is used instead. llm_bind_address: "10.1.71.130" # Firewall scoping (Phase 3) — subnet/hosts allowed to reach the ports above. # Override per-environment; default assumes Hermes runs somewhere on this /24. llm_allowed_source_cidr: "10.1.70.0/24" # --- RETIRED (2026-08-06): Aux / classification instance (port 8000, Phi-4-14B) # and Tool-calling instance (port 8001, Mistral-Small-24B) -------------------- # Consolidated down to a single production model (Qwen2.5-14B-Instruct-1M, # port 8002) serving BOTH the friday and war-machine Hermes profiles. Ryan # explicitly accepted the tradeoffs (single model for chat + tool-calling + # aux duties) over keeping the aux/toolcall split running. # Both llama-server-aux and llama-server-toolcall services were stopped, # disabled, and had their unit files removed from astro-orbiter; their GGUF # weights (phi-4-14b-instruct-Q4_K_M.gguf, mistral-small-24b-instruct-2501- # Q3_K_M.gguf) were deleted from /opt/models (~45GB reclaimed). The # templates/tasks that deployed them have been removed from this role — see # git log for the prior variable definitions and unit templates if a future # rollback needs them restored. # --- Production instance (port 8002, Qwen2.5-14B-Instruct-1M) ---------------- # History (2026-08-06): Qwen2.5-14B-Instruct (base) was deployed to this slot # and DISQUALIFIED — live /v1/models meta reported n_ctx_train=32768, well # under the 64K Hermes floor (the model card's "128K" figure conflated # YaRN-extended inference-time scaling with actual trained context; disabled # by default, not baked in). Llama-3.1-8B-Instruct was tried next — cleared # the context gate (verified live n_ctx_train=131072) but failed the # tool-calling validation harness badly (8/10 hallucination-stress prompts # triggered spurious tool_calls even at temp=0.1 with the correct official # chat template) — purged from disk and Ansible entirely, see git log. # Current model: Qwen2.5-14B-Instruct-1M (bartowski GGUF) — distinct # checkpoint with genuine additional long-context pretraining, NOT the same # weights as the disqualified base model above. Live-verified 2026-08-06: # /v1/models reports n_ctx=65536, n_ctx_train=1010000 (well over the 64K # floor). Tool-calling verified live via a /v1/chat/completions probe with a # tools= payload — returned a well-formed tool_calls response (finish_reason # "tool_calls", valid JSON arguments), no hallucinated calls observed. # PROMOTED TO PRODUCTION (2026-08-06): llm_qwen_service_enabled now defaults # to true — this is the sole model serving both Hermes profiles. Ports # 8000/8001 are permanently freed; no co-residency VRAM gate applies anymore. llm_qwen_service_enabled: true llm_qwen_port: 8002 llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf" llm_qwen_model_min_bytes: 19000000000 # guard threshold; complete file ~20GB llm_qwen_ctx_size: 65536 llm_qwen_parallel: 1 llm_qwen_gpu_layers: 99 llm_qwen_batch_size: 2048 llm_qwen_ubatch_size: 512 llm_qwen_service_name: llama-server-qwen llm_qwen_model_id: Qwen3.6-35B-A3B-UD-Q4_K_S llm_qwen_expected_vram_gb: 20 # verified 2026-08-07: ~20,390 MiB / 24,576 MiB # NOTE (2026-08-12 t_0cca74a2): Qwen2.5-14B-Instruct-1M was superseded by # Qwen3.6-35B-A3B-UD-Q4_K_S (task t_2ffc0f63, 2026-08-07). Defaults updated # to reflect the current production model. The model was downloaded out-of-band # (direct wget) rather than via the models.yml get_url pattern. # llm_qwen_model_url is intentionally not set — see models.yml WARN task for # the HuggingFace URL if a re-download is ever needed. # --- Existing Gemma baseline (rollback target — never modified by this role) - # Populated by Phase 0 discovery (tasks/discover.yml) if not already known. # Set here only as a fallback name to search for; discovery is authoritative. llm_existing_gemma_service_name_guess: llama-server # --- Router mode shadow deployment (port 8003) -------------------------------- # Deploy llama-server in router/supervisor mode (no -m flag) on a shadow port. # Production unit (llama-server-qwen, port 8002) is UNCHANGED until validation # gates pass and Ryan explicitly approves cutover. # # Default: llm_router_enabled: false — all router tasks are no-ops until you # flip this to true (either in host_vars, extra-vars, or the shadow playbook). # # CRITICAL: llm_router_models_max is hardcoded to 1 in the j2 template AND # listed here for documentation. DO NOT raise it without a VRAM budget review. # Default llama-server cap is 4 simultaneous — that would OOM a 24GB card # immediately when Qwen3.6-35B (20GB) is the resident model. # # Added 2026-08-12 (t_0cca74a2): router mode migration — War Machine. llm_router_enabled: false llm_router_port: 8003 llm_router_service_name: llama-server-router llm_router_models_dir: "{{ llm_models_dir }}" # /opt/models — same dir as production llm_router_models_max: 1 # CRITICAL: RTX 3090 24GB, single model only llm_router_ctx_size: 65536 # 64K — must match production (Hermes floor) llm_router_parallel: 1 llm_router_gpu_layers: 99 llm_router_batch_size: 2048 llm_router_ubatch_size: 512 llm_router_cache_type_k: q4_0 # required to fit 64K KV in 24GB llm_router_cache_type_v: q4_0 llm_router_flash_attn: "auto" llm_router_bind_address: "{{ llm_bind_address }}" # 10.1.71.130 llm_router_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24 llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" # verified at Gate 1 llm_router_vram_max_mib: 23000 # Gate 3: fail if exceeded under load