--- # ------------------------------------------------------------------------------ # FILE: roles/llm-inference-multimodel/defaults/main.yml # DESCRIPTION: Overridable defaults for the llm-inference-multimodel role. # Deploy target: astro-orbiter (10.1.71.130, RTX 3090 24GB). # Built ALONGSIDE roles/llm-inference (not a replacement) — that # role's CUDA/build/driver phases are the prerequisite; this role # assumes /opt/llama.cpp/build/bin/llama-server already exists. # # See /home/hermes/astro-orbiter-multi-model-plan.md for the full # approved design (VRAM math, rationale, rollback story). # ------------------------------------------------------------------------------ # Shared llm_service_user: jarvis llm_binary_path: /opt/llama.cpp/build/bin/llama-server llm_models_dir: /opt/models # Bind address — deliberately NOT 0.0.0.0 (see plan §5). Default to the private # LAN interface so both instances are reachable from Hermes but not the world. # Override to 127.0.0.1 if even LAN-wide reachability is unwanted and a reverse # proxy/localhost-only tunnel is used instead. llm_bind_address: "10.1.71.130" # Firewall scoping (Phase 3) — subnet/hosts allowed to reach the ports above. # Override per-environment; default assumes Hermes runs somewhere on this /24. llm_allowed_source_cidr: "10.1.70.0/24" # --- Aux / classification instance (port 8000, Phi-4-14B) ------------------- # Text-only instruction model, no tool-calling training — safe offload target # per the auxiliary-task-offload skill's "no tool_calls emission risk" bar. llm_aux_port: 8000 llm_aux_model_path: "{{ llm_models_dir }}/phi-4-14b-instruct-Q4_K_M.gguf" llm_aux_model_url: "https://huggingface.co/bartowski/phi-4-GGUF/resolve/main/phi-4-Q4_K_M.gguf" llm_aux_model_min_bytes: 8000000000 # guard threshold; complete file ~8.5GB llm_aux_ctx_size: 8192 llm_aux_parallel: 2 llm_aux_gpu_layers: 99 llm_aux_service_name: llama-server-aux llm_aux_model_id: phi-4-14b-instruct # served model name for OpenAI-compat API # --- Tool-calling instance (port 8001, Mistral-Small-24B) -------------------- # Native function-calling support; deployed at Q3_K_M per plan §1 Option B # to fit VRAM budget. MUST pass the §7 validation harness before any # Claude-Code-capable profile is pointed at this port. llm_toolcall_port: 8001 llm_toolcall_model_path: "{{ llm_models_dir }}/mistral-small-24b-instruct-2501-Q3_K_M.gguf" llm_toolcall_model_url: "https://huggingface.co/bartowski/Mistral-Small-24B-Instruct-2501-GGUF/resolve/main/Mistral-Small-24B-Instruct-2501-Q3_K_M.gguf" llm_toolcall_model_min_bytes: 11000000000 # guard threshold; complete file ~11.7GB llm_toolcall_ctx_size: 4096 llm_toolcall_parallel: 1 # 99 (full offload) OOMs on a 24GB GPU when co-resident with llama-server # (phi-4, port 8000, ~10.4GB). 20 layers validated stable alongside it with # headroom to spare (see docs/validation-log.md). llm_toolcall_gpu_layers: 20 # Chat template shipped with Mistral-Small-24B-Instruct-2501 has no tool-call # support (no [AVAILABLE_TOOLS]/[TOOL_CALLS] handling) — see # docs/validation-log.md for the investigation. We use Mistral-Nemo-Instruct-2407's # tool-calling-capable template instead; Mistral-Small-24B shares the same # Mistral tokenizer/instruct format family so the template is compatible. llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-tool-use.jinja" llm_toolcall_service_name: llama-server-toolcall llm_toolcall_model_id: mistral-small-24b-instruct-2501 # --- Shadow instance (port 8002) --------------------------------------------- # History (2026-08-06): Qwen2.5-14B-Instruct (base) was deployed to this slot # and DISQUALIFIED — live /v1/models meta reported n_ctx_train=32768, well # under the 64K Hermes floor (the model card's "128K" figure conflated # YaRN-extended inference-time scaling with actual trained context; disabled # by default, not baked in). Llama-3.1-8B-Instruct was tried next — cleared # the context gate (verified live n_ctx_train=131072) but failed the # tool-calling validation harness badly (8/10 hallucination-stress prompts # triggered spurious tool_calls even at temp=0.1 with the correct official # chat template) — purged from disk and Ansible entirely, see git log. # Current candidate: Qwen2.5-14B-Instruct-1M (bartowski GGUF) — distinct # checkpoint with genuine additional long-context pretraining, NOT the same # weights as the disqualified base model above. MUST re-verify n_ctx_train # live from /v1/models before trusting any claim — the base-Qwen and # Llama-3.1 lessons both apply here. # GATE: VRAM co-residency with Phi-4(8000)+Mistral(8001) not guaranteed — # stop llama-server-toolcall for the shadow-test window if needed (proven # pattern from prior attempts). llm_qwen_service_enabled defaults to false so # a normal playbook run will template the unit but NOT start it. llm_qwen_service_enabled: false llm_qwen_port: 8002 llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf" llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-1M-GGUF/resolve/main/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf" llm_qwen_model_min_bytes: 9500000000 # guard threshold; complete file ~10.5GB llm_qwen_ctx_size: 65536 llm_qwen_parallel: 1 llm_qwen_gpu_layers: 99 llm_qwen_batch_size: 2048 llm_qwen_ubatch_size: 512 llm_qwen_service_name: llama-server-qwen llm_qwen_model_id: qwen2.5-14b-instruct-1m llm_qwen_expected_vram_gb: 11 # updated 2026-08-06 for -1M checkpoint per report math (~10-12GB); prior 16.5 was stale base-Qwen estimate # --- Existing Gemma baseline (rollback target — never modified by this role) - # Populated by Phase 0 discovery (tasks/discover.yml) if not already known. # Set here only as a fallback name to search for; discovery is authoritative. llm_existing_gemma_service_name_guess: llama-server