92 lines
5.0 KiB
YAML
92 lines
5.0 KiB
YAML
---
|
|
# ------------------------------------------------------------------------------
|
|
# FILE: roles/llm-inference-multimodel/defaults/main.yml
|
|
# DESCRIPTION: Overridable defaults for the llm-inference-multimodel role.
|
|
# Deploy target: astro-orbiter (10.1.71.130, RTX 3090 24GB).
|
|
# Built ALONGSIDE roles/llm-inference (not a replacement) — that
|
|
# role's CUDA/build/driver phases are the prerequisite; this role
|
|
# assumes /opt/llama.cpp/build/bin/llama-server already exists.
|
|
#
|
|
# See /home/hermes/astro-orbiter-multi-model-plan.md for the full
|
|
# approved design (VRAM math, rationale, rollback story).
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# Shared
|
|
llm_service_user: jarvis
|
|
llm_binary_path: /opt/llama.cpp/build/bin/llama-server
|
|
llm_models_dir: /opt/models
|
|
|
|
# Bind address — deliberately NOT 0.0.0.0 (see plan §5). Default to the private
|
|
# LAN interface so both instances are reachable from Hermes but not the world.
|
|
# Override to 127.0.0.1 if even LAN-wide reachability is unwanted and a reverse
|
|
# proxy/localhost-only tunnel is used instead.
|
|
llm_bind_address: "10.1.71.130"
|
|
|
|
# Firewall scoping (Phase 3) — subnet/hosts allowed to reach the ports above.
|
|
# Override per-environment; default assumes Hermes runs somewhere on this /24.
|
|
llm_allowed_source_cidr: "10.1.70.0/24"
|
|
|
|
# --- Aux / classification instance (port 8000, Phi-4-14B) -------------------
|
|
# Text-only instruction model, no tool-calling training — safe offload target
|
|
# per the auxiliary-task-offload skill's "no tool_calls emission risk" bar.
|
|
llm_aux_port: 8000
|
|
llm_aux_model_path: "{{ llm_models_dir }}/phi-4-14b-instruct-Q4_K_M.gguf"
|
|
llm_aux_model_url: "https://huggingface.co/bartowski/phi-4-GGUF/resolve/main/phi-4-Q4_K_M.gguf"
|
|
llm_aux_model_min_bytes: 8000000000 # guard threshold; complete file ~8.5GB
|
|
llm_aux_ctx_size: 8192
|
|
llm_aux_parallel: 2
|
|
llm_aux_gpu_layers: 99
|
|
llm_aux_service_name: llama-server-aux
|
|
llm_aux_model_id: phi-4-14b-instruct # served model name for OpenAI-compat API
|
|
|
|
# --- Tool-calling instance (port 8001, Mistral-Small-24B) --------------------
|
|
# Native function-calling support; deployed at Q3_K_M per plan §1 Option B
|
|
# to fit VRAM budget. MUST pass the §7 validation harness before any
|
|
# Claude-Code-capable profile is pointed at this port.
|
|
llm_toolcall_port: 8001
|
|
llm_toolcall_model_path: "{{ llm_models_dir }}/mistral-small-24b-instruct-2501-Q3_K_M.gguf"
|
|
llm_toolcall_model_url: "https://huggingface.co/bartowski/Mistral-Small-24B-Instruct-2501-GGUF/resolve/main/Mistral-Small-24B-Instruct-2501-Q3_K_M.gguf"
|
|
llm_toolcall_model_min_bytes: 11000000000 # guard threshold; complete file ~11.7GB
|
|
llm_toolcall_ctx_size: 4096
|
|
llm_toolcall_parallel: 1
|
|
# 99 (full offload) OOMs on a 24GB GPU when co-resident with llama-server
|
|
# (phi-4, port 8000, ~10.4GB). 20 layers validated stable alongside it with
|
|
# headroom to spare (see docs/validation-log.md).
|
|
llm_toolcall_gpu_layers: 20
|
|
# Chat template shipped with Mistral-Small-24B-Instruct-2501 has no tool-call
|
|
# support (no [AVAILABLE_TOOLS]/[TOOL_CALLS] handling) — see
|
|
# docs/validation-log.md for the investigation. We use Mistral-Nemo-Instruct-2407's
|
|
# tool-calling-capable template instead; Mistral-Small-24B shares the same
|
|
# Mistral tokenizer/instruct format family so the template is compatible.
|
|
llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-tool-use.jinja"
|
|
llm_toolcall_service_name: llama-server-toolcall
|
|
llm_toolcall_model_id: mistral-small-24b-instruct-2501
|
|
|
|
# --- Shadow instance (port 8002, Qwen2.5-14B-Instruct) -----------------------
|
|
# Shadow-deployment candidate to eventually replace the toolcall (8001) slot
|
|
# per /home/hermes/reports/local-llm-64k-context-recommendation.md. Runs
|
|
# alongside 8000/8001 during validation — does NOT replace either.
|
|
# GATE: VRAM co-residency NOT confirmed as of 2026-08-06 — see role README /
|
|
# deployment report. Do not enable/start this service until that is resolved
|
|
# (either free VRAM by stopping llama-server-toolcall for the shadow window,
|
|
# or reduce ctx-size). llm_qwen_service_enabled defaults to false so a normal
|
|
# playbook run will template the unit but NOT start it.
|
|
llm_qwen_service_enabled: false
|
|
llm_qwen_port: 8002
|
|
llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-Q5_K_M.gguf"
|
|
llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-GGUF/resolve/main/Qwen2.5-14B-Instruct-Q5_K_M.gguf"
|
|
llm_qwen_model_min_bytes: 9500000000 # guard threshold; complete file ~10.5GB
|
|
llm_qwen_ctx_size: 65536
|
|
llm_qwen_parallel: 1
|
|
llm_qwen_gpu_layers: 99
|
|
llm_qwen_batch_size: 2048
|
|
llm_qwen_ubatch_size: 512
|
|
llm_qwen_service_name: llama-server-qwen
|
|
llm_qwen_model_id: qwen2.5-14b-instruct
|
|
llm_qwen_expected_vram_gb: 16.5
|
|
|
|
# --- Existing Gemma baseline (rollback target — never modified by this role) -
|
|
# Populated by Phase 0 discovery (tasks/discover.yml) if not already known.
|
|
# Set here only as a fallback name to search for; discovery is authoritative.
|
|
llm_existing_gemma_service_name_guess: llama-server
|