homelab/ansible/host_vars/astro-orbiter/vars.yml

---
# ------------------------------------------------------------------------------
# FILE: ansible/host_vars/astro_orbiter/vars.yml
# HOST: astro-orbiter (10.1.71.130)
# ROLE: llama.cpp LLM inference host — Ryzen 7 5800XT / RTX 3090 (ATX rebuild,
#       2026-08-04). Superseded the prior AMD RX 5700 / Ollama config below;
#       drive was transplanted into new hardware, not reinstalled.
# ------------------------------------------------------------------------------

ansible_host: 10.1.71.130
ansible_user: jarvis
ansible_ssh_private_key_file: ~/.ssh/id_jarvis
ansible_become: true

# LVM root expansion — xlarge template uses sda3 partition, standard VG/LV names
common_expand_root_lvm: true
common_root_pv: /dev/sda3
common_root_vg: ubuntu-vg
common_root_lv: ubuntu-lv

# --- Staged GGUF models for the llama.cpp router (:8002) ---------------------
# Data-driven list consumed by roles/llm-inference-multimodel tasks/models.yml
# (loop -> tasks/stage_model.yml). Each entry is idempotently staged into
# /opt/models: stat + EXACT-size check vs HF manifest; skip (no download, no
# restart) when present + size matches. Source repos are public bartowski GGUFs
# on HuggingFace (no auth). A router restart is notified ONLY when a new GGUF
# is actually downloaded.
# Added 2026-08-12 (War Machine): codify Phi-3.5-mini-instruct-Q8_0 and
# Meta-Llama-3.1-8B-Instruct-Q4_K_M as router models alongside the production
# Qwen3.6-35B-A3B-UD-Q4_K_S. The live files were already present/correct on
# astro-orbiter; this pass codifies them. Future adds = append to this list.
# Router --models-max override for astro-orbiter.
# Default in defaults/main.yml is 1 (conservative). Bumped to 4 on 2026-08-12
# (t_33acbb2e) so the router can keep more than one GGUF resident on-demand
# and LRU-evict when needed.
#
# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83, updated t_f5f7e9ad, updated t_441470b9, updated t_c5cef2b2):
# With models-max=4 and all 6 GGUFs registered, worst case is all 6 loaded simultaneously:
#   Qwen3.8-27B Q4_K_M:               ~20.0GB (weights ~17.1GB + KV ~2.9GB @ 65536 ctx, q4_0) ← CORRECTED (ctx rolled back from 128K to 65536, t_c9fed26c 2026-08-18)
#   Phi-3.5-mini-instruct Q8_0:       ~4.3GB  (weights ~3.8GB + KV ~0.5GB @ 32K ctx)
#   Meta-Llama-3.1-8B Q4_K_M:        ~5.6GB  (weights ~4.6GB + KV ~0.2GB @ 8K ctx)
#   Qwen2.5-Coder-14B Q4_K_M:        ~9.0GB  (weights ~8.4GB + KV ~0.6GB @ 16K ctx)
#   nomic-embed-text-v1.5 Q4_K_M:    ~0.09GB (~84MB, embedding only — no KV cache)
#   Qwen3-8B Q4_K_M:                 ~5.5GB  (weights ~4.68GB + KV ~0.5GB @ 32K ctx, q4_0)
#   Total worst-case:                 ~44.5GB  >>  24GB RTX 3090
#
# OOM RISK: Full co-residency is impossible on 24GB. LRU eviction prevents this
# in practice: models-max=4 means the router can REGISTER 6 models but only keeps
# up to 4 LOADED simultaneously — the router will evict the LRU model when a new
# one is needed. nomic-embed-text-v1.5 is pinned via sleep-idle-seconds=-1 and
# load-on-startup=true but it uses only ~84MB, so it never meaningfully changes
# the budget. In single-user homelab operation, only one generative model is active
# at a time alongside the always-resident embedding model.
# Qwen3.8-27B alone uses ~17,804 MiB (weights+KV @ 65536 ctx); co-residency
# with Coder (~9GB) = ~27GB > 24GB. LRU eviction handles this automatically.
# Ryan should be aware this means model-switching always incurs a ~30-60s
# cold-load latency when switching between Qwen3.8-27B and any other model.
# Proceeding to models-max=4 as instructed; flagged for Ryan's attention.
# Router --models-max override for astro-orbiter.
# UPDATED (t_f5f7e9ad, 2026-08-16): Set to 2 because Qwen3.8-27B-Q4_K_M
# uses 17,804 MiB at 65536 ctx. Only nomic-embed (558MB, pinned) and ONE
# generative model can be resident simultaneously. Co-residency of Qwen3.8
# with any auxiliary model (Phi 8.3GB, Llama 5.9GB, Coder 9GB) exceeds 24GB.
# models-max=2: slot 1 = nomic-embed (pinned, always loaded), slot 2 = LRU
# generative model (Qwen3.8 primary, cold-loaded on first request ~30-60s;
# auxiliary models evict it on demand, and vice versa).
# NOTE: Qwen3.8 does NOT have load-on-startup — it loads on first request.
# This avoids an LRU eviction race with nomic-embed at startup.
# UPDATED (t_72646029, 2026-08-17): CPU offload for Coder + Llama changes the
# constraint. Coder and Llama now use CPU inference (n-gpu-layers=0). GPU-resident
# VRAM: Qwen3.8 (~17,804 MiB at 65536 ctx) + nomic-embed (558 MiB, pinned) plus
# the CUDA-context buffers llama.cpp 6ea215d allocates for the CPU models (~1.4-1.7GB
# each) = ~20,004 MiB steady-state, below the 24,576 MiB physical limit.
# CORRECTED (t_c5cef2b2, 2026-08-19): ctx-size was rolled back from 131072 to 65536
# (t_c9fed26c 2026-08-18). Qwen3.8 VRAM at 65536: 17,804 MiB (not 20,302 MiB).
# models-max raised to 4: nomic (slot 1, pinned) + Qwen3.8 (slot 2, GPU) +
# Llama (slot 3, CPU) + Coder (slot 4, CPU). Phi (GPU, ~8.3GB) and new
# Qwen3-8B (GPU, ~5.5GB) can also be requested but evict Qwen3.8 due to VRAM.
# models-max=4 is required so CPU-offloaded models count as loaded without
# evicting Qwen3.8.
llm_router_models_max: 4

llm_staged_models:
  - filename: "Phi-3.5-mini-instruct-Q8_0.gguf"
    url: "https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q8_0.gguf"
    size_bytes: 4061222688
    source_repo: "bartowski/Phi-3.5-mini-instruct-GGUF"
  - filename: "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"
    url: "https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"
    size_bytes: 4920739232
    source_repo: "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF"
  - filename: "Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf"
    url: "https://huggingface.co/bartowski/Qwen2.5-Coder-14B-Instruct-GGUF/resolve/main/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf"
    size_bytes: 8988111072
    source_repo: "bartowski/Qwen2.5-Coder-14B-Instruct-GGUF"
  - filename: "nomic-embed-text-v1.5-Q4_K_M.gguf"
    url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q4_K_M.gguf"
    size_bytes: 84106624
    source_repo: "nomic-ai/nomic-embed-text-v1.5-GGUF"
  # Added t_c5cef2b2 (2026-08-19, War Machine): Qwen3-8B dense 8B model for
  # aux tasks (routing, rewriting, structured extraction, tool-call construction).
  # Source: bartowski/Qwen_Qwen3-8B-GGUF (public, no auth). HF filename is
  # Qwen_Qwen3-8B-Q4_K_M.gguf; stored locally as Qwen3-8B-Q4_K_M.gguf.
  # Exact size verified from HF manifest (content-length): 5,027,784,224 bytes.
  # VRAM: ~4.68GB weights + ~0.5GB KV @ 32K ctx (q4_0) ≈ 5.2GB total.
  # Thinking mode ON by default; use /no_think for latency-sensitive aux tasks.
  - filename: "Qwen3-8B-Q4_K_M.gguf"
    url: "https://huggingface.co/bartowski/Qwen_Qwen3-8B-GGUF/resolve/main/Qwen_Qwen3-8B-Q4_K_M.gguf"
    size_bytes: 5027784224
    source_repo: "bartowski/Qwen_Qwen3-8B-GGUF"

# --- deploy-vllm role: vllm_models override (t_r1d32b_swap, 2026-09-01) -----
# Ansible's hash_behaviour is "replace" (see ansible.cfg) — a host_vars list
# variable REPLACES the role default list wholesale, it does not deep-merge.
#
# SWAP (Ryan direction, 2026-09-01): Qwen2.5-32B-Instruct-AWQ retired,
# replaced with DeepSeek-R1-Distill-Qwen-32B-AWQ, max_model_len=32768.
# "Single model only" — nomic-embed-text-v1.5 (embedding, :8020) and
# Qwen3-8B-AWQ (aux, :8010, already disabled) are BOTH disabled here.
# DeepSeek gets the full 24GB card to itself. Nothing in production
# consumed nomic-embed at the time of this swap (Hindsight uses its own
# bundled 384-dim embedder; OpenViking pointed at the old llama-swap
# endpoint, already stopped) — confirmed with Ryan before disabling.
#
# Model choice: casperhansen/deepseek-r1-distill-qwen-32b-awq — same
# quantizer/toolchain (AutoAWQ) as the outgoing Qwen2.5-32B-Instruct-AWQ,
# widely used, 4-bit GEMM AWQ, ~19.3GB on disk (4 safetensors shards).
# Architecture: Qwen2ForCausalLM (DeepSeek-R1 distilled onto Qwen2.5-32B
# base) — same vLLM code path as the outgoing model, no new serving
# support needed. Native max_position_embeddings=131072; we cap at 32768
# per the task's explicit max-model-len requirement.
#
# VRAM math: ~19.3GB weights (4-bit AWQ) + KV cache at 32768 ctx (GQA,
# 8 KV heads, 128 head_dim, 64 layers, fp16 KV by default) ≈ 19.3GB +
# ~4GB KV+overhead ≈ 23.3GB — tight but the FULL 24GB card is now
# available (no co-resident nomic-embed/Qwen3-8B taking a share, unlike
# the outgoing Qwen2.5-32B config). gpu_memory_utilization=0.95 (role
# default) + enforce_eager retained as the proven-stable mitigation from
# t_e6facb19/t_ca1af9fb (avoids CUDA graph capture VRAM spike; this host's
# only validated way to avoid crash-loop-to-stabilize behavior on this
# card). If 0.95 OOMs at 32768 ctx once tested live, drop to 0.90 next
# (documented fallback, same pattern as the outgoing model).
#
# DeepSeek-R1 output note: reasoning traces stream in <think> tags before
# the final answer — this is expected R1-distill behavior, not a bug.
# Model card recommends temperature 0.5-0.7 (not 0, not vLLM's greedy
# default) to avoid repetition/incoherence; not set here (server-side
# default), left to be set client-side per the model card's guidance —
# flagging for whoever wires this into Hermes profile configs next.
vllm_models:
  - id: "Gemma-4-26B-A4B-it-AWQ"
    hf_repo: "cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit"
    role: primary
    # NO quantization field set (unlike the AutoAWQ-quantized DeepSeek/
    # Qwen2.5 models above) — live test (2026-09-01) found this repo's
    # config.json declares quant_method: "compressed-tensors" (llm-compressor
    # tool output, not classic AutoAWQ), even though the repo name says
    # "AWQ-4bit". Passing --quantization awq explicitly caused a hard
    # pydantic ValidationError at every single startup attempt: "Quantization
    # method specified in the model config (compressed-tensors) does not
    # match the quantization method specified in the `quantization` argument
    # (awq)." vLLM auto-detects the quant method correctly from the model's
    # own config.json when --quantization is omitted — confirmed fix, clean
    # start. Lesson: don't trust a HF repo's naming convention ("...-AWQ...")
    # for the `quantization:` field here — check config.json's quant_method.
    port: 8000
    # Ryan direction (2026-09-01, t_gemma4_swap): DeepSeek-R1-Distill-Qwen-32B
    # retired after confirming its `auto` tool-choice reliability is a known,
    # documented DeepSeek-R1-distillation limitation (trained on pure
    # reasoning traces, no function-calling data — GitHub-confirmed upstream,
    # not a vLLM config gap). Replaced with Gemma 4 26B A4B (Google,
    # Apache 2.0, US-origin — matches Ryan's standing model-origin
    # preference, unlike Qwen/DeepSeek). Chose MoE (26B A4B, 3.8B active)
    # over the dense 31B variant: ~3.7GB smaller on-disk AWQ footprint
    # (17.2GB vs 20.9GB) buys more KV-cache headroom on this tight 24GB
    # card, and decode should be faster (memory-bandwidth-bound on active
    # params, not total params). Tradeoff accepted: MoE scores lower than
    # dense on the Tau2 tool-use benchmark (68.2% vs 76.9%) but still beats
    # every other size in the family except the 31B on most reasoning
    # benchmarks. Model choice: cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit —
    # AutoAWQ 4-bit group_size=32, MoE expert layers (gate/up/down/router)
    # explicitly excluded from quantization ("ignore" list in config.json)
    # per standard llm-compressor MoE quant practice — only the dense
    # attention/projection layers are 4-bit, experts stay higher precision.
    # Native architecture: Gemma4ForConditionalGeneration (registered
    # natively in this host's installed vLLM 0.28.0 — vllm/model_executor/
    # models/registry.py line 415 — no plugin/trust-remote-code needed).
    # Native max_position_embeddings: 262144 (256K) — Hermes's 64K floor is
    # comfortably covered without any context-extension trick.
    max_model_len: 65536
    # VRAM math (not yet live-validated — see swap validation log below
    # once run): AWQ weights ~17.2GB on disk (dense attn 4-bit + MoE
    # experts higher-precision, per config.json's compressed-tensors
    # ignore list). Starting the KV cache dtype at int4_per_token_head
    # from the outset (rather than fp16 -> fp8 -> int4 trial-and-error like
    # the DeepSeek swap) since that same escalation pattern is expected to
    # repeat on this VRAM-constrained card for any 20+ GB model at >32K ctx.
    kv_cache_dtype: int4_per_token_head
    gpu_memory_utilization: 0.95
    enforce_eager: true
    # Native tool-calling + reasoning support (no `hermes` workaround
    # needed, unlike DeepSeek-R1-Distill): Gemma4EngineToolParser and
    # Gemma4ParserReasoningAdapter are both registered natively in this
    # host's vLLM 0.28.0 (vllm/tool_parsers/__init__.py,
    # vllm/reasoning/__init__.py) — purpose-built for this model's actual
    # output format, not a same-family approximation.
    enable_auto_tool_choice: true
    tool_call_parser: gemma4
    reasoning_parser: gemma4
    enabled: true
  - id: "Qwen3-8B-AWQ"
    hf_repo: "Qwen/Qwen3-8B-AWQ"
    role: aux
    quantization: awq
    port: 8010
    max_model_len: 32768
    gpu_memory_utilization: 0.15
    enforce_eager: true
    enabled: false   # single-model deployment — see swap note above
  - id: "nomic-embed-text-v1.5"
    hf_repo: "nomic-ai/nomic-embed-text-v1.5"
    role: embedding
    quantization: none
    port: 8020
    max_model_len: 2048
    gpu_memory_utilization: 0.05
    trust_remote_code: true
    enabled: false   # single-model deployment — see swap note above

# --- deploy-vllm role: boot persistence (unchanged) -------------------------
# Still permanent/boot-persistent — same policy as the outgoing Qwen2.5-32B
# deployment (t_5508360a), just now serving one model instead of two.
vllm_service_enabled: true
vllm_service_state: started