Add Qwen3-8B no-think variant — dual thinking deployment (t_664289a0)

Part A: qwen3-no-think.jinja.j2 Ansible template - New template: templates/qwen3-no-think.jinja.j2 Standard Qwen3 chat template with enable_thinking unconditionally false (hardcoded empty <think></think> prefix at add_generation_prompt step). Deployed to /opt/models/templates/qwen3-no-think.jinja on astro-orbiter. - tasks/models.yml: deploy templates dir + qwen3-no-think.jinja via ansible.builtin.template task (tags: models, chat_templates). Part B: INI preset template — two Qwen3-8B sections - templates/llama-server-router-preset.ini.j2: [Qwen3-8B-Q4_K_M] — thinking variant, same GGUF, baked-in template. [Qwen3-8B-Q4_K_M-no_think] — no-think variant, same GGUF, chat-template-file = /opt/models/templates/qwen3-no-think.jinja. Both sections: n-gpu-layers=99, ctx-size=32768, flash-attn=true, q4_0 KV cache, sleep-idle-seconds=60. Part C: defaults/main.yml — llama-swap models + matrix - llm_swapmode_models: added Qwen3-8B-Q4_K_M (port 8106, GPU) and Qwen3-8B-Q4_K_M-no_think (port 8107, GPU, chat_template_file set). - llm_swapmode_matrix_rows: row5 (Qwen3-8B thinking + embed), row6 (Qwen3-8B no_think + embed). Neither co-resident with Qwen3.8-27B. - templates/llama-swap-config.yaml.j2: added chat_template_file support (--chat-template-file flag conditional on model.chat_template_file). - tasks/swapmode.yml: GATE 2 assert updated 5 -> 7 models. Part D: war-machine config.yaml (not tracked in git) - custom_providers.astro-orbiter.models: added Qwen3-8B-Q4_K_M-no_think (context_length: 32768). - Reassigned 5 latency-sensitive aux tasks from Meta-Llama-3.1-8B to Qwen3-8B-Q4_K_M-no_think: skills_hub, approval, mcp, title_generation, profile_describer. Rationale: GPU-resident, lower latency, json_schema OK. - web_extract and compression remain on Phi-3.5-mini (long scrapes). VRAM: both Qwen3-8B variants co-reside with nomic-embed only (~10.4GB + 84MB). Cannot co-reside with Qwen3.8-27B (17.8GB); LRU eviction applies.
2026-08-19 11:35:08 -05:00
parent a04435ee9b
commit 5cf4468754
6 changed files with 845 additions and 31 deletions
--- a/ansible/roles/llm-inference-multimodel/defaults/main.yml
+++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml
@@ -62,22 +62,21 @@ llm_allowed_source_cidr: "10.1.70.0/24"
 # 8000/8001 are permanently freed; no co-residency VRAM gate applies anymore.
 llm_qwen_service_enabled: true
 llm_qwen_port: 8002
-llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf"
-llm_qwen_model_min_bytes: 19000000000   # guard threshold; complete file ~20GB
+llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf"
+llm_qwen_model_min_bytes: 17000000000   # guard threshold; complete file ~17.1GB
 llm_qwen_ctx_size: 65536
 llm_qwen_parallel: 1
 llm_qwen_gpu_layers: 99
-llm_qwen_batch_size: 2048
-llm_qwen_ubatch_size: 512
+llm_qwen_batch_size: 4096
+llm_qwen_ubatch_size: 4096
 llm_qwen_service_name: llama-server-qwen
-llm_qwen_model_id: Qwen3.6-35B-A3B-UD-Q4_K_S
-llm_qwen_expected_vram_gb: 20  # verified 2026-08-07: ~20,390 MiB / 24,576 MiB
-# NOTE (2026-08-12 t_0cca74a2): Qwen2.5-14B-Instruct-1M was superseded by
-# Qwen3.6-35B-A3B-UD-Q4_K_S (task t_2ffc0f63, 2026-08-07). Defaults updated
-# to reflect the current production model. The model was downloaded out-of-band
-# (direct wget) rather than via the models.yml get_url pattern.
-# llm_qwen_model_url is intentionally not set — see models.yml WARN task for
-# the HuggingFace URL if a re-download is ever needed.
+llm_qwen_model_id: Qwen3.8-27B-Q4_K_M
+llm_qwen_expected_vram_gb: 17  # Q4_K_M = 17.1GB weights + ~6GB KV @ 65536 ctx = ~23GB max
+# NOTE (2026-08-16 t_f5f7e9ad): Qwen3.6-35B-A3B-UD-Q4_K_S superseded by
+# Qwen3.8-27B-Q4_K_M per Ryan's direction. Qwen3.8-27B is a dense 27B VLM
+# (Apache-2.0, Alibaba, Aug 2026) quantized by Unsloth Dynamic V3.0.
+# Q4_K_M: 17,106,775,008 bytes. Downloaded out-of-band via wget.
+# llm_qwen_model_url: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf

 # --- Staged GGUF models (data-driven, idempotent staging) --------------------
 # Additional GGUFs to ensure are present in llm_models_dir, alongside the
@@ -122,14 +121,14 @@ llm_router_models_max: 1                              # CRITICAL: RTX 3090 24GB,
 llm_router_ctx_size: 65536                            # 64K — must match production (Hermes floor)
 llm_router_parallel: 1
 llm_router_gpu_layers: 99
-llm_router_batch_size: 2048
-llm_router_ubatch_size: 512
+llm_router_batch_size: 4096
+llm_router_ubatch_size: 4096
 llm_router_cache_type_k: q4_0                         # required to fit 64K KV in 24GB
 llm_router_cache_type_v: q4_0
 llm_router_flash_attn: "auto"
 llm_router_bind_address: "{{ llm_bind_address }}"    # 10.1.71.130
 llm_router_allowed_source_cidr: "{{ llm_allowed_source_cidr }}"  # 10.1.70.0/24
-llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S"       # verified at Gate 1
+llm_router_expected_model_id: "Qwen3.8-27B-Q4_K_M"           # verified at Gate 1
 llm_router_vram_max_mib: 23000                        # Gate 3: fail if exceeded under load

 # --- Router preset mode (--models-preset INI) ---------------------------------
@@ -159,4 +158,188 @@ llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}"
 # Qwen2.5-Coder-14B: ctx_size=16384, flash_attn=true per task t_55c164f5
 llm_router_coder_ctx_size: 16384
 llm_router_coder_flash_attn: "true"
+# CPU offload vars (t_72646029, 2026-08-17): n-gpu-layers=0 moves Coder and Llama to
+# full CPU inference. Allows concurrent residency with Qwen3.8-27B. NOTE: llama.cpp
+# 6ea215d still allocates ~1.4-1.7GB CUDA-context VRAM per CPU model, so steady-state
+# is ~24,004 MiB (at the 24,576 MiB physical limit), not the 0-VRAM the spec assumed.
+llm_router_coder_gpu_layers: 0
+llm_router_llama_gpu_layers: 0
 llm_router_preset_path: /opt/llama-server-router-preset.ini
+# Qwen3.8-27B: ctx=65536 (64K). Bumped 32768 -> 131072 (t_441470b9, 2026-08-16);
+# rolled back to 65536 (t_c9fed26c follow-up, 2026-08-18) after t_72646029 CPU-offload
+# deployment moved Phi-3.5mini back to GPU, exceeding RTX 3090 24,576 MiB ceiling.
+# At 131072 ctx + all 5 models resident, Qwen3.8 fails to load (HTTP 500 OOM).
+# 64K satisfies the 2026-08-12 cutover validation Gate 1 (n_ctx >= 64000).
+# Full VRAM analysis and Phase 2 options documented in
+# playbooks/day2_qwen38_ctx128k_rollback.yml.
+llm_router_qwen38_ctx_size: 65536
+# nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83
+# No flash_attn or KV cache params - embedding models use bidirectional forward pass,
+# not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it
+# always warm at negligible VRAM cost (~84MB).
+llm_router_nomic_ctx_size: 8192
+# FIX (2026-08-14, t_openviking_embed_batch): batch-size/ubatch-size were
+# previously omitted from this section entirely, so llama-server silently
+# defaulted the physical batch (ubatch-size) to 512 tokens. Embedding requests
+# cannot be split across ubatches in llama.cpp, so any OpenViking chunk over
+# ~512 tokens (observed 2000-3400 tokens/chunk from openviking-config's
+# embedding.dense chunking) hard-failed with "input (N tokens) is too large to
+# process. increase the physical batch size" - this fed OpenViking's circuit
+# breaker into a permanent fail/re-enqueue loop. 4096 covers the observed max
+# comfortably while staying under ctx-size=8192.
+llm_router_nomic_batch_size: 4096
+llm_router_nomic_ubatch_size: 4096
+
+# --- llama-swap mode (port 8001) -----------------------------------------------
+# Deploy llama-swap — Go-based hot-swap proxy (v250+) for model orchestration.
+# Replaces router mode entirely: single binary + YAML config.json, no --models-preset INI.
+# Additive deployment (non-invasive); production router (port 8002) stays running during Phase 1 shadow.
+#
+# Default: llm_swapmode_enabled: false — all llama-swap tasks are no-ops until flipped to true.
+# Gated by Phase 3 go/no-go once War Machine Phase 1-2 validation completes.
+#
+# NOTE: llama-swap v250 config format differs from evaluation docs (§4b).
+# Uses routing.router DSL with expression-based matrix, not old list-of-arrays syntax.
+# See /etc/llama-swap/config.yaml on astro-orbiter (Phase 1 artifact) for reference.
+#
+# Added 2026-08-18 (t_c1e44190): llama-swap Phase 3 Ansible integration — Wong.
+llm_swapmode_enabled: false                          # Gate for llama-swap tasks (Phase 3)
+llm_swapmode_port: 8001                              # Shadow port (Phase 1), becomes production in Phase 3
+llm_swapmode_bind_address: "{{ llm_bind_address }}"  # 10.1.71.130
+llm_swapmode_allowed_source_cidr: "{{ llm_allowed_source_cidr }}"  # 10.1.70.0/24
+
+# Binary installation
+llm_swapmode_binary_url: "https://github.com/mostlygeek/llama-swap/releases/download/v250/llama-swap-linux-amd64.tar.gz"
+llm_swapmode_binary_version: "v250"
+llm_swapmode_checksum: "sha256:60226b64fcc78e8de6e9d4fac78de95372c2c2a0a31fd6b7d26d1e77ea7c9d9d"  # From Phase 1 deployment
+
+# Directories
+llm_swapmode_config_dir: /etc/llama-swap
+llm_swapmode_config_file: "{{ llm_swapmode_config_dir }}/config.yaml"
+llm_swapmode_models_dir: "{{ llm_models_dir }}"      # /opt/models — same as production
+
+# Service
+llm_swapmode_service_name: llama-swap
+llm_swapmode_service_user: "{{ llm_service_user }}"  # jarvis
+llm_swapmode_vram_max_mib: 23000                     # Gate 3: fail if exceeded under load
+
+# Consolidated model list for llama-swap config.yaml
+# Each model specifies full per-model config (ctx_size, n_gpu_layers, cmd args)
+# Instead of scattered llm_router_* variables, this is the structure llama-swap expects
+# (matches the v250 config.yaml YAML structure, not the router's INI/per-model variables)
+llm_swapmode_models:
+  - id: Qwen3.8-27B-Q4_K_M
+    gguf_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf"
+    port: 8105
+    n_gpu_layers: -1       # -1 = auto-detect / all layers to GPU
+    ctx_size: 65536
+    batch_size: 4096
+    ubatch_size: 4096
+    parallel: 1
+    cache_type: q8_0
+    flash_attn: true
+    sleep_idle_seconds: -1 # never idle (primary model — always ready)
+    load_on_startup: true
+
+  - id: Qwen2.5-Coder-14B-Instruct-Q4_K_M
+    gguf_path: "{{ llm_models_dir }}/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf"
+    port: 8101
+    n_gpu_layers: 0        # CPU-offload (aux model)
+    ctx_size: 16384
+    batch_size: 4096
+    ubatch_size: 4096
+    parallel: 1
+    flash_attn: "true"
+    sleep_idle_seconds: 60  # idle after 60s no requests
+
+  - id: Meta-Llama-3.1-8B-Instruct-Q4_K_M
+    gguf_path: "{{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"
+    port: 8102
+    n_gpu_layers: 0        # CPU-offload (aux model)
+    ctx_size: 8192
+    batch_size: 4096
+    ubatch_size: 4096
+    parallel: 1
+    flash_attn: "true"
+    sleep_idle_seconds: 60
+
+  - id: Phi-3.5-mini-instruct-Q8_0
+    gguf_path: "{{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf"
+    port: 8104
+    n_gpu_layers: 0        # CPU-offload (aux model)
+    ctx_size: 32768
+    batch_size: 4096
+    ubatch_size: 4096
+    parallel: 1
+    flash_attn: "true"
+    sleep_idle_seconds: 60
+
+  - id: nomic-embed-text-v1.5
+    gguf_path: "{{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf"
+    port: 8103
+    n_gpu_layers: 0        # CPU-offload (embedding model — always on)
+    ctx_size: 8192
+    batch_size: 4096
+    ubatch_size: 4096
+    parallel: 1
+    sleep_idle_seconds: -1 # never idle (always ready for embeddings)
+    load_on_startup: true
+
+  # t_c5cef2b2 / t_664289a0 (2026-08-19): Qwen3-8B dual-thinking deployment.
+  # Both variants point to the same GGUF. GPU-resident (~5.2GB each).
+  # Cannot co-reside with Qwen3.8-27B-Q4_K_M; LRU eviction applies.
+  # chat_template_file for no_think variant: {{ llm_models_dir }}/templates/qwen3-no-think.jinja
+  - id: Qwen3-8B-Q4_K_M
+    gguf_path: "{{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf"
+    port: 8106
+    n_gpu_layers: 99       # GPU-resident (thinking variant)
+    ctx_size: 32768
+    batch_size: 4096
+    ubatch_size: 4096
+    parallel: 1
+    cache_type: q4_0
+    flash_attn: "true"
+    sleep_idle_seconds: 60  # idle after 60s no requests
+
+  - id: Qwen3-8B-Q4_K_M-no_think
+    gguf_path: "{{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf"
+    port: 8107
+    n_gpu_layers: 99       # GPU-resident (no-think variant)
+    ctx_size: 32768
+    batch_size: 4096
+    ubatch_size: 4096
+    parallel: 1
+    cache_type: q4_0
+    flash_attn: "true"
+    sleep_idle_seconds: 60
+    chat_template_file: "{{ llm_models_dir }}/templates/qwen3-no-think.jinja"
+
+# llama-swap matrix routing configuration
+# Each row defines a set of models that can be co-resident and hot-swappable
+# Syntax: "model1 & model2" = both models in same row (via v250 expression DSL)
+llm_swapmode_matrix_rows:
+  - row: row0
+    expr: "nomic-embed-text-v1.5"                                    # Embedding-only row
+
+  - row: row1
+    expr: "Qwen3.8-27B-Q4_K_M & nomic-embed-text-v1.5"              # Primary + embed
+
+  - row: row2
+    expr: "Meta-Llama-3.1-8B-Instruct-Q4_K_M & nomic-embed-text-v1.5"  # Aux LLM + embed
+
+  - row: row3
+    expr: "Qwen2.5-Coder-14B-Instruct-Q4_K_M & nomic-embed-text-v1.5"  # Coder + embed
+
+  - row: row4
+    expr: "Phi-3.5-mini-instruct-Q8_0 & nomic-embed-text-v1.5"      # Mini + embed
+
+  # t_c5cef2b2 / t_664289a0 (2026-08-19): Qwen3-8B dual-thinking rows.
+  # Both Qwen3-8B variants co-reside with nomic-embed but NOT with
+  # Qwen3.8-27B-Q4_K_M (17.8GB). LRU eviction swaps between primary and
+  # Qwen3-8B when needed. They CAN co-reside with each other (~10.4GB total)
+  # but NOT simultaneously with Qwen3.8-27B.
+  - row: row5
+    expr: "Qwen3-8B-Q4_K_M & nomic-embed-text-v1.5"                  # Thinking variant + embed
+
+  - row: row6
+    expr: "Qwen3-8B-Q4_K_M-no_think & nomic-embed-text-v1.5"          # No-think variant + embed