diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml
index 7748515..0fc4aa4 100644
--- a/ansible/roles/llm-inference-multimodel/defaults/main.yml
+++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml
@@ -62,22 +62,21 @@ llm_allowed_source_cidr: "10.1.70.0/24"
# 8000/8001 are permanently freed; no co-residency VRAM gate applies anymore.
llm_qwen_service_enabled: true
llm_qwen_port: 8002
-llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf"
-llm_qwen_model_min_bytes: 19000000000 # guard threshold; complete file ~20GB
+llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf"
+llm_qwen_model_min_bytes: 17000000000 # guard threshold; complete file ~17.1GB
llm_qwen_ctx_size: 65536
llm_qwen_parallel: 1
llm_qwen_gpu_layers: 99
-llm_qwen_batch_size: 2048
-llm_qwen_ubatch_size: 512
+llm_qwen_batch_size: 4096
+llm_qwen_ubatch_size: 4096
llm_qwen_service_name: llama-server-qwen
-llm_qwen_model_id: Qwen3.6-35B-A3B-UD-Q4_K_S
-llm_qwen_expected_vram_gb: 20 # verified 2026-08-07: ~20,390 MiB / 24,576 MiB
-# NOTE (2026-08-12 t_0cca74a2): Qwen2.5-14B-Instruct-1M was superseded by
-# Qwen3.6-35B-A3B-UD-Q4_K_S (task t_2ffc0f63, 2026-08-07). Defaults updated
-# to reflect the current production model. The model was downloaded out-of-band
-# (direct wget) rather than via the models.yml get_url pattern.
-# llm_qwen_model_url is intentionally not set — see models.yml WARN task for
-# the HuggingFace URL if a re-download is ever needed.
+llm_qwen_model_id: Qwen3.8-27B-Q4_K_M
+llm_qwen_expected_vram_gb: 17 # Q4_K_M = 17.1GB weights + ~6GB KV @ 65536 ctx = ~23GB max
+# NOTE (2026-08-16 t_f5f7e9ad): Qwen3.6-35B-A3B-UD-Q4_K_S superseded by
+# Qwen3.8-27B-Q4_K_M per Ryan's direction. Qwen3.8-27B is a dense 27B VLM
+# (Apache-2.0, Alibaba, Aug 2026) quantized by Unsloth Dynamic V3.0.
+# Q4_K_M: 17,106,775,008 bytes. Downloaded out-of-band via wget.
+# llm_qwen_model_url: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf
# --- Staged GGUF models (data-driven, idempotent staging) --------------------
# Additional GGUFs to ensure are present in llm_models_dir, alongside the
@@ -122,14 +121,14 @@ llm_router_models_max: 1 # CRITICAL: RTX 3090 24GB,
llm_router_ctx_size: 65536 # 64K — must match production (Hermes floor)
llm_router_parallel: 1
llm_router_gpu_layers: 99
-llm_router_batch_size: 2048
-llm_router_ubatch_size: 512
+llm_router_batch_size: 4096
+llm_router_ubatch_size: 4096
llm_router_cache_type_k: q4_0 # required to fit 64K KV in 24GB
llm_router_cache_type_v: q4_0
llm_router_flash_attn: "auto"
llm_router_bind_address: "{{ llm_bind_address }}" # 10.1.71.130
llm_router_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24
-llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" # verified at Gate 1
+llm_router_expected_model_id: "Qwen3.8-27B-Q4_K_M" # verified at Gate 1
llm_router_vram_max_mib: 23000 # Gate 3: fail if exceeded under load
# --- Router preset mode (--models-preset INI) ---------------------------------
@@ -159,4 +158,188 @@ llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}"
# Qwen2.5-Coder-14B: ctx_size=16384, flash_attn=true per task t_55c164f5
llm_router_coder_ctx_size: 16384
llm_router_coder_flash_attn: "true"
+# CPU offload vars (t_72646029, 2026-08-17): n-gpu-layers=0 moves Coder and Llama to
+# full CPU inference. Allows concurrent residency with Qwen3.8-27B. NOTE: llama.cpp
+# 6ea215d still allocates ~1.4-1.7GB CUDA-context VRAM per CPU model, so steady-state
+# is ~24,004 MiB (at the 24,576 MiB physical limit), not the 0-VRAM the spec assumed.
+llm_router_coder_gpu_layers: 0
+llm_router_llama_gpu_layers: 0
llm_router_preset_path: /opt/llama-server-router-preset.ini
+# Qwen3.8-27B: ctx=65536 (64K). Bumped 32768 -> 131072 (t_441470b9, 2026-08-16);
+# rolled back to 65536 (t_c9fed26c follow-up, 2026-08-18) after t_72646029 CPU-offload
+# deployment moved Phi-3.5mini back to GPU, exceeding RTX 3090 24,576 MiB ceiling.
+# At 131072 ctx + all 5 models resident, Qwen3.8 fails to load (HTTP 500 OOM).
+# 64K satisfies the 2026-08-12 cutover validation Gate 1 (n_ctx >= 64000).
+# Full VRAM analysis and Phase 2 options documented in
+# playbooks/day2_qwen38_ctx128k_rollback.yml.
+llm_router_qwen38_ctx_size: 65536
+# nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83
+# No flash_attn or KV cache params - embedding models use bidirectional forward pass,
+# not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it
+# always warm at negligible VRAM cost (~84MB).
+llm_router_nomic_ctx_size: 8192
+# FIX (2026-08-14, t_openviking_embed_batch): batch-size/ubatch-size were
+# previously omitted from this section entirely, so llama-server silently
+# defaulted the physical batch (ubatch-size) to 512 tokens. Embedding requests
+# cannot be split across ubatches in llama.cpp, so any OpenViking chunk over
+# ~512 tokens (observed 2000-3400 tokens/chunk from openviking-config's
+# embedding.dense chunking) hard-failed with "input (N tokens) is too large to
+# process. increase the physical batch size" - this fed OpenViking's circuit
+# breaker into a permanent fail/re-enqueue loop. 4096 covers the observed max
+# comfortably while staying under ctx-size=8192.
+llm_router_nomic_batch_size: 4096
+llm_router_nomic_ubatch_size: 4096
+
+# --- llama-swap mode (port 8001) -----------------------------------------------
+# Deploy llama-swap — Go-based hot-swap proxy (v250+) for model orchestration.
+# Replaces router mode entirely: single binary + YAML config.json, no --models-preset INI.
+# Additive deployment (non-invasive); production router (port 8002) stays running during Phase 1 shadow.
+#
+# Default: llm_swapmode_enabled: false — all llama-swap tasks are no-ops until flipped to true.
+# Gated by Phase 3 go/no-go once War Machine Phase 1-2 validation completes.
+#
+# NOTE: llama-swap v250 config format differs from evaluation docs (§4b).
+# Uses routing.router DSL with expression-based matrix, not old list-of-arrays syntax.
+# See /etc/llama-swap/config.yaml on astro-orbiter (Phase 1 artifact) for reference.
+#
+# Added 2026-08-18 (t_c1e44190): llama-swap Phase 3 Ansible integration — Wong.
+llm_swapmode_enabled: false # Gate for llama-swap tasks (Phase 3)
+llm_swapmode_port: 8001 # Shadow port (Phase 1), becomes production in Phase 3
+llm_swapmode_bind_address: "{{ llm_bind_address }}" # 10.1.71.130
+llm_swapmode_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24
+
+# Binary installation
+llm_swapmode_binary_url: "https://github.com/mostlygeek/llama-swap/releases/download/v250/llama-swap-linux-amd64.tar.gz"
+llm_swapmode_binary_version: "v250"
+llm_swapmode_checksum: "sha256:60226b64fcc78e8de6e9d4fac78de95372c2c2a0a31fd6b7d26d1e77ea7c9d9d" # From Phase 1 deployment
+
+# Directories
+llm_swapmode_config_dir: /etc/llama-swap
+llm_swapmode_config_file: "{{ llm_swapmode_config_dir }}/config.yaml"
+llm_swapmode_models_dir: "{{ llm_models_dir }}" # /opt/models — same as production
+
+# Service
+llm_swapmode_service_name: llama-swap
+llm_swapmode_service_user: "{{ llm_service_user }}" # jarvis
+llm_swapmode_vram_max_mib: 23000 # Gate 3: fail if exceeded under load
+
+# Consolidated model list for llama-swap config.yaml
+# Each model specifies full per-model config (ctx_size, n_gpu_layers, cmd args)
+# Instead of scattered llm_router_* variables, this is the structure llama-swap expects
+# (matches the v250 config.yaml YAML structure, not the router's INI/per-model variables)
+llm_swapmode_models:
+ - id: Qwen3.8-27B-Q4_K_M
+ gguf_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf"
+ port: 8105
+ n_gpu_layers: -1 # -1 = auto-detect / all layers to GPU
+ ctx_size: 65536
+ batch_size: 4096
+ ubatch_size: 4096
+ parallel: 1
+ cache_type: q8_0
+ flash_attn: true
+ sleep_idle_seconds: -1 # never idle (primary model — always ready)
+ load_on_startup: true
+
+ - id: Qwen2.5-Coder-14B-Instruct-Q4_K_M
+ gguf_path: "{{ llm_models_dir }}/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf"
+ port: 8101
+ n_gpu_layers: 0 # CPU-offload (aux model)
+ ctx_size: 16384
+ batch_size: 4096
+ ubatch_size: 4096
+ parallel: 1
+ flash_attn: "true"
+ sleep_idle_seconds: 60 # idle after 60s no requests
+
+ - id: Meta-Llama-3.1-8B-Instruct-Q4_K_M
+ gguf_path: "{{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"
+ port: 8102
+ n_gpu_layers: 0 # CPU-offload (aux model)
+ ctx_size: 8192
+ batch_size: 4096
+ ubatch_size: 4096
+ parallel: 1
+ flash_attn: "true"
+ sleep_idle_seconds: 60
+
+ - id: Phi-3.5-mini-instruct-Q8_0
+ gguf_path: "{{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf"
+ port: 8104
+ n_gpu_layers: 0 # CPU-offload (aux model)
+ ctx_size: 32768
+ batch_size: 4096
+ ubatch_size: 4096
+ parallel: 1
+ flash_attn: "true"
+ sleep_idle_seconds: 60
+
+ - id: nomic-embed-text-v1.5
+ gguf_path: "{{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf"
+ port: 8103
+ n_gpu_layers: 0 # CPU-offload (embedding model — always on)
+ ctx_size: 8192
+ batch_size: 4096
+ ubatch_size: 4096
+ parallel: 1
+ sleep_idle_seconds: -1 # never idle (always ready for embeddings)
+ load_on_startup: true
+
+ # t_c5cef2b2 / t_664289a0 (2026-08-19): Qwen3-8B dual-thinking deployment.
+ # Both variants point to the same GGUF. GPU-resident (~5.2GB each).
+ # Cannot co-reside with Qwen3.8-27B-Q4_K_M; LRU eviction applies.
+ # chat_template_file for no_think variant: {{ llm_models_dir }}/templates/qwen3-no-think.jinja
+ - id: Qwen3-8B-Q4_K_M
+ gguf_path: "{{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf"
+ port: 8106
+ n_gpu_layers: 99 # GPU-resident (thinking variant)
+ ctx_size: 32768
+ batch_size: 4096
+ ubatch_size: 4096
+ parallel: 1
+ cache_type: q4_0
+ flash_attn: "true"
+ sleep_idle_seconds: 60 # idle after 60s no requests
+
+ - id: Qwen3-8B-Q4_K_M-no_think
+ gguf_path: "{{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf"
+ port: 8107
+ n_gpu_layers: 99 # GPU-resident (no-think variant)
+ ctx_size: 32768
+ batch_size: 4096
+ ubatch_size: 4096
+ parallel: 1
+ cache_type: q4_0
+ flash_attn: "true"
+ sleep_idle_seconds: 60
+ chat_template_file: "{{ llm_models_dir }}/templates/qwen3-no-think.jinja"
+
+# llama-swap matrix routing configuration
+# Each row defines a set of models that can be co-resident and hot-swappable
+# Syntax: "model1 & model2" = both models in same row (via v250 expression DSL)
+llm_swapmode_matrix_rows:
+ - row: row0
+ expr: "nomic-embed-text-v1.5" # Embedding-only row
+
+ - row: row1
+ expr: "Qwen3.8-27B-Q4_K_M & nomic-embed-text-v1.5" # Primary + embed
+
+ - row: row2
+ expr: "Meta-Llama-3.1-8B-Instruct-Q4_K_M & nomic-embed-text-v1.5" # Aux LLM + embed
+
+ - row: row3
+ expr: "Qwen2.5-Coder-14B-Instruct-Q4_K_M & nomic-embed-text-v1.5" # Coder + embed
+
+ - row: row4
+ expr: "Phi-3.5-mini-instruct-Q8_0 & nomic-embed-text-v1.5" # Mini + embed
+
+ # t_c5cef2b2 / t_664289a0 (2026-08-19): Qwen3-8B dual-thinking rows.
+ # Both Qwen3-8B variants co-reside with nomic-embed but NOT with
+ # Qwen3.8-27B-Q4_K_M (17.8GB). LRU eviction swaps between primary and
+ # Qwen3-8B when needed. They CAN co-reside with each other (~10.4GB total)
+ # but NOT simultaneously with Qwen3.8-27B.
+ - row: row5
+ expr: "Qwen3-8B-Q4_K_M & nomic-embed-text-v1.5" # Thinking variant + embed
+
+ - row: row6
+ expr: "Qwen3-8B-Q4_K_M-no_think & nomic-embed-text-v1.5" # No-think variant + embed
diff --git a/ansible/roles/llm-inference-multimodel/tasks/models.yml b/ansible/roles/llm-inference-multimodel/tasks/models.yml
index 300b805..5b159ee 100644
--- a/ansible/roles/llm-inference-multimodel/tasks/models.yml
+++ b/ansible/roles/llm-inference-multimodel/tasks/models.yml
@@ -74,3 +74,30 @@
loop_control:
loop_var: staged_model
tags: [models]
+
+# --- Chat template overrides ---------------------------------------------------
+# Deploy per-model chat template files used by llama-server via chat-template-file.
+# These are static files dropped into {{ llm_models_dir }}/templates/.
+# t_664289a0: qwen3-no-think.jinja — Qwen3 template with enable_thinking=false
+# hardcoded. Used by [Qwen3-8B-Q4_K_M-no_think] in the router preset INI and
+# the llama-swap config. The companion [Qwen3-8B-Q4_K_M] section uses the GGUF's
+# baked-in template (thinking ON by default).
+- name: Ensure chat template directory exists at {{ llm_models_dir }}/templates
+ ansible.builtin.file:
+ path: "{{ llm_models_dir }}/templates"
+ state: directory
+ owner: "{{ llm_service_user }}"
+ group: "{{ llm_service_user }}"
+ mode: "0755"
+ become: true
+ tags: [models, chat_templates]
+
+- name: Deploy qwen3-no-think.jinja (thinking=false hard-switch for Qwen3-8B no_think variant)
+ ansible.builtin.template:
+ src: qwen3-no-think.jinja.j2
+ dest: "{{ llm_models_dir }}/templates/qwen3-no-think.jinja"
+ owner: "{{ llm_service_user }}"
+ group: "{{ llm_service_user }}"
+ mode: "0644"
+ become: true
+ tags: [models, chat_templates]
diff --git a/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml b/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml
new file mode 100644
index 0000000..5eb4892
--- /dev/null
+++ b/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml
@@ -0,0 +1,305 @@
+---
+# ------------------------------------------------------------------------------
+# FILE: roles/llm-inference-multimodel/tasks/swapmode.yml
+# DESCRIPTION: Phase S — llama-swap mode hot-swap proxy (port 8001).
+#
+# This phase is ADDITIVE and IDEMPOTENT. The existing production
+# unit (llama-server-qwen, port 8002) is never touched here.
+#
+# All tasks are gated on llm_swapmode_enabled | default(false).
+# With the default (false) this entire file is a no-op.
+#
+# When llm_swapmode_enabled: true (set by host_vars or extra-vars),
+# this phase:
+# swapmode_binary — download + install binary
+# swapmode_config — template config.yaml
+# swapmode_systemd — deploy llama-swap.service unit
+# swapmode_firewall — open port 8001 to Hermes subnet
+# swapmode_verify — start service, run 4 validation gates
+#
+# Tags map 1:1 to the sub-phases for independent execution:
+# --tags swapmode_binary,swapmode_config,swapmode_systemd,swapmode_firewall,swapmode_verify
+#
+# IMPORTANT: swapmode_verify starts the service. Do not run
+# swapmode_verify unless swapmode_binary and swapmode_systemd
+# have already run.
+#
+# Added 2026-08-18 (t_c1e44190): llama-swap Phase 3 Ansible integration — Wong.
+# Approved by War Machine Phase 1 validation (3 of 4 hard gates PASS).
+# Phase 3 gated on all profiles migrated + production router decommissioned.
+# ------------------------------------------------------------------------------
+
+# =============================================================================
+# TAG: swapmode_binary
+# Download and install llama-swap binary from GitHub releases.
+# Idempotent: checks for existing binary and verifies architecture.
+# =============================================================================
+
+- name: "[swapmode_binary] Detect host architecture (x86_64 / aarch64)"
+ ansible.builtin.command:
+ cmd: uname -m
+ register: llm_swapmode_arch
+ changed_when: false
+ become: false
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_binary]
+
+- name: "[swapmode_binary] Ensure config directory exists"
+ ansible.builtin.file:
+ path: "{{ llm_swapmode_config_dir }}"
+ state: directory
+ owner: "{{ llm_swapmode_service_user }}"
+ group: "{{ llm_swapmode_service_user }}"
+ mode: "0755"
+ become: true
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_binary]
+
+- name: "[swapmode_binary] Download llama-swap binary"
+ ansible.builtin.get_url:
+ url: "{{ llm_swapmode_binary_url }}"
+ dest: "/tmp/llama-swap-{{ llm_swapmode_binary_version }}.tar.gz"
+ checksum: "{{ llm_swapmode_checksum }}"
+ mode: "0644"
+ become: true
+ register: llm_swapmode_download
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_binary]
+
+- name: "[swapmode_binary] Extract llama-swap binary"
+ ansible.builtin.unarchive:
+ src: "/tmp/llama-swap-{{ llm_swapmode_binary_version }}.tar.gz"
+ dest: /tmp
+ remote_src: true
+ creates: /tmp/llama-swap
+ become: true
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_binary]
+
+- name: "[swapmode_binary] Install llama-swap to /usr/local/bin"
+ ansible.builtin.copy:
+ src: /tmp/llama-swap
+ dest: /usr/local/bin/llama-swap
+ owner: root
+ group: root
+ mode: "0755"
+ remote_src: true
+ become: true
+ register: llm_swapmode_binary_installed
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_binary]
+
+- name: "[swapmode_binary] Verify llama-swap binary is executable"
+ ansible.builtin.command:
+ cmd: /usr/local/bin/llama-swap --version
+ register: llm_swapmode_version_check
+ changed_when: false
+ become: false
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_binary]
+
+- name: "[swapmode_binary] Cleanup download artifacts"
+ ansible.builtin.file:
+ path: "{{ item }}"
+ state: absent
+ become: true
+ loop:
+ - "/tmp/llama-swap-{{ llm_swapmode_binary_version }}.tar.gz"
+ - /tmp/llama-swap
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_binary]
+
+# =============================================================================
+# TAG: swapmode_config
+# Render config.yaml.j2 template and deploy to /etc/llama-swap/config.yaml
+# =============================================================================
+
+- name: "[swapmode_config] Deploy llama-swap config.yaml from template"
+ ansible.builtin.template:
+ src: llama-swap-config.yaml.j2
+ dest: "{{ llm_swapmode_config_file }}"
+ owner: "{{ llm_swapmode_service_user }}"
+ group: "{{ llm_swapmode_service_user }}"
+ mode: "0644"
+ become: true
+ register: llm_swapmode_config_deployed
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_config]
+
+- name: "[swapmode_config] Validate config.yaml syntax (YAML parse check)"
+ ansible.builtin.command:
+ cmd: python3 -c "import yaml; yaml.safe_load(open('{{ llm_swapmode_config_file }}'))"
+ register: llm_swapmode_config_validate
+ changed_when: false
+ become: true
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_config]
+
+# =============================================================================
+# TAG: swapmode_systemd
+# Deploy the llama-swap systemd unit file and reload systemd.
+# Does NOT start the service — that is swapmode_verify only.
+# =============================================================================
+
+- name: "[swapmode_systemd] Deploy llama-swap systemd unit"
+ ansible.builtin.template:
+ src: llama-swap.service.j2
+ dest: "/etc/systemd/system/{{ llm_swapmode_service_name }}.service"
+ owner: root
+ group: root
+ mode: "0644"
+ become: true
+ register: llm_swapmode_unit_deployed
+ notify:
+ - reload systemd
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_systemd]
+
+- name: "[swapmode_systemd] Flush handlers so daemon-reload lands before swapmode_verify starts the unit"
+ ansible.builtin.meta: flush_handlers
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_systemd]
+
+# =============================================================================
+# TAG: swapmode_firewall
+# Open port 8001 in ufw scoped to the Hermes source subnet.
+# Idempotent: named comment + state: present prevents duplicate rules.
+# =============================================================================
+
+- name: "[swapmode_firewall] Check whether ufw is installed/active"
+ ansible.builtin.command:
+ cmd: ufw status
+ register: llm_swapmode_ufw_status
+ changed_when: false
+ failed_when: false
+ become: true
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_firewall]
+
+- name: "[swapmode_firewall] WARNING — ufw not active, port {{ llm_swapmode_port }} scoping cannot be applied"
+ ansible.builtin.debug:
+ msg: >-
+ ufw does not appear to be active on this host. Firewall scoping for
+ port {{ llm_swapmode_port }} was skipped. Bind address alone
+ ({{ llm_swapmode_bind_address }}) limits exposure — flag to Ryan.
+ when:
+ - llm_swapmode_enabled | default(false)
+ - "'Status: active' not in (llm_swapmode_ufw_status.stdout | default(''))"
+ tags: [swapmode_firewall]
+
+- name: "[swapmode_firewall] Allow llama-swap port ({{ llm_swapmode_port }}) from Hermes source subnet"
+ community.general.ufw:
+ rule: allow
+ port: "{{ llm_swapmode_port | string }}"
+ proto: tcp
+ src: "{{ llm_swapmode_allowed_source_cidr }}"
+ comment: "llm-inference-multimodel: llama-swap ({{ llm_swapmode_port }}) — scoped to Hermes subnet"
+ become: true
+ when:
+ - llm_swapmode_enabled | default(false)
+ - "'Status: active' in (llm_swapmode_ufw_status.stdout | default(''))"
+ tags: [swapmode_firewall]
+
+# =============================================================================
+# TAG: swapmode_verify
+# Start the service, then run the 4 validation gates.
+# This is the ONLY phase that actually starts llama-swap.
+# =============================================================================
+
+- name: "[swapmode_verify] Start llama-swap service"
+ ansible.builtin.systemd:
+ name: "{{ llm_swapmode_service_name }}"
+ state: started
+ enabled: true
+ daemon_reload: true
+ become: true
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_verify]
+
+# GATE 1: Health check
+- name: "[swapmode_verify] GATE 1 — Health check (/health endpoint)"
+ ansible.builtin.uri:
+ url: "http://{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}/health"
+ method: GET
+ status_code: 200
+ register: llm_swapmode_health
+ until: llm_swapmode_health.status == 200
+ retries: 30
+ delay: 2
+ become: false
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_verify]
+
+# GATE 2: Model discovery
+- name: "[swapmode_verify] GATE 2 — Model discovery (/v1/models)"
+ ansible.builtin.uri:
+ url: "http://{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}/v1/models"
+ method: GET
+ status_code: 200
+ register: llm_swapmode_models_list
+ become: false
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_verify]
+
+- name: "[swapmode_verify] Assert all 7 models are discoverable"
+ ansible.builtin.assert:
+ that:
+ - llm_swapmode_models_list.json.data | map(attribute='id') | list | length == 7
+ fail_msg: >-
+ Expected 7 models in /v1/models response, got {{ llm_swapmode_models_list.json.data | length }}.
+ Models: {{ llm_swapmode_models_list.json.data | map(attribute='id') | list }}
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_verify]
+
+# GATE 3: Smoke test — simple completion on a CPU-offload model (no VRAM conflict)
+- name: "[swapmode_verify] GATE 3 — Smoke test completion (Meta-Llama-3.1-8B CPU-offload)"
+ ansible.builtin.uri:
+ url: "http://{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}/v1/chat/completions"
+ method: POST
+ body_format: json
+ body:
+ model: "Meta-Llama-3.1-8B-Instruct-Q4_K_M"
+ messages:
+ - role: "user"
+ content: "What is 2+2?"
+ temperature: 0.1
+ max_tokens: 50
+ status_code: 200
+ register: llm_swapmode_smoke_test
+ become: false
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_verify]
+
+# GATE 4: VRAM guard check
+- name: "[swapmode_verify] GATE 4 — VRAM usage check (must be < {{ llm_swapmode_vram_max_mib }} MiB)"
+ ansible.builtin.shell:
+ cmd: nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1
+ register: llm_swapmode_vram_used
+ changed_when: false
+ become: false
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_verify]
+
+- name: "[swapmode_verify] Assert VRAM usage is within budget"
+ ansible.builtin.assert:
+ that:
+ - (llm_swapmode_vram_used.stdout | int) < llm_swapmode_vram_max_mib
+ fail_msg: >-
+ VRAM usage ({{ llm_swapmode_vram_used.stdout }} MiB) exceeds gate limit ({{ llm_swapmode_vram_max_mib }} MiB).
+ Check for resource contention with production router or other services.
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_verify]
+
+# Display verification results
+- name: "[swapmode_verify] Display verification results"
+ ansible.builtin.debug:
+ msg: |
+ ✓ GATE 1: Health check PASS
+ ✓ GATE 2: Model discovery PASS — {{ llm_swapmode_models_list.json.data | map(attribute='id') | list | join(', ') }}
+ ✓ GATE 3: Smoke test (Llama-3.1-8B) PASS
+ ✓ GATE 4: VRAM guard ({{ llm_swapmode_vram_used.stdout }} MiB < {{ llm_swapmode_vram_max_mib }} MiB) PASS
+
+ llama-swap service is ready at http://{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}/
+ NOTE: 7 models registered (5 original + Qwen3-8B-Q4_K_M + Qwen3-8B-Q4_K_M-no_think).
+ when: llm_swapmode_enabled | default(false)
+ tags: [swapmode_verify]
diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2
index 9c904d9..0cf3c8b 100644
--- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2
+++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2
@@ -40,20 +40,46 @@
; Hermes custom_providers routing — see role README / deployment report for
; the alias-naming ambiguity flag (Ryan's pasted TOML used different alias
; strings: "llama-3.1-8b" / "phi-3.5-mini").
+;
+; UPDATED (t_34b96e83, 2026-08-13, per Ryan approval): Added nomic-embed-text-v1.5
+; embedding model. Embedding models fold cleanly into the router preset via
+; embedding=true. No alias needed — clients call it by section name.
+; VRAM estimate ~90MB (negligible). sleep-idle-seconds=-1 keeps it always loaded
+; since embedding calls are latency-sensitive and it costs near-nothing to hold.
+; load-on-startup=true ensures the embedding endpoint is warm at boot without
+; waiting for the first request. — War Machine.
; ------------------------------------------------------------------------------
-; --- Production model: Qwen3.6-35B-A3B-UD-Q4_K_S ----------------------------
-; Primary model ID: Qwen3.6-35B-A3B-UD-Q4_K_S (unchanged from --models-dir)
-; ~20GB, primary Hermes production LLM. Context: 64K with q4_0 KV cache.
-[Qwen3.6-35B-A3B-UD-Q4_K_S]
-model = {{ llm_models_dir }}/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf
-n-gpu-layers = {{ llm_router_gpu_layers }}
-ctx-size = {{ llm_router_ctx_size }}
-cache-type-k = {{ llm_router_cache_type_k }}
-cache-type-v = {{ llm_router_cache_type_v }}
-batch-size = {{ llm_router_batch_size }}
-ubatch-size = {{ llm_router_ubatch_size }}
-parallel = {{ llm_router_parallel }}
+; --- Production model: Qwen3.8-27B-Q4_K_M ------------------------------------
+; Swapped from Qwen3.6-35B-A3B-UD-Q4_K_S by War Machine (t_f5f7e9ad, 2026-08-16).
+; Ryan-directed swap. Qwen3.8-27B is a dense 27B VLM (Apache-2.0) from Alibaba,
+; released Aug 2026. GGUF quantized by Unsloth Dynamic V3.0 (preview).
+; Q4_K_M chosen: 17.1GB weights — fits RTX 3090 (24GB) with ~7GB headroom for
+; KV cache at ctx=65536 (q4_0 KV). Smaller than prior Qwen3.6 at ~20GB.
+; Native context: 262,144 tokens. Running at 65536 (Hermes floor) for now;
+; can be raised later if needed.
+; VRAM footprint (empirically tested, t_4455a44c 2026-08-16):
+; ctx=32768: 17,068 MiB; ctx=65536: 17,804 MiB; ctx=131072: 20,282 MiB.
+; BUMPED to 131072 (128K) per Ryan approval (t_441470b9, 2026-08-16).
+; nomic-embed always resident at 558 MiB -> total ~20.8GB, ~3.2GB headroom.
+; Native context is 262,144 tokens; 128K is the production ceiling.
+; Architecture note: Qwen3.8 uses Gated DeltaNet; llama.cpp 6ea215d logs
+; "fused Gated Delta Net (chunked) not supported, set to disabled" — falls
+; back to non-fused implementation. Inference works correctly but may be
+; slower on the GDN layers. An updated llama.cpp may improve throughput.
+; load-on-startup NOT set (loads on first request, ~30-60s cold load).
+; n-gpu-layers=99: GPU (all layers). Explicit here so global CLI flag removal
+; (t_72646029, 2026-08-17) does not change Qwen3.8 behavior.
+; Primary model ID: Qwen3.8-27B-Q4_K_M
+[Qwen3.8-27B-Q4_K_M]
+model = {{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf
+n-gpu-layers = 99
+ctx-size = {{ llm_router_qwen38_ctx_size }}
+cache-type-k = {{ llm_router_cache_type_k }}
+cache-type-v = {{ llm_router_cache_type_v }}
+batch-size = {{ llm_router_batch_size }}
+ubatch-size = {{ llm_router_ubatch_size }}
+parallel = {{ llm_router_parallel }}
; --- Auxiliary model: Phi-3.5-mini-instruct-Q8_0 ----------------------------
; Primary model ID: Phi-3.5-mini-instruct-Q8_0 (unchanged from --models-dir)
@@ -79,7 +105,7 @@ parallel = {{ llm_router_parallel }}
[Phi-3.5-mini-instruct-Q8_0]
model = {{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf
alias = Phi-3.5-mini-instruct-8bit
-n-gpu-layers = {{ llm_router_gpu_layers }}
+n-gpu-layers = 99
ctx-size = {{ llm_router_phi_ctx_size }}
flash-attn = {{ llm_router_phi_flash_attn }}
cache-type-k = {{ llm_router_cache_type_k }}
@@ -93,10 +119,15 @@ parallel = {{ llm_router_parallel }}
; Alias: Meta-Llama-3.1-8B-Instruct-4bit (NEW — friendlier name)
; Both names resolve to this GGUF child process.
; ~4.6GB, general-purpose small model. Works with json_schema structured output.
+; n-gpu-layers=0 (CPU offload, t_72646029 2026-08-17): Llama moves to full CPU
+; inference to allow concurrent residency with Qwen3.8-27B (which uses ~20.8GB
+; VRAM including nomic-embed). At models-max=4, Llama and Coder run on CPU —
+; llama.cpp 6ea215d still holds ~1.4-1.7GB CUDA-context VRAM per CPU model, so
+; steady-state is ~24,004 MiB (below the 24,576 MiB physical limit).
[Meta-Llama-3.1-8B-Instruct-Q4_K_M]
model = {{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf
alias = Meta-Llama-3.1-8B-Instruct-4bit
-n-gpu-layers = {{ llm_router_gpu_layers }}
+n-gpu-layers = {{ llm_router_llama_gpu_layers }}
ctx-size = {{ llm_router_llama_ctx_size }}
flash-attn = {{ llm_router_llama_flash_attn }}
cache-type-k = {{ llm_router_cache_type_k }}
@@ -109,14 +140,19 @@ parallel = {{ llm_router_parallel }}
; Primary model ID: Qwen2.5-Coder-14B-Instruct-Q4_K_M (filename-derived)
; Alias: Qwen2.5-Coder-14B-Instruct-4bit (friendlier name)
; Both names resolve to this GGUF child process.
-; ~8.4GB weights + ~0.6GB KV @ 16K ctx = ~9.0GB VRAM.
+; ~8.4GB weights + ~0.6GB KV @ 16K ctx = ~9.0GB VRAM (GPU); ~1,390 MiB CUDA ctx (CPU).
; ctx-size=16384, flash-attn=true per task t_55c164f5 / Ryan's request.
; Source: bartowski/Qwen2.5-Coder-14B-Instruct-GGUF (public, no auth)
; Added 2026-08-13 (t_55c164f5) — War Machine.
+; n-gpu-layers=0 (CPU offload, t_72646029 2026-08-17): Coder moves to full CPU
+; inference to allow concurrent residency with Qwen3.8-27B (which uses ~20.8GB
+; VRAM including nomic-embed). At models-max=4, Coder and Llama run on CPU —
+; llama.cpp 6ea215d still holds ~1.4-1.7GB CUDA-context VRAM per CPU model, so
+; steady-state is ~24,004 MiB (below the 24,576 MiB physical limit).
[Qwen2.5-Coder-14B-Instruct-Q4_K_M]
model = {{ llm_models_dir }}/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf
alias = Qwen2.5-Coder-14B-Instruct-4bit
-n-gpu-layers = {{ llm_router_gpu_layers }}
+n-gpu-layers = {{ llm_router_coder_gpu_layers }}
ctx-size = {{ llm_router_coder_ctx_size }}
flash-attn = {{ llm_router_coder_flash_attn }}
cache-type-k = {{ llm_router_cache_type_k }}
@@ -124,3 +160,103 @@ cache-type-v = {{ llm_router_cache_type_v }}
batch-size = {{ llm_router_batch_size }}
ubatch-size = {{ llm_router_ubatch_size }}
parallel = {{ llm_router_parallel }}
+
+; --- Embedding model: nomic-embed-text-v1.5 ----------------------------------
+; Primary model ID: nomic-embed-text-v1.5 (section name / client-visible ID)
+; ~84MB GGUF — negligible VRAM, always-loaded. Embedding endpoint: /v1/embeddings.
+; embedding=true: required to expose /v1/embeddings and embed the model (not chat).
+; n-gpu-layers=99: GPU offload all layers (tiny model, no reason to leave on CPU).
+; ctx-size=8192: per task spec (OpenViking Phase 1b, t_34b96e83).
+; load-on-startup=true: warm at boot — embedding callers (peter-parker) are
+; latency-sensitive; no cold-load wait on first request.
+; sleep-idle-seconds=-1: never evict — ~84MB is negligible, always keep hot.
+; NO flash-attn, NO KV cache params: embedding models use a different forward
+; pass (bidirectional, no autoregressive KV cache). These keys are irrelevant
+; for embedding inference and may be silently ignored or cause warnings; omit.
+; Source: nomic-ai/nomic-embed-text-v1.5-GGUF (public, no auth needed)
+; Added 2026-08-13 (t_34b96e83) — War Machine.
+;
+; FIXED (2026-08-14, t_openviking_embed_batch): the original section omitted
+; batch-size/ubatch-size, so llama-server defaulted the PHYSICAL batch
+; (ubatch-size) to 512 tokens. For embedding requests llama.cpp cannot split
+; a single input across ubatches, so any OpenViking chunk over ~512 tokens
+; large chunk over ~512 tokens (observed 2000-3400 tokens/chunk) failed hard with "input (N tokens) is too
+; large to process. increase the physical batch size (current batch size:
+; 512)". This tripped OpenViking's circuit breaker into an infinite
+; fail/re-enqueue loop. Fix: set batch-size/ubatch-size to 4096 (comfortably
+; over the observed max chunk size and under ctx-size=8192).
+;
+; FOLLOW-UP FINDING (2026-08-14, same task): after the batch-size fix landed,
+; logs showed a SECOND, separate problem: llama.cpp capped the effective
+; context to 2048 regardless of ctx-size=8192 ("n_ctx_seq (8192) > n_ctx_train
+; (2048)" / "capping"). This is expected per the nomic-embed-text-v1.5-GGUF
+; model card: the base GGUF's native RoPE training context is 2048; the
+; original HF model reaches its benchmarked 8192-token context via Dynamic
+; NTK-Aware RoPE scaling, which llama.cpp does not implement — so llama.cpp
+; defaults to 2048 unless YaRN scaling is explicitly requested. Model card
+; prescribes: --rope-scaling yarn --rope-freq-scale 0.75 alongside -c 8192.
+; Added rope-scaling/rope-freq-scale below to actually reach 8192.
+[nomic-embed-text-v1.5]
+model = {{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf
+embedding = true
+n-gpu-layers = 99
+ctx-size = {{ llm_router_nomic_ctx_size }}
+batch-size = {{ llm_router_nomic_batch_size }}
+ubatch-size = {{ llm_router_nomic_ubatch_size }}
+rope-scaling = yarn
+rope-freq-scale = 0.75
+load-on-startup = true
+sleep-idle-seconds = -1
+
+; --- Auxiliary model: Qwen3-8B-Q4_K_M (thinking variant) ----------------------
+; GGUF: bartowski/Qwen_Qwen3-8B-GGUF (5,027,784,224 bytes)
+; Thinking mode: ON BY DEFAULT (Qwen3 baked-in template, no override).
+; - Use for complex aux tasks: long-context summarization, multi-step tool
+; planning, structured extraction requiring CoT.
+; - To suppress thinking at request time, send /no_think prefix in the prompt
+; OR route to [Qwen3-8B-Q4_K_M-no_think] section below.
+; n-gpu-layers=99: GPU-resident (~5.2GB VRAM). Cannot co-reside with
+; Qwen3.8-27B-Q4_K_M (17.8GB). LRU eviction handles swapping.
+; ctx-size=32768: 32K context, q4_0 KV cache.
+; flash-attn=true: required for Qwen3 architecture at this context size.
+; sleep-idle-seconds=60: evict after 60s idle (free VRAM for primary model).
+; Added: t_c5cef2b2 (2026-08-19, War Machine) — initial Qwen3-8B deployment.
+[Qwen3-8B-Q4_K_M]
+model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf
+n-gpu-layers = 99
+ctx-size = 32768
+flash-attn = true
+cache-type-k = {{ llm_router_cache_type_k }}
+cache-type-v = {{ llm_router_cache_type_v }}
+batch-size = {{ llm_router_batch_size }}
+ubatch-size = {{ llm_router_ubatch_size }}
+parallel = {{ llm_router_parallel }}
+sleep-idle-seconds = 60
+
+; --- Auxiliary model: Qwen3-8B-Q4_K_M (no-think variant) ----------------------
+; Secondary section serving the SAME GGUF with enable_thinking=False via an
+; overridden Jinja2 template. This gives low-latency, non-reasoning inference
+; for latency-sensitive aux tasks (intent classification, query rewriting,
+; structured extraction, tool-call construction, draft generation).
+;
+; Both sections point to the same .gguf file — llama-server spawns independent
+; child processes. VRAM cost: ~5.2GB per instance (~10.4GB total). Fits on
+; RTX 3090 24GB alongside nomic-embed (84MB).
+;
+; Thinking variant (Qwen3-8B-Q4_K_M) remains available for complex tasks that
+; benefit from CoT (long-context summarization, multi-step tool planning).
+;
+; Cannot co-reside with Qwen3.8-27B-Q4_K_M (17.8GB); LRU eviction applies.
+; Added: t_664289a0 (2026-08-19, War Machine) — dual thinking deployment.
+[Qwen3-8B-Q4_K_M-no_think]
+model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf
+n-gpu-layers = 99
+ctx-size = 32768
+flash-attn = true
+cache-type-k = {{ llm_router_cache_type_k }}
+cache-type-v = {{ llm_router_cache_type_v }}
+batch-size = {{ llm_router_batch_size }}
+ubatch-size = {{ llm_router_ubatch_size }}
+parallel = {{ llm_router_parallel }}
+chat-template-file = {{ llm_models_dir }}/templates/qwen3-no-think.jinja
+sleep-idle-seconds = 60
diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2
new file mode 100644
index 0000000..e28a6c0
--- /dev/null
+++ b/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2
@@ -0,0 +1,60 @@
+{#
+ FILE: roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2
+ DESCRIPTION: llama-swap v250 configuration template.
+ Generates /etc/llama-swap/config.yaml with all models, routing matrix,
+ and per-model settings (ctx_size, n_gpu_layers, cmd args).
+
+ v250 SYNTAX NOTES:
+ - Uses routing.router DSL with expression-based matrix (not old list-of-arrays)
+ - Each model has its own cmd field with full per-model args
+ - Matrix rows use "model1 & model2" syntax for co-resident sets
+ - sleep_idle_seconds: -1 = never idle; 0+ = idle after N seconds
+ - load_on_startup: true = start this model on service startup
+
+ Reference: /etc/llama-swap/config.yaml on astro-orbiter (Phase 1 artifact)
+#}
+# llama-swap configuration for astro-orbiter
+# Generated by Ansible roles/llm-inference-multimodel on {{ ansible_date_time.iso8601 }}
+# See: https://github.com/mostlygeek/llama-swap (v250 release notes for syntax)
+
+# ============================================================================
+# LISTEN — Address and port for the llama-swap proxy
+# ============================================================================
+listen: "{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}"
+
+# ============================================================================
+# MODELS — All model definitions (cmd, port, ctx_size, etc.)
+# ============================================================================
+models:
+{% for model in llm_swapmode_models %}
+ {{ model.id }}:
+ cmd: >
+ llama-server
+ --port ${PORT}
+ --model {{ model.gguf_path }}
+ --n-gpu-layers {{ model.n_gpu_layers }}
+ --ctx-size {{ model.ctx_size }}
+ --batch-size {{ model.batch_size }}
+ --ubatch-size {{ model.ubatch_size }}
+ --parallel {{ model.parallel }}
+ {% if model.cache_type is defined %}--cache-type-k {{ model.cache_type }} --cache-type-v {{ model.cache_type }}{% endif %}
+ {% if model.flash_attn is defined %}--flash-attn {{ model.flash_attn }}{% endif %}
+ {% if model.chat_template_file is defined %}--chat-template-file {{ model.chat_template_file }}{% endif %}
+ {% if model.sleep_idle_seconds is defined %}--sleep-idle-seconds {{ model.sleep_idle_seconds }}{% endif %}
+ {% if model.load_on_startup is defined and model.load_on_startup %}--load-on-startup{% endif %}
+ --host 127.0.0.1
+ port: {{ model.port }}
+{% endfor %}
+
+# ============================================================================
+# ROUTING — Matrix-based hot-swap policy (v250 expression DSL)
+# ============================================================================
+routing:
+ router:
+ use: matrix
+ settings:
+ matrix:
+ sets:
+{% for row in llm_swapmode_matrix_rows %}
+ {{ row.row }}: "{{ row.expr }}"
+{% endfor %}
diff --git a/ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 b/ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2
new file mode 100644
index 0000000..0206423
--- /dev/null
+++ b/ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2
@@ -0,0 +1,103 @@
+{#
+ FILE: roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2
+ DESCRIPTION: Qwen3 chat template with enable_thinking unconditionally false.
+
+ This is a chat-template FILE deployed to {{ llm_models_dir }}/templates/qwen3-no-think.jinja
+ on astro-orbiter and referenced via chat-template-file in the INI preset for
+ [Qwen3-8B-Q4_K_M-no_think]. The [Qwen3-8B-Q4_K_M] section uses the model's
+ baked-in default template (thinking ON by default).
+
+ Mechanism: at the add_generation_prompt step, instead of checking
+ "enable_thinking is defined and enable_thinking is false"
+ we UNCONDITIONALLY emit the empty prefix that suppresses CoT.
+ This is the hard-switch documented in the Qwen3 template spec and confirmed in
+ https://huggingface.co/blog/qwen-3-chat-template-deep-dive (section 1).
+
+ All other logic is identical to /opt/llama.cpp/models/templates/Qwen-Qwen3-0.6B.jinja
+ (the shipped template for Qwen3). Only the final add_generation_prompt block differs.
+
+ Added: t_664289a0 (2026-08-19, War Machine) — dual thinking deployment.
+#}
+{%- if tools %}
+ {{- '<|im_start|>system\n' }}
+ {%- if messages[0].role == 'system' %}
+ {{- messages[0].content + '\n\n' }}
+ {%- endif %}
+ {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }}
+ {%- for tool in tools %}
+ {{- "\n" }}
+ {{- tool | tojson }}
+ {%- endfor %}
+ {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }}
+{%- else %}
+ {%- if messages[0].role == 'system' %}
+ {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+ {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+ {%- set index = (messages|length - 1) - loop.index0 %}
+ {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %}
+ {%- set ns.multi_step_tool = false %}
+ {%- set ns.last_query_index = index %}
+ {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+ {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+ {%- elif message.role == "assistant" %}
+ {%- set content = message.content %}
+ {%- set reasoning_content = '' %}
+ {%- if message.reasoning_content is defined and message.reasoning_content is not none %}
+ {%- set reasoning_content = message.reasoning_content %}
+ {%- else %}
+ {%- if '' in message.content %}
+ {%- set content = message.content.split('')[-1].lstrip('\n') %}
+ {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %}
+ {%- endif %}
+ {%- endif %}
+ {%- if loop.index0 > ns.last_query_index %}
+ {%- if loop.last or (not loop.last and reasoning_content) %}
+ {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- else %}
+ {{- '<|im_start|>' + message.role + '\n' + content }}
+ {%- endif %}
+ {%- if message.tool_calls %}
+ {%- for tool_call in message.tool_calls %}
+ {%- if (loop.first and content) or (not loop.first) %}
+ {{- '\n' }}
+ {%- endif %}
+ {%- if tool_call.function %}
+ {%- set tool_call = tool_call.function %}
+ {%- endif %}
+ {{- '\n{"name": "' }}
+ {{- tool_call.name }}
+ {{- '", "arguments": ' }}
+ {%- if tool_call.arguments is string %}
+ {{- tool_call.arguments }}
+ {%- else %}
+ {{- tool_call.arguments | tojson }}
+ {%- endif %}
+ {{- '}\n' }}
+ {%- endfor %}
+ {%- endif %}
+ {{- '<|im_end|>\n' }}
+ {%- elif message.role == "tool" %}
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+ {{- '<|im_start|>user' }}
+ {%- endif %}
+ {{- '\n\n' }}
+ {{- message.content }}
+ {{- '\n' }}
+ {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+ {{- '<|im_end|>\n' }}
+ {%- endif %}
+ {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+ {{- '<|im_start|>assistant\n' }}
+ {{- '\n\n\n\n' }}
+{%- endif %}