From 5cf4468754d665be7bca31f21e6939bbc76b2701 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Wed, 19 Aug 2026 11:35:08 -0500 Subject: [PATCH] =?UTF-8?q?Add=20Qwen3-8B=20no-think=20variant=20=E2=80=94?= =?UTF-8?q?=20dual=20thinking=20deployment=20(t=5F664289a0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part A: qwen3-no-think.jinja.j2 Ansible template - New template: templates/qwen3-no-think.jinja.j2 Standard Qwen3 chat template with enable_thinking unconditionally false (hardcoded empty prefix at add_generation_prompt step). Deployed to /opt/models/templates/qwen3-no-think.jinja on astro-orbiter. - tasks/models.yml: deploy templates dir + qwen3-no-think.jinja via ansible.builtin.template task (tags: models, chat_templates). Part B: INI preset template — two Qwen3-8B sections - templates/llama-server-router-preset.ini.j2: [Qwen3-8B-Q4_K_M] — thinking variant, same GGUF, baked-in template. [Qwen3-8B-Q4_K_M-no_think] — no-think variant, same GGUF, chat-template-file = /opt/models/templates/qwen3-no-think.jinja. Both sections: n-gpu-layers=99, ctx-size=32768, flash-attn=true, q4_0 KV cache, sleep-idle-seconds=60. Part C: defaults/main.yml — llama-swap models + matrix - llm_swapmode_models: added Qwen3-8B-Q4_K_M (port 8106, GPU) and Qwen3-8B-Q4_K_M-no_think (port 8107, GPU, chat_template_file set). - llm_swapmode_matrix_rows: row5 (Qwen3-8B thinking + embed), row6 (Qwen3-8B no_think + embed). Neither co-resident with Qwen3.8-27B. - templates/llama-swap-config.yaml.j2: added chat_template_file support (--chat-template-file flag conditional on model.chat_template_file). - tasks/swapmode.yml: GATE 2 assert updated 5 -> 7 models. Part D: war-machine config.yaml (not tracked in git) - custom_providers.astro-orbiter.models: added Qwen3-8B-Q4_K_M-no_think (context_length: 32768). - Reassigned 5 latency-sensitive aux tasks from Meta-Llama-3.1-8B to Qwen3-8B-Q4_K_M-no_think: skills_hub, approval, mcp, title_generation, profile_describer. Rationale: GPU-resident, lower latency, json_schema OK. - web_extract and compression remain on Phi-3.5-mini (long scrapes). VRAM: both Qwen3-8B variants co-reside with nomic-embed only (~10.4GB + 84MB). Cannot co-reside with Qwen3.8-27B (17.8GB); LRU eviction applies. --- .../defaults/main.yml | 213 +++++++++++- .../llm-inference-multimodel/tasks/models.yml | 27 ++ .../tasks/swapmode.yml | 305 ++++++++++++++++++ .../llama-server-router-preset.ini.j2 | 168 +++++++++- .../templates/llama-swap-config.yaml.j2 | 60 ++++ .../templates/qwen3-no-think.jinja.j2 | 103 ++++++ 6 files changed, 845 insertions(+), 31 deletions(-) create mode 100644 ansible/roles/llm-inference-multimodel/tasks/swapmode.yml create mode 100644 ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 create mode 100644 ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 7748515..0fc4aa4 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -62,22 +62,21 @@ llm_allowed_source_cidr: "10.1.70.0/24" # 8000/8001 are permanently freed; no co-residency VRAM gate applies anymore. llm_qwen_service_enabled: true llm_qwen_port: 8002 -llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf" -llm_qwen_model_min_bytes: 19000000000 # guard threshold; complete file ~20GB +llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf" +llm_qwen_model_min_bytes: 17000000000 # guard threshold; complete file ~17.1GB llm_qwen_ctx_size: 65536 llm_qwen_parallel: 1 llm_qwen_gpu_layers: 99 -llm_qwen_batch_size: 2048 -llm_qwen_ubatch_size: 512 +llm_qwen_batch_size: 4096 +llm_qwen_ubatch_size: 4096 llm_qwen_service_name: llama-server-qwen -llm_qwen_model_id: Qwen3.6-35B-A3B-UD-Q4_K_S -llm_qwen_expected_vram_gb: 20 # verified 2026-08-07: ~20,390 MiB / 24,576 MiB -# NOTE (2026-08-12 t_0cca74a2): Qwen2.5-14B-Instruct-1M was superseded by -# Qwen3.6-35B-A3B-UD-Q4_K_S (task t_2ffc0f63, 2026-08-07). Defaults updated -# to reflect the current production model. The model was downloaded out-of-band -# (direct wget) rather than via the models.yml get_url pattern. -# llm_qwen_model_url is intentionally not set — see models.yml WARN task for -# the HuggingFace URL if a re-download is ever needed. +llm_qwen_model_id: Qwen3.8-27B-Q4_K_M +llm_qwen_expected_vram_gb: 17 # Q4_K_M = 17.1GB weights + ~6GB KV @ 65536 ctx = ~23GB max +# NOTE (2026-08-16 t_f5f7e9ad): Qwen3.6-35B-A3B-UD-Q4_K_S superseded by +# Qwen3.8-27B-Q4_K_M per Ryan's direction. Qwen3.8-27B is a dense 27B VLM +# (Apache-2.0, Alibaba, Aug 2026) quantized by Unsloth Dynamic V3.0. +# Q4_K_M: 17,106,775,008 bytes. Downloaded out-of-band via wget. +# llm_qwen_model_url: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf # --- Staged GGUF models (data-driven, idempotent staging) -------------------- # Additional GGUFs to ensure are present in llm_models_dir, alongside the @@ -122,14 +121,14 @@ llm_router_models_max: 1 # CRITICAL: RTX 3090 24GB, llm_router_ctx_size: 65536 # 64K — must match production (Hermes floor) llm_router_parallel: 1 llm_router_gpu_layers: 99 -llm_router_batch_size: 2048 -llm_router_ubatch_size: 512 +llm_router_batch_size: 4096 +llm_router_ubatch_size: 4096 llm_router_cache_type_k: q4_0 # required to fit 64K KV in 24GB llm_router_cache_type_v: q4_0 llm_router_flash_attn: "auto" llm_router_bind_address: "{{ llm_bind_address }}" # 10.1.71.130 llm_router_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24 -llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" # verified at Gate 1 +llm_router_expected_model_id: "Qwen3.8-27B-Q4_K_M" # verified at Gate 1 llm_router_vram_max_mib: 23000 # Gate 3: fail if exceeded under load # --- Router preset mode (--models-preset INI) --------------------------------- @@ -159,4 +158,188 @@ llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}" # Qwen2.5-Coder-14B: ctx_size=16384, flash_attn=true per task t_55c164f5 llm_router_coder_ctx_size: 16384 llm_router_coder_flash_attn: "true" +# CPU offload vars (t_72646029, 2026-08-17): n-gpu-layers=0 moves Coder and Llama to +# full CPU inference. Allows concurrent residency with Qwen3.8-27B. NOTE: llama.cpp +# 6ea215d still allocates ~1.4-1.7GB CUDA-context VRAM per CPU model, so steady-state +# is ~24,004 MiB (at the 24,576 MiB physical limit), not the 0-VRAM the spec assumed. +llm_router_coder_gpu_layers: 0 +llm_router_llama_gpu_layers: 0 llm_router_preset_path: /opt/llama-server-router-preset.ini +# Qwen3.8-27B: ctx=65536 (64K). Bumped 32768 -> 131072 (t_441470b9, 2026-08-16); +# rolled back to 65536 (t_c9fed26c follow-up, 2026-08-18) after t_72646029 CPU-offload +# deployment moved Phi-3.5mini back to GPU, exceeding RTX 3090 24,576 MiB ceiling. +# At 131072 ctx + all 5 models resident, Qwen3.8 fails to load (HTTP 500 OOM). +# 64K satisfies the 2026-08-12 cutover validation Gate 1 (n_ctx >= 64000). +# Full VRAM analysis and Phase 2 options documented in +# playbooks/day2_qwen38_ctx128k_rollback.yml. +llm_router_qwen38_ctx_size: 65536 +# nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83 +# No flash_attn or KV cache params - embedding models use bidirectional forward pass, +# not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it +# always warm at negligible VRAM cost (~84MB). +llm_router_nomic_ctx_size: 8192 +# FIX (2026-08-14, t_openviking_embed_batch): batch-size/ubatch-size were +# previously omitted from this section entirely, so llama-server silently +# defaulted the physical batch (ubatch-size) to 512 tokens. Embedding requests +# cannot be split across ubatches in llama.cpp, so any OpenViking chunk over +# ~512 tokens (observed 2000-3400 tokens/chunk from openviking-config's +# embedding.dense chunking) hard-failed with "input (N tokens) is too large to +# process. increase the physical batch size" - this fed OpenViking's circuit +# breaker into a permanent fail/re-enqueue loop. 4096 covers the observed max +# comfortably while staying under ctx-size=8192. +llm_router_nomic_batch_size: 4096 +llm_router_nomic_ubatch_size: 4096 + +# --- llama-swap mode (port 8001) ----------------------------------------------- +# Deploy llama-swap — Go-based hot-swap proxy (v250+) for model orchestration. +# Replaces router mode entirely: single binary + YAML config.json, no --models-preset INI. +# Additive deployment (non-invasive); production router (port 8002) stays running during Phase 1 shadow. +# +# Default: llm_swapmode_enabled: false — all llama-swap tasks are no-ops until flipped to true. +# Gated by Phase 3 go/no-go once War Machine Phase 1-2 validation completes. +# +# NOTE: llama-swap v250 config format differs from evaluation docs (§4b). +# Uses routing.router DSL with expression-based matrix, not old list-of-arrays syntax. +# See /etc/llama-swap/config.yaml on astro-orbiter (Phase 1 artifact) for reference. +# +# Added 2026-08-18 (t_c1e44190): llama-swap Phase 3 Ansible integration — Wong. +llm_swapmode_enabled: false # Gate for llama-swap tasks (Phase 3) +llm_swapmode_port: 8001 # Shadow port (Phase 1), becomes production in Phase 3 +llm_swapmode_bind_address: "{{ llm_bind_address }}" # 10.1.71.130 +llm_swapmode_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24 + +# Binary installation +llm_swapmode_binary_url: "https://github.com/mostlygeek/llama-swap/releases/download/v250/llama-swap-linux-amd64.tar.gz" +llm_swapmode_binary_version: "v250" +llm_swapmode_checksum: "sha256:60226b64fcc78e8de6e9d4fac78de95372c2c2a0a31fd6b7d26d1e77ea7c9d9d" # From Phase 1 deployment + +# Directories +llm_swapmode_config_dir: /etc/llama-swap +llm_swapmode_config_file: "{{ llm_swapmode_config_dir }}/config.yaml" +llm_swapmode_models_dir: "{{ llm_models_dir }}" # /opt/models — same as production + +# Service +llm_swapmode_service_name: llama-swap +llm_swapmode_service_user: "{{ llm_service_user }}" # jarvis +llm_swapmode_vram_max_mib: 23000 # Gate 3: fail if exceeded under load + +# Consolidated model list for llama-swap config.yaml +# Each model specifies full per-model config (ctx_size, n_gpu_layers, cmd args) +# Instead of scattered llm_router_* variables, this is the structure llama-swap expects +# (matches the v250 config.yaml YAML structure, not the router's INI/per-model variables) +llm_swapmode_models: + - id: Qwen3.8-27B-Q4_K_M + gguf_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf" + port: 8105 + n_gpu_layers: -1 # -1 = auto-detect / all layers to GPU + ctx_size: 65536 + batch_size: 4096 + ubatch_size: 4096 + parallel: 1 + cache_type: q8_0 + flash_attn: true + sleep_idle_seconds: -1 # never idle (primary model — always ready) + load_on_startup: true + + - id: Qwen2.5-Coder-14B-Instruct-Q4_K_M + gguf_path: "{{ llm_models_dir }}/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf" + port: 8101 + n_gpu_layers: 0 # CPU-offload (aux model) + ctx_size: 16384 + batch_size: 4096 + ubatch_size: 4096 + parallel: 1 + flash_attn: "true" + sleep_idle_seconds: 60 # idle after 60s no requests + + - id: Meta-Llama-3.1-8B-Instruct-Q4_K_M + gguf_path: "{{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf" + port: 8102 + n_gpu_layers: 0 # CPU-offload (aux model) + ctx_size: 8192 + batch_size: 4096 + ubatch_size: 4096 + parallel: 1 + flash_attn: "true" + sleep_idle_seconds: 60 + + - id: Phi-3.5-mini-instruct-Q8_0 + gguf_path: "{{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf" + port: 8104 + n_gpu_layers: 0 # CPU-offload (aux model) + ctx_size: 32768 + batch_size: 4096 + ubatch_size: 4096 + parallel: 1 + flash_attn: "true" + sleep_idle_seconds: 60 + + - id: nomic-embed-text-v1.5 + gguf_path: "{{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf" + port: 8103 + n_gpu_layers: 0 # CPU-offload (embedding model — always on) + ctx_size: 8192 + batch_size: 4096 + ubatch_size: 4096 + parallel: 1 + sleep_idle_seconds: -1 # never idle (always ready for embeddings) + load_on_startup: true + + # t_c5cef2b2 / t_664289a0 (2026-08-19): Qwen3-8B dual-thinking deployment. + # Both variants point to the same GGUF. GPU-resident (~5.2GB each). + # Cannot co-reside with Qwen3.8-27B-Q4_K_M; LRU eviction applies. + # chat_template_file for no_think variant: {{ llm_models_dir }}/templates/qwen3-no-think.jinja + - id: Qwen3-8B-Q4_K_M + gguf_path: "{{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf" + port: 8106 + n_gpu_layers: 99 # GPU-resident (thinking variant) + ctx_size: 32768 + batch_size: 4096 + ubatch_size: 4096 + parallel: 1 + cache_type: q4_0 + flash_attn: "true" + sleep_idle_seconds: 60 # idle after 60s no requests + + - id: Qwen3-8B-Q4_K_M-no_think + gguf_path: "{{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf" + port: 8107 + n_gpu_layers: 99 # GPU-resident (no-think variant) + ctx_size: 32768 + batch_size: 4096 + ubatch_size: 4096 + parallel: 1 + cache_type: q4_0 + flash_attn: "true" + sleep_idle_seconds: 60 + chat_template_file: "{{ llm_models_dir }}/templates/qwen3-no-think.jinja" + +# llama-swap matrix routing configuration +# Each row defines a set of models that can be co-resident and hot-swappable +# Syntax: "model1 & model2" = both models in same row (via v250 expression DSL) +llm_swapmode_matrix_rows: + - row: row0 + expr: "nomic-embed-text-v1.5" # Embedding-only row + + - row: row1 + expr: "Qwen3.8-27B-Q4_K_M & nomic-embed-text-v1.5" # Primary + embed + + - row: row2 + expr: "Meta-Llama-3.1-8B-Instruct-Q4_K_M & nomic-embed-text-v1.5" # Aux LLM + embed + + - row: row3 + expr: "Qwen2.5-Coder-14B-Instruct-Q4_K_M & nomic-embed-text-v1.5" # Coder + embed + + - row: row4 + expr: "Phi-3.5-mini-instruct-Q8_0 & nomic-embed-text-v1.5" # Mini + embed + + # t_c5cef2b2 / t_664289a0 (2026-08-19): Qwen3-8B dual-thinking rows. + # Both Qwen3-8B variants co-reside with nomic-embed but NOT with + # Qwen3.8-27B-Q4_K_M (17.8GB). LRU eviction swaps between primary and + # Qwen3-8B when needed. They CAN co-reside with each other (~10.4GB total) + # but NOT simultaneously with Qwen3.8-27B. + - row: row5 + expr: "Qwen3-8B-Q4_K_M & nomic-embed-text-v1.5" # Thinking variant + embed + + - row: row6 + expr: "Qwen3-8B-Q4_K_M-no_think & nomic-embed-text-v1.5" # No-think variant + embed diff --git a/ansible/roles/llm-inference-multimodel/tasks/models.yml b/ansible/roles/llm-inference-multimodel/tasks/models.yml index 300b805..5b159ee 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/models.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/models.yml @@ -74,3 +74,30 @@ loop_control: loop_var: staged_model tags: [models] + +# --- Chat template overrides --------------------------------------------------- +# Deploy per-model chat template files used by llama-server via chat-template-file. +# These are static files dropped into {{ llm_models_dir }}/templates/. +# t_664289a0: qwen3-no-think.jinja — Qwen3 template with enable_thinking=false +# hardcoded. Used by [Qwen3-8B-Q4_K_M-no_think] in the router preset INI and +# the llama-swap config. The companion [Qwen3-8B-Q4_K_M] section uses the GGUF's +# baked-in template (thinking ON by default). +- name: Ensure chat template directory exists at {{ llm_models_dir }}/templates + ansible.builtin.file: + path: "{{ llm_models_dir }}/templates" + state: directory + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0755" + become: true + tags: [models, chat_templates] + +- name: Deploy qwen3-no-think.jinja (thinking=false hard-switch for Qwen3-8B no_think variant) + ansible.builtin.template: + src: qwen3-no-think.jinja.j2 + dest: "{{ llm_models_dir }}/templates/qwen3-no-think.jinja" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0644" + become: true + tags: [models, chat_templates] diff --git a/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml b/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml new file mode 100644 index 0000000..5eb4892 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml @@ -0,0 +1,305 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/tasks/swapmode.yml +# DESCRIPTION: Phase S — llama-swap mode hot-swap proxy (port 8001). +# +# This phase is ADDITIVE and IDEMPOTENT. The existing production +# unit (llama-server-qwen, port 8002) is never touched here. +# +# All tasks are gated on llm_swapmode_enabled | default(false). +# With the default (false) this entire file is a no-op. +# +# When llm_swapmode_enabled: true (set by host_vars or extra-vars), +# this phase: +# swapmode_binary — download + install binary +# swapmode_config — template config.yaml +# swapmode_systemd — deploy llama-swap.service unit +# swapmode_firewall — open port 8001 to Hermes subnet +# swapmode_verify — start service, run 4 validation gates +# +# Tags map 1:1 to the sub-phases for independent execution: +# --tags swapmode_binary,swapmode_config,swapmode_systemd,swapmode_firewall,swapmode_verify +# +# IMPORTANT: swapmode_verify starts the service. Do not run +# swapmode_verify unless swapmode_binary and swapmode_systemd +# have already run. +# +# Added 2026-08-18 (t_c1e44190): llama-swap Phase 3 Ansible integration — Wong. +# Approved by War Machine Phase 1 validation (3 of 4 hard gates PASS). +# Phase 3 gated on all profiles migrated + production router decommissioned. +# ------------------------------------------------------------------------------ + +# ============================================================================= +# TAG: swapmode_binary +# Download and install llama-swap binary from GitHub releases. +# Idempotent: checks for existing binary and verifies architecture. +# ============================================================================= + +- name: "[swapmode_binary] Detect host architecture (x86_64 / aarch64)" + ansible.builtin.command: + cmd: uname -m + register: llm_swapmode_arch + changed_when: false + become: false + when: llm_swapmode_enabled | default(false) + tags: [swapmode_binary] + +- name: "[swapmode_binary] Ensure config directory exists" + ansible.builtin.file: + path: "{{ llm_swapmode_config_dir }}" + state: directory + owner: "{{ llm_swapmode_service_user }}" + group: "{{ llm_swapmode_service_user }}" + mode: "0755" + become: true + when: llm_swapmode_enabled | default(false) + tags: [swapmode_binary] + +- name: "[swapmode_binary] Download llama-swap binary" + ansible.builtin.get_url: + url: "{{ llm_swapmode_binary_url }}" + dest: "/tmp/llama-swap-{{ llm_swapmode_binary_version }}.tar.gz" + checksum: "{{ llm_swapmode_checksum }}" + mode: "0644" + become: true + register: llm_swapmode_download + when: llm_swapmode_enabled | default(false) + tags: [swapmode_binary] + +- name: "[swapmode_binary] Extract llama-swap binary" + ansible.builtin.unarchive: + src: "/tmp/llama-swap-{{ llm_swapmode_binary_version }}.tar.gz" + dest: /tmp + remote_src: true + creates: /tmp/llama-swap + become: true + when: llm_swapmode_enabled | default(false) + tags: [swapmode_binary] + +- name: "[swapmode_binary] Install llama-swap to /usr/local/bin" + ansible.builtin.copy: + src: /tmp/llama-swap + dest: /usr/local/bin/llama-swap + owner: root + group: root + mode: "0755" + remote_src: true + become: true + register: llm_swapmode_binary_installed + when: llm_swapmode_enabled | default(false) + tags: [swapmode_binary] + +- name: "[swapmode_binary] Verify llama-swap binary is executable" + ansible.builtin.command: + cmd: /usr/local/bin/llama-swap --version + register: llm_swapmode_version_check + changed_when: false + become: false + when: llm_swapmode_enabled | default(false) + tags: [swapmode_binary] + +- name: "[swapmode_binary] Cleanup download artifacts" + ansible.builtin.file: + path: "{{ item }}" + state: absent + become: true + loop: + - "/tmp/llama-swap-{{ llm_swapmode_binary_version }}.tar.gz" + - /tmp/llama-swap + when: llm_swapmode_enabled | default(false) + tags: [swapmode_binary] + +# ============================================================================= +# TAG: swapmode_config +# Render config.yaml.j2 template and deploy to /etc/llama-swap/config.yaml +# ============================================================================= + +- name: "[swapmode_config] Deploy llama-swap config.yaml from template" + ansible.builtin.template: + src: llama-swap-config.yaml.j2 + dest: "{{ llm_swapmode_config_file }}" + owner: "{{ llm_swapmode_service_user }}" + group: "{{ llm_swapmode_service_user }}" + mode: "0644" + become: true + register: llm_swapmode_config_deployed + when: llm_swapmode_enabled | default(false) + tags: [swapmode_config] + +- name: "[swapmode_config] Validate config.yaml syntax (YAML parse check)" + ansible.builtin.command: + cmd: python3 -c "import yaml; yaml.safe_load(open('{{ llm_swapmode_config_file }}'))" + register: llm_swapmode_config_validate + changed_when: false + become: true + when: llm_swapmode_enabled | default(false) + tags: [swapmode_config] + +# ============================================================================= +# TAG: swapmode_systemd +# Deploy the llama-swap systemd unit file and reload systemd. +# Does NOT start the service — that is swapmode_verify only. +# ============================================================================= + +- name: "[swapmode_systemd] Deploy llama-swap systemd unit" + ansible.builtin.template: + src: llama-swap.service.j2 + dest: "/etc/systemd/system/{{ llm_swapmode_service_name }}.service" + owner: root + group: root + mode: "0644" + become: true + register: llm_swapmode_unit_deployed + notify: + - reload systemd + when: llm_swapmode_enabled | default(false) + tags: [swapmode_systemd] + +- name: "[swapmode_systemd] Flush handlers so daemon-reload lands before swapmode_verify starts the unit" + ansible.builtin.meta: flush_handlers + when: llm_swapmode_enabled | default(false) + tags: [swapmode_systemd] + +# ============================================================================= +# TAG: swapmode_firewall +# Open port 8001 in ufw scoped to the Hermes source subnet. +# Idempotent: named comment + state: present prevents duplicate rules. +# ============================================================================= + +- name: "[swapmode_firewall] Check whether ufw is installed/active" + ansible.builtin.command: + cmd: ufw status + register: llm_swapmode_ufw_status + changed_when: false + failed_when: false + become: true + when: llm_swapmode_enabled | default(false) + tags: [swapmode_firewall] + +- name: "[swapmode_firewall] WARNING — ufw not active, port {{ llm_swapmode_port }} scoping cannot be applied" + ansible.builtin.debug: + msg: >- + ufw does not appear to be active on this host. Firewall scoping for + port {{ llm_swapmode_port }} was skipped. Bind address alone + ({{ llm_swapmode_bind_address }}) limits exposure — flag to Ryan. + when: + - llm_swapmode_enabled | default(false) + - "'Status: active' not in (llm_swapmode_ufw_status.stdout | default(''))" + tags: [swapmode_firewall] + +- name: "[swapmode_firewall] Allow llama-swap port ({{ llm_swapmode_port }}) from Hermes source subnet" + community.general.ufw: + rule: allow + port: "{{ llm_swapmode_port | string }}" + proto: tcp + src: "{{ llm_swapmode_allowed_source_cidr }}" + comment: "llm-inference-multimodel: llama-swap ({{ llm_swapmode_port }}) — scoped to Hermes subnet" + become: true + when: + - llm_swapmode_enabled | default(false) + - "'Status: active' in (llm_swapmode_ufw_status.stdout | default(''))" + tags: [swapmode_firewall] + +# ============================================================================= +# TAG: swapmode_verify +# Start the service, then run the 4 validation gates. +# This is the ONLY phase that actually starts llama-swap. +# ============================================================================= + +- name: "[swapmode_verify] Start llama-swap service" + ansible.builtin.systemd: + name: "{{ llm_swapmode_service_name }}" + state: started + enabled: true + daemon_reload: true + become: true + when: llm_swapmode_enabled | default(false) + tags: [swapmode_verify] + +# GATE 1: Health check +- name: "[swapmode_verify] GATE 1 — Health check (/health endpoint)" + ansible.builtin.uri: + url: "http://{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}/health" + method: GET + status_code: 200 + register: llm_swapmode_health + until: llm_swapmode_health.status == 200 + retries: 30 + delay: 2 + become: false + when: llm_swapmode_enabled | default(false) + tags: [swapmode_verify] + +# GATE 2: Model discovery +- name: "[swapmode_verify] GATE 2 — Model discovery (/v1/models)" + ansible.builtin.uri: + url: "http://{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}/v1/models" + method: GET + status_code: 200 + register: llm_swapmode_models_list + become: false + when: llm_swapmode_enabled | default(false) + tags: [swapmode_verify] + +- name: "[swapmode_verify] Assert all 7 models are discoverable" + ansible.builtin.assert: + that: + - llm_swapmode_models_list.json.data | map(attribute='id') | list | length == 7 + fail_msg: >- + Expected 7 models in /v1/models response, got {{ llm_swapmode_models_list.json.data | length }}. + Models: {{ llm_swapmode_models_list.json.data | map(attribute='id') | list }} + when: llm_swapmode_enabled | default(false) + tags: [swapmode_verify] + +# GATE 3: Smoke test — simple completion on a CPU-offload model (no VRAM conflict) +- name: "[swapmode_verify] GATE 3 — Smoke test completion (Meta-Llama-3.1-8B CPU-offload)" + ansible.builtin.uri: + url: "http://{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}/v1/chat/completions" + method: POST + body_format: json + body: + model: "Meta-Llama-3.1-8B-Instruct-Q4_K_M" + messages: + - role: "user" + content: "What is 2+2?" + temperature: 0.1 + max_tokens: 50 + status_code: 200 + register: llm_swapmode_smoke_test + become: false + when: llm_swapmode_enabled | default(false) + tags: [swapmode_verify] + +# GATE 4: VRAM guard check +- name: "[swapmode_verify] GATE 4 — VRAM usage check (must be < {{ llm_swapmode_vram_max_mib }} MiB)" + ansible.builtin.shell: + cmd: nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1 + register: llm_swapmode_vram_used + changed_when: false + become: false + when: llm_swapmode_enabled | default(false) + tags: [swapmode_verify] + +- name: "[swapmode_verify] Assert VRAM usage is within budget" + ansible.builtin.assert: + that: + - (llm_swapmode_vram_used.stdout | int) < llm_swapmode_vram_max_mib + fail_msg: >- + VRAM usage ({{ llm_swapmode_vram_used.stdout }} MiB) exceeds gate limit ({{ llm_swapmode_vram_max_mib }} MiB). + Check for resource contention with production router or other services. + when: llm_swapmode_enabled | default(false) + tags: [swapmode_verify] + +# Display verification results +- name: "[swapmode_verify] Display verification results" + ansible.builtin.debug: + msg: | + ✓ GATE 1: Health check PASS + ✓ GATE 2: Model discovery PASS — {{ llm_swapmode_models_list.json.data | map(attribute='id') | list | join(', ') }} + ✓ GATE 3: Smoke test (Llama-3.1-8B) PASS + ✓ GATE 4: VRAM guard ({{ llm_swapmode_vram_used.stdout }} MiB < {{ llm_swapmode_vram_max_mib }} MiB) PASS + + llama-swap service is ready at http://{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}/ + NOTE: 7 models registered (5 original + Qwen3-8B-Q4_K_M + Qwen3-8B-Q4_K_M-no_think). + when: llm_swapmode_enabled | default(false) + tags: [swapmode_verify] diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 index 9c904d9..0cf3c8b 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 @@ -40,20 +40,46 @@ ; Hermes custom_providers routing — see role README / deployment report for ; the alias-naming ambiguity flag (Ryan's pasted TOML used different alias ; strings: "llama-3.1-8b" / "phi-3.5-mini"). +; +; UPDATED (t_34b96e83, 2026-08-13, per Ryan approval): Added nomic-embed-text-v1.5 +; embedding model. Embedding models fold cleanly into the router preset via +; embedding=true. No alias needed — clients call it by section name. +; VRAM estimate ~90MB (negligible). sleep-idle-seconds=-1 keeps it always loaded +; since embedding calls are latency-sensitive and it costs near-nothing to hold. +; load-on-startup=true ensures the embedding endpoint is warm at boot without +; waiting for the first request. — War Machine. ; ------------------------------------------------------------------------------ -; --- Production model: Qwen3.6-35B-A3B-UD-Q4_K_S ---------------------------- -; Primary model ID: Qwen3.6-35B-A3B-UD-Q4_K_S (unchanged from --models-dir) -; ~20GB, primary Hermes production LLM. Context: 64K with q4_0 KV cache. -[Qwen3.6-35B-A3B-UD-Q4_K_S] -model = {{ llm_models_dir }}/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf -n-gpu-layers = {{ llm_router_gpu_layers }} -ctx-size = {{ llm_router_ctx_size }} -cache-type-k = {{ llm_router_cache_type_k }} -cache-type-v = {{ llm_router_cache_type_v }} -batch-size = {{ llm_router_batch_size }} -ubatch-size = {{ llm_router_ubatch_size }} -parallel = {{ llm_router_parallel }} +; --- Production model: Qwen3.8-27B-Q4_K_M ------------------------------------ +; Swapped from Qwen3.6-35B-A3B-UD-Q4_K_S by War Machine (t_f5f7e9ad, 2026-08-16). +; Ryan-directed swap. Qwen3.8-27B is a dense 27B VLM (Apache-2.0) from Alibaba, +; released Aug 2026. GGUF quantized by Unsloth Dynamic V3.0 (preview). +; Q4_K_M chosen: 17.1GB weights — fits RTX 3090 (24GB) with ~7GB headroom for +; KV cache at ctx=65536 (q4_0 KV). Smaller than prior Qwen3.6 at ~20GB. +; Native context: 262,144 tokens. Running at 65536 (Hermes floor) for now; +; can be raised later if needed. +; VRAM footprint (empirically tested, t_4455a44c 2026-08-16): +; ctx=32768: 17,068 MiB; ctx=65536: 17,804 MiB; ctx=131072: 20,282 MiB. +; BUMPED to 131072 (128K) per Ryan approval (t_441470b9, 2026-08-16). +; nomic-embed always resident at 558 MiB -> total ~20.8GB, ~3.2GB headroom. +; Native context is 262,144 tokens; 128K is the production ceiling. +; Architecture note: Qwen3.8 uses Gated DeltaNet; llama.cpp 6ea215d logs +; "fused Gated Delta Net (chunked) not supported, set to disabled" — falls +; back to non-fused implementation. Inference works correctly but may be +; slower on the GDN layers. An updated llama.cpp may improve throughput. +; load-on-startup NOT set (loads on first request, ~30-60s cold load). +; n-gpu-layers=99: GPU (all layers). Explicit here so global CLI flag removal +; (t_72646029, 2026-08-17) does not change Qwen3.8 behavior. +; Primary model ID: Qwen3.8-27B-Q4_K_M +[Qwen3.8-27B-Q4_K_M] +model = {{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf +n-gpu-layers = 99 +ctx-size = {{ llm_router_qwen38_ctx_size }} +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} ; --- Auxiliary model: Phi-3.5-mini-instruct-Q8_0 ---------------------------- ; Primary model ID: Phi-3.5-mini-instruct-Q8_0 (unchanged from --models-dir) @@ -79,7 +105,7 @@ parallel = {{ llm_router_parallel }} [Phi-3.5-mini-instruct-Q8_0] model = {{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf alias = Phi-3.5-mini-instruct-8bit -n-gpu-layers = {{ llm_router_gpu_layers }} +n-gpu-layers = 99 ctx-size = {{ llm_router_phi_ctx_size }} flash-attn = {{ llm_router_phi_flash_attn }} cache-type-k = {{ llm_router_cache_type_k }} @@ -93,10 +119,15 @@ parallel = {{ llm_router_parallel }} ; Alias: Meta-Llama-3.1-8B-Instruct-4bit (NEW — friendlier name) ; Both names resolve to this GGUF child process. ; ~4.6GB, general-purpose small model. Works with json_schema structured output. +; n-gpu-layers=0 (CPU offload, t_72646029 2026-08-17): Llama moves to full CPU +; inference to allow concurrent residency with Qwen3.8-27B (which uses ~20.8GB +; VRAM including nomic-embed). At models-max=4, Llama and Coder run on CPU — +; llama.cpp 6ea215d still holds ~1.4-1.7GB CUDA-context VRAM per CPU model, so +; steady-state is ~24,004 MiB (below the 24,576 MiB physical limit). [Meta-Llama-3.1-8B-Instruct-Q4_K_M] model = {{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf alias = Meta-Llama-3.1-8B-Instruct-4bit -n-gpu-layers = {{ llm_router_gpu_layers }} +n-gpu-layers = {{ llm_router_llama_gpu_layers }} ctx-size = {{ llm_router_llama_ctx_size }} flash-attn = {{ llm_router_llama_flash_attn }} cache-type-k = {{ llm_router_cache_type_k }} @@ -109,14 +140,19 @@ parallel = {{ llm_router_parallel }} ; Primary model ID: Qwen2.5-Coder-14B-Instruct-Q4_K_M (filename-derived) ; Alias: Qwen2.5-Coder-14B-Instruct-4bit (friendlier name) ; Both names resolve to this GGUF child process. -; ~8.4GB weights + ~0.6GB KV @ 16K ctx = ~9.0GB VRAM. +; ~8.4GB weights + ~0.6GB KV @ 16K ctx = ~9.0GB VRAM (GPU); ~1,390 MiB CUDA ctx (CPU). ; ctx-size=16384, flash-attn=true per task t_55c164f5 / Ryan's request. ; Source: bartowski/Qwen2.5-Coder-14B-Instruct-GGUF (public, no auth) ; Added 2026-08-13 (t_55c164f5) — War Machine. +; n-gpu-layers=0 (CPU offload, t_72646029 2026-08-17): Coder moves to full CPU +; inference to allow concurrent residency with Qwen3.8-27B (which uses ~20.8GB +; VRAM including nomic-embed). At models-max=4, Coder and Llama run on CPU — +; llama.cpp 6ea215d still holds ~1.4-1.7GB CUDA-context VRAM per CPU model, so +; steady-state is ~24,004 MiB (below the 24,576 MiB physical limit). [Qwen2.5-Coder-14B-Instruct-Q4_K_M] model = {{ llm_models_dir }}/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf alias = Qwen2.5-Coder-14B-Instruct-4bit -n-gpu-layers = {{ llm_router_gpu_layers }} +n-gpu-layers = {{ llm_router_coder_gpu_layers }} ctx-size = {{ llm_router_coder_ctx_size }} flash-attn = {{ llm_router_coder_flash_attn }} cache-type-k = {{ llm_router_cache_type_k }} @@ -124,3 +160,103 @@ cache-type-v = {{ llm_router_cache_type_v }} batch-size = {{ llm_router_batch_size }} ubatch-size = {{ llm_router_ubatch_size }} parallel = {{ llm_router_parallel }} + +; --- Embedding model: nomic-embed-text-v1.5 ---------------------------------- +; Primary model ID: nomic-embed-text-v1.5 (section name / client-visible ID) +; ~84MB GGUF — negligible VRAM, always-loaded. Embedding endpoint: /v1/embeddings. +; embedding=true: required to expose /v1/embeddings and embed the model (not chat). +; n-gpu-layers=99: GPU offload all layers (tiny model, no reason to leave on CPU). +; ctx-size=8192: per task spec (OpenViking Phase 1b, t_34b96e83). +; load-on-startup=true: warm at boot — embedding callers (peter-parker) are +; latency-sensitive; no cold-load wait on first request. +; sleep-idle-seconds=-1: never evict — ~84MB is negligible, always keep hot. +; NO flash-attn, NO KV cache params: embedding models use a different forward +; pass (bidirectional, no autoregressive KV cache). These keys are irrelevant +; for embedding inference and may be silently ignored or cause warnings; omit. +; Source: nomic-ai/nomic-embed-text-v1.5-GGUF (public, no auth needed) +; Added 2026-08-13 (t_34b96e83) — War Machine. +; +; FIXED (2026-08-14, t_openviking_embed_batch): the original section omitted +; batch-size/ubatch-size, so llama-server defaulted the PHYSICAL batch +; (ubatch-size) to 512 tokens. For embedding requests llama.cpp cannot split +; a single input across ubatches, so any OpenViking chunk over ~512 tokens +; large chunk over ~512 tokens (observed 2000-3400 tokens/chunk) failed hard with "input (N tokens) is too +; large to process. increase the physical batch size (current batch size: +; 512)". This tripped OpenViking's circuit breaker into an infinite +; fail/re-enqueue loop. Fix: set batch-size/ubatch-size to 4096 (comfortably +; over the observed max chunk size and under ctx-size=8192). +; +; FOLLOW-UP FINDING (2026-08-14, same task): after the batch-size fix landed, +; logs showed a SECOND, separate problem: llama.cpp capped the effective +; context to 2048 regardless of ctx-size=8192 ("n_ctx_seq (8192) > n_ctx_train +; (2048)" / "capping"). This is expected per the nomic-embed-text-v1.5-GGUF +; model card: the base GGUF's native RoPE training context is 2048; the +; original HF model reaches its benchmarked 8192-token context via Dynamic +; NTK-Aware RoPE scaling, which llama.cpp does not implement — so llama.cpp +; defaults to 2048 unless YaRN scaling is explicitly requested. Model card +; prescribes: --rope-scaling yarn --rope-freq-scale 0.75 alongside -c 8192. +; Added rope-scaling/rope-freq-scale below to actually reach 8192. +[nomic-embed-text-v1.5] +model = {{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf +embedding = true +n-gpu-layers = 99 +ctx-size = {{ llm_router_nomic_ctx_size }} +batch-size = {{ llm_router_nomic_batch_size }} +ubatch-size = {{ llm_router_nomic_ubatch_size }} +rope-scaling = yarn +rope-freq-scale = 0.75 +load-on-startup = true +sleep-idle-seconds = -1 + +; --- Auxiliary model: Qwen3-8B-Q4_K_M (thinking variant) ---------------------- +; GGUF: bartowski/Qwen_Qwen3-8B-GGUF (5,027,784,224 bytes) +; Thinking mode: ON BY DEFAULT (Qwen3 baked-in template, no override). +; - Use for complex aux tasks: long-context summarization, multi-step tool +; planning, structured extraction requiring CoT. +; - To suppress thinking at request time, send /no_think prefix in the prompt +; OR route to [Qwen3-8B-Q4_K_M-no_think] section below. +; n-gpu-layers=99: GPU-resident (~5.2GB VRAM). Cannot co-reside with +; Qwen3.8-27B-Q4_K_M (17.8GB). LRU eviction handles swapping. +; ctx-size=32768: 32K context, q4_0 KV cache. +; flash-attn=true: required for Qwen3 architecture at this context size. +; sleep-idle-seconds=60: evict after 60s idle (free VRAM for primary model). +; Added: t_c5cef2b2 (2026-08-19, War Machine) — initial Qwen3-8B deployment. +[Qwen3-8B-Q4_K_M] +model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf +n-gpu-layers = 99 +ctx-size = 32768 +flash-attn = true +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} +sleep-idle-seconds = 60 + +; --- Auxiliary model: Qwen3-8B-Q4_K_M (no-think variant) ---------------------- +; Secondary section serving the SAME GGUF with enable_thinking=False via an +; overridden Jinja2 template. This gives low-latency, non-reasoning inference +; for latency-sensitive aux tasks (intent classification, query rewriting, +; structured extraction, tool-call construction, draft generation). +; +; Both sections point to the same .gguf file — llama-server spawns independent +; child processes. VRAM cost: ~5.2GB per instance (~10.4GB total). Fits on +; RTX 3090 24GB alongside nomic-embed (84MB). +; +; Thinking variant (Qwen3-8B-Q4_K_M) remains available for complex tasks that +; benefit from CoT (long-context summarization, multi-step tool planning). +; +; Cannot co-reside with Qwen3.8-27B-Q4_K_M (17.8GB); LRU eviction applies. +; Added: t_664289a0 (2026-08-19, War Machine) — dual thinking deployment. +[Qwen3-8B-Q4_K_M-no_think] +model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf +n-gpu-layers = 99 +ctx-size = 32768 +flash-attn = true +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} +chat-template-file = {{ llm_models_dir }}/templates/qwen3-no-think.jinja +sleep-idle-seconds = 60 diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 new file mode 100644 index 0000000..e28a6c0 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 @@ -0,0 +1,60 @@ +{# + FILE: roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 + DESCRIPTION: llama-swap v250 configuration template. + Generates /etc/llama-swap/config.yaml with all models, routing matrix, + and per-model settings (ctx_size, n_gpu_layers, cmd args). + + v250 SYNTAX NOTES: + - Uses routing.router DSL with expression-based matrix (not old list-of-arrays) + - Each model has its own cmd field with full per-model args + - Matrix rows use "model1 & model2" syntax for co-resident sets + - sleep_idle_seconds: -1 = never idle; 0+ = idle after N seconds + - load_on_startup: true = start this model on service startup + + Reference: /etc/llama-swap/config.yaml on astro-orbiter (Phase 1 artifact) +#} +# llama-swap configuration for astro-orbiter +# Generated by Ansible roles/llm-inference-multimodel on {{ ansible_date_time.iso8601 }} +# See: https://github.com/mostlygeek/llama-swap (v250 release notes for syntax) + +# ============================================================================ +# LISTEN — Address and port for the llama-swap proxy +# ============================================================================ +listen: "{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}" + +# ============================================================================ +# MODELS — All model definitions (cmd, port, ctx_size, etc.) +# ============================================================================ +models: +{% for model in llm_swapmode_models %} + {{ model.id }}: + cmd: > + llama-server + --port ${PORT} + --model {{ model.gguf_path }} + --n-gpu-layers {{ model.n_gpu_layers }} + --ctx-size {{ model.ctx_size }} + --batch-size {{ model.batch_size }} + --ubatch-size {{ model.ubatch_size }} + --parallel {{ model.parallel }} + {% if model.cache_type is defined %}--cache-type-k {{ model.cache_type }} --cache-type-v {{ model.cache_type }}{% endif %} + {% if model.flash_attn is defined %}--flash-attn {{ model.flash_attn }}{% endif %} + {% if model.chat_template_file is defined %}--chat-template-file {{ model.chat_template_file }}{% endif %} + {% if model.sleep_idle_seconds is defined %}--sleep-idle-seconds {{ model.sleep_idle_seconds }}{% endif %} + {% if model.load_on_startup is defined and model.load_on_startup %}--load-on-startup{% endif %} + --host 127.0.0.1 + port: {{ model.port }} +{% endfor %} + +# ============================================================================ +# ROUTING — Matrix-based hot-swap policy (v250 expression DSL) +# ============================================================================ +routing: + router: + use: matrix + settings: + matrix: + sets: +{% for row in llm_swapmode_matrix_rows %} + {{ row.row }}: "{{ row.expr }}" +{% endfor %} diff --git a/ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 b/ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 new file mode 100644 index 0000000..0206423 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 @@ -0,0 +1,103 @@ +{# + FILE: roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 + DESCRIPTION: Qwen3 chat template with enable_thinking unconditionally false. + + This is a chat-template FILE deployed to {{ llm_models_dir }}/templates/qwen3-no-think.jinja + on astro-orbiter and referenced via chat-template-file in the INI preset for + [Qwen3-8B-Q4_K_M-no_think]. The [Qwen3-8B-Q4_K_M] section uses the model's + baked-in default template (thinking ON by default). + + Mechanism: at the add_generation_prompt step, instead of checking + "enable_thinking is defined and enable_thinking is false" + we UNCONDITIONALLY emit the empty prefix that suppresses CoT. + This is the hard-switch documented in the Qwen3 template spec and confirmed in + https://huggingface.co/blog/qwen-3-chat-template-deep-dive (section 1). + + All other logic is identical to /opt/llama.cpp/models/templates/Qwen-Qwen3-0.6B.jinja + (the shipped template for Qwen3). Only the final add_generation_prompt block differs. + + Added: t_664289a0 (2026-08-19, War Machine) — dual thinking deployment. +#} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {{- '\n\n\n\n' }} +{%- endif %}