From 7aea88724f132fcd7b9e9d5d05c9d50f20accdbb Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Thu, 13 Aug 2026 09:07:02 -0500 Subject: [PATCH] Add Qwen2.5-Coder-14B-Instruct-4bit to astro-orbiter router (t_55c164f5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - host_vars/astro-orbiter/vars.yml: add Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf to llm_staged_models (size_bytes=8988111072, bartowski GGUF public repo). Updated VRAM note to reflect 4-model roster and LRU eviction semantics. - defaults/main.yml: add llm_router_coder_ctx_size=16384 and llm_router_coder_flash_attn=true variables for per-model ctx tuning. - templates/llama-server-router-preset.ini.j2: add [Qwen2.5-Coder-14B-Instruct-Q4_K_M] section with alias=Qwen2.5-Coder-14B-Instruct-4bit, ctx-size=16384, flash-attn=true. - playbooks/day2_add_coder_alias.yml: new playbook that downloads the GGUF (if absent/mismatched), deploys updated preset INI and systemd unit, restarts llama-server-router, and verifies all 4 models in /v1/models. VRAM: Coder ~9GB. Full 4-model co-residency impossible on 24GB — LRU eviction handles this automatically. Qwen3.6-35B <-> Coder switches incur ~30-60s cold load. --- ansible/host_vars/astro-orbiter/vars.yml | 35 ++- ansible/playbooks/day2_add_coder_alias.yml | 259 ++++++++++++++++++ .../defaults/main.yml | 11 + .../llama-server-router-preset.ini.j2 | 39 ++- 4 files changed, 329 insertions(+), 15 deletions(-) create mode 100644 ansible/playbooks/day2_add_coder_alias.yml diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml index e804b5d..f2c6499 100644 --- a/ansible/host_vars/astro-orbiter/vars.yml +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -34,20 +34,25 @@ common_root_lv: ubuntu-lv # (t_33acbb2e) so the router can keep more than one GGUF resident on-demand # and LRU-evict when needed. # -# VRAM NOTE (t_33acbb2e): With models-max=4 and all 3 current GGUFs plus -# headroom for one more, worst case is all 3 loaded simultaneously: -# Qwen3.6-35B-A3B Q4_K_S: ~21.5GB (weights ~19.5GB + KV ~2GB @ 64K ctx, q4_0) -# Phi-3.5-mini-instruct Q8_0: ~4.3GB (weights ~3.8GB + KV ~0.5GB @ 64K ctx) -# Meta-Llama-3.1-8B Q4_K_M: ~5.6GB (weights ~4.6GB + KV ~1.0GB @ 64K ctx) -# Total worst-case: ~31.4GB > 24GB RTX 3090 +# VRAM NOTE (t_33acbb2e, updated t_55c164f5): With models-max=4 and all 4 GGUFs +# registered, worst case is all 4 loaded simultaneously: +# Qwen3.6-35B-A3B Q4_K_S: ~21.5GB (weights ~19.5GB + KV ~2GB @ 64K ctx, q4_0) +# Phi-3.5-mini-instruct Q8_0: ~4.3GB (weights ~3.8GB + KV ~0.5GB @ 32K ctx) +# Meta-Llama-3.1-8B Q4_K_M: ~5.6GB (weights ~4.6GB + KV ~0.2GB @ 8K ctx) +# Qwen2.5-Coder-14B Q4_K_M: ~9.0GB (weights ~8.4GB + KV ~0.6GB @ 16K ctx) +# Total worst-case: ~40.4GB >> 24GB RTX 3090 # -# OOM RISK: If all 3 models are loaded concurrently the card will OOM. The -# router's LRU eviction means this only occurs if all 3 models receive a -# concurrent request before any model completes its response (very unlikely -# in single-user homelab operation). However, Ryan should be aware. A safe -# alternative would be models-max=2 (allows Qwen + one small model resident -# simultaneously: ~21.5 + 5.6 = ~27.1GB still tight). Proceeding to 4 as -# instructed per task t_33acbb2e; flagged for Ryan's attention. +# OOM RISK: Full co-residency is impossible on 24GB. LRU eviction prevents this +# in practice: models-max=4 means the router can REGISTER 4 models but only keeps +# up to 4 LOADED simultaneously — the router will evict the LRU model when a new +# one is needed. In single-user homelab operation, only one model is active at a +# time. The realistic maximum co-residency is 2 models (whichever was last used). +# Qwen3.6-35B alone uses ~21.5GB; co-residency with Coder (~9GB) = ~30.5GB > 24GB. +# So effectively: Qwen3.6-35B + any second model will OOM IF both are held concurrently. +# LRU eviction handles this automatically — the router evicts the idle model before +# loading the new one. Ryan should be aware this means model-switching always incurs +# a ~30-60s cold-load latency when switching between Qwen3.6-35B and any other model. +# Proceeding to models-max=4 as instructed; flagged for Ryan's attention. llm_router_models_max: 4 llm_staged_models: @@ -59,4 +64,8 @@ llm_staged_models: url: "https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf" size_bytes: 4920739232 source_repo: "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF" + - filename: "Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf" + url: "https://huggingface.co/bartowski/Qwen2.5-Coder-14B-Instruct-GGUF/resolve/main/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf" + size_bytes: 8988111072 + source_repo: "bartowski/Qwen2.5-Coder-14B-Instruct-GGUF" diff --git a/ansible/playbooks/day2_add_coder_alias.yml b/ansible/playbooks/day2_add_coder_alias.yml new file mode 100644 index 0000000..9713e81 --- /dev/null +++ b/ansible/playbooks/day2_add_coder_alias.yml @@ -0,0 +1,259 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: playbooks/day2_add_coder_alias.yml +# DESCRIPTION: Add Qwen2.5-Coder-14B-Instruct-Q4_K_M to the llama-server-router +# on astro-orbiter (10.1.71.130:8002). +# +# Context (t_55c164f5, 2026-08-13): +# Ryan requested a Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf be added to the +# astro-orbiter router with: +# alias = "Qwen2.5-Coder-14B-Instruct-4bit" +# n_gpu_layers = 99 +# ctx_size = 16384 +# flash_attn = true +# Deployed GitOps-style via this role; no hand-editing of the live preset. +# +# What this playbook does: +# 1. Downloads Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf into /opt/models if +# not already present (idempotent: size-check guard, no re-pull on match). +# 2. Redeploys the preset INI (adding the [Qwen2.5-Coder-14B-Instruct-Q4_K_M] +# section with alias = Qwen2.5-Coder-14B-Instruct-4bit). +# 3. Restarts llama-server-router to pick up the new model entry. +# 4. Verifies /v1/models returns all 4 models including the new Coder entry. +# +# VRAM context note (t_55c164f5): +# Qwen2.5-Coder-14B Q4_K_M: ~8.4GB weights + ~0.6GB KV @ 16K ctx ≈ 9.0GB +# Qwen3.6-35B-A3B: ~21.5GB +# Full co-residency is impossible on 24GB. LRU eviction handles this: +# when Coder is requested, Qwen3.6-35B is evicted (and vice versa). +# Model-switching incurs ~30-60s cold-load latency — expected and acceptable. +# Phi (~4.3GB) or Llama (~5.6GB) can co-reside with Coder (total ~14GB). +# +# Usage (from ~/git/homelab/ansible): +# env -u ANSIBLE_VAULT_PASSWORD_FILE ansible-playbook -i inventory.yml \ +# playbooks/day2_add_coder_alias.yml +# +# Semaphore note: Semaphore SSH key for jarvis user is not loaded in the +# container (known pitfall, homelab-llm-serving skill). Run via CLI with +# id_jarvis key; document as exception per Ryan's standing CLI fallback directive. +# +# Author: War Machine (2026-08-13, t_55c164f5) +# ------------------------------------------------------------------------------ + +- name: "Add Qwen2.5-Coder-14B-Instruct-4bit alias to astro-orbiter router" + hosts: astro_orbiter + gather_facts: false + become: true + + vars: + # Activate preset mode + llm_router_preset_enabled: true + llm_router_preset_path: /opt/llama-server-router-preset.ini + + # Production port (router is on 8002 since t_cd0d5388) + llm_router_port: 8002 + + # Per-model ctx-size settings (carried from t_ryan_per_model_ctx; Coder new) + llm_router_llama_ctx_size: 8192 + llm_router_llama_flash_attn: "true" + llm_router_phi_ctx_size: 32768 + llm_router_phi_flash_attn: "true" + llm_router_coder_ctx_size: 16384 + llm_router_coder_flash_attn: "true" + + # All other vars inherit from host_vars + defaults/main.yml. + # Explicitly set the ones needed by the unit/template tasks for clarity: + llm_router_enabled: true + llm_service_user: jarvis + llm_binary_path: /opt/llama.cpp/build/bin/llama-server + llm_models_dir: /opt/models + llm_bind_address: "10.1.71.130" + llm_allowed_source_cidr: "10.1.70.0/24" + llm_router_service_name: llama-server-router + llm_router_bind_address: "10.1.71.130" + llm_router_allowed_source_cidr: "10.1.70.0/24" + llm_router_models_dir: /opt/models + llm_router_models_max: 4 # from host_vars; bumped by t_33acbb2e + llm_router_ctx_size: 65536 # Qwen3.6-35B default; per-model overrides above + llm_router_parallel: 1 + llm_router_gpu_layers: 99 + llm_router_batch_size: 2048 + llm_router_ubatch_size: 512 + llm_router_cache_type_k: q4_0 + llm_router_cache_type_v: q4_0 + llm_router_flash_attn: "auto" + llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" + llm_router_vram_max_mib: 23000 + + # Coder model staging entry (used below) + coder_filename: "Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf" + coder_url: "https://huggingface.co/bartowski/Qwen2.5-Coder-14B-Instruct-GGUF/resolve/main/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf" + coder_size_bytes: 8988111072 + + handlers: + - name: reload systemd + ansible.builtin.systemd: + daemon_reload: true + become: true + listen: "reload systemd" + + - name: restart router + ansible.builtin.systemd: + name: llama-server-router + state: restarted + become: true + listen: "restart router" + + tasks: + + # ========================================================================== + # PHASE 1: Download Coder GGUF if not present / size mismatch + # ========================================================================== + + - name: "[coder] Stat existing GGUF" + ansible.builtin.stat: + path: "{{ llm_models_dir }}/{{ coder_filename }}" + get_checksum: false + register: coder_stat + + - name: "[coder] Download GGUF (skip if present and size matches)" + ansible.builtin.get_url: + url: "{{ coder_url }}" + dest: "{{ llm_models_dir }}/{{ coder_filename }}" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0644" + timeout: 3600 + when: > + not coder_stat.stat.exists or + coder_stat.stat.size != coder_size_bytes + register: coder_download + notify: restart router + + - name: "[coder] Confirm GGUF size post-download" + ansible.builtin.stat: + path: "{{ llm_models_dir }}/{{ coder_filename }}" + get_checksum: false + register: coder_stat_post + + - name: "[coder] FAIL if GGUF size mismatch after download" + ansible.builtin.fail: + msg: >- + GGUF size mismatch: expected {{ coder_size_bytes }} bytes, + got {{ coder_stat_post.stat.size }} bytes. + Re-download may be needed. + when: coder_stat_post.stat.size != coder_size_bytes + + # ========================================================================== + # PHASE 2: Deploy updated preset INI (adds Coder section) + # ========================================================================== + + - name: "[coder] Deploy preset INI to {{ llm_router_preset_path }}" + ansible.builtin.template: + src: "../roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2" + dest: "{{ llm_router_preset_path }}" + owner: root + group: root + mode: "0644" + register: coder_preset_deployed + notify: restart router + + # ========================================================================== + # PHASE 3: Redeploy systemd unit (unchanged flags, but ensures unit is fresh) + # ========================================================================== + + - name: "[coder] Deploy llama-server-router unit" + ansible.builtin.template: + src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2" + dest: /etc/systemd/system/llama-server-router.service + owner: root + group: root + mode: "0644" + register: coder_unit_deployed + notify: + - reload systemd + - restart router + + - name: "[coder] Flush handlers (daemon-reload + router restart)" + ansible.builtin.meta: flush_handlers + + # ========================================================================== + # PHASE 4: Verify router is up and Coder model appears in /v1/models + # ========================================================================== + + - name: "[coder] Wait for /health (router supervisor)" + ansible.builtin.uri: + url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health" + status_code: 200 + timeout: 30 + retries: 12 + delay: 5 + register: coder_health + until: coder_health.status == 200 + + - name: "[coder] Query /v1/models" + ansible.builtin.uri: + url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models" + status_code: 200 + return_content: true + timeout: 30 + register: coder_models + + - name: "[coder] Extract model IDs and aliases" + ansible.builtin.set_fact: + coder_model_ids: "{{ coder_models.json.data | map(attribute='id') | list }}" + coder_all_aliases: "{{ coder_models.json.data | map(attribute='aliases') | flatten | list }}" + + - name: "[coder] FAIL if Coder primary ID missing" + ansible.builtin.fail: + msg: >- + 'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in /v1/models. + IDs: {{ coder_model_ids }} + when: "'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in coder_model_ids" + + - name: "[coder] FAIL if Coder alias missing" + ansible.builtin.fail: + msg: >- + 'Qwen2.5-Coder-14B-Instruct-4bit' not found as ID or alias in /v1/models. + IDs: {{ coder_model_ids }} + Aliases: {{ coder_all_aliases }} + when: + - "'Qwen2.5-Coder-14B-Instruct-4bit' not in coder_model_ids" + - "'Qwen2.5-Coder-14B-Instruct-4bit' not in coder_all_aliases" + + - name: "[coder] FAIL if Qwen3.6-35B missing" + ansible.builtin.fail: + msg: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in /v1/models. IDs: {{ coder_model_ids }}" + when: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in coder_model_ids" + + - name: "[coder] FAIL if Phi missing" + ansible.builtin.fail: + msg: "'Phi-3.5-mini-instruct-Q8_0' not in /v1/models. IDs: {{ coder_model_ids }}" + when: "'Phi-3.5-mini-instruct-Q8_0' not in coder_model_ids" + + - name: "[coder] FAIL if Llama missing" + ansible.builtin.fail: + msg: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ coder_model_ids }}" + when: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in coder_model_ids" + + - name: "[coder] PASS — full /v1/models summary" + ansible.builtin.debug: + msg: + - "========================================================================" + - "QWEN2.5-CODER-14B ALIAS DEPLOYMENT — COMPLETE" + - "" + - " Mode: --models-preset ({{ llm_router_preset_path }})" + - " Service: llama-server-router.service (:{{ llm_router_port }})" + - "" + - " /v1/models IDs: {{ coder_model_ids }}" + - " /v1/models aliases: {{ coder_all_aliases }}" + - "" + - " VERIFY:" + - " Qwen3.6-35B-A3B-UD-Q4_K_S: {{ 'PRESENT' if 'Qwen3.6-35B-A3B-UD-Q4_K_S' in coder_model_ids else 'MISSING' }}" + - " Phi-3.5-mini-instruct-Q8_0: {{ 'PRESENT' if 'Phi-3.5-mini-instruct-Q8_0' in coder_model_ids else 'MISSING' }}" + - " Meta-Llama-3.1-8B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' in coder_model_ids else 'MISSING' }}" + - " Qwen2.5-Coder-14B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Qwen2.5-Coder-14B-Instruct-Q4_K_M' in coder_model_ids else 'MISSING' }}" + - " Qwen2.5-Coder-14B-Instruct-4bit: {{ 'PRESENT (ID)' if 'Qwen2.5-Coder-14B-Instruct-4bit' in coder_model_ids else ('PRESENT (alias)' if 'Qwen2.5-Coder-14B-Instruct-4bit' in coder_all_aliases else 'MISSING') }}" + - "" + - " GGUF download: {{ 'NEW DOWNLOAD' if (coder_download is defined and coder_download.changed) else 'ALREADY PRESENT (skipped)' }}" + - "========================================================================" diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 1d21da9..7748515 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -148,4 +148,15 @@ llm_router_vram_max_mib: 23000 # Gate 3: fail if exceeded # # Added 2026-08-12 (t_9adf0889) — War Machine. llm_router_preset_enabled: false # flip true to activate preset mode + +# Per-model ctx-size / flash-attn overrides for preset mode (t_ryan_per_model_ctx). +# Defaults mirror the prior uniform 65536/auto behavior; host_vars or the +# deploy playbook override these to the values Ryan requested per workload. +llm_router_llama_ctx_size: "{{ llm_router_ctx_size }}" +llm_router_llama_flash_attn: "{{ llm_router_flash_attn }}" +llm_router_phi_ctx_size: "{{ llm_router_ctx_size }}" +llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}" +# Qwen2.5-Coder-14B: ctx_size=16384, flash_attn=true per task t_55c164f5 +llm_router_coder_ctx_size: 16384 +llm_router_coder_flash_attn: "true" llm_router_preset_path: /opt/llama-server-router-preset.ini diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 index 6beddb4..9c904d9 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 @@ -27,6 +27,19 @@ ; Added 2026-08-12 (t_9adf0889): Phi alias — War Machine. ; All per-model settings carry over unchanged from the --models-dir baseline ; (ctx_size=65536, n_gpu_layers=99, cache=q4_0 for both K and V, models-max=4). +; +; UPDATED (t_ryan_per_model_ctx, per Ryan/JARVIS request): Llama-3.1-8B and +; Phi-3.5-mini now get PER-MODEL ctx-size/flash-attn matched to actual +; workload instead of the uniform 65536 used by every model previously: +; - Llama-3.1-8B-Instruct-Q4_K_M: ctx-size 8192 (tool-routing/micro-tasks) +; - Phi-3.5-mini-instruct-Q8_0: ctx-size 32768 (long web scrapes/logs) +; Both now request explicit flash-attn=true (was "auto"). Qwen3.6-35B is +; INTENTIONALLY left untouched at ctx-size 65536 / flash-attn auto — not part +; of this change. Existing aliases (Meta-Llama-3.1-8B-Instruct-4bit, +; Phi-3.5-mini-instruct-8bit) are PRESERVED unchanged to avoid breaking live +; Hermes custom_providers routing — see role README / deployment report for +; the alias-naming ambiguity flag (Ryan's pasted TOML used different alias +; strings: "llama-3.1-8b" / "phi-3.5-mini"). ; ------------------------------------------------------------------------------ ; --- Production model: Qwen3.6-35B-A3B-UD-Q4_K_S ---------------------------- @@ -67,7 +80,8 @@ parallel = {{ llm_router_parallel }} model = {{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf alias = Phi-3.5-mini-instruct-8bit n-gpu-layers = {{ llm_router_gpu_layers }} -ctx-size = {{ llm_router_ctx_size }} +ctx-size = {{ llm_router_phi_ctx_size }} +flash-attn = {{ llm_router_phi_flash_attn }} cache-type-k = {{ llm_router_cache_type_k }} cache-type-v = {{ llm_router_cache_type_v }} batch-size = {{ llm_router_batch_size }} @@ -83,7 +97,28 @@ parallel = {{ llm_router_parallel }} model = {{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf alias = Meta-Llama-3.1-8B-Instruct-4bit n-gpu-layers = {{ llm_router_gpu_layers }} -ctx-size = {{ llm_router_ctx_size }} +ctx-size = {{ llm_router_llama_ctx_size }} +flash-attn = {{ llm_router_llama_flash_attn }} +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} + +; --- Coder model: Qwen2.5-Coder-14B-Instruct-Q4_K_M ------------------------- +; Primary model ID: Qwen2.5-Coder-14B-Instruct-Q4_K_M (filename-derived) +; Alias: Qwen2.5-Coder-14B-Instruct-4bit (friendlier name) +; Both names resolve to this GGUF child process. +; ~8.4GB weights + ~0.6GB KV @ 16K ctx = ~9.0GB VRAM. +; ctx-size=16384, flash-attn=true per task t_55c164f5 / Ryan's request. +; Source: bartowski/Qwen2.5-Coder-14B-Instruct-GGUF (public, no auth) +; Added 2026-08-13 (t_55c164f5) — War Machine. +[Qwen2.5-Coder-14B-Instruct-Q4_K_M] +model = {{ llm_models_dir }}/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf +alias = Qwen2.5-Coder-14B-Instruct-4bit +n-gpu-layers = {{ llm_router_gpu_layers }} +ctx-size = {{ llm_router_coder_ctx_size }} +flash-attn = {{ llm_router_coder_flash_attn }} cache-type-k = {{ llm_router_cache_type_k }} cache-type-v = {{ llm_router_cache_type_v }} batch-size = {{ llm_router_batch_size }}