--- # ------------------------------------------------------------------------------ # FILE: playbooks/day2_add_coder_alias.yml # DESCRIPTION: Add Qwen2.5-Coder-14B-Instruct-Q4_K_M to the llama-server-router # on astro-orbiter (10.1.71.130:8002). # # Context (t_55c164f5, 2026-08-13): # Ryan requested a Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf be added to the # astro-orbiter router with: # alias = "Qwen2.5-Coder-14B-Instruct-4bit" # n_gpu_layers = 99 # ctx_size = 16384 # flash_attn = true # Deployed GitOps-style via this role; no hand-editing of the live preset. # # What this playbook does: # 1. Downloads Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf into /opt/models if # not already present (idempotent: size-check guard, no re-pull on match). # 2. Redeploys the preset INI (adding the [Qwen2.5-Coder-14B-Instruct-Q4_K_M] # section with alias = Qwen2.5-Coder-14B-Instruct-4bit). # 3. Restarts llama-server-router to pick up the new model entry. # 4. Verifies /v1/models returns all 4 models including the new Coder entry. # # VRAM context note (t_55c164f5): # Qwen2.5-Coder-14B Q4_K_M: ~8.4GB weights + ~0.6GB KV @ 16K ctx ≈ 9.0GB # Qwen3.6-35B-A3B: ~21.5GB # Full co-residency is impossible on 24GB. LRU eviction handles this: # when Coder is requested, Qwen3.6-35B is evicted (and vice versa). # Model-switching incurs ~30-60s cold-load latency — expected and acceptable. # Phi (~4.3GB) or Llama (~5.6GB) can co-reside with Coder (total ~14GB). # # Usage (from ~/git/homelab/ansible): # env -u ANSIBLE_VAULT_PASSWORD_FILE ansible-playbook -i inventory.yml \ # playbooks/day2_add_coder_alias.yml # # Semaphore note: Semaphore SSH key for jarvis user is not loaded in the # container (known pitfall, homelab-llm-serving skill). Run via CLI with # id_jarvis key; document as exception per Ryan's standing CLI fallback directive. # # Author: War Machine (2026-08-13, t_55c164f5) # ------------------------------------------------------------------------------ - name: "Add Qwen2.5-Coder-14B-Instruct-4bit alias to astro-orbiter router" hosts: astro_orbiter gather_facts: false become: true vars: # Activate preset mode llm_router_preset_enabled: true llm_router_preset_path: /opt/llama-server-router-preset.ini # Production port (router is on 8002 since t_cd0d5388) llm_router_port: 8002 # Per-model ctx-size settings (carried from t_ryan_per_model_ctx; Coder new) llm_router_llama_ctx_size: 8192 llm_router_llama_flash_attn: "true" llm_router_phi_ctx_size: 32768 llm_router_phi_flash_attn: "true" llm_router_coder_ctx_size: 16384 llm_router_coder_flash_attn: "true" # All other vars inherit from host_vars + defaults/main.yml. # Explicitly set the ones needed by the unit/template tasks for clarity: llm_router_enabled: true llm_service_user: jarvis llm_binary_path: /opt/llama.cpp/build/bin/llama-server llm_models_dir: /opt/models llm_bind_address: "10.1.71.130" llm_allowed_source_cidr: "10.1.70.0/24" llm_router_service_name: llama-server-router llm_router_bind_address: "10.1.71.130" llm_router_allowed_source_cidr: "10.1.70.0/24" llm_router_models_dir: /opt/models llm_router_models_max: 4 # from host_vars; bumped by t_33acbb2e llm_router_ctx_size: 65536 # Qwen3.6-35B default; per-model overrides above llm_router_parallel: 1 llm_router_gpu_layers: 99 llm_router_batch_size: 2048 llm_router_ubatch_size: 512 llm_router_cache_type_k: q4_0 llm_router_cache_type_v: q4_0 llm_router_flash_attn: "auto" llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" llm_router_vram_max_mib: 23000 # Coder model staging entry (used below) coder_filename: "Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf" coder_url: "https://huggingface.co/bartowski/Qwen2.5-Coder-14B-Instruct-GGUF/resolve/main/Qwen2.5-Coder-14B-Instruct-Q4_K_M.gguf" coder_size_bytes: 8988111072 handlers: - name: reload systemd ansible.builtin.systemd: daemon_reload: true become: true listen: "reload systemd" - name: restart router ansible.builtin.systemd: name: llama-server-router state: restarted become: true listen: "restart router" tasks: # ========================================================================== # PHASE 1: Download Coder GGUF if not present / size mismatch # ========================================================================== - name: "[coder] Stat existing GGUF" ansible.builtin.stat: path: "{{ llm_models_dir }}/{{ coder_filename }}" get_checksum: false register: coder_stat - name: "[coder] Download GGUF (skip if present and size matches)" ansible.builtin.get_url: url: "{{ coder_url }}" dest: "{{ llm_models_dir }}/{{ coder_filename }}" owner: "{{ llm_service_user }}" group: "{{ llm_service_user }}" mode: "0644" timeout: 3600 when: > not coder_stat.stat.exists or coder_stat.stat.size != coder_size_bytes register: coder_download notify: restart router - name: "[coder] Confirm GGUF size post-download" ansible.builtin.stat: path: "{{ llm_models_dir }}/{{ coder_filename }}" get_checksum: false register: coder_stat_post - name: "[coder] FAIL if GGUF size mismatch after download" ansible.builtin.fail: msg: >- GGUF size mismatch: expected {{ coder_size_bytes }} bytes, got {{ coder_stat_post.stat.size }} bytes. Re-download may be needed. when: coder_stat_post.stat.size != coder_size_bytes # ========================================================================== # PHASE 2: Deploy updated preset INI (adds Coder section) # ========================================================================== - name: "[coder] Deploy preset INI to {{ llm_router_preset_path }}" ansible.builtin.template: src: "../roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2" dest: "{{ llm_router_preset_path }}" owner: root group: root mode: "0644" register: coder_preset_deployed notify: restart router # ========================================================================== # PHASE 3: Redeploy systemd unit (unchanged flags, but ensures unit is fresh) # ========================================================================== - name: "[coder] Deploy llama-server-router unit" ansible.builtin.template: src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2" dest: /etc/systemd/system/llama-server-router.service owner: root group: root mode: "0644" register: coder_unit_deployed notify: - reload systemd - restart router - name: "[coder] Flush handlers (daemon-reload + router restart)" ansible.builtin.meta: flush_handlers # ========================================================================== # PHASE 4: Verify router is up and Coder model appears in /v1/models # ========================================================================== - name: "[coder] Wait for /health (router supervisor)" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health" status_code: 200 timeout: 30 retries: 12 delay: 5 register: coder_health until: coder_health.status == 200 - name: "[coder] Query /v1/models" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models" status_code: 200 return_content: true timeout: 30 register: coder_models - name: "[coder] Extract model IDs and aliases" ansible.builtin.set_fact: coder_model_ids: "{{ coder_models.json.data | map(attribute='id') | list }}" coder_all_aliases: "{{ coder_models.json.data | map(attribute='aliases') | flatten | list }}" - name: "[coder] FAIL if Coder primary ID missing" ansible.builtin.fail: msg: >- 'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ coder_model_ids }} when: "'Qwen2.5-Coder-14B-Instruct-Q4_K_M' not in coder_model_ids" - name: "[coder] FAIL if Coder alias missing" ansible.builtin.fail: msg: >- 'Qwen2.5-Coder-14B-Instruct-4bit' not found as ID or alias in /v1/models. IDs: {{ coder_model_ids }} Aliases: {{ coder_all_aliases }} when: - "'Qwen2.5-Coder-14B-Instruct-4bit' not in coder_model_ids" - "'Qwen2.5-Coder-14B-Instruct-4bit' not in coder_all_aliases" - name: "[coder] FAIL if Qwen3.6-35B missing" ansible.builtin.fail: msg: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in /v1/models. IDs: {{ coder_model_ids }}" when: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in coder_model_ids" - name: "[coder] FAIL if Phi missing" ansible.builtin.fail: msg: "'Phi-3.5-mini-instruct-Q8_0' not in /v1/models. IDs: {{ coder_model_ids }}" when: "'Phi-3.5-mini-instruct-Q8_0' not in coder_model_ids" - name: "[coder] FAIL if Llama missing" ansible.builtin.fail: msg: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ coder_model_ids }}" when: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in coder_model_ids" - name: "[coder] PASS — full /v1/models summary" ansible.builtin.debug: msg: - "========================================================================" - "QWEN2.5-CODER-14B ALIAS DEPLOYMENT — COMPLETE" - "" - " Mode: --models-preset ({{ llm_router_preset_path }})" - " Service: llama-server-router.service (:{{ llm_router_port }})" - "" - " /v1/models IDs: {{ coder_model_ids }}" - " /v1/models aliases: {{ coder_all_aliases }}" - "" - " VERIFY:" - " Qwen3.6-35B-A3B-UD-Q4_K_S: {{ 'PRESENT' if 'Qwen3.6-35B-A3B-UD-Q4_K_S' in coder_model_ids else 'MISSING' }}" - " Phi-3.5-mini-instruct-Q8_0: {{ 'PRESENT' if 'Phi-3.5-mini-instruct-Q8_0' in coder_model_ids else 'MISSING' }}" - " Meta-Llama-3.1-8B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' in coder_model_ids else 'MISSING' }}" - " Qwen2.5-Coder-14B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Qwen2.5-Coder-14B-Instruct-Q4_K_M' in coder_model_ids else 'MISSING' }}" - " Qwen2.5-Coder-14B-Instruct-4bit: {{ 'PRESENT (ID)' if 'Qwen2.5-Coder-14B-Instruct-4bit' in coder_model_ids else ('PRESENT (alias)' if 'Qwen2.5-Coder-14B-Instruct-4bit' in coder_all_aliases else 'MISSING') }}" - "" - " GGUF download: {{ 'NEW DOWNLOAD' if (coder_download is defined and coder_download.changed) else 'ALREADY PRESENT (skipped)' }}" - "========================================================================"