--- # ------------------------------------------------------------------------------ # FILE: playbooks/day2_per_model_ctx_size.yml # DESCRIPTION: Right-size --ctx-size per model workload on llama-server-router # (already in --models-preset mode since t_9adf0889). # # Context (t_ryan_per_model_ctx, 2026-08-13, requested by Ryan via JARVIS): # All 3 preset models currently launch with a uniform --ctx-size 65536. # This playbook narrows two of them to match actual workload: # - Meta-Llama-3.1-8B-Instruct-Q4_K_M (alias Meta-Llama-3.1-8B-Instruct-4bit): # ctx-size 65536 -> 8192 (tool-routing / micro-tasks: title gen, MCP # tool calls, approval checks) # - Phi-3.5-mini-instruct-Q8_0 (alias Phi-3.5-mini-instruct-8bit): # ctx-size 65536 -> 32768 (long web scrapes / session-log compression) # Both also move flash-attn from "auto" to explicit "true" per Ryan's spec. # Qwen3.6-35B-A3B-UD-Q4_K_S is INTENTIONALLY left untouched at 65536/auto. # # Existing aliases (Meta-Llama-3.1-8B-Instruct-4bit, Phi-3.5-mini-instruct-8bit) # are PRESERVED as-is. Ryan's pasted TOML used different alias strings # ("llama-3.1-8b", "phi-3.5-mini") but renaming aliases was not explicitly # requested and would break live Hermes custom_providers routing — flagged # in the deployment report rather than applied silently. # # IMPORTANT — Hermes side effect: /home/hermes/.hermes/config.yaml declares # context_length: 65536 for both these models under custom_providers. This # playbook does NOT touch that file (out of role/agent scope) but the value # becomes STALE the moment this playbook lands. Flag to JARVIS/Maria Hill. # # Usage (from ~/git/homelab/ansible): # ansible-playbook -i inventory.yml playbooks/day2_per_model_ctx_size.yml # # Author: War Machine (2026-08-13, t_ryan_per_model_ctx) # ------------------------------------------------------------------------------ - name: "Right-size per-model ctx-size on llama-server-router (Llama 8k, Phi 32k)" hosts: astro_orbiter gather_facts: false become: true vars: # Preset mode already active in production (t_9adf0889) — keep it on. llm_router_preset_enabled: true llm_router_preset_path: /opt/llama-server-router-preset.ini llm_router_enabled: true # Production port llm_router_port: 8002 llm_router_bind_address: "10.1.71.130" llm_router_allowed_source_cidr: "10.1.70.0/24" llm_bind_address: "10.1.71.130" llm_allowed_source_cidr: "10.1.70.0/24" llm_service_user: jarvis llm_binary_path: /opt/llama.cpp/build/bin/llama-server llm_models_dir: /opt/models llm_router_service_name: llama-server-router llm_router_models_dir: /opt/models llm_router_models_max: 4 llm_router_parallel: 1 llm_router_gpu_layers: 99 llm_router_batch_size: 2048 llm_router_ubatch_size: 512 llm_router_cache_type_k: q4_0 llm_router_cache_type_v: q4_0 # Qwen — untouched baseline (also used as router-wide fallback default) llm_router_ctx_size: 65536 llm_router_flash_attn: "auto" llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" llm_router_vram_max_mib: 23000 # --- THE CHANGE: per-model overrides --- llm_router_llama_ctx_size: 8192 llm_router_llama_flash_attn: "true" llm_router_phi_ctx_size: 32768 llm_router_phi_flash_attn: "true" handlers: - name: reload systemd ansible.builtin.systemd: daemon_reload: true become: true listen: "reload systemd" - name: restart router ansible.builtin.systemd: name: llama-server-router state: restarted become: true listen: "restart router" tasks: # ========================================================================== # PHASE 1: Deploy the preset INI with new per-model ctx-size/flash-attn # ========================================================================== - name: "[ctx-resize] Deploy preset INI to {{ llm_router_preset_path }}" ansible.builtin.template: src: "../roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2" dest: "{{ llm_router_preset_path }}" owner: root group: root mode: "0644" register: ctx_resize_preset_deployed notify: - restart router - name: "[ctx-resize] Deploy router systemd unit (drop global --ctx-size/--flash-attn in preset mode)" ansible.builtin.template: src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2" dest: /etc/systemd/system/llama-server-router.service owner: root group: root mode: "0644" register: ctx_resize_unit_deployed notify: - reload systemd - restart router - name: "[ctx-resize] Flush handlers (daemon-reload + router restart if changed)" ansible.builtin.meta: flush_handlers # ========================================================================== # PHASE 2: Verify # ========================================================================== - name: "[ctx-resize] Wait for /health" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health" status_code: 200 timeout: 30 retries: 12 delay: 5 register: ctx_resize_health until: ctx_resize_health.status == 200 - name: "[ctx-resize] Query /v1/models" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models" status_code: 200 return_content: true timeout: 30 register: ctx_resize_models - name: "[ctx-resize] Trigger load — Llama (confirms actual load + captures live args)" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions" method: POST body_format: json body: model: "Meta-Llama-3.1-8B-Instruct-Q4_K_M" messages: - role: user content: "Reply with one word: hello" max_tokens: 5 temperature: 0.0 status_code: 200 return_content: true timeout: 120 register: ctx_resize_llama_warmup - name: "[ctx-resize] Trigger load — Phi (confirms actual load + captures live args)" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions" method: POST body_format: json body: model: "Phi-3.5-mini-instruct-Q8_0" messages: - role: user content: "Reply with one word: hello" max_tokens: 5 temperature: 0.0 status_code: 200 return_content: true timeout: 120 register: ctx_resize_phi_warmup - name: "[ctx-resize] Re-query /v1/models after warmup (final state)" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models" status_code: 200 return_content: true timeout: 30 register: ctx_resize_models_final - name: "[ctx-resize] Extract Llama args" ansible.builtin.set_fact: ctx_resize_llama_args: >- {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Meta-Llama-3.1-8B-Instruct-Q4_K_M') | first).status.args }} ctx_resize_llama_status: >- {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Meta-Llama-3.1-8B-Instruct-Q4_K_M') | first).status.value }} - name: "[ctx-resize] Extract Phi args" ansible.builtin.set_fact: ctx_resize_phi_args: >- {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Phi-3.5-mini-instruct-Q8_0') | first).status.args }} ctx_resize_phi_status: >- {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Phi-3.5-mini-instruct-Q8_0') | first).status.value }} - name: "[ctx-resize] Extract Qwen args (must be unchanged)" ansible.builtin.set_fact: ctx_resize_qwen_args: >- {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Qwen3.6-35B-A3B-UD-Q4_K_S') | first).status.args }} - name: "[ctx-resize] GATE — Llama ctx-size must be 8192" ansible.builtin.assert: that: - "'8192' in ctx_resize_llama_args" - ctx_resize_llama_args[ctx_resize_llama_args.index('--ctx-size') + 1] == '8192' fail_msg: "Llama ctx-size not 8192. Args: {{ ctx_resize_llama_args }}" success_msg: "Llama ctx-size confirmed 8192." - name: "[ctx-resize] GATE — Llama flash-attn must be true" ansible.builtin.assert: that: - ctx_resize_llama_args[ctx_resize_llama_args.index('--flash-attn') + 1] == 'true' fail_msg: "Llama flash-attn not true. Args: {{ ctx_resize_llama_args }}" success_msg: "Llama flash-attn confirmed true." - name: "[ctx-resize] GATE — Llama loaded successfully" ansible.builtin.assert: that: - ctx_resize_llama_status == 'loaded' fail_msg: "Llama status is '{{ ctx_resize_llama_status }}', expected 'loaded'." success_msg: "Llama status confirmed 'loaded'." - name: "[ctx-resize] GATE — Phi ctx-size must be 32768" ansible.builtin.assert: that: - ctx_resize_phi_args[ctx_resize_phi_args.index('--ctx-size') + 1] == '32768' fail_msg: "Phi ctx-size not 32768. Args: {{ ctx_resize_phi_args }}" success_msg: "Phi ctx-size confirmed 32768." - name: "[ctx-resize] GATE — Phi flash-attn must be true" ansible.builtin.assert: that: - ctx_resize_phi_args[ctx_resize_phi_args.index('--flash-attn') + 1] == 'true' fail_msg: "Phi flash-attn not true. Args: {{ ctx_resize_phi_args }}" success_msg: "Phi flash-attn confirmed true." - name: "[ctx-resize] GATE — Phi loaded successfully" ansible.builtin.assert: that: - ctx_resize_phi_status == 'loaded' fail_msg: "Phi status is '{{ ctx_resize_phi_status }}', expected 'loaded'." success_msg: "Phi status confirmed 'loaded'." - name: "[ctx-resize] GATE — Qwen ctx-size UNCHANGED at 65536" ansible.builtin.assert: that: - ctx_resize_qwen_args[ctx_resize_qwen_args.index('--ctx-size') + 1] == '65536' fail_msg: "Qwen ctx-size changed unexpectedly! Args: {{ ctx_resize_qwen_args }}" success_msg: "Qwen ctx-size confirmed UNCHANGED at 65536." - name: "[ctx-resize] PASS — summary" ansible.builtin.debug: msg: - "================================================================" - "PER-MODEL CTX-SIZE DEPLOYMENT — COMPLETE" - "" - " Llama-3.1-8B (Meta-Llama-3.1-8B-Instruct-Q4_K_M):" - " status: {{ ctx_resize_llama_status }}" - " args: {{ ctx_resize_llama_args }}" - "" - " Phi-3.5-mini (Phi-3.5-mini-instruct-Q8_0):" - " status: {{ ctx_resize_phi_status }}" - " args: {{ ctx_resize_phi_args }}" - "" - " Qwen3.6-35B-A3B-UD-Q4_K_S: UNCHANGED (ctx-size 65536, args: {{ ctx_resize_qwen_args }})" - "" - " ACTION NEEDED: /home/hermes/.hermes/config.yaml custom_providers" - " context_length: 65536 for both Meta-Llama-3.1-8B-Instruct-4bit and" - " Phi-3.5-mini-instruct-8bit is now STALE (actual: 8192 / 32768)." - " Flag to JARVIS/Maria Hill for correction — NOT done by this playbook." - "================================================================"