homelab/ansible/playbooks/day2_per_model_ctx_size.yml

---
# ------------------------------------------------------------------------------
# FILE: playbooks/day2_per_model_ctx_size.yml
# DESCRIPTION: Right-size --ctx-size per model workload on llama-server-router
#              (already in --models-preset mode since t_9adf0889).
#
# Context (t_ryan_per_model_ctx, 2026-08-13, requested by Ryan via JARVIS):
#   All 3 preset models currently launch with a uniform --ctx-size 65536.
#   This playbook narrows two of them to match actual workload:
#     - Meta-Llama-3.1-8B-Instruct-Q4_K_M (alias Meta-Llama-3.1-8B-Instruct-4bit):
#         ctx-size 65536 -> 8192  (tool-routing / micro-tasks: title gen, MCP
#         tool calls, approval checks)
#     - Phi-3.5-mini-instruct-Q8_0 (alias Phi-3.5-mini-instruct-8bit):
#         ctx-size 65536 -> 32768 (long web scrapes / session-log compression)
#   Both also move flash-attn from "auto" to explicit "true" per Ryan's spec.
#   Qwen3.6-35B-A3B-UD-Q4_K_S is INTENTIONALLY left untouched at 65536/auto.
#
#   Existing aliases (Meta-Llama-3.1-8B-Instruct-4bit, Phi-3.5-mini-instruct-8bit)
#   are PRESERVED as-is. Ryan's pasted TOML used different alias strings
#   ("llama-3.1-8b", "phi-3.5-mini") but renaming aliases was not explicitly
#   requested and would break live Hermes custom_providers routing — flagged
#   in the deployment report rather than applied silently.
#
# IMPORTANT — Hermes side effect: /home/hermes/.hermes/config.yaml declares
# context_length: 65536 for both these models under custom_providers. This
# playbook does NOT touch that file (out of role/agent scope) but the value
# becomes STALE the moment this playbook lands. Flag to JARVIS/Maria Hill.
#
# Usage (from ~/git/homelab/ansible):
#   ansible-playbook -i inventory.yml playbooks/day2_per_model_ctx_size.yml
#
# Author: War Machine (2026-08-13, t_ryan_per_model_ctx)
# ------------------------------------------------------------------------------

- name: "Right-size per-model ctx-size on llama-server-router (Llama 8k, Phi 32k)"
  hosts: astro_orbiter
  gather_facts: false
  become: true

  vars:
    # Preset mode already active in production (t_9adf0889) — keep it on.
    llm_router_preset_enabled: true
    llm_router_preset_path: /opt/llama-server-router-preset.ini
    llm_router_enabled: true

    # Production port
    llm_router_port: 8002
    llm_router_bind_address: "10.1.71.130"
    llm_router_allowed_source_cidr: "10.1.70.0/24"
    llm_bind_address: "10.1.71.130"
    llm_allowed_source_cidr: "10.1.70.0/24"

    llm_service_user: jarvis
    llm_binary_path: /opt/llama.cpp/build/bin/llama-server
    llm_models_dir: /opt/models
    llm_router_service_name: llama-server-router
    llm_router_models_dir: /opt/models
    llm_router_models_max: 4
    llm_router_parallel: 1
    llm_router_gpu_layers: 99
    llm_router_batch_size: 2048
    llm_router_ubatch_size: 512
    llm_router_cache_type_k: q4_0
    llm_router_cache_type_v: q4_0

    # Qwen — untouched baseline (also used as router-wide fallback default)
    llm_router_ctx_size: 65536
    llm_router_flash_attn: "auto"
    llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S"
    llm_router_vram_max_mib: 23000

    # --- THE CHANGE: per-model overrides ---
    llm_router_llama_ctx_size: 8192
    llm_router_llama_flash_attn: "true"
    llm_router_phi_ctx_size: 32768
    llm_router_phi_flash_attn: "true"

  handlers:
    - name: reload systemd
      ansible.builtin.systemd:
        daemon_reload: true
      become: true
      listen: "reload systemd"

    - name: restart router
      ansible.builtin.systemd:
        name: llama-server-router
        state: restarted
      become: true
      listen: "restart router"

  tasks:

    # ==========================================================================
    # PHASE 1: Deploy the preset INI with new per-model ctx-size/flash-attn
    # ==========================================================================

    - name: "[ctx-resize] Deploy preset INI to {{ llm_router_preset_path }}"
      ansible.builtin.template:
        src: "../roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2"
        dest: "{{ llm_router_preset_path }}"
        owner: root
        group: root
        mode: "0644"
      register: ctx_resize_preset_deployed
      notify:
        - restart router

    - name: "[ctx-resize] Deploy router systemd unit (drop global --ctx-size/--flash-attn in preset mode)"
      ansible.builtin.template:
        src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2"
        dest: /etc/systemd/system/llama-server-router.service
        owner: root
        group: root
        mode: "0644"
      register: ctx_resize_unit_deployed
      notify:
        - reload systemd
        - restart router

    - name: "[ctx-resize] Flush handlers (daemon-reload + router restart if changed)"
      ansible.builtin.meta: flush_handlers

    # ==========================================================================
    # PHASE 2: Verify
    # ==========================================================================

    - name: "[ctx-resize] Wait for /health"
      ansible.builtin.uri:
        url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health"
        status_code: 200
        timeout: 30
      retries: 12
      delay: 5
      register: ctx_resize_health
      until: ctx_resize_health.status == 200

    - name: "[ctx-resize] Query /v1/models"
      ansible.builtin.uri:
        url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models"
        status_code: 200
        return_content: true
        timeout: 30
      register: ctx_resize_models

    - name: "[ctx-resize] Trigger load — Llama (confirms actual load + captures live args)"
      ansible.builtin.uri:
        url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions"
        method: POST
        body_format: json
        body:
          model: "Meta-Llama-3.1-8B-Instruct-Q4_K_M"
          messages:
            - role: user
              content: "Reply with one word: hello"
          max_tokens: 5
          temperature: 0.0
        status_code: 200
        return_content: true
        timeout: 120
      register: ctx_resize_llama_warmup

    - name: "[ctx-resize] Trigger load — Phi (confirms actual load + captures live args)"
      ansible.builtin.uri:
        url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions"
        method: POST
        body_format: json
        body:
          model: "Phi-3.5-mini-instruct-Q8_0"
          messages:
            - role: user
              content: "Reply with one word: hello"
          max_tokens: 5
          temperature: 0.0
        status_code: 200
        return_content: true
        timeout: 120
      register: ctx_resize_phi_warmup

    - name: "[ctx-resize] Re-query /v1/models after warmup (final state)"
      ansible.builtin.uri:
        url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models"
        status_code: 200
        return_content: true
        timeout: 30
      register: ctx_resize_models_final

    - name: "[ctx-resize] Extract Llama args"
      ansible.builtin.set_fact:
        ctx_resize_llama_args: >-
          {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Meta-Llama-3.1-8B-Instruct-Q4_K_M') | first).status.args }}
        ctx_resize_llama_status: >-
          {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Meta-Llama-3.1-8B-Instruct-Q4_K_M') | first).status.value }}

    - name: "[ctx-resize] Extract Phi args"
      ansible.builtin.set_fact:
        ctx_resize_phi_args: >-
          {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Phi-3.5-mini-instruct-Q8_0') | first).status.args }}
        ctx_resize_phi_status: >-
          {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Phi-3.5-mini-instruct-Q8_0') | first).status.value }}

    - name: "[ctx-resize] Extract Qwen args (must be unchanged)"
      ansible.builtin.set_fact:
        ctx_resize_qwen_args: >-
          {{ (ctx_resize_models_final.json.data | selectattr('id', 'equalto', 'Qwen3.6-35B-A3B-UD-Q4_K_S') | first).status.args }}

    - name: "[ctx-resize] GATE — Llama ctx-size must be 8192"
      ansible.builtin.assert:
        that:
          - "'8192' in ctx_resize_llama_args"
          - ctx_resize_llama_args[ctx_resize_llama_args.index('--ctx-size') + 1] == '8192'
        fail_msg: "Llama ctx-size not 8192. Args: {{ ctx_resize_llama_args }}"
        success_msg: "Llama ctx-size confirmed 8192."

    - name: "[ctx-resize] GATE — Llama flash-attn must be true"
      ansible.builtin.assert:
        that:
          - ctx_resize_llama_args[ctx_resize_llama_args.index('--flash-attn') + 1] == 'true'
        fail_msg: "Llama flash-attn not true. Args: {{ ctx_resize_llama_args }}"
        success_msg: "Llama flash-attn confirmed true."

    - name: "[ctx-resize] GATE — Llama loaded successfully"
      ansible.builtin.assert:
        that:
          - ctx_resize_llama_status == 'loaded'
        fail_msg: "Llama status is '{{ ctx_resize_llama_status }}', expected 'loaded'."
        success_msg: "Llama status confirmed 'loaded'."

    - name: "[ctx-resize] GATE — Phi ctx-size must be 32768"
      ansible.builtin.assert:
        that:
          - ctx_resize_phi_args[ctx_resize_phi_args.index('--ctx-size') + 1] == '32768'
        fail_msg: "Phi ctx-size not 32768. Args: {{ ctx_resize_phi_args }}"
        success_msg: "Phi ctx-size confirmed 32768."

    - name: "[ctx-resize] GATE — Phi flash-attn must be true"
      ansible.builtin.assert:
        that:
          - ctx_resize_phi_args[ctx_resize_phi_args.index('--flash-attn') + 1] == 'true'
        fail_msg: "Phi flash-attn not true. Args: {{ ctx_resize_phi_args }}"
        success_msg: "Phi flash-attn confirmed true."

    - name: "[ctx-resize] GATE — Phi loaded successfully"
      ansible.builtin.assert:
        that:
          - ctx_resize_phi_status == 'loaded'
        fail_msg: "Phi status is '{{ ctx_resize_phi_status }}', expected 'loaded'."
        success_msg: "Phi status confirmed 'loaded'."

    - name: "[ctx-resize] GATE — Qwen ctx-size UNCHANGED at 65536"
      ansible.builtin.assert:
        that:
          - ctx_resize_qwen_args[ctx_resize_qwen_args.index('--ctx-size') + 1] == '65536'
        fail_msg: "Qwen ctx-size changed unexpectedly! Args: {{ ctx_resize_qwen_args }}"
        success_msg: "Qwen ctx-size confirmed UNCHANGED at 65536."

    - name: "[ctx-resize] PASS — summary"
      ansible.builtin.debug:
        msg:
          - "================================================================"
          - "PER-MODEL CTX-SIZE DEPLOYMENT — COMPLETE"
          - ""
          - "  Llama-3.1-8B (Meta-Llama-3.1-8B-Instruct-Q4_K_M):"
          - "    status:     {{ ctx_resize_llama_status }}"
          - "    args:       {{ ctx_resize_llama_args }}"
          - ""
          - "  Phi-3.5-mini (Phi-3.5-mini-instruct-Q8_0):"
          - "    status:     {{ ctx_resize_phi_status }}"
          - "    args:       {{ ctx_resize_phi_args }}"
          - ""
          - "  Qwen3.6-35B-A3B-UD-Q4_K_S: UNCHANGED (ctx-size 65536, args: {{ ctx_resize_qwen_args }})"
          - ""
          - "  ACTION NEEDED: /home/hermes/.hermes/config.yaml custom_providers"
          - "  context_length: 65536 for both Meta-Llama-3.1-8B-Instruct-4bit and"
          - "  Phi-3.5-mini-instruct-8bit is now STALE (actual: 8192 / 32768)."
          - "  Flag to JARVIS/Maria Hill for correction — NOT done by this playbook."
          - "================================================================"