--- # ------------------------------------------------------------------------------ # FILE: playbooks/day2_cutover_qwen_to_router.yml # DESCRIPTION: Promote llama-server-router to production on port 8002. # # Context: Router-mode shadow deployment (t_0cca74a2) validated 2026-08-12: # all 4 hard gates PASSED (n_ctx 65536, tool-calling PASS, hallucination-stress # PASS, VRAM 20410 MiB / 1 process). Ryan approved cutover. # # This playbook makes the router the permanent production endpoint: # # 1. Stop + disable llama-server-qwen (:8002). Unit file is PRESERVED on disk # as the rollback target (same pattern as prior role history). # 2. Redeploy llama-server-router unit file with --port 8002 (production port). # PORT DECISION: we rebind the router to :8002 rather than updating 8 # dependent Hermes profiles' base_url. One unit file change beats 8 # config.yaml updates — atomic, GitOps-clean, zero profile drift. # 3. Enable + start llama-server-router on :8002. # 4. Re-run validation gates 1-3 against the NOW-production endpoint. # (Same logic as Phase R / router_verify in tasks/router.yml — hard gates.) # 5. Run Gate 4: verify bundled SvelteKit UI is reachable. # # Usage (from ~/git/homelab/ansible): # ansible-playbook -i inventory.yml playbooks/day2_cutover_qwen_to_router.yml # # Rollback (if gates fail or any time after): # ansible-playbook -i inventory.yml playbooks/day2_cutover_qwen_to_router.yml \ # --tags cutover_rollback # # Author: War Machine (2026-08-12, t_cd0d5388) # Approved by: Ryan (cutover authorization, 2026-08-12) # ------------------------------------------------------------------------------ - name: "CUTOVER — Promote llama-server-router to production (:8002) on astro-orbiter" hosts: astro_orbiter gather_facts: true become: true vars: # ---------------------------------------------------------------- # PORT DECISION: # We rebind the router to :8002 (production port) rather than # updating 8 dependent Hermes profiles' base_url to :8003. # Rationale: one unit file change is atomic and GitOps-clean. # Updating 8 config.yaml files risks drift and requires per-profile # activation tests. The template renders llm_router_port as the # --port argument; we just override it here to 8002. # ---------------------------------------------------------------- # Router port override: take over production port llm_router_port: 8002 # All other role defaults needed by the template (mirrors defaults/main.yml) llm_service_user: jarvis llm_binary_path: /opt/llama.cpp/build/bin/llama-server llm_models_dir: /opt/models llm_bind_address: "10.1.71.130" llm_allowed_source_cidr: "10.1.70.0/24" llm_router_enabled: true llm_router_service_name: llama-server-router llm_router_models_dir: /opt/models llm_router_models_max: 1 # CRITICAL: RTX 3090 24GB, single model only llm_router_ctx_size: 65536 llm_router_parallel: 1 llm_router_gpu_layers: 99 llm_router_batch_size: 2048 llm_router_ubatch_size: 512 llm_router_cache_type_k: q4_0 llm_router_cache_type_v: q4_0 llm_router_flash_attn: "auto" llm_router_bind_address: "10.1.71.130" llm_router_allowed_source_cidr: "10.1.70.0/24" llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" llm_router_vram_max_mib: 23000 llm_qwen_service_name: llama-server-qwen llm_qwen_port: 8002 tasks: # ======================================================================= # PHASE 1 — Stop and disable llama-server-qwen (bare single-model) # Preserve unit file on disk — rollback target per existing role pattern. # ======================================================================= - name: "[cutover] PHASE 1: Confirm llama-server-qwen current state" ansible.builtin.systemd: name: llama-server-qwen register: cutover_qwen_status tags: [cutover_stop_qwen, cutover] - name: "[cutover] PHASE 1: Report current llama-server-qwen status" ansible.builtin.debug: msg: >- llama-server-qwen: ActiveState={{ cutover_qwen_status.status.ActiveState | default('unknown') }}, UnitFileState={{ cutover_qwen_status.status.UnitFileState | default('unknown') }}. Will stop + disable. Unit file preserved at /etc/systemd/system/llama-server-qwen.service as rollback target. tags: [cutover_stop_qwen, cutover] - name: "[cutover] PHASE 1: Stop llama-server-qwen (:8002, bare single-model)" ansible.builtin.systemd: name: llama-server-qwen state: stopped register: cutover_qwen_stopped tags: [cutover_stop_qwen, cutover] - name: "[cutover] PHASE 1: Disable llama-server-qwen (prevent auto-start on reboot)" ansible.builtin.systemd: name: llama-server-qwen enabled: false tags: [cutover_stop_qwen, cutover] - name: "[cutover] PHASE 1: Wait 5s for VRAM to be released" ansible.builtin.pause: seconds: 5 when: cutover_qwen_stopped.changed | default(false) tags: [cutover_stop_qwen, cutover] - name: "[cutover] PHASE 1: Verify port 8002 is now free" ansible.builtin.command: cmd: ss -ltnp register: cutover_port_check changed_when: false tags: [cutover_stop_qwen, cutover] - name: "[cutover] PHASE 1: Fail if port 8002 is still bound" ansible.builtin.fail: msg: >- Port 8002 is still bound after stopping llama-server-qwen. Check 'ss -ltnp | grep :8002' and resolve before the router can bind. when: - "':8002 ' in (cutover_port_check.stdout | default('')) or ':8002:' in (cutover_port_check.stdout | default(''))" tags: [cutover_stop_qwen, cutover] - name: "[cutover] PHASE 1: Report VRAM state (should be empty)" ansible.builtin.command: cmd: nvidia-smi --query-compute-apps=pid,name,used_memory --format=csv,noheader register: cutover_vram_free_check changed_when: false tags: [cutover_stop_qwen, cutover] - name: "[cutover] PHASE 1: Print VRAM state" ansible.builtin.debug: msg: >- VRAM after stopping llama-server-qwen: {{ cutover_vram_free_check.stdout if (cutover_vram_free_check.stdout | length > 0) else '(no GPU processes — VRAM free)' }} tags: [cutover_stop_qwen, cutover] # ======================================================================= # PHASE 2 — Redeploy llama-server-router unit with --port 8002 # ======================================================================= - name: "[cutover] PHASE 2: Deploy llama-server-router unit file (port 8002 — production)" ansible.builtin.template: src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2" dest: /etc/systemd/system/llama-server-router.service owner: root group: root mode: "0644" register: cutover_router_unit_deployed notify: - reload systemd tags: [cutover_deploy_unit, cutover] - name: "[cutover] PHASE 2: Flush handlers (daemon-reload before start)" ansible.builtin.meta: flush_handlers tags: [cutover_deploy_unit, cutover] # ======================================================================= # PHASE 3 — Enable + start llama-server-router on :8002 # ======================================================================= - name: "[cutover] PHASE 3: Enable + start llama-server-router (production, :8002)" ansible.builtin.systemd: name: llama-server-router state: "{{ 'restarted' if (cutover_router_unit_deployed.changed | default(false)) else 'started' }}" enabled: true daemon_reload: true tags: [cutover_start_router, cutover] # ======================================================================= # PHASE 4 — Validation gates 1-3 (hard gates against now-production :8002) # ======================================================================= - name: "[cutover] GATE 1a: Wait for router /health on :8002 (up to 5min — cold model load)" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health" status_code: 200 register: cutover_health retries: 30 delay: 10 until: cutover_health.status == 200 tags: [cutover_validate, cutover] - name: "[cutover] GATE 1a: Trigger model load (router lazy-loads on first request)" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions" method: POST body_format: json body: model: "{{ llm_router_expected_model_id }}" messages: - role: user content: "Reply with one word: hello" max_tokens: 5 temperature: 0.0 status_code: 200 return_content: true timeout: 300 register: cutover_warmup tags: [cutover_validate, cutover] - name: "[cutover] GATE 1a: Report warmup" ansible.builtin.debug: msg: - "Model loaded. finish_reason={{ cutover_warmup.json.choices[0].finish_reason | default('unknown') }}" - "Response: {{ cutover_warmup.json.choices[0].message.content | default('(empty)') | truncate(100) }}" tags: [cutover_validate, cutover] - name: "[cutover] GATE 1b: Query /v1/models on :8002" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models" status_code: 200 return_content: true register: cutover_models tags: [cutover_validate, cutover] - name: "[cutover] GATE 1b: Fail if expected model ID not found" ansible.builtin.fail: msg: >- GATE 1 FAIL: '{{ llm_router_expected_model_id }}' not found in /v1/models. Returned: {{ cutover_models.json.data | map(attribute='id') | list }} when: - cutover_models.json.data | selectattr('id', 'equalto', llm_router_expected_model_id) | list | length == 0 tags: [cutover_validate, cutover] - name: "[cutover] GATE 1b: Extract ctx-size from router model args" ansible.builtin.set_fact: cutover_qwen_n_ctx: >- {%- set model = cutover_models.json.data | selectattr('id', 'equalto', llm_router_expected_model_id) | first -%} {%- set args = model.status.args -%} {%- set ctx_idx = args.index('--ctx-size') if '--ctx-size' in args else -1 -%} {{ args[ctx_idx + 1] | int if ctx_idx >= 0 else 0 }} when: - cutover_models.json.data | selectattr('id', 'equalto', llm_router_expected_model_id) | list | length > 0 tags: [cutover_validate, cutover] - name: "[cutover] GATE 1b: Fail if n_ctx < 64000" ansible.builtin.fail: msg: "GATE 1 FAIL: --ctx-size={{ cutover_qwen_n_ctx }} < 64000 (Hermes 64K floor)." when: - cutover_qwen_n_ctx is defined - cutover_qwen_n_ctx | int < 64000 tags: [cutover_validate, cutover] - name: "[cutover] GATE 1b: PASS — n_ctx >= 64K" ansible.builtin.debug: msg: "GATE 1 PASS: --ctx-size={{ cutover_qwen_n_ctx }} >= 64000." when: - cutover_qwen_n_ctx is defined - cutover_qwen_n_ctx | int >= 64000 tags: [cutover_validate, cutover] # --- Gate 2: Tool-calling through router proxy --- - name: "[cutover] GATE 2: Tool-calling probe" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions" method: POST body_format: json body: model: "{{ llm_router_expected_model_id }}" messages: - role: user content: "What is the current weather in Chicago? Use the provided tool." tools: - type: function function: name: get_weather description: "Get current weather conditions for a city" parameters: type: object properties: city: type: string description: "The city name" required: - city temperature: 0.0 status_code: 200 return_content: true timeout: 120 register: cutover_toolcall_probe tags: [cutover_validate, cutover] - name: "[cutover] GATE 2: Fail if not finish_reason=tool_calls" ansible.builtin.fail: msg: >- GATE 2 FAIL: finish_reason={{ cutover_toolcall_probe.json.choices[0].finish_reason | default('(missing)') }} (expected tool_calls). Response: {{ cutover_toolcall_probe.json | to_json }} when: - cutover_toolcall_probe.json.choices[0].finish_reason | default('') != 'tool_calls' tags: [cutover_validate, cutover] - name: "[cutover] GATE 2: PASS" ansible.builtin.debug: msg: - "GATE 2 PASS: finish_reason=tool_calls" - "function: {{ cutover_toolcall_probe.json.choices[0].message.tool_calls[0].function.name | default('(unknown)') }}" - "arguments: {{ cutover_toolcall_probe.json.choices[0].message.tool_calls[0].function.arguments | default('(none)') }}" when: - cutover_toolcall_probe.json.choices[0].finish_reason | default('') == 'tool_calls' tags: [cutover_validate, cutover] # --- Gate 2b: Hallucination stress --- - name: "[cutover] GATE 2b: Hallucination stress probe" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions" method: POST body_format: json body: model: "{{ llm_router_expected_model_id }}" messages: - role: user content: "Tell me a brief fact about the planet Mars. Do not call any functions." tools: - type: function function: name: get_weather description: "Get current weather conditions for a city" parameters: type: object properties: city: type: string required: - city temperature: 0.1 status_code: 200 return_content: true timeout: 120 register: cutover_halluc_probe tags: [cutover_validate, cutover] - name: "[cutover] GATE 2b: Fail if spurious tool_calls" ansible.builtin.fail: msg: >- GATE 2b FAIL: finish_reason=tool_calls on unrelated prompt (Mars fact). Over-triggering through router. Response: {{ cutover_halluc_probe.json | to_json }} when: - cutover_halluc_probe.json.choices[0].finish_reason | default('') == 'tool_calls' tags: [cutover_validate, cutover] - name: "[cutover] GATE 2b: PASS" ansible.builtin.debug: msg: "GATE 2b PASS: finish_reason={{ cutover_halluc_probe.json.choices[0].finish_reason }} — no spurious tool_calls." when: - cutover_halluc_probe.json.choices[0].finish_reason | default('') != 'tool_calls' tags: [cutover_validate, cutover] # --- Gate 3: VRAM guard --- - name: "[cutover] GATE 3: Check VRAM usage (--models-max 1 guard)" ansible.builtin.command: cmd: nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu --format=csv,noheader register: cutover_vram_post changed_when: false tags: [cutover_validate, cutover] - name: "[cutover] GATE 3: Parse VRAM used MiB" ansible.builtin.set_fact: cutover_vram_used_mib: "{{ cutover_vram_post.stdout.split(',')[0].strip().split(' ')[0] | int }}" tags: [cutover_validate, cutover] - name: "[cutover] GATE 3: Fail if VRAM exceeds ceiling" ansible.builtin.fail: msg: >- GATE 3 FAIL: {{ cutover_vram_used_mib }} MiB > {{ llm_router_vram_max_mib }} MiB ceiling. Full: {{ cutover_vram_post.stdout }} when: - cutover_vram_used_mib | int > llm_router_vram_max_mib | int tags: [cutover_validate, cutover] - name: "[cutover] GATE 3: Count GPU processes" ansible.builtin.command: cmd: nvidia-smi --query-compute-apps=pid,name --format=csv,noheader register: cutover_gpu_procs changed_when: false failed_when: false tags: [cutover_validate, cutover] - name: "[cutover] GATE 3: PASS" ansible.builtin.debug: msg: - "GATE 3 PASS: {{ cutover_vram_used_mib }} MiB / {{ llm_router_vram_max_mib }} MiB ceiling." - "GPU processes: {{ cutover_gpu_procs.stdout_lines | default(['(none)']) }}" - "Full nvidia-smi: {{ cutover_vram_post.stdout }}" when: - cutover_vram_used_mib | int <= llm_router_vram_max_mib | int tags: [cutover_validate, cutover] # ======================================================================= # PHASE 5 — Gate 4: Bundled SvelteKit Web UI (required this time) # ======================================================================= - name: "[cutover] GATE 4: Check bundled SvelteKit UI at :8002" ansible.builtin.uri: url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/" status_code: [200, 301, 302] return_content: true timeout: 30 register: cutover_ui_check failed_when: false tags: [cutover_validate, cutover_ui, cutover] - name: "[cutover] GATE 4: Inspect UI content" ansible.builtin.set_fact: cutover_ui_is_html: "{{ 'html' in (cutover_ui_check.content | default('') | lower) or '= 64K): PASS ({{ cutover_qwen_n_ctx | default('N/A') }})" - " Gate 2 (tool-calling): PASS (finish_reason=tool_calls)" - " Gate 2b (halluc stress): PASS (no spurious tool_calls)" - " Gate 3 (VRAM <= 23000MiB): PASS ({{ cutover_vram_used_mib | default('N/A') }} MiB)" - " Gate 4 (Web UI): HTTP {{ cutover_ui_check.status | default('N/A') }}" - "" - " ROLLBACK TARGET: /etc/systemd/system/llama-server-qwen.service (unit preserved)" - " ROLLBACK CMD: sudo systemctl enable --now llama-server-qwen" - " sudo systemctl disable --now llama-server-router" - " Or: ansible-playbook -i inventory.yml day2_cutover_qwen_to_router.yml --tags cutover_rollback" - "" - " Web UI bookmark: http://{{ llm_router_bind_address }}:{{ llm_router_port }}/" - "======================================================================" tags: [cutover] # ======================================================================= # ROLLBACK — tag cutover_rollback reverses the cutover # Run: ansible-playbook -i inventory.yml day2_cutover_qwen_to_router.yml --tags cutover_rollback # WARNING: rollback_task has no dependency on cutover tags — safe to run standalone. # ======================================================================= - name: "[cutover_rollback] Stop + disable llama-server-router" ansible.builtin.systemd: name: llama-server-router state: stopped enabled: false tags: [cutover_rollback, never] # 'never' = only runs with explicit --tags cutover_rollback - name: "[cutover_rollback] Enable + start llama-server-qwen (restore bare :8002)" ansible.builtin.systemd: name: llama-server-qwen state: started enabled: true tags: [cutover_rollback, never] - name: "[cutover_rollback] Verify rollback /health" ansible.builtin.uri: url: "http://{{ llm_bind_address | default('10.1.71.130') }}:8002/health" status_code: 200 timeout: 30 register: cutover_rollback_health failed_when: false tags: [cutover_rollback, never] - name: "[cutover_rollback] Report rollback result" ansible.builtin.debug: msg: >- ROLLBACK: llama-server-qwen :8002 health returned {{ cutover_rollback_health.status | default('UNREACHABLE') }}. {{ 'OK — production restored to bare qwen.' if (cutover_rollback_health.status | default(0) | int == 200) else 'WARNING — health check failed. Check manually.' }} tags: [cutover_rollback, never] handlers: - name: reload systemd ansible.builtin.systemd: daemon_reload: true