feat(llm-router): switch to --models-preset mode; add Phi-3.5-mini-instruct-8bit alias (t_9adf0889)
- Add templates/llama-server-router-preset.ini.j2: defines all 3 router GGUFs (Qwen3.6-35B, Phi-3.5-mini-Q8_0, Meta-Llama-3.1-8B) with explicit ctx-size, gpu-layers, cache settings carried over from --models-dir baseline. The Phi entry adds alias=Phi-3.5-mini-instruct-8bit (Ryan's aux.title_generation target). - Update templates/llama-server-router.service.j2: Jinja2 conditional emits --models-preset <path> when llm_router_preset_enabled=true, otherwise --models-dir (backward compat, default unchanged). - Add tasks/preset.yml: deploy preset INI, restart router on change, verify both Phi-3.5-mini-instruct-Q8_0 (primary) and Phi-3.5-mini-instruct-8bit (alias) appear in /v1/models, plus Qwen and Llama IDs unchanged. - Update defaults/main.yml: add llm_router_preset_enabled (default false) and llm_router_preset_path=/opt/llama-server-router-preset.ini. - Update tasks/main.yml: import preset.yml as Phase P (gated, no-op by default). - Update handlers/main.yml: add 'restart router' handler for preset changes. - Add playbooks/day2_add_phi_alias.yml: single-command deployment. GH #22364 note: --models-preset mode creates an extra 'default' entry in /v1/models — cosmetic, does not affect model selection by name.
This commit is contained in:
Hermes Agent service account
203
ansible/playbooks/day2_add_phi_alias.yml
Normal file
203
ansible/playbooks/day2_add_phi_alias.yml
Normal file
@@ -0,0 +1,203 @@
|
||||
---
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: playbooks/day2_add_phi_alias.yml
|
||||
# DESCRIPTION: Add Phi-3.5-mini-instruct-8bit alias to the llama-server-router
|
||||
# by switching from --models-dir to --models-preset INI mode.
|
||||
#
|
||||
# Context (t_9adf0889, 2026-08-12):
|
||||
# Ryan's Hermes config (auxiliary.title_generation.model) points to
|
||||
# "Phi-3.5-mini-instruct-8bit" but the router only exposes the GGUF
|
||||
# filename-derived ID "Phi-3.5-mini-instruct-Q8_0". They are the same file.
|
||||
# This playbook adds the alias so both names work without changing Ryan's
|
||||
# Hermes config.
|
||||
#
|
||||
# What this playbook does:
|
||||
# 1. Deploys the preset INI template (llama-server-router-preset.ini.j2)
|
||||
# to /opt/llama-server-router-preset.ini on astro-orbiter.
|
||||
# 2. Redeploys the systemd unit (llama-server-router.service) with
|
||||
# --models-preset instead of --models-dir.
|
||||
# 3. Restarts llama-server-router to pick up the new flag.
|
||||
# 4. Verifies that /v1/models returns:
|
||||
# - Phi-3.5-mini-instruct-Q8_0 (original ID — must still work)
|
||||
# - Phi-3.5-mini-instruct-8bit (new alias — Ryan's config target)
|
||||
# - Qwen3.6-35B-A3B-UD-Q4_K_S (unchanged)
|
||||
# - Meta-Llama-3.1-8B-Instruct-Q4_K_M (unchanged)
|
||||
#
|
||||
# Known upstream behavior:
|
||||
# GH #22364: --models-preset creates an extra "default" entry in /v1/models.
|
||||
# This is cosmetic only and does not affect model selection by name.
|
||||
#
|
||||
# Usage (from ~/git/homelab/ansible):
|
||||
# ansible-playbook -i inventory.yml playbooks/day2_add_phi_alias.yml
|
||||
#
|
||||
# Semaphore note (t_9adf0889): Semaphore SSH key for jarvis user is not loaded
|
||||
# in the container (known pitfall, homelab-llm-serving skill). Run via CLI with
|
||||
# id_jarvis key; document as exception per Ryan's standing CLI fallback directive.
|
||||
#
|
||||
# Author: War Machine (2026-08-12, t_9adf0889)
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
- name: "Add Phi-3.5-mini-instruct-8bit alias — switch router to preset mode"
|
||||
hosts: astro_orbiter
|
||||
gather_facts: false
|
||||
become: true
|
||||
|
||||
vars:
|
||||
# Activate preset mode and provide the on-disk INI path
|
||||
llm_router_preset_enabled: true
|
||||
llm_router_preset_path: /opt/llama-server-router-preset.ini
|
||||
|
||||
# Production port (router is already on 8002 since t_cd0d5388)
|
||||
llm_router_port: 8002
|
||||
|
||||
# All other vars inherit from host_vars + defaults/main.yml.
|
||||
# Explicitly set the ones needed by the unit template for clarity:
|
||||
llm_router_enabled: true
|
||||
llm_service_user: jarvis
|
||||
llm_binary_path: /opt/llama.cpp/build/bin/llama-server
|
||||
llm_models_dir: /opt/models
|
||||
llm_bind_address: "10.1.71.130"
|
||||
llm_allowed_source_cidr: "10.1.70.0/24"
|
||||
llm_router_service_name: llama-server-router
|
||||
llm_router_bind_address: "10.1.71.130"
|
||||
llm_router_allowed_source_cidr: "10.1.70.0/24"
|
||||
llm_router_models_dir: /opt/models
|
||||
llm_router_models_max: 4 # from host_vars; bumped by t_33acbb2e
|
||||
llm_router_ctx_size: 65536
|
||||
llm_router_parallel: 1
|
||||
llm_router_gpu_layers: 99
|
||||
llm_router_batch_size: 2048
|
||||
llm_router_ubatch_size: 512
|
||||
llm_router_cache_type_k: q4_0
|
||||
llm_router_cache_type_v: q4_0
|
||||
llm_router_flash_attn: "auto"
|
||||
llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S"
|
||||
llm_router_vram_max_mib: 23000
|
||||
|
||||
handlers:
|
||||
- name: reload systemd
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
become: true
|
||||
listen: "reload systemd"
|
||||
|
||||
- name: restart router
|
||||
ansible.builtin.systemd:
|
||||
name: llama-server-router
|
||||
state: restarted
|
||||
become: true
|
||||
listen: "restart router"
|
||||
|
||||
tasks:
|
||||
|
||||
# ==========================================================================
|
||||
# PHASE 1: Deploy the preset INI
|
||||
# ==========================================================================
|
||||
|
||||
- name: "[phi-alias] Deploy preset INI to {{ llm_router_preset_path }}"
|
||||
ansible.builtin.template:
|
||||
src: "../roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2"
|
||||
dest: "{{ llm_router_preset_path }}"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: phi_alias_preset_deployed
|
||||
notify:
|
||||
- restart router
|
||||
|
||||
# ==========================================================================
|
||||
# PHASE 2: Redeploy systemd unit with --models-preset flag
|
||||
# ==========================================================================
|
||||
|
||||
- name: "[phi-alias] Deploy llama-server-router unit (--models-preset mode)"
|
||||
ansible.builtin.template:
|
||||
src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2"
|
||||
dest: /etc/systemd/system/llama-server-router.service
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: phi_alias_unit_deployed
|
||||
notify:
|
||||
- reload systemd
|
||||
- restart router
|
||||
|
||||
- name: "[phi-alias] Flush handlers (daemon-reload + router restart)"
|
||||
ansible.builtin.meta: flush_handlers
|
||||
|
||||
# ==========================================================================
|
||||
# PHASE 3: Verify alias is present
|
||||
# ==========================================================================
|
||||
|
||||
- name: "[phi-alias] Wait for /health (router supervisor, no model needed)"
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health"
|
||||
status_code: 200
|
||||
timeout: 30
|
||||
retries: 12
|
||||
delay: 5
|
||||
register: phi_alias_health
|
||||
until: phi_alias_health.status == 200
|
||||
|
||||
- name: "[phi-alias] Query /v1/models"
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models"
|
||||
status_code: 200
|
||||
return_content: true
|
||||
timeout: 30
|
||||
register: phi_alias_models
|
||||
|
||||
- name: "[phi-alias] Extract model IDs and aliases"
|
||||
ansible.builtin.set_fact:
|
||||
phi_alias_model_ids: "{{ phi_alias_models.json.data | map(attribute='id') | list }}"
|
||||
phi_alias_all_aliases: "{{ phi_alias_models.json.data | map(attribute='aliases') | flatten | list }}"
|
||||
phi_alias_model_sources: "{{ phi_alias_models.json.data | map(attribute='source') | list }}"
|
||||
|
||||
- name: "[phi-alias] FAIL if Phi original ID missing"
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
'Phi-3.5-mini-instruct-Q8_0' not in /v1/models.
|
||||
IDs: {{ phi_alias_model_ids }}
|
||||
when: "'Phi-3.5-mini-instruct-Q8_0' not in phi_alias_model_ids"
|
||||
|
||||
- name: "[phi-alias] FAIL if Phi alias missing"
|
||||
ansible.builtin.fail:
|
||||
msg: >-
|
||||
'Phi-3.5-mini-instruct-8bit' not found as ID or alias in /v1/models.
|
||||
IDs: {{ phi_alias_model_ids }}
|
||||
Aliases: {{ phi_alias_all_aliases }}
|
||||
when:
|
||||
- "'Phi-3.5-mini-instruct-8bit' not in phi_alias_model_ids"
|
||||
- "'Phi-3.5-mini-instruct-8bit' not in phi_alias_all_aliases"
|
||||
|
||||
- name: "[phi-alias] FAIL if Qwen missing"
|
||||
ansible.builtin.fail:
|
||||
msg: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in /v1/models. IDs: {{ phi_alias_model_ids }}"
|
||||
when: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in phi_alias_model_ids"
|
||||
|
||||
- name: "[phi-alias] FAIL if Llama missing"
|
||||
ansible.builtin.fail:
|
||||
msg: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ phi_alias_model_ids }}"
|
||||
when: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in phi_alias_model_ids"
|
||||
|
||||
- name: "[phi-alias] PASS — full /v1/models summary"
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "========================================================================"
|
||||
- "PHI ALIAS DEPLOYMENT — COMPLETE"
|
||||
- ""
|
||||
- " Mode: --models-preset ({{ llm_router_preset_path }})"
|
||||
- " Service: llama-server-router.service (:{{ llm_router_port }})"
|
||||
- ""
|
||||
- " /v1/models IDs: {{ phi_alias_model_ids }}"
|
||||
- " /v1/models aliases: {{ phi_alias_all_aliases }}"
|
||||
- " Sources: {{ phi_alias_model_sources }}"
|
||||
- ""
|
||||
- " VERIFY:"
|
||||
- " Phi-3.5-mini-instruct-Q8_0: {{ 'PRESENT' if 'Phi-3.5-mini-instruct-Q8_0' in phi_alias_model_ids else 'MISSING' }}"
|
||||
- " Phi-3.5-mini-instruct-8bit: {{ 'PRESENT (ID)' if 'Phi-3.5-mini-instruct-8bit' in phi_alias_model_ids else ('PRESENT (alias)' if 'Phi-3.5-mini-instruct-8bit' in phi_alias_all_aliases else 'MISSING') }}"
|
||||
- " Qwen3.6-35B-A3B-UD-Q4_K_S: {{ 'PRESENT' if 'Qwen3.6-35B-A3B-UD-Q4_K_S' in phi_alias_model_ids else 'MISSING' }}"
|
||||
- " Meta-Llama-3.1-8B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' in phi_alias_model_ids else 'MISSING' }}"
|
||||
- ""
|
||||
- " GH #22364: if 'default' appears in IDs above, that is expected"
|
||||
- " in --models-preset mode. Cosmetic only."
|
||||
- "========================================================================"
|
||||
Reference in New Issue
Block a user