rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
3dc58cf644e1b9180e5ac8b279545abc42760e3f
homelab/ansible/roles/llm-inference-multimodel/tasks/preset.yml
Hermes Agent service account a47b29d49f feat(llm-router): switch to --models-preset mode; add Phi-3.5-mini-instruct-8bit alias (t_9adf0889) 
- Add templates/llama-server-router-preset.ini.j2: defines all 3 router GGUFs
  (Qwen3.6-35B, Phi-3.5-mini-Q8_0, Meta-Llama-3.1-8B) with explicit ctx-size,
  gpu-layers, cache settings carried over from --models-dir baseline. The Phi
  entry adds alias=Phi-3.5-mini-instruct-8bit (Ryan's aux.title_generation target).

- Update templates/llama-server-router.service.j2: Jinja2 conditional emits
  --models-preset <path> when llm_router_preset_enabled=true, otherwise
  --models-dir (backward compat, default unchanged).

- Add tasks/preset.yml: deploy preset INI, restart router on change, verify
  both Phi-3.5-mini-instruct-Q8_0 (primary) and Phi-3.5-mini-instruct-8bit
  (alias) appear in /v1/models, plus Qwen and Llama IDs unchanged.

- Update defaults/main.yml: add llm_router_preset_enabled (default false) and
  llm_router_preset_path=/opt/llama-server-router-preset.ini.

- Update tasks/main.yml: import preset.yml as Phase P (gated, no-op by default).
- Update handlers/main.yml: add 'restart router' handler for preset changes.
- Add playbooks/day2_add_phi_alias.yml: single-command deployment.

GH #22364 note: --models-preset mode creates an extra 'default' entry in
/v1/models — cosmetic, does not affect model selection by name.
2026-08-12 22:58:12 -05:00

162 lines
6.9 KiB
YAML
Raw Blame History

---
# ------------------------------------------------------------------------------
# FILE: roles/llm-inference-multimodel/tasks/preset.yml
# DESCRIPTION: Phase P — deploy --models-preset INI file and switch the router
# from --models-dir to --models-preset mode.
#
# Purpose: preset mode is required to add model aliases. The alias
# field in the INI lets the Phi-3.5-mini-instruct-Q8_0 entry also
# respond to "Phi-3.5-mini-instruct-8bit" — Ryan's Hermes
# auxiliary.title_generation already uses this friendlier name.
#
# This phase is gated on llm_router_preset_enabled | default(false).
# With the default (false) it is a no-op. Flip to true in a playbook
# or extra-vars to activate.
#
# Tag: router_preset (deploy INI + restart service on change)
#
# Order: run AFTER router_systemd. The systemd unit template
# conditionally emits --models-preset when llm_router_preset_enabled
# is true; this phase deploys the INI the unit references.
#
# Added 2026-08-12 (t_9adf0889): Phi alias — War Machine.
# ------------------------------------------------------------------------------
# =============================================================================
# TAG: router_preset
# Deploy the preset INI and restart the router if the INI or unit changed.
# =============================================================================
- name: "[router_preset] Deploy preset INI to {{ llm_router_preset_path }}"
ansible.builtin.template:
src: llama-server-router-preset.ini.j2
dest: "{{ llm_router_preset_path }}"
owner: root
group: root
mode: "0644"
become: true
register: llm_router_preset_deployed
notify:
- restart router
when: llm_router_preset_enabled | default(false)
tags: [router_preset]
- name: "[router_preset] Report preset INI deployment result"
ansible.builtin.debug:
msg: >-
Preset INI {{ 'deployed (changed)' if llm_router_preset_deployed.changed | default(false) else 'already up-to-date (no change)' }}
at {{ llm_router_preset_path }}.
when: llm_router_preset_enabled | default(false)
tags: [router_preset]
- name: "[router_preset] Flush handlers so router restarts before verification"
ansible.builtin.meta: flush_handlers
when: llm_router_preset_enabled | default(false)
tags: [router_preset]
# =============================================================================
# Verification: confirm both the original ID and the alias appear in /v1/models
# =============================================================================
- name: "[router_preset] Wait for router /health after potential restart"
ansible.builtin.uri:
url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health"
status_code: 200
timeout: 30
retries: 12
delay: 5
register: llm_preset_health
until: llm_preset_health.status == 200
when: llm_router_preset_enabled | default(false)
tags: [router_preset]
- name: "[router_preset] Query /v1/models to verify preset loaded"
ansible.builtin.uri:
url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models"
status_code: 200
return_content: true
timeout: 30
register: llm_preset_models
when: llm_router_preset_enabled | default(false)
tags: [router_preset]
- name: "[router_preset] Extract model IDs and aliases from /v1/models"
ansible.builtin.set_fact:
llm_preset_model_ids: >-
{{ llm_preset_models.json.data | map(attribute='id') | list }}
llm_preset_model_aliases: >-
{{ llm_preset_models.json.data | map(attribute='aliases') | flatten | list }}
when:
- llm_router_preset_enabled | default(false)
- llm_preset_models is defined
tags: [router_preset]
- name: "[router_preset] VERIFY: Phi original ID (Phi-3.5-mini-instruct-Q8_0) present in /v1/models"
ansible.builtin.fail:
msg: >-
PRESET VERIFY FAIL: 'Phi-3.5-mini-instruct-Q8_0' not found in /v1/models IDs.
Returned IDs: {{ llm_preset_model_ids | default([]) }}
Check the preset INI section name.
when:
- llm_router_preset_enabled | default(false)
- llm_preset_model_ids is defined
- "'Phi-3.5-mini-instruct-Q8_0' not in llm_preset_model_ids"
tags: [router_preset]
- name: "[router_preset] VERIFY: Phi alias (Phi-3.5-mini-instruct-8bit) present in /v1/models aliases"
ansible.builtin.fail:
msg: >-
PRESET VERIFY FAIL: alias 'Phi-3.5-mini-instruct-8bit' not found in /v1/models.
Returned IDs: {{ llm_preset_model_ids | default([]) }}
Returned aliases: {{ llm_preset_model_aliases | default([]) }}
Check the 'alias' field in the preset INI for the [Phi-3.5-mini-instruct-Q8_0] section.
when:
- llm_router_preset_enabled | default(false)
- llm_preset_model_ids is defined
- "'Phi-3.5-mini-instruct-8bit' not in llm_preset_model_ids and 'Phi-3.5-mini-instruct-8bit' not in llm_preset_model_aliases"
tags: [router_preset]
- name: "[router_preset] VERIFY: Qwen model ID unchanged"
ansible.builtin.fail:
msg: >-
PRESET VERIFY FAIL: 'Qwen3.6-35B-A3B-UD-Q4_K_S' not found in /v1/models.
Returned IDs: {{ llm_preset_model_ids | default([]) }}
when:
- llm_router_preset_enabled | default(false)
- llm_preset_model_ids is defined
- "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in llm_preset_model_ids"
tags: [router_preset]
- name: "[router_preset] VERIFY: Llama model ID unchanged"
ansible.builtin.fail:
msg: >-
PRESET VERIFY FAIL: 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not found in /v1/models.
Returned IDs: {{ llm_preset_model_ids | default([]) }}
when:
- llm_router_preset_enabled | default(false)
- llm_preset_model_ids is defined
- "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in llm_preset_model_ids"
tags: [router_preset]
- name: "[router_preset] VERIFY PASS — preset mode active, all models and alias confirmed"
ansible.builtin.debug:
msg:
- "======================================================================"
- "PRESET VERIFY PASS: --models-preset mode active on llama-server-router."
- ""
- " /v1/models IDs: {{ llm_preset_model_ids | default([]) }}"
- " /v1/models aliases: {{ llm_preset_model_aliases | default([]) }}"
- ""
- " 'Phi-3.5-mini-instruct-Q8_0' => PRESENT (primary ID)"
- " 'Phi-3.5-mini-instruct-8bit' => PRESENT (alias — Ryan's title_generation target)"
- " 'Qwen3.6-35B-A3B-UD-Q4_K_S' => PRESENT"
- " 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' => PRESENT"
- ""
- " GH #22364 note: a 'default' model entry may also appear above."
- " This is an expected upstream behavior in preset mode — cosmetic only."
- "======================================================================"
when:
- llm_router_preset_enabled | default(false)
- llm_preset_model_ids is defined
tags: [router_preset]
Reference in New Issue View Git Blame Copy Permalink