diff --git a/ansible/playbooks/day2_add_phi_alias.yml b/ansible/playbooks/day2_add_phi_alias.yml new file mode 100644 index 0000000..0fc8def --- /dev/null +++ b/ansible/playbooks/day2_add_phi_alias.yml @@ -0,0 +1,203 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: playbooks/day2_add_phi_alias.yml +# DESCRIPTION: Add Phi-3.5-mini-instruct-8bit alias to the llama-server-router +# by switching from --models-dir to --models-preset INI mode. +# +# Context (t_9adf0889, 2026-08-12): +# Ryan's Hermes config (auxiliary.title_generation.model) points to +# "Phi-3.5-mini-instruct-8bit" but the router only exposes the GGUF +# filename-derived ID "Phi-3.5-mini-instruct-Q8_0". They are the same file. +# This playbook adds the alias so both names work without changing Ryan's +# Hermes config. +# +# What this playbook does: +# 1. Deploys the preset INI template (llama-server-router-preset.ini.j2) +# to /opt/llama-server-router-preset.ini on astro-orbiter. +# 2. Redeploys the systemd unit (llama-server-router.service) with +# --models-preset instead of --models-dir. +# 3. Restarts llama-server-router to pick up the new flag. +# 4. Verifies that /v1/models returns: +# - Phi-3.5-mini-instruct-Q8_0 (original ID — must still work) +# - Phi-3.5-mini-instruct-8bit (new alias — Ryan's config target) +# - Qwen3.6-35B-A3B-UD-Q4_K_S (unchanged) +# - Meta-Llama-3.1-8B-Instruct-Q4_K_M (unchanged) +# +# Known upstream behavior: +# GH #22364: --models-preset creates an extra "default" entry in /v1/models. +# This is cosmetic only and does not affect model selection by name. +# +# Usage (from ~/git/homelab/ansible): +# ansible-playbook -i inventory.yml playbooks/day2_add_phi_alias.yml +# +# Semaphore note (t_9adf0889): Semaphore SSH key for jarvis user is not loaded +# in the container (known pitfall, homelab-llm-serving skill). Run via CLI with +# id_jarvis key; document as exception per Ryan's standing CLI fallback directive. +# +# Author: War Machine (2026-08-12, t_9adf0889) +# ------------------------------------------------------------------------------ + +- name: "Add Phi-3.5-mini-instruct-8bit alias — switch router to preset mode" + hosts: astro_orbiter + gather_facts: false + become: true + + vars: + # Activate preset mode and provide the on-disk INI path + llm_router_preset_enabled: true + llm_router_preset_path: /opt/llama-server-router-preset.ini + + # Production port (router is already on 8002 since t_cd0d5388) + llm_router_port: 8002 + + # All other vars inherit from host_vars + defaults/main.yml. + # Explicitly set the ones needed by the unit template for clarity: + llm_router_enabled: true + llm_service_user: jarvis + llm_binary_path: /opt/llama.cpp/build/bin/llama-server + llm_models_dir: /opt/models + llm_bind_address: "10.1.71.130" + llm_allowed_source_cidr: "10.1.70.0/24" + llm_router_service_name: llama-server-router + llm_router_bind_address: "10.1.71.130" + llm_router_allowed_source_cidr: "10.1.70.0/24" + llm_router_models_dir: /opt/models + llm_router_models_max: 4 # from host_vars; bumped by t_33acbb2e + llm_router_ctx_size: 65536 + llm_router_parallel: 1 + llm_router_gpu_layers: 99 + llm_router_batch_size: 2048 + llm_router_ubatch_size: 512 + llm_router_cache_type_k: q4_0 + llm_router_cache_type_v: q4_0 + llm_router_flash_attn: "auto" + llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" + llm_router_vram_max_mib: 23000 + + handlers: + - name: reload systemd + ansible.builtin.systemd: + daemon_reload: true + become: true + listen: "reload systemd" + + - name: restart router + ansible.builtin.systemd: + name: llama-server-router + state: restarted + become: true + listen: "restart router" + + tasks: + + # ========================================================================== + # PHASE 1: Deploy the preset INI + # ========================================================================== + + - name: "[phi-alias] Deploy preset INI to {{ llm_router_preset_path }}" + ansible.builtin.template: + src: "../roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2" + dest: "{{ llm_router_preset_path }}" + owner: root + group: root + mode: "0644" + register: phi_alias_preset_deployed + notify: + - restart router + + # ========================================================================== + # PHASE 2: Redeploy systemd unit with --models-preset flag + # ========================================================================== + + - name: "[phi-alias] Deploy llama-server-router unit (--models-preset mode)" + ansible.builtin.template: + src: "../roles/llm-inference-multimodel/templates/llama-server-router.service.j2" + dest: /etc/systemd/system/llama-server-router.service + owner: root + group: root + mode: "0644" + register: phi_alias_unit_deployed + notify: + - reload systemd + - restart router + + - name: "[phi-alias] Flush handlers (daemon-reload + router restart)" + ansible.builtin.meta: flush_handlers + + # ========================================================================== + # PHASE 3: Verify alias is present + # ========================================================================== + + - name: "[phi-alias] Wait for /health (router supervisor, no model needed)" + ansible.builtin.uri: + url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health" + status_code: 200 + timeout: 30 + retries: 12 + delay: 5 + register: phi_alias_health + until: phi_alias_health.status == 200 + + - name: "[phi-alias] Query /v1/models" + ansible.builtin.uri: + url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models" + status_code: 200 + return_content: true + timeout: 30 + register: phi_alias_models + + - name: "[phi-alias] Extract model IDs and aliases" + ansible.builtin.set_fact: + phi_alias_model_ids: "{{ phi_alias_models.json.data | map(attribute='id') | list }}" + phi_alias_all_aliases: "{{ phi_alias_models.json.data | map(attribute='aliases') | flatten | list }}" + phi_alias_model_sources: "{{ phi_alias_models.json.data | map(attribute='source') | list }}" + + - name: "[phi-alias] FAIL if Phi original ID missing" + ansible.builtin.fail: + msg: >- + 'Phi-3.5-mini-instruct-Q8_0' not in /v1/models. + IDs: {{ phi_alias_model_ids }} + when: "'Phi-3.5-mini-instruct-Q8_0' not in phi_alias_model_ids" + + - name: "[phi-alias] FAIL if Phi alias missing" + ansible.builtin.fail: + msg: >- + 'Phi-3.5-mini-instruct-8bit' not found as ID or alias in /v1/models. + IDs: {{ phi_alias_model_ids }} + Aliases: {{ phi_alias_all_aliases }} + when: + - "'Phi-3.5-mini-instruct-8bit' not in phi_alias_model_ids" + - "'Phi-3.5-mini-instruct-8bit' not in phi_alias_all_aliases" + + - name: "[phi-alias] FAIL if Qwen missing" + ansible.builtin.fail: + msg: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in /v1/models. IDs: {{ phi_alias_model_ids }}" + when: "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in phi_alias_model_ids" + + - name: "[phi-alias] FAIL if Llama missing" + ansible.builtin.fail: + msg: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in /v1/models. IDs: {{ phi_alias_model_ids }}" + when: "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in phi_alias_model_ids" + + - name: "[phi-alias] PASS — full /v1/models summary" + ansible.builtin.debug: + msg: + - "========================================================================" + - "PHI ALIAS DEPLOYMENT — COMPLETE" + - "" + - " Mode: --models-preset ({{ llm_router_preset_path }})" + - " Service: llama-server-router.service (:{{ llm_router_port }})" + - "" + - " /v1/models IDs: {{ phi_alias_model_ids }}" + - " /v1/models aliases: {{ phi_alias_all_aliases }}" + - " Sources: {{ phi_alias_model_sources }}" + - "" + - " VERIFY:" + - " Phi-3.5-mini-instruct-Q8_0: {{ 'PRESENT' if 'Phi-3.5-mini-instruct-Q8_0' in phi_alias_model_ids else 'MISSING' }}" + - " Phi-3.5-mini-instruct-8bit: {{ 'PRESENT (ID)' if 'Phi-3.5-mini-instruct-8bit' in phi_alias_model_ids else ('PRESENT (alias)' if 'Phi-3.5-mini-instruct-8bit' in phi_alias_all_aliases else 'MISSING') }}" + - " Qwen3.6-35B-A3B-UD-Q4_K_S: {{ 'PRESENT' if 'Qwen3.6-35B-A3B-UD-Q4_K_S' in phi_alias_model_ids else 'MISSING' }}" + - " Meta-Llama-3.1-8B-Instruct-Q4_K_M: {{ 'PRESENT' if 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' in phi_alias_model_ids else 'MISSING' }}" + - "" + - " GH #22364: if 'default' appears in IDs above, that is expected" + - " in --models-preset mode. Cosmetic only." + - "========================================================================" diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 96802d5..1d21da9 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -131,3 +131,21 @@ llm_router_bind_address: "{{ llm_bind_address }}" # 10.1.71.130 llm_router_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24 llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" # verified at Gate 1 llm_router_vram_max_mib: 23000 # Gate 3: fail if exceeded under load + +# --- Router preset mode (--models-preset INI) --------------------------------- +# Set llm_router_preset_enabled: true to switch from --models-dir to +# --models-preset. Preset mode is REQUIRED to support model aliases. +# The template at llama-server-router-preset.ini.j2 defines all 3 router models: +# - Qwen3.6-35B-A3B-UD-Q4_K_S (no alias — primary ID unchanged) +# - Phi-3.5-mini-instruct-Q8_0 (alias: Phi-3.5-mini-instruct-8bit) <-- t_9adf0889 +# - Meta-Llama-3.1-8B-Instruct-Q4_K_M (no alias — primary ID unchanged) +# +# llm_router_preset_path: on-disk path where the rendered INI is deployed. +# Default: /opt/llama-server-router-preset.ini (owned by root, readable by all). +# +# GH #22364 note: --models-preset causes an extra "default" entry in /v1/models. +# This is cosmetic and does not affect model selection by name. Accept it. +# +# Added 2026-08-12 (t_9adf0889) — War Machine. +llm_router_preset_enabled: false # flip true to activate preset mode +llm_router_preset_path: /opt/llama-server-router-preset.ini diff --git a/ansible/roles/llm-inference-multimodel/handlers/main.yml b/ansible/roles/llm-inference-multimodel/handlers/main.yml index 51cbcdc..fff0259 100644 --- a/ansible/roles/llm-inference-multimodel/handlers/main.yml +++ b/ansible/roles/llm-inference-multimodel/handlers/main.yml @@ -35,3 +35,12 @@ name: "{{ llm_router_service_name | default('llama-server-router') }}" state: restarted become: true + +# Restart handler for preset mode changes (notified by tasks/preset.yml when +# the preset INI or unit file changes). Distinct from "restart llama-server-router +# on new GGUF" so each change path can notify independently. +- name: restart router + ansible.builtin.systemd: + name: "{{ llm_router_service_name | default('llama-server-router') }}" + state: restarted + become: true diff --git a/ansible/roles/llm-inference-multimodel/tasks/main.yml b/ansible/roles/llm-inference-multimodel/tasks/main.yml index 9d6264c..03a4aa4 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/main.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/main.yml @@ -50,3 +50,13 @@ - include_tasks: router.yml when: llm_router_enabled | default(false) tags: [always] + +# Phase P — Router preset mode / alias deployment +# Gates on llm_router_preset_enabled (default false — complete no-op until enabled). +# Use playbooks/day2_add_phi_alias.yml which sets llm_router_preset_enabled: true. +# Purpose: switch from --models-dir to --models-preset to enable model aliases. +# The Phi-3.5-mini-instruct-Q8_0 entry gains alias "Phi-3.5-mini-instruct-8bit". +# Added 2026-08-12 (t_9adf0889): Phi alias — War Machine. +- include_tasks: preset.yml + when: llm_router_preset_enabled | default(false) + tags: [always] diff --git a/ansible/roles/llm-inference-multimodel/tasks/preset.yml b/ansible/roles/llm-inference-multimodel/tasks/preset.yml new file mode 100644 index 0000000..cf17c24 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/tasks/preset.yml @@ -0,0 +1,161 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/tasks/preset.yml +# DESCRIPTION: Phase P — deploy --models-preset INI file and switch the router +# from --models-dir to --models-preset mode. +# +# Purpose: preset mode is required to add model aliases. The alias +# field in the INI lets the Phi-3.5-mini-instruct-Q8_0 entry also +# respond to "Phi-3.5-mini-instruct-8bit" — Ryan's Hermes +# auxiliary.title_generation already uses this friendlier name. +# +# This phase is gated on llm_router_preset_enabled | default(false). +# With the default (false) it is a no-op. Flip to true in a playbook +# or extra-vars to activate. +# +# Tag: router_preset (deploy INI + restart service on change) +# +# Order: run AFTER router_systemd. The systemd unit template +# conditionally emits --models-preset when llm_router_preset_enabled +# is true; this phase deploys the INI the unit references. +# +# Added 2026-08-12 (t_9adf0889): Phi alias — War Machine. +# ------------------------------------------------------------------------------ + +# ============================================================================= +# TAG: router_preset +# Deploy the preset INI and restart the router if the INI or unit changed. +# ============================================================================= + +- name: "[router_preset] Deploy preset INI to {{ llm_router_preset_path }}" + ansible.builtin.template: + src: llama-server-router-preset.ini.j2 + dest: "{{ llm_router_preset_path }}" + owner: root + group: root + mode: "0644" + become: true + register: llm_router_preset_deployed + notify: + - restart router + when: llm_router_preset_enabled | default(false) + tags: [router_preset] + +- name: "[router_preset] Report preset INI deployment result" + ansible.builtin.debug: + msg: >- + Preset INI {{ 'deployed (changed)' if llm_router_preset_deployed.changed | default(false) else 'already up-to-date (no change)' }} + at {{ llm_router_preset_path }}. + when: llm_router_preset_enabled | default(false) + tags: [router_preset] + +- name: "[router_preset] Flush handlers so router restarts before verification" + ansible.builtin.meta: flush_handlers + when: llm_router_preset_enabled | default(false) + tags: [router_preset] + +# ============================================================================= +# Verification: confirm both the original ID and the alias appear in /v1/models +# ============================================================================= + +- name: "[router_preset] Wait for router /health after potential restart" + ansible.builtin.uri: + url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health" + status_code: 200 + timeout: 30 + retries: 12 + delay: 5 + register: llm_preset_health + until: llm_preset_health.status == 200 + when: llm_router_preset_enabled | default(false) + tags: [router_preset] + +- name: "[router_preset] Query /v1/models to verify preset loaded" + ansible.builtin.uri: + url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models" + status_code: 200 + return_content: true + timeout: 30 + register: llm_preset_models + when: llm_router_preset_enabled | default(false) + tags: [router_preset] + +- name: "[router_preset] Extract model IDs and aliases from /v1/models" + ansible.builtin.set_fact: + llm_preset_model_ids: >- + {{ llm_preset_models.json.data | map(attribute='id') | list }} + llm_preset_model_aliases: >- + {{ llm_preset_models.json.data | map(attribute='aliases') | flatten | list }} + when: + - llm_router_preset_enabled | default(false) + - llm_preset_models is defined + tags: [router_preset] + +- name: "[router_preset] VERIFY: Phi original ID (Phi-3.5-mini-instruct-Q8_0) present in /v1/models" + ansible.builtin.fail: + msg: >- + PRESET VERIFY FAIL: 'Phi-3.5-mini-instruct-Q8_0' not found in /v1/models IDs. + Returned IDs: {{ llm_preset_model_ids | default([]) }} + Check the preset INI section name. + when: + - llm_router_preset_enabled | default(false) + - llm_preset_model_ids is defined + - "'Phi-3.5-mini-instruct-Q8_0' not in llm_preset_model_ids" + tags: [router_preset] + +- name: "[router_preset] VERIFY: Phi alias (Phi-3.5-mini-instruct-8bit) present in /v1/models aliases" + ansible.builtin.fail: + msg: >- + PRESET VERIFY FAIL: alias 'Phi-3.5-mini-instruct-8bit' not found in /v1/models. + Returned IDs: {{ llm_preset_model_ids | default([]) }} + Returned aliases: {{ llm_preset_model_aliases | default([]) }} + Check the 'alias' field in the preset INI for the [Phi-3.5-mini-instruct-Q8_0] section. + when: + - llm_router_preset_enabled | default(false) + - llm_preset_model_ids is defined + - "'Phi-3.5-mini-instruct-8bit' not in llm_preset_model_ids and 'Phi-3.5-mini-instruct-8bit' not in llm_preset_model_aliases" + tags: [router_preset] + +- name: "[router_preset] VERIFY: Qwen model ID unchanged" + ansible.builtin.fail: + msg: >- + PRESET VERIFY FAIL: 'Qwen3.6-35B-A3B-UD-Q4_K_S' not found in /v1/models. + Returned IDs: {{ llm_preset_model_ids | default([]) }} + when: + - llm_router_preset_enabled | default(false) + - llm_preset_model_ids is defined + - "'Qwen3.6-35B-A3B-UD-Q4_K_S' not in llm_preset_model_ids" + tags: [router_preset] + +- name: "[router_preset] VERIFY: Llama model ID unchanged" + ansible.builtin.fail: + msg: >- + PRESET VERIFY FAIL: 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not found in /v1/models. + Returned IDs: {{ llm_preset_model_ids | default([]) }} + when: + - llm_router_preset_enabled | default(false) + - llm_preset_model_ids is defined + - "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' not in llm_preset_model_ids" + tags: [router_preset] + +- name: "[router_preset] VERIFY PASS — preset mode active, all models and alias confirmed" + ansible.builtin.debug: + msg: + - "======================================================================" + - "PRESET VERIFY PASS: --models-preset mode active on llama-server-router." + - "" + - " /v1/models IDs: {{ llm_preset_model_ids | default([]) }}" + - " /v1/models aliases: {{ llm_preset_model_aliases | default([]) }}" + - "" + - " 'Phi-3.5-mini-instruct-Q8_0' => PRESENT (primary ID)" + - " 'Phi-3.5-mini-instruct-8bit' => PRESENT (alias — Ryan's title_generation target)" + - " 'Qwen3.6-35B-A3B-UD-Q4_K_S' => PRESENT" + - " 'Meta-Llama-3.1-8B-Instruct-Q4_K_M' => PRESENT" + - "" + - " GH #22364 note: a 'default' model entry may also appear above." + - " This is an expected upstream behavior in preset mode — cosmetic only." + - "======================================================================" + when: + - llm_router_preset_enabled | default(false) + - llm_preset_model_ids is defined + tags: [router_preset] diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 new file mode 100644 index 0000000..c503f22 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 @@ -0,0 +1,74 @@ +; ------------------------------------------------------------------------------ +; FILE: roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 +; DESCRIPTION: llama.cpp --models-preset INI for llama-server-router. +; +; Purpose: define all router-served GGUFs as named model entries so that: +; - Each model is explicitly named and configured (no auto-discovery surprises) +; - Aliases can be added per model (impossible with --models-dir alone) +; - The Phi-3.5-mini-instruct-Q8_0 entry carries the alias +; "Phi-3.5-mini-instruct-8bit" — Ryan's Hermes auxiliary.title_generation +; already references this friendlier name; both names resolve to the same +; GGUF child process. +; +; INI format notes (llama.cpp preset.md): +; - Section header (e.g. [Phi-3.5-mini-instruct-Q8_0]) is the primary model ID +; that appears in /v1/models and that clients send in the "model" field. +; - `alias` adds an ADDITIONAL name — both the section name and the alias work. +; - The `model` key is the absolute path to the GGUF file. +; - All other keys map directly to llama-server CLI flags (underscores or hyphens). +; +; Known upstream issues (Aug 2026): +; - GH #22364: --models-preset creates an extra "default" model entry in +; /v1/models. This is cosmetic — it has no functional effect on model +; selection by name. Document and move on. +; - GH #23460: can't pass per-model samplers via --models-preset in router +; mode. Non-issue: Hermes always sends sampling params in the request body. +; +; Added 2026-08-12 (t_9adf0889): Phi alias — War Machine. +; All per-model settings carry over unchanged from the --models-dir baseline +; (ctx_size=65536, n_gpu_layers=99, cache=q4_0 for both K and V, models-max=4). +; ------------------------------------------------------------------------------ + +; --- Production model: Qwen3.6-35B-A3B-UD-Q4_K_S ---------------------------- +; Primary model ID: Qwen3.6-35B-A3B-UD-Q4_K_S (unchanged from --models-dir) +; ~20GB, primary Hermes production LLM. Context: 64K with q4_0 KV cache. +[Qwen3.6-35B-A3B-UD-Q4_K_S] +model = {{ llm_models_dir }}/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf +n-gpu-layers = {{ llm_router_gpu_layers }} +ctx-size = {{ llm_router_ctx_size }} +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} + +; --- Auxiliary model: Phi-3.5-mini-instruct-Q8_0 ---------------------------- +; Primary model ID: Phi-3.5-mini-instruct-Q8_0 (unchanged from --models-dir) +; Alias: Phi-3.5-mini-instruct-8bit (NEW — Ryan's config target) +; Both names resolve to this GGUF child process. +; ~3.8GB, auxiliary.title_generation consumer in Ryan's Hermes config. +; The alias is what this entire task is about — once deployed, the +; "Auxiliary title generation failed" warning will be gone. +[Phi-3.5-mini-instruct-Q8_0] +model = {{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf +alias = Phi-3.5-mini-instruct-8bit +n-gpu-layers = {{ llm_router_gpu_layers }} +ctx-size = {{ llm_router_ctx_size }} +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} + +; --- Auxiliary model: Meta-Llama-3.1-8B-Instruct-Q4_K_M -------------------- +; Primary model ID: Meta-Llama-3.1-8B-Instruct-Q4_K_M (unchanged) +; ~4.6GB, general-purpose small model. +[Meta-Llama-3.1-8B-Instruct-Q4_K_M] +model = {{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf +n-gpu-layers = {{ llm_router_gpu_layers }} +ctx-size = {{ llm_router_ctx_size }} +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router.service.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router.service.j2 index 2e637e1..ac9ce40 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router.service.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router.service.j2 @@ -10,7 +10,11 @@ User={{ llm_service_user }} Group={{ llm_service_user }} Environment="HOME=/home/{{ llm_service_user }}" ExecStart={{ llm_binary_path }} \ +{% if llm_router_preset_enabled | default(false) %} + --models-preset {{ llm_router_preset_path }} \ +{% else %} --models-dir {{ llm_router_models_dir }} \ +{% endif %} --models-max {{ llm_router_models_max }} \ --host {{ llm_router_bind_address }} \ --port {{ llm_router_port }} \ @@ -24,19 +28,23 @@ ExecStart={{ llm_binary_path }} \ --parallel {{ llm_router_parallel }} \ --metrics -# ROUTER MODE NOTES (2026-08-12, t_0cca74a2): +# ROUTER MODE NOTES (2026-08-12, t_0cca74a2 / updated t_9adf0889): # - NO -m/--model flag: this is what enables llama-server router/supervisor mode. -# Without -m, llama-server discovers all .gguf files in --models-dir, spawning -# each as its own child process on demand (LRU-eviction when over models-max). +# Without -m, llama-server discovers all .gguf files in --models-dir, or uses +# the per-model definitions in a --models-preset INI file. +# - PRESET MODE (t_9adf0889, 2026-08-12): +# llm_router_preset_enabled=true switches from --models-dir to --models-preset. +# Preset mode adds alias support (--models-dir cannot assign aliases). +# The preset INI is at {{ llm_router_preset_path | default('/opt/llama-server-router-preset.ini') }}. +# Both the section name and the alias field in the INI work as model IDs. +# GH #22364 (extra "default" entry in /v1/models) is expected in preset mode — cosmetic. # - --models-max {{ llm_router_models_max }} is driven by llm_router_models_max # (default 1 in defaults/main.yml; overridden to 4 in host_vars/astro-orbiter # as of t_33acbb2e after VRAM budget review — see host_vars for OOM risk note). # Default llama-server cap is 4 simultaneous — OOM on 24GB if all 3 current # GGUFs load at once. LRU eviction mitigates in practice but review before adding # models. See host_vars/astro-orbiter/vars.yml for full VRAM breakdown. -# - --models-dir /opt/models: auto-discovers all .gguf files. Keep that directory -# clean (Qwen-only) to avoid spurious extra entries in /v1/models. -# - Clients select a model via "model": "" in their +# - Clients select a model via "model": "" in their # chat completion request. Hermes sends model: "" on every request already. # - Cold model load on first request: ~30-60s for Qwen3.6-35B. First response # will be slow. This is expected. Document in runbook.