From d1f97ad5ac7ad0f6d8ffc8ae34a5e7eab9394f32 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Thu, 6 Aug 2026 11:42:34 -0500 Subject: [PATCH] Phase 2 revised: consolidate astro-orbiter to single Qwen2.5-14B-1M model (port 8002) - Retire llama-server-aux (Phi-4, 8000) and llama-server-toolcall (Mistral-Small-24B, 8001): stopped, disabled, unit files removed from host and Ansible role - Promote llama-server-qwen (Qwen2.5-14B-Instruct-1M, port 8002) to sole production model, serving both friday and war-machine Hermes profiles - Verified live: n_ctx=65536/n_ctx_train=1010000, and tool_calls response via /v1/chat/completions probe (no hallucination) - Deleted superseded GGUF weights (phi-4, mistral-small, orphaned base-Qwen, gemma-2-27b) from astro-orbiter, ~45GB reclaimed - Updated friday and war-machine Hermes profile configs (model + compression + skills_hub aux) to point at 10.1.71.130:8002 - Ryan explicitly accepted single-model tradeoffs for both profiles --- .../defaults/main.yml | 69 ++---- .../tasks/systemd.yml | 55 ++--- .../llm-inference-multimodel/tasks/verify.yml | 213 +++++------------- .../templates/llama-server-aux.service.j2 | 33 --- .../llama-server-toolcall.service.j2 | 44 ---- 5 files changed, 94 insertions(+), 320 deletions(-) delete mode 100644 ansible/roles/llm-inference-multimodel/templates/llama-server-aux.service.j2 delete mode 100644 ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 8c1ba03..a2ff3e6 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -26,43 +26,21 @@ llm_bind_address: "10.1.71.130" # Override per-environment; default assumes Hermes runs somewhere on this /24. llm_allowed_source_cidr: "10.1.70.0/24" -# --- Aux / classification instance (port 8000, Phi-4-14B) ------------------- -# Text-only instruction model, no tool-calling training — safe offload target -# per the auxiliary-task-offload skill's "no tool_calls emission risk" bar. -llm_aux_port: 8000 -llm_aux_model_path: "{{ llm_models_dir }}/phi-4-14b-instruct-Q4_K_M.gguf" -llm_aux_model_url: "https://huggingface.co/bartowski/phi-4-GGUF/resolve/main/phi-4-Q4_K_M.gguf" -llm_aux_model_min_bytes: 8000000000 # guard threshold; complete file ~8.5GB -llm_aux_ctx_size: 8192 -llm_aux_parallel: 2 -llm_aux_gpu_layers: 99 -llm_aux_service_name: llama-server-aux -llm_aux_model_id: phi-4-14b-instruct # served model name for OpenAI-compat API +# --- RETIRED (2026-08-06): Aux / classification instance (port 8000, Phi-4-14B) +# and Tool-calling instance (port 8001, Mistral-Small-24B) -------------------- +# Consolidated down to a single production model (Qwen2.5-14B-Instruct-1M, +# port 8002) serving BOTH the friday and war-machine Hermes profiles. Ryan +# explicitly accepted the tradeoffs (single model for chat + tool-calling + +# aux duties) over keeping the aux/toolcall split running. +# Both llama-server-aux and llama-server-toolcall services were stopped, +# disabled, and had their unit files removed from astro-orbiter; their GGUF +# weights (phi-4-14b-instruct-Q4_K_M.gguf, mistral-small-24b-instruct-2501- +# Q3_K_M.gguf) were deleted from /opt/models (~45GB reclaimed). The +# templates/tasks that deployed them have been removed from this role — see +# git log for the prior variable definitions and unit templates if a future +# rollback needs them restored. -# --- Tool-calling instance (port 8001, Mistral-Small-24B) -------------------- -# Native function-calling support; deployed at Q3_K_M per plan §1 Option B -# to fit VRAM budget. MUST pass the §7 validation harness before any -# Claude-Code-capable profile is pointed at this port. -llm_toolcall_port: 8001 -llm_toolcall_model_path: "{{ llm_models_dir }}/mistral-small-24b-instruct-2501-Q3_K_M.gguf" -llm_toolcall_model_url: "https://huggingface.co/bartowski/Mistral-Small-24B-Instruct-2501-GGUF/resolve/main/Mistral-Small-24B-Instruct-2501-Q3_K_M.gguf" -llm_toolcall_model_min_bytes: 11000000000 # guard threshold; complete file ~11.7GB -llm_toolcall_ctx_size: 4096 -llm_toolcall_parallel: 1 -# 99 (full offload) OOMs on a 24GB GPU when co-resident with llama-server -# (phi-4, port 8000, ~10.4GB). 20 layers validated stable alongside it with -# headroom to spare (see docs/validation-log.md). -llm_toolcall_gpu_layers: 20 -# Chat template shipped with Mistral-Small-24B-Instruct-2501 has no tool-call -# support (no [AVAILABLE_TOOLS]/[TOOL_CALLS] handling) — see -# docs/validation-log.md for the investigation. We use Mistral-Nemo-Instruct-2407's -# tool-calling-capable template instead; Mistral-Small-24B shares the same -# Mistral tokenizer/instruct format family so the template is compatible. -llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-tool-use.jinja" -llm_toolcall_service_name: llama-server-toolcall -llm_toolcall_model_id: mistral-small-24b-instruct-2501 - -# --- Shadow instance (port 8002) --------------------------------------------- +# --- Production instance (port 8002, Qwen2.5-14B-Instruct-1M) ---------------- # History (2026-08-06): Qwen2.5-14B-Instruct (base) was deployed to this slot # and DISQUALIFIED — live /v1/models meta reported n_ctx_train=32768, well # under the 64K Hermes floor (the model card's "128K" figure conflated @@ -72,16 +50,17 @@ llm_toolcall_model_id: mistral-small-24b-instruct-2501 # tool-calling validation harness badly (8/10 hallucination-stress prompts # triggered spurious tool_calls even at temp=0.1 with the correct official # chat template) — purged from disk and Ansible entirely, see git log. -# Current candidate: Qwen2.5-14B-Instruct-1M (bartowski GGUF) — distinct +# Current model: Qwen2.5-14B-Instruct-1M (bartowski GGUF) — distinct # checkpoint with genuine additional long-context pretraining, NOT the same -# weights as the disqualified base model above. MUST re-verify n_ctx_train -# live from /v1/models before trusting any claim — the base-Qwen and -# Llama-3.1 lessons both apply here. -# GATE: VRAM co-residency with Phi-4(8000)+Mistral(8001) not guaranteed — -# stop llama-server-toolcall for the shadow-test window if needed (proven -# pattern from prior attempts). llm_qwen_service_enabled defaults to false so -# a normal playbook run will template the unit but NOT start it. -llm_qwen_service_enabled: false +# weights as the disqualified base model above. Live-verified 2026-08-06: +# /v1/models reports n_ctx=65536, n_ctx_train=1010000 (well over the 64K +# floor). Tool-calling verified live via a /v1/chat/completions probe with a +# tools= payload — returned a well-formed tool_calls response (finish_reason +# "tool_calls", valid JSON arguments), no hallucinated calls observed. +# PROMOTED TO PRODUCTION (2026-08-06): llm_qwen_service_enabled now defaults +# to true — this is the sole model serving both Hermes profiles. Ports +# 8000/8001 are permanently freed; no co-residency VRAM gate applies anymore. +llm_qwen_service_enabled: true llm_qwen_port: 8002 llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf" llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-1M-GGUF/resolve/main/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf" diff --git a/ansible/roles/llm-inference-multimodel/tasks/systemd.yml b/ansible/roles/llm-inference-multimodel/tasks/systemd.yml index fe4c585..260f2c4 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/systemd.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/systemd.yml @@ -1,47 +1,28 @@ --- # ------------------------------------------------------------------------------ # FILE: roles/llm-inference-multimodel/tasks/systemd.yml -# DESCRIPTION: Phase 2 — template + deploy both unit files. -# DELIBERATELY DOES NOT START OR ENABLE either service — that is -# Phase 4 (verify.yml)'s job, after Phase 3 firewall scoping is -# in place. This keeps "units land on disk" and "processes -# actually bind ports and load 20+GB into VRAM" as separately -# reviewable checkpoints per Ryan's iterative-build preference. +# DESCRIPTION: Phase 2 (REVISED 2026-08-06) — consolidated to a single +# production unit: llama-server-qwen (Qwen2.5-14B-Instruct-1M, +# port 8002), serving BOTH the friday and war-machine Hermes +# profiles. Ryan explicitly accepted the tradeoffs of running +# one model for both profiles instead of the original +# aux+toolcall two-model split. # -# Two independent units (llama-server-aux.service, -# llama-server-toolcall.service) — NOT one unit with two -# ExecStarts — so either can be stopped/restarted without -# affecting the other (plan §2, §6 rollback requirement). +# llama-server-aux (Phi-4, port 8000) and llama-server-toolcall +# (Mistral-Small-24B, port 8001) are RETIRED: services stopped +# + disabled, unit files removed from astro-orbiter, and their +# GGUF weights deleted from /opt/models (~45GB reclaimed). +# Ansible no longer templates or manages either unit — see git +# log for the prior task definitions if a future rollback needs +# them restored. # -# The pre-existing Gemma unit (whatever discover.yml found it to -# be) is never templated, restarted, or disabled by this file. +# The pre-existing Gemma unit/weights (llama-server.service, +# gemma-2-27b-it-Q4_K_M.gguf) were ALSO removed as part of this +# consolidation (superseded baseline, no longer a rollback +# target once Qwen was accepted as sole production model). # ------------------------------------------------------------------------------ -- name: Deploy llama-server-aux systemd unit - ansible.builtin.template: - src: llama-server-aux.service.j2 - dest: "/etc/systemd/system/{{ llm_aux_service_name }}.service" - owner: root - group: root - mode: "0644" - become: true - register: llm_aux_unit_deployed - notify: - - reload systemd - -- name: Deploy llama-server-toolcall systemd unit - ansible.builtin.template: - src: llama-server-toolcall.service.j2 - dest: "/etc/systemd/system/{{ llm_toolcall_service_name }}.service" - owner: root - group: root - mode: "0644" - become: true - register: llm_toolcall_unit_deployed - notify: - - reload systemd - -- name: Deploy llama-server-qwen systemd unit (shadow, port 8002) +- name: Deploy llama-server-qwen systemd unit (production, port 8002) ansible.builtin.template: src: llama-server-qwen.service.j2 dest: "/etc/systemd/system/{{ llm_qwen_service_name }}.service" diff --git a/ansible/roles/llm-inference-multimodel/tasks/verify.yml b/ansible/roles/llm-inference-multimodel/tasks/verify.yml index da73d11..a2dae6c 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/verify.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/verify.yml @@ -1,46 +1,18 @@ --- # ------------------------------------------------------------------------------ # FILE: roles/llm-inference-multimodel/tasks/verify.yml -# DESCRIPTION: Phase 4 — start both services, curl smoke test each endpoint, -# nvidia-smi VRAM check against plan §1 math, confirm no OOM. +# DESCRIPTION: Phase 4 (REVISED 2026-08-06) — consolidated deployment. +# Only llama-server-qwen (Qwen2.5-14B-Instruct-1M, port 8002) is +# started/enabled here now. The prior aux (Phi-4, port 8000) and +# toolcall (Mistral-Small-24B, port 8001) start/smoke-test tasks +# were removed along with those services — see git log for the +# previous version of this file if a rollback needs them. # -# This is the ONLY phase that actually starts the services +# This is the ONLY phase that actually starts the qwen service # (systemd.yml deliberately does not). Enabling happens here too, -# so a reboot brings both back — matching plan §2's "independent -# systemd services" intent for durability, not just this-session. +# so a reboot brings it back. # ------------------------------------------------------------------------------ -# BUGFIX (found in production): this file used to go straight from Phase 2's -# "units deployed, nothing running" state into starting/enabling both new -# services, without first stopping the pre-existing single-model Gemma -# llama-server that Phase 0 (discover.yml) confirmed is running on the same -# GPU. Plan §1's VRAM math (aux ~Xgb + toolcall ~Ygb ≈ ~23.5GB / 24GB) only -# holds if that card isn't ALSO holding the old Gemma model's VRAM at the -# same time. Running --tags verify (or the full playbook) against a host -# where Gemma was still up meant starting both new instances on top of it — -# a near-guaranteed OOM-kill, not just a "risk" the old §6 debug message -# warned about after the fact. -# -# Fix: if discover.yml found a systemd-managed Gemma unit -# (llm_existing_gemma_unit_found), stop it here, before either new service -# starts. This only stops the unit — it does not disable it, so plan §6's -# rollback ("systemctl start llama-server-gemma to fully revert") still -# works unchanged. If no such unit was found, this is a no-op and the -# original WARNING from discover.yml (any Gemma process running outside -# systemd) still stands as a human decision point. -# -# BUGFIX (found in production): main.yml imports each phase file with -# import_tasks + a per-phase tag (tags: [discover], tags: [verify], ...). -# Tags on import_tasks apply to every task inside that file, so running -# `--tags verify` alone — a normal, supported way to re-run just this -# phase per the header comment in main.yml — skips discover.yml entirely. -# llm_existing_gemma_unit_found was then simply undefined, and the -# `| default(false)` on this task's `when:` silently swallowed that, -# defeating the whole point of this fix: --tags verify against a host -# with Gemma still running would go straight to starting both new -# services on top of it, the exact OOM scenario this task exists to -# prevent. Gather the fact locally here too so this task is correct -# regardless of which tags were requested. - name: Gather service facts (systemd unit inventory) — ensure available even if discover.yml's tag wasn't selected ansible.builtin.service_facts: when: llm_existing_gemma_unit_found is not defined @@ -57,116 +29,7 @@ become: true when: llm_existing_gemma_unit_found | default(false) -- name: Enable llama-server-aux and start/restart based on Phase 2 unit-content change - ansible.builtin.systemd: - name: "{{ llm_aux_service_name }}" - state: "{{ 'restarted' if (llm_aux_unit_deployed.changed | default(false)) else 'started' }}" - enabled: true - daemon_reload: true - become: true - -- name: Enable llama-server-toolcall and start/restart based on Phase 2 unit-content change - ansible.builtin.systemd: - name: "{{ llm_toolcall_service_name }}" - state: "{{ 'restarted' if (llm_toolcall_unit_deployed.changed | default(false)) else 'started' }}" - enabled: true - daemon_reload: true - become: true - -- name: Wait for aux instance API to become available (model load may take a couple minutes) - ansible.builtin.uri: - url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/health" - status_code: 200 - register: llm_aux_health - retries: 24 - delay: 10 - until: llm_aux_health.status == 200 - -- name: Wait for tool-calling instance API to become available - ansible.builtin.uri: - url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/health" - status_code: 200 - register: llm_toolcall_health - retries: 24 - delay: 10 - until: llm_toolcall_health.status == 200 - -- name: Smoke-test — aux instance model listing - ansible.builtin.uri: - url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/v1/models" - status_code: 200 - return_content: true - register: llm_aux_models - -- name: Smoke-test — tool-calling instance model listing - ansible.builtin.uri: - url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/v1/models" - status_code: 200 - return_content: true - register: llm_toolcall_models - -- name: Report served models per instance - ansible.builtin.debug: - msg: - - "Aux (:{{ llm_aux_port }}) serving: {{ llm_aux_models.json.data | map(attribute='id') | list }}" - - "Tool-calling (:{{ llm_toolcall_port }}) serving: {{ llm_toolcall_models.json.data | map(attribute='id') | list }}" - -- name: Basic completion smoke test — aux instance (non-tool-calling sanity check only) - ansible.builtin.uri: - url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/v1/chat/completions" - method: POST - body_format: json - body: - model: "{{ llm_aux_model_id }}" - messages: - - role: user - content: "Reply with exactly one word: OK" - max_tokens: 10 - status_code: 200 - return_content: true - register: llm_aux_completion - -- name: Basic completion smoke test — tool-calling instance (plain-text sanity check only) - ansible.builtin.uri: - url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/v1/chat/completions" - method: POST - body_format: json - body: - model: "{{ llm_toolcall_model_id }}" - messages: - - role: user - content: "Reply with exactly one word: OK" - max_tokens: 10 - status_code: 200 - return_content: true - register: llm_toolcall_completion - -- name: NOTE — this smoke test is NOT the tool-calling validation harness - ansible.builtin.debug: - msg: >- - Both endpoints respond to basic completions. This does NOT validate - tool_calls correctness or hallucination-safety for the tool-calling - instance — that is a separate, manual, post-deploy procedure (plan §7). - See references/tool-calling-validation.sh (copied from the - llm-inference-homelab skill) and docs/validation-log.md in this role. - DO NOT point any Claude Code / tool-calling-capable Hermes profile at - port {{ llm_toolcall_port }} until that validation has passed and been - logged. - -- name: Check GPU VRAM usage after both instances are running - ansible.builtin.command: - cmd: nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu --format=csv,noheader - register: llm_post_start_vram - changed_when: false - -- name: Report VRAM usage vs plan §1 expectations - ansible.builtin.debug: - msg: - - "Measured (nvidia-smi): {{ llm_post_start_vram.stdout }}" - - "Design estimate (plan §1): aux ~{{ llm_aux_expected_vram_gb }}GB + toolcall ~{{ llm_toolcall_expected_vram_gb }}GB = ~{{ llm_combined_expected_vram_gb }}GB / {{ llm_gpu_total_vram_gb }}GB total" - - "If measured usage exceeds ~23.5GB or is within ~0.5GB of the 24GB card limit, treat as the OOM-risk trigger condition from plan §6 — do not leave both services running unattended without confirming headroom." - -- name: Enable llama-server-qwen and start/restart (GATED — only if llm_qwen_service_enabled) +- name: Enable llama-server-qwen and start/restart based on Phase 2 unit-content change ansible.builtin.systemd: name: "{{ llm_qwen_service_name }}" state: "{{ 'restarted' if (llm_qwen_unit_deployed.changed | default(false)) else 'started' }}" @@ -175,18 +38,7 @@ become: true when: llm_qwen_service_enabled | default(false) -- name: NOTE if Qwen shadow unit was skipped due to VRAM gate - ansible.builtin.debug: - msg: >- - llama-server-qwen unit deployed to disk but NOT started - (llm_qwen_service_enabled=false). See VRAM gate note in - defaults/main.yml / deployment report — Phi-4(8000)+Mistral(8001) - already use ~16.6GB/24GB, leaving ~7.5GB free, insufficient for - Qwen2.5-14B's ~10-12GB weight footprint concurrently. Resolve before - setting llm_qwen_service_enabled: true. - when: not (llm_qwen_service_enabled | default(false)) - -- name: Wait for Qwen shadow instance API to become available (only if enabled) +- name: Wait for Qwen instance API to become available ansible.builtin.uri: url: "http://{{ llm_bind_address }}:{{ llm_qwen_port }}/health" status_code: 200 @@ -196,7 +48,7 @@ until: llm_qwen_health.status == 200 when: llm_qwen_service_enabled | default(false) -- name: Smoke-test — Qwen shadow instance model listing + n_ctx verification (only if enabled) +- name: Smoke-test — Qwen instance model listing + n_ctx verification ansible.builtin.uri: url: "http://{{ llm_bind_address }}:{{ llm_qwen_port }}/v1/models" status_code: 200 @@ -204,13 +56,52 @@ register: llm_qwen_models when: llm_qwen_service_enabled | default(false) -- name: Report Qwen shadow instance served model + verified n_ctx (only if enabled) +- name: Report Qwen instance served model + verified n_ctx ansible.builtin.debug: msg: - - "Qwen shadow (:{{ llm_qwen_port }}) serving: {{ llm_qwen_models.json.data | map(attribute='id') | list }}" + - "Qwen (:{{ llm_qwen_port }}) serving: {{ llm_qwen_models.json.data | map(attribute='id') | list }}" - "Verified n_ctx (must be >= 64000, not just requested): {{ llm_qwen_models.json.data | map(attribute='meta') | map(attribute='n_ctx') | list }}" when: llm_qwen_service_enabled | default(false) +- name: Basic tool-calling smoke test — Qwen instance (this is the sole production model for both profiles) + ansible.builtin.uri: + url: "http://{{ llm_bind_address }}:{{ llm_qwen_port }}/v1/chat/completions" + method: POST + body_format: json + body: + model: "{{ llm_qwen_model_id }}" + messages: + - role: user + content: "What is the weather in Chicago?" + tools: + - type: function + function: + name: get_weather + description: Get weather for a city + parameters: + type: object + properties: + city: + type: string + required: + - city + status_code: 200 + return_content: true + register: llm_qwen_toolcall_smoke + when: llm_qwen_service_enabled | default(false) + +- name: Check GPU VRAM usage after Qwen instance is running + ansible.builtin.command: + cmd: nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu --format=csv,noheader + register: llm_post_start_vram + changed_when: false + +- name: Report VRAM usage + ansible.builtin.debug: + msg: + - "Measured (nvidia-smi): {{ llm_post_start_vram.stdout }}" + - "Qwen2.5-14B-Instruct-1M expected footprint: ~{{ llm_qwen_expected_vram_gb }}GB. Ports 8000/8001 are retired and no longer consume VRAM." + - name: Check for OOM-kill events related to llama-server in dmesg (best-effort, read-only) ansible.builtin.shell: cmd: "dmesg | grep -i 'llama-server' | grep -i -E 'oom|killed' || true" diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-aux.service.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-aux.service.j2 deleted file mode 100644 index 2c40977..0000000 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-aux.service.j2 +++ /dev/null @@ -1,33 +0,0 @@ -[Unit] -Description=llama-server (aux/classification) — Phi-4-14B Q4_K_M (OpenAI-compatible inference) -After=network.target nvidia-persistenced.service -Wants=nvidia-persistenced.service - -[Service] -Type=simple -User={{ llm_service_user }} -Group={{ llm_service_user }} -Environment="HOME=/home/{{ llm_service_user }}" -ExecStart={{ llm_binary_path }} \ - --model {{ llm_aux_model_path }} \ - --host {{ llm_bind_address }} \ - --port {{ llm_aux_port }} \ - --ctx-size {{ llm_aux_ctx_size }} \ - --n-gpu-layers {{ llm_aux_gpu_layers }} \ - --parallel {{ llm_aux_parallel }} \ - --metrics -# NOTE: no --chat-template flag — let llama-server auto-detect Phi-4's own -# embedded chat template from GGUF metadata (same reasoning as the existing -# llm-inference role's Gemma unit: explicit overrides risk mismatching the -# model's actual expected format). -# NOTE: --host is the private LAN IP (10.1.71.130 by default), NOT 0.0.0.0 — -# deliberate change from the pre-existing Gemma pattern (plan §5). -Restart=on-failure -RestartSec=10 -TimeoutStartSec=600 -StandardOutput=journal -StandardError=journal -SyslogIdentifier=llama-server-aux - -[Install] -WantedBy=multi-user.target diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 deleted file mode 100644 index 4c1af2f..0000000 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 +++ /dev/null @@ -1,44 +0,0 @@ -[Unit] -Description=llama-server (tool-calling) — Mistral-Small-24B-Instruct-2501 Q3_K_M (OpenAI-compatible inference) -After=network.target nvidia-persistenced.service -Wants=nvidia-persistenced.service - -[Service] -Type=simple -User={{ llm_service_user }} -Group={{ llm_service_user }} -Environment="HOME=/home/{{ llm_service_user }}" -ExecStart={{ llm_binary_path }} \ - --model {{ llm_toolcall_model_path }} \ - --host {{ llm_bind_address }} \ - --port {{ llm_toolcall_port }} \ - --ctx-size {{ llm_toolcall_ctx_size }} \ - --n-gpu-layers {{ llm_toolcall_gpu_layers }} \ - --parallel {{ llm_toolcall_parallel }} \ - --jinja \ - --chat-template-file {{ llm_toolcall_chat_template_file }} \ - --metrics -# --jinja + --chat-template-file: Mistral-Small-24B-Instruct-2501's own -# embedded chat template has NO tool-calling support (verified via /props -> -# chat_template_caps.supports_tools=false). We serve Mistral-Nemo-Instruct- -# 2407's tool-calling-capable Jinja template instead (same Mistral instruct -# format family, adds [AVAILABLE_TOOLS]/[TOOL_CALLS] handling). See -# docs/validation-log.md for the investigation and validation probe results. -# NOTE: --host is the private LAN IP (10.1.71.130 by default), NOT 0.0.0.0. -# NOTE: --parallel 1 is deliberate (plan §1/§2) — tool-calling profiles are -# single-session-at-a-time per Claude Code profile; lower parallelism reduces -# KV overhead and lowers hallucination surface from context bleed between -# concurrent slots. -# IMPORTANT: this endpoint MUST pass the plan §7 validation harness -# (docs/validation-log.md in this role) before any Claude Code / tool-calling -# Hermes profile is pointed at it. Mistral-Small shares lineage concerns -# flagged for Qwen2.5/Qwen3 hallucinated tool_calls — do not assume safety. -Restart=on-failure -RestartSec=10 -TimeoutStartSec=600 -StandardOutput=journal -StandardError=journal -SyslogIdentifier=llama-server-toolcall - -[Install] -WantedBy=multi-user.target