rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm-inference-multimodel: fix tool-calling support (jinja template + gpu-layers=20 for VRAM fit)

Browse Source
This commit is contained in:
Hermes Agent service account
2026-08-05 17:13:32 -05:00
parent a76ad3195c
commit 5dc76a8348
4 changed files with 132 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
ansible/roles/llm-inference-multimodel/defaults/main.yml
View File

@@ -49,7 +49,16 @@ llm_toolcall_model_url: "https://huggingface.co/bartowski/Mistral-Small-24B-Inst
llm_toolcall_model_min_bytes: 11000000000 # guard threshold; complete file ~11.7GB
llm_toolcall_ctx_size: 4096
llm_toolcall_parallel: 1
llm_toolcall_gpu_layers: 99
# 99 (full offload) OOMs on a 24GB GPU when co-resident with llama-server
# (phi-4, port 8000, ~10.4GB). 20 layers validated stable alongside it with
# headroom to spare (see docs/validation-log.md).
llm_toolcall_gpu_layers: 20
# Chat template shipped with Mistral-Small-24B-Instruct-2501 has no tool-call
# support (no [AVAILABLE_TOOLS]/[TOOL_CALLS] handling) — see
# docs/validation-log.md for the investigation. We use Mistral-Nemo-Instruct-2407's
# tool-calling-capable template instead; Mistral-Small-24B shares the same
# Mistral tokenizer/instruct format family so the template is compatible.
llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-tool-use.jinja"
llm_toolcall_service_name: llama-server-toolcall
llm_toolcall_model_id: mistral-small-24b-instruct-2501