llm-inference-multimodel: fix tool-calling support (jinja template + gpu-layers=20 for VRAM fit)

ansible/roles/llm-inference-multimodel/defaults/main.yml

@@ -49,7 +49,16 @@ llm_toolcall_model_url: "https://huggingface.co/bartowski/Mistral-Small-24B-Inst
 llm_toolcall_model_min_bytes: 11000000000   # guard threshold; complete file ~11.7GB
 llm_toolcall_ctx_size: 4096
 llm_toolcall_parallel: 1
-llm_toolcall_gpu_layers: 99
+# 99 (full offload) OOMs on a 24GB GPU when co-resident with llama-server
+# (phi-4, port 8000, ~10.4GB). 20 layers validated stable alongside it with
+# headroom to spare (see docs/validation-log.md).
+llm_toolcall_gpu_layers: 20
+# Chat template shipped with Mistral-Small-24B-Instruct-2501 has no tool-call
+# support (no [AVAILABLE_TOOLS]/[TOOL_CALLS] handling) — see
+# docs/validation-log.md for the investigation. We use Mistral-Nemo-Instruct-2407's
+# tool-calling-capable template instead; Mistral-Small-24B shares the same
+# Mistral tokenizer/instruct format family so the template is compatible.
+llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-tool-use.jinja"
 llm_toolcall_service_name: llama-server-toolcall
 llm_toolcall_model_id: mistral-small-24b-instruct-2501
 

ansible/roles/llm-inference-multimodel/files/mistral-small-tool-use.jinja

@@ -0,0 +1,87 @@
+{%- if messages[0]["role"] == "system" %}
+    {%- set system_message = messages[0]["content"] %}
+    {%- set loop_messages = messages[1:] %}
+{%- else %}
+    {%- set loop_messages = messages %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
+
+{#- This block checks for alternating user/assistant messages, skipping tool calling messages #}
+{%- set ns = namespace() %}
+{%- set ns.index = 0 %}
+{%- for message in loop_messages %}
+    {%- if not (message.role == "tool" or message.role == "tool_results" or (message.tool_calls is defined and message.tool_calls is not none)) %}
+        {%- if (message["role"] == "user") != (ns.index % 2 == 0) %}
+            {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
+        {%- endif %}
+        {%- set ns.index = ns.index + 1 %}
+    {%- endif %}
+{%- endfor %}
+
+{{- bos_token }}
+{%- for message in loop_messages %}
+    {%- if message["role"] == "user" %}
+        {%- if tools is not none and (message == user_messages[-1]) %}
+            {{- "[AVAILABLE_TOOLS][" }}
+            {%- for tool in tools %}
+                {%- set tool = tool.function %}
+                {{- '{"type": "function", "function": {' }}
+                {%- for key, val in tool.items() if key != "return" %}
+                    {%- if val is string %}
+                        {{- '"' + key + '": "' + val + '"' }}
+                    {%- else %}
+                        {{- '"' + key + '": ' + val|tojson }}
+                    {%- endif %}
+                    {%- if not loop.last %}
+                        {{- ", " }}
+                    {%- endif %}
+                {%- endfor %}
+                {{- "}}" }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- else %}
+                    {{- "]" }}
+                {%- endif %}
+            {%- endfor %}
+            {{- "[/AVAILABLE_TOOLS]" }}
+            {%- endif %}
+        {%- if loop.last and system_message is defined %}
+            {{- "[INST]" + system_message + "\n\n" + message["content"] + "[/INST]" }}
+        {%- else %}
+            {{- "[INST]" + message["content"] + "[/INST]" }}
+        {%- endif %}
+    {%- elif (message.tool_calls is defined and message.tool_calls is not none) %}
+        {{- "[TOOL_CALLS][" }}
+        {%- for tool_call in message.tool_calls %}
+            {%- set out = tool_call.function|tojson %}
+            {{- out[:-1] }}
+            {%- if not tool_call.id is defined or tool_call.id|length != 9 %}
+                {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+            {%- endif %}
+            {{- ', "id": "' + tool_call.id + '"}' }}
+            {%- if not loop.last %}
+                {{- ", " }}
+            {%- else %}
+                {{- "]" + eos_token }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif message["role"] == "assistant" %}
+        {{- message["content"] + eos_token}}
+    {%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
+        {%- if message.content is defined and message.content.content is defined %}
+            {%- set content = message.content.content %}
+        {%- else %}
+            {%- set content = message.content %}
+        {%- endif %}
+        {{- '[TOOL_RESULTS]{"content": ' + content|string + ", " }}
+        {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %}
+            {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }}
+        {%- endif %}
+        {{- '"call_id": "' + message.tool_call_id + '"}[/TOOL_RESULTS]' }}
+    {%- else %}
+        {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
+    {%- endif %}
+{%- endfor %}

ansible/roles/llm-inference-multimodel/tasks/models.yml

@@ -70,3 +70,30 @@
     msg:
       - "Aux model:        {{ llm_aux_model_path }}"
       - "Tool-calling model: {{ llm_toolcall_model_path }}"
+
+# --- Tool-calling chat template override -------------------------------------
+# Mistral-Small-24B-Instruct-2501's own embedded/tokenizer_config chat template
+# has NO tool-call handling ([AVAILABLE_TOOLS]/[TOOL_CALLS] blocks) — confirmed
+# via /props chat_template_caps.supports_tools=false against the stock
+# template. Mistral-Nemo-Instruct-2407 ships a template with full tool-calling
+# support and the same Mistral instruct format family, so we serve it via
+# --chat-template-file instead of relying on GGUF-embedded metadata.
+# See docs/validation-log.md for the investigation and probe results.
+
+- name: Create chat templates directory
+  ansible.builtin.file:
+    path: "{{ llm_toolcall_chat_template_file | dirname }}"
+    state: directory
+    owner: "{{ llm_service_user }}"
+    group: "{{ llm_service_user }}"
+    mode: "0755"
+  become: true
+
+- name: Deploy tool-calling-capable chat template (from Mistral-Nemo-Instruct-2407)
+  ansible.builtin.copy:
+    src: mistral-small-tool-use.jinja
+    dest: "{{ llm_toolcall_chat_template_file }}"
+    owner: "{{ llm_service_user }}"
+    group: "{{ llm_service_user }}"
+    mode: "0644"
+  become: true

ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2

@@ -15,9 +15,15 @@ ExecStart={{ llm_binary_path }} \
     --ctx-size {{ llm_toolcall_ctx_size }} \
     --n-gpu-layers {{ llm_toolcall_gpu_layers }} \
     --parallel {{ llm_toolcall_parallel }} \
+    --jinja \
+    --chat-template-file {{ llm_toolcall_chat_template_file }} \
     --metrics
-# NOTE: no --chat-template flag — let llama-server auto-detect Mistral-Small's
+# --jinja + --chat-template-file: Mistral-Small-24B-Instruct-2501's own
-# own embedded chat template from GGUF metadata.
+# embedded chat template has NO tool-calling support (verified via /props ->
+# chat_template_caps.supports_tools=false). We serve Mistral-Nemo-Instruct-
+# 2407's tool-calling-capable Jinja template instead (same Mistral instruct
+# format family, adds [AVAILABLE_TOOLS]/[TOOL_CALLS] handling). See
+# docs/validation-log.md for the investigation and validation probe results.
 # NOTE: --host is the private LAN IP (10.1.71.130 by default), NOT 0.0.0.0.
 # NOTE: --parallel 1 is deliberate (plan §1/§2) — tool-calling profiles are
 # single-session-at-a-time per Claude Code profile; lower parallelism reduces