diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index b392f21..fa80014 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -49,7 +49,16 @@ llm_toolcall_model_url: "https://huggingface.co/bartowski/Mistral-Small-24B-Inst llm_toolcall_model_min_bytes: 11000000000 # guard threshold; complete file ~11.7GB llm_toolcall_ctx_size: 4096 llm_toolcall_parallel: 1 -llm_toolcall_gpu_layers: 99 +# 99 (full offload) OOMs on a 24GB GPU when co-resident with llama-server +# (phi-4, port 8000, ~10.4GB). 20 layers validated stable alongside it with +# headroom to spare (see docs/validation-log.md). +llm_toolcall_gpu_layers: 20 +# Chat template shipped with Mistral-Small-24B-Instruct-2501 has no tool-call +# support (no [AVAILABLE_TOOLS]/[TOOL_CALLS] handling) — see +# docs/validation-log.md for the investigation. We use Mistral-Nemo-Instruct-2407's +# tool-calling-capable template instead; Mistral-Small-24B shares the same +# Mistral tokenizer/instruct format family so the template is compatible. +llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-tool-use.jinja" llm_toolcall_service_name: llama-server-toolcall llm_toolcall_model_id: mistral-small-24b-instruct-2501 diff --git a/ansible/roles/llm-inference-multimodel/files/mistral-small-tool-use.jinja b/ansible/roles/llm-inference-multimodel/files/mistral-small-tool-use.jinja new file mode 100644 index 0000000..9c21a3f --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/files/mistral-small-tool-use.jinja @@ -0,0 +1,87 @@ +{%- if messages[0]["role"] == "system" %} + {%- set system_message = messages[0]["content"] %} + {%- set loop_messages = messages[1:] %} +{%- else %} + {%- set loop_messages = messages %} +{%- endif %} +{%- if not tools is defined %} + {%- set tools = none %} +{%- endif %} +{%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %} + +{#- This block checks for alternating user/assistant messages, skipping tool calling messages #} +{%- set ns = namespace() %} +{%- set ns.index = 0 %} +{%- for message in loop_messages %} + {%- if not (message.role == "tool" or message.role == "tool_results" or (message.tool_calls is defined and message.tool_calls is not none)) %} + {%- if (message["role"] == "user") != (ns.index % 2 == 0) %} + {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }} + {%- endif %} + {%- set ns.index = ns.index + 1 %} + {%- endif %} +{%- endfor %} + +{{- bos_token }} +{%- for message in loop_messages %} + {%- if message["role"] == "user" %} + {%- if tools is not none and (message == user_messages[-1]) %} + {{- "[AVAILABLE_TOOLS][" }} + {%- for tool in tools %} + {%- set tool = tool.function %} + {{- '{"type": "function", "function": {' }} + {%- for key, val in tool.items() if key != "return" %} + {%- if val is string %} + {{- '"' + key + '": "' + val + '"' }} + {%- else %} + {{- '"' + key + '": ' + val|tojson }} + {%- endif %} + {%- if not loop.last %} + {{- ", " }} + {%- endif %} + {%- endfor %} + {{- "}}" }} + {%- if not loop.last %} + {{- ", " }} + {%- else %} + {{- "]" }} + {%- endif %} + {%- endfor %} + {{- "[/AVAILABLE_TOOLS]" }} + {%- endif %} + {%- if loop.last and system_message is defined %} + {{- "[INST]" + system_message + "\n\n" + message["content"] + "[/INST]" }} + {%- else %} + {{- "[INST]" + message["content"] + "[/INST]" }} + {%- endif %} + {%- elif (message.tool_calls is defined and message.tool_calls is not none) %} + {{- "[TOOL_CALLS][" }} + {%- for tool_call in message.tool_calls %} + {%- set out = tool_call.function|tojson %} + {{- out[:-1] }} + {%- if not tool_call.id is defined or tool_call.id|length != 9 %} + {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }} + {%- endif %} + {{- ', "id": "' + tool_call.id + '"}' }} + {%- if not loop.last %} + {{- ", " }} + {%- else %} + {{- "]" + eos_token }} + {%- endif %} + {%- endfor %} + {%- elif message["role"] == "assistant" %} + {{- message["content"] + eos_token}} + {%- elif message["role"] == "tool_results" or message["role"] == "tool" %} + {%- if message.content is defined and message.content.content is defined %} + {%- set content = message.content.content %} + {%- else %} + {%- set content = message.content %} + {%- endif %} + {{- '[TOOL_RESULTS]{"content": ' + content|string + ", " }} + {%- if not message.tool_call_id is defined or message.tool_call_id|length != 9 %} + {{- raise_exception("Tool call IDs should be alphanumeric strings with length 9!") }} + {%- endif %} + {{- '"call_id": "' + message.tool_call_id + '"}[/TOOL_RESULTS]' }} + {%- else %} + {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }} + {%- endif %} +{%- endfor %} diff --git a/ansible/roles/llm-inference-multimodel/tasks/models.yml b/ansible/roles/llm-inference-multimodel/tasks/models.yml index 0f6189f..c9fca48 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/models.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/models.yml @@ -70,3 +70,30 @@ msg: - "Aux model: {{ llm_aux_model_path }}" - "Tool-calling model: {{ llm_toolcall_model_path }}" + +# --- Tool-calling chat template override ------------------------------------- +# Mistral-Small-24B-Instruct-2501's own embedded/tokenizer_config chat template +# has NO tool-call handling ([AVAILABLE_TOOLS]/[TOOL_CALLS] blocks) — confirmed +# via /props chat_template_caps.supports_tools=false against the stock +# template. Mistral-Nemo-Instruct-2407 ships a template with full tool-calling +# support and the same Mistral instruct format family, so we serve it via +# --chat-template-file instead of relying on GGUF-embedded metadata. +# See docs/validation-log.md for the investigation and probe results. + +- name: Create chat templates directory + ansible.builtin.file: + path: "{{ llm_toolcall_chat_template_file | dirname }}" + state: directory + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0755" + become: true + +- name: Deploy tool-calling-capable chat template (from Mistral-Nemo-Instruct-2407) + ansible.builtin.copy: + src: mistral-small-tool-use.jinja + dest: "{{ llm_toolcall_chat_template_file }}" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0644" + become: true diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 index 689bbdb..4c1af2f 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 @@ -15,9 +15,15 @@ ExecStart={{ llm_binary_path }} \ --ctx-size {{ llm_toolcall_ctx_size }} \ --n-gpu-layers {{ llm_toolcall_gpu_layers }} \ --parallel {{ llm_toolcall_parallel }} \ + --jinja \ + --chat-template-file {{ llm_toolcall_chat_template_file }} \ --metrics -# NOTE: no --chat-template flag — let llama-server auto-detect Mistral-Small's -# own embedded chat template from GGUF metadata. +# --jinja + --chat-template-file: Mistral-Small-24B-Instruct-2501's own +# embedded chat template has NO tool-calling support (verified via /props -> +# chat_template_caps.supports_tools=false). We serve Mistral-Nemo-Instruct- +# 2407's tool-calling-capable Jinja template instead (same Mistral instruct +# format family, adds [AVAILABLE_TOOLS]/[TOOL_CALLS] handling). See +# docs/validation-log.md for the investigation and validation probe results. # NOTE: --host is the private LAN IP (10.1.71.130 by default), NOT 0.0.0.0. # NOTE: --parallel 1 is deliberate (plan §1/§2) — tool-calling profiles are # single-session-at-a-time per Claude Code profile; lower parallelism reduces