From 6455d2275237922fb62092324046a6ae50db0572 Mon Sep 17 00:00:00 2001
From: Hermes Agent service account
 <hermes@carousel-of-progress.local.mk-labs.cloud>
Date: Wed, 12 Aug 2026 23:33:20 -0500
Subject: [PATCH] feat(llm-router): add Meta-Llama-3.1-8B-Instruct-4bit alias;
 document Phi json_schema limitation (t_9adf0889)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add Meta-Llama-3.1-8B-Instruct-4bit alias on Meta-Llama-3.1-8B-Instruct-Q4_K_M
  entry in the preset INI (per Ryan dashboard note).
- Document known issue: Phi-3.5-mini json_schema grammar sampler incompatibility
  in router mode (GH #23460 variant — chat template token format mismatch).
  Meta-Llama works with json_schema response_format; confirmed via live test.
- Phi-3.5-mini-instruct-8bit alias remains working for model routing;
  structured output (json_schema) fails due to the model's token format.
---
 .../llama-server-router-preset.ini.j2         | 25 ++++++++++++++++---
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2
index c503f22..6beddb4 100644
--- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2
+++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2
@@ -47,8 +47,22 @@ parallel     = {{ llm_router_parallel }}
 ; Alias:            Phi-3.5-mini-instruct-8bit (NEW — Ryan's config target)
 ; Both names resolve to this GGUF child process.
 ; ~3.8GB, auxiliary.title_generation consumer in Ryan's Hermes config.
-; The alias is what this entire task is about — once deployed, the
-; "Auxiliary title generation failed" warning will be gone.
+; The alias is deployed — "model not found" errors are resolved.
+;
+; KNOWN ISSUE (2026-08-12, t_9adf0889):
+;   json_schema response_format fails for Phi-3.5-mini in llama.cpp router
+;   mode due to chat template grammar sampler incompatibility (GH #23460).
+;   The grammar sampler generates root ::= "assistant|>\n" ... which fails
+;   to initialize. This means Hermes auxiliary.title_generation still errors
+;   with "HTTP 400: Failed to initialize samplers" when json_schema format
+;   is requested. Without response_format (plain text), Phi works fine.
+;
+;   Resolution options:
+;   a) Update auxiliary.title_generation.model in ~/.hermes/config.yaml to
+;      Meta-Llama-3.1-8B-Instruct-Q4_K_M (which supports json_schema) — Ryan
+;      needs to approve this config.yaml write (protected file).
+;   b) Rebuild llama.cpp from a newer commit if this bug is fixed upstream.
+;   c) Accept title generation degradation for Phi-specific structured output.
 [Phi-3.5-mini-instruct-Q8_0]
 model        = {{ llm_models_dir }}/Phi-3.5-mini-instruct-Q8_0.gguf
 alias        = Phi-3.5-mini-instruct-8bit
@@ -61,10 +75,13 @@ ubatch-size  = {{ llm_router_ubatch_size }}
 parallel     = {{ llm_router_parallel }}
 
 ; --- Auxiliary model: Meta-Llama-3.1-8B-Instruct-Q4_K_M --------------------
-; Primary model ID: Meta-Llama-3.1-8B-Instruct-Q4_K_M (unchanged)
-; ~4.6GB, general-purpose small model.
+; Primary model ID: Meta-Llama-3.1-8B-Instruct-Q4_K_M (unchanged from --models-dir)
+; Alias:            Meta-Llama-3.1-8B-Instruct-4bit (NEW — friendlier name)
+; Both names resolve to this GGUF child process.
+; ~4.6GB, general-purpose small model. Works with json_schema structured output.
 [Meta-Llama-3.1-8B-Instruct-Q4_K_M]
 model        = {{ llm_models_dir }}/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf
+alias        = Meta-Llama-3.1-8B-Instruct-4bit
 n-gpu-layers = {{ llm_router_gpu_layers }}
 ctx-size     = {{ llm_router_ctx_size }}
 cache-type-k = {{ llm_router_cache_type_k }}