rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm-inference-multimodel: fix qwen unit -- llama.cpp requires --flash-attn <on|off|auto>, not bare flag

Browse Source
This commit is contained in:
Hermes Agent service account
2026-08-06 09:24:42 -05:00
parent 75cb93f25c
commit d4ff2681ac
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2
View File

@@ -14,7 +14,7 @@ ExecStart={{ llm_binary_path }} \
--port {{ llm_qwen_port }} \ --port {{ llm_qwen_port }} \
--n-gpu-layers {{ llm_qwen_gpu_layers }} \ --n-gpu-layers {{ llm_qwen_gpu_layers }} \
--ctx-size {{ llm_qwen_ctx_size }} \ --ctx-size {{ llm_qwen_ctx_size }} \
--flash-attn \ --flash-attn on \
--cache-type-k q8_0 --cache-type-v q8_0 \ --cache-type-k q8_0 --cache-type-v q8_0 \
--batch-size {{ llm_qwen_batch_size }} --ubatch-size {{ llm_qwen_ubatch_size }} \ --batch-size {{ llm_qwen_batch_size }} --ubatch-size {{ llm_qwen_ubatch_size }} \
--jinja \ --jinja \