aa8e229e64
bitsandbytes peak RAM ~54GB (bf16 load before quantize) — kills 40GB OptiPlex. llama.cpp Q4_K_M GGUF loads pre-quantized: peak RAM ~15.5GB, fits cleanly. Changes: - serve.yml: build llama.cpp with CUDA, download Q4_K_M GGUF from bartowski, disable vllm-serve, deploy llama-server.service - llama-server.service.j2: OpenAI-compatible server on same port 8000, --n-gpu-layers 99 (full GPU offload), --parallel 4, gemma chat template - defaults: llm_gguf_dir, llm_gguf_path, llm_gpu_layers, llm_parallel_slots - handlers: restart llama-server, vllm-serve failed_when=false (may not exist) GGUF: bartowski/gemma-2-27b-it-Q4_K_M.gguf (15.5GB, 24GB VRAM fits w/ ~8GB headroom)
43 lines
1.0 KiB
YAML
43 lines
1.0 KiB
YAML
---
|
|
# ------------------------------------------------------------------------------
|
|
# FILE: roles/llm-inference/handlers/main.yml
|
|
# ------------------------------------------------------------------------------
|
|
|
|
- name: Reload systemd
|
|
ansible.builtin.systemd:
|
|
daemon_reload: true
|
|
listen: "reload systemd"
|
|
|
|
- name: Restart vllm-serve
|
|
ansible.builtin.systemd:
|
|
name: vllm-serve
|
|
state: restarted
|
|
listen: "restart vllm-serve"
|
|
failed_when: false
|
|
|
|
- name: Restart llama-server
|
|
ansible.builtin.systemd:
|
|
name: llama-server
|
|
state: restarted
|
|
listen: "restart llama-server"
|
|
|
|
- name: Restart Hermes on carousel
|
|
ansible.builtin.systemd:
|
|
name: "{{ item }}"
|
|
state: restarted
|
|
scope: user
|
|
loop:
|
|
- hermes-gateway
|
|
- hermes-dashboard
|
|
become: true
|
|
become_user: wed
|
|
delegate_to: carousel-of-progress
|
|
listen: "restart hermes"
|
|
ignore_errors: true
|
|
|
|
- name: Restart nvidia-gpu-exporter
|
|
ansible.builtin.systemd:
|
|
name: nvidia-gpu-exporter
|
|
state: restarted
|
|
listen: "restart nvidia-gpu-exporter"
|