aa8e229e64
bitsandbytes peak RAM ~54GB (bf16 load before quantize) — kills 40GB OptiPlex. llama.cpp Q4_K_M GGUF loads pre-quantized: peak RAM ~15.5GB, fits cleanly. Changes: - serve.yml: build llama.cpp with CUDA, download Q4_K_M GGUF from bartowski, disable vllm-serve, deploy llama-server.service - llama-server.service.j2: OpenAI-compatible server on same port 8000, --n-gpu-layers 99 (full GPU offload), --parallel 4, gemma chat template - defaults: llm_gguf_dir, llm_gguf_path, llm_gpu_layers, llm_parallel_slots - handlers: restart llama-server, vllm-serve failed_when=false (may not exist) GGUF: bartowski/gemma-2-27b-it-Q4_K_M.gguf (15.5GB, 24GB VRAM fits w/ ~8GB headroom)
117 lines
3.5 KiB
YAML
117 lines
3.5 KiB
YAML
---
|
|
# ------------------------------------------------------------------------------
|
|
# FILE: roles/llm-inference/tasks/serve.yml
|
|
# DESCRIPTION: Phase 5 — llama-server (llama.cpp) serving Gemma 2 27B-it GGUF.
|
|
#
|
|
# WHY llama.cpp instead of vLLM:
|
|
# vLLM with bitsandbytes int4 quantizes on-the-fly — loads full bf16 weights
|
|
# (~54GB RAM peak) before compressing, killing the 40GB OptiPlex on warmup.
|
|
# llama.cpp loads the pre-quantized GGUF directly (~15.5GB peak RAM for Q4_K_M).
|
|
# No torch.compile, no warmup spike, OpenAI-compatible API on the same port.
|
|
#
|
|
# GGUF source: bartowski/gemma-2-27b-it-GGUF (Q4_K_M, 15.5GB)
|
|
# Model downloaded to: {{ llm_gguf_path }}
|
|
# ------------------------------------------------------------------------------
|
|
|
|
- name: Install llama.cpp build dependencies
|
|
ansible.builtin.apt:
|
|
name:
|
|
- cmake
|
|
- build-essential
|
|
- libcurl4-openssl-dev
|
|
state: present
|
|
update_cache: false
|
|
|
|
- name: Clone llama.cpp repository
|
|
ansible.builtin.git:
|
|
repo: https://github.com/ggml-org/llama.cpp.git
|
|
dest: /opt/llama.cpp
|
|
depth: 1
|
|
update: false
|
|
become: true
|
|
|
|
- name: Build llama.cpp with CUDA support
|
|
ansible.builtin.command:
|
|
cmd: cmake -B build -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Release
|
|
chdir: /opt/llama.cpp
|
|
creates: /opt/llama.cpp/build/CMakeCache.txt
|
|
become: true
|
|
|
|
- name: Compile llama.cpp (parallel build)
|
|
ansible.builtin.command:
|
|
cmd: cmake --build build --config Release --parallel {{ ansible_processor_vcpus }}
|
|
chdir: /opt/llama.cpp
|
|
creates: /opt/llama.cpp/build/bin/llama-server
|
|
become: true
|
|
timeout: 600
|
|
|
|
- name: Create GGUF model directory
|
|
ansible.builtin.file:
|
|
path: "{{ llm_gguf_dir }}"
|
|
state: directory
|
|
owner: "{{ llm_venv_owner }}"
|
|
group: "{{ llm_venv_owner }}"
|
|
mode: "0755"
|
|
|
|
- name: Download Gemma 2 27B Q4_K_M GGUF from HuggingFace
|
|
ansible.builtin.get_url:
|
|
url: "https://huggingface.co/bartowski/gemma-2-27b-it-GGUF/resolve/main/gemma-2-27b-it-Q4_K_M.gguf"
|
|
dest: "{{ llm_gguf_path }}"
|
|
headers:
|
|
Authorization: "Bearer {{ vault_hf_token }}"
|
|
owner: "{{ llm_venv_owner }}"
|
|
group: "{{ llm_venv_owner }}"
|
|
mode: "0644"
|
|
timeout: 7200
|
|
become: true
|
|
no_log: true
|
|
|
|
- name: Disable and stop vllm-serve if present
|
|
ansible.builtin.systemd:
|
|
name: vllm-serve
|
|
state: stopped
|
|
enabled: false
|
|
failed_when: false
|
|
notify: reload systemd
|
|
|
|
- name: Deploy llama-server systemd service unit
|
|
ansible.builtin.template:
|
|
src: llama-server.service.j2
|
|
dest: /etc/systemd/system/llama-server.service
|
|
owner: root
|
|
group: root
|
|
mode: "0644"
|
|
notify:
|
|
- reload systemd
|
|
- restart llama-server
|
|
|
|
- name: Flush handlers to reload systemd before enabling service
|
|
ansible.builtin.meta: flush_handlers
|
|
|
|
- name: Enable and start llama-server
|
|
ansible.builtin.systemd:
|
|
name: llama-server
|
|
state: started
|
|
enabled: true
|
|
daemon_reload: true
|
|
|
|
- name: Wait for llama-server API to become available (model load ~30s)
|
|
ansible.builtin.uri:
|
|
url: "http://localhost:{{ llm_serve_port }}/health"
|
|
status_code: 200
|
|
register: llama_health
|
|
retries: 18
|
|
delay: 10
|
|
until: llama_health.status == 200
|
|
|
|
- name: Smoke-test — list available models
|
|
ansible.builtin.uri:
|
|
url: "http://localhost:{{ llm_serve_port }}/v1/models"
|
|
status_code: 200
|
|
return_content: true
|
|
register: llama_models
|
|
|
|
- name: Print available models
|
|
ansible.builtin.debug:
|
|
msg: "llama-server serving: {{ llama_models.json.data | map(attribute='id') | list }}"
|