From aa8e229e64f1e1c2dcf9612a0e57f9c043a38dda Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Mon, 3 Aug 2026 12:37:03 -0500 Subject: [PATCH] fix(llm-inference): switch serve phase from vLLM+bitsandbytes to llama.cpp+GGUF MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bitsandbytes peak RAM ~54GB (bf16 load before quantize) — kills 40GB OptiPlex. llama.cpp Q4_K_M GGUF loads pre-quantized: peak RAM ~15.5GB, fits cleanly. Changes: - serve.yml: build llama.cpp with CUDA, download Q4_K_M GGUF from bartowski, disable vllm-serve, deploy llama-server.service - llama-server.service.j2: OpenAI-compatible server on same port 8000, --n-gpu-layers 99 (full GPU offload), --parallel 4, gemma chat template - defaults: llm_gguf_dir, llm_gguf_path, llm_gpu_layers, llm_parallel_slots - handlers: restart llama-server, vllm-serve failed_when=false (may not exist) GGUF: bartowski/gemma-2-27b-it-Q4_K_M.gguf (15.5GB, 24GB VRAM fits w/ ~8GB headroom) --- ansible/roles/llm-inference/defaults/main.yml | 15 ++- ansible/roles/llm-inference/handlers/main.yml | 7 ++ ansible/roles/llm-inference/tasks/serve.yml | 98 ++++++++++++++++--- .../templates/llama-server.service.j2 | 27 +++++ 4 files changed, 128 insertions(+), 19 deletions(-) create mode 100644 ansible/roles/llm-inference/templates/llama-server.service.j2 diff --git a/ansible/roles/llm-inference/defaults/main.yml b/ansible/roles/llm-inference/defaults/main.yml index ad6160c..007ddbf 100644 --- a/ansible/roles/llm-inference/defaults/main.yml +++ b/ansible/roles/llm-inference/defaults/main.yml @@ -17,12 +17,19 @@ llm_venv_owner: jarvis llm_hf_cache_dir: /home/jarvis/.cache/huggingface llm_hf_model: google/gemma-2-27b-it -# vLLM serve +# vLLM serve (deprecated — replaced by llama-server) +# llama-server serve llm_serve_port: 8000 llm_serve_host: "0.0.0.0" -llm_quantization: "bitsandbytes" # int4 — fits 27B in 24GB VRAM (~14GB vs ~54GB bf16) -llm_gpu_memory_utilization: "0.92" # higher utilization to give KV cache room -llm_max_model_len: 4096 # safe ceiling given 15.96GB model + int4 +llm_max_model_len: 8192 +llm_gpu_layers: 99 # offload all layers to GPU +llm_parallel_slots: 4 # concurrent request slots +llm_gguf_dir: /home/jarvis/models +llm_gguf_path: /home/jarvis/models/gemma-2-27b-it-Q4_K_M.gguf + +# Legacy vLLM vars (kept for role documentation, not used by llama-server) +llm_quantization: "bitsandbytes" +llm_gpu_memory_utilization: "0.92" # Monitoring llm_gpu_exporter_version: "1.13.1" diff --git a/ansible/roles/llm-inference/handlers/main.yml b/ansible/roles/llm-inference/handlers/main.yml index 14fa54e..a537089 100644 --- a/ansible/roles/llm-inference/handlers/main.yml +++ b/ansible/roles/llm-inference/handlers/main.yml @@ -13,6 +13,13 @@ name: vllm-serve state: restarted listen: "restart vllm-serve" + failed_when: false + +- name: Restart llama-server + ansible.builtin.systemd: + name: llama-server + state: restarted + listen: "restart llama-server" - name: Restart Hermes on carousel ansible.builtin.systemd: diff --git a/ansible/roles/llm-inference/tasks/serve.yml b/ansible/roles/llm-inference/tasks/serve.yml index 869e228..96feceb 100644 --- a/ansible/roles/llm-inference/tasks/serve.yml +++ b/ansible/roles/llm-inference/tasks/serve.yml @@ -1,48 +1,116 @@ --- # ------------------------------------------------------------------------------ # FILE: roles/llm-inference/tasks/serve.yml -# DESCRIPTION: Phase 5 — systemd vllm-serve service. -# Deploys the service unit, enables on boot, starts it, and -# health-checks the OpenAI-compatible API endpoint. +# DESCRIPTION: Phase 5 — llama-server (llama.cpp) serving Gemma 2 27B-it GGUF. +# +# WHY llama.cpp instead of vLLM: +# vLLM with bitsandbytes int4 quantizes on-the-fly — loads full bf16 weights +# (~54GB RAM peak) before compressing, killing the 40GB OptiPlex on warmup. +# llama.cpp loads the pre-quantized GGUF directly (~15.5GB peak RAM for Q4_K_M). +# No torch.compile, no warmup spike, OpenAI-compatible API on the same port. +# +# GGUF source: bartowski/gemma-2-27b-it-GGUF (Q4_K_M, 15.5GB) +# Model downloaded to: {{ llm_gguf_path }} # ------------------------------------------------------------------------------ -- name: Deploy vllm-serve systemd service unit +- name: Install llama.cpp build dependencies + ansible.builtin.apt: + name: + - cmake + - build-essential + - libcurl4-openssl-dev + state: present + update_cache: false + +- name: Clone llama.cpp repository + ansible.builtin.git: + repo: https://github.com/ggml-org/llama.cpp.git + dest: /opt/llama.cpp + depth: 1 + update: false + become: true + +- name: Build llama.cpp with CUDA support + ansible.builtin.command: + cmd: cmake -B build -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Release + chdir: /opt/llama.cpp + creates: /opt/llama.cpp/build/CMakeCache.txt + become: true + +- name: Compile llama.cpp (parallel build) + ansible.builtin.command: + cmd: cmake --build build --config Release --parallel {{ ansible_processor_vcpus }} + chdir: /opt/llama.cpp + creates: /opt/llama.cpp/build/bin/llama-server + become: true + timeout: 600 + +- name: Create GGUF model directory + ansible.builtin.file: + path: "{{ llm_gguf_dir }}" + state: directory + owner: "{{ llm_venv_owner }}" + group: "{{ llm_venv_owner }}" + mode: "0755" + +- name: Download Gemma 2 27B Q4_K_M GGUF from HuggingFace + ansible.builtin.get_url: + url: "https://huggingface.co/bartowski/gemma-2-27b-it-GGUF/resolve/main/gemma-2-27b-it-Q4_K_M.gguf" + dest: "{{ llm_gguf_path }}" + headers: + Authorization: "Bearer {{ vault_hf_token }}" + owner: "{{ llm_venv_owner }}" + group: "{{ llm_venv_owner }}" + mode: "0644" + timeout: 7200 + become: true + no_log: true + +- name: Disable and stop vllm-serve if present + ansible.builtin.systemd: + name: vllm-serve + state: stopped + enabled: false + failed_when: false + notify: reload systemd + +- name: Deploy llama-server systemd service unit ansible.builtin.template: - src: vllm-serve.service.j2 - dest: /etc/systemd/system/vllm-serve.service + src: llama-server.service.j2 + dest: /etc/systemd/system/llama-server.service owner: root group: root mode: "0644" notify: - reload systemd - - restart vllm-serve + - restart llama-server - name: Flush handlers to reload systemd before enabling service ansible.builtin.meta: flush_handlers -- name: Enable and start vllm-serve +- name: Enable and start llama-server ansible.builtin.systemd: - name: vllm-serve + name: llama-server state: started enabled: true daemon_reload: true -- name: Wait for vLLM API to become available (model load can take ~60s) +- name: Wait for llama-server API to become available (model load ~30s) ansible.builtin.uri: url: "http://localhost:{{ llm_serve_port }}/health" status_code: 200 - register: vllm_health - retries: 30 + register: llama_health + retries: 18 delay: 10 - until: vllm_health.status == 200 + until: llama_health.status == 200 - name: Smoke-test — list available models ansible.builtin.uri: url: "http://localhost:{{ llm_serve_port }}/v1/models" status_code: 200 return_content: true - register: vllm_models + register: llama_models - name: Print available models ansible.builtin.debug: - msg: "vLLM serving: {{ vllm_models.json.data | map(attribute='id') | list }}" + msg: "llama-server serving: {{ llama_models.json.data | map(attribute='id') | list }}" diff --git a/ansible/roles/llm-inference/templates/llama-server.service.j2 b/ansible/roles/llm-inference/templates/llama-server.service.j2 new file mode 100644 index 0000000..7554f71 --- /dev/null +++ b/ansible/roles/llm-inference/templates/llama-server.service.j2 @@ -0,0 +1,27 @@ +[Unit] +Description=llama-server — Gemma 2 27B-it Q4_K_M (OpenAI-compatible inference) +After=network.target nvidia-persistenced.service +Wants=nvidia-persistenced.service + +[Service] +Type=simple +User={{ llm_venv_owner }} +Group={{ llm_venv_owner }} +Environment="HOME=/home/{{ llm_venv_owner }}" +ExecStart=/opt/llama.cpp/build/bin/llama-server \ + --model {{ llm_gguf_path }} \ + --host {{ llm_serve_host }} \ + --port {{ llm_serve_port }} \ + --ctx-size {{ llm_max_model_len }} \ + --n-gpu-layers {{ llm_gpu_layers }} \ + --parallel {{ llm_parallel_slots }} \ + --chat-template gemma +Restart=on-failure +RestartSec=10 +TimeoutStartSec=120 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=llama-server + +[Install] +WantedBy=multi-user.target