fix(llm-inference): switch serve phase from vLLM+bitsandbytes to llama.cpp+GGUF

bitsandbytes peak RAM ~54GB (bf16 load before quantize) — kills 40GB OptiPlex.
llama.cpp Q4_K_M GGUF loads pre-quantized: peak RAM ~15.5GB, fits cleanly.

Changes:
- serve.yml: build llama.cpp with CUDA, download Q4_K_M GGUF from bartowski,
  disable vllm-serve, deploy llama-server.service
- llama-server.service.j2: OpenAI-compatible server on same port 8000,
  --n-gpu-layers 99 (full GPU offload), --parallel 4, gemma chat template
- defaults: llm_gguf_dir, llm_gguf_path, llm_gpu_layers, llm_parallel_slots
- handlers: restart llama-server, vllm-serve failed_when=false (may not exist)

GGUF: bartowski/gemma-2-27b-it-Q4_K_M.gguf (15.5GB, 24GB VRAM fits w/ ~8GB headroom)

ansible/roles/llm-inference/defaults/main.yml

@@ -17,12 +17,19 @@ llm_venv_owner: jarvis
 llm_hf_cache_dir: /home/jarvis/.cache/huggingface
 llm_hf_model: google/gemma-2-27b-it
 
-# vLLM serve
+# vLLM serve (deprecated — replaced by llama-server)
+# llama-server serve
 llm_serve_port: 8000
 llm_serve_host: "0.0.0.0"
-llm_quantization: "bitsandbytes"   # int4 — fits 27B in 24GB VRAM (~14GB vs ~54GB bf16)
+llm_max_model_len: 8192
-llm_gpu_memory_utilization: "0.92" # higher utilization to give KV cache room
+llm_gpu_layers: 99              # offload all layers to GPU
-llm_max_model_len: 4096            # safe ceiling given 15.96GB model + int4
+llm_parallel_slots: 4           # concurrent request slots
+llm_gguf_dir: /home/jarvis/models
+llm_gguf_path: /home/jarvis/models/gemma-2-27b-it-Q4_K_M.gguf
+
+# Legacy vLLM vars (kept for role documentation, not used by llama-server)
+llm_quantization: "bitsandbytes"
+llm_gpu_memory_utilization: "0.92"
 
 # Monitoring
 llm_gpu_exporter_version: "1.13.1"

ansible/roles/llm-inference/handlers/main.yml

@@ -13,6 +13,13 @@
     name: vllm-serve
     state: restarted
   listen: "restart vllm-serve"
+  failed_when: false
+
+- name: Restart llama-server
+  ansible.builtin.systemd:
+    name: llama-server
+    state: restarted
+  listen: "restart llama-server"
 
 - name: Restart Hermes on carousel
   ansible.builtin.systemd:

ansible/roles/llm-inference/tasks/serve.yml

@@ -1,48 +1,116 @@
 ---
 # ------------------------------------------------------------------------------
 # FILE: roles/llm-inference/tasks/serve.yml
-# DESCRIPTION: Phase 5 — systemd vllm-serve service.
+# DESCRIPTION: Phase 5 — llama-server (llama.cpp) serving Gemma 2 27B-it GGUF.
-#              Deploys the service unit, enables on boot, starts it, and
+#
-#              health-checks the OpenAI-compatible API endpoint.
+# WHY llama.cpp instead of vLLM:
+#   vLLM with bitsandbytes int4 quantizes on-the-fly — loads full bf16 weights
+#   (~54GB RAM peak) before compressing, killing the 40GB OptiPlex on warmup.
+#   llama.cpp loads the pre-quantized GGUF directly (~15.5GB peak RAM for Q4_K_M).
+#   No torch.compile, no warmup spike, OpenAI-compatible API on the same port.
+#
+# GGUF source: bartowski/gemma-2-27b-it-GGUF (Q4_K_M, 15.5GB)
+# Model downloaded to: {{ llm_gguf_path }}
 # ------------------------------------------------------------------------------
 
-- name: Deploy vllm-serve systemd service unit
+- name: Install llama.cpp build dependencies
+  ansible.builtin.apt:
+    name:
+      - cmake
+      - build-essential
+      - libcurl4-openssl-dev
+    state: present
+    update_cache: false
+
+- name: Clone llama.cpp repository
+  ansible.builtin.git:
+    repo: https://github.com/ggml-org/llama.cpp.git
+    dest: /opt/llama.cpp
+    depth: 1
+    update: false
+  become: true
+
+- name: Build llama.cpp with CUDA support
+  ansible.builtin.command:
+    cmd: cmake -B build -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Release
+    chdir: /opt/llama.cpp
+    creates: /opt/llama.cpp/build/CMakeCache.txt
+  become: true
+
+- name: Compile llama.cpp (parallel build)
+  ansible.builtin.command:
+    cmd: cmake --build build --config Release --parallel {{ ansible_processor_vcpus }}
+    chdir: /opt/llama.cpp
+    creates: /opt/llama.cpp/build/bin/llama-server
+  become: true
+  timeout: 600
+
+- name: Create GGUF model directory
+  ansible.builtin.file:
+    path: "{{ llm_gguf_dir }}"
+    state: directory
+    owner: "{{ llm_venv_owner }}"
+    group: "{{ llm_venv_owner }}"
+    mode: "0755"
+
+- name: Download Gemma 2 27B Q4_K_M GGUF from HuggingFace
+  ansible.builtin.get_url:
+    url: "https://huggingface.co/bartowski/gemma-2-27b-it-GGUF/resolve/main/gemma-2-27b-it-Q4_K_M.gguf"
+    dest: "{{ llm_gguf_path }}"
+    headers:
+      Authorization: "Bearer {{ vault_hf_token }}"
+    owner: "{{ llm_venv_owner }}"
+    group: "{{ llm_venv_owner }}"
+    mode: "0644"
+    timeout: 7200
+  become: true
+  no_log: true
+
+- name: Disable and stop vllm-serve if present
+  ansible.builtin.systemd:
+    name: vllm-serve
+    state: stopped
+    enabled: false
+  failed_when: false
+  notify: reload systemd
+
+- name: Deploy llama-server systemd service unit
   ansible.builtin.template:
-    src: vllm-serve.service.j2
+    src: llama-server.service.j2
-    dest: /etc/systemd/system/vllm-serve.service
+    dest: /etc/systemd/system/llama-server.service
     owner: root
     group: root
     mode: "0644"
   notify:
     - reload systemd
-    - restart vllm-serve
+    - restart llama-server
 
 - name: Flush handlers to reload systemd before enabling service
   ansible.builtin.meta: flush_handlers
 
-- name: Enable and start vllm-serve
+- name: Enable and start llama-server
   ansible.builtin.systemd:
-    name: vllm-serve
+    name: llama-server
     state: started
     enabled: true
     daemon_reload: true
 
-- name: Wait for vLLM API to become available (model load can take ~60s)
+- name: Wait for llama-server API to become available (model load ~30s)
   ansible.builtin.uri:
     url: "http://localhost:{{ llm_serve_port }}/health"
     status_code: 200
-  register: vllm_health
+  register: llama_health
-  retries: 30
+  retries: 18
   delay: 10
-  until: vllm_health.status == 200
+  until: llama_health.status == 200
 
 - name: Smoke-test — list available models
   ansible.builtin.uri:
     url: "http://localhost:{{ llm_serve_port }}/v1/models"
     status_code: 200
     return_content: true
-  register: vllm_models
+  register: llama_models
 
 - name: Print available models
   ansible.builtin.debug:
-    msg: "vLLM serving: {{ vllm_models.json.data | map(attribute='id') | list }}"
+    msg: "llama-server serving: {{ llama_models.json.data | map(attribute='id') | list }}"

ansible/roles/llm-inference/templates/llama-server.service.j2

@@ -0,0 +1,27 @@
+[Unit]
+Description=llama-server — Gemma 2 27B-it Q4_K_M (OpenAI-compatible inference)
+After=network.target nvidia-persistenced.service
+Wants=nvidia-persistenced.service
+
+[Service]
+Type=simple
+User={{ llm_venv_owner }}
+Group={{ llm_venv_owner }}
+Environment="HOME=/home/{{ llm_venv_owner }}"
+ExecStart=/opt/llama.cpp/build/bin/llama-server \
+    --model {{ llm_gguf_path }} \
+    --host {{ llm_serve_host }} \
+    --port {{ llm_serve_port }} \
+    --ctx-size {{ llm_max_model_len }} \
+    --n-gpu-layers {{ llm_gpu_layers }} \
+    --parallel {{ llm_parallel_slots }} \
+    --chat-template gemma
+Restart=on-failure
+RestartSec=10
+TimeoutStartSec=120
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=llama-server
+
+[Install]
+WantedBy=multi-user.target