fix(llm-inference): bitsandbytes int4 OOM — pending switch to llama.cpp+GGUF

bitsandbytes quantizes on-the-fly: loads full bf16 weights (~54GB RAM peak)
before compressing to int4. Kills the 40GB OptiPlex on torch.compile warmup.

Fix in next commit: switch serve phase to llama.cpp + GGUF Q4_K_M.
Pre-quantized weights load directly — peak RAM ~16GB, no compile overhead.

ansible/roles/llm-inference/defaults/main.yml

@@ -20,8 +20,9 @@ llm_hf_model: google/gemma-2-27b-it
 # vLLM serve
 llm_serve_port: 8000
 llm_serve_host: "0.0.0.0"
-llm_gpu_memory_utilization: "0.90"
+llm_quantization: "bitsandbytes"   # int4 — fits 27B in 24GB VRAM (~14GB vs ~54GB bf16)
-llm_max_model_len: 8192
+llm_gpu_memory_utilization: "0.92" # higher utilization to give KV cache room
+llm_max_model_len: 4096            # safe ceiling given 15.96GB model + int4
 
 # Monitoring
 llm_gpu_exporter_version: "1.13.1"

ansible/roles/llm-inference/tasks/main.yml

@@ -3,25 +3,34 @@
 # FILE: roles/llm-inference/tasks/main.yml
 # DESCRIPTION: Entry point — imports one task file per phase.
 #              Phases are additive; re-running the full playbook is always safe.
+#              Use --tags to run a specific phase subset:
+#                --tags foundation,driver,vllm,model,serve,integration,monitoring
 # ------------------------------------------------------------------------------
 
 # Phase 1 — Foundation
 - import_tasks: foundation.yml
+  tags: [foundation]
 
 # Phase 2 — Driver
 - import_tasks: driver.yml
+  tags: [driver]
 
 # Phase 3 — vLLM
 - import_tasks: vllm.yml
+  tags: [vllm]
 
 # Phase 4 — Model
 - import_tasks: model.yml
+  tags: [model]
 
 # Phase 5 — Serve
 - import_tasks: serve.yml
+  tags: [serve]
 
 # Phase 6 — Integration
 - import_tasks: integration.yml
+  tags: [integration]
 
 # Phase 7 — Monitoring
 - import_tasks: monitoring.yml
+  tags: [monitoring]

ansible/roles/llm-inference/tasks/vllm.yml

@@ -22,9 +22,11 @@
   become: true
   become_user: "{{ llm_venv_owner }}"
 
-- name: Install vLLM
+- name: Install vLLM and bitsandbytes
   ansible.builtin.pip:
-    name: vllm
+    name:
+      - vllm
+      - bitsandbytes
     state: present
     virtualenv: "{{ llm_venv_path }}"
   become: true

ansible/roles/llm-inference/templates/vllm-serve.service.j2

@@ -14,6 +14,7 @@ ExecStart={{ llm_venv_path }}/bin/python -m vllm.entrypoints.openai.api_server \
     --model {{ llm_hf_model }} \
     --host {{ llm_serve_host }} \
     --port {{ llm_serve_port }} \
+    --quantization {{ llm_quantization }} \
     --gpu-memory-utilization {{ llm_gpu_memory_utilization }} \
     --max-model-len {{ llm_max_model_len }} \
     --enable-prefix-caching