From 22a020e4c7f400d1d313a8021b7279b99164d89d Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Mon, 3 Aug 2026 12:35:46 -0500 Subject: [PATCH] =?UTF-8?q?fix(llm-inference):=20bitsandbytes=20int4=20OOM?= =?UTF-8?q?=20=E2=80=94=20pending=20switch=20to=20llama.cpp+GGUF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bitsandbytes quantizes on-the-fly: loads full bf16 weights (~54GB RAM peak) before compressing to int4. Kills the 40GB OptiPlex on torch.compile warmup. Fix in next commit: switch serve phase to llama.cpp + GGUF Q4_K_M. Pre-quantized weights load directly — peak RAM ~16GB, no compile overhead. --- ansible/roles/llm-inference/defaults/main.yml | 5 +++-- ansible/roles/llm-inference/tasks/main.yml | 9 +++++++++ ansible/roles/llm-inference/tasks/vllm.yml | 6 ++++-- .../roles/llm-inference/templates/vllm-serve.service.j2 | 1 + 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/ansible/roles/llm-inference/defaults/main.yml b/ansible/roles/llm-inference/defaults/main.yml index b453472..ad6160c 100644 --- a/ansible/roles/llm-inference/defaults/main.yml +++ b/ansible/roles/llm-inference/defaults/main.yml @@ -20,8 +20,9 @@ llm_hf_model: google/gemma-2-27b-it # vLLM serve llm_serve_port: 8000 llm_serve_host: "0.0.0.0" -llm_gpu_memory_utilization: "0.90" -llm_max_model_len: 8192 +llm_quantization: "bitsandbytes" # int4 — fits 27B in 24GB VRAM (~14GB vs ~54GB bf16) +llm_gpu_memory_utilization: "0.92" # higher utilization to give KV cache room +llm_max_model_len: 4096 # safe ceiling given 15.96GB model + int4 # Monitoring llm_gpu_exporter_version: "1.13.1" diff --git a/ansible/roles/llm-inference/tasks/main.yml b/ansible/roles/llm-inference/tasks/main.yml index 5e41f8d..2495269 100644 --- a/ansible/roles/llm-inference/tasks/main.yml +++ b/ansible/roles/llm-inference/tasks/main.yml @@ -3,25 +3,34 @@ # FILE: roles/llm-inference/tasks/main.yml # DESCRIPTION: Entry point — imports one task file per phase. # Phases are additive; re-running the full playbook is always safe. +# Use --tags to run a specific phase subset: +# --tags foundation,driver,vllm,model,serve,integration,monitoring # ------------------------------------------------------------------------------ # Phase 1 — Foundation - import_tasks: foundation.yml + tags: [foundation] # Phase 2 — Driver - import_tasks: driver.yml + tags: [driver] # Phase 3 — vLLM - import_tasks: vllm.yml + tags: [vllm] # Phase 4 — Model - import_tasks: model.yml + tags: [model] # Phase 5 — Serve - import_tasks: serve.yml + tags: [serve] # Phase 6 — Integration - import_tasks: integration.yml + tags: [integration] # Phase 7 — Monitoring - import_tasks: monitoring.yml + tags: [monitoring] diff --git a/ansible/roles/llm-inference/tasks/vllm.yml b/ansible/roles/llm-inference/tasks/vllm.yml index 9fdbf1b..5b7bef9 100644 --- a/ansible/roles/llm-inference/tasks/vllm.yml +++ b/ansible/roles/llm-inference/tasks/vllm.yml @@ -22,9 +22,11 @@ become: true become_user: "{{ llm_venv_owner }}" -- name: Install vLLM +- name: Install vLLM and bitsandbytes ansible.builtin.pip: - name: vllm + name: + - vllm + - bitsandbytes state: present virtualenv: "{{ llm_venv_path }}" become: true diff --git a/ansible/roles/llm-inference/templates/vllm-serve.service.j2 b/ansible/roles/llm-inference/templates/vllm-serve.service.j2 index 20f948e..4445029 100644 --- a/ansible/roles/llm-inference/templates/vllm-serve.service.j2 +++ b/ansible/roles/llm-inference/templates/vllm-serve.service.j2 @@ -14,6 +14,7 @@ ExecStart={{ llm_venv_path }}/bin/python -m vllm.entrypoints.openai.api_server \ --model {{ llm_hf_model }} \ --host {{ llm_serve_host }} \ --port {{ llm_serve_port }} \ + --quantization {{ llm_quantization }} \ --gpu-memory-utilization {{ llm_gpu_memory_utilization }} \ --max-model-len {{ llm_max_model_len }} \ --enable-prefix-caching