diff --git a/ansible/roles/llm-inference/defaults/main.yml b/ansible/roles/llm-inference/defaults/main.yml index b453472..ad6160c 100644 --- a/ansible/roles/llm-inference/defaults/main.yml +++ b/ansible/roles/llm-inference/defaults/main.yml @@ -20,8 +20,9 @@ llm_hf_model: google/gemma-2-27b-it # vLLM serve llm_serve_port: 8000 llm_serve_host: "0.0.0.0" -llm_gpu_memory_utilization: "0.90" -llm_max_model_len: 8192 +llm_quantization: "bitsandbytes" # int4 — fits 27B in 24GB VRAM (~14GB vs ~54GB bf16) +llm_gpu_memory_utilization: "0.92" # higher utilization to give KV cache room +llm_max_model_len: 4096 # safe ceiling given 15.96GB model + int4 # Monitoring llm_gpu_exporter_version: "1.13.1" diff --git a/ansible/roles/llm-inference/tasks/main.yml b/ansible/roles/llm-inference/tasks/main.yml index 5e41f8d..2495269 100644 --- a/ansible/roles/llm-inference/tasks/main.yml +++ b/ansible/roles/llm-inference/tasks/main.yml @@ -3,25 +3,34 @@ # FILE: roles/llm-inference/tasks/main.yml # DESCRIPTION: Entry point — imports one task file per phase. # Phases are additive; re-running the full playbook is always safe. +# Use --tags to run a specific phase subset: +# --tags foundation,driver,vllm,model,serve,integration,monitoring # ------------------------------------------------------------------------------ # Phase 1 — Foundation - import_tasks: foundation.yml + tags: [foundation] # Phase 2 — Driver - import_tasks: driver.yml + tags: [driver] # Phase 3 — vLLM - import_tasks: vllm.yml + tags: [vllm] # Phase 4 — Model - import_tasks: model.yml + tags: [model] # Phase 5 — Serve - import_tasks: serve.yml + tags: [serve] # Phase 6 — Integration - import_tasks: integration.yml + tags: [integration] # Phase 7 — Monitoring - import_tasks: monitoring.yml + tags: [monitoring] diff --git a/ansible/roles/llm-inference/tasks/vllm.yml b/ansible/roles/llm-inference/tasks/vllm.yml index 9fdbf1b..5b7bef9 100644 --- a/ansible/roles/llm-inference/tasks/vllm.yml +++ b/ansible/roles/llm-inference/tasks/vllm.yml @@ -22,9 +22,11 @@ become: true become_user: "{{ llm_venv_owner }}" -- name: Install vLLM +- name: Install vLLM and bitsandbytes ansible.builtin.pip: - name: vllm + name: + - vllm + - bitsandbytes state: present virtualenv: "{{ llm_venv_path }}" become: true diff --git a/ansible/roles/llm-inference/templates/vllm-serve.service.j2 b/ansible/roles/llm-inference/templates/vllm-serve.service.j2 index 20f948e..4445029 100644 --- a/ansible/roles/llm-inference/templates/vllm-serve.service.j2 +++ b/ansible/roles/llm-inference/templates/vllm-serve.service.j2 @@ -14,6 +14,7 @@ ExecStart={{ llm_venv_path }}/bin/python -m vllm.entrypoints.openai.api_server \ --model {{ llm_hf_model }} \ --host {{ llm_serve_host }} \ --port {{ llm_serve_port }} \ + --quantization {{ llm_quantization }} \ --gpu-memory-utilization {{ llm_gpu_memory_utilization }} \ --max-model-len {{ llm_max_model_len }} \ --enable-prefix-caching