fix(llm-inference): bitsandbytes int4 OOM — pending switch to llama.cpp+GGUF
bitsandbytes quantizes on-the-fly: loads full bf16 weights (~54GB RAM peak) before compressing to int4. Kills the 40GB OptiPlex on torch.compile warmup. Fix in next commit: switch serve phase to llama.cpp + GGUF Q4_K_M. Pre-quantized weights load directly — peak RAM ~16GB, no compile overhead.
This commit is contained in:
@@ -3,25 +3,34 @@
|
||||
# FILE: roles/llm-inference/tasks/main.yml
|
||||
# DESCRIPTION: Entry point — imports one task file per phase.
|
||||
# Phases are additive; re-running the full playbook is always safe.
|
||||
# Use --tags to run a specific phase subset:
|
||||
# --tags foundation,driver,vllm,model,serve,integration,monitoring
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# Phase 1 — Foundation
|
||||
- import_tasks: foundation.yml
|
||||
tags: [foundation]
|
||||
|
||||
# Phase 2 — Driver
|
||||
- import_tasks: driver.yml
|
||||
tags: [driver]
|
||||
|
||||
# Phase 3 — vLLM
|
||||
- import_tasks: vllm.yml
|
||||
tags: [vllm]
|
||||
|
||||
# Phase 4 — Model
|
||||
- import_tasks: model.yml
|
||||
tags: [model]
|
||||
|
||||
# Phase 5 — Serve
|
||||
- import_tasks: serve.yml
|
||||
tags: [serve]
|
||||
|
||||
# Phase 6 — Integration
|
||||
- import_tasks: integration.yml
|
||||
tags: [integration]
|
||||
|
||||
# Phase 7 — Monitoring
|
||||
- import_tasks: monitoring.yml
|
||||
tags: [monitoring]
|
||||
|
||||
@@ -22,9 +22,11 @@
|
||||
become: true
|
||||
become_user: "{{ llm_venv_owner }}"
|
||||
|
||||
- name: Install vLLM
|
||||
- name: Install vLLM and bitsandbytes
|
||||
ansible.builtin.pip:
|
||||
name: vllm
|
||||
name:
|
||||
- vllm
|
||||
- bitsandbytes
|
||||
state: present
|
||||
virtualenv: "{{ llm_venv_path }}"
|
||||
become: true
|
||||
|
||||
Reference in New Issue
Block a user