homelab/ansible/roles/llm-inference/tasks/serve.yml

---
# ------------------------------------------------------------------------------
# FILE: roles/llm-inference/tasks/serve.yml
# DESCRIPTION: Phase 5 — llama-server (llama.cpp) serving Gemma 2 27B-it GGUF.
#
# WHY llama.cpp instead of vLLM:
#   vLLM with bitsandbytes int4 quantizes on-the-fly — loads full bf16 weights
#   (~54GB RAM peak) before compressing, killing the 40GB OptiPlex on warmup.
#   llama.cpp loads the pre-quantized GGUF directly (~15.5GB peak RAM for Q4_K_M).
#   No torch.compile, no warmup spike, OpenAI-compatible API on the same port.
#
# GGUF source: bartowski/gemma-2-27b-it-GGUF (Q4_K_M, 15.5GB)
# Model downloaded to: {{ llm_gguf_path }}
# ------------------------------------------------------------------------------

- name: Install llama.cpp build dependencies
  ansible.builtin.apt:
    name:
      - cmake
      - build-essential
      - libcurl4-openssl-dev
    state: present
    update_cache: false

# NOTE: nvidia-driver-595-open provides the runtime driver only (nvidia-smi,
# libcuda.so) — it does NOT ship nvcc/CUDA headers needed to build GGML_CUDA=ON.
# Ubuntu 24.04's nvidia-cuda-toolkit (12.0.x) is sufficient to build llama.cpp
# against; it does not need to match the 595 driver's CUDA 13.2 runtime version.
- name: Install NVIDIA CUDA toolkit (nvcc) for building llama.cpp with CUDA support
  ansible.builtin.apt:
    name: nvidia-cuda-toolkit
    state: present
    update_cache: false
  become: true

- name: Clone llama.cpp repository
  ansible.builtin.git:
    repo: https://github.com/ggml-org/llama.cpp.git
    dest: /opt/llama.cpp
    depth: 1
    update: false
  become: true

- name: Check for incomplete/stale llama.cpp CMake configuration
  ansible.builtin.stat:
    path: /opt/llama.cpp/build/Makefile
  register: llama_cmake_generated

- name: Remove stale llama.cpp build dir if CMake configure never completed
  ansible.builtin.file:
    path: /opt/llama.cpp/build
    state: absent
  become: true
  when:
    - not llama_cmake_generated.stat.exists
    - not (ansible_check_mode | default(false))

- name: Build llama.cpp with CUDA support
  ansible.builtin.command:
    cmd: cmake -B build -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Release
    chdir: /opt/llama.cpp
    creates: /opt/llama.cpp/build/CMakeCache.txt
  become: true

- name: Compile llama.cpp (parallel build)
  ansible.builtin.command:
    cmd: cmake --build build --config Release --parallel {{ ansible_processor_vcpus }}
    chdir: /opt/llama.cpp
    creates: /opt/llama.cpp/build/bin/llama-server
  become: true
  timeout: 600

- name: Create GGUF model directory
  ansible.builtin.file:
    path: "{{ llm_gguf_dir }}"
    state: directory
    owner: "{{ llm_venv_owner }}"
    group: "{{ llm_venv_owner }}"
    mode: "0755"

- name: Check whether GGUF already exists (avoid re-downloading 16.6GB on every run)
  ansible.builtin.stat:
    path: "{{ llm_gguf_path }}"
  register: llm_gguf_stat

- name: Download Gemma 2 27B Q4_K_M GGUF from HuggingFace
  ansible.builtin.get_url:
    url: "https://huggingface.co/bartowski/gemma-2-27b-it-GGUF/resolve/main/gemma-2-27b-it-Q4_K_M.gguf"
    dest: "{{ llm_gguf_path }}"
    headers:
      Authorization: "Bearer {{ vault_hf_token }}"
    owner: "{{ llm_venv_owner }}"
    group: "{{ llm_venv_owner }}"
    mode: "0644"
    timeout: 7200
    force: false
  become: true
  no_log: true
  # Idempotency: skip entirely once the file exists and is reasonably sized
  # (the finished GGUF is ~16.6GB; guard against a truncated partial download
  # being mistaken for complete by only trusting files > 15GB).
  when: not llm_gguf_stat.stat.exists or (llm_gguf_stat.stat.size | int) < 15000000000

- name: Disable and stop vllm-serve if present
  ansible.builtin.systemd:
    name: vllm-serve
    state: stopped
    enabled: false
  failed_when: false
  notify: reload systemd

- name: Deploy llama-server systemd service unit
  ansible.builtin.template:
    src: llama-server.service.j2
    dest: /etc/systemd/system/llama-server.service
    owner: root
    group: root
    mode: "0644"
  notify:
    - reload systemd
    - restart llama-server

- name: Flush handlers to reload systemd before enabling service
  ansible.builtin.meta: flush_handlers

- name: Enable and start llama-server
  ansible.builtin.systemd:
    name: llama-server
    state: started
    enabled: true
    daemon_reload: true

- name: Wait for llama-server API to become available (model load ~30s)
  ansible.builtin.uri:
    url: "http://localhost:{{ llm_serve_port }}/health"
    status_code: 200
  register: llama_health
  retries: 18
  delay: 10
  until: llama_health.status == 200

- name: Smoke-test — list available models
  ansible.builtin.uri:
    url: "http://localhost:{{ llm_serve_port }}/v1/models"
    status_code: 200
    return_content: true
  register: llama_models

- name: Print available models
  ansible.builtin.debug:
    msg: "llama-server serving: {{ llama_models.json.data | map(attribute='id') | list }}"