54 lines
2.1 KiB
Django/Jinja
54 lines
2.1 KiB
Django/Jinja
{#
|
|
FILE: roles/llm-inference-multimodel/templates/llama-swap.service.j2
|
|
DESCRIPTION: llama-swap systemd unit template.
|
|
Single Go binary, no subprocess management — just a /usr/local/bin/llama-swap
|
|
process reading /etc/llama-swap/config.yaml.
|
|
|
|
Design:
|
|
- Type=simple (no forking)
|
|
- User={{ llm_swapmode_service_user }} (jarvis)
|
|
- Restart=on-failure, RestartSec=10
|
|
- Logs to journald (StandardOutput/StandardError=journal)
|
|
- After nvidia-persistenced.service (NVIDIA driver dependency)
|
|
|
|
Config location: /etc/llama-swap/config.yaml (rendered by swapmode_config phase)
|
|
Listen address: 127.0.0.1 inside the container (exposed by --listen flag)
|
|
#}
|
|
[Unit]
|
|
Description=llama-swap — hot-swap model proxy (port {{ llm_swapmode_port }})
|
|
Documentation=https://github.com/mostlygeek/llama-swap
|
|
After=network.target nvidia-persistenced.service
|
|
Wants=nvidia-persistenced.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
User={{ llm_swapmode_service_user }}
|
|
Group={{ llm_swapmode_service_user }}
|
|
Environment="HOME=/home/{{ llm_swapmode_service_user }}"
|
|
|
|
ExecStart=/usr/local/bin/llama-swap \
|
|
--config {{ llm_swapmode_config_file }} \
|
|
--listen {{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}
|
|
|
|
# LLAMA-SWAP NOTES (2026-08-18, t_c1e44190):
|
|
# - Single Go binary, zero runtime dependencies (llama.cpp statically linked).
|
|
# - Upstream servers (llama-server instances) are spawned on-demand per config.yaml model definitions.
|
|
# - --listen can override config.yaml's listen key; this flag takes precedence.
|
|
# Double-check consistency between ExecStart and config.yaml.
|
|
# - CUDA_VISIBLE_DEVICES can be set via Environment= if GPU isolation is needed.
|
|
# Default: inherit from parent (systemd likely has it unset, picks all GPUs).
|
|
# - No jinja flag needed: llama.cpp model templates are embedded in each model's GGUF.
|
|
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
TimeoutStartSec=600
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
SyslogIdentifier=llama-swap
|
|
|
|
# Resource limits (optional; adjust per VRAM budget)
|
|
# MemoryMax=24G # Enforce hard limit; uncomment if runaway is a concern
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|