aff792a061
- Prometheus scrape configs for node/gpu/llama-server exporters on astro-orbiter now declared in cluster/applications/monitoring/values.yaml (additionalScrapeConfigs), applied via ArgoCD sync instead of an imperative kubectl secret patch from the Ansible role. - Grafana dashboard for astro-orbiter LLM inference added as a ConfigMap in cluster/applications/monitoring/dashboards.yaml (grafana_dashboard=1 sidecar label), replacing the role's ad-hoc kubectl apply of a rendered Jinja template. - ansible/roles/llm-inference/tasks/monitoring.yml: removed the kubectl get/patch/apply tasks and orphaned grafana-llm-dashboard.json.j2 template; role now only stands up node_exporter + nvidia_gpu_exporter and verifies they're reachable — cluster-facing config lives in Git. - host_vars/vars.yml + inventory.yml: finalize astro-orbiter as the llama.cpp/RTX 3090 host (jarvis user, ssh key), drop stale Ollama/AMD vars and ollama_server inventory group superseded by the ATX rebuild.
33 lines
1.1 KiB
Django/Jinja
33 lines
1.1 KiB
Django/Jinja
[Unit]
|
|
Description=llama-server — Gemma 2 27B-it Q4_K_M (OpenAI-compatible inference)
|
|
After=network.target nvidia-persistenced.service
|
|
Wants=nvidia-persistenced.service
|
|
|
|
[Service]
|
|
Type=simple
|
|
User={{ llm_venv_owner }}
|
|
Group={{ llm_venv_owner }}
|
|
Environment="HOME=/home/{{ llm_venv_owner }}"
|
|
ExecStart=/opt/llama.cpp/build/bin/llama-server \
|
|
--model {{ llm_gguf_path }} \
|
|
--host {{ llm_serve_host }} \
|
|
--port {{ llm_serve_port }} \
|
|
--ctx-size {{ llm_max_model_len }} \
|
|
--n-gpu-layers {{ llm_gpu_layers }} \
|
|
--parallel {{ llm_parallel_slots }} \
|
|
--metrics
|
|
# NOTE: no --chat-template flag — llama-server auto-detects and uses the
|
|
# GGUF's own embedded Jinja chat template (verified correct Gemma-2
|
|
# start_of_turn/end_of_turn format for bartowski's gemma-2-27b-it-Q4_K_M).
|
|
# The built-in "--chat-template gemma" name does NOT match this model's
|
|
# expected format on this llama.cpp build and produced garbled completions.
|
|
Restart=on-failure
|
|
RestartSec=10
|
|
TimeoutStartSec=120
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
SyslogIdentifier=llama-server
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|