Merge origin/main — integrate monitoring/Phase3 updates with Qwen3-8B no-think deployment
Resolved add/add conflicts in: - defaults/main.yml: kept our version (5 original models + Qwen3-8B x2 + rows 5-6) - tasks/swapmode.yml: kept our version (7-model GATE 2 assert) - templates/llama-server-router-preset.ini.j2: kept our version (+Qwen3-8B sections) - templates/llama-swap-config.yaml.j2: kept our version (+chat_template_file support) Remote changes incorporated from origin/main (14 commits): - Ansible Phase 3 integration (llama-swap.service.j2, tasks/monitoring.yml) - Prometheus monitoring: PrometheusRule, Grafana dashboard, scrape config - VRAM exporter script, llama-swap-phase3 cutover results - Day2 playbooks: nomic_embed, cpu_offload_aux, per_model_ctx, qwen38_ctx128k - Router: CPU-offload Coder-14B + Llama-3.1-8B - host_vars/astro-orbiter/vars.yml updates
This commit is contained in:
Hermes Agent service account
@@ -18,9 +18,11 @@ ExecStart={{ llm_binary_path }} \
|
||||
--models-max {{ llm_router_models_max }} \
|
||||
--host {{ llm_router_bind_address }} \
|
||||
--port {{ llm_router_port }} \
|
||||
{% if not (llm_router_preset_enabled | default(false)) %}
|
||||
--n-gpu-layers {{ llm_router_gpu_layers }} \
|
||||
--ctx-size {{ llm_router_ctx_size }} \
|
||||
--flash-attn {{ llm_router_flash_attn }} \
|
||||
{% endif %}
|
||||
--cache-type-k {{ llm_router_cache_type_k }} \
|
||||
--cache-type-v {{ llm_router_cache_type_v }} \
|
||||
--batch-size {{ llm_router_batch_size }} \
|
||||
@@ -28,7 +30,7 @@ ExecStart={{ llm_binary_path }} \
|
||||
--parallel {{ llm_router_parallel }} \
|
||||
--metrics
|
||||
|
||||
# ROUTER MODE NOTES (2026-08-12, t_0cca74a2 / updated t_9adf0889):
|
||||
# ROUTER MODE NOTES (2026-08-12, t_0cca74a2 / updated t_9adf0889 / updated t_72646029):
|
||||
# - NO -m/--model flag: this is what enables llama-server router/supervisor mode.
|
||||
# Without -m, llama-server discovers all .gguf files in --models-dir, or uses
|
||||
# the per-model definitions in a --models-preset INI file.
|
||||
@@ -38,12 +40,16 @@ ExecStart={{ llm_binary_path }} \
|
||||
# The preset INI is at {{ llm_router_preset_path | default('/opt/llama-server-router-preset.ini') }}.
|
||||
# Both the section name and the alias field in the INI work as model IDs.
|
||||
# GH #22364 (extra "default" entry in /v1/models) is expected in preset mode — cosmetic.
|
||||
# - --n-gpu-layers is INTENTIONALLY OMITTED from preset mode (t_72646029, 2026-08-17):
|
||||
# In --models-preset mode every model section in the INI sets n-gpu-layers explicitly.
|
||||
# A global CLI --n-gpu-layers has HIGHEST precedence in llama.cpp (CLI > model-section > global-INI)
|
||||
# and would override per-model INI values (e.g. n-gpu-layers=0 for CPU offload).
|
||||
# When preset mode is disabled (--models-dir), --n-gpu-layers is emitted normally.
|
||||
# - --models-max {{ llm_router_models_max }} is driven by llm_router_models_max
|
||||
# (default 1 in defaults/main.yml; overridden to 4 in host_vars/astro-orbiter
|
||||
# as of t_33acbb2e after VRAM budget review — see host_vars for OOM risk note).
|
||||
# Default llama-server cap is 4 simultaneous — OOM on 24GB if all 3 current
|
||||
# GGUFs load at once. LRU eviction mitigates in practice but review before adding
|
||||
# models. See host_vars/astro-orbiter/vars.yml for full VRAM breakdown.
|
||||
# as of t_72646029 after CPU-offload enabling — CPU models count against models-max
|
||||
# and hold ~1.4-1.7GB CUDA-context VRAM each (llama.cpp 6ea215d allocates it even at
|
||||
# n-gpu-layers=0); steady-state ~24,004 MiB, below the 24,576 MiB physical limit).
|
||||
# - Clients select a model via "model": "<section-name-or-alias>" in their
|
||||
# chat completion request. Hermes sends model: "<id>" on every request already.
|
||||
# - Cold model load on first request: ~30-60s for Qwen3.6-35B. First response
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
# ==============================================================================
|
||||
# FILE: roles/llm-inference-multimodel/templates/llama-swap-alerts.yml.j2
|
||||
# DESCRIPTION: PrometheusRule CustomResource for llama-swap alert rules.
|
||||
# Defines CRITICAL, WARNING, and INFO alerts per the Ciro Luciotta
|
||||
# monitoring pattern (references/monitoring-llm-homelab-ciro-luciotta-2026.md).
|
||||
#
|
||||
# Deployed by ArgoCD as a K8s resource in the monitoring namespace.
|
||||
# Prometheus loads these rules automatically on sync.
|
||||
#
|
||||
# SCOPE: Alerts fire when:
|
||||
# - VRAM exceeds physical limit (24GB) — pending OOM-kill
|
||||
# - KV-cache spills to CPU (>92% utilization) — requests may drop
|
||||
# - Throughput degrades below baseline — model may be throttled
|
||||
#
|
||||
# AUTHOR: Wong (Infrastructure Automation Specialist)
|
||||
# DATE: 2026-08-18
|
||||
# ==============================================================================
|
||||
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: llama-swap-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
prometheus: kube-prometheus
|
||||
spec:
|
||||
groups:
|
||||
- name: llama-swap.rules
|
||||
interval: 30s
|
||||
rules:
|
||||
|
||||
# ====================================================================
|
||||
# CRITICAL: GPU VRAM saturation (OOM risk)
|
||||
# ====================================================================
|
||||
- alert: LlamaSwapVramSaturation
|
||||
expr: (llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576) > {{ llm_swapmode_vram_max_mib | int }}
|
||||
for: 1m
|
||||
labels:
|
||||
severity: critical
|
||||
component: llm-inference
|
||||
annotations:
|
||||
summary: "GPU VRAM saturation on {{ $labels.instance }}"
|
||||
description: |
|
||||
GPU VRAM usage is {{ $value | humanize }}MiB (critical threshold: {{ llm_swapmode_vram_max_mib }}MiB).
|
||||
|
||||
The system is at risk of out-of-memory (OOM) kernel-kill events.
|
||||
Immediate action required:
|
||||
1. Check Prometheus dashboard for request queue depth and active models
|
||||
2. Identify which model(s) are consuming VRAM
|
||||
3. If queue depth is high, consider rate-limiting or routing requests
|
||||
4. If a single request caused the spike, investigate context-window size
|
||||
|
||||
Instance: {{ $labels.instance }}
|
||||
Time: {{ $value | humanizeDuration }}
|
||||
|
||||
# ====================================================================
|
||||
# WARNING: KV-cache spill risk (context cache pressure)
|
||||
# ====================================================================
|
||||
- alert: LlamaSwapVramPressure
|
||||
expr: llamaswap_gpu_memory_util_percent{job="llama-swap"} > 92
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
component: llm-inference
|
||||
annotations:
|
||||
summary: "GPU memory utilization high (possible VRAM pressure)"
|
||||
description: |
|
||||
KV-cache utilization on {{ $labels.model }} is {{ $value | humanizePercentage }}
|
||||
(warning threshold: 92%).
|
||||
|
||||
The model's context cache is nearly full. Requests with large context windows
|
||||
may not fit and could be dropped from the queue. Consider:
|
||||
1. Reviewing incoming request context-window distribution
|
||||
2. Reducing n_ctx for non-critical models (if router mode is active)
|
||||
3. Routing long-context requests to a different model with more capacity
|
||||
4. Investigating whether concurrent requests are competing for KV space
|
||||
|
||||
Model: {{ $labels.model }}
|
||||
Instance: {{ $labels.instance }}
|
||||
|
||||
# ====================================================================
|
||||
# WARNING: Throughput degradation (possible throttling)
|
||||
# ====================================================================
|
||||
- alert: LlamaSwapInferenceStall
|
||||
expr: |
|
||||
(llamaswap_gpu_util_percent{job="llama-swap"} == 0) and (llamaswap_gpu_memory_util_percent{job="llama-swap"} > 50)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
component: llm-inference
|
||||
annotations:
|
||||
summary: "GPU compute stall detected (memory loaded but no utilization)"
|
||||
description: |
|
||||
The RTX 3090 has >50% memory utilization but 0% compute utilization
|
||||
for more than 5 minutes. This may indicate:
|
||||
|
||||
This may indicate:
|
||||
1. Thermal throttling (GPU temperature limiting frequency)
|
||||
2. Memory pressure (even if VRAM not full, latency can increase)
|
||||
3. CPU contention (if models are CPU-offloaded)
|
||||
4. Incoming request rate exceeds model capacity (check queue depth)
|
||||
|
||||
Recommended actions:
|
||||
- Check nvidia-smi output for GPU temperature and throttle flags
|
||||
- Compare queue depth to baseline (alert if >5 sustained)
|
||||
- Check CPU usage and interrupt frequency (vmstat 1 1)
|
||||
- Review log tail for errors or warnings from llama-swap
|
||||
|
||||
Model: {{ $labels.model }}
|
||||
Instance: {{ $labels.instance }}
|
||||
|
||||
# ====================================================================
|
||||
# INFO: Scrape failures (monitoring health)
|
||||
# ====================================================================
|
||||
- alert: LlamaSwapScrapeFailed
|
||||
expr: up{job="llama-swap"} == 0
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
component: monitoring
|
||||
annotations:
|
||||
summary: "llama-swap Prometheus scrape failed"
|
||||
description: |
|
||||
Prometheus cannot scrape llama-swap's /metrics endpoint at
|
||||
http://{{ $labels.instance }}/metrics (HTTP {{ $value }} or timeout).
|
||||
|
||||
The monitoring pipeline is degraded. Check:
|
||||
1. llama-swap service status: systemctl status llama-swap
|
||||
2. Network reachability: curl http://{{ $labels.instance }}/metrics
|
||||
3. Prometheus scrape logs in Prometheus UI (Alerts -> llama-swap)
|
||||
|
||||
Instance: {{ $labels.instance }}
|
||||
@@ -0,0 +1,534 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": "-- Grafana --",
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"gnetId": null,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"panels": [
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "MiB",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"max": 24576,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 23000
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 24000
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"last",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "llamaswap_gpu_memory_used_bytes{job=\"llama-swap\"} / 1048576",
|
||||
"interval": "",
|
||||
"legendFormat": "VRAM Used",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "GPU VRAM Usage (MiB)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"mappings": [],
|
||||
"max": 1,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.8
|
||||
},
|
||||
{
|
||||
"color": "orange",
|
||||
"value": 0.92
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0.95
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"values": false,
|
||||
"fields": "",
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
]
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "llamaswap_gpu_memory_util_percent{job=\"llama-swap\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ model }}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "GPU Memory Utilization %",
|
||||
"type": "gauge"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "%",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "llamaswap_gpu_util_percent{job=\"llama-swap\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ model }}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "GPU Utilization %",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "%",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 3
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 5
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg(llamaswap_cpu_util_percent{job=\"llama-swap\"})",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ model }}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "CPU Utilization %",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "W",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 0,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "watt"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"mean"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "llamaswap_gpu_power_draw_watts{job=\"llama-swap\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "{{ model }} (tokens/min)",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "GPU Power Draw (W)",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": "Prometheus",
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisLabel": "load",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"drawStyle": "bars",
|
||||
"fillOpacity": 100,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"tooltip": false,
|
||||
"viz": false,
|
||||
"legend": false
|
||||
},
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "never",
|
||||
"spanNulls": true,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "normal"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 16
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single"
|
||||
}
|
||||
},
|
||||
"pluginVersion": "8.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "llamaswap_load_average{interval=\"5m\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "p95 latency",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "llamaswap_load_average{interval=\"5m\"}",
|
||||
"interval": "",
|
||||
"legendFormat": "p99 latency",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "System Load Average (5m)",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 27,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"llm",
|
||||
"llama-swap",
|
||||
"gpu-monitoring",
|
||||
"ciro-luciotta"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-24h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "",
|
||||
"title": {{ llm_grafana_dashboard_title }},
|
||||
"uid": {{ llm_grafana_dashboard_uid }},
|
||||
"version": 1
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
# ==============================================================================
|
||||
# FILE: roles/llm-inference-multimodel/templates/llama-swap-prometheus-scrape.yml.j2
|
||||
# DESCRIPTION: Prometheus scrape job configuration for llama-swap's native
|
||||
# /metrics endpoint (OpenMetrics format).
|
||||
#
|
||||
# This template is rendered and deployed to the Prometheus
|
||||
# config via GitOps (cluster/applications/monitoring/values.yaml).
|
||||
# Does NOT include this file inline here; it is referenced and
|
||||
# rendered by Ansible roles/llm-inference-multimodel/tasks/*.yml.
|
||||
#
|
||||
# TARGET HOST: astro-orbiter ({{ llm_bind_address }}:{{ llm_swapmode_port }})
|
||||
# METRICS: llamacpp_tokens_predicted_total, llamacpp_kv_cache_usage_ratio,
|
||||
# llamacpp_time_predict_ms, llamacpp_queue_size, etc. (per llama.cpp)
|
||||
#
|
||||
# AUTHOR: Wong (Infrastructure Automation Specialist)
|
||||
# DATE: 2026-08-18
|
||||
# ==============================================================================
|
||||
|
||||
---
|
||||
- job_name: llama-swap
|
||||
static_configs:
|
||||
- targets: ["{{ llm_bind_address }}:{{ llm_swapmode_port }}"]
|
||||
labels:
|
||||
component: llm-inference
|
||||
service: llama-swap
|
||||
environment: homelab
|
||||
scrape_interval: 30s
|
||||
scrape_timeout: 10s
|
||||
honor_labels: true
|
||||
metrics_path: /metrics
|
||||
|
||||
# Relabeling: extract model name from metric labels for dashboard grouping
|
||||
# llama-swap exposes llamaswap_* metrics (GPU VRAM, utilization, power, CPU,
|
||||
# network, load average). Per-model inference metrics are not available at the
|
||||
# proxy level. Filter to keep only llamaswap_* metrics to reduce cardinality.
|
||||
metric_relabel_configs:
|
||||
- source_labels: [__name__]
|
||||
regex: 'llamaswap_.*'
|
||||
action: keep
|
||||
@@ -0,0 +1,53 @@
|
||||
{#
|
||||
FILE: roles/llm-inference-multimodel/templates/llama-swap.service.j2
|
||||
DESCRIPTION: llama-swap systemd unit template.
|
||||
Single Go binary, no subprocess management — just a /usr/local/bin/llama-swap
|
||||
process reading /etc/llama-swap/config.yaml.
|
||||
|
||||
Design:
|
||||
- Type=simple (no forking)
|
||||
- User={{ llm_swapmode_service_user }} (jarvis)
|
||||
- Restart=on-failure, RestartSec=10
|
||||
- Logs to journald (StandardOutput/StandardError=journal)
|
||||
- After nvidia-persistenced.service (NVIDIA driver dependency)
|
||||
|
||||
Config location: /etc/llama-swap/config.yaml (rendered by swapmode_config phase)
|
||||
Listen address: 127.0.0.1 inside the container (exposed by --listen flag)
|
||||
#}
|
||||
[Unit]
|
||||
Description=llama-swap — hot-swap model proxy (port {{ llm_swapmode_port }})
|
||||
Documentation=https://github.com/mostlygeek/llama-swap
|
||||
After=network.target nvidia-persistenced.service
|
||||
Wants=nvidia-persistenced.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User={{ llm_swapmode_service_user }}
|
||||
Group={{ llm_swapmode_service_user }}
|
||||
Environment="HOME=/home/{{ llm_swapmode_service_user }}"
|
||||
|
||||
ExecStart=/usr/local/bin/llama-swap \
|
||||
--config {{ llm_swapmode_config_file }} \
|
||||
--listen {{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}
|
||||
|
||||
# LLAMA-SWAP NOTES (2026-08-18, t_c1e44190):
|
||||
# - Single Go binary, zero runtime dependencies (llama.cpp statically linked).
|
||||
# - Upstream servers (llama-server instances) are spawned on-demand per config.yaml model definitions.
|
||||
# - --listen can override config.yaml's listen key; this flag takes precedence.
|
||||
# Double-check consistency between ExecStart and config.yaml.
|
||||
# - CUDA_VISIBLE_DEVICES can be set via Environment= if GPU isolation is needed.
|
||||
# Default: inherit from parent (systemd likely has it unset, picks all GPUs).
|
||||
# - No jinja flag needed: llama.cpp model templates are embedded in each model's GGUF.
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=10
|
||||
TimeoutStartSec=600
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=llama-swap
|
||||
|
||||
# Resource limits (optional; adjust per VRAM budget)
|
||||
# MemoryMax=24G # Enforce hard limit; uncomment if runaway is a concern
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in New Issue
Block a user