Changes: - host_vars/astro-orbiter/vars.yml: add llm_router_models_max: 4 (overrides conservative default of 1). Detailed VRAM OOM risk note included inline: worst-case 3-model co-residency ~31GB > 24GB RTX 3090. LRU eviction mitigates in single-user operation; flagged for Ryan's review. - playbooks/day2_bump_router_models_max.yml: new targeted playbook; deploys updated router unit, restarts the live service, verifies /health 200 and /v1/models lists all 3 GGUFs post-restart. - group_vars/all/semaphore.yml: add llm_router_update_unit template pointing at the new playbook. - roles/llm-inference-multimodel/defaults/main.yml: update comment to reflect the var is now overridden in host_vars rather than 'hardcoded to 1'. - roles/llm-inference-multimodel/templates/llama-server-router.service.j2: correct stale 'HARDCODED TO 1' comment — value is variable-driven. Constraints honored: - --parallel 1 left untouched (not in scope, not modified anywhere) - No ad-hoc SSH/systemctl/curl state mutation; all execution via Semaphore - No installed/vendored code patched
194 lines
7.8 KiB
YAML
194 lines
7.8 KiB
YAML
---
|
|
# ============================================================================
|
|
# Semaphore configuration-as-code
|
|
# ============================================================================
|
|
# Drives a freshly-deployed Semaphore instance into its desired state via
|
|
# the Semaphore REST API. Idempotent: every object is checked first; only
|
|
# missing ones are created. Existing objects are left alone.
|
|
#
|
|
# Loaded from group_vars/all/semaphore.yml so that the configuration is
|
|
# version-controlled in the homelab repo and survives a wipe-and-redeploy
|
|
# of the Semaphore VM.
|
|
# ============================================================================
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# API connection (defaults to the local Traefik-fronted service-name URL).
|
|
# Override semaphore_api_url to point at a specific instance if needed.
|
|
# ---------------------------------------------------------------------------
|
|
semaphore_api_url: "https://semaphore.local.mk-labs.cloud/api"
|
|
semaphore_api_validate_certs: true
|
|
semaphore_api_token: "{{ vault_semaphore_api_token }}"
|
|
|
|
# Feature flag — keeps day1_deploy_semaphore.yml deploy-only by default.
|
|
# Set true to also run the configuration pass.
|
|
semaphore_configure: false
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Declarative configuration of the Semaphore instance.
|
|
# ---------------------------------------------------------------------------
|
|
#
|
|
# Top-level shape:
|
|
#
|
|
# semaphore_config:
|
|
# project: single dict — the lab uses one project ("mk-labs")
|
|
# keys: list of credentials Semaphore stores
|
|
# repositories: git repos Semaphore can clone
|
|
# inventories: Ansible inventories from those repos
|
|
# environments: env-var bundles
|
|
# templates: task templates that tie everything together
|
|
#
|
|
# Each list element has a unique "name" used as the natural identity key.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
semaphore_config:
|
|
project:
|
|
name: mk-labs
|
|
alert: false
|
|
max_parallel_tasks: 0 # 0 = unlimited
|
|
|
|
keys:
|
|
# The ansible-vault password. login_password type with empty login
|
|
# — only the password field is consumed by Semaphore at runtime.
|
|
- name: ansible-vault-pass
|
|
type: login_password
|
|
login: ""
|
|
password: "{{ vault_ansible_vault_password }}"
|
|
|
|
# SSH key for the gitea deploy access (clone the homelab repo).
|
|
- name: gitea-deploy
|
|
type: ssh
|
|
ssh_login: git
|
|
ssh_private_key: "{{ vault_gitea_deploy_key }}"
|
|
|
|
# SSH key for the universal automation account 'wed' — pre-baked in
|
|
# every mk-labs VM template. This is the canonical user Semaphore
|
|
# uses to reach the fleet.
|
|
- name: wed-ssh
|
|
type: ssh
|
|
ssh_login: wed
|
|
ssh_private_key: "{{ vault_wed_ssh_private_key }}"
|
|
|
|
# SSH key Semaphore can use to reach the fleet as jarvis (admin
|
|
# account provisioned by linux-baseline). Retained for jobs that
|
|
# specifically need jarvis-level access; the default is wed-ssh.
|
|
- name: jarvis-ssh
|
|
type: ssh
|
|
ssh_login: jarvis
|
|
ssh_private_key: "{{ vault_jarvis_ssh_private_key }}"
|
|
|
|
repositories:
|
|
- name: homelab
|
|
git_url: "ssh://git@gitea.mk-labs.cloud:2221/rblundon/homelab.git"
|
|
git_branch: main
|
|
ssh_key: gitea-deploy
|
|
|
|
inventories:
|
|
- name: production
|
|
type: file
|
|
inventory_file: ansible/inventory.yml
|
|
repository: homelab
|
|
# wed is the universal automation account pre-baked in every VM
|
|
# template. Semaphore uses it for fleet-wide jobs.
|
|
ssh_key: wed-ssh
|
|
# become_key is Semaphore's sudo PASSWORD slot, not a second SSH
|
|
# key. wed has passwordless sudo on every host, so reference the
|
|
# built-in "None" key. (Semaphore rejects an SSH-type key here.)
|
|
become_key: None
|
|
|
|
environments:
|
|
- name: default
|
|
env:
|
|
ANSIBLE_HOST_KEY_CHECKING: "False"
|
|
ANSIBLE_FORCE_COLOR: "True"
|
|
# Semaphore runs ansible-playbook from the cloned REPO ROOT (not
|
|
# from the playbook's directory as I first assumed). Path is
|
|
# therefore relative to repo root, not playbook dir.
|
|
ANSIBLE_ROLES_PATH: "ansible/roles"
|
|
# Collections are installed by the semaphore role into a host-side
|
|
# directory bind-mounted into the container at this path.
|
|
ANSIBLE_COLLECTIONS_PATH: "/opt/ansible-collections"
|
|
|
|
templates:
|
|
- name: "day0_linux_baseline"
|
|
description: "Apply the mk-labs Linux baseline to one or more hosts."
|
|
app: ansible
|
|
playbook: ansible/playbooks/day0_linux_baseline.yml
|
|
inventory: production
|
|
repository: homelab
|
|
environment: default
|
|
vault_password: ansible-vault-pass
|
|
arguments: '["--diff"]'
|
|
survey_vars:
|
|
- name: target
|
|
title: "Target host or group"
|
|
description: "Inventory target (e.g. figment, semaphore_server, all)"
|
|
required: true
|
|
type: TextVar
|
|
default_value: "all"
|
|
|
|
- name: "day1_deploy_semaphore"
|
|
description: "Re-deploy Semaphore + PostgreSQL on figment."
|
|
app: ansible
|
|
playbook: ansible/playbooks/day1_deploy_semaphore.yml
|
|
inventory: production
|
|
repository: homelab
|
|
environment: default
|
|
vault_password: ansible-vault-pass
|
|
arguments: '["--diff"]'
|
|
|
|
- name: "day0_linux_baseline_check"
|
|
description: "Dry-run the baseline — shows diffs, applies nothing."
|
|
app: ansible
|
|
playbook: ansible/playbooks/day0_linux_baseline.yml
|
|
inventory: production
|
|
repository: homelab
|
|
environment: default
|
|
vault_password: ansible-vault-pass
|
|
arguments: '["--check","--diff"]'
|
|
survey_vars:
|
|
- name: target
|
|
title: "Target host or group"
|
|
description: "Inventory target (e.g. figment, semaphore_server, all)"
|
|
required: true
|
|
type: TextVar
|
|
default_value: "all"
|
|
|
|
- name: "llm_inference_multimodel_stage_models"
|
|
description: >-
|
|
Stage additional GGUF models into /opt/models on astro-orbiter via the
|
|
llm-inference-multimodel role (--tags models only). Idempotent: skips
|
|
files already present at the correct byte size. Notifies the
|
|
llama-server-router restart handler ONLY when a new GGUF is actually
|
|
downloaded. Does NOT touch Phase 4 (verify) or the legacy
|
|
llama-server-qwen service. Safe to run repeatedly.
|
|
app: ansible
|
|
playbook: ansible/playbooks/day1_deploy_llm_inference_multimodel.yml
|
|
inventory: production
|
|
repository: homelab
|
|
environment: default
|
|
vault_password: ansible-vault-pass
|
|
arguments: '["--tags","models","--diff"]'
|
|
# Scoped to --tags models:
|
|
# Phase 0 (discover) -- skipped (no tag)
|
|
# Phase 1 (models) -- RUN (idempotent GGUF staging via stage_model.yml)
|
|
# Phase 2 (systemd) -- skipped
|
|
# Phase 3 (firewall) -- skipped
|
|
# Phase 4 (verify) -- SKIPPED (collision risk: verify.yml would start
|
|
# llama-server-qwen on :8002, conflicting with the
|
|
# production llama-server-router.service. Excluded
|
|
# here deliberately. See t_730f9584.)
|
|
|
|
- name: "llm_router_update_unit"
|
|
description: >-
|
|
Re-render and reload the llama-server-router systemd unit on astro-orbiter,
|
|
then restart the live service so new args (e.g. --models-max) take effect.
|
|
Drives playbooks/day2_bump_router_models_max.yml. Added 2026-08-12 (t_33acbb2e):
|
|
bump --models-max 1 -> 4 with full VRAM budget note in host_vars.
|
|
app: ansible
|
|
playbook: ansible/playbooks/day2_bump_router_models_max.yml
|
|
inventory: production
|
|
repository: homelab
|
|
environment: default
|
|
vault_password: ansible-vault-pass
|
|
arguments: '["--diff"]'
|