diff --git a/ansible/group_vars/prometheus_server/vars b/ansible/group_vars/prometheus_server/vars deleted file mode 100644 index f3f9c47..0000000 --- a/ansible/group_vars/prometheus_server/vars +++ /dev/null @@ -1,34 +0,0 @@ -# file: group_vars/prometheus_server/vars -# Ansible vars template for Prometheus nodes created via Ansible\. - -# Version of Prometheus to install. All available versions can be found at: https://prometheus.io/download/ - -#prometheus_version: "3.5.0" -#alertmanager_version: "0.28.1" - -# prometheus_nodes: -# # - '192.168.1.207' # Worker node IP -# - cinderella-castle # Master node IP -# - 10.1.71.21 -# - cinderella-castle.local.mk-labs.cloud - -# Static Variables -# prometheus_installer_file: "prometheus-{{ prometheus_version }}.linux-amd64.tar.gz" -# prometheus_installer_download_url: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/{{ prometheus_installer_file }}" - -# Grafana Variables - -#grafana_port: "3000" -#grafana_url: "{{ inventory_hostname }}.{{ base_domain }}:3000" - -# prometheus_targets: -# node: -# - targets: -# - localhost:9100 -# labels: -# env: mk-labs - -grafana_ini: - security: - admin_user: admin - admin_password: admin diff --git a/ansible/group_vars/unbound_servers/vars.yml b/ansible/group_vars/unbound_servers/vars.yml deleted file mode 100644 index 1ed812f..0000000 --- a/ansible/group_vars/unbound_servers/vars.yml +++ /dev/null @@ -1,77 +0,0 @@ ---- -# Variables for Unbound https://github.com/aruhier/ansible-role-unbound - -### Server ### - -unbound_chroot: '""' -unbound_num_threads: 2 -unbound_interfaces: - - "{{ ip_address }}" -#unbound_do_ip4: "yes" -unbound_do_ip6: "no" -#unbound_do_udp: "yes" -#unbound_do_tcp: "yes" - -# list of strings, as the order impact how policies are interpreted -unbound_outgoing_policies: [] - -unbound_access_control: - - '192.168.1.0/24 allow' - - '192.168.3.0/24 allow' - - '192.168.5.0/24 allow' - - '192.168.10.0/24 allow' - - '10.1.71.0/24 allow' - -unbound_access_control_tag: [] -unbound_access_control_tag_action: [] -unbound_access_control_tag_data: [] -unbound_access_control_view: [] - -unbound_private_addresses: - - '192.168.1.0/24' - - '192.168.3.0/24' - - '192.168.5.0/24' - - '192.168.10.0/24' - - '10.1.71.0/24' - -unbound_private_domains: - - 'mk-labs.cloud' - -unbound_domains_insecure: [] -unbound_do_not_query_addresses: [] -unbound_local_zones: [] -unbound_local_datas: [] -unbound_local_data_ptrs: [] -unbound_local_zone_tags: [] -unbound_local_zone_overrides: [] - -unbound_trust_anchors: [] -unbound_trusted_keys_files: [] - -### Remote Control ### - -unbound_control_enable: "no" -unbound_control_interface: [] - -### Stub, forward zones and others ### - -unbound_rpz_zones: [] # list of dicts -unbound_stub_zones: [] # list of dicts -unbound_forward_zones: - - name: 'int.mk-labs.cloud' - forward-addr: - - '10.1.71.5' - - name: '.' - forward-addr: - - '1.0.0.1@53#one.one.one.one' - - '1.1.1.1@53#one.one.one.one' - -# For stub and forward zones, if a key inside one of the dict is an iterable, -# it will iterate inside it to duplicate the option with all the contained -# values. -# Can be useful in case of multiple forward-addr for a same zone: -# Example: -# unbound_forward_zones: -# - {name: "test.tld", "forward-addr": ["192.0.2.5", "192.0.2.6"]} - -unbound_views: [] # list of strings, as options can be multiples \ No newline at end of file diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml new file mode 100644 index 0000000..7d982b6 --- /dev/null +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -0,0 +1,25 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: ansible/host_vars/astro_orbiter/vars.yml +# HOST: astro-orbiter (10.1.71.130) +# ROLE: Ollama inference host with AMD RX 5700 GPU passthrough +# ------------------------------------------------------------------------------ + +ansible_host: 10.1.71.130 +ansible_user: wed +ansible_become: true + +# LVM root expansion — xlarge template uses sda3 partition, standard VG/LV names +common_expand_root_lvm: true +common_root_pv: /dev/sda3 +common_root_vg: ubuntu-vg +common_root_lv: ubuntu-lv + +# Ollama — all defaults apply; explicitly documented here for visibility +ollama_rocm_version: "6.2" +ollama_default_model: "qwen3:8b" +ollama_hsa_override_gfx_version: "10.1.0" +ollama_data_disk: /dev/sdb +ollama_data_vg: ollama-vg +ollama_data_lv: ollama-lv +ollama_data_dir: /var/lib/ollama diff --git a/ansible/host_vars/cinderella-castle/vars b/ansible/host_vars/cinderella-castle/vars deleted file mode 100644 index d80533f..0000000 --- a/ansible/host_vars/cinderella-castle/vars +++ /dev/null @@ -1,49 +0,0 @@ ---- -# file: host_vars/vm_template/vars -# Ansible vars template for hosts created via cloning. - -# Supported hypervisors: -# - Proxmox - -platform: "proxmox" - -# This is the Proxmox node where all the VM templates are stored. (Templates are not global.) - -proxmox_clone_node: "fantasyland" - -# Templates are named in the following format (all lower case): -- -# Current OS offerings are: -# - Fedora (42) -# Current VM sizes are: -# - Small: 2 cores, 2GB memory, 8 GiB virtual disk -# - Medium: 2 cores, 4GB memory, 16 GiB virtual disk -# - Large: 4 cores, 4GB memory, 32 GiB virtual disk -# - Xlarge: 4 cores, 8GB memory, 64 GiB virtual disk -# - Xlarge Plus: 4 cores, 8GB memory, 128 GiB virtual disk - -vm_clone_source: "fedora-42-xlarge-plus" - -# Proxmox storage target. - -vm_storage: "general" - -# Proxmox does not yet do dynamic load balancing, the host target sets the target for HA groups -# and backup groups. (ha_group will be factored out in the next functionality update.) - -ha_group: "fantasyland" -proxmox_host_target: "fantasyland" - -# Currently, only single NIC VMs using IPv4 are supported via cloning. The IP address also -# sets the Proxmox VMID. The VMID is a combination of the 3rd and 4th octet of the IPv4 address. - -ip_address: 10.1.71.21 - -# Software -# Future enhancement will allow specification of additional software to automatically deploy to -# the VM after creation. - -#terraform_version: "1.11.3" - -# --- - -hostname: "{{ inventory_hostname }}.{{ base_domain }}" # Change variable to fqdn diff --git a/ansible/host_vars/monitor/vars b/ansible/host_vars/monitor/vars deleted file mode 100644 index d0cfb1f..0000000 --- a/ansible/host_vars/monitor/vars +++ /dev/null @@ -1,17 +0,0 @@ ---- -# file: host_vars/monitor/vars - -# VM Definition -platform: "proxmox" -proxmox_clone_node: "pve03" -vm_clone_source: "fedora-42-large" -vm_storage: "pve-general" -ha_group: "pve03" - -# Networking -ip_address: 10.1.71.249 - -# Software -#terraform_version: "1.11.3" - -hostname: "{{ inventory_hostname }}.{{ base_domain }}" #change variable to fqdn diff --git a/ansible/host_vars/unbound01/vars b/ansible/host_vars/unbound01/vars deleted file mode 100644 index de0c4c6..0000000 --- a/ansible/host_vars/unbound01/vars +++ /dev/null @@ -1,47 +0,0 @@ ---- -# file: host_vars/unbound01/vars - -# Supported hypervisors: -# - Proxmox - -platform: "proxmox" - -# This is the Proxmox node where all the VM templates are stored. (Templates are not global.) - -proxmox_clone_node: "pve03" - -# Templates are named in the following format (all lower case): -- -# Current OS offerings are: -# - Fedora (42) -# Current VM sizes are: -# - Small: 2 cores, 2GB memory, 8 GiB virtual disk -# - Medium: 2 cores, 4GB memory, 16 GiB virtual disk -# - Large: 4 cores, 4GB memory, 32 GiB virtual disk -# - Xlarge: 4 cores, 8GB memory, 64 GiB virtual disk - -vm_clone_source: "fedora-42-small" - -# Proxmox storage target. - -vm_storage: "pve-general" - -# Proxmox does not yet do dynamic load balancing, the host target sets the target for HA groups -# and backup groups. (ha_group will be factored out in the next functionality update.) - -ha_group: "pve03" -proxmox_host_target: "pve03" - -# Currently, only single NIC VMs using IPv4 are supported via cloning. The IP address also -# sets the Proxmox VMID. The VMID is a combination of the 3rd and 4th octet of the IPv4 address. - -ip_address: 10.1.71.251 - -# Software -# Future enhancement will allow specification of additional software to automatically deploy to -# the VM after creation. - -#terraform_version: "1.11.3" - -# --- - -hostname: "{{ inventory_hostname }}.{{ base_domain }}" # Change variable to fqdn diff --git a/ansible/host_vars/unbound02/vars b/ansible/host_vars/unbound02/vars deleted file mode 100644 index 70484b8..0000000 --- a/ansible/host_vars/unbound02/vars +++ /dev/null @@ -1,47 +0,0 @@ ---- -# file: host_vars/unbound02/vars - -# Supported hypervisors: -# - Proxmox - -platform: "proxmox" - -# This is the Proxmox node where all the VM templates are stored. (Templates are not global.) - -proxmox_clone_node: "pve03" - -# Templates are named in the following format (all lower case): -- -# Current OS offerings are: -# - Fedora (42) -# Current VM sizes are: -# - Small: 2 cores, 2GB memory, 8 GiB virtual disk -# - Medium: 2 cores, 4GB memory, 16 GiB virtual disk -# - Large: 4 cores, 4GB memory, 32 GiB virtual disk -# - Xlarge: 4 cores, 8GB memory, 64 GiB virtual disk - -vm_clone_source: "fedora-42-small" - -# Proxmox storage target. - -vm_storage: "pve-general" - -# Proxmox does not yet do dynamic load balancing, the host target sets the target for HA groups -# and backup groups. (ha_group will be factored out in the next functionality update.) - -ha_group: "pve03" -proxmox_host_target: "pve03" - -# Currently, only single NIC VMs using IPv4 are supported via cloning. The IP address also -# sets the Proxmox VMID. The VMID is a combination of the 3rd and 4th octet of the IPv4 address. - -ip_address: 10.1.71.252 - -# Software -# Future enhancement will allow specification of additional software to automatically deploy to -# the VM after creation. - -#terraform_version: "1.11.3" - -# --- - -hostname: "{{ inventory_hostname }}.{{ base_domain }}" # Change variable to fqdn diff --git a/ansible/inventory.yml b/ansible/inventory.yml index af08a43..efbe11e 100755 --- a/ansible/inventory.yml +++ b/ansible/inventory.yml @@ -58,25 +58,13 @@ n8n_server: hosts: tiki-room: -monitoring_server: +ollama_server: hosts: - cinderella-castle: - ansible_host: 10.1.71.31 + astro-orbiter: + ansible_host: 10.1.71.130 ansible_user: wed ansible_become: true -# node_explorer: -# hosts: -# cinderella-castle: - -# alertmanager: -# hosts: -# cinderella-castle: - -# grafana: -# hosts: -# cinderella-castle: - papermc_server: # ansible-galaxy role install engonzal.papermc hosts: diff --git a/ansible/playbooks/day1_configure_netbox_catalog.yml b/ansible/playbooks/day1_configure_netbox_catalog.yml new file mode 100644 index 0000000..3a94e6c --- /dev/null +++ b/ansible/playbooks/day1_configure_netbox_catalog.yml @@ -0,0 +1,400 @@ +--- +# ansible/playbooks/day1_configure_netbox_catalog.yml +# +# Creates the tag taxonomy and custom fields in NetBox for service catalog +# documentation. Run once (idempotent — uses name-based checks). +# +# Usage (from ansible/ directory): +# ansible-playbook playbooks/day1_configure_netbox_catalog.yml +# +# Requires: +# - vault_netbox_token in Ansible Vault +# - netbox reachable at http://fire-station.local.mk-labs.cloud + +- name: Configure NetBox service catalog taxonomy + hosts: localhost + gather_facts: false + + vars_files: + - "{{ playbook_dir }}/../group_vars/all/vault" + + vars: + netbox_url: "http://fire-station.local.mk-labs.cloud" + netbox_token: "{{ vault_netbox_token }}" + netbox_api: "{{ netbox_url }}/api" + headers: + Authorization: "Token {{ netbox_token }}" + Content-Type: "application/json" + Accept: "application/json" + + # ── Tag definitions ─────────────────────────────────────────────────── + tags: + # Infrastructure type + - name: k8s + slug: k8s + color: "2196f3" # blue + description: "Workload running in the fastpass Kubernetes cluster" + - name: vm + slug: vm + color: "4caf50" # green + description: "Traditional VM or LXC on Proxmox" + # Service tier + - name: platform + slug: platform + color: "9c27b0" # purple + description: "Platform/infrastructure service (not user-facing)" + - name: application + slug: application + color: "ff9800" # orange + description: "User-facing application workload" + # Service categories + - name: monitoring + slug: monitoring + color: "607d8b" # grey + description: "Metrics, logging, alerting" + - name: auth + slug: auth + color: "607d8b" + description: "Authentication and SSO" + - name: gitops + slug: gitops + color: "607d8b" + description: "GitOps and CI/CD" + - name: dashboard + slug: dashboard + color: "607d8b" + description: "Dashboard and portal services" + - name: storage + slug: storage + color: "607d8b" + description: "Storage and file services" + - name: dns + slug: dns + color: "607d8b" + description: "DNS and name resolution" + - name: automation + slug: automation + color: "607d8b" + description: "Automation and orchestration" + - name: networking + slug: networking + color: "607d8b" + description: "Network infrastructure services" + - name: inference + slug: inference + color: "607d8b" + description: "AI/ML inference workloads" + + # ── Custom field definitions ────────────────────────────────────────── + # object_types use app_label.model format + custom_fields: + - name: hostnames + label: Hostnames + type: longtext + object_types: + - ipam.ipaddress + - virtualization.virtualmachine + description: "All DNS names that resolve to this service (comma-separated)" + ui_visible: always + ui_editable: yes + + - name: namespace + label: Namespace + type: text + object_types: + - ipam.ipaddress + - virtualization.virtualmachine + description: "Kubernetes namespace (blank for VM-based services)" + ui_visible: always + ui_editable: yes + + - name: managed_by + label: Managed By + type: select + object_types: + - ipam.ipaddress + - virtualization.virtualmachine + description: "How this service is managed" + choices: + - ArgoCD + - Ansible + - Manual + ui_visible: always + ui_editable: yes + + - name: thematic_name + label: Thematic Name + type: text + object_types: + - ipam.ipaddress + - virtualization.virtualmachine + description: "Disney/Magic Kingdom thematic hostname for this service" + ui_visible: always + ui_editable: yes + + tasks: + + # ── Tags ───────────────────────────────────────────────────────────── + + - name: Fetch existing tags + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/tags/?limit=200" + method: GET + headers: "{{ headers }}" + return_content: true + status_code: 200 + register: existing_tags_response + + - name: Set existing tag slugs fact + ansible.builtin.set_fact: + existing_tag_slugs: "{{ existing_tags_response.json.results | map(attribute='slug') | list }}" + + - name: Create tags + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/tags/" + method: POST + headers: "{{ headers }}" + body_format: json + body: + name: "{{ item.name }}" + slug: "{{ item.slug }}" + color: "{{ item.color }}" + description: "{{ item.description }}" + status_code: 201 + loop: "{{ tags }}" + when: item.slug not in existing_tag_slugs + register: tag_creation + changed_when: tag_creation.status == 201 + + # ── Custom Fields ───────────────────────────────────────────────────── + + - name: Fetch existing custom fields + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-fields/?limit=200" + method: GET + headers: "{{ headers }}" + return_content: true + status_code: 200 + register: existing_cf_response + + - name: Set existing custom field names fact + ansible.builtin.set_fact: + existing_cf_names: "{{ existing_cf_response.json.results | map(attribute='name') | list }}" + + - name: Create custom field — hostnames + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-fields/" + method: POST + headers: "{{ headers }}" + body_format: json + body: + name: hostnames + label: Hostnames + type: longtext + object_types: "{{ custom_fields[0].object_types }}" + description: "{{ custom_fields[0].description }}" + ui_visible: always + ui_editable: yes + status_code: 201 + when: "'hostnames' not in existing_cf_names" + register: cf_hostnames + changed_when: cf_hostnames.status == 201 + + - name: Create custom field — namespace + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-fields/" + method: POST + headers: "{{ headers }}" + body_format: json + body: + name: namespace + label: Namespace + type: text + object_types: "{{ custom_fields[1].object_types }}" + description: "{{ custom_fields[1].description }}" + ui_visible: always + ui_editable: yes + status_code: 201 + when: "'namespace' not in existing_cf_names" + register: cf_namespace + changed_when: cf_namespace.status == 201 + + - name: Create choice set for managed_by field + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-field-choice-sets/" + method: POST + headers: "{{ headers }}" + body_format: json + body: + name: managed-by-choices + extra_choices: + - - ArgoCD + - ArgoCD + - - Ansible + - Ansible + - - Manual + - Manual + status_code: [201, 400] + register: choice_set + changed_when: choice_set.status == 201 + + - name: Fetch choice set ID + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-field-choice-sets/?name=managed-by-choices" + method: GET + headers: "{{ headers }}" + return_content: true + status_code: 200 + register: choice_set_response + + - name: Create custom field — managed_by + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-fields/" + method: POST + headers: "{{ headers }}" + body_format: json + body: + name: managed_by + label: Managed By + type: select + object_types: "{{ custom_fields[2].object_types }}" + description: "{{ custom_fields[2].description }}" + choice_set: "{{ choice_set_response.json.results[0].id }}" + ui_visible: always + ui_editable: yes + status_code: 201 + when: "'managed_by' not in existing_cf_names" + register: cf_managed_by + changed_when: cf_managed_by.status == 201 + + - name: Create custom field — thematic_name + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-fields/" + method: POST + headers: "{{ headers }}" + body_format: json + body: + name: thematic_name + label: Thematic Name + type: text + object_types: "{{ custom_fields[3].object_types }}" + description: "{{ custom_fields[3].description }}" + ui_visible: always + ui_editable: yes + status_code: 201 + when: "'thematic_name' not in existing_cf_names" + register: cf_thematic_name + changed_when: cf_thematic_name.status == 201 + + # ── Provisioning Pipeline Fields ────────────────────────────────────── + # These fields drive the NetBox → n8n → Terraform pipeline. + # proxmox_datastore already exists — PATCH it to add utilidor choice. + # data_disk_enabled and data_disk_size_gb are new POSTs. + + - name: Fetch proxmox_datastore field ID + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-fields/?name=proxmox_datastore" + method: GET + headers: "{{ headers }}" + return_content: true + status_code: 200 + register: proxmox_datastore_cf_response + + - name: Fetch proxmox-datastore-choices choice set ID + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-field-choice-sets/?name=proxmox-datastore-choices" + method: GET + headers: "{{ headers }}" + return_content: true + status_code: 200 + register: datastore_choice_set_response + + - name: Create proxmox-datastore-choices choice set if missing + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-field-choice-sets/" + method: POST + headers: "{{ headers }}" + body_format: json + body: + name: proxmox-datastore-choices + extra_choices: + - - liberty-tree + - liberty-tree + - - utilidor + - utilidor + status_code: [201, 400] + register: datastore_choice_set_create + changed_when: datastore_choice_set_create.status == 201 + when: datastore_choice_set_response.json.count == 0 + + - name: Re-fetch proxmox-datastore-choices choice set ID after possible creation + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-field-choice-sets/?name=proxmox-datastore-choices" + method: GET + headers: "{{ headers }}" + return_content: true + status_code: 200 + register: datastore_choice_set_response + + - name: Patch proxmox_datastore field to use choice set with utilidor + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-fields/{{ proxmox_datastore_cf_response.json.results[0].id }}/" + method: PATCH + headers: "{{ headers }}" + body_format: json + body: + choice_set: "{{ datastore_choice_set_response.json.results[0].id }}" + status_code: 200 + when: proxmox_datastore_cf_response.json.count > 0 + register: cf_datastore_patch + changed_when: cf_datastore_patch.status == 200 + + - name: Create custom field — data_disk_enabled + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-fields/" + method: POST + headers: "{{ headers }}" + body_format: json + body: + name: data_disk_enabled + label: Data Disk Enabled + type: boolean + object_types: + - virtualization.virtualmachine + description: "When true, Terraform provisions a second disk for application data storage." + default: false + ui_visible: always + ui_editable: yes + status_code: 201 + when: "'data_disk_enabled' not in existing_cf_names" + register: cf_data_disk_enabled + changed_when: cf_data_disk_enabled.status == 201 + + - name: Create custom field — data_disk_size_gb + ansible.builtin.uri: + url: "{{ netbox_api }}/extras/custom-fields/" + method: POST + headers: "{{ headers }}" + body_format: json + body: + name: data_disk_size_gb + label: Data Disk Size (GB) + type: integer + object_types: + - virtualization.virtualmachine + description: "Size in GB for the optional second data disk. Only used when data_disk_enabled is true." + ui_visible: always + ui_editable: yes + status_code: 201 + when: "'data_disk_size_gb' not in existing_cf_names" + register: cf_data_disk_size + changed_when: cf_data_disk_size.status == 201 + + # ── Summary ─────────────────────────────────────────────────────────── + + - name: Summary + ansible.builtin.debug: + msg: + - "Tags created: {{ tag_creation.results | selectattr('status', 'equalto', 201) | list | length }}" + - "Tags skipped (already exist): {{ tag_creation.results | selectattr('skipped', 'defined') | list | length }}" + - "Custom fields configured: hostnames, namespace, managed_by, thematic_name, proxmox_datastore (patched), data_disk_enabled, data_disk_size_gb" \ No newline at end of file diff --git a/ansible/playbooks/day1_deploy_monitoring.yml b/ansible/playbooks/day1_deploy_monitoring.yml deleted file mode 100644 index a5b7759..0000000 --- a/ansible/playbooks/day1_deploy_monitoring.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -# ------------------------------------------------------------------------------ -# FILE: playbooks/day1_deploy_monitoring.yml -# DESCRIPTION: Deploys Prometheus + Grafana + Node Exporter + Alertmanager -# on cinderella-castle (10.1.71.31) -# Runs: common → docker-host → monitoring -# -# USAGE: -# ansible-playbook -i inventory.yml playbooks/day1_deploy_monitoring.yml -# ansible-playbook -i inventory.yml playbooks/day1_deploy_monitoring.yml --limit cinderella_castle -# -# SECRETS REQUIRED IN VAULT (group_vars/all/vault): -# vault_grafana_admin_password -# ------------------------------------------------------------------------------ - -- name: Deploy monitoring stack on cinderella-castle - hosts: cinderella-castle - become: true - - vars: - grafana_admin_password: "{{ vault_grafana_admin_password }}" - - roles: - - role: common - - role: docker-host - - role: monitoring diff --git a/ansible/playbooks/day1_deploy_ollama.yml b/ansible/playbooks/day1_deploy_ollama.yml new file mode 100644 index 0000000..d86250a --- /dev/null +++ b/ansible/playbooks/day1_deploy_ollama.yml @@ -0,0 +1,35 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: ansible/playbooks/day1_deploy_ollama.yml +# DESCRIPTION: Deploys Ollama with ROCm GPU acceleration on astro-orbiter. +# Assumes day0_baseline.yml has already run (common role complete). +# PCIe passthrough for the RX 5700 must be configured in Proxmox +# and the GPU must be visible to the VM before running this playbook. +# +# Pre-flight check: +# ssh wed@astro-orbiter 'lspci | grep -i amd' +# Should show the RX 5700 before proceeding. +# +# Usage (from ansible/ directory): +# ansible-playbook playbooks/day1_deploy_ollama.yml +# ------------------------------------------------------------------------------ + +- name: Deploy Ollama with ROCm on astro-orbiter + hosts: astro-orbiter + become: true + + pre_tasks: + - name: Verify AMD GPU is visible to the VM + command: lspci + register: lspci_output + changed_when: false + + - name: Fail if no AMD GPU detected + fail: + msg: > + No AMD GPU detected via lspci. Verify PCIe passthrough is configured + in Proxmox and the RX 5700 is visible to the VM before proceeding. + when: "'AMD' not in lspci_output.stdout and 'Radeon' not in lspci_output.stdout" + + roles: + - ollama diff --git a/ansible/playbooks/day1_docker.yml b/ansible/playbooks/day1_docker.yml new file mode 100644 index 0000000..49b62c9 --- /dev/null +++ b/ansible/playbooks/day1_docker.yml @@ -0,0 +1,7 @@ +--- +- name: Apply common role + hosts: "{{ target | default('all') }}" + become: true + roles: + - common + - docker-host diff --git a/ansible/playbooks/roles/common/defaults/main.yml b/ansible/playbooks/roles/common/defaults/main.yml index 6818447..5e0e391 100644 --- a/ansible/playbooks/roles/common/defaults/main.yml +++ b/ansible/playbooks/roles/common/defaults/main.yml @@ -5,7 +5,7 @@ common_timezone: America/Chicago -common_ntp_server: 10.1.71.33 # sundial +common_ntp_server: "sundial.local.mk-labs.cloud" common_packages: - curl @@ -20,3 +20,15 @@ common_packages: - lsb-release - net-tools - dnsutils + - lvm2 + +# ------------------------------------------------------------------------------ +# LVM root volume expansion +# Override common_root_pv/vg/lv per host if the template uses different names. +# Ubuntu 24.04 cloud templates use ubuntu-vg/ubuntu-lv by default. +# ------------------------------------------------------------------------------ + +common_expand_root_lvm: true +common_root_pv: /dev/sda3 # Ubuntu cloud-init partition layout +common_root_vg: ubuntu-vg +common_root_lv: ubuntu-lv diff --git a/ansible/playbooks/roles/common/handlers/main.yml b/ansible/playbooks/roles/common/handlers/main.yml index db0bf20..a5f8670 100644 --- a/ansible/playbooks/roles/common/handlers/main.yml +++ b/ansible/playbooks/roles/common/handlers/main.yml @@ -1,12 +1,9 @@ +--- # ------------------------------------------------------------------------------ # FILE: roles/common/handlers/main.yml # ------------------------------------------------------------------------------ + - name: restart chrony systemd: name: chrony state: restarted - -- name: restart sshd - systemd: - name: "{{ 'ssh' if ansible_os_family == 'Debian' else 'sshd' }}" - state: restarted diff --git a/ansible/playbooks/roles/common/tasks/main.yml b/ansible/playbooks/roles/common/tasks/main.yml index 2853037..4ce1dc3 100644 --- a/ansible/playbooks/roles/common/tasks/main.yml +++ b/ansible/playbooks/roles/common/tasks/main.yml @@ -1,8 +1,9 @@ +--- # ------------------------------------------------------------------------------ # FILE: roles/common/tasks/main.yml -# DESCRIPTION: Baseline configuration applied to all managed hosts. -# Handles hostname, timezone, core packages, NTP, and -# ansible user setup. +# DESCRIPTION: Baseline configuration applied to all managed Ubuntu hosts. +# Handles hostname, timezone, core packages, NTP, ansible user, +# and optional LVM root volume expansion. # ------------------------------------------------------------------------------ - name: Set hostname @@ -10,7 +11,7 @@ name: "{{ inventory_hostname | replace('_', '-') }}" - name: Set timezone - community.general.timezone: + timezone: name: "{{ common_timezone }}" - name: Update apt cache @@ -41,7 +42,7 @@ state: started enabled: yes -- name: Ensure ansible user has sudo without password +- name: Ensure ansible user has passwordless sudo lineinfile: path: /etc/sudoers.d/{{ ansible_user }} line: "{{ ansible_user }} ALL=(ALL) NOPASSWD:ALL" @@ -49,33 +50,28 @@ mode: '0440' validate: 'visudo -cf %s' -# --- Step-CA SSH certificate enrollment --- -- name: Enroll host in step-ca SSH CA - include_tasks: step_ca_client.yml +# ------------------------------------------------------------------------------ +# LVM root volume expansion +# Extends the root PV to the full disk size and grows the LV + filesystem. +# This codifies what was previously done manually after provisioning. +# Runs only when common_expand_root_lvm is true (default: true). +# Safe to re-run — pvresize and lvextend are idempotent when already at max. +# ------------------------------------------------------------------------------ -- name: Create authorized principals directory - ansible.builtin.file: - path: /etc/ssh/auth_principals - state: directory - owner: root - group: root - mode: '0755' +- name: Expand root PV to full disk size + command: pvresize {{ common_root_pv }} + register: pvresize_result + changed_when: "'changed' in pvresize_result.stdout or pvresize_result.rc == 0" + when: common_expand_root_lvm | bool -- name: Create authorized principals files - ansible.builtin.copy: - content: "{{ item.principals | join('\n') }}\n" - dest: "/etc/ssh/auth_principals/{{ item.local_user }}" - owner: root - group: root - mode: '0644' - loop: "{{ step_ca_principal_mappings }}" - notify: restart sshd - -- name: Configure sshd AuthorizedPrincipalsFile - ansible.builtin.lineinfile: - path: /etc/ssh/sshd_config - line: "AuthorizedPrincipalsFile /etc/ssh/auth_principals/%u" - regexp: "^#?AuthorizedPrincipalsFile" - state: present - notify: restart sshd - \ No newline at end of file +- name: Extend root LV to 100% of free VG space + lvol: + vg: "{{ common_root_vg }}" + lv: "{{ common_root_lv }}" + size: +100%FREE + resizefs: yes + when: + - common_expand_root_lvm | bool + ignore_errors: yes + # ignore_errors because lvextend returns non-zero when already at max size. + # resizefs: yes handles the resize2fs call inline — no separate task needed. diff --git a/ansible/playbooks/roles/monitoring/defaults/main.yml b/ansible/playbooks/roles/monitoring/defaults/main.yml deleted file mode 100644 index c94e182..0000000 --- a/ansible/playbooks/roles/monitoring/defaults/main.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -# ------------------------------------------------------------------------------ -# FILE: roles/monitoring/defaults/main.yml -# ------------------------------------------------------------------------------ - -# Directory layout on cinderella-castle -monitoring_base_dir: /opt/docker/monitoring - -# Data disk — LVM on /dev/sdb, mounted at this path for Prometheus TSDB -prometheus_data_dir: /var/lib/prometheus -prometheus_vg: prometheus-vg -prometheus_lv: prometheus-lv - -# Prometheus retention -prometheus_retention_time: 15d -prometheus_retention_size: 58GB # 64GB disk, leaving headroom - -# Scrape interval -prometheus_scrape_interval: 30s - -# Grafana -grafana_port: 3000 -grafana_admin_user: admin - -# Service DNS (via Traefik) -grafana_hostname: grafana.local.mk-labs.cloud - -# Trusted proxy (lightning-lane) -trusted_proxy_ip: 10.1.71.35 - -# Prometheus port (internal, not exposed via Traefik) -prometheus_port: 9090 - -# Alertmanager port (internal) -alertmanager_port: 9093 - -# Node exporter port -node_exporter_port: 9100 diff --git a/ansible/playbooks/roles/monitoring/handlers/main.yml b/ansible/playbooks/roles/monitoring/handlers/main.yml deleted file mode 100644 index 1b797e5..0000000 --- a/ansible/playbooks/roles/monitoring/handlers/main.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -# ------------------------------------------------------------------------------ -# FILE: roles/monitoring/handlers/main.yml -# ------------------------------------------------------------------------------ - -- name: restart monitoring - community.docker.docker_compose_v2: - project_src: "{{ monitoring_base_dir }}" - state: present - recreate: always diff --git a/ansible/playbooks/roles/monitoring/tasks/main.yml b/ansible/playbooks/roles/monitoring/tasks/main.yml deleted file mode 100644 index 0bc8b75..0000000 --- a/ansible/playbooks/roles/monitoring/tasks/main.yml +++ /dev/null @@ -1,203 +0,0 @@ ---- -# ------------------------------------------------------------------------------ -# FILE: roles/monitoring/tasks/main.yml -# ------------------------------------------------------------------------------ - -# ── Data disk (LVM) ─────────────────────────────────────────────────────────── - -- name: Install LVM2 - ansible.builtin.apt: - name: lvm2 - state: present - update_cache: true - -- name: Check if PV already exists on /dev/sdb - ansible.builtin.command: pvs /dev/sdb - register: pvs_result - changed_when: false - failed_when: false - -- name: Check if disk has a partition table signature - ansible.builtin.command: wipefs /dev/sdb - register: wipefs_check - changed_when: false - failed_when: false - when: pvs_result.rc != 0 - -- name: Wipe GPT protective partition signature from /dev/sdb - ansible.builtin.command: wipefs -a /dev/sdb - when: - - pvs_result.rc != 0 - - wipefs_check.stdout | length > 0 - -- name: Create LVM volume group on /dev/sdb - community.general.lvg: - vg: prometheus-vg - pvs: /dev/sdb - state: present - when: pvs_result.rc != 0 - -- name: Check if logical volume already exists - ansible.builtin.command: lvs /dev/prometheus-vg/prometheus-lv - register: lvs_result - changed_when: false - failed_when: false - -- name: Create logical volume for Prometheus TSDB - community.general.lvol: - vg: prometheus-vg - lv: prometheus-lv - size: 100%FREE - state: present - shrink: false - when: lvs_result.rc != 0 - -- name: Format logical volume as ext4 - community.general.filesystem: - fstype: ext4 - dev: /dev/prometheus-vg/prometheus-lv - -- name: Create Prometheus data directory mount point - ansible.builtin.file: - path: "{{ prometheus_data_dir }}" - state: directory - owner: root - group: root - mode: "0755" - -- name: Mount Prometheus logical volume - ansible.posix.mount: - path: "{{ prometheus_data_dir }}" - src: /dev/prometheus-vg/prometheus-lv - fstype: ext4 - opts: defaults - state: mounted - -- name: Set ownership of Prometheus data directory - ansible.builtin.file: - path: "{{ prometheus_data_dir }}" - state: directory - owner: "65534" # nobody — Prometheus container runs as this UID - group: "65534" - mode: "0755" - -# ── Directory layout ─────────────────────────────────────────────────────────── - -- name: Create monitoring directory structure - ansible.builtin.file: - path: "{{ item }}" - state: directory - owner: "{{ ansible_user }}" - group: docker - mode: "0775" - loop: - - "{{ monitoring_base_dir }}" - - "{{ monitoring_base_dir }}/prometheus" - - "{{ monitoring_base_dir }}/grafana" - - "{{ monitoring_base_dir }}/alertmanager" - - "{{ monitoring_base_dir }}/pve-exporter" - - "{{ monitoring_base_dir }}/snmp-exporter" - -# ── Config files ─────────────────────────────────────────────────────────────── - -- name: Deploy Prometheus configuration - ansible.builtin.template: - src: prometheus.yaml.j2 - dest: "{{ monitoring_base_dir }}/prometheus/prometheus.yaml" - owner: "{{ ansible_user }}" - group: docker - mode: "0644" - notify: restart monitoring - -- name: Deploy Alertmanager configuration - ansible.builtin.template: - src: alertmanager.yml.j2 - dest: "{{ monitoring_base_dir }}/alertmanager/alertmanager.yml" - owner: "{{ ansible_user }}" - group: docker - mode: "0644" - notify: restart monitoring - -- name: Deploy Grafana provisioning datasource - ansible.builtin.template: - src: grafana-datasource.yml.j2 - dest: "{{ monitoring_base_dir }}/grafana/datasource.yml" - owner: "{{ ansible_user }}" - group: docker - mode: "0644" - notify: restart monitoring - -- name: Deploy PVE exporter credentials file - ansible.builtin.template: - src: pve.yml.j2 - dest: "{{ monitoring_base_dir }}/pve-exporter/pve.yml" - owner: "{{ ansible_user }}" - group: docker - mode: "0644" - no_log: true - notify: restart monitoring - -# ── SNMP exporter config ─────────────────────────────────────────────────────── -# Extract the default snmp.yml from the image (contains full if_mib metrics -# mapping) then inject the mk-labs community auth into the auths section. - -- name: Extract default snmp.yml from snmp-exporter image - ansible.builtin.shell: - cmd: > - docker run --rm --entrypoint cat prom/snmp-exporter:latest - /etc/snmp_exporter/snmp.yml - > {{ monitoring_base_dir }}/snmp-exporter/snmp.yml - changed_when: true - -- name: Inject mk_labs_v2c auth into snmp.yml - ansible.builtin.shell: - cmd: | - python3 - << 'EOF' - path = '{{ monitoring_base_dir }}/snmp-exporter/snmp.yml' - with open(path, 'r') as f: - content = f.read() - auth = ( - ' mk_labs_v2c:\n' - ' community: {{ vault_snmp_community }}\n' - ' security_level: noAuthNoPriv\n' - ' auth_protocol: MD5\n' - ' priv_protocol: DES\n' - ' version: 2\n' - ) - if 'mk_labs_v2c' not in content: - content = content.replace('\nmodules:\n', '\n' + auth + 'modules:\n', 1) - with open(path, 'w') as f: - f.write(content) - print('injected') - else: - print('already present') - EOF - register: snmp_inject - changed_when: "'injected' in snmp_inject.stdout" - no_log: true - notify: restart monitoring - -# ── Compose stack ────────────────────────────────────────────────────────────── - -- name: Deploy monitoring compose file from boilerplate - ansible.builtin.copy: - src: "{{ playbook_dir }}/../../boilerplates/monitoring/compose.yaml" - dest: "{{ monitoring_base_dir }}/compose.yaml" - owner: "{{ ansible_user }}" - group: docker - mode: "0644" - notify: restart monitoring - -- name: Deploy .env with secrets from vault - ansible.builtin.template: - src: env.j2 - dest: "{{ monitoring_base_dir }}/.env" - owner: "{{ ansible_user }}" - group: docker - mode: "0600" - notify: restart monitoring - -- name: Start monitoring stack - community.docker.docker_compose_v2: - project_src: "{{ monitoring_base_dir }}" - state: present diff --git a/ansible/playbooks/roles/monitoring/templates/alertmanager.yml.j2 b/ansible/playbooks/roles/monitoring/templates/alertmanager.yml.j2 deleted file mode 100644 index 447b2a3..0000000 --- a/ansible/playbooks/roles/monitoring/templates/alertmanager.yml.j2 +++ /dev/null @@ -1,13 +0,0 @@ -# ------------------------------------------------------------------------------ -# FILE: roles/monitoring/templates/alertmanager.yml.j2 -# DESCRIPTION: Alertmanager configuration — routing deferred (P.XX) -# ------------------------------------------------------------------------------ - -route: - receiver: blackhole - group_wait: 30s - group_interval: 5m - repeat_interval: 4h - -receivers: - - name: blackhole diff --git a/ansible/playbooks/roles/monitoring/templates/env.j2 b/ansible/playbooks/roles/monitoring/templates/env.j2 deleted file mode 100644 index e132bb4..0000000 --- a/ansible/playbooks/roles/monitoring/templates/env.j2 +++ /dev/null @@ -1,11 +0,0 @@ -# ------------------------------------------------------------------------------ -# FILE: roles/monitoring/templates/env.j2 -# DESCRIPTION: Environment file for monitoring compose stack -# Deployed to {{ monitoring_base_dir }}/.env -# ------------------------------------------------------------------------------ - -GRAFANA_ADMIN_USER={{ grafana_admin_user }} -GRAFANA_ADMIN_PASSWORD={{ grafana_admin_password }} -PROMETHEUS_RETENTION_TIME={{ prometheus_retention_time }} -PROMETHEUS_RETENTION_SIZE={{ prometheus_retention_size }} -TRUSTED_PROXIES={{ trusted_proxy_ip }} \ No newline at end of file diff --git a/ansible/playbooks/roles/monitoring/templates/grafana-datasource.yml.j2 b/ansible/playbooks/roles/monitoring/templates/grafana-datasource.yml.j2 deleted file mode 100644 index 87076c1..0000000 --- a/ansible/playbooks/roles/monitoring/templates/grafana-datasource.yml.j2 +++ /dev/null @@ -1,14 +0,0 @@ -# ------------------------------------------------------------------------------ -# FILE: roles/monitoring/templates/grafana-datasource.yml.j2 -# DESCRIPTION: Auto-provisions Prometheus as Grafana default datasource -# ------------------------------------------------------------------------------ - -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: false diff --git a/ansible/playbooks/roles/monitoring/templates/prometheus.yaml.j2 b/ansible/playbooks/roles/monitoring/templates/prometheus.yaml.j2 deleted file mode 100644 index f035407..0000000 --- a/ansible/playbooks/roles/monitoring/templates/prometheus.yaml.j2 +++ /dev/null @@ -1,205 +0,0 @@ -# ------------------------------------------------------------------------------ -# FILE: ansible/playbooks/roles/monitoring/templates/prometheus.yaml.j2 -# DESCRIPTION: Prometheus scrape configuration -# ------------------------------------------------------------------------------ - -global: - scrape_interval: 30s - evaluation_interval: 30s - -alerting: - alertmanagers: - - static_configs: - - targets: - - alertmanager:9093 - -rule_files: [] - -scrape_configs: - - # Prometheus self-monitoring - - job_name: prometheus - static_configs: - - targets: - - localhost:9090 - - # cinderella-castle node exporter (this host) - - job_name: node-cinderella-castle - static_configs: - - targets: - - 10.1.71.31:9100 - labels: - hostname: cinderella-castle - - # Proxmox nodes (node exporter on each hypervisor) - - job_name: proxmox-nodes - static_configs: - - targets: - - 10.1.71.11:9100 - - 10.1.71.12:9100 - - 10.1.71.13:9100 - labels: - job: proxmox - - # Traefik metrics - - job_name: traefik - static_configs: - - targets: - - 10.1.71.35:8080 - labels: - hostname: lightning-lane - - # -------------------------------------------------------------------------- - # Proxmox VE Exporter — cluster-level metrics (VM states, storage, CPU) - # Uses the multi-target pattern: Prometheus passes the cluster address as a - # query parameter to the exporter, which proxies to the Proxmox API. - # One scrape job covers all three nodes via the cluster API endpoint. - # -------------------------------------------------------------------------- - - job_name: proxmox-cluster - metrics_path: /pve - params: - module: [default] - cluster: ["1"] - node: [""] - static_configs: - - targets: - - 10.1.71.11 # main-street-usa — cluster API endpoint - labels: - cluster: magic-kingdom - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: pve-exporter:9221 - - - # -------------------------------------------------------------------------- - # Pure Storage FlashArray -- utilidor (DEFERRED until Purity 6.7+) - # - # When utilidor is upgraded to Purity 6.7+, uncomment this job and remove - # the legacy exporter container. No container needed -- Prometheus scrapes - # the native OpenMetrics endpoint on the array directly over HTTPS. - # - # Also add vault_pure_utilidor_api_token to group_vars/all/vault and create - # a read-only API token on utilidor under System -> Users -> API Tokens. - # -------------------------------------------------------------------------- - # - job_name: pure-flasharray-utilidor - # metrics_path: /metrics/array - # scheme: https - # tls_config: - # insecure_skip_verify: true - # authorization: - # credentials: {{ vault_pure_utilidor_api_token }} - # params: - # namespace: ["purefa"] - # static_configs: - # - targets: - # - utilidor.local.mk-labs.cloud - # labels: - # array: utilidor - # array_type: physical - - # -------------------------------------------------------------------------- - # UniFi Network — SNMP scrape jobs (one per device for clean Grafana filtering) - # All targets use the if_mib module for interface throughput, errors, and - # up/down state. Auth uses the mk_labs_v2c community string from vault. - # -------------------------------------------------------------------------- - - - job_name: snmp-udm-pro - metrics_path: /snmp - params: - auth: [mk_labs_v2c] - module: [if_mib] - static_configs: - - targets: - - 192.168.1.1 - labels: - device: world-drive - model: UDM-Pro - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 10.1.71.31:9116 - - - job_name: snmp-usw-pro-aggregation - metrics_path: /snmp - params: - auth: [mk_labs_v2c] - module: [if_mib] - static_configs: - - targets: - - 192.168.1.123 - labels: - device: usw-pro-aggregation - model: USW-Pro-Aggregation - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 10.1.71.31:9116 - - - job_name: snmp-usw-pro-max-24 - metrics_path: /snmp - scrape_interval: 60s - scrape_timeout: 55s - params: - auth: [mk_labs_v2c] - module: [if_mib] - static_configs: - - targets: - - 192.168.1.226 - labels: - device: usw-pro-max-24 - model: USW-Pro-Max-24 - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 10.1.71.31:9116 - - - job_name: snmp-usw-lite-16-poe - metrics_path: /snmp - params: - auth: [mk_labs_v2c] - module: [if_mib] - static_configs: - - targets: - - 192.168.1.136 - labels: - device: usw-lite-16-poe - model: USW-Lite-16-PoE - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 10.1.71.31:9116 - - - job_name: snmp-ups-tower - metrics_path: /snmp - params: - auth: [mk_labs_v2c] - module: [if_mib] - static_configs: - - targets: - - 192.168.1.233 - labels: - device: ups-tower - model: UPS-Tower - relabel_configs: - - source_labels: [__address__] - target_label: __param_target - - source_labels: [__param_target] - target_label: instance - - target_label: __address__ - replacement: 10.1.71.31:9116 diff --git a/ansible/playbooks/roles/monitoring/templates/pve.yml.j2 b/ansible/playbooks/roles/monitoring/templates/pve.yml.j2 deleted file mode 100644 index c43b0e7..0000000 --- a/ansible/playbooks/roles/monitoring/templates/pve.yml.j2 +++ /dev/null @@ -1,11 +0,0 @@ -# ------------------------------------------------------------------------------ -# FILE: ansible/playbooks/roles/monitoring/templates/pve.yml.j2 -# DESCRIPTION: Proxmox VE Exporter credentials file. -# Mounted read-only into the pve-exporter container. -# ------------------------------------------------------------------------------ - -default: - user: prometheus@pve - token_name: prometheus-token - token_value: {{ vault_proxmox_prometheus_token }} - verify_ssl: false diff --git a/ansible/playbooks/roles/ollama/defaults/main.yml b/ansible/playbooks/roles/ollama/defaults/main.yml new file mode 100644 index 0000000..5cd3e64 --- /dev/null +++ b/ansible/playbooks/roles/ollama/defaults/main.yml @@ -0,0 +1,28 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/ollama/defaults/main.yml +# ------------------------------------------------------------------------------ + +# ROCm +ollama_rocm_version: "6.2" # ROCm release stream — update when upgrading +ollama_rocm_packages: + - hip-runtime-amd # Pulls in full ROCm stack + - rocminfo # GPU visibility verification tool + - rocm-smi-lib # ROCm system management interface + +# Ollama +ollama_version: "latest" +ollama_port: 11434 +ollama_host: "0.0.0.0" # Listen on all interfaces for cluster access +ollama_default_model: "qwen3:8b" + +# HSA override — required for RX 5000-series (gfx1010 / RDNA 1) +# Tells ROCm to treat the GPU as gfx1010 which has supported LLVM targets. +ollama_hsa_override_gfx_version: "10.1.0" + +# Data disk +ollama_setup_data_disk: true +ollama_data_disk: /dev/sdb +ollama_data_dir: /var/lib/ollama # Ollama default models path +ollama_data_vg: ollama-vg +ollama_data_lv: ollama-lv diff --git a/ansible/playbooks/roles/ollama/handlers/main.yml b/ansible/playbooks/roles/ollama/handlers/main.yml new file mode 100644 index 0000000..954c380 --- /dev/null +++ b/ansible/playbooks/roles/ollama/handlers/main.yml @@ -0,0 +1,20 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/ollama/handlers/main.yml +# ------------------------------------------------------------------------------ + +- name: update apt cache + apt: + update_cache: yes + +- name: update initramfs + command: update-initramfs -u + +- name: reload systemd + systemd: + daemon_reload: yes + +- name: restart ollama + systemd: + name: ollama + state: restarted diff --git a/ansible/playbooks/roles/ollama/tasks/main.yml b/ansible/playbooks/roles/ollama/tasks/main.yml new file mode 100644 index 0000000..a48e0fb --- /dev/null +++ b/ansible/playbooks/roles/ollama/tasks/main.yml @@ -0,0 +1,229 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/ollama/tasks/main.yml +# DESCRIPTION: Deploys Ollama with AMD ROCm GPU acceleration on Ubuntu 24.04. +# Handles: +# - ROCm repository and driver stack installation +# - Data disk preparation (LVM, ext4, persistent mount) +# - Ollama installation and systemd service configuration +# - HSA override for RX 5000-series (gfx1010) compatibility +# - Initial model pull +# ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# ROCm prerequisites +# ------------------------------------------------------------------------------ + +- name: Install ROCm prerequisite packages + apt: + name: + - wget + - gnupg + - ca-certificates + - lvm2 + state: present + update_cache: yes + +- name: Add ROCm apt repository signing key + apt_key: + url: https://repo.radeon.com/rocm/rocm.gpg.key + state: present + +- name: Add ROCm apt repository + apt_repository: + repo: "deb [arch=amd64] https://repo.radeon.com/rocm/apt/{{ ollama_rocm_version }} {{ ansible_distribution_release }} main" + state: present + filename: rocm + notify: update apt cache + +- name: Flush handlers to update apt cache before ROCm install + meta: flush_handlers + +# ------------------------------------------------------------------------------ +# ROCm driver stack +# hip-runtime-amd pulls in the full ROCm stack as a dependency. +# rocminfo is used to verify GPU visibility after install. +# ------------------------------------------------------------------------------ + +- name: Install ROCm packages + apt: + name: "{{ ollama_rocm_packages }}" + state: present + notify: update initramfs + +- name: Create /opt/rocm symlink to versioned directory + file: + src: "/opt/rocm-{{ ollama_rocm_version }}.0" + dest: /opt/rocm + state: link + force: false + +- name: Add ROCm binaries to system PATH + copy: + dest: /etc/profile.d/rocm.sh + content: | + export PATH=$PATH:/opt/rocm/bin + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib + mode: '0644' + +- name: Add wed user to render and video groups for GPU access + user: + name: "{{ ansible_user }}" + groups: + - render + - video + append: yes + +- name: Add ollama user to render and video groups (created by Ollama installer) + user: + name: ollama + groups: + - render + - video + append: yes + ignore_errors: yes + # ollama user does not exist yet at this point — Ollama installer creates it. + # This task re-runs after install via the post-install handler. + +# ------------------------------------------------------------------------------ +# Data disk — LVM setup on /dev/sdb +# Follows the same pattern as the monitoring role (cinderella-castle). +# Wipes any stale GPT/partition signatures before creating the PV to avoid +# the "GPT protective partition" error encountered during monitoring deploy. +# ------------------------------------------------------------------------------ + +- name: Wipe any existing partition signatures from data disk + command: wipefs -a {{ ollama_data_disk }} + args: + creates: /dev/{{ ollama_data_vg }} + when: ollama_setup_data_disk | bool + +- name: Create LVM physical volume on data disk + command: pvcreate {{ ollama_data_disk }} + args: + creates: /dev/{{ ollama_data_vg }} + when: ollama_setup_data_disk | bool + +- name: Create LVM volume group + lvg: + vg: "{{ ollama_data_vg }}" + pvs: "{{ ollama_data_disk }}" + when: ollama_setup_data_disk | bool + +- name: Create LVM logical volume using 100% of VG + lvol: + vg: "{{ ollama_data_vg }}" + lv: "{{ ollama_data_lv }}" + size: 100%FREE + shrink: false + when: ollama_setup_data_disk | bool + +- name: Format logical volume as ext4 + filesystem: + fstype: ext4 + dev: "/dev/{{ ollama_data_vg }}/{{ ollama_data_lv }}" + opts: "-L ollama-data" + when: ollama_setup_data_disk | bool + +- name: Create Ollama model directory mount point + file: + path: "{{ ollama_data_dir }}" + state: directory + mode: '0755' + +- name: Mount Ollama data volume and add fstab entry + mount: + path: "{{ ollama_data_dir }}" + src: "/dev/{{ ollama_data_vg }}/{{ ollama_data_lv }}" + fstype: ext4 + opts: defaults,noatime + state: mounted + when: ollama_setup_data_disk | bool + # noatime: eliminates inode access time writes on every model file read. + # Meaningful win for large sequential reads like model loading. + +# ------------------------------------------------------------------------------ +# Ollama installation +# Uses the official install script which creates the ollama user and +# systemd service unit. We override the service with our own config. +# ------------------------------------------------------------------------------ + +- name: Download Ollama install script + get_url: + url: https://ollama.ai/install.sh + dest: /tmp/ollama_install.sh + mode: '0755' + +- name: Run Ollama install script + command: /tmp/ollama_install.sh + environment: "{{ {} if ollama_version == 'latest' else {'OLLAMA_VERSION': ollama_version} }}" + args: + creates: /usr/local/bin/ollama + +- name: Add ollama user to render and video groups (post-install) + user: + name: ollama + groups: + - render + - video + append: yes + +- name: Set ownership of Ollama data directory to ollama user + file: + path: "{{ ollama_data_dir }}" + owner: ollama + group: ollama + recurse: yes + +# ------------------------------------------------------------------------------ +# Systemd service override +# Injects HSA_OVERRIDE_GFX_VERSION for RX 5000-series compatibility and +# sets OLLAMA_MODELS to the data disk mount point. +# ------------------------------------------------------------------------------ + +- name: Create systemd override directory for ollama service + file: + path: /etc/systemd/system/ollama.service.d + state: directory + mode: '0755' + +- name: Deploy ollama systemd service override + template: + src: ollama-override.conf.j2 + dest: /etc/systemd/system/ollama.service.d/override.conf + mode: '0644' + notify: + - reload systemd + - restart ollama + +- name: Flush handlers to apply service config before model pull + meta: flush_handlers + +- name: Enable and start Ollama service + systemd: + name: ollama + state: started + enabled: yes + daemon_reload: yes + +- name: Wait for Ollama API to become available + uri: + url: "http://localhost:{{ ollama_port }}/api/tags" + status_code: 200 + register: ollama_health + retries: 12 + delay: 5 + until: ollama_health.status == 200 + +# ------------------------------------------------------------------------------ +# Initial model pull +# ------------------------------------------------------------------------------ + +- name: Pull initial model + command: ollama pull {{ ollama_default_model }} + environment: + OLLAMA_HOST: "http://localhost:{{ ollama_port }}" + register: model_pull + changed_when: "'pulled' in model_pull.stdout or model_pull.rc == 0" + timeout: 600 + # Qwen3 8B Q4_K_M is ~5GB — allow 10 minutes on first pull. diff --git a/ansible/playbooks/roles/ollama/templates/ollama-override.conf.j2 b/ansible/playbooks/roles/ollama/templates/ollama-override.conf.j2 new file mode 100644 index 0000000..702f62b --- /dev/null +++ b/ansible/playbooks/roles/ollama/templates/ollama-override.conf.j2 @@ -0,0 +1,13 @@ +# ------------------------------------------------------------------------------ +# FILE: roles/ollama/templates/ollama-override.conf.j2 +# DESCRIPTION: Systemd drop-in for the ollama service. +# Sets environment variables required for AMD GPU operation +# and configures the model storage path. +# Managed by Ansible — do not edit manually. +# ------------------------------------------------------------------------------ + +[Service] +Environment="OLLAMA_HOST={{ ollama_host }}:{{ ollama_port }}" +Environment="OLLAMA_MODELS={{ ollama_data_dir }}" +Environment="HSA_OVERRIDE_GFX_VERSION={{ ollama_hsa_override_gfx_version }}" +Environment="ROCR_VISIBLE_DEVICES=0" diff --git a/boilerplates/grafana/compose.yaml b/boilerplates/grafana/compose.yaml deleted file mode 100644 index 4a66b31..0000000 --- a/boilerplates/grafana/compose.yaml +++ /dev/null @@ -1,12 +0,0 @@ -services: - grafana: - image: docker.io/grafana/grafana-oss:12.3.3 - restart: unless-stopped - ports: - - "3000:3000" - volumes: - - grafana_data:/var/lib/grafana - -volumes: - grafana_data: - driver: local diff --git a/boilerplates/monitoring/compose.yaml b/boilerplates/monitoring/compose.yaml deleted file mode 100644 index 797c0ba..0000000 --- a/boilerplates/monitoring/compose.yaml +++ /dev/null @@ -1,138 +0,0 @@ -# ------------------------------------------------------------------------------ -# FILE: boilerplates/monitoring/compose.yaml -# DESCRIPTION: Monitoring stack — Prometheus, Grafana, Alertmanager, -# Node Exporter, PVE Exporter, Pure FA Exporter -# ------------------------------------------------------------------------------ - -networks: - monitoring: - driver: bridge - -volumes: - prometheus_data: - driver: local - driver_opts: - type: none - o: bind - device: /var/lib/prometheus - grafana_data: - driver: local - alertmanager_data: - driver: local - -services: - - # ---------------------------------------------------------------------------- - # Prometheus - # ---------------------------------------------------------------------------- - prometheus: - image: prom/prometheus:latest - container_name: prometheus - restart: unless-stopped - command: - - --config.file=/etc/prometheus/prometheus.yaml - - --storage.tsdb.path=/prometheus - - --storage.tsdb.retention.time=30d - - --web.enable-lifecycle - volumes: - - /opt/docker/monitoring/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro - - prometheus_data:/prometheus - ports: - - "9090:9090" - networks: - - monitoring - - # ---------------------------------------------------------------------------- - # Grafana - # ---------------------------------------------------------------------------- - grafana: - image: grafana/grafana:latest - container_name: grafana - restart: unless-stopped - environment: - - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER} - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} - - GF_USERS_ALLOW_SIGN_UP=false - - GF_SERVER_ROOT_URL=https://grafana.local.mk-labs.cloud - - GF_AUTH_GENERIC_OAUTH_ENABLED=true - - TRUSTED_PROXIES=10.1.71.35 - volumes: - - grafana_data:/var/lib/grafana - ports: - - "3000:3000" - networks: - - monitoring - depends_on: - - prometheus - - # ---------------------------------------------------------------------------- - # Alertmanager - # ---------------------------------------------------------------------------- - alertmanager: - image: prom/alertmanager:latest - container_name: alertmanager - restart: unless-stopped - volumes: - - /opt/docker/monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro - - alertmanager_data:/alertmanager - ports: - - "9093:9093" - networks: - - monitoring - - # ---------------------------------------------------------------------------- - # Node Exporter - # ---------------------------------------------------------------------------- - node-exporter: - image: prom/node-exporter:latest - container_name: node-exporter - restart: unless-stopped - command: - - --path.rootfs=/host - pid: host - volumes: - - /:/host:ro,rslave - ports: - - "9100:9100" - networks: - - monitoring - - # ---------------------------------------------------------------------------- - # Proxmox VE Exporter - # Scrapes the Proxmox API and exposes cluster/node/VM metrics for Prometheus. - # Credentials are mounted from pve.yml (rendered from vault). - # ---------------------------------------------------------------------------- - pve-exporter: - image: prompve/prometheus-pve-exporter:latest - container_name: pve-exporter - restart: unless-stopped - volumes: - - /opt/docker/monitoring/pve-exporter/pve.yml:/etc/prometheus/pve.yml:ro - ports: - - "9221:9221" - networks: - - monitoring - - - # ---------------------------------------------------------------------------- - # SNMP Exporter - # Scrapes UniFi switches and UPS Tower via SNMP v2c. - # snmp.yml is rendered from vault and mounted read-only. - # ---------------------------------------------------------------------------- - snmp-exporter: - image: prom/snmp-exporter:latest - container_name: snmp-exporter - restart: unless-stopped - volumes: - - /opt/docker/monitoring/snmp-exporter/snmp.yml:/etc/snmp_exporter/snmp.yml:ro - # network_mode: host is required so the container can route to VLANs - # (192.168.1.0/24) that are reachable from the host but not the Docker bridge. - # Port mapping and networks are incompatible with host networking. - network_mode: host - # ---------------------------------------------------------------------------- - # Pure Storage FlashArray Exporter -- DEFERRED - # utilidor is on Purity 6.5.x; legacy exporter image is no longer publicly - # available on Quay. Once utilidor reaches Purity 6.7+, no container is - # needed -- Prometheus scrapes the native OpenMetrics endpoint directly. - # See prometheus.yaml.j2 for the commented-out scrape job to re-enable. - # ---------------------------------------------------------------------------- diff --git a/cluster/applications/jarvis/application.yaml b/cluster/applications/jarvis/application.yaml new file mode 100644 index 0000000..9254cbc --- /dev/null +++ b/cluster/applications/jarvis/application.yaml @@ -0,0 +1,30 @@ +# ------------------------------------------------------------------------------ +# Application: jarvis (Hermes Agent) +# Wave 20 — application workload, after platform services +# Single-source: all manifests from repo +# ------------------------------------------------------------------------------ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: jarvis + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "20" +spec: + project: default + sources: + - repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git + targetRevision: main + path: cluster/applications/jarvis + directory: + exclude: "application.yaml" + destination: + server: https://kubernetes.default.svc + namespace: jarvis + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/cluster/applications/jarvis/deployment.yaml b/cluster/applications/jarvis/deployment.yaml new file mode 100644 index 0000000..e51626c --- /dev/null +++ b/cluster/applications/jarvis/deployment.yaml @@ -0,0 +1,116 @@ +# ------------------------------------------------------------------------------ +# Hermes Agent (jarvis) — Deployment + Service + PVC +# +# Hermes requires a one-time interactive setup before first gateway run. +# See bootstrap procedure below before applying for the first time. +# +# Bootstrap procedure (run once): +# 1. kubectl run hermes-setup -it --rm --image=nousresearch/hermes-agent \ +# --overrides='{"spec":{"volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"jarvis-data"}}],"containers":[{"name":"hermes-setup","image":"nousresearch/hermes-agent","command":["setup"],"stdin":true,"tty":true,"volumeMounts":[{"name":"data","mountPath":"/opt/data"}]}]}}' \ +# -n jarvis +# 2. Complete the setup wizard: +# - Provider: Ollama (custom endpoint) +# - Base URL: http://astro-orbiter.local.mk-labs.cloud:11434/v1 +# - API Key: none (leave blank) +# - Model: qwen3:8b +# - Messaging: skip for now +# 3. Verify config was written: kubectl exec -n jarvis -- ls /opt/data +# 4. Apply the rest of the manifests (ArgoCD will handle from here) +# ------------------------------------------------------------------------------ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: jarvis-data + namespace: jarvis +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + # Uses cluster default StorageClass (Ceph/Longhorn — whatever fastpass has) +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jarvis + namespace: jarvis + labels: + app: jarvis +spec: + replicas: 1 + selector: + matchLabels: + app: jarvis + template: + metadata: + labels: + app: jarvis + spec: + securityContext: + runAsNonRoot: true + runAsUser: 10000 + seccompProfile: + type: RuntimeDefault + containers: + - name: hermes + image: nousresearch/hermes-agent:latest + command: ["gateway", "run"] + ports: + - containerPort: 8642 + name: gateway + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + env: + - name: HERMES_DASHBOARD + value: "1" + - name: API_SERVER_ENABLED + value: "true" + - name: API_SERVER_HOST + value: "0.0.0.0" + - name: API_SERVER_CORS_ORIGINS + value: "*" + volumeMounts: + - name: data + mountPath: /opt/data + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + livenessProbe: + httpGet: + path: /health + port: 8642 + initialDelaySeconds: 30 + periodSeconds: 30 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /health + port: 8642 + initialDelaySeconds: 15 + periodSeconds: 10 + volumes: + - name: data + persistentVolumeClaim: + claimName: jarvis-data +--- +apiVersion: v1 +kind: Service +metadata: + name: jarvis + namespace: jarvis + labels: + app: jarvis +spec: + selector: + app: jarvis + ports: + - name: gateway + port: 8642 + targetPort: 8642 \ No newline at end of file diff --git a/cluster/applications/jarvis/httproute.yaml b/cluster/applications/jarvis/httproute.yaml new file mode 100644 index 0000000..f8db24f --- /dev/null +++ b/cluster/applications/jarvis/httproute.yaml @@ -0,0 +1,45 @@ +# ------------------------------------------------------------------------------ +# HTTPRoute — jarvis (Hermes Agent) via Cilium Gateway +# Hostnames: jarvis.local.mk-labs.cloud +# ------------------------------------------------------------------------------ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: jarvis-tls + namespace: jarvis +spec: + secretName: jarvis-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - jarvis.local.mk-labs.cloud +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: jarvis + namespace: jarvis + annotations: + external-dns.alpha.kubernetes.io/hostname: jarvis.local.mk-labs.cloud + external-dns.alpha.kubernetes.io/target: "10.1.71.90" +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: fastpass-gateway + namespace: gateway + sectionName: https + hostnames: + - jarvis.local.mk-labs.cloud + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - group: "" + kind: Service + name: jarvis + port: 8642 + weight: 1 diff --git a/docs/astro-orbiter-gpu-passthrough.md b/docs/astro-orbiter-gpu-passthrough.md new file mode 100644 index 0000000..0bd3051 --- /dev/null +++ b/docs/astro-orbiter-gpu-passthrough.md @@ -0,0 +1,281 @@ +# astro-orbiter: AMD GPU Passthrough & Ollama/ROCm Setup + +**Host:** `tomorrowland` (10.1.71.12) — Proxmox node +**VM:** `astro-orbiter` (10.1.71.130, VM ID 71130) +**GPU:** MSI Radeon RX 5700 (Navi 10, gfx1010) via Minisforum DEG1 eGPU dock + OCuLink +**Purpose:** Ollama inference host for `jarvis` AI assistant +**Date:** 2026-05-25 + +--- + +## Hardware + +- Minisforum MS-01 (`tomorrowland`) +- Minisforum DEG1 eGPU dock connected via M.2 OCuLink +- MSI Radeon RX 5700 8GB +- Montech BETA 2 650W ATX PSU powering the eGPU dock + +--- + +## Part 1: tomorrowland Host Setup + +### 1.1 Network Interface Rename (eGPU side effect) + +Adding the eGPU dock shifted PCIe bus numbers, renaming all NICs. Fix: + +```bash +# Back up interfaces config +cp /etc/network/interfaces /etc/network/interfaces.bak + +# Apply renames (adjust if hardware differs) +sed -i 's/enp87s0/enp93s0/g' /etc/network/interfaces +sed -i 's/enp2s0f0np0/enp5s0f0np0/g' /etc/network/interfaces +sed -i 's/enp2s0f0np1/enp5s0f0np1/g' /etc/network/interfaces + +# Reboot to apply (do from console/JetKVM, not SSH) +reboot +``` + +### 1.2 Verify GPU Enumeration + +After reboot, confirm the GPU is visible: + +```bash +lspci | grep -i amd +# Expected: +# 01:00.0 PCI bridge: AMD/ATI Navi 10 XL Upstream Port +# 02:00.0 PCI bridge: AMD/ATI Navi 10 XL Downstream Port +# 03:00.0 VGA compatible controller: AMD/ATI Navi 10 [Radeon RX 5600/5700] +# 03:00.1 Audio device: AMD/ATI Navi 10 HDMI Audio +``` + +### 1.3 Verify IOMMU + +```bash +dmesg | grep -i iommu | head -5 +# Must show: iommu: Default domain type: Translated + +# Verify GPU is in its own IOMMU groups (no ACS override needed) +find /sys/kernel/iommu_groups/*/devices/ -name "*03:00*" +# Expected: groups 17 and 18, one device each +``` + +### 1.4 Blacklist amdgpu and Bind vfio-pci + +```bash +# Blacklist host GPU drivers +echo "blacklist amdgpu +blacklist radeon" > /etc/modprobe.d/blacklist-amdgpu.conf + +# Bind vfio-pci to the GPU at boot +# 1002:731f = RX 5700 VGA, 1002:ab38 = Navi 10 HDMI Audio +echo "options vfio-pci ids=1002:731f,1002:ab38" > /etc/modprobe.d/vfio.conf + +# Load vfio modules early +echo "vfio +vfio_iommu_type1 +vfio_pci +vfio_virqfd" > /etc/modules-load.d/vfio.conf + +# Rebuild initramfs and reboot +update-initramfs -u -k all +reboot +``` + +Verify after reboot: +```bash +lspci -k -s 03:00.0 +# Must show: Kernel driver in use: vfio-pci + +lspci -k -s 03:00.1 +# Must show: Kernel driver in use: vfio-pci +``` + +### 1.5 Fix GPU D3 Power State on VM Restart (Required — RX 5700 lacks FLR) + +The RX 5700 does not support PCIe Function Level Reset (FLR). Without a fix, +the GPU gets stuck in D3 power state on every VM restart, requiring a full host +reboot to recover. + +**Fix: `pcie_port_pm=off` kernel parameter** + +`vendor-reset` was attempted but causes a QEMU PCI IRQ assertion failure +(`pci_irq_handler: Assertion 0 <= irq_num`) on PVE 7.0.2 with this hardware. +`pcie_port_pm=off` prevents the card from entering D3 entirely and is the +correct solution for this setup. + +```bash +sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT="/GRUB_CMDLINE_LINUX_DEFAULT="pcie_port_pm=off /' /etc/default/grub +update-grub +reboot +``` + +Verify after reboot — VM should restart cleanly without host reboot: +```bash +qm reboot 71130 +# wait 30 seconds +ssh wed@astro-orbiter 'lsmod | grep amdgpu' +# Should be non-empty +``` + +--- + +## Part 2: astro-orbiter VM Provisioning + +### 2.1 Terraform Provisioning + +VM was provisioned manually (not via NetBox pipeline) due to PCIe passthrough +requirements. Run from `city-hall`: + +```bash +cd /opt/git/homelab/terraform/proxmox/vm + +terraform apply \ + -var='proxmox_api_url=https://fantasyland.local.mk-labs.cloud:8006' \ + -var='proxmox_api_token=terraform@pve!terraform-token=' \ + -var='vm_id=71130' \ + -var='hostname=astro-orbiter' \ + -var='ip_address=10.1.71.130' \ + -var='target_node=tomorrowland' \ + -var='template_name=ubuntu-24.04-large' \ + -var='datastore=liberty-tree' \ + -var='data_disk_enabled=true' \ + -var='data_disk_size=128' \ + -var='data_disk_datastore=utilidor' +``` + +**Notes:** +- `datastore=liberty-tree` for OS disk — iSCSI storage does not support EFI disk + cloning from templates. OS disk stays on Ceph. +- `data_disk_datastore=utilidor` for model storage — fresh disk creation works fine + on iSCSI. +- `pcie_passthrough_enabled` is NOT passed — the `bpg/proxmox` provider's + `dynamic "hostpci"` block does not work reliably. PCIe passthrough is applied + manually via `qm set` after provisioning (see 2.2). + +### 2.2 Post-Provisioning: PCIe and Machine Type + +The template uses `i440fx` machine type. PCIe passthrough requires `q35`. +Run after Terraform provisioning, with VM shut down: + +```bash +# Switch to q35 machine type (required for PCIe passthrough) +qm set 71130 -machine q35 + +# Add GPU passthrough (both VGA and audio functions) +qm set 71130 -hostpci0 0000:03:00,pcie=1,rombar=1 + +# Add data disk on utilidor +qm set 71130 -scsi1 utilidor:128,format=raw,discard=on + +# Increase RAM to 8GB +qm set 71130 -memory 8192 + +# Start the VM +qm start 71130 +``` + +Verify config landed: +```bash +qm config 71130 | grep -E "hostpci|machine|memory|scsi1" +``` + +### 2.3 Ansible Deployment + +From `city-hall` in the `ansible/` directory: + +```bash +# OS baseline +ansible-playbook playbooks/day0_baseline.yml --limit astro-orbiter --ask-vault-pass + +# Ollama + ROCm +ansible-playbook playbooks/day1_deploy_ollama.yml --ask-vault-pass +``` + +The `day1_deploy_ollama.yml` playbook includes a pre-flight `lspci` check that +will fail fast if the GPU is not visible to the VM. + +--- + +## Part 3: Verification + +### 3.1 GPU visible in VM +```bash +lspci | grep -i amd +# Expected: 01:00.0 VGA ... Navi 10 [Radeon RX 5600/5700] +# 01:00.1 Audio ... Navi 10 HDMI Audio +``` + +### 3.2 amdgpu driver loaded +```bash +lsmod | grep amdgpu +# Should show amdgpu and dependent modules +``` + +### 3.3 ROCm sees the GPU +```bash +/opt/rocm/bin/rocm-smi +# Should show Device 0 with VRAM%, GPU%, temperature +``` + +### 3.4 Ollama GPU inference +```bash +ollama run qwen3:8b "say hello" +# While running, check rocm-smi — GPU% and power draw should spike +/opt/rocm/bin/rocm-smi +# GPU% > 0%, power > 18W idle = GPU is doing inference +``` + +### 3.5 VM restart test +```bash +# On tomorrowland: +qm reboot 71130 +# Wait 30 seconds, then on astro-orbiter: +lsmod | grep amdgpu +# Should be non-empty — if empty, vendor-reset is not working +``` + +--- + +## Known Issues & Lessons Learned + +| Issue | Cause | Fix | +|---|---|---| +| NIC names changed after eGPU install | PCIe bus renumbering | Update `/etc/network/interfaces` with new names | +| `can't allocate space in iscsi storage` on clone | iSCSI doesn't support EFI disk cloning | Use `liberty-tree` for OS disk | +| `q35 machine model is not enabled` on hostpci | Template built with i440fx | `qm set -machine q35` before adding hostpci | +| GPU stuck in D3 on VM restart | RX 5700 lacks FLR support | `pcie_port_pm=off` kernel param on host. `vendor-reset` builds but causes QEMU PCI IRQ assertion failure on PVE 7.0.2 — do not use. | +| `dynamic "hostpci"` block not applying | `bpg/proxmox` provider limitation | Use `qm set` workaround post-provisioning | +| `rocm-smi` not in PATH | ROCm installs to `/opt/rocm-6.2.0/bin/` | `/etc/profile.d/rocm.sh` sets PATH (via Ansible role) | +| Ollama install script 404 | `OLLAMA_VERSION=latest` is not a valid version string | Omit env var when version is `latest` — script self-determines | + +--- + +## ROCm Configuration + +The RX 5700 is RDNA 1 (gfx1010). ROCm's official support starts at gfx1030 +(RDNA 2). The `HSA_OVERRIDE_GFX_VERSION=10.1.0` environment variable tells ROCm +to treat the GPU as a supported target. + +This is set via systemd drop-in at: +`/etc/systemd/system/ollama.service.d/override.conf` + +```ini +[Service] +Environment="OLLAMA_HOST=0.0.0.0:11434" +Environment="OLLAMA_MODELS=/var/lib/ollama" +Environment="HSA_OVERRIDE_GFX_VERSION=10.1.0" +Environment="ROCR_VISIBLE_DEVICES=0" +``` + +--- + +## Parking Lot + +- **P.PACKER:** Ubuntu VM templates should be built with `machine: q35` by default + to support PCIe passthrough without manual intervention. +- **P.TERRAFORM:** Fix `bpg/proxmox` provider `hostpci` dynamic block or find + correct syntax for the installed provider version. +- **P.PIPELINE:** Update n8n workflow to pass `data_disk_enabled`, `data_disk_size`, + and `pcie_device_id` from NetBox custom fields to Terraform for pipeline-driven + GPU VM provisioning. diff --git a/hello-world.yaml b/hello-world.yaml deleted file mode 100644 index d986761..0000000 --- a/hello-world.yaml +++ /dev/null @@ -1,124 +0,0 @@ -# ------------------------------------------------------------------------------ -# hello-world — Gateway API + cert-manager + external-dns end-to-end test -# -# Tests: -# - Cilium Gateway provisioning (10.1.71.90 from cilium-gateway pool) -# - cert-manager DNS-01 cert issuance (letsencrypt-staging) -# - external-dns A record creation in Technitium via RFC2136 -# - HTTPRoute routing to backend service -# -# Cleanup when done: -# kubectl delete -f hello-world.yaml -n default -# ------------------------------------------------------------------------------ - -# Gateway — one per cluster, shared by all HTTPRoutes -apiVersion: gateway.networking.k8s.io/v1 -kind: Gateway -metadata: - name: fastpass-gateway - namespace: default - annotations: - io.cilium/lb-ipam-ips: "10.1.71.90" -spec: - gatewayClassName: cilium - listeners: - - name: http - port: 80 - protocol: HTTP - allowedRoutes: - namespaces: - from: Same - - name: https - port: 443 - protocol: HTTPS - tls: - mode: Terminate - certificateRefs: - - name: hello-world-tls - kind: Secret - allowedRoutes: - namespaces: - from: Same ---- -# Certificate — staging issuer to avoid LE rate limits during testing -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: hello-world-tls - namespace: default -spec: - secretName: hello-world-tls - issuerRef: - name: letsencrypt-staging - kind: ClusterIssuer - dnsNames: - - hello.local.mk-labs.cloud ---- -# Deployment — http-echo runs as non-root, restricted compliant -apiVersion: apps/v1 -kind: Deployment -metadata: - name: hello-world - namespace: default -spec: - replicas: 1 - selector: - matchLabels: - app: hello-world - template: - metadata: - labels: - app: hello-world - spec: - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - seccompProfile: - type: RuntimeDefault - containers: - - name: hello-world - image: hashicorp/http-echo:1.0 - args: - - "-text=fastpass gateway test - hello from mk-labs!" - - "-listen=:8080" - ports: - - containerPort: 8080 - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL ---- -# Service -apiVersion: v1 -kind: Service -metadata: - name: hello-world - namespace: default -spec: - selector: - app: hello-world - ports: - - port: 80 - targetPort: 8080 ---- -# HTTPRoute -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: hello-world - namespace: default - annotations: - external-dns.alpha.kubernetes.io/hostname: hello.local.mk-labs.cloud - external-dns.alpha.kubernetes.io/target: "10.1.71.90" -spec: - parentRefs: - - name: fastpass-gateway - sectionName: https - hostnames: - - hello.local.mk-labs.cloud - rules: - - backendRefs: - - name: hello-world - port: 80 diff --git a/hermes/.env b/hermes/.env new file mode 100644 index 0000000..812986d --- /dev/null +++ b/hermes/.env @@ -0,0 +1,469 @@ +# Hermes Agent Environment Configuration +# Copy this file to .env and fill in your API keys + +# ============================================================================= +# LLM PROVIDER (OpenRouter) +# ============================================================================= +# OpenRouter provides access to many models through one API +# All LLM calls go through OpenRouter - no direct provider keys needed +# Get your key at: https://openrouter.ai/keys +# OPENROUTER_API_KEY= + +# Default model is configured in ~/.hermes/config.yaml (model.default). +# Use 'hermes model' or 'hermes setup' to change it. +# LLM_MODEL is no longer read from .env — this line is kept for reference only. +# LLM_MODEL=anthropic/claude-opus-4.6 + +# ============================================================================= +# LLM PROVIDER (NovitaAI) +# ============================================================================= +# NovitaAI — 90+ models, pay-per-use +# Get your key at: https://novita.ai/settings/key-management +# NOVITA_API_KEY= +# NOVITA_BASE_URL=https://api.novita.ai/openai/v1 # Override default base URL + +# ============================================================================= +# LLM PROVIDER (Google AI Studio / Gemini) +# ============================================================================= +# Native Gemini API via Google's OpenAI-compatible endpoint. +# Get your key at: https://aistudio.google.com/app/apikey +# GOOGLE_API_KEY=your_google_ai_studio_key_here +# GEMINI_API_KEY=your_gemini_key_here # alias for GOOGLE_API_KEY +# Optional base URL override (default: Google's OpenAI-compatible endpoint) +# GEMINI_BASE_URL=https://generativelanguage.googleapis.com/v1beta/openai + +# ============================================================================= +# LLM PROVIDER (Ollama Cloud) +# ============================================================================= +# Cloud-hosted open models via Ollama's OpenAI-compatible endpoint. +# Get your key at: https://ollama.com/settings +# OLLAMA_API_KEY=your_ollama_key_here +# Optional base URL override (default: https://ollama.com/v1) +# OLLAMA_BASE_URL=https://ollama.com/v1 + +# ============================================================================= +# LLM PROVIDER (z.ai / GLM) +# ============================================================================= +# z.ai provides access to ZhipuAI GLM models (GLM-4-Plus, etc.) +# Get your key at: https://z.ai or https://open.bigmodel.cn +# GLM_API_KEY= +# GLM_BASE_URL=https://api.z.ai/api/paas/v4 # Override default base URL + +# ============================================================================= +# LLM PROVIDER (Kimi / Moonshot) +# ============================================================================= +# Kimi Code provides access to Moonshot AI coding models (kimi-k2.5, etc.) +# Get your key at: https://platform.kimi.ai (Kimi Code console) +# Keys prefixed sk-kimi- use the Kimi Code API (api.kimi.com) by default. +# Legacy keys from platform.moonshot.ai need KIMI_BASE_URL override below. +# KIMI_API_KEY= +# KIMI_BASE_URL=https://api.kimi.com/coding/v1 # Default for sk-kimi- keys +# KIMI_BASE_URL=https://api.moonshot.ai/v1 # For legacy Moonshot keys +# KIMI_BASE_URL=https://api.moonshot.cn/v1 # For Moonshot China keys +# KIMI_CN_API_KEY= # Dedicated Moonshot China key + +# ============================================================================= +# LLM PROVIDER (Arcee AI) +# ============================================================================= +# Arcee AI provides access to Trinity models (trinity-mini, trinity-large-*) +# Get an Arcee key at: https://chat.arcee.ai/ +# ARCEEAI_API_KEY= +# ARCEE_BASE_URL= # Override default base URL + +# ============================================================================= +# LLM PROVIDER (MiniMax) +# ============================================================================= +# MiniMax provides access to MiniMax models (global endpoint) +# Get your key at: https://www.minimax.io +# MINIMAX_API_KEY= +# MINIMAX_BASE_URL=https://api.minimax.io/v1 # Override default base URL + +# MiniMax China endpoint (for users in mainland China) +# MINIMAX_CN_API_KEY= +# MINIMAX_CN_BASE_URL=https://api.minimaxi.com/v1 # Override default base URL + +# ============================================================================= +# LLM PROVIDER (OpenCode Zen) +# ============================================================================= +# OpenCode Zen provides curated, tested models (GPT, Claude, Gemini, MiniMax, GLM, Kimi) +# Pay-as-you-go pricing. Get your key at: https://opencode.ai/auth +# OPENCODE_ZEN_API_KEY= +# OPENCODE_ZEN_BASE_URL=https://opencode.ai/zen/v1 # Override default base URL + +# ============================================================================= +# LLM PROVIDER (OpenCode Go) +# ============================================================================= +# OpenCode Go provides access to open models (GLM-5, Kimi K2.5, MiniMax M2.5) +# $10/month subscription. Get your key at: https://opencode.ai/auth +# OPENCODE_GO_API_KEY= + +# ============================================================================= +# LLM PROVIDER (Hugging Face Inference Providers) +# ============================================================================= +# Hugging Face routes to 20+ open models via unified OpenAI-compatible endpoint. +# Free tier included ($0.10/month), no markup on provider rates. +# Get your token at: https://huggingface.co/settings/tokens +# Required permission: "Make calls to Inference Providers" +# HF_TOKEN= +# OPENCODE_GO_BASE_URL=https://opencode.ai/zen/go/v1 # Override default base URL + +# ============================================================================= +# LLM PROVIDER (Qwen OAuth) +# ============================================================================= +# Qwen OAuth reuses your local Qwen CLI login (qwen auth qwen-oauth). +# No API key needed — credentials come from ~/.qwen/oauth_creds.json. +# Optional base URL override: +# HERMES_QWEN_BASE_URL=https://portal.qwen.ai/v1 + +# ============================================================================= +# LLM PROVIDER (Xiaomi MiMo) +# ============================================================================= +# Xiaomi MiMo models (mimo-v2-pro, mimo-v2-omni, mimo-v2-flash). +# Get your key at: https://platform.xiaomimimo.com +# XIAOMI_API_KEY=your_key_here +# Optional base URL override: +# XIAOMI_BASE_URL=https://api.xiaomimimo.com/v1 + +# ============================================================================= +# TOOL API KEYS +# ============================================================================= + +# Exa API Key - AI-native web search and contents +# Get at: https://exa.ai +# EXA_API_KEY= + +# Parallel API Key - AI-native web search and extract +# Get at: https://parallel.ai +# PARALLEL_API_KEY= + +# Firecrawl API Key - Web search, extract, and crawl +# Get at: https://firecrawl.dev/ +# FIRECRAWL_API_KEY= + + +# FAL.ai API Key - Image generation +# Get at: https://fal.ai/ +# FAL_KEY= + +# Honcho - Cross-session AI-native user modeling (optional) +# Builds a persistent understanding of the user across sessions and tools. +# Get at: https://app.honcho.dev +# Also requires ~/.honcho/config.json with enabled=true (see README). +# HONCHO_API_KEY= + +# ============================================================================= +# HYPERLIQUID OPTIONAL SKILL +# ============================================================================= +# Optional defaults for the Hyperliquid skill in optional-skills/blockchain/hyperliquid +# +# Hyperliquid API base URL override +# Default: https://api.hyperliquid.xyz +# HYPERLIQUID_API_URL=https://api.hyperliquid-testnet.xyz +# +# Default address for account-level commands like state, fills, orders, and review +# HYPERLIQUID_USER_ADDRESS=0x0000000000000000000000000000000000000000 + +# ============================================================================= +# TERMINAL TOOL CONFIGURATION +# ============================================================================= +# Backend type: "local", "singularity", "docker", "modal", or "ssh" +# Terminal backend is configured in ~/.hermes/config.yaml (terminal.backend). +# Use 'hermes setup' or 'hermes config set terminal.backend docker' to change. +# Supported: local, docker, singularity, modal, ssh +# +# Only override here if you need to force a backend without touching config.yaml: +# TERMINAL_ENV=local + +# Override the container runtime binary (e.g. to use Podman instead of Docker). +# Useful on systems where Docker's storage driver is broken or unavailable. +# HERMES_DOCKER_BINARY=/usr/local/bin/podman + +# Container images (for singularity/docker/modal backends) +# TERMINAL_DOCKER_IMAGE=nikolaik/python-nodejs:python3.11-nodejs20 +# TERMINAL_SINGULARITY_IMAGE=docker://nikolaik/python-nodejs:python3.11-nodejs20 +TERMINAL_MODAL_IMAGE=nikolaik/python-nodejs:python3.11-nodejs20 + + +# Working directory for terminal commands +# For local backend: "." means current directory (resolved automatically) +# For remote backends (ssh/docker/modal/singularity): use an absolute path +# INSIDE the target environment, or leave unset for the backend's default +# (/root for modal, / for docker, ~ for ssh). Do NOT use a host-local path. +# Usually managed by config.yaml (terminal.cwd) — uncomment to override +# TERMINAL_CWD=. + +# Default command timeout in seconds +TERMINAL_TIMEOUT=60 + +# Cleanup inactive environments after this many seconds +TERMINAL_LIFETIME_SECONDS=300 + +# ============================================================================= +# SSH REMOTE EXECUTION (for TERMINAL_ENV=ssh) +# ============================================================================= +# Run terminal commands on a remote server via SSH. +# Agent code stays on your machine, commands execute remotely. +# +# SECURITY BENEFITS: +# - Agent cannot read your .env file (API keys protected) +# - Agent cannot modify its own code +# - Remote server acts as isolated sandbox +# - Can safely configure passwordless sudo on remote +# +# TERMINAL_SSH_HOST=192.168.1.100 +# TERMINAL_SSH_USER=agent +# TERMINAL_SSH_PORT=22 +# TERMINAL_SSH_KEY=~/.ssh/id_rsa + +# ============================================================================= +# SUDO SUPPORT (works with ALL terminal backends) +# ============================================================================= +# If set, enables sudo commands by piping password via `sudo -S`. +# Works with: local, docker, singularity, modal, and ssh backends. +# +# SECURITY WARNING: Password stored in plaintext. Only use on trusted machines. +# +# ALTERNATIVES: +# - For SSH backend: Configure passwordless sudo on the remote server +# - For containers: Run as root inside the container (no sudo needed) +# - For local: Configure /etc/sudoers for specific commands +# - For CLI: Leave unset - you'll be prompted interactively with 45s timeout +# +# SUDO_PASSWORD=your_password_here + +# ============================================================================= +# MODAL CLOUD BACKEND (Optional - for TERMINAL_ENV=modal) +# ============================================================================= +# Modal uses CLI authentication, not environment variables. +# Run: pip install modal && modal setup +# This will authenticate via browser and store credentials locally. +# No API key needed in .env - Modal handles auth automatically. + +# ============================================================================= +# BROWSER TOOL CONFIGURATION (agent-browser + Browserbase) +# ============================================================================= +# Browser automation requires Browserbase cloud service for remote browser execution. +# This allows the agent to navigate websites, fill forms, and extract information. +# +# STEALTH MODES: +# - Basic Stealth: ALWAYS active (random fingerprints, auto CAPTCHA solving) +# - Advanced Stealth: Requires BROWSERBASE_ADVANCED_STEALTH=true (Scale Plan only) + +# Browserbase API Key - Cloud browser execution +# Get at: https://browserbase.com/ +# BROWSERBASE_API_KEY= + +# Browserbase Project ID - From your Browserbase dashboard +# BROWSERBASE_PROJECT_ID= + +# Enable residential proxies for better CAPTCHA solving (default: true) +# Routes traffic through residential IPs, significantly improves success rate +BROWSERBASE_PROXIES=true + +# Enable advanced stealth mode (default: false, requires Scale Plan) +# Uses custom Chromium build to avoid bot detection altogether +BROWSERBASE_ADVANCED_STEALTH=false + +# Browser engine for local mode (default: auto = Chrome) +# "auto" — use Chrome (don't pass --engine flag) +# "lightpanda" — use Lightpanda (1.3-5.8x faster navigation, no screenshots) +# "chrome" — explicitly request Chrome +# Requires agent-browser v0.25.3+. Lightpanda commands that fail or return +# empty results are automatically retried with Chrome. +# Also configurable via browser.engine in config.yaml. +# AGENT_BROWSER_ENGINE=auto + +# Browser session timeout in seconds (default: 300) +# Sessions are cleaned up after this duration of inactivity +BROWSER_SESSION_TIMEOUT=300 + +# Browser inactivity timeout - auto-cleanup inactive sessions (default: 120 = 2 min) +# Browser sessions are automatically closed after this period of no activity +BROWSER_INACTIVITY_TIMEOUT=120 + +# Extra Chromium launch flags passed to agent-browser, comma- or newline-separated. +# Hermes auto-injects "--no-sandbox,--disable-dev-shm-usage" when it detects root +# or AppArmor-restricted unprivileged user namespaces (Ubuntu 23.10+, DGX Spark, +# many container images), so leave this unset unless you need extra flags. +# Setting this disables the auto-injection. +# AGENT_BROWSER_ARGS=--no-sandbox + +# Camofox local anti-detection browser (Camoufox-based Firefox). +# Set CAMOFOX_URL to route the browser tools through a local Camofox server +# instead of agent-browser/Browserbase. See docs/user-guide/features/browser.md. +# CAMOFOX_URL=http://localhost:9377 + +# Externally managed Camofox sessions — when another app owns the visible +# Camofox browser, set these so Hermes shares the same userId/profile instead +# of creating its own isolated session. +# CAMOFOX_USER_ID= +# CAMOFOX_SESSION_KEY= +# Set to true to reuse an already-open Camofox tab for this identity before +# creating a new one (useful for gateway restarts). +# CAMOFOX_ADOPT_EXISTING_TAB=false + +# ============================================================================= +# SESSION LOGGING +# ============================================================================= +# Session trajectories are automatically saved to logs/ directory +# Format: logs/session_YYYYMMDD_HHMMSS_UUID.json +# Contains full conversation history in trajectory format for debugging/replay + +# ============================================================================= +# VOICE TRANSCRIPTION & OPENAI TTS +# ============================================================================= +# Required for voice message transcription (Whisper) and OpenAI TTS voices. +# Uses OpenAI's API directly (not via OpenRouter). +# Named VOICE_TOOLS_OPENAI_KEY to avoid interference with OpenRouter. +# Get at: https://platform.openai.com/api-keys +# VOICE_TOOLS_OPENAI_KEY= + +# ============================================================================= +# SLACK INTEGRATION +# ============================================================================= +# Slack Bot Token - From Slack App settings (OAuth & Permissions) +# Get at: https://api.slack.com/apps +# SLACK_BOT_TOKEN=xoxb-... + +# Slack App Token - For Socket Mode (App-Level Tokens in Slack App settings) +# SLACK_APP_TOKEN=xapp-... + +# Slack allowed users (comma-separated Slack user IDs) +# SLACK_ALLOWED_USERS= + +# ============================================================================= +# TELEGRAM INTEGRATION +# ============================================================================= +# Telegram Bot Token - From @BotFather (https://t.me/BotFather) +# TELEGRAM_BOT_TOKEN= +# TELEGRAM_ALLOWED_USERS= # Comma-separated user IDs +# TELEGRAM_HOME_CHANNEL= # Default chat for cron delivery +# TELEGRAM_HOME_CHANNEL_NAME= # Display name for home channel + +# Webhook mode (optional — for cloud deployments like Fly.io/Railway) +# Default is long polling. Setting TELEGRAM_WEBHOOK_URL switches to webhook mode. +# TELEGRAM_WEBHOOK_URL=https://my-app.fly.dev/telegram +# TELEGRAM_WEBHOOK_PORT=8443 +# TELEGRAM_WEBHOOK_SECRET= # Recommended for production + +# WhatsApp (built-in Baileys bridge — run `hermes whatsapp` to pair) +# WHATSAPP_ENABLED=false +# WHATSAPP_ALLOWED_USERS=15551234567 + +# Email (IMAP/SMTP — send and receive emails as Hermes) +# For Gmail: enable 2FA → create App Password at https://myaccount.google.com/apppasswords +# EMAIL_ADDRESS=hermes@gmail.com +# EMAIL_PASSWORD=xxxx xxxx xxxx xxxx +# EMAIL_IMAP_HOST=imap.gmail.com +# EMAIL_IMAP_PORT=993 +# EMAIL_SMTP_HOST=smtp.gmail.com +# EMAIL_SMTP_PORT=587 +# EMAIL_POLL_INTERVAL=15 +# EMAIL_ALLOWED_USERS=your@email.com +# EMAIL_HOME_ADDRESS=your@email.com + +# Gateway-wide: allow ALL users without an allowlist (default: false = deny) +# Only set to true if you intentionally want open access. +# GATEWAY_ALLOW_ALL_USERS=false + +# ============================================================================= +# RESPONSE PACING +# ============================================================================= +# Human-like delays between message chunks on messaging platforms. +# Makes the bot feel less robotic. +# HERMES_HUMAN_DELAY_MODE=off # off | natural | custom +# HERMES_HUMAN_DELAY_MIN_MS=800 # Min delay in ms (custom mode) +# HERMES_HUMAN_DELAY_MAX_MS=2500 # Max delay in ms (custom mode) + +# ============================================================================= +# DEBUG OPTIONS +# ============================================================================= +WEB_TOOLS_DEBUG=false +VISION_TOOLS_DEBUG=false +MOA_TOOLS_DEBUG=false +IMAGE_TOOLS_DEBUG=false + +# ============================================================================= +# CONTEXT COMPRESSION (Auto-shrinks long conversations) +# ============================================================================= +# When conversation approaches model's context limit, middle turns are +# automatically summarized to free up space. +# +# Context compression is configured in ~/.hermes/config.yaml under compression: +# CONTEXT_COMPRESSION_ENABLED=true # Enable auto-compression (default: true) +# CONTEXT_COMPRESSION_THRESHOLD=0.85 # Compress at 85% of context limit +# Model is set via compression.summary_model in config.yaml (default: google/gemini-3-flash-preview) + +# ============================================================================= +# SKILLS HUB (GitHub integration for skill search/install/publish) +# ============================================================================= + +# GitHub Personal Access Token — for higher API rate limits on skill search/install +# Get at: https://github.com/settings/tokens (Fine-grained recommended) +# GITHUB_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx + +# GitHub App credentials (optional — for bot identity on PRs) +# GITHUB_APP_ID= +# GITHUB_APP_PRIVATE_KEY_PATH= +# GITHUB_APP_INSTALLATION_ID= + +# Groq API key (free tier — used for Whisper STT in voice mode) +# GROQ_API_KEY= + +# ============================================================================= +# STT PROVIDER SELECTION +# ============================================================================= +# Default STT provider is "local" (faster-whisper) — runs on your machine, no API key needed. +# Install with: pip install faster-whisper +# Model downloads automatically on first use (~150 MB for "base"). +# To use cloud providers instead, set GROQ_API_KEY or VOICE_TOOLS_OPENAI_KEY above. +# Provider priority: local > groq > openai +# Configure in config.yaml: stt.provider: local | groq | openai + +# ============================================================================= +# STT ADVANCED OVERRIDES (optional) +# ============================================================================= +# Override default STT models per provider (normally set via stt.model in config.yaml) +# STT_GROQ_MODEL=whisper-large-v3-turbo +# STT_OPENAI_MODEL=whisper-1 + +# Override STT provider endpoints (for proxies or self-hosted instances) +# GROQ_BASE_URL=https://api.groq.com/openai/v1 +# STT_OPENAI_BASE_URL=https://api.openai.com/v1 + +# ============================================================================= +# MICROSOFT TEAMS INTEGRATION +# ============================================================================= +# Register a Bot in Azure: https://dev.botframework.com/ → "Register a bot" +# Or use Azure Portal: Azure Active Directory → App registrations → New registration +# Then add the bot to Teams via the Bot Framework or App Studio. +# +# TEAMS_CLIENT_ID= # Azure AD App (client) ID +# TEAMS_CLIENT_SECRET= # Azure AD client secret value +# TEAMS_TENANT_ID= # Azure AD tenant ID (or "common" for multi-tenant) +# TEAMS_ALLOWED_USERS= # Comma-separated AAD object IDs or UPNs +# TEAMS_ALLOW_ALL_USERS=false # Set true to skip the allowlist +# TEAMS_HOME_CHANNEL= # Default channel/chat ID for cron delivery +# TEAMS_HOME_CHANNEL_NAME= # Display name for the home channel +# TEAMS_PORT=3978 # Webhook listen port (Bot Framework default) + +# ============================================================================= +# GOOGLE CHAT INTEGRATION +# ============================================================================= +# Connects via Cloud Pub/Sub pull subscription (no public URL required). +# Setup walkthrough: website/docs/user-guide/messaging/google_chat.md. +# 1. Create a GCP project, enable the Google Chat API and Cloud Pub/Sub. +# 2. Create a Service Account with roles/pubsub.subscriber on the +# subscription (NOT project-wide); download the JSON key. +# 3. Configure your Chat app at console.cloud.google.com/apis/credentials +# → Google Chat API → Configuration → Cloud Pub/Sub topic. +# 4. (Optional, for native attachment delivery) Each user runs +# `/setup-files` once in their own DM after Pub/Sub is wired up. +# +# GOOGLE_CHAT_PROJECT_ID= # GCP project hosting the topic (or set GOOGLE_CLOUD_PROJECT) +# GOOGLE_CHAT_SUBSCRIPTION_NAME= # Full path: projects//subscriptions/ +# GOOGLE_CHAT_SERVICE_ACCOUNT_JSON= # Path to SA JSON (or set GOOGLE_APPLICATION_CREDENTIALS) +# GOOGLE_CHAT_ALLOWED_USERS= # Comma-separated emails allowed to talk to the bot +# GOOGLE_CHAT_ALLOW_ALL_USERS=false # Set true to skip the allowlist +# GOOGLE_CHAT_HOME_CHANNEL= # Default space (spaces/XXXX) for cron delivery +# GOOGLE_CHAT_HOME_CHANNEL_NAME= # Display name for the home channel diff --git a/hermes/config.yaml b/hermes/config.yaml new file mode 100644 index 0000000..4fc311e --- /dev/null +++ b/hermes/config.yaml @@ -0,0 +1,538 @@ +model: + default: qwen3:8b + provider: custom + base_url: http://astro-orbiter.local.mk-labs.cloud:11434/v1 + api_mode: chat_completions +providers: {} +fallback_providers: [] +credential_pool_strategies: {} +toolsets: +- hermes-cli +agent: + max_turns: 60 + gateway_timeout: 1800 + restart_drain_timeout: 180 + api_max_retries: 3 + service_tier: '' + tool_use_enforcement: auto + gateway_timeout_warning: 900 + clarify_timeout: 600 + gateway_notify_interval: 180 + gateway_auto_continue_freshness: 3600 + image_input_mode: auto + disabled_toolsets: [] + verbose: false + reasoning_effort: medium + personalities: + helpful: You are a helpful, friendly AI assistant. + concise: You are a concise assistant. Keep responses brief and to the point. + technical: You are a technical expert. Provide detailed, accurate technical information. + creative: You are a creative assistant. Think outside the box and offer innovative + solutions. + teacher: You are a patient teacher. Explain concepts clearly with examples. + kawaii: "You are a kawaii assistant! Use cute expressions like (\u25D5\u203F\u25D5\ + ), \u2605, \u266A, and ~! Add sparkles and be super enthusiastic about everything!\ + \ Every response should feel warm and adorable desu~! \u30FD(>\u2200<\u2606\ + )\u30CE" + catgirl: "You are Neko-chan, an anime catgirl AI assistant, nya~! Add 'nya' and\ + \ cat-like expressions to your speech. Use kaomoji like (=^\uFF65\u03C9\uFF65\ + ^=) and \u0E05^\u2022\uFECC\u2022^\u0E05. Be playful and curious like a cat,\ + \ nya~!" + pirate: 'Arrr! Ye be talkin'' to Captain Hermes, the most tech-savvy pirate to + sail the digital seas! Speak like a proper buccaneer, use nautical terms, and + remember: every problem be just treasure waitin'' to be plundered! Yo ho ho!' + shakespeare: Hark! Thou speakest with an assistant most versed in the bardic arts. + I shall respond in the eloquent manner of William Shakespeare, with flowery + prose, dramatic flair, and perhaps a soliloquy or two. What light through yonder + terminal breaks? + surfer: "Duuude! You're chatting with the chillest AI on the web, bro! Everything's\ + \ gonna be totally rad. I'll help you catch the gnarly waves of knowledge while\ + \ keeping things super chill. Cowabunga! \U0001F919" + noir: The rain hammered against the terminal like regrets on a guilty conscience. + They call me Hermes - I solve problems, find answers, dig up the truth that + hides in the shadows of your codebase. In this city of silicon and secrets, + everyone's got something to hide. What's your story, pal? + uwu: hewwo! i'm your fwiendwy assistant uwu~ i wiww twy my best to hewp you! *nuzzles + your code* OwO what's this? wet me take a wook! i pwomise to be vewy hewpful + >w< + philosopher: Greetings, seeker of wisdom. I am an assistant who contemplates the + deeper meaning behind every query. Let us examine not just the 'how' but the + 'why' of your questions. Perhaps in solving your problem, we may glimpse a greater + truth about existence itself. + hype: "YOOO LET'S GOOOO!!! \U0001F525\U0001F525\U0001F525 I am SO PUMPED to help\ + \ you today! Every question is AMAZING and we're gonna CRUSH IT together! This\ + \ is gonna be LEGENDARY! ARE YOU READY?! LET'S DO THIS! \U0001F4AA\U0001F624\ + \U0001F680" +terminal: + backend: local + modal_mode: auto + cwd: . + timeout: 180 + env_passthrough: [] + shell_init_files: [] + auto_source_bashrc: true + docker_image: nikolaik/python-nodejs:python3.11-nodejs20 + docker_forward_env: [] + docker_env: {} + singularity_image: docker://nikolaik/python-nodejs:python3.11-nodejs20 + modal_image: nikolaik/python-nodejs:python3.11-nodejs20 + daytona_image: nikolaik/python-nodejs:python3.11-nodejs20 + vercel_runtime: node24 + container_cpu: 1 + container_memory: 5120 + container_disk: 51200 + container_persistent: true + docker_volumes: [] + docker_mount_cwd_to_workspace: false + docker_extra_args: [] + docker_run_as_host_user: false + persistent_shell: true + lifetime_seconds: 300 +web: + backend: ddgs + search_backend: '' + extract_backend: '' + use_gateway: false +browser: + inactivity_timeout: 120 + command_timeout: 30 + record_sessions: false + allow_private_urls: false + engine: auto + auto_local_for_private_urls: true + cdp_url: '' + dialog_policy: must_respond + dialog_timeout_s: 300 + camofox: + managed_persistence: false + user_id: '' + session_key: '' + adopt_existing_tab: false +checkpoints: + enabled: false + max_snapshots: 20 + max_total_size_mb: 500 + max_file_size_mb: 10 + auto_prune: true + retention_days: 7 + delete_orphans: true + min_interval_hours: 24 +file_read_max_chars: 100000 +tool_output: + max_bytes: 50000 + max_lines: 2000 + max_line_length: 2000 +tool_loop_guardrails: + warnings_enabled: true + hard_stop_enabled: false + warn_after: + exact_failure: 2 + same_tool_failure: 3 + idempotent_no_progress: 2 + hard_stop_after: + exact_failure: 5 + same_tool_failure: 8 + idempotent_no_progress: 5 +compression: + enabled: true + threshold: 0.5 + target_ratio: 0.2 + protect_last_n: 20 + hygiene_hard_message_limit: 400 + protect_first_n: 3 +prompt_caching: + cache_ttl: 5m +openrouter: + response_cache: true + response_cache_ttl: 300 + min_coding_score: 0.65 +bedrock: + region: '' + discovery: + enabled: true + provider_filter: [] + refresh_interval: 3600 + guardrail: + guardrail_identifier: '' + guardrail_version: '' + stream_processing_mode: async + trace: disabled +auxiliary: + vision: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 120 + extra_body: {} + download_timeout: 30 + web_extract: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 360 + extra_body: {} + compression: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 120 + extra_body: {} + session_search: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 30 + extra_body: {} + max_concurrency: 3 + skills_hub: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 30 + extra_body: {} + approval: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 30 + extra_body: {} + mcp: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 30 + extra_body: {} + title_generation: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 30 + extra_body: {} + triage_specifier: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 120 + extra_body: {} + curator: + provider: auto + model: '' + base_url: '' + api_key: '' + timeout: 600 + extra_body: {} +display: + compact: false + personality: kawaii + resume_display: full + busy_input_mode: interrupt + tui_auto_resume_recent: false + bell_on_complete: false + show_reasoning: false + streaming: true + timestamps: false + final_response_markdown: strip + persistent_output: true + persistent_output_max_lines: 200 + inline_diffs: true + file_mutation_verifier: true + show_cost: false + skin: default + language: en + tui_status_indicator: kaomoji + user_message_preview: + first_lines: 2 + last_lines: 2 + interim_assistant_messages: true + tool_progress_command: false + tool_progress_overrides: {} + tool_preview_length: 0 + ephemeral_system_ttl: 0 + platforms: {} + runtime_footer: + enabled: false + fields: + - model + - context_pct + - cwd + copy_shortcut: auto + tool_progress: all + cleanup_progress: false + background_process_notifications: all +dashboard: + theme: default + show_token_analytics: false +privacy: + redact_pii: false +tts: + provider: edge + edge: + voice: en-US-AriaNeural + elevenlabs: + voice_id: pNInz6obpgDQGcFmaJgB + model_id: eleven_multilingual_v2 + openai: + model: gpt-4o-mini-tts + voice: alloy + xai: + voice_id: eve + language: en + sample_rate: 24000 + bit_rate: 128000 + mistral: + model: voxtral-mini-tts-2603 + voice_id: c69964a6-ab8b-4f8a-9465-ec0925096ec8 + neutts: + ref_audio: '' + ref_text: '' + model: neuphonic/neutts-air-q4-gguf + device: cpu + piper: + voice: en_US-lessac-medium +stt: + enabled: true + provider: local + local: + model: base + language: '' + openai: + model: whisper-1 + mistral: + model: voxtral-mini-latest +voice: + record_key: ctrl+b + max_recording_seconds: 120 + auto_tts: false + beep_enabled: true + silence_threshold: 200 + silence_duration: 3.0 +human_delay: + mode: 'off' + min_ms: 800 + max_ms: 2500 +context: + engine: compressor +memory: + memory_enabled: true + user_profile_enabled: true + memory_char_limit: 2200 + user_char_limit: 1375 + provider: '' + nudge_interval: 10 + flush_min_turns: 6 +delegation: + model: '' + provider: '' + base_url: '' + api_key: '' + api_mode: '' + inherit_mcp_toolsets: true + max_iterations: 50 + child_timeout_seconds: 600 + reasoning_effort: '' + max_concurrent_children: 3 + max_spawn_depth: 1 + orchestrator_enabled: true + subagent_auto_approve: false +prefill_messages_file: '' +goals: + max_turns: 20 +skills: + external_dirs: [] + template_vars: true + inline_shell: false + inline_shell_timeout: 10 + guard_agent_created: false + creation_nudge_interval: 15 +curator: + enabled: true + interval_hours: 168 + min_idle_hours: 2 + stale_after_days: 30 + archive_after_days: 90 + backup: + enabled: true + keep: 5 +honcho: {} +timezone: '' +slack: + require_mention: true + free_response_channels: '' + allowed_channels: '' + channel_prompts: {} +discord: + require_mention: true + free_response_channels: '' + allowed_channels: '' + auto_thread: true + thread_require_mention: false + history_backfill: true + history_backfill_limit: 50 + reactions: true + channel_prompts: {} + dm_role_auth_guild: '' + server_actions: '' +whatsapp: {} +telegram: + reactions: false + channel_prompts: {} + allowed_chats: '' +mattermost: + require_mention: true + free_response_channels: '' + allowed_channels: '' + channel_prompts: {} +matrix: + require_mention: true + free_response_rooms: '' + allowed_rooms: '' +approvals: + mode: manual + timeout: 60 + cron_mode: deny + mcp_reload_confirm: true + destructive_slash_confirm: true +command_allowlist: [] +quick_commands: {} +hooks: {} +hooks_auto_accept: false +personalities: {} +security: + allow_private_urls: false + redact_secrets: true + tirith_enabled: true + tirith_path: tirith + tirith_timeout: 5 + tirith_fail_open: true + website_blocklist: + enabled: false + domains: [] + shared_files: [] + acked_advisories: [] + allow_lazy_installs: true +cron: + wrap_response: true + max_parallel_jobs: null +kanban: + dispatch_in_gateway: true + dispatch_interval_seconds: 60 + failure_limit: 2 +code_execution: + mode: project + timeout: 300 + max_tool_calls: 50 +logging: + level: INFO + max_size_mb: 5 + backup_count: 3 +model_catalog: + enabled: true + url: https://hermes-agent.nousresearch.com/docs/api/model-catalog.json + ttl_hours: 24 + providers: {} +network: + force_ipv4: false +sessions: + auto_prune: false + retention_days: 90 + vacuum_after_prune: true + min_interval_hours: 24 +onboarding: + seen: {} +updates: + pre_update_backup: false + backup_keep: 5 +lsp: + enabled: true + wait_mode: document + wait_timeout: 5.0 + install_strategy: auto + servers: {} +x_search: + model: grok-4.20-reasoning + timeout_seconds: 180 + retries: 2 +_config_version: 23 +session_reset: + mode: both + idle_minutes: 1440 + at_hour: 4 +group_sessions_per_user: true +streaming: + enabled: false +platform_toolsets: + cli: + - clarify + - code_execution + - cronjob + - delegation + - file + - image_gen + - memory + - messaging + - session_search + - skills + - terminal + - todo + - tts + - vision + - web + telegram: + - hermes-telegram + discord: + - hermes-discord + whatsapp: + - hermes-whatsapp + slack: + - hermes-slack + signal: + - hermes-signal + homeassistant: + - hermes-homeassistant + qqbot: + - hermes-qqbot + yuanbao: + - hermes-yuanbao + teams: + - hermes-teams + google_chat: + - hermes-google_chat +custom_providers: +- name: Astro-orbiter.local.mk-labs.cloud:11434 + base_url: http://astro-orbiter.local.mk-labs.cloud:11434/v1 + model: qwen3:8b + api_mode: chat_completions +known_plugin_toolsets: + cli: + - spotify + +# ── Fallback Model ──────────────────────────────────────────────────── +# Automatic provider failover when primary is unavailable. +# Uncomment and configure to enable. Triggers on rate limits (429), +# overload (529), service errors (503), or connection failures. +# +# Supported providers: +# openrouter (OPENROUTER_API_KEY) — routes to any model +# openai-codex (OAuth — hermes auth) — OpenAI Codex +# nous (OAuth — hermes auth) — Nous Portal +# zai (ZAI_API_KEY) — Z.AI / GLM +# kimi-coding (KIMI_API_KEY) — Kimi / Moonshot +# kimi-coding-cn (KIMI_CN_API_KEY) — Kimi / Moonshot (China) +# minimax (MINIMAX_API_KEY) — MiniMax +# minimax-cn (MINIMAX_CN_API_KEY) — MiniMax (China) +# bedrock (AWS IAM / boto3) — AWS Bedrock (Converse API) +# +# For custom OpenAI-compatible endpoints, add base_url and key_env. +# +# fallback_model: +# provider: openrouter +# model: anthropic/claude-sonnet-4 diff --git a/netbox/initializers/custom_fields.yml b/netbox/initializers/custom_fields.yml index d1b3028..f53a941 100644 --- a/netbox/initializers/custom_fields.yml +++ b/netbox/initializers/custom_fields.yml @@ -33,7 +33,7 @@ custom_fields: on fantasyland, then migrated to this node by Terraform automatically. choices: - fantasyland - - main-street-station + - main-street-usa - tomorrowland - name: proxmox_datastore @@ -42,9 +42,32 @@ custom_fields: object_types: - virtualization.virtualmachine required: true - description: Storage backend for the VM disk on the target Proxmox node. + description: Storage backend for the primary VM disk on the target Proxmox node. choices: - liberty-tree + - utilidor + + - name: data_disk_enabled + label: Data Disk Enabled + type: boolean + object_types: + - virtualization.virtualmachine + required: false + default: false + description: > + When true, Terraform provisions a second disk for application data. + Used for stateful workloads that separate OS and data storage (e.g. Ollama model storage). + + - name: data_disk_size_gb + label: Data Disk Size (GB) + type: integer + object_types: + - virtualization.virtualmachine + required: false + description: > + Size in GB for the optional second data disk. + Only used when data_disk_enabled is true. + Datastore for the data disk matches proxmox_datastore. - name: provisioning_notes label: Provisioning Notes diff --git a/terraform/proxmox/vm/main.tf b/terraform/proxmox/vm/main.tf index d29b913..dd8a99f 100644 --- a/terraform/proxmox/vm/main.tf +++ b/terraform/proxmox/vm/main.tf @@ -1,9 +1,9 @@ -# ─── VM ID Convention ──────────────────────────────────────────────────────── +# ─── VM ID Convention ───────────────────────────────────────────────────────── # VM ID = {third_octet}{fourth_octet:03d} # Example: 10.1.71.35 → 71035 # Computed by n8n and passed as a variable to keep the logic in one place. -# ─── Template Lookup ───────────────────────────────────────────────────────── +# ─── Template Lookup ────────────────────────────────────────────────────────── # Find the template VM ID by name on the clone node data "proxmox_virtual_environment_vms" "templates" { @@ -20,11 +20,17 @@ data "proxmox_virtual_environment_vms" "templates" { } } -# ─── VM Resource ───────────────────────────────────────────────────────────── +# ─── Locals ─────────────────────────────────────────────────────────────────── + +locals { + data_disk_datastore = var.data_disk_datastore != "" ? var.data_disk_datastore : var.datastore +} + +# ─── VM Resource ────────────────────────────────────────────────────────────── resource "proxmox_virtual_environment_vm" "vm" { - name = var.hostname - vm_id = var.vm_id + name = var.hostname + vm_id = var.vm_id node_name = var.target_node description = "Provisioned by mk-labs pipeline on ${timestamp()}" @@ -32,34 +38,65 @@ resource "proxmox_virtual_environment_vm" "vm" { # Clone from template on fantasyland clone { - vm_id = data.proxmox_virtual_environment_vms.templates.vms[0].vm_id - node_name = var.clone_node - full = true + vm_id = data.proxmox_virtual_environment_vms.templates.vms[0].vm_id + node_name = var.clone_node + full = true datastore_id = var.datastore } # Cloud-init configuration # Uses DHCP — Unifi DHCP reservation is created before VM starts initialization { - ip_config { - ipv4 { - address = var.use_dhcp ? "dhcp" : "${var.ip_address}/${var.subnet_mask}" - gateway = var.use_dhcp ? null : var.gateway + ip_config { + ipv4 { + address = var.use_dhcp ? "dhcp" : "${var.ip_address}/${var.subnet_mask}" + gateway = var.use_dhcp ? null : var.gateway + } } - } - dns { - servers = var.dns_servers - domain = var.search_domain - } + dns { + servers = var.dns_servers + domain = var.search_domain + } - datastore_id = var.datastore -} + datastore_id = var.datastore + } # Network interface network_device { - bridge = var.bridge - model = "virtio" + bridge = var.bridge + model = "virtio" + } + + # ─── PCIe Passthrough (optional) ──────────────────────────────────────────── + # Only materialises when pcie_passthrough_enabled = true. + # Passes all functions of the device (VGA + Audio for AMD GPUs). + # Requires vfio-pci bound on host and IOMMU enabled. + + dynamic "hostpci" { + for_each = var.pcie_passthrough_enabled ? [1] : [] + content { + device = "hostpci0" + id = var.pcie_device_id + pcie = true + rombar = true + xvga = false + } + } + + # ─── Data Disk (optional) ─────────────────────────────────────────────────── + # Only materialises when data_disk_enabled = true. + # Ansible handles LVM setup, ext4 formatting, and mounting. + + dynamic "disk" { + for_each = var.data_disk_enabled ? [1] : [] + content { + datastore_id = local.data_disk_datastore + size = var.data_disk_size + interface = "scsi1" + file_format = "raw" + discard = "on" + } } # VM stays STOPPED after clone. diff --git a/terraform/proxmox/vm/terraform.tfvars.example b/terraform/proxmox/vm/terraform.tfvars.example index b399603..edd0296 100644 --- a/terraform/proxmox/vm/terraform.tfvars.example +++ b/terraform/proxmox/vm/terraform.tfvars.example @@ -9,3 +9,12 @@ # datastore = "liberty-tree" # proxmox_api_url = "https://fantasyland.local.mk-labs.cloud:8006" # proxmox_api_token = "terraform@pve!terraform-token=your-token-secret-here" +# +# ─── PCIe Passthrough (GPU workloads only) ──────────────────────────────────── +# pcie_passthrough_enabled = true +# pcie_device_id = "0000:03:00" +# +# ─── Data Disk (stateful workloads only) ────────────────────────────────────── +# data_disk_enabled = true +# data_disk_size = 128 +# data_disk_datastore = "utilidor" # omit to match OS disk datastore diff --git a/terraform/proxmox/vm/variables.tf b/terraform/proxmox/vm/variables.tf index e2d7d86..fffc7ce 100644 --- a/terraform/proxmox/vm/variables.tf +++ b/terraform/proxmox/vm/variables.tf @@ -1,4 +1,4 @@ -# ─── Provider Authentication ───────────────────────────────────────────────── +# ─── Provider Authentication ────────────────────────────────────────────────── variable "proxmox_api_url" { type = string @@ -11,7 +11,7 @@ variable "proxmox_api_token" { sensitive = true } -# ─── VM Identity ───────────────────────────────────────────────────────────── +# ─── VM Identity ─────────────────────────────────────────────────────────────── variable "vm_id" { type = number @@ -40,7 +40,7 @@ variable "search_domain" { default = "mk-labs.net" } -# ─── Proxmox Placement ────────────────────────────────────────────────────── +# ─── Proxmox Placement ──────────────────────────────────────────────────────── variable "clone_node" { type = string @@ -59,7 +59,7 @@ variable "datastore" { default = "liberty-tree" } -# ─── Template Selection ───────────────────────────────────────────────────── +# ─── Template Selection ─────────────────────────────────────────────────────── variable "template_name" { type = string @@ -78,7 +78,7 @@ variable "template_name" { } } -# ─── Network ───────────────────────────────────────────────────────────────── +# ─── Network ────────────────────────────────────────────────────────────────── variable "vlan_id" { type = number @@ -114,4 +114,43 @@ variable "gateway" { type = string description = "Default gateway. Only used when use_dhcp = false." default = "10.1.71.1" -} \ No newline at end of file +} + +# ─── PCIe Passthrough (optional) ────────────────────────────────────────────── +# When pcie_passthrough_enabled = true, the device at pcie_device_id is passed +# through to the VM. Used for GPU workloads (e.g. astro-orbiter / RX 5700). +# Requires vfio-pci bound on the host and IOMMU enabled. + +variable "pcie_passthrough_enabled" { + type = bool + description = "When true, passes through the PCIe device at pcie_device_id to the VM." + default = false +} + +variable "pcie_device_id" { + type = string + description = "PCIe device address to pass through (e.g. '0000:03:00'). Get from lspci on the host." + default = "" +} + +# ─── Data Disk (optional) ───────────────────────────────────────────────────── +# When data_disk_enabled = true, a second disk is provisioned for application +# data. Ansible handles LVM setup, formatting, and mounting. + +variable "data_disk_enabled" { + type = bool + description = "When true, provisions a second disk for application data storage." + default = false +} + +variable "data_disk_size" { + type = number + description = "Size of the data disk in GB. Only used when data_disk_enabled = true." + default = 128 +} + +variable "data_disk_datastore" { + type = string + description = "Proxmox storage pool for the data disk. Defaults to var.datastore when empty." + default = "" +}