feat(fastpass): Talos cluster provisioning and bootstrap

- Terraform: VM provisioning, Unifi DHCP, Technitium DNS
- talhelper: cluster config for 6-node Talos cluster
- Cilium 1.19.4 CNI with Talos-compatible security context
- docs: city-hall setup guide and bootstrap runbook
This commit is contained in:
2026-05-17 16:08:19 -05:00
parent 9f3ac95d8d
commit 97e9889251
16 changed files with 1205 additions and 0 deletions

View File

@@ -0,0 +1,60 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/bpg/proxmox" {
version = "0.106.0"
constraints = ">= 0.74.0"
hashes = [
"h1:CnUVTBB9ZlJRH9xhyxPIAY++4gSRqMeLMZokzgN1V2A=",
"zh:1a8d717dada526bf0aab51683352390e194bf5c05c8c69464e00befd113121d6",
"zh:31db845b45df5839b442104e31aa41f832eb1f363f5278df290f1a906b731150",
"zh:36af78f4395ac64e5a997fad6cfb18f369e5cae7e4895086ef7d39c65e3d6324",
"zh:526e87482db9a0f8c7a22b430214886a3df4fd08c6949b7235d9a088850fd736",
"zh:6c99f833606527b86fed0ef74bf8723e118b98a8067ff9f3745e90b095a4bc2b",
"zh:7aa85292c4891e85a817821dd31d736e977eabe3a04d29913d1cb5dfca2020ef",
"zh:879dc2b315093c2818c8ff289fcb20d6925dab57e7d758b5e646c6b35860ac37",
"zh:955eb04b8df5438a717af9766f02b48e919c0a33c97590905ee04f3106adb299",
"zh:981eb203d9cfbc735bd792a7654319a5a9b8d68a485d0399b4f018557550f0ce",
"zh:a0e3b74af3976aa4f9e68a7c8bde620abda2d4f0d29859032f9d6d2df3e4c200",
"zh:b767b70e56dc47238ef9f541c29a16e97e7f19b63b303d551593b620d0c9f609",
"zh:edf48f3699cf8597d07d7ab8923e5353da4337dfa9462c07c289643309bf7494",
"zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
"zh:faf3b240bce18a4cf91359b0bdb017f9b2a8f4c38202c4c959c3149f69bf321e",
]
}
provider "registry.terraform.io/darkhonor/technitium" {
version = "1.1.0"
constraints = ">= 0.3.0"
hashes = [
"h1:+PIjpgfSqFxA3893Jvd8Xs4xiuoiGr/4Lu4QXOZWW2g=",
"zh:0855bb0e2a5bd9ec68a6a7cf117980189b2f2030232ef7ae6815c97118eb0908",
"zh:1cbca86ce5f77e8aa32f863f2f40046a12b32f4d94860f7f0a03d24a6091ffd4",
"zh:268dc7849a9fee58efcfbe662fc4a0f915bb5a4e1f55746228926a02bcbde408",
"zh:6e878f8bdf8b468166774766879094d39626bd03995b0942ce156448c22988f9",
"zh:7006c9b909905f5fe8cee43610d4676ec555666ff26862b8821122e428327cd3",
"zh:97d3c059d8a44d7525232f1471b49f638ea071a85955d82630d1c7682a58dab2",
]
}
provider "registry.terraform.io/paultyng/unifi" {
version = "0.41.0"
constraints = "0.41.0"
hashes = [
"h1:vE4C9E7NtYUC7otQrAvNNkDFBHqEMxkUlcqJhLWchLY=",
"zh:03ddd3aee05a08e1446f75a7b3f52810181d3307728cba08ce8fb67f109a9c00",
"zh:11b14b79ad02b0a55fd6116b10c0eb6fab432dd7d1f3527af0e2055adf292451",
"zh:18c0eb19889927f115a1e05d64f59b4e8d530ccdf1a8b574940a86be20973564",
"zh:2df9ca0c21830d2757758e574b19d0d4e54965ce80dbbfb3f124db1dac3d7e8f",
"zh:36274af3b7e8b08ba69c04a226c63e0dd2ec386c583288ebd7bc2a30e349ee8f",
"zh:413eb222ef30889bab33ccbfc46c9fb64307555da34eac4625d51e696ac72e1d",
"zh:4839814ff9f405a13397ffadd6f1052c770b88802280a4d8cde066f9a19718c7",
"zh:9547b7831852cc5b9c0fd13ab447d48539eae94582c8725ad255af36e31fb5d9",
"zh:a855c89b12326eb1c89bbf292a2bb1de3651794e3409d5012076ada89aabdc8a",
"zh:aef12a33b90fd77a9bf4e9d397966ccbfa4a037a648a1725074aff2db2d90fb0",
"zh:b3c72a6a02e29b4d21aa0d0831a272ca7cb82c3f8c2c3c7f09fcc2d2dcd78752",
"zh:c8354eaaab5f526e8e530b098544c7583a0f0b5b27d67500c7b3e9da56a3a7e5",
"zh:dc29f1e70f20ce86d3c6a66c7a817616f993a1cf9d941604dfd5222a06992c4c",
"zh:e772779333419f34d2c6da333c7f7d235a5a34f21ea47636b548e132aed74f3b",
]
}

View File

@@ -0,0 +1,151 @@
# ─── fastpass Talos Cluster ───────────────────────────────────────────────────
#
# Provisions all infrastructure for the fastpass Kubernetes cluster:
# - Talos ISO download to Proxmox
# - 3 control plane VMs (space-mountain, big-thunder-mountain, splash-mountain)
# - 3 worker VMs (jungle-cruise, haunted-mansion, peter-pans-flight)
# - Unifi DHCP reservations (MAC → IP) for all 6 nodes
# - Technitium DNS A records for all 6 nodes + cluster VIP
#
# After apply:
# Run talhelper from city-hall to generate and apply machine configs.
# See talos/talhelper/ for cluster configuration.
#
# IP Allocation (10.1.71.64/26 — Container Orchestration):
# .65 — Kubernetes API VIP (Talos native VIP, not a VM)
# .66 — space-mountain (control plane, main-street-usa)
# .67 — big-thunder-mountain (control plane, tomorrowland)
# .68 — splash-mountain (control plane, fantasyland)
# .69 — jungle-cruise (worker, main-street-usa)
# .70 — haunted-mansion (worker, tomorrowland)
# .71 — peter-pans-flight (worker, fantasyland)
# .80-.99 — Cilium LB pool (managed by Cilium, not Terraform)
locals {
# Node definitions — single source of truth for the entire cluster.
# VM ID convention: {third_octet}{fourth_octet:03d}
nodes = {
"space-mountain" = {
role = "controlplane"
target_node = "main-street-usa"
ip_address = "10.1.71.66"
vm_id = 71066
cpu_cores = 4
memory_mb = 8192
disk_size = 50
}
"big-thunder-mountain" = {
role = "controlplane"
target_node = "tomorrowland"
ip_address = "10.1.71.67"
vm_id = 71067
cpu_cores = 4
memory_mb = 8192
disk_size = 50
}
"splash-mountain" = {
role = "controlplane"
target_node = "fantasyland"
ip_address = "10.1.71.68"
vm_id = 71068
cpu_cores = 4
memory_mb = 8192
disk_size = 50
}
"jungle-cruise" = {
role = "worker"
target_node = "main-street-usa"
ip_address = "10.1.71.69"
vm_id = 71069
cpu_cores = 8
memory_mb = 16384
disk_size = 100
}
"haunted-mansion" = {
role = "worker"
target_node = "tomorrowland"
ip_address = "10.1.71.70"
vm_id = 71070
cpu_cores = 8
memory_mb = 16384
disk_size = 100
}
"peter-pans-flight" = {
role = "worker"
target_node = "fantasyland"
ip_address = "10.1.71.71"
vm_id = 71071
cpu_cores = 8
memory_mb = 16384
disk_size = 100
}
}
}
# ─── Talos ISO ───────────────────────────────────────────────────────────────
# ISO is pre-uploaded to the shared 'templates' datastore (Ceph-backed,
# visible to all nodes). File ID format: <datastore>:iso/<filename>
locals {
talos_iso_file_id = "${var.iso_datastore}:iso/metal-amd64.iso"
}
# ─── Cluster Nodes ───────────────────────────────────────────────────────────
# Iterate over the node map and call the node module for each.
module "nodes" {
source = "../modules/node"
for_each = local.nodes
hostname = each.key
vm_id = each.value.vm_id
node_role = each.value.role
target_node = each.value.target_node
iso_file_id = local.talos_iso_file_id
cpu_cores = each.value.cpu_cores
memory_mb = each.value.memory_mb
disk_size = each.value.disk_size
datastore = var.datastore
bridge = var.bridge
}
# ─── Unifi DHCP Reservations ─────────────────────────────────────────────────
# Create a static MAC→IP reservation for each node.
# Reservations are created before VMs start so the IP is guaranteed
# on first DHCP broadcast.
resource "unifi_user" "nodes" {
for_each = local.nodes
name = each.key
mac = lower(module.nodes[each.key].mac_address)
fixed_ip = each.value.ip_address
note = "Managed by mk-labs Terraform — fastpass cluster ${each.value.role}"
depends_on = [module.nodes]
}
# ─── DNS A Records ───────────────────────────────────────────────────────────
# Create A records for all 6 nodes in the internal zone.
# The cluster VIP (.65) is managed by Talos natively — no VM, but still
# needs a DNS entry for kubectl and talhelper to resolve the endpoint.
resource "technitium_record" "nodes" {
for_each = local.nodes
zone = var.dns_zone
name = "${each.key}.${var.dns_zone}"
type = "A"
ttl = 300
value = each.value.ip_address
}
resource "technitium_record" "vip" {
zone = var.dns_zone
name = "${var.cluster_name}-api.${var.dns_zone}"
type = "A"
ttl = 300
value = "10.1.71.65"
}

View File

@@ -0,0 +1,43 @@
# ─── Post-Apply Reminders ────────────────────────────────────────────────────
output "ha_rule_instructions" {
description = "Manual steps required after the Talos cluster is fully installed and the ISO is no longer needed."
value = <<-EOT
── Post-Cluster-Install: HA Setup ──────────────────────────────────────────
Once Talos is fully installed and running (ISO no longer needed for boot),
complete the following steps:
1. Register each VM with the Proxmox HA manager:
ha-manager add vm:71066 --state started
ha-manager add vm:71067 --state started
ha-manager add vm:71068 --state started
ha-manager add vm:71069 --state started
ha-manager add vm:71070 --state started
ha-manager add vm:71071 --state started
2. Add VMs to the existing node-affinity rules (check current resources
first with pvesh get /cluster/ha/rules/<rule>):
prefer_main-street-usa (space-mountain:71066, jungle-cruise:71069):
pvesh set /cluster/ha/rules/prefer_main-street-usa \
--type node-affinity \
--resources "<existing>,vm:71066,vm:71069"
prefer_tomorrowland (big-thunder-mountain:71067, haunted-mansion:71070):
pvesh set /cluster/ha/rules/prefer_tomorrowland \
--type node-affinity \
--resources "<existing>,vm:71067,vm:71070"
prefer_fantasyland (splash-mountain:71068, peter-pans-flight:71071):
pvesh set /cluster/ha/rules/prefer_fantasyland \
--type node-affinity \
--resources "<existing>,vm:71068,vm:71071"
3. Eject the Talos ISO from each VM in the Proxmox UI or via:
qm set <vmid> --ide2 none
────────────────────────────────────────────────────────────────────────────
EOT
}

View File

@@ -0,0 +1,48 @@
terraform {
required_version = ">= 1.5.0"
required_providers {
proxmox = {
source = "bpg/proxmox"
version = ">= 0.74.0"
}
unifi = {
source = "paultyng/unifi"
version = "0.41.0"
}
technitium = {
source = "darkhonor/technitium"
version = ">= 0.3.0"
}
}
}
# ─── Proxmox Provider ────────────────────────────────────────────────────────
provider "proxmox" {
endpoint = var.proxmox_api_url
api_token = var.proxmox_api_token
insecure = true
ssh {
agent = true
}
}
# ─── Unifi Provider ──────────────────────────────────────────────────────────
provider "unifi" {
username = var.unifi_username
password = var.unifi_password
api_url = var.unifi_api_url
allow_insecure = true
}
# ─── Technitium DNS Provider ─────────────────────────────────────────────────
provider "technitium" {
server_url = var.technitium_url
api_token = var.technitium_api_token
}

View File

@@ -0,0 +1,32 @@
# ─── fastpass Cluster — Variable Values ──────────────────────────────────────
# Copy to fastpass.tfvars and populate.
# Sensitive values (passwords, tokens) are passed via CLI or environment:
#
# terraform apply \
# -var-file="tfvars/fastpass.tfvars" \
# -state="states/fastpass.tfstate" \
# -var="proxmox_api_token=terraform@pve!terraform-token=<secret>" \
# -var="unifi_password=<secret>" \
# -var="technitium_password=<secret>"
# ─── Proxmox ─────────────────────────────────────────────────────────────────
proxmox_api_url = "https://fantasyland.local.mk-labs.cloud:8006"
# ─── Unifi ───────────────────────────────────────────────────────────────────
unifi_api_url = "https://10.1.71.1:443"
unifi_username = "terraform"
# ─── Technitium DNS ───────────────────────────────────────────────────────────
technitium_url = "http://monorail.local.mk-labs.cloud:5380"
# ─── Cluster ─────────────────────────────────────────────────────────────────
cluster_name = "fastpass"
talos_version = "v1.13.2"
dns_zone = "local.mk-labs.cloud"
# ─── ISO ─────────────────────────────────────────────────────────────────────
iso_datastore = "templates"
# ─── VM Defaults ─────────────────────────────────────────────────────────────
datastore = "liberty-tree"
bridge = "vmbr0"

View File

@@ -0,0 +1,40 @@
# ─── fastpass Cluster — Variable Values ──────────────────────────────────────
# Copy to fastpass.tfvars and populate.
# Sensitive values (passwords, tokens) are passed via CLI or environment:
#
# terraform apply \
# -var-file="tfvars/fastpass.tfvars" \
# -state="states/fastpass.tfstate" \
# -var="proxmox_api_token=terraform@pve!terraform-token=<secret>" \
# -var="unifi_password=<secret>" \
# -var="technitium_password=<secret>"
# ─── Proxmox ─────────────────────────────────────────────────────────────────
proxmox_api_url = "https://fantasyland.local.mk-labs.cloud:8006"
proxmox_api_token = "terraform@pve!terraform-token=REPLACE_ME"
# ─── Unifi ───────────────────────────────────────────────────────────────────
unifi_api_url = "https://10.1.71.1"
unifi_username = "terraform"
unifi_password = "REPLACE_ME"
# Retrieve with: curl -sk -X GET https://10.1.71.1/api/network | jq '.data[] | select(.name=="Server Trusted") | ._id'
unifi_network_id = "REPLACE_ME"
# ─── Technitium DNS ───────────────────────────────────────────────────────────
technitium_url = "http://10.1.71.32:5380"
technitium_username = "admin"
technitium_password = "REPLACE_ME"
# ─── Cluster ─────────────────────────────────────────────────────────────────
cluster_name = "fastpass"
talos_version = "v1.9.5"
dns_zone = "local.mk-labs.cloud"
# ─── ISO ─────────────────────────────────────────────────────────────────────
iso_node = "fantasyland"
iso_datastore = "local"
# ─── VM Defaults ─────────────────────────────────────────────────────────────
datastore = "liberty-tree"
bridge = "vmbr0"
vlan_id = 71

View File

@@ -0,0 +1,79 @@
# ─── Proxmox Authentication ──────────────────────────────────────────────────
variable "proxmox_api_url" {
type = string
description = "Proxmox API endpoint URL (e.g., https://fantasyland.local.mk-labs.cloud:8006)."
}
variable "proxmox_api_token" {
type = string
description = "Proxmox API token. Format: user@realm!token-name=token-secret."
sensitive = true
}
# ─── Unifi Authentication ────────────────────────────────────────────────────
variable "unifi_api_url" {
type = string
description = "Unifi controller API URL (e.g., https://10.1.71.1:443)."
}
variable "unifi_username" {
type = string
description = "Unifi controller username."
}
variable "unifi_password" {
type = string
description = "Unifi controller password."
sensitive = true
}
# ─── Technitium DNS Authentication ───────────────────────────────────────────
variable "technitium_url" {
type = string
description = "Technitium DNS server URL (e.g., http://10.1.71.32:5380)."
}
variable "technitium_api_token" {
type = string
description = "Technitium API token. Create under Administration → API Tokens."
sensitive = true
}
# ─── Cluster Configuration ───────────────────────────────────────────────────
variable "cluster_name" {
type = string
description = "Talos cluster name."
default = "fastpass"
}
variable "dns_zone" {
type = string
description = "Internal DNS zone for A record creation."
default = "local.mk-labs.cloud"
}
# ─── ISO Storage ─────────────────────────────────────────────────────────────
variable "iso_datastore" {
type = string
description = "Proxmox datastore for ISO storage. Must be visible to all nodes — use a shared/Ceph-backed datastore, not node-local storage."
default = "templates"
}
# ─── Shared VM Defaults ──────────────────────────────────────────────────────
variable "datastore" {
type = string
description = "Proxmox storage pool for VM disks."
default = "liberty-tree"
}
variable "bridge" {
type = string
description = "Proxmox network bridge."
default = "vmbr0"
}

View File

@@ -0,0 +1,94 @@
# ─── Talos Node Resource ─────────────────────────────────────────────────────
# Creates a Proxmox VM booting from the Talos ISO.
# No cloud-init, no template clone — Talos is configured via the Talos API
# after boot using talhelper-generated machine configs from city-hall.
#
# Boot sequence:
# 1. Terraform creates VM (stopped) + DHCP reservation
# 2. Unifi assigns reserved IP when VM starts
# 3. talhelper applies machine config via Talos API (port 50000)
# 4. Cluster bootstraps; ISO is no longer needed after first boot
# (Talos installs itself to disk on first config apply)
resource "proxmox_virtual_environment_vm" "node" {
name = var.hostname
vm_id = var.vm_id
node_name = var.target_node
description = "Talos ${var.node_role} node — fastpass cluster. Managed by mk-labs Terraform."
tags = ["managed", "talos", "fastpass", var.node_role]
bios = "ovmf"
# ─── EFI Disk ─────────────────────────────────────────────────────────────
# Required for OVMF/UEFI boot. Stores EFI variables.
efi_disk {
datastore_id = var.datastore
file_format = "raw"
type = "4m"
}
# ─── Boot: Talos ISO ──────────────────────────────────────────────────────
# Talos boots from ISO, applies machine config via API, then installs
# itself to the disk. After first boot the ISO is not needed.
cdrom {
file_id = var.iso_file_id
interface = "ide2"
}
boot_order = ["scsi0", "ide2"]
# ─── Disk ─────────────────────────────────────────────────────────────────
# Single disk — Talos manages its own partition layout.
# No separate data disk; persistent app storage comes from Pure Storage CSI.
disk {
datastore_id = var.datastore
size = var.disk_size
interface = "scsi0"
file_format = "raw"
ssd = true
discard = "on"
}
# ─── CPU ──────────────────────────────────────────────────────────────────
cpu {
cores = var.cpu_cores
type = "x86-64-v2-AES"
}
# ─── Memory ───────────────────────────────────────────────────────────────
memory {
dedicated = var.memory_mb
}
# ─── Network ──────────────────────────────────────────────────────────────
# Single NIC on VLAN 71 (Server Trusted).
# MAC is deterministic via Terraform so Unifi DHCP reservation matches.
network_device {
bridge = var.bridge
model = "virtio"
}
# ─── QEMU Guest Agent ─────────────────────────────────────────────────────
# Requires the qemu-guest-agent Talos system extension (set in talhelper).
agent {
enabled = true
}
# ─── Start Behavior ───────────────────────────────────────────────────────
# VM starts immediately — Talos needs to boot to accept machine configs.
# Unifi DHCP reservation is created before this resource so the IP is
# already reserved when the NIC first broadcasts a DHCP request.
started = true
# ─── Lifecycle ────────────────────────────────────────────────────────────
lifecycle {
ignore_changes = [
description,
tags,
# cdrom can be ejected manually after first boot without Terraform drift
cdrom,
# node_name changes after HA migrates VMs to their intended hosts
node_name,
]
}
}

View File

@@ -0,0 +1,18 @@
# ─── Node Outputs ────────────────────────────────────────────────────────────
# Consumed by the cluster root module to wire up DHCP reservations and
# DNS records without hardcoding MAC addresses.
output "vm_id" {
description = "Proxmox VM ID."
value = proxmox_virtual_environment_vm.node.vm_id
}
output "mac_address" {
description = "MAC address of the primary network interface. Used to create the Unifi DHCP reservation."
value = proxmox_virtual_environment_vm.node.network_device[0].mac_address
}
output "hostname" {
description = "Node hostname. Passed through for convenience when building DNS records."
value = proxmox_virtual_environment_vm.node.name
}

View File

@@ -0,0 +1,8 @@
terraform {
required_providers {
proxmox = {
source = "bpg/proxmox"
version = ">= 0.74.0"
}
}
}

View File

@@ -0,0 +1,71 @@
# ─── Provider Authentication ─────────────────────────────────────────────────
# Passed through from the cluster root module — not declared here since
# provider config lives in the root module's providers.tf.
# ─── Node Identity ───────────────────────────────────────────────────────────
variable "hostname" {
type = string
description = "Node hostname (e.g., space-mountain). Used as the Proxmox VM name."
}
variable "vm_id" {
type = number
description = "Proxmox VM ID. Convention: {third_octet}{fourth_octet:03d} (e.g., 10.1.71.66 → 71066)."
}
variable "node_role" {
type = string
description = "Role of the node in the cluster. Used for tagging."
validation {
condition = contains(["controlplane", "worker"], var.node_role)
error_message = "node_role must be 'controlplane' or 'worker'."
}
}
# ─── Proxmox Placement ───────────────────────────────────────────────────────
variable "target_node" {
type = string
description = "Proxmox node to place this VM on (e.g., main-street-usa, tomorrowland, fantasyland)."
}
variable "datastore" {
type = string
description = "Proxmox storage pool for the VM disk."
default = "liberty-tree"
}
# ─── Boot Image ──────────────────────────────────────────────────────────────
variable "iso_file_id" {
type = string
description = "Proxmox file ID for the Talos ISO (e.g., local:iso/talos-v1.9.5-metal-amd64.iso). Output from proxmox_virtual_environment_download_file."
}
# ─── Hardware ────────────────────────────────────────────────────────────────
variable "cpu_cores" {
type = number
description = "Number of vCPU cores."
}
variable "memory_mb" {
type = number
description = "RAM in megabytes."
}
variable "disk_size" {
type = number
description = "OS disk size in gigabytes. Talos manages its own partition layout."
}
# ─── Network ─────────────────────────────────────────────────────────────────
variable "bridge" {
type = string
description = "Proxmox network bridge."
default = "vmbr0"
}