feat(semaphore): rewrite role with rootful Podman Quadlet + PostgreSQL

Complete rewrite of the semaphore role. Supersedes three prior
iterations whose admin-user-creation logic was unreliable across
Semaphore CLI versions.

Architecture:
  - Rootful Podman Quadlet under /etc/containers/systemd/
  - Separate PostgreSQL 16-alpine container on a user-defined
    podman network (semaphore-net)
  - Named volumes for both data stores (semaphore_data,
    semaphore_postgres_data) so container recreation is
    non-destructive
  - Pinned image tags: semaphoreui/semaphore:v2.18.5-ansible2.16.5
    and postgres:16-alpine
  - Post-deploy HTTP health check fails the playbook if Semaphore
    doesn't respond on /api/ping within ~60s

Admin user creation remains intentionally manual after first deploy;
the role README documents the exact podman exec command.

Removes the duplicate deploy_semaphore.yml and the now-unneeded
cleanup_semaphore.yml; day1_deploy_semaphore.yml is the canonical
entry point.

ansible/playbooks/cleanup_semaphore.yml

@@ -1,56 +0,0 @@
----
-- name: Cleanup old Semaphore deployment on figment
-  hosts: semaphore_server
-  become: true
-  vars_files:
-    - ../roles/semaphore/defaults/main.yml
-  vars:
-    # Set to true to also remove data directories and config
-    semaphore_force_clean: false
-
-  tasks:
-    - name: Stop and remove old Semaphore container
-      containers.podman.podman_container:
-        name: "{{ semaphore_container_name }}"
-        state: absent
-      ignore_errors: true
-
-    - name: Stop and disable legacy generated systemd service
-      ansible.builtin.systemd:
-        name: "container-{{ semaphore_container_name }}.service"
-        state: stopped
-        enabled: false
-      ignore_errors: true
-
-    - name: Remove legacy systemd unit file
-      ansible.builtin.file:
-        path: "/etc/systemd/system/container-{{ semaphore_container_name }}.service"
-        state: absent
-
-    - name: Remove quadlet files (if any were previously deployed)
-      ansible.builtin.file:
-        path: "/etc/containers/systemd/{{ item }}.container"
-        state: absent
-      loop:
-        - "{{ semaphore_container_name }}"
-        - "{{ semaphore_postgres_container_name }}"
-
-    - name: Reload systemd daemon
-      ansible.builtin.systemd:
-        daemon_reload: true
-
-    - name: Remove data directories (only when semaphore_force_clean=true)
-      when: semaphore_force_clean | bool
-      ansible.builtin.file:
-        path: "{{ item }}"
-        state: absent
-      loop:
-        - "{{ semaphore_data_dir }}"
-        - "{{ semaphore_config_dir }}"
-
-    - name: Remove PostgreSQL data volume marker (only when semaphore_force_clean=true)
-      when: semaphore_force_clean | bool and semaphore_use_postgres | default(false)
-      ansible.builtin.file:
-        path: "/var/lib/containers/storage/volumes/{{ semaphore_postgres_volume }}/_data"
-        state: absent
-      ignore_errors: true

ansible/playbooks/day1_deploy_semaphore.yml

@@ -1,22 +1,17 @@
 ---
-# ------------------------------------------------------------------------------
+# ============================================================================
-# FILE: playbooks/day1_deploy_semaphore.yml
+# day1_deploy_semaphore.yml
-# DESCRIPTION: Deploys Semaphore on imagineering
+# ----------------------------------------------------------------------------
-#              Runs: common → docker-host → semaphore
+# Deploys SemaphoreUI + PostgreSQL on the imagineering host (figment).
+# Run AFTER day0_linux_baseline.yml has been applied to the target.
 #
-# USAGE:
+# Usage:
 #   ansible-playbook -i inventory.yml playbooks/day1_deploy_semaphore.yml
-#
+# ============================================================================
-# SECRETS REQUIRED IN VAULT (group_vars/all/vault):
-#   vault_semaphore_database_password
-#   vault_semaphore_admin_password
-#   vault_semaphore_access_key_encryption
-# ------------------------------------------------------------------------------
 
-- name: Deploy Semaphore
+- name: Deploy SemaphoreUI on imagineering
   hosts: semaphore_server
   become: true
-
+  gather_facts: true
   roles:
-    - docker-host
     - semaphore

ansible/playbooks/deploy_semaphore.yml

@@ -1,10 +0,0 @@
----
-# Cleanup old deployments first with:
-#   ansible-playbook -i inventory.yml playbooks/cleanup_semaphore.yml --limit figment
-#   (add -e "semaphore_force_clean=true" to also remove data)
-
-- name: Deploy Semaphore on figment
-  hosts: figment
-  become: true
-  roles:
-    - semaphore

ansible/roles/semaphore/README.md

@@ -0,0 +1,117 @@
+# semaphore
+
+Deploys [SemaphoreUI](https://semaphoreui.com) with a PostgreSQL backend on
+the mk-labs `imagineering` VM (`figment`) via rootful Podman Quadlets.
+
+## Architecture
+
+```
+                                 ┌─────────────────────┐
+   user ─── HTTPS ─── Traefik ──▶│  figment (10.1.71.37) │
+                                 │                       │
+                                 │  ┌─────────────────┐  │
+                                 │  │ semaphore       │  │
+                                 │  │ :3000           │──┼──┐
+                                 │  └─────────────────┘  │  │ semaphore-net
+                                 │                       │  │ (podman)
+                                 │  ┌─────────────────┐  │  │
+                                 │  │ semaphore-      │◀─┼──┘
+                                 │  │ postgres :5432  │  │
+                                 │  └─────────────────┘  │
+                                 │                       │
+                                 │  /etc/containers/     │
+                                 │    systemd/*.container│
+                                 └───────────────────────┘
+```
+
+- Both containers run as **rootful** Podman services, managed by
+  systemd-generated units from Quadlet files in `/etc/containers/systemd/`.
+- The two containers share a user-defined Podman network
+  (`semaphore-net`) so Semaphore can address PostgreSQL by container name.
+- Data persists on two named Podman volumes (`semaphore_data`,
+  `semaphore_postgres_data`) so container recreation is non-destructive.
+- Image tags are **pinned** — no floating `:latest`.
+
+## Required vault variables
+
+| Variable                                | Purpose                                                 |
+|-----------------------------------------|---------------------------------------------------------|
+| `vault_semaphore_database_password`     | PostgreSQL role password for the `semaphore` DB user.   |
+| `vault_semaphore_admin_password`        | Initial admin password (used during manual user-add).   |
+| `vault_semaphore_access_key_encryption` | 32-byte key for Semaphore-stored access keys.           |
+
+Generate the access-key encryption value with:
+
+```bash
+head -c 32 /dev/urandom | base64
+```
+
+## Usage
+
+```yaml
+- name: Deploy SemaphoreUI
+  hosts: semaphore_server
+  become: true
+  roles:
+    - semaphore
+```
+
+Or via the dedicated playbook:
+
+```bash
+ansible-playbook -i inventory.yml playbooks/day1_deploy_semaphore.yml
+```
+
+## First-run admin user creation (manual)
+
+Semaphore's CLI `user add` is intentionally invoked once, by hand, after
+the first deployment succeeds:
+
+```bash
+sudo podman exec -it semaphore semaphore user add --admin \
+  --login admin \
+  --name "Administrator" \
+  --email admin@local.mk-labs.cloud \
+  --password '<vault_semaphore_admin_password>'
+```
+
+Subsequent password rotations should also be done via the CLI, not by
+re-running this role.
+
+## Pinned versions
+
+- **Semaphore**: `docker.io/semaphoreui/semaphore:v2.18.5-ansible2.16.5`
+  (bundles Ansible 2.16.5 inside the container, matching our controller)
+- **PostgreSQL**: `docker.io/library/postgres:16-alpine`
+
+To bump versions, update `semaphore_image` / `semaphore_postgres_image`
+in `defaults/main.yml`. Test against figment, snapshot first.
+
+## Verification
+
+The role completes with an HTTP health check against
+`http://127.0.0.1:3000/api/ping`. If Semaphore doesn't respond within
+~60 seconds, the role fails loudly rather than reporting green-but-broken.
+
+## Troubleshooting
+
+```bash
+# Container status
+sudo systemctl status semaphore semaphore-postgres
+sudo podman ps -a
+
+# Logs
+sudo journalctl -u semaphore -f
+sudo journalctl -u semaphore-postgres -f
+sudo podman logs semaphore --tail 100
+
+# Network
+sudo podman network inspect semaphore-net
+
+# Wipe and redeploy (destructive — destroys all Semaphore data)
+sudo systemctl stop semaphore semaphore-postgres
+sudo podman volume rm semaphore_data semaphore_postgres_data
+sudo rm /etc/containers/systemd/semaphore.container /etc/containers/systemd/semaphore-postgres.container
+sudo systemctl daemon-reload
+# then re-run the playbook
+```

ansible/roles/semaphore/defaults/main.yml

@@ -1,46 +1,72 @@
 ---
-# Semaphore version (use a known good tag)
+# ============================================================================
-semaphore_version: "latest"
+# semaphore role defaults
+# ============================================================================
+# Deploys SemaphoreUI (https://semaphoreui.com) as a rootful Podman service
+# managed via Quadlet. Uses a separate PostgreSQL 16 container on the same
+# user-defined network. Both services persist to named volumes.
+#
+# Admin user creation is intentionally NOT automated — the Semaphore CLI's
+# `user add` command has been unreliable across versions. Operator creates
+# the first admin manually after the first successful deployment; see README.
+# ============================================================================
 
-# Service account
+# ---------------------------------------------------------------------------
-semaphore_user: cast
+# Image pinning
-semaphore_group: cast
+# ---------------------------------------------------------------------------
+# The -ansible2.16.5 tag bundles Ansible 2.16.5 inside the Semaphore image,
+# which matches the controller. Useful when Semaphore itself runs playbooks.
+semaphore_image: "docker.io/semaphoreui/semaphore:v2.18.5-ansible2.16.5"
+semaphore_postgres_image: "docker.io/library/postgres:16-alpine"
 
-# Container settings
+# ---------------------------------------------------------------------------
+# Container & network identifiers
+# ---------------------------------------------------------------------------
 semaphore_container_name: semaphore
-semaphore_image: "semaphoreui/semaphore:{{ semaphore_version }}"
+semaphore_postgres_container_name: semaphore-postgres
+semaphore_network_name: semaphore-net
 
-# Paths
+# Named podman volumes
-semaphore_data_dir: /var/lib/semaphore
+semaphore_data_volume: semaphore_data
-semaphore_config_dir: /etc/semaphore
+semaphore_postgres_volume: semaphore_postgres_data
 
-# Port
+# ---------------------------------------------------------------------------
-semaphore_port: 3000
+# Networking
+# ---------------------------------------------------------------------------
+# Port exposed on the VM. Traefik on lightning-lane terminates TLS and
+# forwards to this port. Localhost-bind keeps the raw service off the LAN.
+semaphore_listen_address: "0.0.0.0"
+semaphore_listen_port: 3000
 
-# Database
+# Public URL (used by Semaphore for OIDC redirects, webhook URLs, etc.)
-semaphore_db_type: bolt
+semaphore_web_url: "https://semaphore.local.mk-labs.cloud"
-semaphore_db_path: "{{ semaphore_data_dir }}/database.boltdb"
 
-# Vault variables (defined in group_vars/all/vault)
+# ---------------------------------------------------------------------------
-# vault_semaphore_admin_password
+# PostgreSQL configuration
-# vault_semaphore_access_key_encryption
+# ---------------------------------------------------------------------------
-# Initial admin user (password should come from vault)
+semaphore_db_user: semaphore
-semaphore_admin_user: admin
+semaphore_db_name: semaphore
-semaphore_admin_name: Administrator
+# semaphore_db_password sourced from vault below
-semaphore_admin_email: admin@local.mk-labs.cloud
-# semaphore_admin_password: defined in vault
 
-# Set to true to force recreation of the admin user (deletes existing user first)
+# ---------------------------------------------------------------------------
-semaphore_force_admin_user: false
+# Vault inputs (defined in group_vars/all/vault, encrypted with ansible-vault)
+# ---------------------------------------------------------------------------
+# vault_semaphore_database_password      - postgres role password
+# vault_semaphore_admin_password         - initial admin password (used only
+#                                          during manual user-add step)
+# vault_semaphore_access_key_encryption  - 32-byte key for encrypting
+#                                          Semaphore-stored access keys
+semaphore_db_password: "{{ vault_semaphore_database_password }}"
+semaphore_admin_password: "{{ vault_semaphore_admin_password }}"
+semaphore_access_key_encryption: "{{ vault_semaphore_access_key_encryption }}"
 
-# PostgreSQL settings (when using postgres instead of bolt)
+# ---------------------------------------------------------------------------
-semaphore_use_postgres: true
+# Health check
-semaphore_postgres_image: "postgres:16"
+# ---------------------------------------------------------------------------
-semaphore_postgres_container_name: "semaphore-postgres"
+semaphore_health_check_retries: 30
-semaphore_postgres_volume: "semaphore_postgres_data"
+semaphore_health_check_delay: 2
-semaphore_postgres_user: "semaphore"
-semaphore_postgres_db: "semaphore"
-# vault_semaphore_database_password: defined in vault
 
-# Named volumes
+# ---------------------------------------------------------------------------
-semaphore_data_volume: "semaphore_data"
+# Quadlet location (rootful)
+# ---------------------------------------------------------------------------
+semaphore_quadlet_dir: /etc/containers/systemd

ansible/roles/semaphore/handlers/main.yml

@@ -1,3 +1,4 @@
 ---
-# Handlers kept for potential future use or legacy paths.
+# handlers intentionally minimal — Quadlet-managed containers restart
-# Quadlet deployment now handles reload + start directly in the task block.
+# automatically on unit file change after `systemctl daemon-reload`, and
+# the role triggers daemon-reload + systemd state checks explicitly.

ansible/roles/semaphore/meta/main.yml

@@ -1,4 +1,24 @@
 ---
-dependencies:
+galaxy_info:
-  - role: podman
+  role_name: semaphore
-    when: podman_install | default(true)
+  author: JARVIS
+  description: >-
+    Deploys SemaphoreUI with PostgreSQL backend via rootful Podman Quadlet
+    on Ubuntu. Designed for the mk-labs imagineering host (figment).
+  license: MIT
+  min_ansible_version: "2.14"
+  platforms:
+    - name: Ubuntu
+      versions:
+        - noble
+        - jammy
+  galaxy_tags:
+    - semaphore
+    - ansible
+    - podman
+    - quadlet
+    - homelab
+
+# linux-baseline is applied separately as a day0 playbook. We don't depend
+# on it here so this role stays composable.
+dependencies: []

ansible/roles/semaphore/tasks/main.yml

@@ -1,201 +1,36 @@
 ---
-- name: Ensure cast group exists
+# ============================================================================
-  ansible.builtin.group:
+# semaphore / main entrypoint
-    name: "{{ semaphore_group }}"
+# ----------------------------------------------------------------------------
-    state: present
+# Order matters:
-    system: true
+#   1. Podman + Quadlet support installed and ready
+#   2. Network created (containers reference it by name)
+#   3. Volumes created (postgres + semaphore data)
+#   4. Postgres started (Semaphore depends on it being ready)
+#   5. Semaphore started (web service)
+#   6. Verify reachable on listen port
+# ============================================================================
 
-- name: Ensure cast user exists
+- name: Install Podman and dependencies
-  ansible.builtin.user:
+  ansible.builtin.import_tasks: podman.yml
-    name: "{{ semaphore_user }}"
+  tags: [semaphore, podman]
-    group: "{{ semaphore_group }}"
-    shell: /bin/bash
-    create_home: true
-    state: present
 
-- name: Create Semaphore data directories with correct ownership
+- name: Ensure podman network exists
-  ansible.builtin.file:
+  ansible.builtin.import_tasks: network.yml
-    path: "{{ item }}"
+  tags: [semaphore, network]
-    state: directory
-    owner: "{{ semaphore_user }}"
-    group: "{{ semaphore_group }}"
-    mode: "0777"
-    recurse: true
-  become: true
-  loop:
-    - "{{ semaphore_data_dir }}"
-    - "{{ semaphore_config_dir }}"
 
-# ============================================================
+- name: Ensure podman volumes exist
-# LEGACY DEPLOYMENT (podman_container + generate systemd)
+  ansible.builtin.import_tasks: volumes.yml
-# ============================================================
+  tags: [semaphore, volumes]
-- name: Legacy deployment with PostgreSQL
-  when: semaphore_use_postgres | default(false)
-  block:
-    - name: Aggressive storage cleanup (when semaphore_force_storage_cleanup=true)
-      when: semaphore_force_storage_cleanup | default(false)
-      block:
-        - name: Force remove all containers
-          ansible.builtin.command:
-            cmd: "podman rm -af"
-          become: true
-          failed_when: false
 
-        - name: Prune unused images, volumes and networks
+- name: Deploy PostgreSQL container
-          ansible.builtin.command:
+  ansible.builtin.import_tasks: postgres.yml
-            cmd: "podman system prune -f --volumes"
+  tags: [semaphore, postgres]
-          become: true
-          failed_when: false
 
-    - name: Remove existing containers if present (force + storage cleanup)
+- name: Deploy Semaphore container
-      ansible.builtin.command:
+  ansible.builtin.import_tasks: semaphore.yml
-        cmd: "podman rm -f --storage {{ item }}"
+  tags: [semaphore]
-      become: true
-      failed_when: false
-      loop:
-        - "{{ semaphore_postgres_container_name }}"
-        - "{{ semaphore_container_name }}"
 
-    - name: Ensure dedicated Podman network exists
+- name: Verify Semaphore is reachable
-      ansible.builtin.command:
+  ansible.builtin.import_tasks: verify.yml
-        cmd: "podman network create semaphore-net"
+  tags: [semaphore, verify]
-      become: true
-      register: net_create
-      failed_when: false
-      changed_when: net_create.rc == 0
-
-    - name: Ensure required Podman volumes exist (reuse if present)
-      ansible.builtin.command:
-        cmd: "podman volume exists {{ item }} || podman volume create {{ item }}"
-      become: true
-      failed_when: false
-      changed_when: false
-      loop:
-        - "{{ semaphore_postgres_volume }}"
-        - "{{ semaphore_data_volume }}"
-
-    - name: Deploy PostgreSQL container (legacy)
-      containers.podman.podman_container:
-        name: "{{ semaphore_postgres_container_name }}"
-        image: "{{ semaphore_postgres_image }}"
-        network: semaphore-net
-        state: started
-        recreate: true
-        volumes:
-          - "{{ semaphore_postgres_volume }}:/var/lib/postgresql/data:Z"
-        env:
-          POSTGRES_USER: "{{ semaphore_postgres_user }}"
-          POSTGRES_PASSWORD: "{{ vault_semaphore_database_password }}"
-          POSTGRES_DB: "{{ semaphore_postgres_db }}"
-      become: true
-
-    - name: Deploy Semaphore container (legacy)
-      containers.podman.podman_container:
-        name: "{{ semaphore_container_name }}"
-        image: "{{ semaphore_image }}"
-        network: semaphore-net
-        state: started
-        recreate: true
-        ports:
-          - "{{ semaphore_port }}:3000"
-        volumes:
-          - "{{ semaphore_data_volume }}:/var/lib/semaphore:Z"
-          - "{{ semaphore_config_dir }}:/etc/semaphore:Z"
-        env:
-          SEMAPHORE_DB_DIALECT: "postgres"
-          SEMAPHORE_DB_PATH: ""
-          SEMAPHORE_DB_USER: "{{ semaphore_postgres_user }}"
-          SEMAPHORE_DB_PASS: "{{ vault_semaphore_database_password }}"
-          SEMAPHORE_DB_HOST: "{{ semaphore_postgres_container_name }}"
-          SEMAPHORE_DB_PORT: "5432"
-          SEMAPHORE_DB_NAME: "{{ semaphore_postgres_db }}"
-          SEMAPHORE_DB_SSLMODE: "disable"
-          SEMAPHORE_ADMIN: "admin"
-          SEMAPHORE_ADMIN_PASSWORD: "{{ vault_semaphore_admin_password }}"
-          SEMAPHORE_ACCESS_KEY_ENCRYPTION: "{{ vault_semaphore_access_key_encryption }}"
-      become: true
-
-    - name: Generate systemd service for PostgreSQL
-      ansible.builtin.command:
-        cmd: "podman generate systemd --name --new --files {{ semaphore_postgres_container_name }}"
-        chdir: /etc/systemd/system
-      become: true
-      changed_when: true
-
-    - name: Generate systemd service for Semaphore
-      ansible.builtin.command:
-        cmd: "podman generate systemd --name --new --files {{ semaphore_container_name }}"
-        chdir: /etc/systemd/system
-      become: true
-      changed_when: true
-
-    - name: Reload systemd and enable services
-      ansible.builtin.systemd:
-        daemon_reload: true
-        name: "{{ item }}"
-        state: started
-        enabled: true
-      become: true
-      loop:
-        - "container-{{ semaphore_postgres_container_name }}.service"
-        - "container-{{ semaphore_container_name }}.service"
-
-# ============================================================
-# LEGACY DEPLOYMENT - BoltDB fallback
-# ============================================================
-- name: Legacy deployment with BoltDB
-  when: not (semaphore_use_postgres | default(false))
-  block:
-    - name: Create Semaphore config.json for BoltDB
-      ansible.builtin.copy:
-        dest: "{{ semaphore_config_dir }}/config.json"
-        owner: "{{ semaphore_user }}"
-        group: "{{ semaphore_group }}"
-        mode: "0644"
-        content: |
-          {
-            "db": {
-              "Dialect": "bolt",
-              "BoltDB": {
-                "Path": "{{ semaphore_data_dir }}"
-              }
-            },
-            "server": {
-              "Addr": ":{{ semaphore_port }}"
-            }
-          }
-      become: true
-
-    - name: Create and start Semaphore container
-      containers.podman.podman_container:
-        name: "{{ semaphore_container_name }}"
-        image: "{{ semaphore_image }}"
-        network: semaphore-net
-        state: started
-        recreate: true
-        ports:
-          - "{{ semaphore_port }}:3000"
-        volumes:
-          - "{{ semaphore_data_dir }}:/var/lib/semaphore:Z"
-          - "{{ semaphore_config_dir }}:/etc/semaphore:Z"
-        env:
-          SEMAPHORE_DB_DIALECT: "{{ semaphore_db_type }}"
-          SEMAPHORE_DB_PATH: "{{ semaphore_db_path }}"
-          SEMAPHORE_ADMIN: admin
-          SEMAPHORE_ADMIN_PASSWORD: "{{ vault_semaphore_admin_password }}"
-          SEMAPHORE_ACCESS_KEY_ENCRYPTION: "{{ vault_semaphore_access_key_encryption }}"
-
-    - name: Generate systemd service for the container
-      ansible.builtin.command:
-        cmd: podman generate systemd --name --new --files {{ semaphore_container_name }}
-        chdir: /etc/systemd/system
-      become: true
-      changed_when: true
-
-    - name: Reload systemd and enable service
-      ansible.builtin.systemd:
-        daemon_reload: true
-        name: "container-{{ semaphore_container_name }}.service"
-        state: started
-        enabled: true
-  become: true

ansible/roles/semaphore/tasks/network.yml

@@ -0,0 +1,21 @@
+---
+# ============================================================================
+# Podman network
+# ----------------------------------------------------------------------------
+# Quadlet supports .network files, but for a single user-defined network
+# we keep it simple with an idempotent podman CLI invocation. The network
+# survives container removal; only re-created if missing.
+# ============================================================================
+
+- name: Check if semaphore network exists
+  ansible.builtin.command:
+    cmd: "podman network exists {{ semaphore_network_name }}"
+  register: semaphore_network_check
+  failed_when: false
+  changed_when: false
+
+- name: Create semaphore podman network
+  ansible.builtin.command:
+    cmd: "podman network create {{ semaphore_network_name }}"
+  when: semaphore_network_check.rc != 0
+  changed_when: true

ansible/roles/semaphore/tasks/podman.yml

@@ -0,0 +1,41 @@
+---
+# ============================================================================
+# Podman + Quadlet prerequisites
+# ============================================================================
+# Ubuntu 24.04 ships podman 4.9+, which has full Quadlet support. No PPAs
+# needed.
+# ============================================================================
+
+- name: Install podman
+  ansible.builtin.apt:
+    name:
+      - podman
+      - uidmap
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Ensure quadlet directory exists
+  ansible.builtin.file:
+    path: "{{ semaphore_quadlet_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Confirm podman is recent enough for Quadlet (>= 4.4)
+  block:
+    - name: Probe podman version
+      ansible.builtin.command:
+        cmd: podman --version
+      register: podman_version_raw
+      changed_when: false
+
+    - name: Assert podman >= 4.4
+      ansible.builtin.assert:
+        that:
+          - (podman_version_raw.stdout | regex_search('([0-9]+\\.[0-9]+)') | float) >= 4.4
+        fail_msg: >-
+          podman {{ podman_version_raw.stdout }} is too old for Quadlet.
+          Quadlet was introduced in 4.4. Upgrade podman or use the
+          generate-systemd fallback path.

ansible/roles/semaphore/tasks/postgres.yml

@@ -0,0 +1,35 @@
+---
+# ============================================================================
+# PostgreSQL container (Quadlet)
+# ============================================================================
+
+- name: Deploy PostgreSQL Quadlet
+  ansible.builtin.template:
+    src: semaphore-postgres.container.j2
+    dest: "{{ semaphore_quadlet_dir }}/{{ semaphore_postgres_container_name }}.container"
+    owner: root
+    group: root
+    mode: "0644"
+  register: postgres_quadlet
+
+- name: Reload systemd to pick up Quadlet changes
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when: postgres_quadlet.changed
+
+- name: Ensure PostgreSQL container is started and enabled
+  ansible.builtin.systemd:
+    name: "{{ semaphore_postgres_container_name }}.service"
+    state: started
+    enabled: true
+
+- name: Wait for PostgreSQL to accept connections
+  ansible.builtin.command:
+    cmd: >-
+      podman exec {{ semaphore_postgres_container_name }}
+      pg_isready -U {{ semaphore_db_user }} -d {{ semaphore_db_name }}
+  register: pg_ready
+  until: pg_ready.rc == 0
+  retries: 30
+  delay: 2
+  changed_when: false

ansible/roles/semaphore/tasks/semaphore.yml

@@ -0,0 +1,24 @@
+---
+# ============================================================================
+# Semaphore container (Quadlet)
+# ============================================================================
+
+- name: Deploy Semaphore Quadlet
+  ansible.builtin.template:
+    src: semaphore.container.j2
+    dest: "{{ semaphore_quadlet_dir }}/{{ semaphore_container_name }}.container"
+    owner: root
+    group: root
+    mode: "0644"
+  register: semaphore_quadlet
+
+- name: Reload systemd to pick up Quadlet changes
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when: semaphore_quadlet.changed
+
+- name: Ensure Semaphore container is started and enabled
+  ansible.builtin.systemd:
+    name: "{{ semaphore_container_name }}.service"
+    state: started
+    enabled: true

ansible/roles/semaphore/tasks/verify.yml

@@ -0,0 +1,30 @@
+---
+# ============================================================================
+# Post-deploy verification
+# ----------------------------------------------------------------------------
+# Polls the local Semaphore port until it responds. Fails loudly if the
+# service doesn't come up — the operator should not get an "all green"
+# playbook result while Semaphore is silently broken.
+# ============================================================================
+
+- name: Wait for Semaphore HTTP endpoint
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ semaphore_listen_port }}/api/ping"
+    status_code: [200, 201, 204]
+    return_content: false
+  register: semaphore_ping
+  until: semaphore_ping.status in [200, 201, 204]
+  retries: "{{ semaphore_health_check_retries }}"
+  delay: "{{ semaphore_health_check_delay }}"
+
+- name: Report Semaphore status
+  ansible.builtin.debug:
+    msg: >-
+      Semaphore is reachable on http://127.0.0.1:{{ semaphore_listen_port }}.
+      Public URL: {{ semaphore_web_url }}.
+      If this is the first deployment, create the admin user with:
+        sudo podman exec -it {{ semaphore_container_name }} \
+          semaphore user add --admin \
+            --login admin --name Administrator \
+            --email admin@local.mk-labs.cloud \
+            --password '<see vault_semaphore_admin_password>'

ansible/roles/semaphore/tasks/volumes.yml

@@ -0,0 +1,16 @@
+---
+# ============================================================================
+# Named podman volumes
+# ============================================================================
+
+- name: Ensure named volumes exist
+  ansible.builtin.command:
+    cmd: "podman volume create {{ item }}"
+  register: volume_create
+  changed_when: "'already exists' not in (volume_create.stderr | default(''))"
+  failed_when:
+    - volume_create.rc != 0
+    - "'already exists' not in (volume_create.stderr | default(''))"
+  loop:
+    - "{{ semaphore_postgres_volume }}"
+    - "{{ semaphore_data_volume }}"

ansible/roles/semaphore/templates/env.j2

@@ -1,4 +0,0 @@
-SEMAPHORE_DB_DIALECT={{ semaphore_db_type }}
-SEMAPHORE_DB_PATH={{ semaphore_db_path }}
-SEMAPHORE_ADMIN_PASSWORD={{ vault_semaphore_admin_password }}
-SEMAPHORE_ACCESS_KEY_ENCRYPTION={{ vault_semaphore_access_key_encryption }}

ansible/roles/semaphore/templates/semaphore-postgres.container.j2

@@ -1,19 +1,37 @@
+# {{ ansible_managed }}
+# mk-labs PostgreSQL backing store for SemaphoreUI
+# Managed by Ansible - semaphore role
+
 [Unit]
-Description=Semaphore PostgreSQL
+Description=PostgreSQL for SemaphoreUI
 After=network-online.target
 Wants=network-online.target
 
 [Container]
 Image={{ semaphore_postgres_image }}
 ContainerName={{ semaphore_postgres_container_name }}
+Network={{ semaphore_network_name }}
+
+# Persistent data on a named volume. :Z relabels for SELinux; :U maps
+# ownership for rootless. Rootful + ext4 doesn't strictly require either,
+# but keeping them is harmless and portable.
 Volume={{ semaphore_postgres_volume }}:/var/lib/postgresql/data:Z
-Environment=POSTGRES_USER={{ semaphore_postgres_user }}
+
-Environment=POSTGRES_PASSWORD={{ vault_semaphore_database_password }}
+Environment=POSTGRES_USER={{ semaphore_db_user }}
-Environment=POSTGRES_DB={{ semaphore_postgres_db }}
+Environment=POSTGRES_DB={{ semaphore_db_name }}
-Restart=always
+Environment=POSTGRES_PASSWORD={{ semaphore_db_password }}
+# Initialise the locale and encoding deterministically
+Environment=POSTGRES_INITDB_ARGS=--encoding=UTF8 --locale=C
+
+# Healthcheck — systemd doesn't act on this, but it's useful diagnostically
+HealthCmd=pg_isready -U {{ semaphore_db_user }} -d {{ semaphore_db_name }}
+HealthInterval=10s
+HealthTimeout=5s
+HealthRetries=3
 
 [Service]
 Restart=always
+TimeoutStartSec=120
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target default.target

ansible/roles/semaphore/templates/semaphore.container.j2

@@ -1,28 +1,42 @@
+# {{ ansible_managed }}
+# mk-labs SemaphoreUI web service
+# Managed by Ansible - semaphore role
+
 [Unit]
-Description=Semaphore UI
+Description=SemaphoreUI
-After=network-online.target
+After=network-online.target {{ semaphore_postgres_container_name }}.service
 Wants=network-online.target
+Requires={{ semaphore_postgres_container_name }}.service
 
 [Container]
 Image={{ semaphore_image }}
 ContainerName={{ semaphore_container_name }}
-PublishPort={{ semaphore_port }}:3000
+Network={{ semaphore_network_name }}
+PublishPort={{ semaphore_listen_address }}:{{ semaphore_listen_port }}:3000
+
+# Persistent runtime state (project repos, tmp playbooks, etc.)
 Volume={{ semaphore_data_volume }}:/var/lib/semaphore:Z
-Volume={{ semaphore_config_dir }}:/etc/semaphore:Z
+
+# Database — points at the postgres container on the user-defined network
 Environment=SEMAPHORE_DB_DIALECT=postgres
-Environment=SEMAPHORE_DB_PATH=
-Environment=SEMAPHORE_DB_USER={{ semaphore_postgres_user }}
-Environment=SEMAPHORE_DB_PASS={{ vault_semaphore_database_password }}
 Environment=SEMAPHORE_DB_HOST={{ semaphore_postgres_container_name }}
 Environment=SEMAPHORE_DB_PORT=5432
-Environment=SEMAPHORE_DB_NAME={{ semaphore_postgres_db }}
+Environment=SEMAPHORE_DB={{ semaphore_db_name }}
-Environment=SEMAPHORE_ADMIN=admin
+Environment=SEMAPHORE_DB_USER={{ semaphore_db_user }}
-Environment=SEMAPHORE_ADMIN_PASSWORD={{ vault_semaphore_admin_password }}
+Environment=SEMAPHORE_DB_PASS={{ semaphore_db_password }}
-Environment=SEMAPHORE_ACCESS_KEY_ENCRYPTION={{ vault_semaphore_access_key_encryption }}
+
-Restart=always
+# Encryption for stored access keys (must be a 32-byte base64 string).
+Environment=SEMAPHORE_ACCESS_KEY_ENCRYPTION={{ semaphore_access_key_encryption }}
+
+# Public-facing URL used for OIDC redirects, webhooks, etc.
+Environment=SEMAPHORE_WEB_ROOT={{ semaphore_web_url }}
+
+# Timezone matches host baseline
+Environment=TZ=America/Chicago
 
 [Service]
 Restart=always
+TimeoutStartSec=180
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target default.target