feat(semaphore): rewrite role with rootful Podman Quadlet + PostgreSQL

Complete rewrite of the semaphore role. Supersedes three prior
iterations whose admin-user-creation logic was unreliable across
Semaphore CLI versions.

Architecture:
  - Rootful Podman Quadlet under /etc/containers/systemd/
  - Separate PostgreSQL 16-alpine container on a user-defined
    podman network (semaphore-net)
  - Named volumes for both data stores (semaphore_data,
    semaphore_postgres_data) so container recreation is
    non-destructive
  - Pinned image tags: semaphoreui/semaphore:v2.18.5-ansible2.16.5
    and postgres:16-alpine
  - Post-deploy HTTP health check fails the playbook if Semaphore
    doesn't respond on /api/ping within ~60s

Admin user creation remains intentionally manual after first deploy;
the role README documents the exact podman exec command.

Removes the duplicate deploy_semaphore.yml and the now-unneeded
cleanup_semaphore.yml; day1_deploy_semaphore.yml is the canonical
entry point.

ansible/roles/semaphore/tasks/main.yml

@@ -1,201 +1,36 @@
 ---
-- name: Ensure cast group exists
-  ansible.builtin.group:
-    name: "{{ semaphore_group }}"
-    state: present
-    system: true
+# ============================================================================
+# semaphore / main entrypoint
+# ----------------------------------------------------------------------------
+# Order matters:
+#   1. Podman + Quadlet support installed and ready
+#   2. Network created (containers reference it by name)
+#   3. Volumes created (postgres + semaphore data)
+#   4. Postgres started (Semaphore depends on it being ready)
+#   5. Semaphore started (web service)
+#   6. Verify reachable on listen port
+# ============================================================================
 
-- name: Ensure cast user exists
-  ansible.builtin.user:
-    name: "{{ semaphore_user }}"
-    group: "{{ semaphore_group }}"
-    shell: /bin/bash
-    create_home: true
-    state: present
+- name: Install Podman and dependencies
+  ansible.builtin.import_tasks: podman.yml
+  tags: [semaphore, podman]
 
-- name: Create Semaphore data directories with correct ownership
-  ansible.builtin.file:
-    path: "{{ item }}"
-    state: directory
-    owner: "{{ semaphore_user }}"
-    group: "{{ semaphore_group }}"
-    mode: "0777"
-    recurse: true
-  become: true
-  loop:
-    - "{{ semaphore_data_dir }}"
-    - "{{ semaphore_config_dir }}"
+- name: Ensure podman network exists
+  ansible.builtin.import_tasks: network.yml
+  tags: [semaphore, network]
 
-# ============================================================
-# LEGACY DEPLOYMENT (podman_container + generate systemd)
-# ============================================================
-- name: Legacy deployment with PostgreSQL
-  when: semaphore_use_postgres | default(false)
-  block:
-    - name: Aggressive storage cleanup (when semaphore_force_storage_cleanup=true)
-      when: semaphore_force_storage_cleanup | default(false)
-      block:
-        - name: Force remove all containers
-          ansible.builtin.command:
-            cmd: "podman rm -af"
-          become: true
-          failed_when: false
+- name: Ensure podman volumes exist
+  ansible.builtin.import_tasks: volumes.yml
+  tags: [semaphore, volumes]
 
-        - name: Prune unused images, volumes and networks
-          ansible.builtin.command:
-            cmd: "podman system prune -f --volumes"
-          become: true
-          failed_when: false
+- name: Deploy PostgreSQL container
+  ansible.builtin.import_tasks: postgres.yml
+  tags: [semaphore, postgres]
 
-    - name: Remove existing containers if present (force + storage cleanup)
-      ansible.builtin.command:
-        cmd: "podman rm -f --storage {{ item }}"
-      become: true
-      failed_when: false
-      loop:
-        - "{{ semaphore_postgres_container_name }}"
-        - "{{ semaphore_container_name }}"
+- name: Deploy Semaphore container
+  ansible.builtin.import_tasks: semaphore.yml
+  tags: [semaphore]
 
-    - name: Ensure dedicated Podman network exists
-      ansible.builtin.command:
-        cmd: "podman network create semaphore-net"
-      become: true
-      register: net_create
-      failed_when: false
-      changed_when: net_create.rc == 0
-
-    - name: Ensure required Podman volumes exist (reuse if present)
-      ansible.builtin.command:
-        cmd: "podman volume exists {{ item }} || podman volume create {{ item }}"
-      become: true
-      failed_when: false
-      changed_when: false
-      loop:
-        - "{{ semaphore_postgres_volume }}"
-        - "{{ semaphore_data_volume }}"
-
-    - name: Deploy PostgreSQL container (legacy)
-      containers.podman.podman_container:
-        name: "{{ semaphore_postgres_container_name }}"
-        image: "{{ semaphore_postgres_image }}"
-        network: semaphore-net
-        state: started
-        recreate: true
-        volumes:
-          - "{{ semaphore_postgres_volume }}:/var/lib/postgresql/data:Z"
-        env:
-          POSTGRES_USER: "{{ semaphore_postgres_user }}"
-          POSTGRES_PASSWORD: "{{ vault_semaphore_database_password }}"
-          POSTGRES_DB: "{{ semaphore_postgres_db }}"
-      become: true
-
-    - name: Deploy Semaphore container (legacy)
-      containers.podman.podman_container:
-        name: "{{ semaphore_container_name }}"
-        image: "{{ semaphore_image }}"
-        network: semaphore-net
-        state: started
-        recreate: true
-        ports:
-          - "{{ semaphore_port }}:3000"
-        volumes:
-          - "{{ semaphore_data_volume }}:/var/lib/semaphore:Z"
-          - "{{ semaphore_config_dir }}:/etc/semaphore:Z"
-        env:
-          SEMAPHORE_DB_DIALECT: "postgres"
-          SEMAPHORE_DB_PATH: ""
-          SEMAPHORE_DB_USER: "{{ semaphore_postgres_user }}"
-          SEMAPHORE_DB_PASS: "{{ vault_semaphore_database_password }}"
-          SEMAPHORE_DB_HOST: "{{ semaphore_postgres_container_name }}"
-          SEMAPHORE_DB_PORT: "5432"
-          SEMAPHORE_DB_NAME: "{{ semaphore_postgres_db }}"
-          SEMAPHORE_DB_SSLMODE: "disable"
-          SEMAPHORE_ADMIN: "admin"
-          SEMAPHORE_ADMIN_PASSWORD: "{{ vault_semaphore_admin_password }}"
-          SEMAPHORE_ACCESS_KEY_ENCRYPTION: "{{ vault_semaphore_access_key_encryption }}"
-      become: true
-
-    - name: Generate systemd service for PostgreSQL
-      ansible.builtin.command:
-        cmd: "podman generate systemd --name --new --files {{ semaphore_postgres_container_name }}"
-        chdir: /etc/systemd/system
-      become: true
-      changed_when: true
-
-    - name: Generate systemd service for Semaphore
-      ansible.builtin.command:
-        cmd: "podman generate systemd --name --new --files {{ semaphore_container_name }}"
-        chdir: /etc/systemd/system
-      become: true
-      changed_when: true
-
-    - name: Reload systemd and enable services
-      ansible.builtin.systemd:
-        daemon_reload: true
-        name: "{{ item }}"
-        state: started
-        enabled: true
-      become: true
-      loop:
-        - "container-{{ semaphore_postgres_container_name }}.service"
-        - "container-{{ semaphore_container_name }}.service"
-
-# ============================================================
-# LEGACY DEPLOYMENT - BoltDB fallback
-# ============================================================
-- name: Legacy deployment with BoltDB
-  when: not (semaphore_use_postgres | default(false))
-  block:
-    - name: Create Semaphore config.json for BoltDB
-      ansible.builtin.copy:
-        dest: "{{ semaphore_config_dir }}/config.json"
-        owner: "{{ semaphore_user }}"
-        group: "{{ semaphore_group }}"
-        mode: "0644"
-        content: |
-          {
-            "db": {
-              "Dialect": "bolt",
-              "BoltDB": {
-                "Path": "{{ semaphore_data_dir }}"
-              }
-            },
-            "server": {
-              "Addr": ":{{ semaphore_port }}"
-            }
-          }
-      become: true
-
-    - name: Create and start Semaphore container
-      containers.podman.podman_container:
-        name: "{{ semaphore_container_name }}"
-        image: "{{ semaphore_image }}"
-        network: semaphore-net
-        state: started
-        recreate: true
-        ports:
-          - "{{ semaphore_port }}:3000"
-        volumes:
-          - "{{ semaphore_data_dir }}:/var/lib/semaphore:Z"
-          - "{{ semaphore_config_dir }}:/etc/semaphore:Z"
-        env:
-          SEMAPHORE_DB_DIALECT: "{{ semaphore_db_type }}"
-          SEMAPHORE_DB_PATH: "{{ semaphore_db_path }}"
-          SEMAPHORE_ADMIN: admin
-          SEMAPHORE_ADMIN_PASSWORD: "{{ vault_semaphore_admin_password }}"
-          SEMAPHORE_ACCESS_KEY_ENCRYPTION: "{{ vault_semaphore_access_key_encryption }}"
-
-    - name: Generate systemd service for the container
-      ansible.builtin.command:
-        cmd: podman generate systemd --name --new --files {{ semaphore_container_name }}
-        chdir: /etc/systemd/system
-      become: true
-      changed_when: true
-
-    - name: Reload systemd and enable service
-      ansible.builtin.systemd:
-        daemon_reload: true
-        name: "container-{{ semaphore_container_name }}.service"
-        state: started
-        enabled: true
-  become: true
+- name: Verify Semaphore is reachable
+  ansible.builtin.import_tasks: verify.yml
+  tags: [semaphore, verify]

ansible/roles/semaphore/tasks/network.yml

@@ -0,0 +1,21 @@
+---
+# ============================================================================
+# Podman network
+# ----------------------------------------------------------------------------
+# Quadlet supports .network files, but for a single user-defined network
+# we keep it simple with an idempotent podman CLI invocation. The network
+# survives container removal; only re-created if missing.
+# ============================================================================
+
+- name: Check if semaphore network exists
+  ansible.builtin.command:
+    cmd: "podman network exists {{ semaphore_network_name }}"
+  register: semaphore_network_check
+  failed_when: false
+  changed_when: false
+
+- name: Create semaphore podman network
+  ansible.builtin.command:
+    cmd: "podman network create {{ semaphore_network_name }}"
+  when: semaphore_network_check.rc != 0
+  changed_when: true

ansible/roles/semaphore/tasks/podman.yml

@@ -0,0 +1,41 @@
+---
+# ============================================================================
+# Podman + Quadlet prerequisites
+# ============================================================================
+# Ubuntu 24.04 ships podman 4.9+, which has full Quadlet support. No PPAs
+# needed.
+# ============================================================================
+
+- name: Install podman
+  ansible.builtin.apt:
+    name:
+      - podman
+      - uidmap
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+
+- name: Ensure quadlet directory exists
+  ansible.builtin.file:
+    path: "{{ semaphore_quadlet_dir }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Confirm podman is recent enough for Quadlet (>= 4.4)
+  block:
+    - name: Probe podman version
+      ansible.builtin.command:
+        cmd: podman --version
+      register: podman_version_raw
+      changed_when: false
+
+    - name: Assert podman >= 4.4
+      ansible.builtin.assert:
+        that:
+          - (podman_version_raw.stdout | regex_search('([0-9]+\\.[0-9]+)') | float) >= 4.4
+        fail_msg: >-
+          podman {{ podman_version_raw.stdout }} is too old for Quadlet.
+          Quadlet was introduced in 4.4. Upgrade podman or use the
+          generate-systemd fallback path.

ansible/roles/semaphore/tasks/postgres.yml

@@ -0,0 +1,35 @@
+---
+# ============================================================================
+# PostgreSQL container (Quadlet)
+# ============================================================================
+
+- name: Deploy PostgreSQL Quadlet
+  ansible.builtin.template:
+    src: semaphore-postgres.container.j2
+    dest: "{{ semaphore_quadlet_dir }}/{{ semaphore_postgres_container_name }}.container"
+    owner: root
+    group: root
+    mode: "0644"
+  register: postgres_quadlet
+
+- name: Reload systemd to pick up Quadlet changes
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when: postgres_quadlet.changed
+
+- name: Ensure PostgreSQL container is started and enabled
+  ansible.builtin.systemd:
+    name: "{{ semaphore_postgres_container_name }}.service"
+    state: started
+    enabled: true
+
+- name: Wait for PostgreSQL to accept connections
+  ansible.builtin.command:
+    cmd: >-
+      podman exec {{ semaphore_postgres_container_name }}
+      pg_isready -U {{ semaphore_db_user }} -d {{ semaphore_db_name }}
+  register: pg_ready
+  until: pg_ready.rc == 0
+  retries: 30
+  delay: 2
+  changed_when: false

ansible/roles/semaphore/tasks/semaphore.yml

@@ -0,0 +1,24 @@
+---
+# ============================================================================
+# Semaphore container (Quadlet)
+# ============================================================================
+
+- name: Deploy Semaphore Quadlet
+  ansible.builtin.template:
+    src: semaphore.container.j2
+    dest: "{{ semaphore_quadlet_dir }}/{{ semaphore_container_name }}.container"
+    owner: root
+    group: root
+    mode: "0644"
+  register: semaphore_quadlet
+
+- name: Reload systemd to pick up Quadlet changes
+  ansible.builtin.systemd:
+    daemon_reload: true
+  when: semaphore_quadlet.changed
+
+- name: Ensure Semaphore container is started and enabled
+  ansible.builtin.systemd:
+    name: "{{ semaphore_container_name }}.service"
+    state: started
+    enabled: true

ansible/roles/semaphore/tasks/verify.yml

@@ -0,0 +1,30 @@
+---
+# ============================================================================
+# Post-deploy verification
+# ----------------------------------------------------------------------------
+# Polls the local Semaphore port until it responds. Fails loudly if the
+# service doesn't come up — the operator should not get an "all green"
+# playbook result while Semaphore is silently broken.
+# ============================================================================
+
+- name: Wait for Semaphore HTTP endpoint
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ semaphore_listen_port }}/api/ping"
+    status_code: [200, 201, 204]
+    return_content: false
+  register: semaphore_ping
+  until: semaphore_ping.status in [200, 201, 204]
+  retries: "{{ semaphore_health_check_retries }}"
+  delay: "{{ semaphore_health_check_delay }}"
+
+- name: Report Semaphore status
+  ansible.builtin.debug:
+    msg: >-
+      Semaphore is reachable on http://127.0.0.1:{{ semaphore_listen_port }}.
+      Public URL: {{ semaphore_web_url }}.
+      If this is the first deployment, create the admin user with:
+        sudo podman exec -it {{ semaphore_container_name }} \
+          semaphore user add --admin \
+            --login admin --name Administrator \
+            --email admin@local.mk-labs.cloud \
+            --password '<see vault_semaphore_admin_password>'

ansible/roles/semaphore/tasks/volumes.yml

@@ -0,0 +1,16 @@
+---
+# ============================================================================
+# Named podman volumes
+# ============================================================================
+
+- name: Ensure named volumes exist
+  ansible.builtin.command:
+    cmd: "podman volume create {{ item }}"
+  register: volume_create
+  changed_when: "'already exists' not in (volume_create.stderr | default(''))"
+  failed_when:
+    - volume_create.rc != 0
+    - "'already exists' not in (volume_create.stderr | default(''))"
+  loop:
+    - "{{ semaphore_postgres_volume }}"
+    - "{{ semaphore_data_volume }}"