Add shared nginx-proxy role; deploy Hermes in Docker; rename Paperclip

nginx-proxy (new role): - Shared nginx + certbot under /opt/nginx, replacing the copies that lived inside the Paperclip compose project - Apps publish via tasks_from: add_vhost (cert marker -> issue cert -> write vhost -> nginx -t -> reload); a vhost that fails nginx -t is rolled back - Certs issued one per domain via Cloudflare DNS challenge; domains registered as marker files in cert-domains/; optional LE staging - Stock nginx.conf plus conf.d (directory mounts, not single-file mounts); catch-all server drops unknown Host/SNI - Containers renamed to nginx and certbot hermes_docker (new role): - Official Hermes image, host networking, gateway and dashboard bound to loopback and published through nginx-proxy - Dashboard login via basic-auth provider, secrets from the vault - Main model set from hermes_model (fantasmic endpoint, key from vault); config.yaml overrides are merged from a 0600 staged file that is removed afterwards, so the key never appears in a command line - Data dir owner-only (HERMES_HOME_MODE=0700) paperclip-docker: - Remove nginx, certbot and docker-compose download tasks; deploy with community.docker.docker_compose_v2 (idempotent, remove_orphans) - Stop passing the Cloudflare token and certbot email to the Paperclip container via env_file - Bind Postgres to 127.0.0.1 (host networking exposed it on all interfaces); stop chowning pgdata, which the postgres image owns - Handler recreates the paperclip service so .env changes apply - Domain m-v.wdw-labs.cloud -> cranium-command.wdw-labs.cloud Playbooks: - Add day1_deploy_nginx_proxy.yml and day1_deploy_hermes_docker.yml - deploy_paperclip_stack.yml: run nginx-proxy before paperclip-docker, CLI-based container checks (no Docker SDK dependency) - Remove deploy-hermes.yml Host vars: add nginx_proxy_certbot_email and nginx_proxy_certbot_staging (staging stays on until the switch to real certs).
2026-09-30 23:12:40 -05:00
1 parent 5b48536c32
commit 881e2e24a8
@@ -0,0 +1,28 @@
+# nginx-proxy
+
+Shared nginx + certbot (Cloudflare DNS challenge) under /opt/nginx. Apps publish
+themselves with `tasks_from: add_vhost`; see the header of tasks/add_vhost.yml.
+
+Layout on the host:
+
+    /opt/nginx/
+      docker-compose.yml  .env (0600)
+      conf.d/        00-global.conf + one <app>.conf per vhost   -> /etc/nginx/conf.d
+      certs/         certbot state (live/, archive/, renewal/)   -> /etc/letsencrypt, /etc/nginx/certs
+      cert-domains/  one empty file per domain (filename = domain)
+      scripts/       certbot-init.sh
+
+Required variables: `nginx_proxy_certbot_email`, and `vault_cloudflare_api_token`
+(or `nginx_proxy_cloudflare_api_token`).
+
+Prerequisite (manual): a DNS record for each app domain pointing at this host.
+
+Staging: set `nginx_proxy_certbot_staging: true` to test the DNS challenge without
+using Let's Encrypt rate limits. Staging certs are skipped-as-existing afterwards, so
+to move a domain to a real cert: `docker exec certbot certbot delete --cert-name <domain>`
+and re-run its play.
+
+Retire a domain: delete its cert-domains/ marker and conf.d/<app>.conf, reload nginx,
+then `docker exec certbot certbot delete --cert-name <domain>`.
+
+Pin nginx older than 1.25.1? Set `nginx_proxy_http2: false` (the template uses `http2 on;`).
@@ -0,0 +1,27 @@
+---
+nginx_proxy_base_path: /opt/nginx
+nginx_proxy_certs_dir: "{{ nginx_proxy_base_path }}/certs"
+nginx_proxy_conf_dir: "{{ nginx_proxy_base_path }}/conf.d"
+nginx_proxy_domains_dir: "{{ nginx_proxy_base_path }}/cert-domains"
+nginx_proxy_scripts_dir: "{{ nginx_proxy_base_path }}/scripts"
+
+nginx_proxy_nginx_container: nginx
+nginx_proxy_certbot_container: certbot
+nginx_proxy_nginx_image: nginx:latest
+nginx_proxy_certbot_image: certbot/dns-cloudflare:latest
+
+# Required: set these in group_vars/host_vars (token comes from the vault)
+nginx_proxy_certbot_email: ""
+nginx_proxy_cloudflare_api_token: "{{ vault_cloudflare_api_token }}"
+
+# true = Let's Encrypt staging (untrusted certs, no rate-limit risk). Use while iterating.
+nginx_proxy_certbot_staging: false
+
+# nginx reloads this often so renewed certs get picked up
+nginx_proxy_reload_interval: 6h
+nginx_proxy_client_max_body_size: 100M
+
+# "http2 on;" needs nginx >= 1.25.1 (fine for nginx:latest). Set false if you pin older.
+nginx_proxy_http2: true
+
+nginx_proxy_pull_policy: missing
@@ -0,0 +1,92 @@
+#!/bin/sh
+# Issues certificates (one per domain) via the Cloudflare DNS challenge.
+#
+#   certbot-init.sh                 -> every domain with a marker file in $DOMAINS_DIR
+#   certbot-init.sh a.example.com   -> only the domains given as arguments
+#
+# Idempotent: domains that already have a certificate are skipped.
+# Renewal is NOT handled here; `certbot renew` covers every existing lineage.
+set -eu
+
+DOMAINS_DIR="${DOMAINS_DIR:-/etc/cert-domains.d}"
+EMAIL="${CERTBOT_EMAIL:-}"
+TOKEN="${CLOUDFLARE_API_TOKEN:-}"
+
+STAGING_ARG=""
+if [ "${CERTBOT_STAGING:-0}" = "1" ]; then
+    STAGING_ARG="--server https://acme-staging-v02.api.letsencrypt.org/directory"
+    echo "NOTE: using the Let's Encrypt STAGING server (certificates will not be trusted)"
+fi
+
+if [ "$#" -gt 0 ]; then
+    DOMAINS="$*"
+elif [ -d "$DOMAINS_DIR" ]; then
+    DOMAINS=$(ls -1 "$DOMAINS_DIR" | grep -E '^[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?\.[A-Za-z]{2,}$' || true)
+else
+    DOMAINS=""
+fi
+DOMAINS=$(printf '%s\n' "$DOMAINS" | tr ', \t' '\n\n\n' | grep -v '^$' | sort -u || true)
+
+if [ -z "$DOMAINS" ]; then
+    echo "No domains registered in $DOMAINS_DIR; nothing to do"
+    exit 0
+fi
+
+if [ -z "$EMAIL" ]; then
+    echo "ERROR: CERTBOT_EMAIL not set"
+    exit 1
+fi
+if [ -z "$TOKEN" ]; then
+    echo "ERROR: CLOUDFLARE_API_TOKEN not set"
+    exit 1
+fi
+
+# Temporary credentials file, shared by all requests and always cleaned up
+CREDS_FILE=$(mktemp /tmp/cloudflare.XXXXXX)
+trap 'rm -f "$CREDS_FILE"' EXIT
+chmod 600 "$CREDS_FILE"
+printf 'dns_cloudflare_api_token = %s\n' "$TOKEN" > "$CREDS_FILE"
+
+FAILED=""
+ISSUED=0
+SKIPPED=0
+
+for DOMAIN in $DOMAINS; do
+    CERT_PATH="/etc/letsencrypt/live/${DOMAIN}/fullchain.pem"
+
+    if [ -f "$CERT_PATH" ]; then
+        echo "Certificate for $DOMAIN already exists at $CERT_PATH"
+        SKIPPED=$((SKIPPED + 1))
+        continue
+    fi
+
+    echo "Requesting certificate for $DOMAIN via Cloudflare DNS challenge..."
+
+    # --cert-name pins the lineage directory to the domain name so the
+    # live/<domain>/ path nginx expects never gets a -0001 suffix.
+    # STAGING_ARG is intentionally unquoted (it expands to two words).
+    # shellcheck disable=SC2086
+    if certbot certonly \
+        --dns-cloudflare \
+        --dns-cloudflare-credentials "$CREDS_FILE" \
+        --agree-tos \
+        --no-eff-email \
+        --email "$EMAIL" \
+        --non-interactive \
+        --cert-name "$DOMAIN" \
+        $STAGING_ARG \
+        -d "$DOMAIN" && [ -f "$CERT_PATH" ]; then
+        echo "Certificate obtained and verified for $DOMAIN at $CERT_PATH"
+        ISSUED=$((ISSUED + 1))
+    else
+        echo "ERROR: Failed to obtain certificate for $DOMAIN"
+        FAILED="$FAILED $DOMAIN"
+    fi
+done
+
+echo "Summary: $ISSUED issued, $SKIPPED already present"
+
+if [ -n "$FAILED" ]; then
+    echo "ERROR: certificate request failed for:$FAILED"
+    exit 1
+fi
@@ -0,0 +1,6 @@
+---
+- name: Reload nginx
+  ansible.builtin.shell: >
+    docker exec {{ nginx_proxy_nginx_container }} nginx -t &&
+    docker exec {{ nginx_proxy_nginx_container }} nginx -s reload
+  changed_when: true
@@ -0,0 +1,118 @@
+---
+# Publish an app through nginx-proxy. Usage:
+#
+#   - ansible.builtin.include_role:
+#       name: nginx-proxy
+#       tasks_from: add_vhost
+#     vars:
+#       vhost_name: paperclip              # required, becomes conf.d/<name>.conf
+#       vhost_domain: "{{ paperclip_domain }}"   # required
+#       vhost_upstream_port: 3100          # required
+#       vhost_upstream_host: 127.0.0.1     # optional
+#       vhost_proxy_timeout: 60s           # optional (use 3600s for WebSockets)
+#       vhost_health_path: /health         # optional, proxied without access log
+#
+# Order: marker -> certificate -> vhost file -> nginx -t -> reload.
+# The vhost is only written once the certificate exists, and a vhost that
+# fails nginx -t is rolled back, so one bad app cannot take the others down.
+- name: Validate vhost parameters
+  ansible.builtin.assert:
+    that:
+      - vhost_name is defined
+      - vhost_domain is defined
+      - vhost_upstream_port is defined
+    fail_msg: "add_vhost needs vhost_name, vhost_domain and vhost_upstream_port"
+    quiet: true
+
+- name: Check that the nginx-proxy containers are running
+  ansible.builtin.command: "docker inspect -f '{% raw %}{{.State.Running}}{% endraw %}' {{ item }}"
+  register: nginx_proxy_running
+  failed_when: nginx_proxy_running.rc != 0 or nginx_proxy_running.stdout | trim != 'true'
+  changed_when: false
+  loop:
+    - "{{ nginx_proxy_nginx_container }}"
+    - "{{ nginx_proxy_certbot_container }}"
+  loop_control:
+    label: "{{ item }}"
+
+- name: "Register cert marker for {{ vhost_domain }}"
+  ansible.builtin.file:
+    path: "{{ nginx_proxy_domains_dir }}/{{ vhost_domain }}"
+    state: touch
+    modification_time: preserve
+    access_time: preserve
+    mode: "0644"
+
+- name: "Check for existing certificate for {{ vhost_domain }}"
+  ansible.builtin.stat:
+    path: "{{ nginx_proxy_certs_dir }}/live/{{ vhost_domain }}/fullchain.pem"
+  register: nginx_proxy_cert_before
+
+# Targeted run: only this domain, so another app's failed issuance
+# cannot fail this play.
+- name: "Issue certificate for {{ vhost_domain }}"
+  ansible.builtin.command: >-
+    docker exec {{ nginx_proxy_certbot_container }}
+    /scripts/certbot-init.sh {{ vhost_domain }}
+  register: nginx_proxy_cert_issue
+  changed_when: "'Certificate obtained' in nginx_proxy_cert_issue.stdout"
+  when: not nginx_proxy_cert_before.stat.exists
+
+- name: "Confirm certificate exists for {{ vhost_domain }}"
+  ansible.builtin.stat:
+    path: "{{ nginx_proxy_certs_dir }}/live/{{ vhost_domain }}/fullchain.pem"
+  register: nginx_proxy_cert_after
+  failed_when: not nginx_proxy_cert_after.stat.exists
+
+- name: "Write {{ vhost_name }} vhost, test, and reload"
+  block:
+    - name: "Template conf.d/{{ vhost_name }}.conf"
+      ansible.builtin.template:
+        src: vhost.conf.j2
+        dest: "{{ nginx_proxy_conf_dir }}/{{ vhost_name }}.conf"
+        mode: "0644"
+        backup: true
+      register: nginx_proxy_vhost_file
+
+    - name: Test nginx config
+      ansible.builtin.command: "docker exec {{ nginx_proxy_nginx_container }} nginx -t"
+      changed_when: false
+
+    - name: Reload nginx
+      ansible.builtin.command: "docker exec {{ nginx_proxy_nginx_container }} nginx -s reload"
+      when: nginx_proxy_vhost_file.changed
+
+    - name: Remove superseded backup
+      ansible.builtin.file:
+        path: "{{ nginx_proxy_vhost_file.backup_file }}"
+        state: absent
+      when: nginx_proxy_vhost_file.backup_file is defined
+
+  rescue:
+    - name: Restore previous vhost
+      ansible.builtin.copy:
+        src: "{{ nginx_proxy_vhost_file.backup_file }}"
+        dest: "{{ nginx_proxy_conf_dir }}/{{ vhost_name }}.conf"
+        remote_src: true
+        mode: "0644"
+      when: nginx_proxy_vhost_file.backup_file | default('') | length > 0
+
+    - name: Remove backup after restoring it
+      ansible.builtin.file:
+        path: "{{ nginx_proxy_vhost_file.backup_file }}"
+        state: absent
+      when: nginx_proxy_vhost_file.backup_file | default('') | length > 0
+
+    - name: Remove the new vhost (there was no previous one)
+      ansible.builtin.file:
+        path: "{{ nginx_proxy_conf_dir }}/{{ vhost_name }}.conf"
+        state: absent
+      when:
+        - nginx_proxy_vhost_file is defined
+        - nginx_proxy_vhost_file.backup_file | default('') | length == 0
+
+    - name: Fail with context
+      ansible.builtin.fail:
+        msg: >-
+          nginx rejected the {{ vhost_name }} vhost; it was rolled back and
+          the running config is unchanged.
@@ -0,0 +1,70 @@
+---
+- name: Assert required variables
+  ansible.builtin.assert:
+    that:
+      - nginx_proxy_certbot_email | length > 0
+      - nginx_proxy_cloudflare_api_token | length > 0
+    fail_msg: >-
+      Set nginx_proxy_certbot_email, and make sure vault_cloudflare_api_token
+      (or nginx_proxy_cloudflare_api_token) is defined.
+    quiet: true
+  tags: [nginx-proxy, preflight]
+
+- name: Create directories
+  ansible.builtin.file:
+    path: "{{ item }}"
+    state: directory
+    mode: "0755"
+  loop:
+    - "{{ nginx_proxy_base_path }}"
+    - "{{ nginx_proxy_certs_dir }}"
+    - "{{ nginx_proxy_conf_dir }}"
+    - "{{ nginx_proxy_domains_dir }}"
+    - "{{ nginx_proxy_scripts_dir }}"
+  tags: [nginx-proxy]
+
+- name: Install certbot script
+  ansible.builtin.copy:
+    src: certbot-init.sh
+    dest: "{{ nginx_proxy_scripts_dir }}/certbot-init.sh"
+    mode: "0755"
+  tags: [nginx-proxy]
+
+- name: Template global nginx settings
+  ansible.builtin.template:
+    src: 00-global.conf.j2
+    dest: "{{ nginx_proxy_conf_dir }}/00-global.conf"
+    mode: "0644"
+  notify: Reload nginx
+  tags: [nginx-proxy]
+
+- name: Template compose env file
+  ansible.builtin.template:
+    src: env.j2
+    dest: "{{ nginx_proxy_base_path }}/.env"
+    mode: "0600"
+  no_log: true
+  tags: [nginx-proxy]
+
+- name: Template compose file
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "{{ nginx_proxy_base_path }}/docker-compose.yml"
+    mode: "0644"
+  tags: [nginx-proxy]
+
+- name: Deploy nginx and certbot
+  community.docker.docker_compose_v2:
+    project_src: "{{ nginx_proxy_base_path }}"
+    pull: "{{ nginx_proxy_pull_policy }}"
+    state: present
+  tags: [nginx-proxy]
+
+- name: Verify nginx config is valid and the container is serving
+  ansible.builtin.command: "docker exec {{ nginx_proxy_nginx_container }} nginx -t"
+  register: nginx_proxy_test
+  until: nginx_proxy_test.rc == 0
+  retries: 6
+  delay: 5
+  changed_when: false
+  tags: [nginx-proxy, verify]
@@ -0,0 +1,31 @@
+# Managed by Ansible (nginx-proxy role)
+# Included inside the stock nginx.conf http { } block, so these are http-level.
+server_tokens off;
+client_max_body_size {{ nginx_proxy_client_max_body_size }};
+
+ssl_protocols TLSv1.2 TLSv1.3;
+ssl_ciphers HIGH:!aNULL:!MD5;
+ssl_prefer_server_ciphers on;
+ssl_session_cache shared:SSL:10m;
+ssl_session_timeout 10m;
+
+# Catch-all: unknown Host headers get nothing, and unknown SNI is rejected
+# before any certificate is presented.
+server {
+    listen 80 default_server;
+    server_name _;
+    access_log off;
+
+    location = /nginx-health {
+        return 200 "ok\n";
+    }
+    location / {
+        return 444;
+    }
+}
+
+server {
+    listen 443 ssl default_server;
+    server_name _;
+    ssl_reject_handshake on;
+}
@@ -0,0 +1,46 @@
+# Managed by Ansible (nginx-proxy role)
+services:
+  certbot:
+    image: {{ nginx_proxy_certbot_image }}
+    container_name: {{ nginx_proxy_certbot_container }}
+    volumes:
+      - {{ nginx_proxy_certs_dir }}:/etc/letsencrypt
+      - {{ nginx_proxy_scripts_dir }}:/scripts:ro
+      - {{ nginx_proxy_domains_dir }}:/etc/cert-domains.d:ro
+    environment:
+      CLOUDFLARE_API_TOKEN: ${CLOUDFLARE_API_TOKEN}
+      CERTBOT_EMAIL: ${CERTBOT_EMAIL}
+      CERTBOT_STAGING: "{{ '1' if nginx_proxy_certbot_staging | bool else '0' }}"
+    entrypoint: /bin/sh
+    command:
+      - -c
+      - |
+        while true; do
+          /scripts/certbot-init.sh || echo "issuance failed; will retry next cycle"
+          certbot renew --quiet
+          sleep 12h
+        done
+    restart: unless-stopped
+
+  nginx:
+    image: {{ nginx_proxy_nginx_image }}
+    container_name: {{ nginx_proxy_nginx_container }}
+    network_mode: host
+    volumes:
+      # Directory mounts (not single files) so Ansible's atomic file
+      # replacement is visible inside the container.
+      - {{ nginx_proxy_conf_dir }}:/etc/nginx/conf.d:ro
+      - {{ nginx_proxy_certs_dir }}:/etc/nginx/certs:ro
+    command:
+      - /bin/sh
+      - -c
+      - |
+        ( while :; do sleep {{ nginx_proxy_reload_interval }}; nginx -s reload; done ) &
+        exec nginx -g 'daemon off;'
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost/nginx-health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
@@ -0,0 +1,3 @@
+# Managed by Ansible (nginx-proxy role)
+CLOUDFLARE_API_TOKEN={{ nginx_proxy_cloudflare_api_token }}
+CERTBOT_EMAIL={{ nginx_proxy_certbot_email }}
@@ -0,0 +1,51 @@
+# Managed by Ansible (nginx-proxy role) - vhost: {{ vhost_name }}
+server {
+    listen 80;
+    server_name {{ vhost_domain }};
+    return 301 https://{{ vhost_domain }}$request_uri;
+}
+
+server {
+    listen 443 ssl;
+{% if nginx_proxy_http2 | bool %}
+    http2 on;
+{% endif %}
+    server_name {{ vhost_domain }};
+
+    ssl_certificate /etc/nginx/certs/live/{{ vhost_domain }}/fullchain.pem;
+    ssl_certificate_key /etc/nginx/certs/live/{{ vhost_domain }}/privkey.pem;
+
+    add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
+    add_header X-Content-Type-Options "nosniff" always;
+    add_header X-Frame-Options "SAMEORIGIN" always;
+    add_header X-XSS-Protection "1; mode=block" always;
+
+    location / {
+        proxy_pass http://{{ vhost_upstream_host | default('127.0.0.1') }}:{{ vhost_upstream_port }};
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Host $host;
+        proxy_set_header X-Forwarded-Port $server_port;
+
+        # WebSocket support
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+
+        proxy_connect_timeout 60s;
+        proxy_send_timeout {{ vhost_proxy_timeout | default('60s') }};
+        proxy_read_timeout {{ vhost_proxy_timeout | default('60s') }};
+
+        proxy_buffering off;
+        proxy_request_buffering off;
+    }
+{% if vhost_health_path | default('') | length > 0 %}
+
+    location {{ vhost_health_path }} {
+        proxy_pass http://{{ vhost_upstream_host | default('127.0.0.1') }}:{{ vhost_upstream_port }}{{ vhost_health_path }};
+        access_log off;
+    }
+{% endif %}
+}