From e569a564178921dca7ceb487874ad0893da7c98d Mon Sep 17 00:00:00 2001
From: Phil <phil@tsol.re>
Date: Thu, 29 Oct 2026 21:04:22 -0600
Subject: [PATCH] Reserve fixed IPs on the lava network so Caddy can always
 start

lava was created by hand without an ip_range, so any container could be
given Caddy's fixed IP (172.20.0.5). After a reboot the containers race
for the low addresses and Caddy often fails with "Address already in use".

- The docker role now creates lava as 172.20.0.0/16 with dynamic
  addresses limited to 172.20.128.0/17, so 172.20.0.x is free for fixed
  IPs, and warns if the network still has the old layout.
- Gitea gets fixed IP 172.20.0.6 once the network has the reserved range,
  since its ufw-docker SSH rule points at its IP.
- ufw-docker allow rules for Caddy (80, 443) and Gitea SSH are managed by
  the playbook and follow IP changes.
- migrate-lava-network.yml recreates the existing network once: it takes
  down every compose project on lava (volumes kept), recreates the
  network, and brings them back up, Caddy first.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 CLAUDE.md                                   |   3 +
 group_vars/all/infrastructure.yml           |   8 +-
 migrate-lava-network.yml                    | 129 ++++++++++++++++++++
 roles/docker/tasks/development/gitea.yml    |   9 +-
 roles/docker/tasks/infrastructure/caddy.yml |  10 +-
 roles/docker/tasks/main.yml                 |  40 ++++++
 roles/docker/templates/gitea-compose.yml.j2 |   5 +
 7 files changed, 201 insertions(+), 3 deletions(-)
 create mode 100644 migrate-lava-network.yml

diff --git a/CLAUDE.md b/CLAUDE.md
index fd69e61..dbf883e 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -112,6 +112,9 @@ Many services depend on Authentik for SSO. When deploying new services, consider
 - Network connectivity requirements within Docker stack
 - Hairpinning fixes for internal service-to-service communication
 
+### Shared Docker Network (`lava`)
+Created by the docker role as `{{ docker.subnet }}` with `ip_range: {{ docker.ip_range }}`. Docker only hands out dynamic addresses from that range, so `172.20.0.x` is reserved for fixed IPs: Caddy (`docker.hairpin_ip`, .5) and Gitea (`docker.gitea_ip`, .6). Both need fixed IPs because ufw-docker firewall rules point at container IPs. A network without the range (the 2024 original) is migrated once with `migrate-lava-network.yml`.
+
 ### Hairpinning Resolution
 Services inside Docker containers cannot reach external domains that resolve to the same server. Fix by adding `extra_hosts` mappings:
 
diff --git a/group_vars/all/infrastructure.yml b/group_vars/all/infrastructure.yml
index 2c1a647..8b01935 100644
--- a/group_vars/all/infrastructure.yml
+++ b/group_vars/all/infrastructure.yml
@@ -4,7 +4,13 @@
 docker:
   network_name: "lava"
   stacks_path: "/opt/stacks"
-  hairpin_ip: "172.20.0.5"
+  hairpin_ip: "172.20.0.5"   # Caddy's fixed IP
+  # Shared network layout. Docker only hands out addresses from ip_range,
+  # so 172.20.0.x is kept free for fixed IPs and can't be taken by others.
+  subnet: "172.20.0.0/16"
+  gateway: "172.20.0.1"
+  ip_range: "172.20.128.0/17"
+  gitea_ip: "172.20.0.6"     # fixed so the ufw-docker SSH rule stays valid
 
 # SMTP configuration
 smtp:
diff --git a/migrate-lava-network.yml b/migrate-lava-network.yml
new file mode 100644
index 0000000..660225d
--- /dev/null
+++ b/migrate-lava-network.yml
@@ -0,0 +1,129 @@
+---
+# One-off: recreate the shared docker network with a reserved address range.
+#
+# Why: lava was created without an ip_range, so any container can be given
+# Caddy's fixed IP (172.20.0.5) and Caddy then fails with "Address already
+# in use". Docker can't change a network's range in place, so every stack on
+# it is taken down, the network is recreated, and the stacks come back up.
+# Volumes (all data) are kept. Expect a few minutes of downtime for everything.
+#
+# Run:   ansible-playbook migrate-lava-network.yml -i hosts.yml --vault-password-file vault_pass
+# Then:  ansible-playbook site.yml -i hosts.yml --vault-password-file vault_pass
+#        (gives gitea its fixed IP and fixes its firewall rule)
+#
+# The list of stacks is saved to /root/lava-migration-<time>.txt first, so a
+# failed run can be finished by hand with "docker compose ... up -d".
+
+- hosts: docker
+  become: true
+  gather_facts: false
+  vars:
+    net: "{{ docker.network_name }}"
+
+  tasks:
+    - name: Pick the recovery file name once
+      ansible.builtin.set_fact:
+        record: "/root/lava-migration-{{ lookup('pipe', 'date +%Y%m%d-%H%M%S') }}.txt"
+
+    - name: Look up the network
+      community.docker.docker_network_info:
+        name: "{{ net }}"
+      register: before
+
+    - name: Stop if the network already has the reserved range
+      ansible.builtin.meta: end_play
+      when: >-
+        before.exists and
+        (before.network.IPAM.Config | map(attribute='IPRange', default='') | first) == docker.ip_range
+
+    # One line per compose project: project|working_dir|config_files
+    - name: List compose projects on the network
+      ansible.builtin.shell: |
+        for id in $(docker ps -aq --filter network={{ net }}); do
+          docker inspect "$id" -f '{{ '{{' }}index .Config.Labels "com.docker.compose.project"{{ '}}' }}|{{ '{{' }}index .Config.Labels "com.docker.compose.project.working_dir"{{ '}}' }}|{{ '{{' }}index .Config.Labels "com.docker.compose.project.config_files"{{ '}}' }}'
+        done | grep -v '^|' | sort -u
+      register: projects_raw
+      changed_when: false
+      check_mode: false
+
+    - name: List containers on the network that aren't from compose
+      ansible.builtin.shell: |
+        for id in $(docker ps -aq --filter network={{ net }}); do
+          p=$(docker inspect "$id" -f '{{ '{{' }}index .Config.Labels "com.docker.compose.project"{{ '}}' }}')
+          [ -z "$p" ] && docker inspect "$id" -f '{{ '{{' }}.Name{{ '}}' }}' | sed 's,^/,,'
+        done; true
+      register: loose_raw
+      changed_when: false
+      check_mode: false
+
+    - name: Parse project list
+      ansible.builtin.set_fact:
+        projects: >-
+          {% set out = [] %}{% for line in projects_raw.stdout_lines %}{% set p = line.split('|') %}{% set _ = out.append({'name': p[0], 'dir': p[1], 'files': p[2]}) %}{% endfor %}{{ out }}
+        loose: "{{ loose_raw.stdout_lines | select | list }}"
+
+    - name: Show what will be migrated
+      ansible.builtin.debug:
+        msg:
+          - "{{ projects | length }} compose projects: {{ projects | map(attribute='name') | join(', ') }}"
+          - "Non-compose containers to detach: {{ loose | join(', ') or 'none' }}"
+
+    - name: Save the project list for recovery
+      ansible.builtin.copy:
+        dest: "{{ record }}"
+        content: "{{ projects_raw.stdout }}\n"
+        mode: '0600'
+
+    - name: Take down every project on the network (volumes are kept)
+      ansible.builtin.command:
+        argv: "{{ ['docker', 'compose', '-p', item.name, '--project-directory', item.dir]
+                  + (item.files.split(',') | map('regex_replace', '^', '--file=') | list)
+                  + ['down'] }}"
+      loop: "{{ projects }}"
+      loop_control:
+        label: "{{ item.name }}"
+
+    - name: Detach non-compose containers
+      ansible.builtin.command: docker network disconnect -f {{ net }} {{ item }}
+      loop: "{{ loose }}"
+
+    - name: Remove the old network
+      community.docker.docker_network:
+        name: "{{ net }}"
+        state: absent
+
+    - name: Create the network with the reserved range
+      community.docker.docker_network:
+        name: "{{ net }}"
+        driver: bridge
+        ipam_config:
+          - subnet: "{{ docker.subnet }}"
+            gateway: "{{ docker.gateway }}"
+            iprange: "{{ docker.ip_range }}"
+
+    # Caddy first so the sites come back as soon as possible
+    - name: Bring projects back up
+      ansible.builtin.command:
+        argv: "{{ ['docker', 'compose', '-p', item.name, '--project-directory', item.dir]
+                  + (item.files.split(',') | map('regex_replace', '^', '--file=') | list)
+                  + ['up', '-d'] }}"
+      loop: "{{ (projects | selectattr('name', 'equalto', 'caddy') | list)
+                + (projects | rejectattr('name', 'equalto', 'caddy') | list) }}"
+      loop_control:
+        label: "{{ item.name }}"
+      register: ups
+      ignore_errors: true
+
+    - name: Make sure public traffic still reaches Caddy
+      ansible.builtin.command: /usr/local/bin/ufw-docker.sh allow caddy-caddy-1 {{ item }}
+      loop: [80/tcp, 443/tcp, 443/udp]
+      register: caddy_ufw
+      changed_when: "'Skipping' not in caddy_ufw.stdout"
+
+    - name: Report
+      ansible.builtin.debug:
+        msg:
+          - "Failed to start: {{ ups.results | selectattr('failed') | map(attribute='item.name') | join(', ') or 'none' }}"
+          - "Recovery list: {{ record }}"
+          - "Detached (reconnect or remove by hand): {{ loose | join(', ') or 'none' }}"
+          - "Next: run site.yml to give gitea its fixed IP and fix its firewall rule."
diff --git a/roles/docker/tasks/development/gitea.yml b/roles/docker/tasks/development/gitea.yml
index 46e8881..7fd9230 100644
--- a/roles/docker/tasks/development/gitea.yml
+++ b/roles/docker/tasks/development/gitea.yml
@@ -18,4 +18,11 @@
   community.docker.docker_compose_v2:
     project_src: /opt/stacks/gitea
     files:
-    - compose.yml
\ No newline at end of file
+    - compose.yml
+
+# Lets git over SSH (host port 222) through. Re-adds the rule if gitea's IP changed.
+- name: allow public ssh to gitea
+  ansible.builtin.command: /usr/local/bin/ufw-docker.sh allow gitea 22/tcp
+  register: gitea_ufw
+  changed_when: "'Skipping' not in gitea_ufw.stdout"
+  when: not ansible_check_mode
diff --git a/roles/docker/tasks/infrastructure/caddy.yml b/roles/docker/tasks/infrastructure/caddy.yml
index 2365217..6fa3c79 100644
--- a/roles/docker/tasks/infrastructure/caddy.yml
+++ b/roles/docker/tasks/infrastructure/caddy.yml
@@ -43,4 +43,12 @@
     project_src: /opt/stacks/caddy
     recreate: never
     files:
-    - compose.yml
\ No newline at end of file
+    - compose.yml
+
+# Lets public traffic through to Caddy. Re-adds the rule if Caddy's IP changed.
+- name: allow public traffic to caddy
+  ansible.builtin.command: /usr/local/bin/ufw-docker.sh allow caddy-caddy-1 {{ item }}
+  loop: [80/tcp, 443/tcp, 443/udp]
+  register: caddy_ufw
+  changed_when: "'Skipping' not in caddy_ufw.stdout"
+  when: not ansible_check_mode
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index 9ef35bd..cb56457 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -56,6 +56,46 @@
     state: restarted
   when: ufw_docker_check.rc != 0
 
+- name: Look up the shared docker network
+  community.docker.docker_network_info:
+    name: "{{ docker.network_name }}"
+  register: lava_net
+  tags: always
+
+# True once the network has the reserved range. Fixed IPs other than Caddy's
+# are only applied then, since today's dynamic containers may hold them.
+- name: Record whether the shared network has the reserved range
+  ansible.builtin.set_fact:
+    lava_reserved: >-
+      {{ lava_net.exists and
+         (lava_net.network.IPAM.Config | map(attribute='IPRange', default='') | first) == docker.ip_range }}
+  tags: always
+
+- name: Create the shared docker network
+  community.docker.docker_network:
+    name: "{{ docker.network_name }}"
+    driver: bridge
+    ipam_config:
+      - subnet: "{{ docker.subnet }}"
+        gateway: "{{ docker.gateway }}"
+        iprange: "{{ docker.ip_range }}"
+  when: not lava_net.exists
+  register: lava_created
+
+- name: Mark a freshly created network as reserved
+  ansible.builtin.set_fact:
+    lava_reserved: true
+  when: lava_created is changed
+
+# Docker can't change a network's address range in place; recreating it
+# needs every container detached first. See migrate-lava-network.yml.
+- name: Warn if the shared network has the old address layout
+  ansible.builtin.debug:
+    msg: >-
+      {{ docker.network_name }} has no reserved range (wanted {{ docker.ip_range }}),
+      so other containers can take Caddy's fixed IP. Run migrate-lava-network.yml.
+  when: lava_net.exists and not (lava_reserved | bool)
+
 - name: make directories
   ansible.builtin.file:
     path: "{{ item }}"
diff --git a/roles/docker/templates/gitea-compose.yml.j2 b/roles/docker/templates/gitea-compose.yml.j2
index 1a6a699..9e90f72 100644
--- a/roles/docker/templates/gitea-compose.yml.j2
+++ b/roles/docker/templates/gitea-compose.yml.j2
@@ -30,6 +30,11 @@ services:
     extra_hosts:
       - '{{ subdomains.auth }}:{{ docker.hairpin_ip }}'
       - '{{ subdomains.git }}:{{ docker.hairpin_ip }}'
+{% if lava_reserved | default(false) | bool %}
+    networks:
+      default:
+        ipv4_address: {{ docker.gitea_ip }}
+{% endif %}
   runner:
     image: gitea/act_runner:nightly
     restart: unless-stopped