From 0875389d024a7cdb8115c30a08589002f9175bfc Mon Sep 17 00:00:00 2001 From: Phil Date: Sat, 31 Oct 2026 09:29:41 -0600 Subject: [PATCH] Add guard rails for several agents working on the repo at once Deploys from unpushed branches were being reverted by the next deploy from main, and in-place vault edits could overwrite each other. - deploy-guard.yml (imported by site.yml and dns.yml) refuses real runs unless the checkout is clean and exactly origin/main; --check is fine. - bin/deploy holds a lock so only one deploy runs at a time. - bin/vault-run edits the vault under a lock with an atomic swap. - bin/worktree starts a task in a fresh worktree from origin/main. - CLAUDE.md explains the workflow. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 31 +++++++++++++++++++++++++++++++ bin/_common.sh | 24 ++++++++++++++++++++++++ bin/deploy | 14 ++++++++++++++ bin/vault-run | 25 +++++++++++++++++++++++++ bin/worktree | 11 +++++++++++ deploy-guard.yml | 40 ++++++++++++++++++++++++++++++++++++++++ dns.yml | 2 ++ site.yml | 2 ++ 8 files changed, 149 insertions(+) create mode 100644 bin/_common.sh create mode 100755 bin/deploy create mode 100755 bin/vault-run create mode 100755 bin/worktree create mode 100644 deploy-guard.yml diff --git a/CLAUDE.md b/CLAUDE.md index edbe848..f32f29a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,6 +8,37 @@ This is a personal infrastructure Ansible playbook that automates deployment and **Important**: Always review `DEPLOYMENT_LEARNINGS.md` when working on this repository for lessons learned and troubleshooting guidance. +## Working alongside other agents (read first) + +Several agents change this repo and deploy to the same server at once. Two +things went wrong before these rules: a deploy from an unpushed branch was +silently reverted by the next deploy from main, and vault edits overwrote each +other. Follow these steps: + +1. **Never edit the main checkout.** Start each task in its own worktree: + `bin/worktree ` (based on the latest origin/main, with the + vault files linked in). +2. **Land on main before deploying.** Commit, `git fetch && git rebase + origin/main`, `git push origin HEAD:main`, then deploy that exact commit. + `deploy-guard.yml` (imported by site.yml and dns.yml) refuses real runs from + a checkout that is dirty or isn't origin/main. `--check` runs are always + allowed; `-e allow_unpushed=true` is for emergencies only, and the fix must + be pushed straight after. +3. **Deploy with `bin/deploy`**, not ansible-playbook directly. It holds a lock + so only one deploy runs at a time (`bin/deploy --tags `; + `PLAYBOOK=dns.yml bin/deploy` for DNS). Use the narrowest tag that covers + your change. +4. **Edit the vault only with `bin/vault-run `.** It locks, backs up, + decrypts to a private temp file passed as `$VAULT_PLAIN`, checks the YAML, + re-encrypts and swaps it in at once. Never `ansible-vault edit`, and don't + decrypt vault.yml in place. +5. **Check the live result after deploying** (e.g. `docker inspect`, a login + test). If someone else's deploy undid yours, rebase, push, and redeploy + rather than deploying an older checkout. +6. **Shared files** (Caddyfile.j2, domains.yml, dns.yml, main.yml task lists) + are touched by everyone: keep edits small, rebase just before pushing, and + resolve conflicts by keeping both sides. + ## Common Commands ### Initial Setup diff --git a/bin/_common.sh b/bin/_common.sh new file mode 100644 index 0000000..ae7c4b2 --- /dev/null +++ b/bin/_common.sh @@ -0,0 +1,24 @@ +# Shared by the bin/ helpers. The main checkout holds the venv and the +# (gitignored) vault files that every worktree uses. +repo=$(git rev-parse --show-toplevel) +main_checkout=$(git -C "$repo" worktree list --porcelain | awk '/^worktree /{print $2; exit}') +lock_dir=${XDG_RUNTIME_DIR:-/tmp}/ansible-selfhosting +mkdir -p "$lock_dir" && chmod 700 "$lock_dir" + +# take_lock NAME: wait for, then hold, a lock shared by every agent on this machine. +take_lock() { + local name=$1 + exec 9>"$lock_dir/$name.lock" + if ! flock -n 9; then + echo "Waiting for $name lock, held by: $(cat "$lock_dir/$name.owner" 2>/dev/null || echo unknown)" >&2 + flock 9 + fi + echo "$(basename "$repo") (pid $$, $(date '+%H:%M:%S'))" > "$lock_dir/$name.owner" +} + +# Worktrees need the gitignored vault files from the main checkout. +link_vault() { + [ "$repo" = "$main_checkout" ] && return + [ -e "$repo/group_vars/all/vault.yml" ] || ln -s "$main_checkout/group_vars/all/vault.yml" "$repo/group_vars/all/vault.yml" + [ -e "$repo/vault_pass" ] || ln -s "$main_checkout/vault_pass" "$repo/vault_pass" +} diff --git a/bin/deploy b/bin/deploy new file mode 100755 index 0000000..80cf7f9 --- /dev/null +++ b/bin/deploy @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Run a playbook while holding the deploy lock, so only one agent deploys at a +# time. Arguments go straight to ansible-playbook. +# bin/deploy --tags gitea (site.yml) +# bin/deploy --check --diff +# PLAYBOOK=dns.yml bin/deploy +# Real runs only work from a clean checkout of origin/main (deploy-guard.yml). +set -euo pipefail +. "$(dirname "$0")/_common.sh" +link_vault +take_lock deploy +cd "$repo" +"$main_checkout/.venv/bin/ansible-playbook" "${PLAYBOOK:-site.yml}" -i hosts.yml \ + --vault-password-file "$main_checkout/vault_pass" "$@" diff --git a/bin/vault-run b/bin/vault-run new file mode 100755 index 0000000..bfeae83 --- /dev/null +++ b/bin/vault-run @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Edit the shared vault safely: lock, back up, decrypt to a private temp file, +# run your command on it, check the YAML, re-encrypt, and swap it in at once. +# The command gets the plaintext path as $VAULT_PLAIN (also appended as the +# last argument). Never use `ansible-vault edit`. +# bin/vault-run python3 my_edit.py +# bin/vault-run nvim +set -euo pipefail +. "$(dirname "$0")/_common.sh" +[ $# -gt 0 ] || { echo "usage: bin/vault-run " >&2; exit 2; } +vault="$main_checkout/group_vars/all/vault.yml" +pass="$main_checkout/vault_pass" +av="$main_checkout/.venv/bin/ansible-vault" +take_lock vault +backups="$HOME/Documents/backups/ansible-vault"; mkdir -p "$backups" +cp -p "$vault" "$backups/vault.yml.$(date +%Y%m%d-%H%M%S)" +tmp=$(mktemp -d "$lock_dir/vault.XXXXXX"); trap 'shred -u "$tmp"/* 2>/dev/null; rm -rf "$tmp"' EXIT +export VAULT_PLAIN="$tmp/vault.yml" +"$av" decrypt --vault-password-file "$pass" --output "$VAULT_PLAIN" "$vault" >/dev/null +"$@" "$VAULT_PLAIN" +"$main_checkout/.venv/bin/python" -c 'import sys,yaml; yaml.safe_load(open(sys.argv[1]))' "$VAULT_PLAIN" +"$av" encrypt --vault-password-file "$pass" --output "$tmp/vault.enc" "$VAULT_PLAIN" >/dev/null +chmod --reference="$vault" "$tmp/vault.enc" 2>/dev/null || true +mv "$tmp/vault.enc" "$vault" +echo "vault updated" diff --git a/bin/worktree b/bin/worktree new file mode 100755 index 0000000..f161e93 --- /dev/null +++ b/bin/worktree @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Start work on a fresh branch in its own worktree, based on the latest main. +# bin/worktree +set -euo pipefail +. "$(dirname "$0")/_common.sh" +name=${1:?usage: bin/worktree } +dest="$(dirname "$main_checkout")/ansible-worktrees/$name" +git -C "$main_checkout" fetch -q origin +git -C "$main_checkout" worktree add -b "$name" "$dest" origin/main +cd "$dest" && repo=$dest && link_vault +echo "Ready: $dest" diff --git a/deploy-guard.yml b/deploy-guard.yml new file mode 100644 index 0000000..3b8c6ef --- /dev/null +++ b/deploy-guard.yml @@ -0,0 +1,40 @@ +--- +# Imported at the top of site.yml and dns.yml. +# +# Several agents work on this repo at once. If one deploys from an unpushed +# branch, the next deploy from main quietly puts the old config back. So a +# real run only goes ahead when this checkout is exactly origin/main: +# land your commit on main first, then deploy that commit. +# +# --check runs are always allowed. -e allow_unpushed=true skips the guard +# for emergencies; push the fix to main straight after. +- name: Only deploy what's on origin/main + hosts: localhost + gather_facts: false + tags: always + tasks: + - name: Check this checkout matches origin/main + ansible.builtin.shell: | + set -eu + git fetch -q origin main + if [ -n "$(git status --porcelain --untracked-files=no)" ]; then + echo "This checkout has uncommitted changes. Commit them, land them on main, then deploy." + exit 1 + fi + head=$(git rev-parse HEAD); main=$(git rev-parse origin/main) + if [ "$head" != "$main" ]; then + echo "HEAD $(git rev-parse --short HEAD) isn't origin/main ($(git rev-parse --short origin/main))." + if git merge-base --is-ancestor "$main" "$head"; then + echo "Your commits aren't on main yet: rebase, push to main, then deploy." + else + echo "main has moved on: rebase onto origin/main first, or other agents' changes get reverted." + fi + exit 1 + fi + args: + chdir: "{{ playbook_dir }}" + changed_when: false + check_mode: false + when: + - not ansible_check_mode + - not (allow_unpushed | default(false) | bool) diff --git a/dns.yml b/dns.yml index 66b9067..31678f1 100644 --- a/dns.yml +++ b/dns.yml @@ -1,5 +1,7 @@ --- # dns.yml +- import_playbook: deploy-guard.yml + - name: Add A Records for thesatelliteoflove.com and nerder.land hosts: localhost gather_facts: false diff --git a/site.yml b/site.yml index e0e5d09..4f7e7de 100644 --- a/site.yml +++ b/site.yml @@ -1,3 +1,5 @@ +- import_playbook: deploy-guard.yml + - hosts: docker become: true roles: