fix(headscale): make the server deployable and manage its config in git

.github/workflows/tower-ansible-deploy.yml

@@ -88,5 +88,8 @@ jobs:
           GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
           SCHEDULED_RUN: ${{ github.event_name == 'schedule' || github.event.inputs.force_schedule_mode == 'true' }}
           DNS_SEARCH_DOMAIN: ${{ secrets.DNS_SEARCH_DOMAIN }}
+          TOWER_HOSTNAME: ${{ secrets.TOWER_HOSTNAME }}
+          TOWER_ACME_EMAIL: ${{ secrets.TOWER_ACME_EMAIL }}
+          TAILNET_BASE_DOMAIN: ${{ secrets.TAILNET_BASE_DOMAIN }}
         run: |
           uv run ansible-playbook -i inventory tower/ansible/playbook.yml --private-key ~/.ssh/id_ed25519

.github/workflows/tower-tofu-apply.yml

@@ -35,6 +35,7 @@ jobs:
       TF_VAR_ssh_authorized_keys: ${{ secrets.SSH_AUTHORIZED_KEYS }}
       TF_VAR_cloudflare_api_token: ${{ secrets.CLOUDFLARE_API_TOKEN }}
       TF_VAR_cloudflare_zone_id: ${{ secrets.CLOUDFLARE_ZONE_ID }}
+      TF_VAR_tower_hostname: ${{ secrets.TOWER_HOSTNAME }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

.github/workflows/validate.yml

@@ -125,6 +125,56 @@ jobs:
       - name: Check formatting
         run: tofu fmt -check -recursive -diff
 
+  headscale:
+    name: Headscale config
+    runs-on: ubuntu-latest
+    steps:
+      # Depth 2: on pull_request the checkout is GitHub's test merge commit,
+      # and the diff below needs its first parent (the base branch).
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          fetch-depth: 2
+
+      # Only pull requests that touch what the check reads run it: the config
+      # and policy, the compose file (it pins the image, so Renovate bumps
+      # count), the script, and this workflow. The job still runs and reports
+      # success otherwise, so it can stay a required status check; a
+      # workflow-level `paths:` filter would leave it pending instead. Merge
+      # queue and manual runs always check.
+      - name: Detect Headscale changes
+        id: changes
+        env:
+          EVENT: ${{ github.event_name }}
+        run: |
+          if [ "$EVENT" != pull_request ]; then
+            echo "run=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          # The test merge commit's first parent is the base branch, so this
+          # is exactly what merging the pull request would change.
+          changed=$(git diff --name-only HEAD^1 HEAD -- \
+            tower/containers/headscale \
+            ci/headscale-check.sh \
+            .github/workflows/validate.yml)
+          if [ -n "$changed" ]; then
+            printf 'Changed:\n%s\n' "$changed"
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No Headscale config, image or check changes; skipping."
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      # Runs the image pinned in tower/containers/headscale/docker-compose.yml
+      # against the committed config and policy, so a Renovate bump that drops
+      # a config key fails here rather than crash-looping on tower.
+      #
+      # The image reference comes from the PR, like the galaxy sources above.
+      # The job holds no credentials and the container gets no network.
+      - name: Check config and policy with the pinned image
+        if: steps.changes.outputs.run == 'true'
+        run: ci/headscale-check.sh
+
   workflows:
     name: Workflow lint
     runs-on: ubuntu-latest

CLAUDE.md

@@ -8,7 +8,7 @@ This repo manages infrastructure for multiple hosts using Ansible + Docker Compo
 
 **Ansible hosts** (`mx1`, `web1`, `web2`, `web3`): each top-level directory contains `ansible/` (playbooks and roles) and `containers/` (Docker Compose stacks).
 
-**OpenTofu hosts** (`tower`): each top-level directory contains `terraform/` (`.tf` files) and `docs/` (node-specific setup guide). Run `tofu` commands from the `terraform/` subdirectory. Variables are passed via `TF_VAR_*` env vars; the S3 backend requires `-backend-config` flags at `init` time — see `tower/docs/setup.md`. OpenTofu version: **1.12.x** (pinned in `.github/workflows/tower-tofu-apply.yml`). Note: `error()` in expressions requires ≥ 1.9. When updating providers, regenerate the lock file with `tofu providers lock -platform=linux_amd64` so CI hashes are included.
+**OpenTofu hosts** (`tower`): each top-level directory contains `terraform/` (`.tf` files) and `docs/` (node-specific setup guide). `tower` is also configured by Ansible: it has `ansible/` and `containers/` like the other hosts, deployed after each OpenTofu apply. Run `tofu` commands from the `terraform/` subdirectory. Variables are passed via `TF_VAR_*` env vars; the S3 backend requires `-backend-config` flags at `init` time — see `tower/docs/setup.md`. OpenTofu version: **1.13.x** (pinned in `.github/workflows/tower-tofu-apply.yml`; Renovate bumps it together with `required_version`). Note: `error()` in expressions requires ≥ 1.9. When updating providers, regenerate the lock file with `tofu providers lock -platform=linux_amd64` so CI hashes are included.
 
 ## Running Playbooks
 
@@ -38,6 +38,10 @@ docker_stacks:
 
 ## Conventions
 
+- **The repository is public.** Do not commit identifying details: hostnames and domains of the hosts (tower's comes
+  from the `TOWER_HOSTNAME` environment secret), names of personal machines or tailnet nodes, what hardware or OS
+  they run, or account identifiers. This applies to comments, docs, commit messages and PR text too. Inject runtime
+  values through environment secrets → `.env` / `TF_VAR_*`, and write `<tower-hostname>`-style placeholders in docs
 - **No `version:` key** in `docker-compose.yml` (deprecated in Compose V2)
 - **Always bind ports to localhost**: `127.0.0.1:<port>:<port>`; use incremental ports (3001, 3002, ...) per host
 - **Prefer `.env` files** over inline `environment:` blocks; set `env_file: - .env` in compose
@@ -93,7 +97,7 @@ security_opt:
 
 **When to add `cap_add` back** (always pair with `cap_drop: [ALL]`):
 - `NET_BIND_SERVICE` — container binds a privileged port (< 1024) directly, e.g. SMTP (25, 465, 587), IMAP (993), HTTPS (443)
-- `NET_ADMIN` — VPN/network management (e.g. headscale)
+- `NET_ADMIN` — VPN/network management (e.g. a WireGuard or Tailscale *client* container; not headscale, which is only a control server and needs just `NET_BIND_SERVICE`)
 - `SYS_NICE` — real-time scheduling (e.g. MySQL)
 - `CHOWN` — init/setup containers that `chown` volume paths on startup
 
@@ -106,7 +110,7 @@ tmpfs:
   - /run        # nginx (PID file)
 ```
 
-Applied to: cloudflared, mailflow frontend + backend, n8n runner.
+Applied to: cloudflared, mailflow frontend + backend, n8n runner. `headscale` is stateful but also runs `read_only: true`: it writes only to its data volume and a `/var/run/headscale` tmpfs.
 
 `calcom` is **not** read-only — its entrypoint installs NPM packages and writes a build cache on startup, which breaks under `read_only: true`. Keep `cap_drop: [ALL]` + `no-new-privileges:true`, omit `read_only`/`tmpfs`.
 
@@ -126,8 +130,8 @@ Applied to: cloudflared, mailflow frontend + backend, n8n runner.
 > worker in one stack and an ordinary hardened twenty.crm worker in another, so
 > "the worker is exempt" is not a safe rule of thumb. And three services
 > deliberately publish a port on all interfaces: the MX (it exists to receive
-> SMTP), headscale (WireGuard endpoint), frankenphp (terminates ACME
-> HTTP-01). Anything else must bind `127.0.0.1`.
+> SMTP), headscale (tailnet control plane + DERP on 443/tcp, STUN on
+> 3478/udp), frankenphp (terminates ACME HTTP-01). Anything else must bind `127.0.0.1`.
 
 ## Adding a Service
 
@@ -139,9 +143,16 @@ Applied to: cloudflared, mailflow frontend + backend, n8n runner.
 ## Validation
 
 `ci/validate.sh` runs every check that CI runs: yamllint, ansible-lint,
-`tofu validate` / `tofu fmt` for `tower`, and a zizmor audit of the validation
-workflow. Run it before pushing; the same failures otherwise surface as a red
-PR.
+`tofu validate` / `tofu fmt` for `tower`, a zizmor audit of the validation
+workflow, and `ci/headscale-check.sh`, which runs the headscale image pinned in
+`tower/containers/headscale/docker-compose.yml` (`configtest`, `policy check`)
+against the committed config and policy and needs docker. Run it before pushing;
+the same failures otherwise surface as a red PR.
+
+Headscale's config and policy live in git at
+`tower/containers/headscale/config/` and are mounted read-only. Never tell
+anyone to edit them on the node. Operations, upgrades, restore, and why its
+users and nodes are not in OpenTofu: `tower/docs/headscale-setup.md`.
 
 Configs live in `.yamllint` and `.ansible-lint`. Two things to know when
 editing them:

README.md

@@ -29,7 +29,7 @@ The repository currently manages the following hosts:
 | `web1` | Ansible + Docker | Web/app services |
 | `web2` | Ansible + Docker | Web/app services |
 | `web3` | Ansible + Docker | Web/app services |
-| `tower` | OpenTofu | ARM VM on OCI — WireGuard VPN endpoint and SSH bastion |
+| `tower` | OpenTofu + Ansible + Docker | ARM VM on OCI — Headscale control server (tailnet coordination + DERP relay) |
 
 See [`tower/docs/setup.md`](tower/docs/setup.md) for the one-time bootstrap steps for the tower node.
 
@@ -107,25 +107,27 @@ If a migration fails, the entire playbook stops to prevent inconsistent states.
 
 ## Validation
 
-Every pull request runs `.github/workflows/validate.yml`, which performs
-static checks only. It never connects to a host, reads a state file, or
-touches a cloud provider, so a broken change is caught before it can reach
-production rather than during a deploy.
+Every pull request runs `.github/workflows/validate.yml`. It never connects
+to a host, reads a state file, or touches a cloud provider, so a broken change
+is caught before it can reach production rather than during a deploy. Every
+check is static except the Headscale one, which runs the pinned image with no
+network access against files from the checkout.
 
 | Check | Tool | Catches |
 |---|---|---|
 | YAML | `yamllint` | Syntax errors, duplicate keys, style drift |
 | Ansible | `ansible-lint` | Broken syntax, missing FQCN, non-idempotent commands, unset file modes |
 | OpenTofu | `tofu validate` / `tofu fmt` | Invalid or unformatted configuration for `tower` |
 | Actions | `zizmor` | Workflow-level privilege footguns |
+| Headscale | `headscale configtest` / `policy check` (`ci/headscale-check.sh`) | Config keys the pinned image rejects (which would crash-loop the container on tower), invalid policy |
 
 ### Running the checks locally
 
 ```bash
 ci/validate.sh
 ```
 
-Needs `ansible-lint`, `yamllint`, `zizmor` and `tofu` on `PATH`. The script
+Needs `ansible-lint`, `yamllint`, `zizmor`, `tofu` and `docker` on `PATH`. The script
 runs the same checks as CI, so failures reproduce locally.
 
 ### The container security contract is not enforced
@@ -147,7 +149,7 @@ twenty.crm worker in another.
 ### Why the validation workflow holds no secrets
 
 `validate.yml` runs on `pull_request`, so it is configured to hold no
-credentials at all — the checks are entirely static analysis:
+credentials at all — the checks only need the checkout:
 
 - It references no `secrets.*` and no `environment:`, so the job runs with no
   access to repository or environment secrets regardless of who opened the
@@ -158,8 +160,13 @@ credentials at all — the checks are entirely static analysis:
 - Third-party actions are pinned to a full commit SHA, fixing the code that
   runs with this job's token.
 
-All four checks are required status checks on `main`, so a pull request cannot
-merge until they pass. If a check is retired or renamed, remove it from the
+The yamllint, ansible-lint, OpenTofu and workflow-lint checks are required
+status checks on `main`, so a pull request cannot merge until they pass. Add
+`Headscale config` to that list too, so that a Renovate bump that breaks the
+config cannot merge. On a pull request that touches none of its inputs (the
+Headscale stack directory, `ci/headscale-check.sh`, `validate.yml`) it skips
+the check but still reports success, so requiring it does not block other
+pull requests. If a check is retired or renamed, remove it from the
 `required_status_checks.contexts` list in the same pull request — otherwise the
 merge blocks waiting on a check that no longer reports.
 

ci/headscale-check.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# Check tower's committed Headscale config and policy with the exact image the
+# compose file pins.
+#
+# Headscale refuses to start on keys its deprecation list marks as removed
+# (`dns_config` in 0.23, `randomize_client_port` in 0.29) and on invalid
+# values, and `docker compose up -d` still reports success when the container
+# then crash-loops, so the deploy cannot catch it. Running the pinned image
+# against the committed files here can. On a Renovate image bump, this is what
+# fails when the new release removed a key the config still uses. Other
+# renamed or dropped keys are ignored without a warning (old configs' unread
+# `ip_prefixes` left no prefix configured, which only failed indirectly), so
+# the upgrade checklist's config-example.yaml diff is still needed.
+#
+# The repo is public, so the server's hostname is not in the config: on tower
+# it arrives through the stack's .env (HEADSCALE_SERVER_URL and
+# HEADSCALE_TLS_LETSENCRYPT_HOSTNAME). This check supplies a placeholder the
+# same way, and fails if a hostname is committed into the file instead.
+#
+# Needs docker. The container gets no network and a throwaway data directory.
+#
+# Usage: ci/headscale-check.sh [ROOT]
+
+set -euo pipefail
+
+ROOT="${1:-.}"
+cd "$ROOT"
+
+stack=tower/containers/headscale
+image=$(awk '$1 == "image:" { print $2; exit }' "$stack/docker-compose.yml")
+if [ -z "$image" ]; then
+  echo "no image: line in $stack/docker-compose.yml" >&2
+  exit 1
+fi
+
+config="$stack/config/config.yaml"
+if grep -nE '^[[:space:]]*(server_url|tls_letsencrypt_hostname)[[:space:]]*:' "$config"; then
+  echo "$config sets the hostname; it must come from the stack's .env (see the file header)" >&2
+  exit 1
+fi
+
+placeholder=headscale.example.com
+
+# Extra `-e KEY=VALUE` pairs may come first; the rest is headscale's argv.
+headscale() {
+  local env=()
+  while [ "${1:-}" = -e ]; do
+    env+=("$1" "$2")
+    shift 2
+  done
+  docker run --rm --network none --read-only \
+    --tmpfs /var/run/headscale \
+    --tmpfs /var/lib/headscale \
+    -e HEADSCALE_SERVER_URL="https://$placeholder" \
+    -e HEADSCALE_TLS_LETSENCRYPT_HOSTNAME="$placeholder" \
+    "${env[@]}" \
+    -v "$PWD/$stack/config:/etc/headscale:ro" \
+    "$image" "$@"
+}
+
+echo "=== headscale configtest ($image)"
+headscale configtest
+
+# MagicDNS and the ACME email are also switched on through .env when their
+# secrets are set (tower/ansible/roles/system/headscale). Check that shape
+# too, with placeholders.
+echo "=== headscale configtest, MagicDNS + ACME email via env"
+headscale \
+  -e HEADSCALE_ACME_EMAIL=ops@example.com \
+  -e HEADSCALE_DNS_MAGIC_DNS=true \
+  -e HEADSCALE_DNS_BASE_DOMAIN=tailnet.internal \
+  configtest
+
+# The bypass flag opens a fresh SQLite database in the tmpfs instead of
+# dialling a running server; --force answers its "is headscale running?"
+# prompt.
+echo "=== headscale policy check"
+headscale policy check --bypass-grpc-and-access-database-directly --force \
+  -f /etc/headscale/policy.hujson

ci/validate.sh

@@ -7,6 +7,7 @@
 # Requirements:
 #   ansible-lint, yamllint, zizmor   (pip, or via the repo venv)
 #   opentofu >= 1.12                 (tofu)
+#   docker                           (headscale config check)
 #
 # Usage: ci/validate.sh [ROOT]
 
@@ -64,6 +65,14 @@ else
   status=1
 fi
 
+# 5. Headscale config and policy, checked by the image the compose file pins.
+if have docker; then
+  run "headscale config" ci/headscale-check.sh
+else
+  printf '\nSKIP: headscale config (docker not installed)\n'
+  status=1
+fi
+
 printf '\n'
 if [ "$status" -eq 0 ]; then
   echo "all checks passed"

renovate.json

@@ -46,6 +46,15 @@
       "matchPackageNames": ["opentofu/opentofu"],
       "groupName": "OpenTofu core version"
     },
+    {
+      "description": "Headscale (0.29+) refuses to start after skipping or downgrading a minor version, and migrates its database on start. One PR per minor so they can be merged in order; the checklist rides in each PR body.",
+      "matchDatasources": ["docker"],
+      "matchPackageNames": ["headscale/headscale"],
+      "separateMultipleMinor": true,
+      "prBodyNotes": [
+        "**Headscale upgrade** — see `tower/docs/headscale-setup.md#upgrading`. Merge minor versions one at a time, oldest first. Read the release notes for removed config keys (the `Headscale config` check runs this image against the committed config) and for the minimum Tailscale client version. The deploy snapshots the database first; rolling back a minor upgrade means restoring that snapshot."
+      ]
+    },
     {
       "description": "Compare gpt-load release candidate tags by their numeric RC suffix.",
       "matchDatasources": ["docker"],

tower/ansible/playbook.yml

@@ -12,6 +12,7 @@
     - role: system/migrations
     - role: system/config
     - role: system/docker-login
+    - role: system/headscale
     - role: system/containers
   collections:
     - devsec.hardening
@@ -27,3 +28,10 @@
           ansible.builtin.include_role:
             name: devsec.hardening.ssh_hardening
           when: scheduled_run
+  post_tasks:
+    # After system/containers has recreated the stack, so the old container no
+    # longer bind-mounts the hand-written config.
+    - name: Retire the hand-written headscale config
+      ansible.builtin.include_role:
+        name: system/headscale
+        tasks_from: retire_legacy_config

tower/ansible/roles/system/containers/defaults/main.yml

@@ -7,5 +7,5 @@ docker_stacks:
     env_file: true
     state: present
   headscale:
-    env_file: false
+    env_file: true
     state: present

tower/ansible/roles/system/headscale/defaults/main.yml

@@ -0,0 +1,29 @@
+---
+# Bare public hostname of tower, e.g. "host.example.com". Kept out of the
+# public repo: it comes from the TOWER_HOSTNAME environment secret (the same
+# one OpenTofu uses for the DNS record). Unset = the headscale stack is left
+# alone, see tasks/main.yml.
+headscale_hostname: "{{ lookup('env', 'TOWER_HOSTNAME') }}"
+
+# Optional, also from environment secrets so nothing identifying is committed.
+# ACME account contact; Let's Encrypt no longer sends expiry reminders, so it
+# only matters for account recovery.
+headscale_acme_email: "{{ lookup('env', 'TOWER_ACME_EMAIL') }}"
+# MagicDNS base domain, e.g. "tailnet.internal". Setting it turns MagicDNS on.
+headscale_dns_base_domain: "{{ lookup('env', 'TAILNET_BASE_DOMAIN') }}"
+
+headscale_stack_dir: /opt/containers/headscale
+# Compose names it <project>-<service>-<index>; the project is the stack dir.
+headscale_container_name: headscale-headscale-1
+# Where the config was hand-written before it moved to git (bind-mounted as a
+# single file by the old compose). See tasks/retire_legacy_config.yml.
+headscale_legacy_config: "{{ headscale_stack_dir }}/config.yaml"
+# The committed config directory on the controller, fingerprinted into .env so
+# a config or policy change always recreates the container.
+headscale_config_src: "{{ playbook_dir }}/../containers/headscale/config"
+headscale_data_dir: "{{ headscale_stack_dir }}/data"
+headscale_backup_dir: "{{ headscale_stack_dir }}/backups"
+# Snapshots kept on the node. Deploys run at least twice a day (the
+# scheduled Ansible run, plus the one chained after the scheduled OpenTofu
+# apply), so 14 covers roughly a week. Must be at least 1.
+headscale_backup_keep: 14