diff --git a/.github/workflows/tower-ansible-deploy.yml b/.github/workflows/tower-ansible-deploy.yml index 99dac937..a76bb300 100644 --- a/.github/workflows/tower-ansible-deploy.yml +++ b/.github/workflows/tower-ansible-deploy.yml @@ -88,5 +88,8 @@ jobs: GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }} SCHEDULED_RUN: ${{ github.event_name == 'schedule' || github.event.inputs.force_schedule_mode == 'true' }} DNS_SEARCH_DOMAIN: ${{ secrets.DNS_SEARCH_DOMAIN }} + TOWER_HOSTNAME: ${{ secrets.TOWER_HOSTNAME }} + TOWER_ACME_EMAIL: ${{ secrets.TOWER_ACME_EMAIL }} + TAILNET_BASE_DOMAIN: ${{ secrets.TAILNET_BASE_DOMAIN }} run: | uv run ansible-playbook -i inventory tower/ansible/playbook.yml --private-key ~/.ssh/id_ed25519 diff --git a/.github/workflows/tower-tofu-apply.yml b/.github/workflows/tower-tofu-apply.yml index 9e7b4535..52f3f490 100644 --- a/.github/workflows/tower-tofu-apply.yml +++ b/.github/workflows/tower-tofu-apply.yml @@ -35,6 +35,7 @@ jobs: TF_VAR_ssh_authorized_keys: ${{ secrets.SSH_AUTHORIZED_KEYS }} TF_VAR_cloudflare_api_token: ${{ secrets.CLOUDFLARE_API_TOKEN }} TF_VAR_cloudflare_zone_id: ${{ secrets.CLOUDFLARE_ZONE_ID }} + TF_VAR_tower_hostname: ${{ secrets.TOWER_HOSTNAME }} steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 419ed53c..810a4fbd 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -125,6 +125,56 @@ jobs: - name: Check formatting run: tofu fmt -check -recursive -diff + headscale: + name: Headscale config + runs-on: ubuntu-latest + steps: + # Depth 2: on pull_request the checkout is GitHub's test merge commit, + # and the diff below needs its first parent (the base branch). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 2 + + # Only pull requests that touch what the check reads run it: the config + # and policy, the compose file (it pins the image, so Renovate bumps + # count), the script, and this workflow. The job still runs and reports + # success otherwise, so it can stay a required status check; a + # workflow-level `paths:` filter would leave it pending instead. Merge + # queue and manual runs always check. + - name: Detect Headscale changes + id: changes + env: + EVENT: ${{ github.event_name }} + run: | + if [ "$EVENT" != pull_request ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + # The test merge commit's first parent is the base branch, so this + # is exactly what merging the pull request would change. + changed=$(git diff --name-only HEAD^1 HEAD -- \ + tower/containers/headscale \ + ci/headscale-check.sh \ + .github/workflows/validate.yml) + if [ -n "$changed" ]; then + printf 'Changed:\n%s\n' "$changed" + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "No Headscale config, image or check changes; skipping." + echo "run=false" >> "$GITHUB_OUTPUT" + fi + + # Runs the image pinned in tower/containers/headscale/docker-compose.yml + # against the committed config and policy, so a Renovate bump that drops + # a config key fails here rather than crash-looping on tower. + # + # The image reference comes from the PR, like the galaxy sources above. + # The job holds no credentials and the container gets no network. + - name: Check config and policy with the pinned image + if: steps.changes.outputs.run == 'true' + run: ci/headscale-check.sh + workflows: name: Workflow lint runs-on: ubuntu-latest diff --git a/CLAUDE.md b/CLAUDE.md index b7a62e22..0cc45e23 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This repo manages infrastructure for multiple hosts using Ansible + Docker Compo **Ansible hosts** (`mx1`, `web1`, `web2`, `web3`): each top-level directory contains `ansible/` (playbooks and roles) and `containers/` (Docker Compose stacks). -**OpenTofu hosts** (`tower`): each top-level directory contains `terraform/` (`.tf` files) and `docs/` (node-specific setup guide). Run `tofu` commands from the `terraform/` subdirectory. Variables are passed via `TF_VAR_*` env vars; the S3 backend requires `-backend-config` flags at `init` time — see `tower/docs/setup.md`. OpenTofu version: **1.12.x** (pinned in `.github/workflows/tower-tofu-apply.yml`). Note: `error()` in expressions requires ≥ 1.9. When updating providers, regenerate the lock file with `tofu providers lock -platform=linux_amd64` so CI hashes are included. +**OpenTofu hosts** (`tower`): each top-level directory contains `terraform/` (`.tf` files) and `docs/` (node-specific setup guide). `tower` is also configured by Ansible: it has `ansible/` and `containers/` like the other hosts, deployed after each OpenTofu apply. Run `tofu` commands from the `terraform/` subdirectory. Variables are passed via `TF_VAR_*` env vars; the S3 backend requires `-backend-config` flags at `init` time — see `tower/docs/setup.md`. OpenTofu version: **1.13.x** (pinned in `.github/workflows/tower-tofu-apply.yml`; Renovate bumps it together with `required_version`). Note: `error()` in expressions requires ≥ 1.9. When updating providers, regenerate the lock file with `tofu providers lock -platform=linux_amd64` so CI hashes are included. ## Running Playbooks @@ -38,6 +38,10 @@ docker_stacks: ## Conventions +- **The repository is public.** Do not commit identifying details: hostnames and domains of the hosts (tower's comes + from the `TOWER_HOSTNAME` environment secret), names of personal machines or tailnet nodes, what hardware or OS + they run, or account identifiers. This applies to comments, docs, commit messages and PR text too. Inject runtime + values through environment secrets → `.env` / `TF_VAR_*`, and write ``-style placeholders in docs - **No `version:` key** in `docker-compose.yml` (deprecated in Compose V2) - **Always bind ports to localhost**: `127.0.0.1::`; use incremental ports (3001, 3002, ...) per host - **Prefer `.env` files** over inline `environment:` blocks; set `env_file: - .env` in compose @@ -93,7 +97,7 @@ security_opt: **When to add `cap_add` back** (always pair with `cap_drop: [ALL]`): - `NET_BIND_SERVICE` — container binds a privileged port (< 1024) directly, e.g. SMTP (25, 465, 587), IMAP (993), HTTPS (443) -- `NET_ADMIN` — VPN/network management (e.g. headscale) +- `NET_ADMIN` — VPN/network management (e.g. a WireGuard or Tailscale *client* container; not headscale, which is only a control server and needs just `NET_BIND_SERVICE`) - `SYS_NICE` — real-time scheduling (e.g. MySQL) - `CHOWN` — init/setup containers that `chown` volume paths on startup @@ -106,7 +110,7 @@ tmpfs: - /run # nginx (PID file) ``` -Applied to: cloudflared, mailflow frontend + backend, n8n runner. +Applied to: cloudflared, mailflow frontend + backend, n8n runner. `headscale` is stateful but also runs `read_only: true`: it writes only to its data volume and a `/var/run/headscale` tmpfs. `calcom` is **not** read-only — its entrypoint installs NPM packages and writes a build cache on startup, which breaks under `read_only: true`. Keep `cap_drop: [ALL]` + `no-new-privileges:true`, omit `read_only`/`tmpfs`. @@ -126,8 +130,8 @@ Applied to: cloudflared, mailflow frontend + backend, n8n runner. > worker in one stack and an ordinary hardened twenty.crm worker in another, so > "the worker is exempt" is not a safe rule of thumb. And three services > deliberately publish a port on all interfaces: the MX (it exists to receive -> SMTP), headscale (WireGuard endpoint), frankenphp (terminates ACME -> HTTP-01). Anything else must bind `127.0.0.1`. +> SMTP), headscale (tailnet control plane + DERP on 443/tcp, STUN on +> 3478/udp), frankenphp (terminates ACME HTTP-01). Anything else must bind `127.0.0.1`. ## Adding a Service @@ -139,9 +143,16 @@ Applied to: cloudflared, mailflow frontend + backend, n8n runner. ## Validation `ci/validate.sh` runs every check that CI runs: yamllint, ansible-lint, -`tofu validate` / `tofu fmt` for `tower`, and a zizmor audit of the validation -workflow. Run it before pushing; the same failures otherwise surface as a red -PR. +`tofu validate` / `tofu fmt` for `tower`, a zizmor audit of the validation +workflow, and `ci/headscale-check.sh`, which runs the headscale image pinned in +`tower/containers/headscale/docker-compose.yml` (`configtest`, `policy check`) +against the committed config and policy and needs docker. Run it before pushing; +the same failures otherwise surface as a red PR. + +Headscale's config and policy live in git at +`tower/containers/headscale/config/` and are mounted read-only. Never tell +anyone to edit them on the node. Operations, upgrades, restore, and why its +users and nodes are not in OpenTofu: `tower/docs/headscale-setup.md`. Configs live in `.yamllint` and `.ansible-lint`. Two things to know when editing them: diff --git a/README.md b/README.md index b19b54bc..fe159a1b 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ The repository currently manages the following hosts: | `web1` | Ansible + Docker | Web/app services | | `web2` | Ansible + Docker | Web/app services | | `web3` | Ansible + Docker | Web/app services | -| `tower` | OpenTofu | ARM VM on OCI — WireGuard VPN endpoint and SSH bastion | +| `tower` | OpenTofu + Ansible + Docker | ARM VM on OCI — Headscale control server (tailnet coordination + DERP relay) | See [`tower/docs/setup.md`](tower/docs/setup.md) for the one-time bootstrap steps for the tower node. @@ -107,10 +107,11 @@ If a migration fails, the entire playbook stops to prevent inconsistent states. ## Validation -Every pull request runs `.github/workflows/validate.yml`, which performs -static checks only. It never connects to a host, reads a state file, or -touches a cloud provider, so a broken change is caught before it can reach -production rather than during a deploy. +Every pull request runs `.github/workflows/validate.yml`. It never connects +to a host, reads a state file, or touches a cloud provider, so a broken change +is caught before it can reach production rather than during a deploy. Every +check is static except the Headscale one, which runs the pinned image with no +network access against files from the checkout. | Check | Tool | Catches | |---|---|---| @@ -118,6 +119,7 @@ production rather than during a deploy. | Ansible | `ansible-lint` | Broken syntax, missing FQCN, non-idempotent commands, unset file modes | | OpenTofu | `tofu validate` / `tofu fmt` | Invalid or unformatted configuration for `tower` | | Actions | `zizmor` | Workflow-level privilege footguns | +| Headscale | `headscale configtest` / `policy check` (`ci/headscale-check.sh`) | Config keys the pinned image rejects (which would crash-loop the container on tower), invalid policy | ### Running the checks locally @@ -125,7 +127,7 @@ production rather than during a deploy. ci/validate.sh ``` -Needs `ansible-lint`, `yamllint`, `zizmor` and `tofu` on `PATH`. The script +Needs `ansible-lint`, `yamllint`, `zizmor`, `tofu` and `docker` on `PATH`. The script runs the same checks as CI, so failures reproduce locally. ### The container security contract is not enforced @@ -147,7 +149,7 @@ twenty.crm worker in another. ### Why the validation workflow holds no secrets `validate.yml` runs on `pull_request`, so it is configured to hold no -credentials at all — the checks are entirely static analysis: +credentials at all — the checks only need the checkout: - It references no `secrets.*` and no `environment:`, so the job runs with no access to repository or environment secrets regardless of who opened the @@ -158,8 +160,13 @@ credentials at all — the checks are entirely static analysis: - Third-party actions are pinned to a full commit SHA, fixing the code that runs with this job's token. -All four checks are required status checks on `main`, so a pull request cannot -merge until they pass. If a check is retired or renamed, remove it from the +The yamllint, ansible-lint, OpenTofu and workflow-lint checks are required +status checks on `main`, so a pull request cannot merge until they pass. Add +`Headscale config` to that list too, so that a Renovate bump that breaks the +config cannot merge. On a pull request that touches none of its inputs (the +Headscale stack directory, `ci/headscale-check.sh`, `validate.yml`) it skips +the check but still reports success, so requiring it does not block other +pull requests. If a check is retired or renamed, remove it from the `required_status_checks.contexts` list in the same pull request — otherwise the merge blocks waiting on a check that no longer reports. diff --git a/ci/headscale-check.sh b/ci/headscale-check.sh new file mode 100755 index 00000000..7beea426 --- /dev/null +++ b/ci/headscale-check.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Check tower's committed Headscale config and policy with the exact image the +# compose file pins. +# +# Headscale refuses to start on keys its deprecation list marks as removed +# (`dns_config` in 0.23, `randomize_client_port` in 0.29) and on invalid +# values, and `docker compose up -d` still reports success when the container +# then crash-loops, so the deploy cannot catch it. Running the pinned image +# against the committed files here can. On a Renovate image bump, this is what +# fails when the new release removed a key the config still uses. Other +# renamed or dropped keys are ignored without a warning (old configs' unread +# `ip_prefixes` left no prefix configured, which only failed indirectly), so +# the upgrade checklist's config-example.yaml diff is still needed. +# +# The repo is public, so the server's hostname is not in the config: on tower +# it arrives through the stack's .env (HEADSCALE_SERVER_URL and +# HEADSCALE_TLS_LETSENCRYPT_HOSTNAME). This check supplies a placeholder the +# same way, and fails if a hostname is committed into the file instead. +# +# Needs docker. The container gets no network and a throwaway data directory. +# +# Usage: ci/headscale-check.sh [ROOT] + +set -euo pipefail + +ROOT="${1:-.}" +cd "$ROOT" + +stack=tower/containers/headscale +image=$(awk '$1 == "image:" { print $2; exit }' "$stack/docker-compose.yml") +if [ -z "$image" ]; then + echo "no image: line in $stack/docker-compose.yml" >&2 + exit 1 +fi + +config="$stack/config/config.yaml" +if grep -nE '^[[:space:]]*(server_url|tls_letsencrypt_hostname)[[:space:]]*:' "$config"; then + echo "$config sets the hostname; it must come from the stack's .env (see the file header)" >&2 + exit 1 +fi + +placeholder=headscale.example.com + +# Extra `-e KEY=VALUE` pairs may come first; the rest is headscale's argv. +headscale() { + local env=() + while [ "${1:-}" = -e ]; do + env+=("$1" "$2") + shift 2 + done + docker run --rm --network none --read-only \ + --tmpfs /var/run/headscale \ + --tmpfs /var/lib/headscale \ + -e HEADSCALE_SERVER_URL="https://$placeholder" \ + -e HEADSCALE_TLS_LETSENCRYPT_HOSTNAME="$placeholder" \ + "${env[@]}" \ + -v "$PWD/$stack/config:/etc/headscale:ro" \ + "$image" "$@" +} + +echo "=== headscale configtest ($image)" +headscale configtest + +# MagicDNS and the ACME email are also switched on through .env when their +# secrets are set (tower/ansible/roles/system/headscale). Check that shape +# too, with placeholders. +echo "=== headscale configtest, MagicDNS + ACME email via env" +headscale \ + -e HEADSCALE_ACME_EMAIL=ops@example.com \ + -e HEADSCALE_DNS_MAGIC_DNS=true \ + -e HEADSCALE_DNS_BASE_DOMAIN=tailnet.internal \ + configtest + +# The bypass flag opens a fresh SQLite database in the tmpfs instead of +# dialling a running server; --force answers its "is headscale running?" +# prompt. +echo "=== headscale policy check" +headscale policy check --bypass-grpc-and-access-database-directly --force \ + -f /etc/headscale/policy.hujson diff --git a/ci/validate.sh b/ci/validate.sh index 5bdbcc6b..0d75e29b 100755 --- a/ci/validate.sh +++ b/ci/validate.sh @@ -7,6 +7,7 @@ # Requirements: # ansible-lint, yamllint, zizmor (pip, or via the repo venv) # opentofu >= 1.12 (tofu) +# docker (headscale config check) # # Usage: ci/validate.sh [ROOT] @@ -64,6 +65,14 @@ else status=1 fi +# 5. Headscale config and policy, checked by the image the compose file pins. +if have docker; then + run "headscale config" ci/headscale-check.sh +else + printf '\nSKIP: headscale config (docker not installed)\n' + status=1 +fi + printf '\n' if [ "$status" -eq 0 ]; then echo "all checks passed" diff --git a/renovate.json b/renovate.json index b46485d6..672c6bb4 100644 --- a/renovate.json +++ b/renovate.json @@ -46,6 +46,15 @@ "matchPackageNames": ["opentofu/opentofu"], "groupName": "OpenTofu core version" }, + { + "description": "Headscale (0.29+) refuses to start after skipping or downgrading a minor version, and migrates its database on start. One PR per minor so they can be merged in order; the checklist rides in each PR body.", + "matchDatasources": ["docker"], + "matchPackageNames": ["headscale/headscale"], + "separateMultipleMinor": true, + "prBodyNotes": [ + "**Headscale upgrade** — see `tower/docs/headscale-setup.md#upgrading`. Merge minor versions one at a time, oldest first. Read the release notes for removed config keys (the `Headscale config` check runs this image against the committed config) and for the minimum Tailscale client version. The deploy snapshots the database first; rolling back a minor upgrade means restoring that snapshot." + ] + }, { "description": "Compare gpt-load release candidate tags by their numeric RC suffix.", "matchDatasources": ["docker"], diff --git a/tower/ansible/playbook.yml b/tower/ansible/playbook.yml index fbaefd95..dc37c998 100644 --- a/tower/ansible/playbook.yml +++ b/tower/ansible/playbook.yml @@ -12,6 +12,7 @@ - role: system/migrations - role: system/config - role: system/docker-login + - role: system/headscale - role: system/containers collections: - devsec.hardening @@ -27,3 +28,10 @@ ansible.builtin.include_role: name: devsec.hardening.ssh_hardening when: scheduled_run + post_tasks: + # After system/containers has recreated the stack, so the old container no + # longer bind-mounts the hand-written config. + - name: Retire the hand-written headscale config + ansible.builtin.include_role: + name: system/headscale + tasks_from: retire_legacy_config diff --git a/tower/ansible/roles/system/containers/defaults/main.yml b/tower/ansible/roles/system/containers/defaults/main.yml index b05238e0..014a8375 100644 --- a/tower/ansible/roles/system/containers/defaults/main.yml +++ b/tower/ansible/roles/system/containers/defaults/main.yml @@ -7,5 +7,5 @@ docker_stacks: env_file: true state: present headscale: - env_file: false + env_file: true state: present diff --git a/tower/ansible/roles/system/headscale/defaults/main.yml b/tower/ansible/roles/system/headscale/defaults/main.yml new file mode 100644 index 00000000..67f21705 --- /dev/null +++ b/tower/ansible/roles/system/headscale/defaults/main.yml @@ -0,0 +1,29 @@ +--- +# Bare public hostname of tower, e.g. "host.example.com". Kept out of the +# public repo: it comes from the TOWER_HOSTNAME environment secret (the same +# one OpenTofu uses for the DNS record). Unset = the headscale stack is left +# alone, see tasks/main.yml. +headscale_hostname: "{{ lookup('env', 'TOWER_HOSTNAME') }}" + +# Optional, also from environment secrets so nothing identifying is committed. +# ACME account contact; Let's Encrypt no longer sends expiry reminders, so it +# only matters for account recovery. +headscale_acme_email: "{{ lookup('env', 'TOWER_ACME_EMAIL') }}" +# MagicDNS base domain, e.g. "tailnet.internal". Setting it turns MagicDNS on. +headscale_dns_base_domain: "{{ lookup('env', 'TAILNET_BASE_DOMAIN') }}" + +headscale_stack_dir: /opt/containers/headscale +# Compose names it --; the project is the stack dir. +headscale_container_name: headscale-headscale-1 +# Where the config was hand-written before it moved to git (bind-mounted as a +# single file by the old compose). See tasks/retire_legacy_config.yml. +headscale_legacy_config: "{{ headscale_stack_dir }}/config.yaml" +# The committed config directory on the controller, fingerprinted into .env so +# a config or policy change always recreates the container. +headscale_config_src: "{{ playbook_dir }}/../containers/headscale/config" +headscale_data_dir: "{{ headscale_stack_dir }}/data" +headscale_backup_dir: "{{ headscale_stack_dir }}/backups" +# Snapshots kept on the node. Deploys run at least twice a day (the +# scheduled Ansible run, plus the one chained after the scheduled OpenTofu +# apply), so 14 covers roughly a week. Must be at least 1. +headscale_backup_keep: 14 diff --git a/tower/ansible/roles/system/headscale/tasks/main.yml b/tower/ansible/roles/system/headscale/tasks/main.yml new file mode 100644 index 00000000..57033483 --- /dev/null +++ b/tower/ansible/roles/system/headscale/tasks/main.yml @@ -0,0 +1,205 @@ +--- +# Runs before system/containers. That role applies image bumps, and headscale +# migrates its database on start and then refuses to run the previous minor +# version, so the snapshot taken here is the only rollback point for a bad +# upgrade. Restore steps: tower/docs/headscale-setup.md. +# +# It also writes the stack's .env, which carries the public hostname so that +# it never has to be committed. system/containers deploys the stack only when +# that file exists (env_file: true in docker_stacks). + +- name: Ensure sqlite3 is installed + ansible.builtin.package: + name: sqlite3 + state: present + +- name: Ensure headscale data and backup directories exist + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0700" + loop: + - "{{ headscale_data_dir }}" + - "{{ headscale_backup_dir }}" + +- name: Check the headscale hostname + ansible.builtin.assert: + that: + - headscale_hostname is match('^([a-z0-9]([a-z0-9-]*[a-z0-9])?\\.)+[a-z]{2,}$') + fail_msg: TOWER_HOSTNAME must be a bare lower-case FQDN, with no scheme, port or path. + quiet: true + when: headscale_hostname | length > 0 + +# Caught here because headscale would otherwise crash-loop on them, and the +# deploy cannot see a crash loop. Messages never print the values (public logs). +- name: Check the optional headscale settings + ansible.builtin.assert: + that: + - headscale_acme_email | length == 0 or headscale_acme_email is match('^[^@\\s]+@[^@\\s]+\\.[^@\\s]+$') + - headscale_dns_base_domain | length == 0 or headscale_dns_base_domain is match('^([a-z0-9]([a-z0-9-]*[a-z0-9])?\\.)+[a-z]{2,}$') + # Headscale refuses a base domain that is server_url's host or a parent of it. + - headscale_dns_base_domain | length == 0 or headscale_hostname != headscale_dns_base_domain + - headscale_dns_base_domain | length == 0 or not headscale_hostname.endswith('.' ~ headscale_dns_base_domain) + fail_msg: >- + TOWER_ACME_EMAIL must be an email address; TAILNET_BASE_DOMAIN must be a + bare lower-case FQDN that is not tower's hostname or a parent of it. + quiet: true + when: headscale_hostname | length > 0 + +# Fingerprint of the committed config/ directory, written into .env below. +# Headscale reads config.yaml and policy.hujson only at start, and a change to +# a bind-mounted file does not make compose recreate the container; only the +# shared "Restart containers" handler would apply it. If a run dies before +# handlers fire, the retry copies nothing new and never notifies, leaving the +# old policy loaded. A changed environment does make `compose up` recreate, +# so this converges on every run regardless of handlers. +- name: Fingerprint the committed headscale config + ansible.builtin.find: + paths: "{{ headscale_config_src }}" + get_checksum: true + delegate_to: localhost + become: false + register: headscale_config_files + +- name: Write the headscale .env + ansible.builtin.copy: + content: | + HEADSCALE_SERVER_URL=https://{{ headscale_hostname }} + HEADSCALE_TLS_LETSENCRYPT_HOSTNAME={{ headscale_hostname }} + {% if headscale_acme_email | length > 0 %} + HEADSCALE_ACME_EMAIL={{ headscale_acme_email }} + {% endif %} + {% if headscale_dns_base_domain | length > 0 %} + HEADSCALE_DNS_MAGIC_DNS=true + HEADSCALE_DNS_BASE_DOMAIN={{ headscale_dns_base_domain }} + {% endif %} + CONFIG_SHA256={{ headscale_config_files.files | sort(attribute='path') | map(attribute='checksum') | join(',') | hash('sha256') }} + dest: "{{ headscale_stack_dir }}/.env" + owner: root + group: root + mode: "0600" + when: headscale_hostname | length > 0 + +- name: Check for the headscale .env + ansible.builtin.stat: + path: "{{ headscale_stack_dir }}/.env" + register: headscale_env + +- name: Report that the headscale stack will not be deployed + ansible.builtin.debug: + msg: >- + TOWER_HOSTNAME is not set and {{ headscale_stack_dir }}/.env does not + exist, so system/containers skips the headscale stack. + when: not headscale_env.stat.exists + +- name: Check for an existing headscale database + ansible.builtin.stat: + path: "{{ headscale_data_dir }}/db.sqlite" + register: headscale_db + +# A snapshot is built under a hidden .partial name and renamed only once the +# database check and key copy have succeeded, so an interrupted or failed +# attempt can never be counted as one of the kept snapshots below. +- name: Find interrupted headscale snapshots + ansible.builtin.find: + paths: "{{ headscale_backup_dir }}" + file_type: directory + hidden: true + patterns: ".*.partial" + register: headscale_partial_snapshots + +- name: Remove interrupted headscale snapshots + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ headscale_partial_snapshots.files }}" + loop_control: + label: "{{ item.path | basename }}" + +- name: Snapshot headscale state + when: headscale_db.stat.exists + block: + # set_fact, not vars: now() is re-evaluated on every reference. + - name: Pick the snapshot directory + ansible.builtin.set_fact: + headscale_snapshot_name: "{{ now(utc=true, fmt='%Y%m%dT%H%M%SZ') }}" + + - name: Name the in-progress snapshot directory + ansible.builtin.set_fact: + headscale_snapshot_dir: "{{ headscale_backup_dir }}/.{{ headscale_snapshot_name }}.partial" + + - name: Create the snapshot directory + ansible.builtin.file: + path: "{{ headscale_snapshot_dir }}" + state: directory + owner: root + group: root + mode: "0700" + + # SQLite's online backup API: consistent while headscale is running and + # writing through the WAL. A file copy of db.sqlite would not be. + - name: Back up the headscale database + ansible.builtin.command: + argv: + - sqlite3 + - "{{ headscale_data_dir }}/db.sqlite" + - ".backup '{{ headscale_snapshot_dir }}/db.sqlite'" + creates: "{{ headscale_snapshot_dir }}/db.sqlite" + + # Fail the deploy rather than upgrade on top of a database we could not + # roll back to. + - name: Verify the database snapshot + ansible.builtin.command: + argv: + - sqlite3 + - "{{ headscale_snapshot_dir }}/db.sqlite" + - "PRAGMA quick_check;" + register: headscale_snapshot_check + changed_when: false + failed_when: headscale_snapshot_check.stdout != 'ok' + + # The Noise key identifies the server to every client: losing it means + # re-registering every node. The DERP key exists only if DERP ever ran. + - name: Find headscale private keys + ansible.builtin.find: + paths: "{{ headscale_data_dir }}" + patterns: "*.key" + register: headscale_keys + + - name: Back up headscale private keys + ansible.builtin.copy: + src: "{{ item.path }}" + dest: "{{ headscale_snapshot_dir }}/{{ item.path | basename }}" + remote_src: true + owner: root + group: root + mode: "0600" + loop: "{{ headscale_keys.files }}" + loop_control: + label: "{{ item.path | basename }}" + + - name: Publish the completed snapshot + ansible.builtin.command: + argv: + - mv + - "{{ headscale_snapshot_dir }}" + - "{{ headscale_backup_dir }}/{{ headscale_snapshot_name }}" + creates: "{{ headscale_backup_dir }}/{{ headscale_snapshot_name }}" + +- name: Find headscale snapshots + ansible.builtin.find: + paths: "{{ headscale_backup_dir }}" + file_type: directory + patterns: "*T*Z" + register: headscale_snapshots + +# Names are UTC timestamps, so sorting by path is sorting by age. +- name: Prune old headscale snapshots + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ (headscale_snapshots.files | sort(attribute='path'))[: -(headscale_backup_keep | int)] }}" + loop_control: + label: "{{ item.path | basename }}" diff --git a/tower/ansible/roles/system/headscale/tasks/retire_legacy_config.yml b/tower/ansible/roles/system/headscale/tasks/retire_legacy_config.yml new file mode 100644 index 00000000..882f4a06 --- /dev/null +++ b/tower/ansible/roles/system/headscale/tasks/retire_legacy_config.yml @@ -0,0 +1,24 @@ +--- +# Run from the playbook's post_tasks, after system/containers. Before this role +# existed, the config was hand-written at the stack root and bind-mounted as a +# single file. Renaming it while a container still mounts it breaks that +# container's next start (dockerd recreates the missing path as an empty +# directory), so rename it only once the running container no longer +# references it. It is kept, under a name nothing mounts, for comparison. + +- name: Inspect the headscale container + community.docker.docker_container_info: + name: "{{ headscale_container_name }}" + register: headscale_container + +- name: Retire the hand-written headscale config + ansible.builtin.command: + argv: + - mv + - "{{ headscale_legacy_config }}" + - "{{ headscale_legacy_config }}.pre-git" + creates: "{{ headscale_legacy_config }}.pre-git" + removes: "{{ headscale_legacy_config }}" + when: + - headscale_container.exists + - headscale_legacy_config not in (headscale_container.container.Mounts | map(attribute='Source') | list) diff --git a/tower/containers/headscale/config/config.yaml b/tower/containers/headscale/config/config.yaml new file mode 100644 index 00000000..254349b6 --- /dev/null +++ b/tower/containers/headscale/config/config.yaml @@ -0,0 +1,136 @@ +--- +# Headscale server configuration for tower. +# +# Ansible copies this directory to /opt/containers/headscale/config/ and the +# compose file mounts it read-only at /etc/headscale. It holds no secrets: +# the Noise/DERP private keys and the SQLite database are generated into +# /opt/containers/headscale/data on first start and never enter git. +# +# Anything identifying is deliberately not in this file (the repo is public). +# Headscale reads any key from HEADSCALE_ environment variables, and the +# system/headscale role writes the stack's .env from environment secrets: +# TOWER_HOSTNAME -> HEADSCALE_SERVER_URL=https:// +# HEADSCALE_TLS_LETSENCRYPT_HOSTNAME= +# TOWER_ACME_EMAIL -> HEADSCALE_ACME_EMAIL (optional) +# TAILNET_BASE_DOMAIN -> MagicDNS on, HEADSCALE_DNS_BASE_DOMAIN (optional) +# Without the hostname headscale refuses to start (server_url must be +# https://...). +# +# Written against headscale v0.29.x. On every minor upgrade, diff this file +# against that release's config-example.yaml. Keys headscale lists as removed +# make it refuse to start (CI's `headscale configtest` job catches those), but +# any other renamed or dropped key is silently ignored. +# https://github.com/juanfont/headscale/blob/v0.29.4/config-example.yaml + +# Control plane, Noise and embedded DERP are all served on 443 inside the +# container. Metrics and gRPC stay on the container's loopback; the CLI talks +# to the unix socket instead. +listen_addr: 0.0.0.0:443 +metrics_listen_addr: 127.0.0.1:9090 +grpc_listen_addr: 127.0.0.1:50443 +grpc_allow_insecure: false + +# Nothing proxies headscale, so never honour forwarded-for headers. +trusted_proxies: [] + +# Every headscale CLI call, including the 30-second `headscale health` +# healthcheck, otherwise asks GitHub for the latest release (thousands of +# unauthenticated API calls a day). Renovate tracks releases instead. +disable_check_updates: true + +noise: + private_key_path: /var/lib/headscale/noise_private.key + +prefixes: + v4: 100.64.0.0/10 + v6: fd7a:115c:a1e0::/48 + allocation: sequential + +# Self-hosted relay only: no Tailscale-operated DERP servers in the map. If +# tower is unreachable, peers that cannot connect directly lose connectivity. +# Add https://controlplane.tailscale.com/derpmap/default to `urls` to fall +# back to Tailscale's relays (traffic stays end-to-end encrypted). +derp: + server: + enabled: true + # 900-999 is the range Tailscale reserves for self-hosted DERP regions + # (tailcfg/derpmap.go), so this cannot collide with one of theirs if their + # map is ever added to `urls` as a fallback. + region_id: 969 + region_code: tower + region_name: eu-1 + # Only nodes registered on this server may relay through it. + verify_clients: true + stun_listen_addr: 0.0.0.0:3478 + private_key_path: /var/lib/headscale/derp_server_private.key + automatically_add_embedded_derp_region: true + urls: [] + paths: [] + auto_update_enabled: false + update_frequency: 3h + +node: + # Untagged nodes never expire unless expired by hand. The tailnet is a few + # always-on servers registered with single-use keys. + expiry: 0 + ephemeral: + inactivity_timeout: 30m + +database: + type: sqlite + sqlite: + path: /var/lib/headscale/db.sqlite + write_ahead_log: true + wal_autocheckpoint: 1000 + +# TLS-ALPN-01 is answered on listen_addr, so no port 80 is needed. The DNS +# record must resolve to tower before the first start. The hostname comes +# from HEADSCALE_TLS_LETSENCRYPT_HOSTNAME (see the top of this file). +acme_url: https://acme-v02.api.letsencrypt.org/directory +# acme_email comes from HEADSCALE_ACME_EMAIL (see the top of this file). +tls_letsencrypt_cache_dir: /var/lib/headscale/cache +tls_letsencrypt_challenge_type: TLS-ALPN-01 +tls_cert_path: "" +tls_key_path: "" + +log: + level: info + format: text + +# Reloaded on restart or SIGHUP. Edit policy.hujson next to this file. +policy: + mode: file + path: /etc/headscale/policy.hujson + +# MagicDNS is off here and switched on by the role (HEADSCALE_DNS_MAGIC_DNS) +# only when TAILNET_BASE_DOMAIN is set: with it on and no base_domain, +# headscale refuses to start. base_domain must not be server_url's host or a +# parent of it, or headscale refuses to start too. +# +# override_local_dns stays false: nodes keep their own resolvers for every +# name outside the tailnet, and nothing replaces them with ours. A node opts +# out of MagicDNS entirely with `tailscale set --accept-dns=false`. +# +# No search domains and no split or global nameservers. A public search +# domain would need split nameservers: on Linux with systemd-resolved it also +# routes its queries to 100.100.100.100, which otherwise has nowhere to +# forward them, so every name under it would fail with SERVFAIL. global would +# only be used as a fallback while an exit node is in use. +dns: + magic_dns: false + base_domain: "" + override_local_dns: false + nameservers: + global: [] + split: {} + search_domains: [] + extra_records: [] + +unix_socket: /var/run/headscale/headscale.sock +unix_socket_permission: "0770" + +logtail: + enabled: false + +auto_update: + enabled: false diff --git a/tower/containers/headscale/config/policy.hujson b/tower/containers/headscale/config/policy.hujson new file mode 100644 index 00000000..7c2ea39a --- /dev/null +++ b/tower/containers/headscale/config/policy.hujson @@ -0,0 +1,21 @@ +// Tailnet access policy for tower's Headscale (HuJSON: comments and trailing +// commas are allowed). +// +// This is deliberately equivalent to having no policy at all: with no "acls" +// or "grants" section, Headscale lets every node reach every other node on +// every port. It is committed so that tightening access is a reviewed change +// to this file rather than an edit on the server, and so CI can check it with +// `headscale policy check` on every pull request. +// +// A deploy that changes this file restarts headscale, which reloads it. +// +// To restrict access, add grants. Once any "grants" (or "acls") key exists, +// everything not granted is denied — an empty "grants": [] blocks all traffic. +// Example: members may only reach each other over SSH and SMB: +// +// "grants": [ +// { "src": ["autogroup:member"], "dst": ["autogroup:member"], "ip": ["tcp:22", "tcp:445"] }, +// ], +// +// Syntax: https://tailscale.com/docs/features/access-control/grants +{} diff --git a/tower/containers/headscale/docker-compose.yml b/tower/containers/headscale/docker-compose.yml index a130fac5..bcaf9b72 100644 --- a/tower/containers/headscale/docker-compose.yml +++ b/tower/containers/headscale/docker-compose.yml @@ -4,14 +4,31 @@ services: image: headscale/headscale:v0.29.4@sha256:8833f828b414c0907b7e5c71da76473216fe17cce0818a166b536ec552c0903f command: serve restart: unless-stopped + # HEADSCALE_SERVER_URL and HEADSCALE_TLS_LETSENCRYPT_HOSTNAME, written by + # the system/headscale role so the hostname stays out of git. + env_file: + - .env + # Published on all interfaces on purpose: this is the tailnet's public + # control plane + DERP relay (443/tcp) and STUN endpoint (3478/udp). ports: - "443:443/tcp" - "3478:3478/udp" volumes: + # config.yaml + policy.hujson, committed next to this file. + - ./config:/etc/headscale:ro + # Keys, SQLite DB and ACME cache. Not in git; snapshotted by the + # system/headscale role before every deploy. - /opt/containers/headscale/data:/var/lib/headscale - - /opt/containers/headscale/config.yaml:/etc/headscale/config.yaml:ro + # Headscale writes only to its data volume and the unix socket directory. + read_only: true tmpfs: - /var/run/headscale + healthcheck: + test: ["CMD", "headscale", "health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s cap_drop: - ALL cap_add: diff --git a/tower/docs/headscale-nodes.md b/tower/docs/headscale-nodes.md index e03369b3..bf90c72d 100644 --- a/tower/docs/headscale-nodes.md +++ b/tower/docs/headscale-nodes.md @@ -1,45 +1,45 @@ # Headscale Nodes -## Current Nodes +Nodes are machines running the standard Tailscale client against tower's Headscale. Their names, addresses and owners +live in Headscale's database, not in this repository. List them on tower with `headscale nodes list` (see +[headscale-setup.md](headscale-setup.md)). -| Name | Expected VPN IP | Role | -|---|---|---| -| `giraffe` | `100.64.x.x` | Unraid NAS/compute node | -| `cetriolo` | `100.64.x.x` | Unraid NAS/compute node | -| `ostrich` | `100.64.x.x` | Unraid NAS/compute node | - -Actual IPs are assigned by Headscale from the `100.64.0.0/10` CGNAT range and can be confirmed with `headscale nodes list` on tower (see [headscale-setup.md](headscale-setup.md)). +Headscale assigns each node an address from `100.64.0.0/10` (plus one from `fd7a:115c:a1e0::/48`) at registration. --- -## Registering an Unraid Node +## Registering a Node -### 1. Install Tailscale on Unraid +### 1. Install Tailscale -Install the Tailscale plugin via **Unraid Community Apps** (search for "Tailscale"). Alternatively, run the official `tailscale/tailscale` Docker container if you prefer a compose-managed approach. Either method works; the plugin is simpler for Unraid. +Install the Tailscale client for the node's OS from , or run the official +`tailscale/tailscale` container. Headscale 0.29 rejects clients older than **v1.80.0**. Check with `tailscale version`. -### 2. Point Tailscale at the custom control server +### 2. Point Tailscale at the control server -Run the following command in the Unraid terminal (or via the plugin's shell): +Create a single-use pre-auth key on tower ([headscale-setup.md](headscale-setup.md#2-create-a-pre-auth-key-per-node)), +then run this on the node: ```bash -tailscale up --login-server=https:// --authkey= +tailscale up --login-server=https:// --authkey= ``` -- Replace `` with the value from the GitHub secret (the public hostname of tower). -- Replace `` with a key generated on tower (see [headscale-setup.md](headscale-setup.md)). +`` is the value of the `TOWER_HOSTNAME` environment secret. The login server URL is stored by the client, +so it must not change later. -On success, Tailscale will print the assigned VPN IP and the node will appear in `headscale nodes list` on tower. +If MagicDNS is enabled ([headscale-setup.md](headscale-setup.md#magicdns)), the node uses it by default. Add +`--accept-dns=false` to keep this node's DNS untouched; `tailscale set --accept-dns=true` turns it on later. Other nodes +reach it as `.` either way. ### 3. Verify the node is registered -On tower: +On tower, from `/opt/containers/headscale/`: ```bash docker compose exec headscale headscale nodes list ``` -The node should appear with a `100.64.x.x` address and an `Online` status. +The node should be listed with a `100.64.x.x` address and show as online. --- @@ -53,7 +53,7 @@ On any registered node: tailscale status ``` -This lists all peers, their VPN IPs, and whether they are online. +This lists all peers with their VPN IPs and whether they are online. A peer reached through the relay shows `relay "tower"`. ### Ping a peer @@ -61,20 +61,15 @@ This lists all peers, their VPN IPs, and whether they are online. tailscale ping ``` -Successful pings confirm the mesh is functional. - -### Check relay vs. direct (P2P) connection - -To see whether traffic to a peer is going direct (P2P) or through the DERP relay on tower: - -```bash -tailscale debug peer-endpoint-changes -``` +Each reply says how it travelled: `via DERP(tower)` means relayed through tower, `via :` means a direct +peer-to-peer path. -Or run a general connectivity check (shows DERP latency, STUN results, UDP availability): +### Check NAT traversal and DERP reachability ```bash tailscale netcheck ``` -Direct connections are preferred. If all traffic routes through DERP, check that UDP port `3478` is reachable from the nodes and that no NAT is blocking peer-to-peer UDP. +This shows UDP availability, the STUN result and the latency to the `tower` DERP region. Direct connections are +preferred. If everything relays through DERP, check that `3478/udp` on tower is reachable from the node and that the +node's NAT allows peer-to-peer UDP. diff --git a/tower/docs/headscale-setup.md b/tower/docs/headscale-setup.md index 176f3ede..a2a31d7a 100644 --- a/tower/docs/headscale-setup.md +++ b/tower/docs/headscale-setup.md @@ -2,123 +2,277 @@ ## Overview -Tower runs [Headscale](https://github.com/juanfont/headscale) — a self-hosted implementation of the Tailscale control plane — with the built-in DERP relay enabled. Nodes (Unraid servers named `giraffe`, `cetriolo`, `ostrich`) run the standard Tailscale client pointed at this control server. The VPN subnet is `100.64.0.0/10`. - -The Headscale container is defined in `tower/containers/headscale/docker-compose.yml` and managed by Ansible like all other stacks. +Tower runs [Headscale](https://github.com/juanfont/headscale), a self-hosted implementation of the Tailscale control +server, with the embedded DERP relay enabled. Nodes run the standard Tailscale client pointed at +`https://`. Node addresses come from `100.64.0.0/10` and `fd7a:115c:a1e0::/48`. + +This repository is public, so tower's hostname is not in its files. It lives in one GitHub environment secret, +`TOWER_HOSTNAME` (environment `tower`, see [setup.md](setup.md#environment-secrets-tower)), as a bare FQDN with no scheme. +OpenTofu uses it as the DNS record name. Ansible writes it into the stack's `.env` as `HEADSCALE_SERVER_URL` and +`HEADSCALE_TLS_LETSENCRYPT_HOSTNAME`, which Headscale reads in place of the matching config keys. Docs and examples +write it as ``. This keeps the name out of the current tree, not secret. Let's Encrypt publishes it in +Certificate Transparency logs, and any name that was ever committed stays in public git history. To keep the host hard +to find, pick a name that has never been in the repository, before registering clients. + +Who manages what: + +| Piece | Where | Managed by | +|---|---|---| +| VM, security list (`443/tcp`, `3478/udp`), DNS record | `tower/terraform/` | OpenTofu | +| Compose stack | `tower/containers/headscale/docker-compose.yml` | Ansible (`system/containers`) | +| Server config and access policy | `tower/containers/headscale/config/` | Git → Ansible, checked by CI | +| Hostname, ACME email, MagicDNS domains (`/opt/containers/headscale/.env`) | `TOWER_HOSTNAME`, `TOWER_ACME_EMAIL`, `TAILNET_*` secrets | Ansible (`system/headscale`) | +| Pre-deploy snapshots, data dir permissions | `tower/ansible/roles/system/headscale/` | Ansible | +| Keys, SQLite DB, ACME cache | `/opt/containers/headscale/data/` on tower | Headscale (never in git) | +| Users, nodes, pre-auth keys | Headscale's database | CLI on tower (see [below](#why-headscale-objects-are-not-in-opentofu)) | + +Nothing in `config/` is secret. The server's private keys are generated into the data directory on first start. + +If `TOWER_HOSTNAME` is unset and no `.env` exists yet, the deploy skips the Headscale stack (the role logs why) and +leaves any running container alone. Without the two variables Headscale refuses to start (`server_url must start with +https://`). --- -## Prerequisites +## Before promoting to production + +1. Add the `TOWER_HOSTNAME` environment secret **before merging**. Without it the OpenTofu apply stops at + variable validation (an unset secret arrives as an empty string), and the Headscale stack is skipped. +2. Run these on tower, since the stack's live state was never checked: + + ```bash + cd /opt/containers/headscale + docker compose ps # want: Up. "Restarting" = crash loop + docker compose logs --tail 50 + ls -la config.yaml data/ # config.yaml should be a file, not a directory + grep server_url config.yaml # must be https://, the TOWER_HOSTNAME value + ``` + +- If clients were registered against a different `server_url`, set `TOWER_HOSTNAME` to that host instead. Clients store + the login server URL, so changing it orphans every registered node. Changing `TOWER_HOSTNAME` also renames the + OpenTofu-managed DNS record in place. If a record for the new name already exists, delete it or `tofu import` it first, + or the apply fails. +- If `data/db.sqlite` exists and nodes are registered, the deploy snapshots it before switching to the committed config + (see [Backup and restore](#backup-and-restore)). +- Once the stack runs on the new compose (so nothing mounts it any more), the deploy renames the hand-written + `config.yaml` to `config.yaml.pre-git`, so it can't be mistaken for the live file. Diff it against + `config/config.yaml`, then delete it. -Before the first run, verify: +After the deploy, verify from **outside** tower: -- **DNS resolves publicly.** Headscale requests a TLS certificate via ACME on startup. The DNS record for your tower hostname must be live and pointing at the tower IP before the container starts, otherwise ACME will fail and the container will crash-loop. -- **Ports are open on the firewall:** - - `443/tcp` — HTTPS (control plane + HTTPS DERP) - - `3478/udp` — STUN (used by DERP for NAT traversal) +```bash +curl -fsS https:///health # {"status":"pass"}; also proves the Let's Encrypt cert is valid +``` + +From a node: `tailscale netcheck` must report `UDP: true` and `IPv4: yes,
`. The mapped address comes from +tower's STUN server, which proves `3478/udp` is reachable. A `tower` region latency on its own does not: netcheck +measures it over HTTPS when STUN fails. If either check fails while the container is healthy, check the host firewall: OCI's Ubuntu images can ship +iptables rules that reject inbound traffic other than SSH, on top of the security list. --- -## Pre-Deployment: Prepare Config Files Manually +## First start + +DNS must already resolve to tower: Headscale fetches its certificate via ACME (TLS-ALPN-01, on port 443) the first time +a client connects. The record is `cloudflare_dns_record.tower` in `tower/terraform/dns.tf`. -The Headscale config is **not generated by Ansible** — you must create it on tower before running the playbook for the first time. +Nothing has to be created on tower by hand. With `TOWER_HOSTNAME` set, Ansible deploys the config, writes `.env`, +creates `/opt/containers/headscale/data` (mode `0700`) and starts the container. Then, from `/opt/containers/headscale/`: -### 1. Create the directories +### 1. Create a user + +A user owns nodes, roughly a Tailscale account. ```bash -mkdir -p /opt/containers/headscale/data +docker compose exec headscale headscale users create +docker compose exec headscale headscale users list # note the ID ``` -### 2. Create `/opt/containers/headscale/config.yaml` +### 2. Create a pre-auth key per node + +`--user` takes the numeric **ID** from `users list`, not the name. Prefer one single-use, short-lived key per node over a +reusable key: + +```bash +docker compose exec headscale headscale preauthkeys create --user --expiration 1h +``` -```yaml -server_url: "https://" +Use it within the hour on the node ([headscale-nodes.md](headscale-nodes.md)). A used or expired key cannot register +anything; already-registered nodes are unaffected by key expiry. -listen_addr: "0.0.0.0:443" -metrics_listen_addr: "127.0.0.1:9090" -grpc_listen_addr: "127.0.0.1:50443" -grpc_allow_insecure: false +--- -noise: - private_key_path: /var/lib/headscale/noise_private.key +## Changing the config or the policy -ip_prefixes: - - "100.64.0.0/10" +Edit `config/config.yaml` or `config/policy.hujson` in a pull request. The **Headscale config** CI job runs the pinned +image's `headscale configtest` and `headscale policy check` against them. A bad policy also stops Headscale from +starting, so a red check here means "this would have taken the tailnet down". -derp: - server: - enabled: true - region_id: 999 - region_code: "tower" - region_name: "Tower (self-hosted)" - stun_listen_addr: "0.0.0.0:3478" - urls: [] - auto_update_enabled: false +On merge, the deploy copies the files and recreates the container, which reloads both. A fingerprint of `config/` is +written into `.env` (`CONFIG_SHA256`), so a retry after an interrupted deploy still recreates it, even when the files on +tower are already up to date. -database: - type: sqlite - sqlite: - path: /var/lib/headscale/db.sqlite +`policy.hujson` is currently allow-all, which is exactly what Headscale does with no policy at all. The file explains how +to restrict it; any `"grants"` key switches the tailnet to deny-by-default. -tls_letsencrypt_hostname: "" -tls_letsencrypt_cache_dir: /var/lib/headscale/cache -tls_letsencrypt_challenge_type: TLS-ALPN-01 +--- -dns_config: - magic_dns: false - nameservers: [] +## MagicDNS -log: - level: info - format: text -``` +MagicDNS is off until the `TAILNET_BASE_DOMAIN` secret is set. With it, the role switches it on through `.env`, and every +node becomes `.`. The base domain is the only search domain pushed to nodes; there are no split or +global nameservers. The role refuses a base domain that is tower's hostname or a parent of it: Headscale would refuse +to start. -Replace `` with the public hostname of your tower. The `tls_letsencrypt_hostname` should be the bare hostname without `https://`. +`override_local_dns` stays `false`. Nodes keep their own resolvers for everything outside the tailnet, and no global +nameservers are pushed (with this setting the client would use them only while an exit node is in use). + +**Per node it is on by default.** Whether a node uses Tailscale's DNS is the client's `--accept-dns` setting, which +defaults to on. To opt a node out, run `tailscale set --accept-dns=false`. To make it opt-in, register nodes with +`tailscale up --accept-dns=false` and turn it on where wanted. In the macOS and iOS apps it is the "Use Tailscale DNS +settings" toggle. `tailscale dns status --all` shows what a node received. + +What a node with it on does, per the Tailscale client source: + +- **Linux without systemd-resolved** (plain `/etc/resolv.conf`): tailscaled takes the file over. It moves the original + to `/etc/resolv.pre-tailscale-backup.conf` and writes `nameserver 100.100.100.100`, the search domains and the + original's search domains. `options` and `domain` lines are dropped while it owns the file. Names outside the tailnet + are forwarded to the original nameservers. A clean stop or `--accept-dns=false` restores the file. After a crash it + stays pointed at 100.100.100.100 until tailscaled starts again (or `tailscaled --cleanup` runs), so lookups fail in + between. +- **Linux with systemd-resolved:** split DNS. Only names under the base domain go to Tailscale. +- **macOS app:** takes over all DNS and forwards names outside the tailnet to the Mac's own resolvers. +- **iOS:** scoped split DNS. Only names under the base domain go to Tailscale. + +Short names resolve to tailnet nodes (`web1` → `web1.`). MagicDNS answers A records only; to get AAAA records for dual-stack nodes, grant the `magicdns-aaaa` attribute in `policy.hujson` (`nodeAttrs`). --- -## First-Run Steps +## Upgrading -These steps are performed once after the container is running for the first time. +Headscale ≥ 0.29 **refuses to start** if a minor version is skipped (0.28 → 0.30) or downgraded. It migrates the database +on start. Renovate raises one PR per minor version (`renovate.json`), and each PR body carries this checklist: -### 1. Retrieve the node key +1. Merge minor versions **one at a time, oldest first**. Patch releases within a minor are always safe. +2. Read the release notes for removed config keys and the minimum Tailscale client version (0.29: **v1.80.0**). Update + the clients first if needed. +3. Diff `config/config.yaml` against that release's `config-example.yaml`. The CI check fails only on keys Headscale + lists as removed (it refuses to start on those). Any other renamed or dropped key is **silently ignored**, and new + options don't show up at all, so this diff is the only way to catch either. +4. Merge. The deploy snapshots the database, then recreates the container on the new image. +5. Check `docker compose ps` (healthy) and `docker compose logs`, then run `tailscale status` on a node. -After the container starts, get the node key that nodes will use to identify this server: +To roll back a minor upgrade, the previous image and the pre-upgrade database must go back together. The order +matters: -```bash -docker compose exec headscale headscale debug node-key -``` +1. Merge a revert of the upgrade PR. The deploy recreates the container on the previous image, which refuses to open the + migrated database and crash-loops. That is expected and changes nothing. +2. On tower, restore the snapshot **the upgrade deploy itself took**, following the steps in + [Backup and restore](#backup-and-restore). The deploy takes its snapshot before it switches the image, so that one is + the newest copy from before the migration. Every later snapshot holds the migrated database, including the one the + revert deploy just took. The deploy log shows which run was the upgrade. -### 2. Create a user +Doing it the other way round (restore first, revert later) leaves a window where the newer image starts on the restored +database, for example on the next scheduled deploy, and migrates it again. -Headscale requires at least one user (equivalent to a Tailscale account/tailnet). Create one for the homelab nodes: +--- -```bash -docker compose exec headscale headscale users create homelab -``` +## Backup and restore + +Before every deploy, `system/headscale` writes a snapshot to `/opt/containers/headscale/backups//`. It +is built under a hidden `..partial` name and renamed only once complete, so an interrupted attempt never takes +one of the kept slots. The next run removes leftover partial directories. Each snapshot holds: -### 3. Generate a preauth key +- `db.sqlite`: an online copy taken with SQLite's backup API (consistent while Headscale runs), checked with + `PRAGMA quick_check`. A failing check fails the deploy before the image is changed. +- `noise_private.key`, `derp_server_private.key`. The Noise key identifies the server to every client. Without it, + every node has to be registered again. -Create a reusable preauth key so nodes can register without manual approval: +The newest 14 are kept (`headscale_backup_keep`), about a week at two deploys a day. These snapshots live on the same +boot volume: they protect against a bad upgrade, **not** against losing the VM. + +To restore: ```bash -docker compose exec headscale headscale preauthkeys create --reusable --expiration 24h +cd /opt/containers/headscale +docker compose stop +cp backups//db.sqlite data/db.sqlite +rm -f data/db.sqlite-wal data/db.sqlite-shm # stale WAL from the newer database +cp backups//*.key data/ # only if the keys were lost +docker compose up -d ``` -Copy the key from the output. Share it securely with each node operator (or use it yourself on each Unraid host). After nodes are registered you can let the key expire or delete it. +`up -d` (not `start`) makes sure the container runs the image `docker-compose.yml` currently pins. `start` would reuse +the existing container, whatever image it was created from. + +Removing the stale `-wal`/`-shm` files matters. SQLite would replay them onto the older `db.sqlite` and corrupt it. --- -## Key Commands Reference +## Key commands + +Run from `/opt/containers/headscale/` on tower. Node and key commands take numeric IDs from the `list` output. | Action | Command | |---|---| +| Health | `docker compose exec headscale headscale health` | | Create a user | `docker compose exec headscale headscale users create ` | -| List users | `docker compose exec headscale headscale users list` | -| Create a preauth key | `docker compose exec headscale headscale preauthkeys create --reusable --expiration 24h` | -| List preauth keys | `docker compose exec headscale headscale preauthkeys list` | -| List registered nodes | `docker compose exec headscale headscale nodes list` | -| Rename a node | `docker compose exec headscale headscale nodes rename --identifier ` | -| Expire a node | `docker compose exec headscale headscale nodes expire --identifier ` | - -All commands are run from the directory containing the Headscale `docker-compose.yml` (`/opt/containers/headscale/` on tower). +| List users (shows IDs) | `docker compose exec headscale headscale users list` | +| Create a single-use pre-auth key | `docker compose exec headscale headscale preauthkeys create --user --expiration 1h` | +| List pre-auth keys | `docker compose exec headscale headscale preauthkeys list` | +| Expire a pre-auth key | `docker compose exec headscale headscale preauthkeys expire --id ` | +| List nodes | `docker compose exec headscale headscale nodes list` | +| Rename a node | `docker compose exec headscale headscale nodes rename --identifier ` | +| Expire a node (forces re-login) | `docker compose exec headscale headscale nodes expire --identifier ` | +| Delete a node | `docker compose exec headscale headscale nodes delete --identifier ` | +| Show / approve advertised routes | `docker compose exec headscale headscale nodes list-routes` / `nodes approve-routes --identifier --routes ` | +| Show the policy file as mounted (not necessarily the loaded one until the next restart) | `docker compose exec headscale headscale policy get` | + +--- + +## Why Headscale objects are not in OpenTofu + +OpenTofu already manages everything below Headscale: the VM, the security-list rules for `443/tcp` and `3478/udp`, and +the DNS record the server URL depends on. Headscale's **static** configuration, the server config and the access policy, +is declarative in git, deployed by Ansible and checked in CI against the real binary. What remains is runtime state: a +few users and nodes, and short-lived pre-auth keys. Moving that into OpenTofu was evaluated and rejected for now. + +**The official `tailscale/tailscale` provider can't be used.** Headscale reimplements the protocol the Tailscale +*client* uses to reach its control server, not Tailscale's admin API. The provider's client builds every request as +`/api/v2/tailnet/{tailnet}/…` (`/acl`, `/keys`, `/devices`, `/dns/…`). Headscale serves a different API at `/api/v1/…` +(`/policy`, `/preauthkey`, `/node/{id}/tags`, `/user`) and has no `/api/v2` routes. Setting the provider's `base_url` to +tower gets 404s. + +**The community `awlsring/headscale` provider doesn't support this version.** It covers users, pre-auth keys, API keys, +node tags, subnet routes and the policy (the policy only with `policy.mode: database`). Its newest line, 0.5.x, last +released 2026-03-19, supports Headscale **0.28.x**. Tower runs 0.29.4, whose API added the `auth` routes and changed the +register endpoint. Headscale also won't downgrade to 0.28 to match it. Every Headscale minor bump would wait on a +third-party provider release. + +**It would cost more than it saves, even if it were compatible:** + +- **A standing admin credential in CI.** OpenTofu needs a Headscale API key, created by hand on the server first, that + expires after 90 days by default. As a GitHub secret it can mint pre-auth keys, which means it can join a machine to + the tailnet next to every node. Today no such credential exists outside tower. +- **Join credentials in state.** Pre-auth keys created by OpenTofu are stored in the S3 state file. +- **Nodes register themselves.** OpenTofu can only adopt a node after `tailscale up`, by its server-assigned ID, for tags + and routes. Every node join becomes two steps. +- **Coupling to the VM's apply.** `tower-tofu-apply.yml` runs daily with `-auto-approve` and discards its output, and the + Ansible deploy (which starts Headscale) runs after it. A Headscale provider in the same root module would make the VM + and DNS apply depend on the app being up and on API compatibility after every Renovate bump. On a rebuilt VM it could + not apply at all until Ansible had run. Avoiding that means a separate root module, state and workflow. +- **The policy already gets the benefit.** In file mode, `policy.hujson` is reviewed in a PR, checked by + `headscale policy check`, and deployed like any other file. + +**When to revisit:** the provider ships and keeps up with Headscale 0.29+ releases, *and* the tailnet grows to the point +where users, tags or routes change often (several operators, OIDC, ACL tags per service). Then it belongs in its own +root module (for example `tower/headscale/`) with its own state and workflow, chained after the Ansible deploy, never in +`tower/terraform/`. + +OpenTofu changes worth making for Headscale, none applied here: + +- **Off-host backups.** An `oci_core_volume_backup_policy` assigned to the boot volume would protect the data directory + against losing the VM. The free tier includes five volume backups. +- **No `AAAA` record yet.** The instance has IPv6, but the `app-infra` network on tower is IPv4-only. Docker's userland + proxy would make STUN report the wrong address to IPv6 clients. Enable IPv6 on that network (or use host networking for + this container) before publishing an `AAAA` record. diff --git a/tower/docs/setup.md b/tower/docs/setup.md index e822919b..452d0875 100644 --- a/tower/docs/setup.md +++ b/tower/docs/setup.md @@ -81,6 +81,9 @@ In your GitHub repository → **Settings** → **Environments** → **tower** (c | `SSH_AUTHORIZED_KEYS` | JSON array of SSH public keys, e.g. `["ssh-ed25519 AAAA...", "ssh-ed25519 BBBB..."]` | | `CLOUDFLARE_API_TOKEN` | API token from Step 4 | | `CLOUDFLARE_ZONE_ID` | Zone ID for your DNS zone from Step 4 | +| `TOWER_HOSTNAME` | Tower's public FQDN, bare (no `https://`). Used as the DNS record name and as Headscale's server URL. Kept out of the repo because it is public; see [headscale-setup.md](headscale-setup.md) | +| `TOWER_ACME_EMAIL` | Optional. Contact address for tower's Let's Encrypt account | +| `TAILNET_BASE_DOMAIN` | Optional. Turns MagicDNS on: nodes become `.`. Use a name nothing public resolves, such as one under `.internal`; it must not be tower's hostname or a parent of it. See [headscale-setup.md](headscale-setup.md#magicdns) | --- diff --git a/tower/terraform/dns.tf b/tower/terraform/dns.tf index 5f00f19d..f4a08336 100644 --- a/tower/terraform/dns.tf +++ b/tower/terraform/dns.tf @@ -4,7 +4,7 @@ provider "cloudflare" { resource "cloudflare_dns_record" "tower" { zone_id = var.cloudflare_zone_id - name = "tower.compute.eu" + name = var.tower_hostname type = "A" content = oci_core_instance.tower.public_ip ttl = 3600 diff --git a/tower/terraform/moved.tf b/tower/terraform/moved.tf index 2a9e8b81..ddd25cf3 100644 --- a/tower/terraform/moved.tf +++ b/tower/terraform/moved.tf @@ -30,12 +30,3 @@ moved { from = oci_core_subnet.gateway to = oci_core_subnet.tower } - -# arm1.turin.oci DNS record deleted — tower.compute.eu replaces it -removed { - from = cloudflare_dns_record.arm1 - - lifecycle { - destroy = true - } -} diff --git a/tower/terraform/terraform.tfvars.example b/tower/terraform/terraform.tfvars.example index f3731052..a0283f32 100644 --- a/tower/terraform/terraform.tfvars.example +++ b/tower/terraform/terraform.tfvars.example @@ -19,5 +19,6 @@ ssh_authorized_keys = [ "ssh-ed25519 AAAAC3... user@host", ] +tower_hostname = "tower.example.com" cloudflare_api_token = "..." cloudflare_zone_id = "..." diff --git a/tower/terraform/variables.tf b/tower/terraform/variables.tf index d51e7c7f..5708c2c8 100644 --- a/tower/terraform/variables.tf +++ b/tower/terraform/variables.tf @@ -80,6 +80,19 @@ variable "cloudflare_api_token" { sensitive = true } +variable "tower_hostname" { + description = "Public FQDN of tower: the DNS record name and the Headscale server URL host. Kept out of the public repo; comes from the TOWER_HOSTNAME environment secret." + type = string + sensitive = true + + # A missing GitHub secret arrives as an empty string, which still counts as + # a set value; without this the apply would try to rename the record to "". + validation { + condition = can(regex("^([a-z0-9]([a-z0-9-]*[a-z0-9])?\\.)+[a-z]{2,}$", var.tower_hostname)) + error_message = "tower_hostname must be a bare lower-case FQDN. Is the TOWER_HOSTNAME secret set?" + } +} + variable "cloudflare_zone_id" { description = "Cloudflare zone ID" type = string