From 3ccdef92f431d5ec3e84a1096b5f6ef58f1c7be7 Mon Sep 17 00:00:00 2001 From: AMAL BABU Date: Tue, 29 Sep 2026 14:43:18 +0530 Subject: [PATCH] Refactor code structure for improved readability and maintainability; removed redundant code blocks and optimized functions. --- .github/workflows/release.yml | 2 +- .github/workflows/test.yml | 6 +- CHANGELOG.md | 69 ++++- README.md | 4 +- ROADMAP.md | 75 +++++ docs/05-QUICKSTART.md | 16 +- docs/10-SINGLE-SERVER.md | 9 +- docs/20-CLUSTER.md | 6 + docs/30-OPERATIONS.md | 26 +- docs/35-CONTINUOUS-DEPLOYMENT.md | 25 ++ docs/40-ADOPTING-A-NEW-PROJECT.md | 4 +- docs/50-WEBUI.md | 44 ++- docs/60-COMMAND-REFERENCE.md | 48 ++- pyproject.toml | 3 +- src/deployctl/__init__.py | 2 +- src/deployctl/cli/commands/ci.py | 56 +++- src/deployctl/cli/commands/deploy.py | 3 + src/deployctl/cli/commands/init.py | 5 +- src/deployctl/cli/commands/migrate_config.py | 91 ++++-- src/deployctl/cli/commands/show.py | 26 +- src/deployctl/cli/commands/transfer.py | 183 +++++++++++ src/deployctl/cli/commands/webui.py | 53 +++- src/deployctl/cli/config.py | 27 ++ src/deployctl/cli/fields.py | 4 +- src/deployctl/cli/github.py | 16 +- src/deployctl/cli/main.py | 2 +- src/deployctl/cli/paths.py | 34 +- src/deployctl/cli/scaffold.py | 22 +- src/deployctl/cli/snapshots.py | 53 ++++ src/deployctl/cli/transfer.py | 291 ++++++++++++++++++ src/deployctl/scripts/common/remote.sh | 32 ++ src/deployctl/scripts/deploy.sh | 16 +- src/deployctl/templates/ci/build-image.yml.j2 | 2 +- src/deployctl/templates/ci/ci.yml.j2 | 7 +- src/deployctl/templates/ci/deploy.yml.j2 | 9 +- src/deployctl/webui/fields/core.toml | 16 +- src/deployctl/webui/panel/live.py | 81 ++++- src/deployctl/webui/panel/routes.py | 106 ++++++- src/deployctl/webui/panel/runner.py | 7 +- src/deployctl/webui/panel/state.py | 14 +- src/deployctl/webui/static/panel.css | 18 ++ src/deployctl/webui/static/panel.js | 51 ++- .../webui/templates/_export_result.html | 16 + .../webui/templates/_import_dialog.html | 36 +++ .../webui/templates/_import_result.html | 40 +++ .../webui/templates/_live_access.html | 37 +++ .../webui/templates/_tab_configure.html | 14 + .../webui/templates/_tab_operate.html | 3 + src/deployctl/webui/templates/_tab_setup.html | 2 + src/deployctl/webui/templates/base.html | 4 +- src/deployctl/webui/templates/index.html | 2 + src/deployctl/webui/templates/start.html | 35 +++ tests/conftest.py | 2 + tests/test_ci_commands.py | 99 +++++- tests/test_ci_deploy.py | 17 + tests/test_deploy_engine.py | 48 +++ tests/test_docs.py | 30 +- tests/test_panel_live.py | 7 + tests/test_panel_transfer.py | 206 +++++++++++++ tests/test_personal_access.py | 144 +++++++++ tests/test_transfer.py | 230 ++++++++++++++ uv.lock | 165 ++++++++++ 62 files changed, 2592 insertions(+), 109 deletions(-) create mode 100644 ROADMAP.md create mode 100644 src/deployctl/cli/commands/transfer.py create mode 100644 src/deployctl/cli/snapshots.py create mode 100644 src/deployctl/cli/transfer.py create mode 100644 src/deployctl/webui/templates/_export_result.html create mode 100644 src/deployctl/webui/templates/_import_dialog.html create mode 100644 src/deployctl/webui/templates/_import_result.html create mode 100644 src/deployctl/webui/templates/_live_access.html create mode 100644 src/deployctl/webui/templates/start.html create mode 100644 tests/test_panel_transfer.py create mode 100644 tests/test_personal_access.py create mode 100644 tests/test_transfer.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 702545d..39f92da 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ permissions: jobs: release: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63c1e4d..c62ca1d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ concurrency: jobs: pytest: name: pytest (Python ${{ matrix.python }}) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 15 strategy: fail-fast: false @@ -38,7 +38,7 @@ jobs: shellcheck: name: shellcheck - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -50,7 +50,7 @@ jobs: package: name: the wheel works on its own - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fa0ccc..a2d75a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,74 @@ # Changelog Versions are git tags (`vX.Y.Z`); projects install one with -`uv tool install git+ssh://git@github.com/hybridinteract/deployctl@vX.Y.Z`. +`uv tool install git+https://github.com/hybridinteract/deployctl@vX.Y.Z`. + +## 0.13.0 + +A second person can deploy the same project: its configuration travels as one encrypted +file, and everyone's own access stays their own. + +**Share the configuration** +- `deployctl config export` writes every environment's shared files — settings and secrets, + everything the project needs to run — to one file, encrypted with a passphrase (AES-256-GCM, + scrypt; a changed byte or a wrong passphrase opens nothing). It lands in `config/exports/`, + owner-only and ignored by git through its own `.gitignore`; the last 5 are kept. + `--generate-passphrase` makes a strong one and prints it once. +- `deployctl config import` loads one — the file given, or the one waiting in + `config/imports/`. It refuses a file made for another repository, and a configuration that + differs from the one here (naming the differing keys, never values) unless `--force`; + `--preview` shows the plan and writes nothing. +- `deployctl config` is now a group: `config show` (plain `deployctl config` still works), + `config export`, `config import`. +- Every panel Save, `config import --force` and `migrate-config --apply` first copies + `config/` into `~/.deployctl/config-backups//` (the last 20). + +**Each person's access is their own** +- The registry login (`REGISTRY_USER`/`REGISTRY_TOKEN`) moves to **`config/local.env`**: + this machine's, loaded over the shared files, never exported, never uploaded to GitHub, and + never listed as an environment. It holds only those keys; anything else there is ignored + and `validate` says so. A login still in a shared file is a `validate` and `ci doctor` + warning, and `ci sync-config` refuses to upload it. +- **CI logs the hosts in with its own short-lived `GITHUB_TOKEN`** (the deploy workflow + grants `packages: read`), not with anyone's token. **Every host is logged out of the + registry once its images are pulled** — no credential outlives a deploy, CI's or a + laptop's. +- `deploy status --json` says why a host cannot be reached: `denied` (this machine's ssh key + is not on it) or `unreachable`. +- `ci doctor` shows your role on the repository; a secrets list GitHub refuses to a + non-admin is a warning, not a failure; "not logged in" and "no access to this repository" + are told apart; and it checks that `ci.yml`'s deploy job grants `packages: read`. + +**Panel** +- A project with no configuration on this machine opens on a **start page with the Import + dialog open**: file and passphrase, a preview, then the full panel seconds later. +- Configure → **Share or back up this configuration**: Export (a download, with the + passphrase shown once) and Import. +- **Your access on this machine** (Setup, and Operate when something is missing): your + registry login, your ssh key on each server — with your public key and the command to add + it when the server refuses it — and your GitHub role. +- It says which project it serves: `deployctl webui` prints it, tabs are titled + `deployctl · · `, and a busy port held by **another project's** panel is + named as such instead of "open it" (`~/.deployctl/panels/.json`). +- `deployctl webui` starts in a clone with no configuration instead of refusing. + +**Also** +- The generated workflows and this repository's own run on `ubuntu-24.04`, not + `ubuntu-latest` (which GitHub moves to Ubuntu 26 on 2026-10-19). +- Install commands use `git+https://` — the repository is public. +- New dependency: `cryptography`. + +**Upgrading a project** + +```bash +uv tool install --force git+https://github.com/hybridinteract/deployctl@v0.13.0 +deployctl migrate-config --apply # your registry login → config/local.env +deployctl ci init --env production --force # CI's own registry login, ubuntu-24.04 +deployctl ci doctor --env production # prints the lines ci.yml's deploy job needs +deployctl ci sync-config --env production # the config without anyone's token +``` + +Then commit the workflows and `ci.yml`, and merge. Until then CI deploys as before. ## 0.12.0 diff --git a/README.md b/README.md index 3471c93..99ccb8d 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ every detail spelled out, plus a troubleshooting table: ### 1 · Install the tool ```bash -uv tool install git+ssh://git@github.com/hybridinteract/deployctl@v0.12.0 +uv tool install git+https://github.com/hybridinteract/deployctl@v0.13.0 deployctl --version ``` @@ -131,7 +131,7 @@ Everything above is a command. Use these for scripting, CI, or when you want the invocation in your shell history: ```bash -uv tool install git+ssh://git@github.com/hybridinteract/deployctl@v0.12.0 # once per machine +uv tool install git+https://github.com/hybridinteract/deployctl@v0.13.0 # once per machine deployctl init --mode single # or --mode cluster; creates deploy/ $EDITOR deploy/config/common.env # project name, domain, image repo $EDITOR deploy/config/production.env # hosts, database, TLS diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..ecc3969 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,75 @@ +# Roadmap + +Work that is decided but not done yet. Each item says why it matters and where the change +goes, so it can be picked up cold. When one ships, move it to the CHANGELOG and delete it +here. + +--- + +## Next + +### Encrypted configuration in git — design first + +**Why.** Since 0.13 a project's configuration travels as one encrypted file +(`config export` / `import`), but a person still passes it on by hand, and two people's +copies can drift until one of them runs `ci status`. Committing the configuration to the +repository — keys readable, each **value** encrypted, the file openable by every listed +person's key and by CI's — would make git the single source of truth: a teammate's access is +adding their public key and re-encrypting; a config change is a pull request whose diff names +the keys that changed and never their values; CI holds one decryption key instead of the +whole config, so `ci sync-config` disappears. + +**Decide before building.** +- The tool: SOPS with age keys (dotenv format built in; a Go binary each machine and CI + needs), dotenvx (keeps `.env`, encrypts values; a Node tool), or age implemented in Python + with `cryptography` (no new binary; format of our own). +- How the panel's Save writes an encrypted file, and how `config/local.env` stays outside it. +- The migration from `config/` + the `DEPLOYCTL_CONFIG` secret, project by project. +- What it costs: encrypted values stay in git history for good — a leaked private key means + rotating every secret it could open. + +### Add and remove a person's ssh key from the panel and CLI + +**Why.** A teammate's ssh key is added by hand today +(`ssh deploy@host 'cat >> ~/.ssh/authorized_keys' < key.pub`), on every host, and removed by +hand when they leave. The panel's *Your access* card already shows the key that needs adding. +`deployctl server keys` / `add-key` / `remove-key` would list, add (every host, the jump host, +proven like `ci setup-key` proves the CI key) and remove them — the human counterpart of +`ci setup-key`. + +--- + +## Later + +### `ci doctor` flags deploy keys that can write +With automatic deploys on, anything that can push to the deploy branch can deploy to +production. A **read-write deploy key** on the repository is exactly that, and nothing +reports it — influen still had one (`deploy-v1`) from its previous tooling after moving to +deployctl. `ci doctor` should list the repository's deploy keys (`gh repo deploy-key list`) +and warn on any with write access. + +### The flaky lock-cancel test +`test_released_when_the_run_is_cancelled` fails about 1 run in 6 under heavy load: a race +between SIGTERM and `wait` in bash 3.2 (macOS), in the test harness rather than in the lock +itself. A separate session was started to fix it; check whether it landed before starting +again. + +### Test client deprecation +Every panel test run warns *"Using `httpx` with `starlette.testclient` is deprecated; install +`httpx2` instead."* Move the dev dependency before a Starlette release removes the old path. + +### Registries other than ghcr.io in the tag picker +"Recent tags from the registry" lists tags only for ghcr.io (`cli/registry.py`). Other +registries work, but the operator types the tag. + +--- + +## 1.0.0 + +After a second project has gone from nothing to deploying on every merge through the panel +(salescrm) — the proof that nothing here is influen-specific. + +## 2.0.0 + +- Remove `IMAGE_TAG` in `config/` (deprecated since 0.10.0, honoured with a warning; + `deployctl migrate-config` removes it). diff --git a/docs/05-QUICKSTART.md b/docs/05-QUICKSTART.md index e084d63..defc4ea 100644 --- a/docs/05-QUICKSTART.md +++ b/docs/05-QUICKSTART.md @@ -14,12 +14,22 @@ optional and nothing here is ambiguous — run the blocks in order, top to botto > 2. **A container image in a registry.** → Don't have one? Step 4 creates the CI workflow > that builds it. +> **Joining a project someone already deploys?** Skip the rest of this page: +> +> 1. Install the tool (Step 1 below) and clone the application's repository. +> 2. Ask whoever runs the project for an export (panel: Configure → *Share or back up* → +> **Export**) and its passphrase, sent separately. +> 3. `deployctl webui` inside the clone — it opens on **Import**. Pick the file, type the +> passphrase, check the preview, import. +> 4. Work through **Your access on this machine** (Setup tab): your own registry login, your +> ssh key on the servers (send them your public key), and `gh auth login`. + --- ## Step 1 · Install the tool ```bash -uv tool install git+ssh://git@github.com/hybridinteract/deployctl@v0.12.0 +uv tool install git+https://github.com/hybridinteract/deployctl@v0.13.0 deployctl --version ``` @@ -127,7 +137,7 @@ Deploy the change). Deploying from this machine is still there, folded under Every button above is a command. Use these for scripting, CI, or an audit trail: ```bash -uv tool install git+ssh://git@github.com/hybridinteract/deployctl@v0.12.0 # once per machine +uv tool install git+https://github.com/hybridinteract/deployctl@v0.13.0 # once per machine deployctl init --mode single --env production $EDITOR deploy/config/common.env # project name, domain, image repo $EDITOR deploy/config/production.env # hosts, database, TLS @@ -158,7 +168,7 @@ run, without touching a host. | Symptom | Do this | |---|---| -| `deployctl: command not found` | Step 1 — `uv tool install git+ssh://git@github.com/hybridinteract/deployctl@v0.12.0`, and make sure uv's tool directory is on your PATH (`uv tool update-shell`). | +| `deployctl: command not found` | Step 1 — `uv tool install git+https://github.com/hybridinteract/deployctl@v0.13.0`, and make sure uv's tool directory is on your PATH (`uv tool update-shell`). | | Commands act on the wrong project | They use the nearest `deploy/` above the current directory. Check with `deployctl envs`, or pass `--project-dir`. | | `no environments configured` | Step 2 has not been run. | | Panel says `port 8765 is already in use` | `deployctl webui --restart` | diff --git a/docs/10-SINGLE-SERVER.md b/docs/10-SINGLE-SERVER.md index 093fbcc..0ab3077 100644 --- a/docs/10-SINGLE-SERVER.md +++ b/docs/10-SINGLE-SERVER.md @@ -26,12 +26,19 @@ horizontal scale. Read **[00-CONCEPTS.md](00-CONCEPTS.md)** first if you have no deployctl init --mode single --env production ``` -It creates `deploy/` in your repository. Fill in the three files. `deploy/config/common.env`: +It creates `deploy/` in your repository. Fill in the files. `deploy/config/common.env`: ```sh PROJECT_NAME=myapp BASE_DOMAIN=example.com IMAGE_REPO=ghcr.io/your-org/myapp +``` + +`deploy/config/local.env` — **yours, on this machine only**: your registry login, used when +you deploy from here and to list image tags. Never exported, never uploaded to GitHub; CI +logs in with its own token, and everyone who deploys keeps their own: + +```sh REGISTRY_USER=your-github-user REGISTRY_TOKEN=ghp_… # classic PAT, read:packages only ``` diff --git a/docs/20-CLUSTER.md b/docs/20-CLUSTER.md index 1625cd9..3b7c84e 100644 --- a/docs/20-CLUSTER.md +++ b/docs/20-CLUSTER.md @@ -129,6 +129,12 @@ deployctl init --mode cluster --env production PROJECT_NAME=myapp BASE_DOMAIN=example.com IMAGE_REPO=ghcr.io/your-org/myapp +``` + +`config/local.env` — yours, on this machine only (never exported or uploaded; CI logs in +with its own token): + +```sh REGISTRY_USER=your-github-user REGISTRY_TOKEN=ghp_… # classic PAT, read:packages only ``` diff --git a/docs/30-OPERATIONS.md b/docs/30-OPERATIONS.md index 54960c0..53083ee 100644 --- a/docs/30-OPERATIONS.md +++ b/docs/30-OPERATIONS.md @@ -33,9 +33,12 @@ Never hand-edit `generated/` or `config/secrets..env`. The first is rebuilt `config/` by step 2 and is safe to delete; the second is minted once and reused so that regenerating never invalidates live sessions. -Everything an operator owns lives in `config/`, and it is gitignored — so back it up (an -encrypted copy, a password manager, a secret store). If it is lost, **do not re-create it -and deploy**: the new render would carry new secrets and blank API keys, and doctor refuses +Everything an operator owns lives in `config/`, and it is gitignored — so keep a copy off +this machine: `deployctl config export` (or Configure → *Share or back up* in the panel) +writes it as one encrypted file, fit for a password manager, and `deployctl config import` +restores it. Every Save, import and migration also snapshots `config/` into +`~/.deployctl/config-backups/` first. If it is lost anyway, **do not re-create it and +deploy**: the new render would carry new secrets and blank API keys, and doctor refuses exactly that (see *"this deploy would CHANGE secrets"* under Troubleshooting). Recover the values from the host's live `.env.` instead. @@ -358,8 +361,14 @@ server-side state that deployctl never overwrites, so nothing repairs it automat ### `doctor` fails: cannot resolve the image In order of likelihood: the tag does not exist (`deployctl image tags`); the package is -private and `REGISTRY_TOKEN` is missing, expired, or not SSO-authorized for the org; CI has -not published anything yet. +private and your `REGISTRY_TOKEN` (`config/local.env`) is missing, expired, or not +SSO-authorized for the org; CI has not published anything yet. + +**In CI**, the hosts log in with the run's own `GITHUB_TOKEN`, which can read the image only +when the job calling the deploy grants `packages: read` — `deployctl ci doctor` checks your +`ci.yml` and prints the lines to add — and the package lets the repository in: the package's +settings → *Manage Actions access* lists the repository (automatic for images the +repository's own workflow published). ### `doctor` fails: architecture mismatch The image was built for a different CPU than the host — classically an `arm64` image built @@ -466,8 +475,11 @@ or add an exclusion in `scripts/common/remote.sh::push_artifacts`. - **Prune old images** on the hosts occasionally; every release leaves one behind: `deployctl deploy shell ` then `docker image prune -a --filter until=720h`. -- **Rotate the registry token** when someone leaves: update `REGISTRY_TOKEN` in - `config/common.env`; the next deploy logs in with the new one. +- **When someone leaves:** remove their ssh key from the deploy user's `authorized_keys` on + every host and their access to the repository. Their registry token was their own + (`config/local.env` on their machine) and never reached GitHub or a server — the hosts are + logged out after every pull — so there is nothing of theirs to rotate there; rotate the + shared secrets they could read (database passwords, API keys) if the departure calls for it. - **Rotate application secrets** deliberately, knowing the cost: `deployctl setup --env --rotate-secrets`, then `deployctl deploy update --env --allow-secret-change` (doctor refuses a changed diff --git a/docs/35-CONTINUOUS-DEPLOYMENT.md b/docs/35-CONTINUOUS-DEPLOYMENT.md index ca5b9e8..874823c 100644 --- a/docs/35-CONTINUOUS-DEPLOYMENT.md +++ b/docs/35-CONTINUOUS-DEPLOYMENT.md @@ -72,6 +72,31 @@ replaces it on every host. The public half is marked `deployctl-ci@/ installed with `restrict` (no forwarding, no PTY). The deploy user is in the `docker` group, which is root-equivalent on the host: treat the key as root. +### The registry login + +CI never uses a person's token. The deploy job hands the hosts the run's own +`GITHUB_TOKEN` — short-lived, read-only for packages — and every host is logged out again +once its image is pulled, so no registry credential outlives a run, CI's or a laptop's. Each +person who deploys by hand keeps their own login in `config/local.env`, which is never part +of the uploaded config: `ci sync-config` refuses while a registry token still sits in a +shared file (`deployctl migrate-config --apply` moves it). + +Two conditions, both checked by `ci doctor`: + +- **The job that calls the deploy grants `packages: read`.** A called workflow gets no more + than its caller allows, and `ci.yml` is your file, so `ci init` never rewrites it — add + under its `deploy:` job: + + ```yaml + permissions: + contents: read + packages: read + ``` + +- **The package lets the repository in** — automatic for an image the repository's own + workflow published; otherwise the package's settings → *Manage Actions access* → add the + repository. + ### Reaching the hosts GitHub's runners connect from a large, changing set of addresses. Three ways in: diff --git a/docs/40-ADOPTING-A-NEW-PROJECT.md b/docs/40-ADOPTING-A-NEW-PROJECT.md index 4d2c154..4c54125 100644 --- a/docs/40-ADOPTING-A-NEW-PROJECT.md +++ b/docs/40-ADOPTING-A-NEW-PROJECT.md @@ -10,7 +10,7 @@ say so rather than forking the tool. ## 1. Install, then initialise the repository ```bash -uv tool install git+ssh://git@github.com/hybridinteract/deployctl@v0.12.0 # once per machine +uv tool install git+https://github.com/hybridinteract/deployctl@v0.13.0 # once per machine cd /path/to/new-project deployctl init --mode single --env production ``` @@ -293,7 +293,7 @@ deploy, restore. ## 8. Upgrading deployctl later ```bash -uv tool install --force git+ssh://git@github.com/hybridinteract/deployctl@ +uv tool install --force git+https://github.com/hybridinteract/deployctl@ deployctl selftest deployctl validate --env deployctl deploy update --env --tag --dry-run diff --git a/docs/50-WEBUI.md b/docs/50-WEBUI.md index d3c1057..f1b061d 100644 --- a/docs/50-WEBUI.md +++ b/docs/50-WEBUI.md @@ -7,7 +7,7 @@ deploying or rolling back a tag, putting a config change live — happens in **O ## Start it ```bash -uv tool install git+ssh://git@github.com/hybridinteract/deployctl@v0.12.0 # once per machine +uv tool install git+https://github.com/hybridinteract/deployctl@v0.13.0 # once per machine deployctl webui # → http://127.0.0.1:8765 ``` @@ -29,7 +29,11 @@ A panel is already serving that port. That is easy to end up with: if it was sta terminal you have since closed, or backgrounded, nothing obvious owns it, and the browser keeps working while `webui` refuses to start. -`deployctl webui` names the culprit rather than leaving you with the bare bind error: +`deployctl webui` names the culprit rather than leaving you with the bare bind error — and, +when that panel belongs to **another project**, says so and suggests another port instead of +"open it": opening it would hand you that other project's buttons. (Each running panel is +recorded in `~/.deployctl/panels/.json`; `webui` also prints which project it serves +when it starts, and every browser tab is titled `deployctl · · `.) ``` [ERROR] port 8765 is already in use @@ -96,6 +100,13 @@ which can send it requests. So: Submitting one blank keeps the stored value; only a non-empty value replaces it. *Preview resolved config* masks credentials by key name **and** by value, so the passwords embedded in `DATABASE_URL` and the Redis URLs are scrubbed too. +- **Import and export keep secrets out of URLs and argv.** Export makes the passphrase on the + server and shows it once; import sends the file and its passphrase in the body of a POST, + twice (preview, then apply), so the server keeps neither between the steps. Both run + `deployctl config export|import` with the passphrase in that one child's environment — + never as an argument, which any process on the machine can read. An upload is staged + owner-only in the git-ignored `config/imports/` and removed once read; the download route + serves only a name `config export` writes, only from `config/exports/`. - **An unknown environment name is a 404**, never a silent fallback to another one — quietly showing staging when the URL said production is how the wrong thing gets deployed. - **No interactive API docs.** `/docs` and `/openapi.json` are disabled: they would be a @@ -147,8 +158,25 @@ and names the one next step with a button to the tab it is done in. The page ope over ssh: for when GitHub or CI is what is broken. Same engine, lock, health gate and revert; outside CI's history of runs. +### A new machine — the start page + +A project with no configuration on this machine — a teammate's fresh clone — opens on a +start page with the **Import** dialog already open: the file someone exported, its +passphrase, then **Preview** (project, who exported it and when, every file it would write, +the keys that differ — names, never values) and **Import**. Seconds later the page reloads +into the full panel. Closing the dialog shows the other way in: `deployctl init`, for a +project that has no configuration anywhere yet. + ### Setup — once per server +**Your access on this machine** heads the tab: what no shared configuration can give you, +each with what to do — your registry login (set in Configure, saved to `config/local.env`), +your ssh key on each server (*the server refused this machine's key* shows your public key and +the one command whoever runs the project uses to add it; *no answer* is a network problem), +and your GitHub access (logged in, and your role: Write deploys and rolls back, Admin also +syncs config and switches automatic deploys). On Operate the same card appears only when +something is missing. + 1. **Prepare the server** — the root script for this environment (`deployctl server bootstrap-script`), how to run it on each host, and the check afterwards. For a *new* server: it upgrades packages and may reboot. @@ -176,7 +204,15 @@ environment that is already deployed is a migration rather than a toggle — see `webui/fields/*.toml` plus your `project/fields.toml`, filtered by mode. Save writes straight to the config files and nothing else — **Apply it →** takes you to *Apply a config change*. The image tag is not here: CI deploys the tag it built, and Operate deploys the -one you pick. +one you pick. *Your registry access* is the one section that is yours alone: it saves to +`config/local.env`, never exported or uploaded. + +**Share or back up this configuration** (top of Configure) — **Export** writes every +environment's settings and secrets as one encrypted file into `config/exports/` and offers it +as a download, with a passphrase shown once: for a teammate, a new laptop, or a copy kept off +this machine. Nobody's registry login, ssh keys or GitHub login are in it. **Import…** opens +the same dialog as the start page; replacing a configuration that differs needs an explicit +*Replace my configuration*, and the current files are snapshotted first. **Logs** — one button per host; streams `docker compose logs -f`. @@ -219,7 +255,7 @@ disagree, the CLI is the truth — and that is a bug worth reporting. **"no environments configured"** — run `deployctl init --mode single|cluster` first. **Dependencies missing** — they are part of the installed tool; reinstall it: -`uv tool install --force git+ssh://git@github.com/hybridinteract/deployctl@`. +`uv tool install --force git+https://github.com/hybridinteract/deployctl@`. **The output pane says "connection lost — the command is still running"** — only the view was lost. Click the job in the bar under the pane's header to re-attach. A job started by a diff --git a/docs/60-COMMAND-REFERENCE.md b/docs/60-COMMAND-REFERENCE.md index 62af5e5..fed5cca 100644 --- a/docs/60-COMMAND-REFERENCE.md +++ b/docs/60-COMMAND-REFERENCE.md @@ -69,10 +69,16 @@ untracked files are never deleted — they are listed. `--from` names the copy w the one found from the current directory. ### `deployctl migrate-config [--apply]` -**Touches: local.** Brings `config/` up to the current tool. Today: removes `IMAGE_TAG` from -every `config/.env` — the tag is decided per command now (see *Image tags*) — and keeps -its value as this machine's last-used tag, so `setup` still works. Prints the plan unless -`--apply` is given; comments and every other line are left as they were. +**Touches: local.** Brings `config/` up to the current tool: + +- removes `IMAGE_TAG` from every `config/.env` — the tag is decided per command now (see + *Image tags*) — keeping its value as this machine's last-used tag, so `setup` still works; +- moves `REGISTRY_USER` / `REGISTRY_TOKEN` out of the shared files into `config/local.env`: + one person's registry login, never exported or uploaded (since 0.13). The shared file keeps + a one-line note saying where it went; a value already in `local.env` wins. + +Prints the plan unless `--apply` is given — never a token's value. `--apply` snapshots +`config/` first; comments and every other line are left as they were. ### `deployctl selftest [--skip-docker]` **Touches: local.** Renders every supported deployment shape into a temporary directory and @@ -86,15 +92,43 @@ you the tool itself is intact. ### `deployctl envs` **Touches: local.** Lists configured environments with their mode, TLS mode and hosts. -### `deployctl config --env E [--key K] [--show-secrets]` +### `deployctl config show --env E [--key K] [--show-secrets]` **Touches: local.** Prints the fully resolved configuration and the source layers it came -from, lowest precedence first. +from, lowest precedence first. Plain `deployctl config --env E [--key K] [--show-secrets]` is +the same command, as it always was. Credentials are masked two ways: by key name (`*_PASSWORD`, `*_TOKEN`, `*_SECRET`, `*_KEY`) and by value, so the passwords embedded in derived URLs — `DATABASE_URL`, `REDIS_URL`, `CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND` — are scrubbed too. `--show-secrets` prints everything in full; `--key K` prints one value and nothing else, for scripting. +### `deployctl config export [--env E]… [--output PATH] [--generate-passphrase] [--json]` +**Touches: local.** Writes this project's configuration to one encrypted file — for a +teammate, a new laptop, or a copy kept off this machine. It holds every shared file of every +environment (`common.env`, `.env`, `app..env`, `secrets..env`; `--env`, +repeatable, limits it) — everything the project needs to run. It never holds anyone's +registry login (`config/local.env`, and those keys wherever they sit), ssh keys or GitHub +login: each person sets up their own. + +AES-256-GCM with a key derived from the passphrase by scrypt; the project, environments and +who exported it are inside the encryption, and a changed byte or a wrong passphrase opens +nothing. Written to `config/exports/` (owner-only, ignored by git through its own +`.gitignore`; the last 5 are kept) unless `--output` (`-o`) names a path — refused inside a git repository +unless that path is ignored. The passphrase is asked for twice (at least 12 characters), +generated and printed once with `--generate-passphrase`, or read from `$DEPLOYCTL_PASSPHRASE` +(how the control panel passes it). `--json` for the panel. + +### `deployctl config import [FILE] [--preview] [--force] [--keep] [--json]` +**Touches: local.** Loads a file made by `config export` into this project's `config/`. Without +`FILE` it takes the one file waiting in `config/imports/` — it lists several rather than +guess, and says where to put one when there is none. It refuses a file made for another +repository, and a configuration that differs from the one here — naming the keys that +differ, never their values — unless `--force`, which snapshots the current files first +(`~/.deployctl/config-backups/`). `--preview` shows what would change and writes nothing. +Files are written owner-only; a file taken from `config/imports/` is deleted afterwards +unless `--keep`. It ends by listing what stays yours to set up: your registry login, your ssh +key on the servers, your GitHub login. + --- ## Images @@ -404,7 +438,7 @@ See [50-WEBUI.md](50-WEBUI.md) for the panel itself and its safety model. | `ASSUME_YES=1` | Skip confirmation prompts. Set automatically by the CLI and the panel. (`backup restore` into the live database confirms in the CLI regardless — see above.) | | `DEPLOYCTL_ALLOW_SECRET_CHANGE=1` | What `--allow-secret-change` sets, for commands that have no flag for it. | | `IMAGE_TAG` | Overrides the configured tag for one invocation — how CI deploys what it just built. | -| `REGISTRY_USER` / `REGISTRY_TOKEN` | Registry credentials, for CI. `GHCR_USER`/`GHCR_TOKEN` are accepted aliases. | +| `REGISTRY_USER` / `REGISTRY_TOKEN` | Registry login for one invocation, over `config/local.env` — how CI logs the hosts in with its run's own `GITHUB_TOKEN`. `GHCR_USER`/`GHCR_TOKEN` are accepted aliases. | | `NO_COLOR=1` | Disable ANSI colour. | Only keys already present in the merged configuration, plus the credential and tag keys diff --git a/pyproject.toml b/pyproject.toml index 5410a48..85ae6a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,8 @@ dependencies = [ "jinja2>=3.1", # templates/ → generated/ "fastapi>=0.110", # the control panel "uvicorn[standard]>=0.29", - "python-multipart>=0.0.9", # form posts from the Configure tab + "python-multipart>=0.0.9", # form posts and uploads in the panel + "cryptography>=43", # config export/import: AES-256-GCM, scrypt ] [project.scripts] diff --git a/src/deployctl/__init__.py b/src/deployctl/__init__.py index dbfa434..504de20 100644 --- a/src/deployctl/__init__.py +++ b/src/deployctl/__init__.py @@ -1,3 +1,3 @@ """deployctl — deploy containerized apps to one server or a fleet over SSH.""" -__version__ = "0.12.0" +__version__ = "0.13.0" diff --git a/src/deployctl/cli/commands/ci.py b/src/deployctl/cli/commands/ci.py index e77ba37..c96a32c 100644 --- a/src/deployctl/cli/commands/ci.py +++ b/src/deployctl/cli/commands/ci.py @@ -31,7 +31,7 @@ from ... import __version__ from .. import bundle, github, paths, secrets, tags, ui -from ..config import Config +from ..config import PERSONAL_KEYS, Config from ..context import env_option, load_config, resolve_env from ..envfile import parse_env_text, patch_env_file, read_env_file from ..runner import run @@ -455,6 +455,14 @@ def sync_config( ui.error(f"config/secrets.{env_name}.env has no {', '.join(unminted)}") ui.hint(f"run `deployctl setup --env {env_name}` first — CI never generates secrets") raise typer.Exit(1) + # One person's registry login must never become CI's: CI logs in with the run's + # own token, and a personal token uploaded here would outlive that person's access. + personal = sorted({key for path in bundle.members(env_name) for key, value in read_env_file(path).items() + if key in PERSONAL_KEYS and value}) + if personal: + ui.error(f"{', '.join(personal)} is in the shared config — that is one person's registry login") + ui.hint("move it to config/local.env first (never uploaded): deployctl migrate-config --apply") + raise typer.Exit(1) files = _collect(env_name) local = bundle.digest(files) @@ -579,11 +587,21 @@ def doctor_items(cfg: Config, repo_level: bool = False) -> list[dict]: env = cfg.env try: github.require() + except github.GitHubError as exc: + _check(items, "github", "fail", "GitHub CLI", str(exc), "gh auth login", value="logged-out") + return items + try: info = github.repo() except github.GitHubError as exc: - _check(items, "github", "fail", "GitHub CLI", str(exc), "gh auth login") + # Logged in, and still refused: GitHub answers "not found" for a private + # repository you have not been given access to — a new teammate's case. + _check(items, "github", "fail", "GitHub", str(exc), + "ask an admin of the repository to add you (Settings → Collaborators), then check again", + value="no-access") return items - _check(items, "github", "ok", "GitHub", f"{info['name']} ({'private' if info['private'] else 'public'})") + role = info["permission"].lower() or "unknown role" + _check(items, "github", "ok", "GitHub", f"{info['name']} ({'private' if info['private'] else 'public'}) · you: {role}", + value=info["permission"]) if cfg.raw["CI_SCOPE"]: _check(items, "scope", "ok", "Where secrets live", cfg.raw["CI_SCOPE"]) @@ -613,13 +631,34 @@ def doctor_items(cfg: Config, repo_level: bool = False) -> list[dict]: else: _check(items, "workflow:ci", "todo", f".github/workflows/{OWNED}", "missing" if not ci_text else "never calls deploy.yml", f"deployctl ci init --env {env}") + if "deploy.yml" in ci_text and not deploy_job_can_pull(ci_text): + # ci.yml is the project's own file, so this is said, never rewritten: a called + # workflow gets no more permission than the job calling it grants. + _check(items, "workflow:ci-packages", "todo", f"{OWNED}: the deploy job may pull images", + "the deploy job does not grant packages: read — CI's own token cannot pull the image", + "add under the `deploy:` job in .github/workflows/ci.yml: " + "permissions: { contents: read, packages: read }") + + personal = sorted({key for path in bundle.members(env) for key, value in read_env_file(path).items() + if key in PERSONAL_KEYS and value}) + if personal: + _check(items, "personal-keys", "warn", "Your registry login is personal", + f"{', '.join(personal)} is in the shared config — exported and uploaded to CI", + "deployctl migrate-config --apply (moves it to config/local.env)") try: names = github.secret_names(scope) values = github.variables(scope) except github.GitHubError as exc: - _check(items, "secrets", "fail", f"Secrets on {_where(cfg, scope)}", str(exc), - f"deployctl ci connect --env {env}") + if info["permission"] not in github.CAN_MANAGE: + # Not missing — out of sight: GitHub shows an organisation repository's + # secrets to admins only. Saying "fail" here would send a teammate + # chasing a problem that does not exist. + _check(items, "secrets", "warn", f"Secrets on {_where(cfg, scope)}", + f"your role ({role}) cannot see them — an admin can check with: deployctl ci doctor") + else: + _check(items, "secrets", "fail", f"Secrets on {_where(cfg, scope)}", str(exc), + f"deployctl ci connect --env {env}") return items repo_values = values if scope is None else _safe_variables(None) @@ -670,6 +709,13 @@ def doctor_items(cfg: Config, repo_level: bool = False) -> list[dict]: return items +def deploy_job_can_pull(ci_text: str) -> bool: + """Whether ci.yml's `deploy:` job grants packages: read (a text check — no YAML parser).""" + match = re.search(r"^ deploy:\n((?:(?: .*|\s*)\n)*)", ci_text + "\n", re.M) + # A line of its own, not a mention in a comment: "# … packages: read …" is prose. + return bool(match) and re.search(r"^\s+packages:\s*read\b", match.group(1), re.M) is not None + + def _safe_variables(scope: str | None) -> dict[str, str]: try: return github.variables(scope) diff --git a/src/deployctl/cli/commands/deploy.py b/src/deployctl/cli/commands/deploy.py index 666c109..cd404c9 100644 --- a/src/deployctl/cli/commands/deploy.py +++ b/src/deployctl/cli/commands/deploy.py @@ -280,7 +280,10 @@ def parse_state(text: str) -> dict: for line in text.splitlines(): parts = line.split("\t") if parts[0] == "HOST" and len(parts) == 4: + # access: reachable | denied (this machine's ssh key is not accepted) | + # unreachable (no answer) — see host_access in scripts/common/remote.sh. hosts[parts[1]] = {"host": parts[1], "role": parts[2], "reachable": parts[3] == "reachable", + "access": parts[3], "tag": "", "deployed_at": "", "deployed_by": "", "services": []} elif parts[0] == "STATE" and len(parts) == 3 and parts[1] in hosts and "=" in parts[2]: key, value = parts[2].split("=", 1) diff --git a/src/deployctl/cli/commands/init.py b/src/deployctl/cli/commands/init.py index 4ddcdb5..8453af8 100644 --- a/src/deployctl/cli/commands/init.py +++ b/src/deployctl/cli/commands/init.py @@ -26,7 +26,10 @@ def init( # Paths as seen from the repository root, where these commands are typed. def shown(path): - return path.relative_to(paths.REPO_ROOT) + try: + return path.relative_to(paths.REPO_ROOT) + except ValueError: + return path written = scaffold.scaffold(env, mode) if written: diff --git a/src/deployctl/cli/commands/migrate_config.py b/src/deployctl/cli/commands/migrate_config.py index 9a0d4ba..db43c7d 100644 --- a/src/deployctl/cli/commands/migrate_config.py +++ b/src/deployctl/cli/commands/migrate_config.py @@ -1,56 +1,111 @@ """``deployctl migrate-config`` — bring a project's config/ up to the current tool. -Today it does one thing: take ``IMAGE_TAG`` out of ``config/.env``. A tag -written there is whatever that machine deployed last, and once CI deploys it is -stale by design (``cli/tags.py``). The value is not lost: it becomes this machine's -"last used" tag, so ``setup`` and ``validate`` keep working until the next deploy. +Two migrations, each shown as a plan before ``--apply`` makes it: + +``IMAGE_TAG`` out of ``config/.env`` + A tag written there is whatever that machine deployed last, and once CI + deploys it is stale by design (``cli/tags.py``). The value is not lost: it + becomes this machine's "last used" tag, so ``setup`` and ``validate`` keep + working until the next deploy. +The registry login into ``config/local.env`` + ``REGISTRY_USER``/``REGISTRY_TOKEN`` are one person's access, not the + project's (``config.PERSONAL_KEYS``). In a shared file they are exported to + everyone given the config and uploaded to CI. They move to this machine's + ``local.env``; the shared file keeps a note saying where they went. """ from __future__ import annotations import typer -from .. import paths, tags, ui -from ..envfile import assignment_key, read_env_file, write_env_file +from .. import paths, snapshots, tags, ui +from ..config import PERSONAL_KEYS +from ..envfile import assignment_key, patch_env_file, read_env_file, write_env_file #: Keys the tool no longer reads from config files. RETIRED = ("IMAGE_TAG",) +_MOVED_NOTE = "# REGISTRY_USER / REGISTRY_TOKEN: in config/local.env — each person's own, never shared" + def migrate_config( apply: bool = typer.Option(False, "--apply", help="Make the changes. Without it, only print the plan."), ) -> None: - """Remove settings the tool no longer reads from config/ (today: IMAGE_TAG). + """Bring config/ up to this version: IMAGE_TAG out, your registry login into local.env. Prints the plan unless --apply is given. A removed IMAGE_TAG becomes this - machine's last-used tag, so local commands keep working. + machine's last-used tag, so local commands keep working. Takes a snapshot of + config/ before changing it. """ - plan: list[tuple[str, str, str]] = [] # (env, key, value) + retired: list[tuple[str, str, str]] = [] # (env, key, value) for env in paths.known_environments(): stored = read_env_file(paths.config_file(env)) for key in RETIRED: if key in stored: - plan.append((env, key, stored[key])) + retired.append((env, key, stored[key])) + + # (shared file, key, value) — every assignment of a personal key in a shared file, + # empty ones included: a blank REGISTRY_TOKEN= there still invites filling it in. + shared_files = [paths.COMMON_CONFIG, *(paths.config_file(env) for env in paths.known_environments())] + personal = [ + (path, key, value) + for path in shared_files + for key, value in read_env_file(path).items() + if key in PERSONAL_KEYS + ] + local = read_env_file(paths.local_config()) ui.header("Migrate config") - if not plan: - ui.ok("nothing to migrate — config/ holds no retired settings") + if not retired and not personal: + ui.ok("nothing to migrate — config/ is current") return - for env, key, value in plan: + for env, key, value in retired: keep = f"; {value} becomes this machine's last-used tag" if key == "IMAGE_TAG" and value else "" print(f" remove {key}={value} from config/{env}.env{keep}") + for path, key, value in personal: + if not value: + print(f" remove {key} (empty) from config/{path.name}") + elif local.get(key): + print(f" remove {key} from config/{path.name} — config/local.env already has yours") + else: + print(f" move {key} from config/{path.name} to config/local.env (yours, never shared)") if not apply: ui.separator() ui.info("nothing changed — this was the plan. Run again with --apply to make it.") return - for env in sorted({env for env, _, _ in plan}): + snapshots.take("migrate-config") + + for env in sorted({env for env, _, _ in retired}): path = paths.config_file(env) - lines = path.read_text().splitlines() - kept = [line for line in lines if assignment_key(line) not in RETIRED] + kept = [line for line in path.read_text().splitlines() if assignment_key(line) not in RETIRED] write_env_file(path, "\n".join(kept) + "\n") - for env, key, value in plan: + for env, key, value in retired: if key == "IMAGE_TAG" and value and not tags.cached(env): tags.remember(env, value) - ui.ok(f"migrated {len(plan)} setting(s) — deploys now take the tag from --tag or the running hosts") + + moving = {key: value for _, key, value in personal if value and not local.get(key)} + if moving: + if not paths.local_config().is_file(): + from ..scaffold import LOCAL_STUB # the explained stub, then the values + + write_env_file(paths.local_config(), LOCAL_STUB) + patch_env_file(paths.local_config(), moving) + for path in sorted({path for path, _, _ in personal}): + out, noted = [], False + for line in path.read_text().splitlines(): + if assignment_key(line) in PERSONAL_KEYS: + if not noted: + out.append(_MOVED_NOTE) + noted = True + continue + out.append(line) + write_env_file(path, "\n".join(out) + "\n") + + ui.ok(f"migrated {len(retired) + len(personal)} setting(s)") + if retired: + ui.info("deploys now take the image tag from --tag or the running hosts") + if personal: + ui.info("your registry login is in config/local.env: yours on this machine, never exported or uploaded") + ui.hint("CI logs in with its own token once its workflows are current: deployctl ci init --force") diff --git a/src/deployctl/cli/commands/show.py b/src/deployctl/cli/commands/show.py index b5d1df5..b2c600a 100644 --- a/src/deployctl/cli/commands/show.py +++ b/src/deployctl/cli/commands/show.py @@ -1,4 +1,4 @@ -"""``deployctl config`` — inspect the resolved configuration.""" +"""``deployctl config`` — inspect the resolved configuration; hand it on (export/import).""" from __future__ import annotations @@ -6,6 +6,25 @@ from .. import paths, ui from ..context import env_option, load_config +from . import transfer as transfer_cmd + +config_app = typer.Typer( + name="config", + help="Show this project's configuration, or hand it on: export to one encrypted file, import one.", + invoke_without_command=True, +) + + +@config_app.callback() +def config_default( + ctx: typer.Context, + env: str = env_option(), + show_secrets: bool = typer.Option(False, "--show-secrets", help="Print secret values in full."), + key: str = typer.Option(None, "--key", "-k", help="Print one value and nothing else (script-friendly)."), +) -> None: + """Plain `deployctl config` is `deployctl config show`, as it always was.""" + if ctx.invoked_subcommand is None: + config(env=env, show_secrets=show_secrets, key=key) def config( @@ -43,6 +62,11 @@ def config( ui.info(f"config files: {paths.CONFIG_DIR.relative_to(paths.ROOT)}/{{common,{cfg.env},secrets.{cfg.env}}}.env") +config_app.command("show")(config) +config_app.command("export")(transfer_cmd.export) +config_app.command("import")(transfer_cmd.import_) + + def envs() -> None: """List the configured environments.""" known = paths.known_environments() diff --git a/src/deployctl/cli/commands/transfer.py b/src/deployctl/cli/commands/transfer.py new file mode 100644 index 0000000..332a50c --- /dev/null +++ b/src/deployctl/cli/commands/transfer.py @@ -0,0 +1,183 @@ +"""``deployctl config export`` / ``import`` — hand a project's configuration on, as one file. + +See cli/transfer.py for what goes in (every environment's shared files) and what +never does (each person's own access). The passphrase comes from a prompt, from +``--generate-passphrase``, or from ``$DEPLOYCTL_PASSPHRASE`` — how the control +panel passes it: never as an argument, where other processes could read it. +""" + +from __future__ import annotations + +import json +import os +import pathlib + +import typer + +from .. import paths, snapshots, transfer, ui +from ..envfile import read_env_file + +_PASSPHRASE_ENV = "DEPLOYCTL_PASSPHRASE" + + +def _emit(as_json: bool, doc: dict) -> None: + if as_json: + print(json.dumps(doc, indent=2)) + + +def _fail(as_json: bool, message: str, hint: str = "") -> None: + if as_json: + _emit(True, {"ok": False, "error": message}) + else: + ui.error(message) + if hint: + ui.hint(hint) + raise typer.Exit(1) + + +def _your_access() -> dict: + """What this machine still needs that no config file can give it.""" + local = read_env_file(paths.local_config()) + return {"registry": bool(local.get("REGISTRY_USER") and local.get("REGISTRY_TOKEN"))} + + +def _print_access(access: dict) -> None: + ui.info("What is yours to set up — never part of a shared config:") + print(f" {'✓' if access['registry'] else '·'} your registry login config/local.env " + "(REGISTRY_USER + a GitHub token with read:packages) — the panel's Configure tab") + print(" · your ssh key on the servers send your public key to whoever runs the project; " + "check with: deployctl deploy status") + print(" · your GitHub login gh auth login — for the CI/CD tab and deploying through GitHub") + + +def export( + env: list[str] = typer.Option(None, "--env", "-e", help="An environment to include (repeatable). Default: every one."), + output: pathlib.Path = typer.Option(None, "--output", "-o", help="Where to write it. Default: config/exports/."), + generate: bool = typer.Option(False, "--generate-passphrase", help="Make a strong passphrase and print it once."), + as_json: bool = typer.Option(False, "--json", help="Machine-readable, for the control panel."), +) -> None: + """Write this project's configuration to one encrypted file — for a teammate, or a backup. + + Holds every shared file of every environment (or those given with --env): all a + project needs to run. Never holds anyone's registry login (config/local.env), + ssh keys or GitHub login. Written to config/exports/ unless -o says otherwise. + """ + environments = env or paths.known_environments() + if not environments: + _fail(as_json, "no environments configured — nothing to export") + + generated = "" + passphrase = os.environ.get(_PASSPHRASE_ENV, "") + if not passphrase and generate: + passphrase = generated = transfer.generate_passphrase() + if not passphrase: + if as_json: + _fail(True, f"no passphrase: set ${_PASSPHRASE_ENV} or pass --generate-passphrase") + passphrase = typer.prompt(f"Passphrase (at least {transfer.PASSPHRASE_MIN} characters)", + hide_input=True, confirmation_prompt=True) + + try: + path, summary = transfer.export(environments, passphrase, output.resolve() if output else None) + except transfer.TransferError as exc: + _fail(as_json, str(exc)) + + if as_json: + _emit(True, {"ok": True, "path": str(path), "name": path.name, "summary": summary, + **({"passphrase": generated} if generated else {})}) + return + ui.header("Config export") + ui.ok(f"wrote {path}") + ui.kv("environments", ", ".join(summary["environments"])) + ui.kv("files", ", ".join(summary["files"])) + ui.kv("never included", "anyone's registry login (config/local.env), ssh keys, GitHub logins") + if generated: + print() + ui.kv("passphrase", generated) + ui.warn("shown once — send it by a different channel than the file") + print() + ui.info("They import it with: deployctl config import (or drop it in config/imports/ and " + "run deployctl config import — the panel offers the same)") + + +def import_( + file: pathlib.Path = typer.Argument(None, help="The file. Default: the one waiting in config/imports/."), + preview: bool = typer.Option(False, "--preview", help="Show what would change and write nothing."), + force: bool = typer.Option(False, "--force", help="Replace a configuration that differs (after a snapshot)."), + keep: bool = typer.Option(False, "--keep", help="Keep the file after importing."), + as_json: bool = typer.Option(False, "--json", help="Machine-readable, for the control panel."), +) -> None: + """Load a configuration exported with `deployctl config export` into this project. + + Refuses a file made for another repository, and a configuration that differs + from the one here unless --force (which snapshots the current one first). What + stays yours to set up — your registry login, ssh key and GitHub login — is listed + afterwards. + """ + try: + path = transfer.pick_import(file) + except transfer.TransferError as exc: + _fail(as_json, str(exc)) + + passphrase = os.environ.get(_PASSPHRASE_ENV, "") + if not passphrase: + if as_json: + _fail(True, f"no passphrase: set ${_PASSPHRASE_ENV}") + passphrase = typer.prompt("Passphrase", hide_input=True) + + try: + payload = transfer.unseal(path.read_bytes(), passphrase) + transfer.check_home(payload) + except transfer.TransferError as exc: + _fail(as_json, str(exc)) + + summary = {key: payload[key] for key in ("project", "repository", "environments", + "created_at", "created_by", "deployctl")} + plan = transfer.compare(payload) + access = _your_access() + + if preview: + if as_json: + _emit(True, {"ok": True, "stage": "preview", "file": path.name, "summary": summary, + "plan": plan, "access": access}) + return + ui.header("Config import — preview") + _print_summary(summary, plan) + return + + if plan["differs"] and not force: + changed = "; ".join(f"{name}: {', '.join(keys)}" for name, keys in plan["differs"].items()) + _fail(as_json, f"the configuration here differs — {changed}", + "--force replaces it (a snapshot of the current one is taken first); --preview shows the plan") + + snapshot = snapshots.take("import") if plan["differs"] else None + written = transfer.apply(payload) if plan["new"] or plan["differs"] else [] + if path.resolve().parent == paths.imports_dir().resolve() and not keep: + path.unlink(missing_ok=True) + + if as_json: + _emit(True, {"ok": True, "stage": "applied", "summary": summary, "written": written, + "snapshot": str(snapshot) if snapshot else None, "access": _your_access()}) + return + ui.header("Config import") + _print_summary(summary, plan) + if written: + ui.ok(f"imported {len(written)} file(s)" + (f"; the previous ones are in {snapshot}" if snapshot else "")) + else: + ui.ok("already current — nothing written") + print() + _print_access(_your_access()) + ui.hint(f"then check you match what CI deploys: deployctl ci status --env {payload['environments'][0]}") + + +def _print_summary(summary: dict, plan: dict) -> None: + ui.kv("project", f"{summary['project']} ({summary['repository'] or 'repository unknown'})") + ui.kv("exported", f"{summary['created_at']} by {summary['created_by']} (deployctl {summary['deployctl']})") + ui.kv("environments", ", ".join(summary["environments"])) + for name in plan["new"]: + print(f" new config/{name}") + for name, keys in plan["differs"].items(): + print(f" replace config/{name} ({len(keys)} differ: {', '.join(keys)})") + for name in plan["same"]: + print(f" same config/{name}") + for name in plan["kept"]: + print(f" kept config/{name} (not in the file; left as it is)") diff --git a/src/deployctl/cli/commands/webui.py b/src/deployctl/cli/commands/webui.py index 33dd507..55e1276 100644 --- a/src/deployctl/cli/commands/webui.py +++ b/src/deployctl/cli/commands/webui.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os import signal import subprocess @@ -11,6 +12,28 @@ import typer from .. import paths, ui +from ..transfer import project_name + + +def _registry(port: int): + """``~/.deployctl/panels/.json``: which project the panel on a port serves.""" + return paths.state_home() / "panels" / f"{port}.json" + + +def _recorded(port: int) -> dict | None: + """The project a running panel serves — None if unknown, or if its process is gone.""" + try: + record = json.loads(_registry(port).read_text()) + os.kill(int(record["pid"]), 0) + except (OSError, ValueError, KeyError, TypeError): + return None + return record + + +def _record(port: int) -> None: + path = _registry(port) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps({"project": project_name(), "root": str(paths.ROOT), "pid": os.getpid()})) def _listener_pid(port: int) -> int | None: @@ -91,25 +114,35 @@ def webui( if existing is not None: ui.error(f"port {port} is already in use") + other = _recorded(port) + if other and other.get("root") != str(paths.ROOT): + # The dangerous case: "open it" would hand this operator another project's panel. + ui.warn(f"it is the panel for {other.get('project')} ({other.get('root')}) — not this project") + print() + ui.info(f"Run this project's panel beside it: deployctl webui --port {port + 1}") + raise typer.Exit(1) ui.hint(f"held by: {_describe(existing)}") print() ui.info("Either reuse it, stop it, or pick another port:") - print(f" open http://127.0.0.1:{port} # it is probably already the panel") - print(f" deployctl webui --restart # stop that one and start fresh") - print(f" deployctl webui --stop # just stop it") + if other: + print(f" open http://127.0.0.1:{port} # it is this project's panel") + else: + print(f" open http://127.0.0.1:{port} # probably a panel — check which project it shows") + print(" deployctl webui --restart # stop that one and start fresh") + print(" deployctl webui --stop # just stop it") print(f" deployctl webui --port {port + 1} # run alongside it") raise typer.Exit(1) - if not paths.known_environments(): - ui.error("no environments configured yet") - ui.hint("run: deployctl init --mode single|cluster") + if not paths.is_project_root(paths.ROOT): + ui.error(f"no deployctl project here ({paths.ROOT} has no project/project.env)") + ui.hint("run it from inside the application's repository — or start one: deployctl init --mode single|cluster") raise typer.Exit(2) try: import uvicorn # noqa: F401 except ImportError: ui.error("the panel's dependencies are missing from this installation") - ui.hint("reinstall deployctl: uv tool install --force git+ssh://git@github.com/hybridinteract/deployctl") + ui.hint("reinstall deployctl: uv tool install --force git+https://github.com/hybridinteract/deployctl") raise typer.Exit(1) from None env = os.environ.copy() @@ -117,7 +150,10 @@ def webui( env["DEPLOYCTL_PROJECT"] = str(paths.ROOT) ui.header("deployctl control panel") + ui.kv("serving", f"{project_name()} ({paths.ROOT})") ui.info(f"http://127.0.0.1:{port} (Ctrl+C to stop)") + if not paths.known_environments(): + ui.info("no configuration yet — the panel opens on Import (a teammate's export) or how to start one") ui.warn("localhost only, no authentication — never expose this port") print() @@ -127,6 +163,7 @@ def webui( "--host", "127.0.0.1", "--port", str(port)] if reload: argv += ["--reload", "--reload-dir", str(paths.TOOL_ROOT)] + _record(port) try: subprocess.run(argv, cwd=str(paths.workdir()), env=env, check=True) except KeyboardInterrupt: @@ -134,3 +171,5 @@ def webui( ui.info("panel stopped") except subprocess.CalledProcessError as exc: raise typer.Exit(exc.returncode) from exc + finally: + _registry(port).unlink(missing_ok=True) diff --git a/src/deployctl/cli/config.py b/src/deployctl/cli/config.py index 713f32c..35239e7 100644 --- a/src/deployctl/cli/config.py +++ b/src/deployctl/cli/config.py @@ -46,6 +46,11 @@ "GHCR_TOKEN", }) +#: One operator's own access, not the project's: kept in config/local.env, never +#: exported, never uploaded to CI (which logs in with its own short-lived token). +#: Each person who deploys sets their own; the shared config never holds them. +PERSONAL_KEYS = frozenset({"REGISTRY_USER", "REGISTRY_TOKEN", "GHCR_USER", "GHCR_TOKEN"}) + #: Legacy names accepted as aliases: alias -> canonical. ALIASES = { "GHCR_USER": "REGISTRY_USER", @@ -372,6 +377,24 @@ def warn(message: str, hint: str = "") -> None: "REGISTRY_TOKEN is set but REGISTRY_USER is empty", "docker login needs both; the token alone cannot authenticate", ) + # One person's access belongs to that person (PERSONAL_KEYS): in a shared file + # it is exported to everyone and uploaded to CI. + shared = {**read_env_file(paths.COMMON_CONFIG), **read_env_file(paths.config_file(self.env))} + misplaced = sorted(key for key in PERSONAL_KEYS if shared.get(key)) + if misplaced: + warn( + f"{', '.join(misplaced)} is in the shared config — that is one person's registry login", + "it is exported to anyone given the config and uploaded to CI; move it to config/local.env: " + "deployctl migrate-config --apply", + ) + ignored = sorted(key for key in read_env_file(paths.local_config()) if key not in PERSONAL_KEYS) + if ignored: + warn( + f"config/local.env: {', '.join(ignored)} ignored", + "local.env holds only your registry login (REGISTRY_USER, REGISTRY_TOKEN) — anything else " + "would make this machine deploy what CI does not; put shared values in config/common.env " + f"or config/{self.env}.env", + ) if self.raw_input.get("IMAGE_TAG") and not os.environ.get("IMAGE_TAG"): warn( @@ -583,6 +606,10 @@ def load(env: str) -> Config: _layer(merged, common, "config/common.env", sources) _layer(merged, env_values, f"config/{env}.env", sources) _layer(merged, read_env_file(paths.secrets_file(env)), f"config/secrets.{env}.env", sources) + # This machine's own access, over the shared files — and only that: any other key + # here would make one operator deploy something CI does not (validate says so). + local = {k: v for k, v in read_env_file(paths.local_config()).items() if k in PERSONAL_KEYS} + _layer(merged, local, "config/local.env", sources) # Aliases, before the environment layer so GHCR_TOKEN in the shell still works. for alias, canonical in ALIASES.items(): diff --git a/src/deployctl/cli/fields.py b/src/deployctl/cli/fields.py index 6714dbf..94d659a 100644 --- a/src/deployctl/cli/fields.py +++ b/src/deployctl/cli/fields.py @@ -12,6 +12,8 @@ ``env`` → ``config/.env`` (this environment) ``app`` → ``config/app..env`` (project application keys, re-applied after every regenerate) +``local`` → ``config/local.env`` (this machine's own registry login — never + exported, never uploaded to CI) """ from __future__ import annotations @@ -22,7 +24,7 @@ from . import paths -VALID_TARGETS = ("common", "env", "app") +VALID_TARGETS = ("common", "env", "app", "local") VALID_TYPES = ("text", "password", "number", "select", "textarea") diff --git a/src/deployctl/cli/github.py b/src/deployctl/cli/github.py index c97cbca..e059e2f 100644 --- a/src/deployctl/cli/github.py +++ b/src/deployctl/cli/github.py @@ -45,8 +45,12 @@ def require() -> None: def repo() -> dict: - """``{"name": "owner/repo", "owner": ..., "private": bool, "default_branch": ...}``.""" - out = _ok(gh(["repo", "view", "--json", "nameWithOwner,owner,visibility,defaultBranchRef"]), + """``{"name": "owner/repo", "owner", "private", "default_branch", "permission"}``. + + ``permission`` is the logged-in user's role on it — ADMIN, MAINTAIN, WRITE, + TRIAGE or READ: what decides which CI/CD actions they can take. + """ + out = _ok(gh(["repo", "view", "--json", "nameWithOwner,owner,visibility,defaultBranchRef,viewerPermission"]), "reading the repository") data = json.loads(out) return { @@ -54,9 +58,17 @@ def repo() -> dict: "owner": data["owner"]["login"], "private": data["visibility"] != "PUBLIC", "default_branch": (data.get("defaultBranchRef") or {}).get("name", ""), + "permission": data.get("viewerPermission") or "", } +#: What each role may do here: WRITE runs deploys and rollbacks; changing secrets +#: and variables (sync-config, setup-key, pin-hosts, auto-deploy) needs ADMIN — +#: GitHub requires it for an organisation repository's secrets. +CAN_DEPLOY = frozenset({"ADMIN", "MAINTAIN", "WRITE"}) +CAN_MANAGE = frozenset({"ADMIN"}) + + def plan(owner: str) -> str | None: """The owner's plan (``free``, ``team``, …), or None when it cannot be seen. diff --git a/src/deployctl/cli/main.py b/src/deployctl/cli/main.py index b2dae2e..7536ccf 100644 --- a/src/deployctl/cli/main.py +++ b/src/deployctl/cli/main.py @@ -73,7 +73,7 @@ def _global_options( app.command("init")(init_cmd.init) app.command("setup")(setup_cmd.setup) app.command("validate")(validate_cmd.validate) -app.command("config")(show_cmd.config) +app.add_typer(show_cmd.config_app) app.command("envs")(show_cmd.envs) app.command("selftest")(selftest_cmd.selftest) app.command("webui")(webui_cmd.webui) diff --git a/src/deployctl/cli/paths.py b/src/deployctl/cli/paths.py index 867990e..b144a7a 100644 --- a/src/deployctl/cli/paths.py +++ b/src/deployctl/cli/paths.py @@ -155,6 +155,32 @@ def app_values_file(env: str) -> pathlib.Path: return CONFIG_DIR / f"app.{env}.env" +def state_home() -> pathlib.Path: + """deployctl's own folder in the user's home: database-dump copies, config + snapshots, running panels. ``$DEPLOYCTL_HOME`` moves it (the tests do).""" + return pathlib.Path(os.environ.get("DEPLOYCTL_HOME") or "~/.deployctl").expanduser() + + +def local_config() -> pathlib.Path: + """This machine's own values: the operator's registry login (``config/local.env``). + + Beside the shared files, loaded over them, and never exported or uploaded — two + people deploying one project each keep their own credentials here, so the shared + config stays identical on every machine and in CI. + """ + return CONFIG_DIR / "local.env" + + +def exports_dir() -> pathlib.Path: + """Where ``deployctl config export`` writes, unless told otherwise.""" + return CONFIG_DIR / "exports" + + +def imports_dir() -> pathlib.Path: + """Where ``deployctl config import`` looks when given no file.""" + return CONFIG_DIR / "imports" + + def profile_file(mode: str) -> pathlib.Path: """Path to a mode's default profile (``profiles/.env``).""" return PROFILES_DIR / f"{mode}.env" @@ -163,16 +189,16 @@ def profile_file(mode: str) -> pathlib.Path: def known_environments() -> list[str]: """Environment names that have a config file. - ``common.env`` is shared, and ``secrets..env`` / ``app..env`` hold - values for an environment that is already listed via its own file — none of - them is an environment in its own right. + ``common.env`` is shared, ``local.env`` is this machine's, and + ``secrets..env`` / ``app..env`` hold values for an environment that is + already listed via its own file — none of them is an environment in its own right. """ if not CONFIG_DIR.is_dir(): return [] return sorted( p.stem for p in CONFIG_DIR.glob("*.env") - if p.stem != "common" and not p.name.startswith(("secrets.", "app.")) + if p.stem not in ("common", "local") and not p.name.startswith(("secrets.", "app.")) ) diff --git a/src/deployctl/cli/scaffold.py b/src/deployctl/cli/scaffold.py index 52900a7..c9a2de1 100644 --- a/src/deployctl/cli/scaffold.py +++ b/src/deployctl/cli/scaffold.py @@ -16,7 +16,7 @@ # ============================================================================ # Shared configuration — applies to every environment. # Per-environment settings live in config/.env and win over this file. -# NEVER COMMIT: this file holds registry credentials. +# Shared with everyone who deploys (deployctl config export). NEVER COMMIT. # ============================================================================ # Container/network/volume name prefix. Lowercase letters, digits and hyphens; @@ -30,10 +30,20 @@ # machine (deployctl image push). Every host pulls from here; nothing is ever # built on a deployment target. IMAGE_REPO=ghcr.io/your-org/your-app +""" + +LOCAL_STUB = """\ +# ============================================================================ +# YOUR access — this machine only. Never exported, never uploaded to GitHub. +# Everyone who deploys this project keeps their own copy of this file; CI logs +# in to the registry with its own short-lived token instead. +# NEVER COMMIT. +# ============================================================================ -# Registry credentials used to `docker login` on each host before pulling. -# A GitHub classic PAT with ONLY the read:packages scope is enough; if the org -# uses SSO, authorize the token for it. GHCR_USER/GHCR_TOKEN also work. +# Your registry login: used to `docker login` on each host when YOU deploy from +# this machine, and to list image tags. A GitHub classic PAT with ONLY the +# read:packages scope is enough (write:packages to `deployctl image push`); if +# the org uses SSO, authorize the token for it. GHCR_USER/GHCR_TOKEN also work. REGISTRY_USER= REGISTRY_TOKEN= """ @@ -268,6 +278,8 @@ # config/ holds the secrets, generated/ is rebuilt from it on every deploy. config/*.env !config/*.env.example +config/exports/ +config/imports/ generated/* !generated/.gitkeep backups/ @@ -296,6 +308,8 @@ def scaffold(env: str, mode: str) -> list[pathlib.Path]: if _write_if_missing(paths.COMMON_CONFIG, COMMON_STUB, secret=True): written.append(paths.COMMON_CONFIG) + if _write_if_missing(paths.local_config(), LOCAL_STUB, secret=True): + written.append(paths.local_config()) stub = SINGLE_STUB if mode == "single" else CLUSTER_STUB api_subdomain = "api" if env == "production" else f"api-{env}" diff --git a/src/deployctl/cli/snapshots.py b/src/deployctl/cli/snapshots.py new file mode 100644 index 0000000..462b260 --- /dev/null +++ b/src/deployctl/cli/snapshots.py @@ -0,0 +1,53 @@ +""" +Automatic copies of ``config/`` taken before deployctl overwrites it. + +A panel Save, ``config import --force`` and ``migrate-config --apply`` all rewrite +files an operator cannot regenerate — the secrets in them were minted once, and +GitHub's copy cannot be read back. Each takes a snapshot first, into +``~/.deployctl/config-backups//