diff --git a/.socraticodecontextartifacts.json b/.socraticodecontextartifacts.json index 0beb30c0..aac6e626 100644 --- a/.socraticodecontextartifacts.json +++ b/.socraticodecontextartifacts.json @@ -98,7 +98,7 @@ { "name": "dependencies-reference", "path": "./docs/DEPENDENCIES.md", - "description": "Where co-core comes from and its pin (#183). co-core, co-core-aio and co-core-sync resolve from cannobserv's private co-pypi Artifact Registry index, declared explicit with a [tool.uv.sources] entry each, the lock carrying every file's sha256; it replaced the GCS wheelhouse and scripts/sync_wheelhouse.py. Which commands need the index token (any uv lock, a cold-cache uv sync) and which don't (uv run, uv lock --check, the unit's ExecStart, even offline). The wrapper scripts/co-pypi.sh: one read-only key, REPLICATOR_CO_PYPI_CREDENTIALS, never the writer's GOOGLE_APPLICATION_CREDENTIALS; an allowlisted environment so neither the writer's key path nor the repo .env PATs reach uv; the token never printed. CI's WIF step hands the token to uv sync alone. Moving the pin: edit, wrapped uv lock, `co-pypi.sh wheels OLD NEW DIR`, then scripts/co_core_closure_diff.py OLD NEW --wheels DIR. Then the pin itself (moved from docs/DEPLOYMENT.md): pin the current minor, why the patch floor is load-bearing under extra=\"ignore\", the 0.10 -> 0.13 jump taken on evidence, and the import-closure diff every pin move takes, its verdicts and the two lazy __getattr__ sites it lists as expected (#145). Last, the uv_build build-backend bound (#195): one minor, the host's uv's; ci.yml's setup-uv version: inputs repeat it verbatim; bump when uv sync warns the bound excludes the running uv; the warning is noise (uv still direct-builds, no fetch); required-version rejected because it would refuse the unit's uv run." + "description": "Where co-core comes from and its pin (#183). co-core, co-core-aio and co-core-sync resolve from cannobserv's private co-pypi Artifact Registry index, declared explicit with a [tool.uv.sources] entry each, the lock carrying every file's sha256; it replaced the GCS wheelhouse and scripts/sync_wheelhouse.py. Which commands need the index token (any uv lock, a cold-cache uv sync) and which don't (uv run, uv lock --check, the unit's ExecStart, even offline). The wrapper scripts/co-pypi.sh is deploying-releases' shared copy, never edited (#201): one read-only key, CO_PYPI_KEY else /etc/replicator/co-pypi-reader.json, never the writer's GOOGLE_APPLICATION_CREDENTIALS; an allowlisted environment so neither the writer's key path nor the repo .env PATs reach uv; the token never printed; run from the lock's non-default co-pypi group. CI's WIF step hands the token to uv sync alone. Moving the pin: edit, wrapped uv lock, scripts/co_pypi_wheels.py under the wrapper (uv has no download command), then scripts/co_core_closure_diff.py OLD NEW --wheels DIR. Then the pin itself (moved from docs/DEPLOYMENT.md): pin the current minor, why the patch floor is load-bearing under extra=\"ignore\", the 0.10 -> 0.13 jump taken on evidence, and the import-closure diff every pin move takes, its verdicts and the two lazy __getattr__ sites it lists as expected (#145). Last, the uv_build build-backend bound (#195): one minor, the host's uv's; ci.yml's setup-uv version: inputs repeat it verbatim; bump when uv sync warns the bound excludes the running uv; the warning is noise (uv still direct-builds, no fetch); required-version rejected because it would refuse the unit's uv run." }, { "name": "memory-protection-reference", @@ -148,7 +148,7 @@ { "name": "co-pypi-wrapper", "path": "./scripts/co_pypi_token.py", - "description": "The co-pypi index wrapper (#183), run as `bash scripts/co-pypi.sh uv …` or `… wheels OLD NEW DIR`. Mints an Artifact Registry access token from the read-only key REPLICATOR_CO_PYPI_CREDENTIALS names — never GOOGLE_APPLICATION_CREDENTIALS, which on this host is the worker's writer — and execs uv with UV_INDEX_CO_PYPI_USERNAME/PASSWORD from an allowlisted environment (co-pypi.sh applies the same list to the outer uv run), so the writer's key path and the repo .env PATs never reach uv. The wheels verb fetches co-core wheels for scripts/co_core_closure_diff.py, sha256-checked against the index, following same-host redirects only. The token is never printed, logged, put in argv or written. CI never runs it: its WIF step hands the token to uv sync." + "description": "The co-pypi index wrapper (#183), deploying-releases' shared marked copy since #201 (install.sh --co-pypi; never edited, tests/test_co_pypi.py checks the marker), run as `bash scripts/co-pypi.sh uv …`. Mints an Artifact Registry access token from the read-only key CO_PYPI_KEY names, else /etc/<[project].name>/co-pypi-reader.json (here /etc/replicator/co-pypi-reader.json) — never GOOGLE_APPLICATION_CREDENTIALS, which on this host is the worker's writer — and execs uv with UV_INDEX_CO_PYPI_USERNAME/PASSWORD from an allowlisted environment (co-pypi.sh applies the same list to the outer uv run, which runs the lock's co-pypi group --frozen), so the writer's key path and the repo .env PATs never reach uv. The token is never printed, logged, put in argv or written. Wheel files for the closure diff come from scripts/co_pypi_wheels.py, run under it. CI never runs it: its WIF step hands the token to uv sync." }, { "name": "redis-floor-guard", @@ -168,7 +168,7 @@ { "name": "release-deploy-hooks", "path": "./scripts/deploy_hooks.sh", - "description": "This repo's half of the release deploy (#151), sourced by scripts/deploy.sh, the deploying-releases core (gregoryfoster/skills#372) installed as a marked copy and never edited. CI=github anonymous with the four ci.yml job display names, UV_WRAPPER=scripts/co-pypi.sh (this repo's own until #201) with REPLICATOR_CO_PYPI_CREDENTIALS defaulted to /etc/replicator/co-pypi-reader.json, src and scripts compiled, src.worker.main and scripts.inflight import-probed, five HOST_CONFIGS compared never installed, KEEP=0 for disk. post_build_check runs the release's own #52 writer guard (src/worker/checkout.py) so a release that would refuse its writers never goes live; verify proves replicator.service with verify_started_on, then its \"worker ready\" journal line naming the build. Held by tests/deploy/. Design: docs/plans/2026-11-14-releases-design.md." + "description": "This repo's half of the release deploy (#151), sourced by scripts/deploy.sh, the deploying-releases core (gregoryfoster/skills#372) installed as a marked copy and never edited. CI=github anonymous with the four ci.yml job display names, UV_WRAPPER=scripts/co-pypi.sh (the shared wrapper since #201, whose key defaults to /etc/replicator/co-pypi-reader.json, so no key variable is exported), src and scripts compiled, src.worker.main and scripts.inflight import-probed, five HOST_CONFIGS compared never installed, KEEP=0 for disk. post_build_check runs the release's own #52 writer guard (src/worker/checkout.py) so a release that would refuse its writers never goes live; verify proves replicator.service with verify_started_on, then its \"worker ready\" journal line naming the build. Held by tests/deploy/. Design: docs/plans/2026-11-14-releases-design.md." } ] } diff --git a/README.md b/README.md index ddb164e8..af704879 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ reads `XAUTOCLAIM`'s three-element reply, added in Redis server 7.0. ```bash # co-core, co-core-aio and co-core-sync come from the private co-pypi index, whose token -# the wrapper mints from REPLICATOR_CO_PYPI_CREDENTIALS (docs/DEPENDENCIES.md). +# the shared wrapper mints from /etc/replicator/co-pypi-reader.json (docs/DEPENDENCIES.md). . scripts/load_env.sh bash scripts/co-pypi.sh uv sync uv run pre-commit install @@ -118,7 +118,6 @@ rather than deleting bytes a consumer was promised. | Variable | Set on this VM | Purpose | |---|---|---| | `GOOGLE_APPLICATION_CREDENTIALS` | `/etc/replicator/co-gcs-replicator-writer.json` | The worker's ADC — the writer SA (`co-gcs-replicator-writer@co-gcs`, #114), never the co-pypi reader | -| `REPLICATOR_CO_PYPI_CREDENTIALS` | `/etc/replicator/co-pypi-reader.json` | Read-only key for the co-pypi index, read by `scripts/co-pypi.sh` in the deploy shell, never by the worker | | `REPLICATOR_REDIS_URL` | `redis://replicator:@broker:6379/0` | Change-bus client URL — the `replicator` ACL user on `co-broker`, over the tailnet | | `REPLICATOR_BLOB_BACKEND` | `gcs` | Temp blobs live in an object store — **not** the `local` default | | `REPLICATOR_BLOB_BUCKET` | `co-gcs-blobs` | The temp-blob bucket that backend writes into | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index baaf5ba9..37433ebf 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -64,7 +64,7 @@ src/worker/checkout.py — Is this checkout main's code? Asked before a write id src/core/logging.py — build_json_formatter() + ColorMessageFilter + configure_logging() + get_logger() src/core/log_config.json — uvicorn --log-config; routes uvicorn's own loggers through that formatter src/core/config.py — Settings / env access (see Environment Variables) -scripts/ — co-pypi.sh + co_pypi_token.py, co_core_closure_diff.py, check_redis_floor.sh, rcli.sh, load_env.sh, deploy.sh (the deploying-releases core, never edited) + deploy_hooks.sh, notify_failure.sh, browser_exec.sh, seed_fetch.py, seed_capture.py, inflight.py, rehearse_reconnect.sh +scripts/ — co-pypi.sh + co_pypi_token.py (deploying-releases' copies, never edited), co_pypi_wheels.py, co_core_closure_diff.py, check_redis_floor.sh, rcli.sh, load_env.sh, deploy.sh (the deploying-releases core, never edited) + deploy_hooks.sh, notify_failure.sh, browser_exec.sh, seed_fetch.py, seed_capture.py, inflight.py, rehearse_reconnect.sh scripts/rcli.sh — sourced: redis-cli with the URL's credential in REDISCLI_AUTH, never argv (#127) scripts/load_env.sh — sourced: parses both env files into the shell, never runs them (#170) scripts/notify_failure.sh — the OnFailure= handler's body: records the incident, dispatches it when configured (#94) diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 75ff5301..28779354 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -3,10 +3,11 @@ ## Setup ```bash -. scripts/load_env.sh # REPLICATOR_CO_PYPI_CREDENTIALS, the index's read-only key bash scripts/co-pypi.sh uv sync # co-core comes from the private co-pypi index bash scripts/co-pypi.sh uv lock # every relock, even a PyPI-only one -bash scripts/co-pypi.sh wheels OLD NEW DIR # every pin move: fetch, then diff the closure +# every pin move: fetch the wheels, then diff the closure +bash scripts/co-pypi.sh uv run --no-project python scripts/co_pypi_wheels.py \ + --dest DIR --version OLD --version NEW co-core co-core-aio co-core-sync uv run --no-project python scripts/co_core_closure_diff.py OLD NEW --wheels DIR uv run pre-commit install ``` diff --git a/docs/DEPENDENCIES.md b/docs/DEPENDENCIES.md index 0207e823..30dbf27b 100644 --- a/docs/DEPENDENCIES.md +++ b/docs/DEPENDENCIES.md @@ -24,39 +24,43 @@ package: uv asks it for those three only, so neither side can shadow the other. | any `uv lock`, even a PyPI-only bump; `uv sync` on a cold cache | **required**: a 401 on co-core without it | So every release build, a dev venv's sync and every relock go through the wrapper. -`scripts/deploy.sh` runs it as `UV_WRAPPER`, and `scripts/deploy_hooks.sh` names the -key, so a deploy needs no `load_env.sh` (#151): +`scripts/deploy.sh` runs it as `UV_WRAPPER` (#151); neither it nor a shell needs an +env file for the key: ```bash -. scripts/load_env.sh # REPLICATOR_CO_PYPI_CREDENTIALS bash scripts/co-pypi.sh uv sync --frozen # a dev venv bash scripts/co-pypi.sh uv lock ``` -**The wrapper.** `scripts/co-pypi.sh` runs `scripts/co_pypi_token.py`, which mints an -access token from the read-only key `REPLICATOR_CO_PYPI_CREDENTIALS` names -(`co-pypi-reader@`, `/etc/replicator/co-pypi-reader.json`) and `exec`s uv with -`UV_INDEX_CO_PYPI_USERNAME` / `_PASSWORD`. Each rule is tested -(`tests/test_co_pypi_token.py`): - -- **One key, no fallback.** Never `GOOGLE_APPLICATION_CREDENTIALS`: on this host - that is the worker's *writer* (#114). Unset, the wrapper exits 2 and names the - variable. +**The wrapper is the cohort's, never edited here** (#201). `scripts/co-pypi.sh` and +`scripts/co_pypi_token.py` are deploying-releases' marked copies, installed with +`install.sh --co-pypi`; a change goes upstream, then a reinstall, and +`tests/test_co_pypi.py` fails on a hand edit. The minter mints an access token from +the read-only `co-pypi-reader@` key and `exec`s uv with `UV_INDEX_CO_PYPI_USERNAME` / +`_PASSWORD`. Its rules are tested in the skill's own suite: + +- **One key, never the writer's.** `CO_PYPI_KEY` when set, else + `/etc/<[project].name>/co-pypi-reader.json` — here `/etc/replicator/co-pypi-reader.json`, + so nothing sets it. Never `GOOGLE_APPLICATION_CREDENTIALS`: on this host that is the + worker's *writer* (#114). - **An allowlisted environment.** The deploy shell carries the writer's key path and the repo `.env`'s org-wide PATs, and uv (with the `uv_build` backend it runs) - starts with neither. The shell script and the Python hold the same list; a - variable uv turns out to need goes on both. + starts with neither. - **The token is never printed, logged, put in argv or written.** Anything but - `uv` or `wheels` is refused before a mint. + `uv` is refused before a mint. +- **It runs from the lock's `co-pypi` group** (`google-auth[requests]`), `--frozen`, + so nothing in the process holding the key resolves at run time. The group is not a + default one: neither a release nor CI syncs it. ruff excludes the minter, whose + marker a `--fix` would break. **CI never runs it.** Each job's read-only WIF step mints the token (`token_format: access_token`, id `co_pypi`) and writes no credentials file; the `uv sync` step alone receives it, masked (`tests/ci/test_workflow_pins.py`). -**Limits.** The token reaches what the wrapped uv spawns (the build backend), -bounded by the reader's role and the token's hour. The `wheels` verb sends it to -`urllib`, for the index host alone and across same-host redirects only: the index -answers a file with a 307. +**Limits.** The token reaches what the wrapped uv spawns (the build backend, and +the wheel fetcher below), bounded by the reader's role and the token's hour. The +fetcher sends it to `urllib`, for the index host alone and across same-host +redirects only: the index answers a file with a 307. ## Moving the pin @@ -66,19 +70,26 @@ answers a file with a 307. 3. Diff the import closure ([below](#the-co-core-pin-and-why-the-patch-floor-is-load-bearing)): ```bash - bash scripts/co-pypi.sh wheels OLD NEW /tmp/co-core-wheels + bash scripts/co-pypi.sh uv run --no-project python scripts/co_pypi_wheels.py \ + --dest /tmp/co-core-wheels --version OLD --version NEW co-core co-core-aio co-core-sync uv run --no-project python scripts/co_core_closure_diff.py OLD NEW --wheels /tmp/co-core-wheels ``` 4. Paste the report into the pin comment and raise the floor by the rules below. -`wheels` checks each file against the index's sha256. The closure diff's -real-wheel test reads `REPLICATOR_TEST_CLOSURE_WHEELS`, a directory `wheels 0.36.0 -0.37.0` filled, and skips without it, in CI too. +**The fetcher** (#201) exists because uv has no download command and leaves no `.whl` +behind [measured, uv 0.13.5]. It runs under the wrapper and reads the pair the wrapper +hands uv, so it holds no key of its own; packages and versions are arguments, the +index is `pyproject.toml`'s `co-pypi`. Each file is checked against the index's sha256 +and renamed into place; a missing version or a mismatch exits 1 naming the file. It is +standard library only and names nothing of this repo's, built to be proposed to +deploying-releases beside the wrapper. The closure diff's real-wheel test reads +`REPLICATOR_TEST_CLOSURE_WHEELS`, a directory it filled at 0.36.0 and 0.37.0, and +skips without it, in CI too. **Ask the index, not a cache.** The #77 review reported 0.13.1 "not mirrored" from a -local `ls` while the index already carried it. Anything local is a cache; `wheels` -or `uv lock` asks the index. +local `ls` while the index already carried it. Anything local is a cache; the +fetcher or `uv lock` asks the index. ## The co-core pin, and why the patch floor is load-bearing diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 322ddf0e..1ea63f49 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -21,8 +21,10 @@ A third carrier, `.claude/settings*.json`, is agent tooling read by the harness Read by neither env file — test-only, defined in `tests/conftest.py`: the `REPLICATOR_TEST_*` variables (`_REDIS_URL`, `_GCS_BUCKET`, `_BLOB_BUCKET`, `_PERMANENT_BUCKET`, `_GCS_CREDENTIALS`, `_CLOSURE_WHEELS`) are in [TESTING.md](TESTING.md#variables), beside the fixtures and markers they gate. The GCS ones never go in `/etc/replicator/.env`. +Read by neither env file either: `CO_PYPI_KEY`, the shared `scripts/co-pypi.sh`'s read-only key (`co-pypi-reader@`). **Unset, and needs no setting:** unset, the wrapper reads `/etc/<[project].name>/co-pypi-reader.json`, this host's key. Unprefixed, being the cohort wrapper's name (#201). Never the worker's, and never `GOOGLE_APPLICATION_CREDENTIALS`, the **replication writer**: nothing that resolves packages holds write on permanent artifacts (#29). [DEPENDENCIES.md](DEPENDENCIES.md) + In `/etc/replicator/.env` (read by the service): -- `GOOGLE_APPLICATION_CREDENTIALS` — the worker's ADC: the writer SA key (`/etc/replicator/co-gcs-replicator-writer.json`, `co-gcs-replicator-writer` since #114; it replaced `co-gcs-replicator`). **Not** the co-pypi reader — that is `REPLICATOR_CO_PYPI_CREDENTIALS` below, split off precisely so the worker and the package index never share an identity (#29); this line said otherwise until #86 verified the file +- `GOOGLE_APPLICATION_CREDENTIALS` — the worker's ADC: the writer SA key (`/etc/replicator/co-gcs-replicator-writer.json`, `co-gcs-replicator-writer` since #114; it replaced `co-gcs-replicator`). **Not** the co-pypi reader — that is `CO_PYPI_KEY`'s default above, split off precisely so the worker and the package index never share an identity (#29); this line said otherwise until #86 verified the file - `REPLICATOR_REDIS_URL` — change-bus client URL; default `redis://localhost:6379/0`. On `co-replicator` it names the `replicator` ACL user at `broker:6379/0` over the tailnet, with the username explicit — without one the login goes to `default` and is refused (broker#46) - The **start-guard variables** — `REPLICATOR_REDIS_FLOOR_TIMEOUT` and `REPLICATOR_REDIS_FLOOR_WAIT` — and the release check's override, `REPLICATOR_ALLOW_ANY_CHECKOUT`, are in [DEPLOYMENT.md](DEPLOYMENT.md#environment-variables), with the guards they set. They are read from this same file; only the prose moved. - The **blob and temp-store variables** — `REPLICATOR_BLOB_BACKEND`, `_BUCKET`, `_PREFIX`, `_TIMEOUT_SECONDS`, `_DIR`, `_TTL_SECONDS`, `_SWEEP_INTERVAL_SECONDS`, `_TEMP_GRACE_SECONDS`, `_MAX_TOTAL_BYTES`, `REPLICATOR_MAX_BLOB_BYTES`, `REPLICATOR_PERMANENT_BUCKET` and `REPLICATOR_CAPTURE_BUCKET` (#189) — are in [STORAGE.md](STORAGE.md#variables), beside the store whose behaviour they set. They are read from this same file; only the prose moved. @@ -62,7 +64,6 @@ In `/etc/replicator/.env` (read by the service): - `REPLICATOR_DEDUPE_TTL_SECONDS` — lifetime of the `replicator:cmd::` dedupe key; default `86400`. The key is written *after* a completing close, so the TTL bounds a re-fetch window and never a correctness window — what the keys guard, and what an empty namespace costs, is [CONVENTIONS.md](CONVENTIONS.md#the-replicatorcmd-keys) (#80) - `REPLICATOR_REPLICATE_CONSUMER_GROUP` — the `content.replicate` group; default `replicator.replicate`. **Must differ from `REPLICATOR_CONSUMER_GROUP`**, and settings construction refuses the collision — but not for the reason this line gave until CR round 2. A consumer group is identified by *(stream key, group name)*, so the same name on both command streams is two unrelated groups whose pending-entry lists are separate; `claim_stale` on one provably cannot reach the other's (`test_a_group_name_is_scoped_to_its_stream`, live broker). It is refused because the consumer-name override is chosen *by group*, so two groups spelled alike leave no way to tell which override applies — and because two same-named groups in `XINFO` misread easily on a broker three services share - `REPLICATOR_REPLICATE_WRITE_TIMEOUT_SECONDS` — how long one conditional create may run; default `120`. Surfaced rather than inherited from the SDK's own default, which nobody chose (CR #38): a hung write holds its PEL entry for the whole window, and on the write side that window is also how long a large blob has to reach a permanent store over whatever link this host has. **The invariant is `write_timeout < REPLICATOR_CLAIM_MIN_IDLE_MS`**, so a write finishes before its entry becomes claimable — raise the reclaim window first, then this. **The shipped pair does not satisfy it** (120 s against a 60 s window) and that is deliberate rather than overlooked: a write still running when its entry goes claimable can be started a second time by a competing consumer, and under T4 that is a duplicate upload, not a duplicate artifact — one call gets `WROTE`, the other `ALREADY_IDENTICAL`, and both publish the same `public_url`, which MUST-4 already requires issuers to tolerate. The cost is egress, not correctness. The fetch path has carried the same relation since it shipped (`REPLICATOR_MAX_FETCH_TIMEOUT_SECONDS` is also 120). Close the gap on either side if duplicate uploads start showing up in the journal -- `REPLICATOR_CO_PYPI_CREDENTIALS` — the read-only key (`/etc/replicator/co-pypi-reader.json`, `co-pypi-reader@`) that `scripts/co-pypi.sh` mints the co-pypi index's token from, in the deploy shell. **Read by the wrapper, never by the worker**, and there is no fallback: the worker's ADC is the **replication writer**, which has no business reading the index, and nothing that resolves packages has any business holding write on permanent artifacts (#29). Renamed from `REPLICATOR_WHEELHOUSE_CREDENTIALS` with the wheelhouse's retirement (#183): [DEPENDENCIES.md](DEPENDENCIES.md) - `REPLICATOR_REPLICATION_ALIASES_FILE` — path to the JSON alias table; **unset by default, and that is the safe posture**. Nothing provisioned means every `content.replicate` command is refused with `alias_unknown`, so enabling replication to a destination is an explicit operator act on the VM rather than a consequence of a message arriving (contract T5 — it matters most for `ia`, whose items cannot be deleted). A *path* rather than the table itself because the provisioned set is host state but is a table, and one variable per alias per field is a shape env does not hold. Format: ```json diff --git a/docs/TESTING.md b/docs/TESTING.md index 5adbdc6b..a7cbf940 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -21,7 +21,7 @@ test-only variables, defined in `tests/conftest.py`. - `REPLICATOR_TEST_BLOB_BUCKET` — the temp-blob bucket `@pytest.mark.gcs` may write to (#7); `co-gcs-test-blobs`, provisioned 2026-08-20. **No default**, same rule as above, and deliberately a *second* variable: the two destinations carry opposite grants — the replicate SA holds no `delete` at all, which is what puts T4 at IAM rather than only in our code, while the temp-store tests create objects and clean up after themselves. One bucket serving both would mean either the temp tests cannot tidy or the permanent store can be erased - `REPLICATOR_TEST_PERMANENT_BUCKET` — the permanent-store test twin `@pytest.mark.gcs` may write to and replicate from (#114); `co-gcs-test-replicator`. **No default**, same rule; the store guard admits it beside `REPLICATOR_TEST_BLOB_BUCKET` - `REPLICATOR_TEST_GCS_CREDENTIALS` — path to the test SA key (`/etc/replicator/co-gcs-test-replicator-writer.json` here, the WIF credentials file in CI). Also no default, and mapped onto `GOOGLE_APPLICATION_CREDENTIALS` for marked tests only — the bucket name alone cannot catch a run pointed at the right bucket with the production identity, which is reachable because AGENTS.md's Common Commands snippet loads `/etc/replicator/.env` -- `REPLICATOR_TEST_CLOSURE_WHEELS` — a directory `bash scripts/co-pypi.sh wheels 0.36.0 0.37.0 DIR` filled, read by `tests/test_co_core_closure_diff.py`'s real-wheel check alone (not `conftest.py`). Without it that check skips, CI included (#183) +- `REPLICATOR_TEST_CLOSURE_WHEELS` — a directory of 0.36.0 and 0.37.0 wheels ([the fetch](DEPENDENCIES.md#moving-the-pin)), read by `tests/test_co_core_closure_diff.py`'s real-wheel check alone (not `conftest.py`). Without it that check skips, CI included (#183) All four GCS ones belong in the repo `.env` or the invoking shell, **never in `/etc/replicator/.env`** — that is the file the service reads, and a delete-capable identity is precisely the grant T4 exists to withhold from the worker. The resource itself: [**The GCS test bucket**](INFRASTRUCTURE.md#the-gcs-test-bucket--the-opposite-grant-on-purpose-38-50) diff --git a/pyproject.toml b/pyproject.toml index 0be996a0..dc7b5398 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -483,6 +483,10 @@ dev = [ "ruff>=0.16,<0.17", "ty", # beta — deliberately unpinned; non-gating advisory checker ] +# The shared co-pypi wrapper's minter runs from this group alone, --frozen, +# before any token exists, so it holds nothing from the index (#201, +# tests/test_co_pypi.py). Not a default group: neither a release nor CI syncs it. +co-pypi = ["google-auth[requests]>=2,<3"] # co-core, co-core-aio and co-core-sync resolve from cannobserv's co-pypi # Artifact Registry index ("Path C", #183), and only they do: `explicit` means @@ -545,7 +549,8 @@ module-root = "" [tool.ruff] line-length = 100 target-version = "py312" -extend-exclude = ["skills-vendor/"] +# The installed co-pypi minter is a marked copy: a --fix or format would break its marker. +extend-exclude = ["skills-vendor/", "scripts/co_pypi_token.py"] [tool.ruff.lint] select = ["E", "F", "I", "W", "UP", "B904", "PLC0415", "FAST", "ASYNC"] @@ -558,6 +563,6 @@ known-first-party = ["src"] # ty walks the whole tree by default. skills-vendor/ is read-only third-party # skill content — 9 of 10 diagnostics came from there, drowning the one real -# signal. Mirrors ruff's extend-exclude. +# signal. Mirrors ruff's skills-vendor/ exclude. [tool.ty.src] exclude = ["skills-vendor/"] diff --git a/scripts/co-pypi.sh b/scripts/co-pypi.sh index 985fb261..8d50594c 100755 --- a/scripts/co-pypi.sh +++ b/scripts/co-pypi.sh @@ -1,21 +1,29 @@ #!/usr/bin/env bash -# Run one uv command, or fetch co-core wheels, with a co-pypi token (#183). +# deploying-releases-co-pypi: version 1 +# deploying-releases-co-pypi-sha256: ff3579e022238c0b032d0a7f6ad14fdf3beb3ed787c690449ad867b2a7b788ce # -# . scripts/load_env.sh # REPLICATOR_CO_PYPI_CREDENTIALS -# bash scripts/co-pypi.sh uv sync --frozen # a dev venv; deploys build through it (UV_WRAPPER) -# bash scripts/co-pypi.sh uv lock -# bash scripts/co-pypi.sh wheels OLD NEW DIR # the closure diff's wheels +# Run one uv command with a co-pypi access token in its environment +# (gregoryfoster/skills#632). Installed by deploying-releases' install.sh +# --co-pypi, with scripts/co_pypi_token.py; never edited here. # -# The token is minted by scripts/co_pypi_token.py and given to the wrapped uv -# alone; it is never printed. The outer `uv run` starts from an allowlisted -# environment, so the writer's GOOGLE_APPLICATION_CREDENTIALS and the repo -# .env's PATs reach neither uv. The two lists match co_pypi_token.py's ALLOWED -# and ALLOWED_PREFIXES, plus the key variable (tests/test_co_pypi_token.py). -# --no-project: the project can't be synced to run this, since that sync is what -# needs the token. Details: docs/DEPENDENCIES.md. +# bash scripts/co-pypi.sh uv sync --locked --group dev # a dev tree +# bash scripts/co-pypi.sh uv lock # a pin change +# +# The token is minted by scripts/co_pypi_token.py from the co-pypi-reader key +# (CO_PYPI_KEY, else /etc/<[project].name>/co-pypi-reader.json) and given to the +# wrapped uv alone; it is never printed. The outer `uv run` starts from an +# allowlisted environment, so a loaded .env's PATs and a shell-wide +# GOOGLE_APPLICATION_CREDENTIALS reach neither uv. The two lists match +# co_pypi_token.py's ALLOWED and ALLOWED_PREFIXES, plus CO_PYPI_KEY. That +# module's own dependency is uv.lock's co-pypi group, run --frozen in this +# checkout (--project): the group comes from public PyPI, so it syncs before +# any token exists. A lock that holds no such group yet (a repo's first +# `uv lock`) can't be written without the token, so that run alone resolves +# the group's pin from PyPI, and says so. The wrapped uv runs where this was +# called from: scripts/deploy.sh calls it in a release. set -euo pipefail -ALLOWED=(PATH HOME USER LOGNAME LANG TERM TMPDIR http_proxy https_proxy no_proxy all_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY ALL_PROXY REPLICATOR_CO_PYPI_CREDENTIALS) +ALLOWED=(PATH HOME USER LOGNAME LANG TERM TMPDIR http_proxy https_proxy no_proxy all_proxy HTTP_PROXY HTTPS_PROXY NO_PROXY ALL_PROXY CO_PYPI_KEY) PREFIXES=(LC_ XDG_ SSL_CERT_ UV_) keep=() @@ -27,5 +35,12 @@ while IFS= read -r name; do done < <(compgen -e) HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec env -i "${keep[@]}" uv run --no-project --with 'google-auth[requests]>=2,<3' \ - python "$HERE/co_pypi_token.py" "$@" +ROOT="$(dirname "$HERE")" +if grep -q '^co-pypi = \[' "$ROOT/uv.lock" 2>/dev/null; then + group=(--project "$ROOT" --frozen --only-group co-pypi) +else + echo "co-pypi.sh: $ROOT/uv.lock holds no co-pypi group yet, so this run resolves" \ + "google-auth from PyPI; commit the lock it writes" >&2 + group=(--no-project --with 'google-auth[requests]>=2,<3') +fi +exec env -i "${keep[@]}" uv run "${group[@]}" python "$HERE/co_pypi_token.py" "$@" diff --git a/scripts/co_core_closure_diff.py b/scripts/co_core_closure_diff.py index 90a2f61a..623eee8b 100644 --- a/scripts/co_core_closure_diff.py +++ b/scripts/co_core_closure_diff.py @@ -7,7 +7,9 @@ module of the named packages it reaches, and compares the two versions file by file:: - bash scripts/co-pypi.sh wheels 0.36.0 0.37.0 /tmp/co-core-wheels + bash scripts/co-pypi.sh uv run --no-project python scripts/co_pypi_wheels.py \ + --dest /tmp/co-core-wheels --version 0.36.0 --version 0.37.0 \ + co-core co-core-aio co-core-sync uv run --no-project python scripts/co_core_closure_diff.py 0.36.0 0.37.0 \ --wheels /tmp/co-core-wheels [--diff] @@ -34,8 +36,8 @@ Nothing in it is Replicator's: roots, packages and the wheel directory are flags, and the defaults are this repo's. The directory has no default: the wheels come -from the co-pypi index through the wrapper's ``wheels`` verb (#183), which -checks each against the index's sha256, into any scratch directory. A +from the co-pypi index through ``scripts/co_pypi_wheels.py`` under the wrapper +(#183, #201), which checks each against the index's sha256, into any scratch directory. A ``--package`` names both the import package and the wheel's distribution, which co-core's three share. It reads the wheels in place with ``zipfile`` and installs nothing, and it's standard library only, so ``uv run --no-project`` @@ -59,6 +61,8 @@ DEFAULT_ROOTS = ["src", "scripts"] DEFAULT_PACKAGES = ["co_core", "co_core_aio", "co_core_sync"] +# What the error messages tell you to run: the wheels fetcher, under the co-pypi wrapper (#201). +FETCH = "bash scripts/co-pypi.sh uv run --no-project python scripts/co_pypi_wheels.py" class MissingWheelError(Exception): @@ -113,7 +117,8 @@ def load_wheels(wheels: Path, packages: Sequence[str], version: str) -> Wheels: if missing: names = ", ".join(str(wheel_path(wheels, p, version)) for p in missing) raise MissingWheelError( - f"missing: {names}. Fetch them first: bash scripts/co-pypi.sh wheels OLD NEW {wheels}" + f"missing: {names}. Fetch them first: {FETCH} --dest {wheels} --version {version}" + f" {' '.join(packages)}" ) files: dict[str, bytes] = {} requires: dict[str, frozenset[str]] = {} @@ -124,7 +129,8 @@ def load_wheels(wheels: Path, packages: Sequence[str], version: str) -> Wheels: except zipfile.BadZipFile as exc: # The fetch verifies each file's sha256, so this is a hand-placed file. raise MissingWheelError( - f"not a readable wheel: {path}. Fetch it again: bash scripts/co-pypi.sh wheels" + f"not a readable wheel: {path}. Fetch it again:" + f" {FETCH} --dest {wheels} --version {version} {package}" ) from exc with z: for name in z.namelist(): @@ -421,7 +427,7 @@ def main(argv: Sequence[str] | None = None) -> int: "--package", action="append", help=f"repeatable; default {DEFAULT_PACKAGES}" ) parser.add_argument( - "--wheels", required=True, help="the directory `co-pypi.sh wheels OLD NEW DIR` filled" + "--wheels", required=True, help="the directory scripts/co_pypi_wheels.py --dest filled" ) parser.add_argument("--diff", action="store_true", help="add each changed file's code diff") args = parser.parse_args(argv) diff --git a/scripts/co_pypi_token.py b/scripts/co_pypi_token.py index 3ed82958..99931070 100644 --- a/scripts/co_pypi_token.py +++ b/scripts/co_pypi_token.py @@ -1,69 +1,77 @@ -"""Run one ``uv`` command, or fetch co-core wheels, with a co-pypi access token (#183). +# scripts/co_pypi_token.py — the co-pypi wrapper's minter (deploying-releases). +# deploying-releases-co-pypi: version 1 +# deploying-releases-co-pypi-sha256: f6c3fedd2c7dbebdfc502606d75a57f62639b9b152cec2979111bcbb36cb8c7d +"""Run one ``uv`` command with a co-pypi access token in its environment. - . scripts/load_env.sh # REPLICATOR_CO_PYPI_CREDENTIALS - bash scripts/co-pypi.sh uv sync --frozen # a dev venv; deploys build through it (UV_WRAPPER) - bash scripts/co-pypi.sh uv lock - bash scripts/co-pypi.sh wheels OLD NEW DIR # the closure diff's wheels + bash scripts/co-pypi.sh uv sync --locked --group dev + bash scripts/co-pypi.sh uv lock --upgrade-package co-core -co-core, co-core-aio and co-core-sync resolve from the co-pypi Artifact Registry -index (``pyproject.toml``'s ``[[tool.uv.index]]``). It takes HTTP basic auth: the +Installed by deploying-releases' ``install.sh --co-pypi`` with +``scripts/co-pypi.sh`` (gregoryfoster/skills#632), and never edited here: an +upgrade is a reviewable diff. Extracted from CannObserv/broker at ``bbf1c72f``, +which ported processor#121's locked group and replicator#183's allowlist. + +cannobserv's packages resolve from its co-pypi Artifact Registry index +(``pyproject.toml``'s ``[[tool.uv.index]]``). It takes HTTP basic auth: the username ``oauth2accesstoken`` and an OAuth2 access token as the password, which -uv reads from ``UV_INDEX_CO_PYPI_USERNAME`` / ``UV_INDEX_CO_PYPI_PASSWORD``. The -host has no gcloud, so this mints the token from the read-only key named by -``REPLICATOR_CO_PYPI_CREDENTIALS`` (``co-pypi-reader@``) and ``exec``s ``uv``. -Ported from CannObserv/watcher#358's wrapper, with two departures. - -**The writer never reaches uv.** Here ``GOOGLE_APPLICATION_CREDENTIALS`` is the -worker's *writer* (#114), and the deploy shell also carries org-wide -``GH_TOKEN_*`` PATs. So the key variable is the only source, with no ADC -fallback, and ``uv`` starts from an **allowlisted** environment, not the -caller's: ``scripts/co-pypi.sh`` applies the list to the outer ``uv run`` and -:func:`child_environment` again to the wrapped command. CI never runs this; its -WIF step hands the token straight to ``uv sync``. +uv reads from ``UV_INDEX_CO_PYPI_USERNAME`` / ``UV_INDEX_CO_PYPI_PASSWORD``. A +host has no gcloud, so this mints the token from the read-only +``co-pypi-reader`` key (``roles/artifactregistry.reader`` on co-pypi alone) and +``exec``s the command with the pair in that child's environment alone. + +**The key:** ``CO_PYPI_KEY`` when set, else ``/etc//co-pypi-reader.json``, +```` being ``[project].name`` in the ``pyproject.toml`` beside this +script's directory. Never ``GOOGLE_APPLICATION_CREDENTIALS``: set shell-wide, +that names an identity to every Google client in the shell. The key path is +explicit, so google-auth's ADC search never runs. + +**The caller's environment never reaches uv:** ``scripts/co-pypi.sh`` starts +the outer ``uv run`` from an allowlist and :func:`child_environment` cuts the +wrapped command's again, so the PATs a loaded ``.env`` carries reach neither. **The token is never printed, logged, put in argv or written anywhere.** It -lives in this process and the wrapped ``uv``'s environment (an hour at most). -Anything but ``uv`` or ``wheels`` is refused before a token is minted. -**Accepted limit:** whatever that ``uv`` spawns inherits the pair (the -``uv_build`` backend). ``wheels`` sends it to ``urllib``, for the index host -alone, across same-host redirects only (:class:`IndexHostRedirect`). - -Exit codes: the wrapped command's own · ``0`` wheels fetched · ``2`` usage, a -refused command or no key variable · ``1`` the mint or a fetch failed · ``127`` -the command could not be run. +lives in this process and the wrapped ``uv``'s environment, an hour at most. +Anything but ``uv`` is refused before a token is minted. **Accepted limit:** +whatever that ``uv`` spawns inherits the pair (the build backend, building the +editable project). uv has no per-index credential scoping short of a keyring or +netrc file; the bound is the identity's read-only role and the token's hour. + +Its own dependency is uv.lock's ``co-pypi`` group, run ``--frozen`` in this +script's checkout, so nothing in the process that holds the key is resolved at +run time. The one exception is a lock that holds no such group yet: a repo's +first ``uv lock`` can't be written without the token, so ``co-pypi.sh`` +resolves the group's pin from PyPI for that run and says so. + +Only a command that downloads from the index needs this: ``uv lock``, and a +``uv sync`` that needs a private file neither installed nor cached. +``uv lock --locked`` and ``uv audit --frozen`` don't. + +Exit codes: the wrapped command's own - ``2`` usage or a refused command - +``1`` the mint failed (the key and its path included) - ``127`` the command +could not be run. """ from __future__ import annotations -import base64 -import hashlib -import http.client import os import sys -import tempfile -import urllib.error -import urllib.request +import tomllib from collections.abc import Callable, Mapping, Sequence -from html.parser import HTMLParser from pathlib import Path -from urllib.parse import urljoin, urlsplit -from google.auth.transport.requests import Request -from google.oauth2 import service_account - -INDEX_URL = "https://us-west1-python.pkg.dev/co-gcs/co-pypi/simple/" -INDEX_HOST = urlsplit(INDEX_URL).hostname -PACKAGES = ("co-core", "co-core-aio", "co-core-sync") -KEY_VAR = "REPLICATOR_CO_PYPI_CREDENTIALS" USERNAME_VAR = "UV_INDEX_CO_PYPI_USERNAME" PASSWORD_VAR = "UV_INDEX_CO_PYPI_PASSWORD" USERNAME = "oauth2accesstoken" +KEY_VAR = "CO_PYPI_KEY" +# The host's key: root: 640, beside the project's settings. +KEY_PATH = "/etc/{name}/co-pypi-reader.json" +PYPROJECT = Path(__file__).resolve().parent.parent / "pyproject.toml" # Artifact Registry's documented scope for a token-as-password read; the -# identity's role, artifactregistry.reader, is what bounds it. +# identity's IAM role, artifactregistry.reader on co-pypi, is what bounds it. SCOPES = ["https://www.googleapis.com/auth/cloud-platform"] -# What uv may see. scripts/co-pypi.sh carries the same two lists, plus KEY_VAR -# (tests/test_co_pypi_token.py holds them equal). VIRTUAL_ENV is left out: the -# outer `uv run` sets it to its ephemeral env. +# What uv may see. scripts/co-pypi.sh carries the same two lists, plus KEY_VAR. +# VIRTUAL_ENV is left out: the outer `uv run` sets it to the checkout's venv, +# and a wrapped `uv sync` in a release would warn about it. ALLOWED = frozenset( { "PATH", "HOME", "USER", "LOGNAME", "LANG", "TERM", "TMPDIR", @@ -72,166 +80,84 @@ } ) # fmt: skip ALLOWED_PREFIXES = ("LC_", "XDG_", "SSL_CERT_", "UV_") -USAGE = "usage: bash scripts/co-pypi.sh uv [args...] | wheels OLD NEW DIR" - - -class FetchError(Exception): - """A wheel could not be fetched or verified. Never carries the token.""" +USAGE = "usage: bash scripts/co-pypi.sh uv [args...]" def child_environment(environ: Mapping[str, str]) -> dict[str, str]: """*environ* cut to the allowlist.""" - return {k: v for k, v in environ.items() if k in ALLOWED or k.startswith(ALLOWED_PREFIXES)} + return { + k: v + for k, v in environ.items() + if k in ALLOWED or k.startswith(ALLOWED_PREFIXES) + } -def mint_token(key_path: str) -> str: - """An access token for the service account whose key is at *key_path*. +def key_path(environ: Mapping[str, str], pyproject: Path = PYPROJECT) -> str: + """``CO_PYPI_KEY``, else the project's default; raises when neither is known.""" + explicit = environ.get(KEY_VAR, "").strip() + if explicit: + return explicit + try: + name = tomllib.loads(pyproject.read_text())["project"]["name"] + except (OSError, tomllib.TOMLDecodeError, KeyError, TypeError) as exc: + raise LookupError( + f"{KEY_VAR} is unset and {pyproject} names no [project].name" + ) from exc + if not isinstance(name, str) or not name or "/" in name or name.startswith("."): + raise LookupError( + f"{KEY_VAR} is unset and [project].name {name!r} is no directory name" + ) + return KEY_PATH.format(name=name) - The path is explicit, so google-auth's ADC search — which would find - ``GOOGLE_APPLICATION_CREDENTIALS`` — never runs. - """ - credentials = service_account.Credentials.from_service_account_file(key_path, scopes=SCOPES) + +def mint_token(path: str) -> str: + """An access token for the service account whose key is at *path*.""" + # Imported here, so the rest of this module runs without the co-pypi group. + from google.auth.transport.requests import Request + from google.oauth2 import service_account + + credentials = service_account.Credentials.from_service_account_file( + path, scopes=SCOPES + ) credentials.refresh(Request()) if not credentials.token: raise RuntimeError("the token endpoint returned no access token") return credentials.token -class IndexHostRedirect(urllib.request.HTTPRedirectHandler): - """Follow a redirect only to https on the index host. - - The index answers a file with a same-host 307 [measured 2026-11-10], and - urllib copies the ``Authorization`` header onto any redirect. - """ - - def redirect_request(self, req, fp, code, msg, headers, newurl): - target = urlsplit(urljoin(req.full_url, newurl)) - if target.scheme != "https" or target.hostname != INDEX_HOST: - raise urllib.error.HTTPError( - req.full_url, code, f"refused a redirect off {INDEX_HOST}", headers, fp - ) - return super().redirect_request(req, fp, code, msg, headers, newurl) - - -def _open(request: urllib.request.Request): - return urllib.request.build_opener(IndexHostRedirect).open(request, timeout=60) - - -class _Links(HTMLParser): - """Every ```` on a PEP 503 page.""" - - def __init__(self): - super().__init__() - self.hrefs: list[str] = [] - - def handle_starttag(self, tag, attrs): - if tag == "a": - self.hrefs.extend(v for k, v in attrs if k == "href" and v) - - -def _get(opener, url: str, auth: str) -> bytes: - if urlsplit(url).scheme != "https" or urlsplit(url).hostname != INDEX_HOST: - raise FetchError(f"refused {url}: not on {INDEX_HOST}") - try: - with opener(urllib.request.Request(url, headers={"Authorization": auth})) as response: - return response.read() - except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: - # A status, a timeout or a dropped connection (CR 4): each names the URL - # and the failure, never the request's headers. - raise FetchError(f"{url}: {type(exc).__name__}: {exc}") from exc - - -def fetch_wheels( - token: str, - versions: Sequence[str], - dest: Path, - *, - opener: Callable[[urllib.request.Request], object] = _open, -) -> list[Path]: - """Download each package's ``py3-none-any`` wheel at each version into *dest*. - - Each file is checked against the index's ``#sha256=`` before it is renamed - into place, so *dest* never holds a partial or unverified wheel. - """ - auth = "Basic " + base64.b64encode(f"{USERNAME}:{token}".encode()).decode() - dest.mkdir(parents=True, exist_ok=True) - fetched: list[Path] = [] - for package in PACKAGES: - page_url = f"{INDEX_URL}{package}/" - parser = _Links() - parser.feed(_get(opener, page_url, auth).decode()) - links = {} - for href in parser.hrefs: - url, _, fragment = urljoin(page_url, href).partition("#") - links[url.rsplit("/", 1)[-1]] = (url, fragment.removeprefix("sha256=")) - for version in versions: - name = f"{package.replace('-', '_')}-{version}-py3-none-any.whl" - if name not in links: - raise FetchError(f"{name} is not on {page_url}") - url, digest = links[name] - data = _get(opener, url, auth) - if not digest or hashlib.sha256(data).hexdigest() != digest: - raise FetchError( - f"sha256 mismatch for {name}: the index says {digest or 'nothing'}" - ) - fd, tmp = tempfile.mkstemp(dir=dest, prefix=f".{name}.", suffix=".part") - try: - with os.fdopen(fd, "wb") as f: - f.write(data) - os.replace(tmp, dest / name) - finally: - Path(tmp).unlink(missing_ok=True) - fetched.append(dest / name) - return fetched - - def main( argv: Sequence[str], environ: Mapping[str, str], *, mint: Callable[[str], str] = mint_token, execvpe: Callable[[str, list[str], dict[str, str]], object] = os.execvpe, - fetch: Callable[[str, list[str], Path], object] = fetch_wheels, + pyproject: Path = PYPROJECT, ) -> int | None: - """Exec *argv* (a ``uv`` command) with the co-pypi pair, or fetch wheels. + """Exec *argv* (a ``uv`` command) with the co-pypi pair in its environment. - Returns an exit code; on a ``uv`` success *execvpe* replaces the process. + Returns an exit code on failure; on success *execvpe* replaces the process, + so in production nothing returns. """ - verb = Path(argv[0]).name if argv else "" - if verb not in ("uv", "wheels") or (verb == "wheels" and len(argv) != 4): - print(f"error: co-pypi wraps uv, or fetches wheels. {USAGE}", file=sys.stderr) - return 2 - - key_path = environ.get(KEY_VAR, "").strip() - if not key_path: - print( - f"error: {KEY_VAR} is not set: it names the co-pypi-reader key" - " (never GOOGLE_APPLICATION_CREDENTIALS, the writer). Load it: . scripts/load_env.sh", - file=sys.stderr, - ) + if not argv or Path(argv[0]).name != "uv": + print(f"error: co-pypi wraps uv alone. {USAGE}", file=sys.stderr) return 2 try: - token = mint(key_path) + path = key_path(environ, pyproject) + except LookupError as exc: + print(f"error: no co-pypi-reader key: {exc}. Set {KEY_VAR}.", file=sys.stderr) + return 1 + try: + token = mint(path) except Exception as exc: # broad by design: any failure here runs nothing # The exception is the token endpoint's or the key file's, never the token. print( - f"error: could not mint a co-pypi access token from {KEY_VAR}={key_path}:" - f" {type(exc).__name__}: {exc}", + f"error: could not mint a co-pypi access token from {path}:" + f" {type(exc).__name__}: {exc}. Set {KEY_VAR}, or install the reader key there.", file=sys.stderr, ) return 1 - if verb == "wheels": - try: - fetched = fetch(token, list(argv[1:3]), Path(argv[3])) - except FetchError as exc: - print(f"error: {exc}", file=sys.stderr) - return 1 - for path in fetched or []: - print(path) - return 0 - env = child_environment(environ) env[USERNAME_VAR] = USERNAME env[PASSWORD_VAR] = token diff --git a/scripts/co_pypi_wheels.py b/scripts/co_pypi_wheels.py new file mode 100644 index 00000000..cccd019c --- /dev/null +++ b/scripts/co_pypi_wheels.py @@ -0,0 +1,234 @@ +# scripts/co_pypi_wheels.py — fetch wheel files from the co-pypi index, hash-checked. +"""Download each package's ``py3-none-any`` wheel at each version from co-pypi. + + bash scripts/co-pypi.sh uv run --no-project python scripts/co_pypi_wheels.py \\ + --dest DIR --version OLD --version NEW PACKAGE [PACKAGE ...] + +uv has no download command and leaves no ``.whl`` behind: an install, and its +cache, hold unpacked trees. A tool that reads wheel files (an import-closure +diff across a pin move, say) needs the files themselves, so this fetches them. + +**Run under the co-pypi wrapper, and it handles no key.** The wrapper mints the +token from the co-pypi-reader key (``CO_PYPI_KEY``, else +``/etc/<[project].name>/co-pypi-reader.json``) and hands +``UV_INDEX_CO_PYPI_USERNAME`` / ``UV_INDEX_CO_PYPI_PASSWORD`` to the uv it +runs, from an allowlisted environment; ``uv run`` hands them on to this. So +this reads the pair uv would, and holds nothing the wrapper doesn't. Run +without the pair, it exits 2 and names the wrapper. + +**The index** is ``pyproject.toml``'s ``[[tool.uv.index]]`` named ``co-pypi``: +the name the pair's variables encode. That ``pyproject.toml`` is the one beside +this script's directory, as the wrapper's minter reads it. + +**Output:** ``DIR/--py3-none-any.whl``, ```` normalized +as a wheel's (``co-core`` → ``co_core``), the index page looked up by PEP 503's. +Each file is checked against the sha256 the index's page gives it, written to +a temp file and renamed into place, so ``DIR`` never holds a partial or +unverified wheel. A wheel already there is replaced. Each written path is +printed, one per line. + +**The token goes to the index host alone.** A link off that host is refused, +and a redirect is followed only to https on it: the index answers a file with +a same-host 307, and urllib copies the Authorization header onto any redirect. +The token is never printed, logged, put in argv or written. Standard library +only, so ``uv run --no-project`` resolves nothing. + +Exit codes: ``0`` every wheel fetched and verified · ``1`` a fetch failed (a +missing version or package, a sha256 mismatch, the network, an unusable ``DIR``), +naming the file, URL or directory; files verified before it stay · ``2`` usage, no +co-pypi index in ``pyproject.toml``, or not run under the wrapper. +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import http.client +import os +import re +import sys +import tempfile +import tomllib +import urllib.error +import urllib.request +from collections.abc import Callable, Mapping, Sequence +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import urljoin, urlsplit + +INDEX_NAME = "co-pypi" +USERNAME_VAR = "UV_INDEX_CO_PYPI_USERNAME" +PASSWORD_VAR = "UV_INDEX_CO_PYPI_PASSWORD" +PYPROJECT = Path(__file__).resolve().parent.parent / "pyproject.toml" +WRAPPED = "bash scripts/co-pypi.sh uv run --no-project python scripts/co_pypi_wheels.py" +TIMEOUT = 60 + + +class FetchError(Exception): + """A wheel could not be fetched or verified. Never carries the token.""" + + +def index_url(pyproject: Path = PYPROJECT) -> str: + """The URL of *pyproject*'s ``[[tool.uv.index]]`` named ``co-pypi``.""" + try: + indexes = tomllib.loads(pyproject.read_text())["tool"]["uv"]["index"] + [url] = [i["url"] for i in indexes if i.get("name") == INDEX_NAME] + except (OSError, tomllib.TOMLDecodeError, KeyError, TypeError, ValueError) as exc: + raise LookupError(f"{pyproject} declares no one [[tool.uv.index]] {INDEX_NAME!r}") from exc + return url if url.endswith("/") else url + "/" + + +def page_name(package: str) -> str: + """PEP 503: the index page's name.""" + return re.sub(r"[-_.]+", "-", package).lower() + + +def wheel_name(package: str, version: str) -> str: + """The wheel spec's file name: the distribution normalized with underscores.""" + return f"{re.sub(r'[-_.]+', '_', package).lower()}-{version}-py3-none-any.whl" + + +class IndexHostRedirect(urllib.request.HTTPRedirectHandler): + """Follow a redirect only to https on *host*.""" + + def __init__(self, host: str): + super().__init__() + self.host = host + + def redirect_request(self, req, fp, code, msg, headers, newurl): + target = urlsplit(urljoin(req.full_url, newurl)) + if target.scheme != "https" or target.hostname != self.host: + raise urllib.error.HTTPError( + req.full_url, code, f"refused a redirect off {self.host}", headers, fp + ) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +class _Links(HTMLParser): + """Every ```` on a PEP 503 page.""" + + def __init__(self): + super().__init__() + self.hrefs: list[str] = [] + + def handle_starttag(self, tag, attrs): + if tag == "a": + self.hrefs.extend(v for k, v in attrs if k == "href" and v) + + +def fetch_wheels( + index: str, + auth: tuple[str, str], + packages: Sequence[str], + versions: Sequence[str], + dest: Path, + *, + opener: Callable[[urllib.request.Request], object] | None = None, +) -> list[Path]: + """Each package's wheel at each version into *dest*, checked, in order.""" + host = urlsplit(index).hostname + if opener is None: + built = urllib.request.build_opener(IndexHostRedirect(host)) + + def opener(request): + return built.open(request, timeout=TIMEOUT) + + header = "Basic " + base64.b64encode(f"{auth[0]}:{auth[1]}".encode()).decode() + + def get(url: str) -> bytes: + if urlsplit(url).scheme != "https" or urlsplit(url).hostname != host: + raise FetchError(f"refused {url}: not on {host}") + request = urllib.request.Request(url, headers={"Authorization": header}) + try: + with opener(request) as response: + return response.read() + except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: + # A status, a timeout or a dropped connection: the URL and the + # failure, never the request's headers. + raise FetchError(f"{url}: {type(exc).__name__}: {exc}") from exc + + try: + dest.mkdir(parents=True, exist_ok=True) + except OSError as exc: + raise FetchError(f"{dest}: {type(exc).__name__}: {exc}") from exc + fetched: list[Path] = [] + for package in packages: + page = f"{index}{page_name(package)}/" + parser = _Links() + try: + parser.feed(get(page).decode()) + except UnicodeDecodeError as exc: + raise FetchError(f"{page}: not a UTF-8 index page") from exc + links = {} + for href in parser.hrefs: + url, _, fragment = urljoin(page, href).partition("#") + links[url.rsplit("/", 1)[-1]] = (url, fragment.removeprefix("sha256=")) + for version in versions: + name = wheel_name(package, version) + if name not in links: + raise FetchError(f"{name} is not on {page}") + url, digest = links[name] + data = get(url) + if not digest or hashlib.sha256(data).hexdigest() != digest: + raise FetchError( + f"sha256 mismatch for {name}: the index says {digest or 'nothing'}" + ) + try: + fd, tmp = tempfile.mkstemp(dir=dest, prefix=f".{name}.", suffix=".part") + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + os.replace(tmp, dest / name) + finally: + Path(tmp).unlink(missing_ok=True) + except OSError as exc: + raise FetchError(f"{dest / name}: {type(exc).__name__}: {exc}") from exc + fetched.append(dest / name) + return fetched + + +def main( + argv: Sequence[str], + environ: Mapping[str, str], + *, + fetch: Callable[..., Sequence[Path]] = fetch_wheels, + pyproject: Path = PYPROJECT, +) -> int: + parser = argparse.ArgumentParser( + prog="co_pypi_wheels.py", + description="Fetch py3-none-any wheels from the co-pypi index, sha256-checked." + f" Run under the wrapper: {WRAPPED} --dest DIR --version V [--version V ...] PACKAGE ...", + ) + parser.add_argument("--dest", required=True, type=Path, help="the directory to fill") + parser.add_argument( + "--version", action="append", required=True, dest="versions", help="repeatable" + ) + parser.add_argument("packages", nargs="+", metavar="PACKAGE") + args = parser.parse_args(argv) + + user, password = environ.get(USERNAME_VAR, ""), environ.get(PASSWORD_VAR, "") + if not user or not password: + print( + f"error: no co-pypi credentials ({USERNAME_VAR}, {PASSWORD_VAR}):" + f" run under the wrapper, {WRAPPED} …", + file=sys.stderr, + ) + return 2 + try: + index = index_url(pyproject) + except LookupError as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + try: + fetched = fetch(index, (user, password), args.packages, args.versions, args.dest) + except FetchError as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + for path in fetched: + print(path) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:], os.environ)) diff --git a/scripts/deploy_hooks.sh b/scripts/deploy_hooks.sh index 89e26bf8..56604c86 100644 --- a/scripts/deploy_hooks.sh +++ b/scripts/deploy_hooks.sh @@ -38,14 +38,11 @@ CI_JOBS=( ) PYTHON=/usr/bin/python3.12 -# co-core resolves from the private co-pypi index (#183), through this repo's own -# wrapper until #201 swaps in the shared one (install.sh --co-pypi --replace). +# co-core resolves from the private co-pypi index (#183), through the shared +# wrapper (install.sh --co-pypi, #201). A deploy's shell loads no env file, and +# needs none: the wrapper's key defaults to /etc/<[project].name>/co-pypi-reader.json, +# this host's /etc/replicator/co-pypi-reader.json. UV_WRAPPER=scripts/co-pypi.sh -# This wrapper has no default key path (scripts/co_pypi_token.py refuses an unset -# variable), and a deploy's shell loads no env file: shipping-work runs the deploy -# command bare. The shared wrapper defaults to this same path, so this line goes -# with #201. -export REPLICATOR_CO_PYPI_CREDENTIALS="${REPLICATOR_CO_PYPI_CREDENTIALS:-/etc/replicator/co-pypi-reader.json}" # scripts/ too: patching-hosts runs scripts.inflight from the live release. SOURCE_DIRS=(src scripts) IMPORT_MODULES=(src.worker.main scripts.inflight) diff --git a/tests/deploy/test_deploy_hooks.py b/tests/deploy/test_deploy_hooks.py index 7f975a29..d1d6c5fa 100644 --- a/tests/deploy/test_deploy_hooks.py +++ b/tests/deploy/test_deploy_hooks.py @@ -8,6 +8,7 @@ from __future__ import annotations +import importlib.util import json import subprocess from pathlib import Path @@ -63,11 +64,17 @@ def test_keep_is_zero_spares(): def test_the_wrapper_finds_its_key_without_an_env_file(): - assert (REPO / sourced('printf "%s\\n" "$UV_WRAPPER"')[0]).is_file() - assert sourced( - 'env -u REPLICATOR_CO_PYPI_CREDENTIALS bash -c \'source "$0"; ' - 'printenv REPLICATOR_CO_PYPI_CREDENTIALS\' "$1"' - ) == ["/etc/replicator/co-pypi-reader.json"] + """A deploy loads no env file, so the shared wrapper's own default must name + the host's key (#201): the hooks export no key variable at all.""" + wrapper = sourced('printf "%s\\n" "$UV_WRAPPER"')[0] + assert wrapper == "scripts/co-pypi.sh" and (REPO / wrapper).is_file() + # From an empty environment, so a loaded shell's own variables don't count. + exported = sourced('env -i PATH="$PATH" bash -c \'source "$0"; compgen -e\' "$1"') + assert not [v for v in exported if "CO_PYPI" in v], "the wrapper's default is the key" + spec = importlib.util.spec_from_file_location("minter", REPO / "scripts" / "co_pypi_token.py") + minter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(minter) + assert minter.key_path({}) == "/etc/replicator/co-pypi-reader.json" WANT = "0123456789ab" diff --git a/tests/test_co_core_closure_diff.py b/tests/test_co_core_closure_diff.py index 656080d7..d2f6c20d 100644 --- a/tests/test_co_core_closure_diff.py +++ b/tests/test_co_core_closure_diff.py @@ -9,7 +9,7 @@ The synthetic wheels are built in ``tmp_path``, so none of this needs the index. The one real-wheel check reads the directory ``REPLICATOR_TEST_CLOSURE_WHEELS`` -names, filled by ``bash scripts/co-pypi.sh wheels 0.36.0 0.37.0 DIR`` (#183), +names, filled by ``scripts/co_pypi_wheels.py`` at 0.36.0 and 0.37.0 (#183, #201), and skips without it. """ @@ -32,6 +32,8 @@ REPO_ROOT = Path(__file__).resolve().parents[1] PACKAGES = ["co_core", "co_core_aio"] +# The fetch the error messages name (#201): the wheels fetcher, under the shared wrapper. +FETCH = "bash scripts/co-pypi.sh uv run --no-project python scripts/co_pypi_wheels.py" def _wheel(wheels: Path, package: str, version: str, files: dict, requires=()) -> None: @@ -350,8 +352,10 @@ def test_diff_adds_a_code_level_diff_of_each_changed_file_only(repo): def test_a_missing_wheel_names_the_fetch(repo): - with pytest.raises(MissingWheelError, match="co-pypi.sh wheels"): + with pytest.raises(MissingWheelError, match="scripts/co_pypi_wheels.py") as exc: load_wheels(repo / "wheels", PACKAGES, "9.9.9") + # The command to run, whole: the wrapper, this directory, the version, the packages. + assert f"{FETCH} --dest {repo / 'wheels'} --version 9.9.9 " in str(exc.value) # --- the CLI ------------------------------------------------------------------ @@ -380,7 +384,8 @@ def test_main_exits_two_on_a_missing_wheel(repo, capsys): captured = capsys.readouterr() assert captured.out == "" assert "co_core-9.9.9-py3-none-any.whl" in captured.err - assert "bash scripts/co-pypi.sh wheels" in captured.err + assert FETCH in captured.err + assert "co-pypi.sh wheels" not in captured.err def test_main_exits_two_on_a_corrupt_wheel(repo, capsys): @@ -388,7 +393,7 @@ def test_main_exits_two_on_a_corrupt_wheel(repo, capsys): assert _main(repo, "1.0.0", "1.1.0") == 2 err = capsys.readouterr().err assert "co_core-1.1.0-py3-none-any.whl" in err - assert "bash scripts/co-pypi.sh wheels" in err + assert f"{FETCH} --dest {repo / 'wheels'} --version 1.1.0 co_core" in err def test_main_exits_two_on_a_missing_root(repo, capsys): @@ -428,7 +433,7 @@ def test_the_retired_flag_is_gone(repo): @pytest.mark.skipif( not all(w.exists() for w in _REAL_WHEELS), reason="REPLICATOR_TEST_CLOSURE_WHEELS has no 0.36.0/0.37.0 wheels" - " (bash scripts/co-pypi.sh wheels 0.36.0 0.37.0 DIR)", + " (scripts/co_pypi_wheels.py --version 0.36.0 --version 0.37.0, under co-pypi.sh)", ) def test_reproduces_136_for_0_36_to_0_37(): """#136's result: 57 files at each version, six not identical. diff --git a/tests/test_co_pypi.py b/tests/test_co_pypi.py new file mode 100644 index 00000000..df663991 --- /dev/null +++ b/tests/test_co_pypi.py @@ -0,0 +1,174 @@ +"""Private dependencies resolve from the co-pypi index, through the installed wrapper. + +Copied in with deploying-releases' ``install.sh --co-pypi`` (gregoryfoster/skills#632); +the repo owns it after. Pure file reads: no network, no key. + +* **The wrapper is the installed one, unedited.** ``scripts/co-pypi.sh`` and + ``scripts/co_pypi_token.py`` match their own sha256 markers. A change belongs + upstream, then ``install.sh --co-pypi``. +* **The URL is the org registry's byte for byte**, trailing slash included: + Dependabot's updater matches it to the org's registry by string. +* **``explicit = true``, and a ``[tool.uv.sources]`` entry per private name.** + An explicit index serves only the names pinned to it, so a private name never + comes from PyPI, where it may be unclaimed, and nothing else comes from here. +* **Every private package is declared directly** (a dependency, an extra or a + group, here or in a workspace member). uv binds a source to a direct + dependency only, so one reached transitively would go to PyPI. +* **The lock carries each file's sha256**, and ``uv sync --locked`` refuses a + file that misses it. A ``.wheelhouse`` lock is refused by the deploy. +* **The ``co-pypi`` group needs nothing from the index:** the wrapper syncs it + before any token exists. +* **The repo's ruff excludes the minter,** which a ``--fix`` would otherwise edit. +""" + +from __future__ import annotations + +import hashlib +import re +import tomllib +from pathlib import Path + +import pytest +from packaging.requirements import Requirement +from packaging.utils import canonicalize_name + +REPO = Path(__file__).resolve().parents[1] +while not (REPO / "pyproject.toml").is_file() and REPO.parent != REPO: + REPO = REPO.parent +PYPROJECT = tomllib.loads((REPO / "pyproject.toml").read_text()) +LOCK = tomllib.loads((REPO / "uv.lock").read_text()) +WRAPPER = REPO / "scripts" / "co-pypi.sh" +MINTER = REPO / "scripts" / "co_pypi_token.py" +MARK_SHA = "# deploying-releases-co-pypi-sha256: " + +INDEX = "co-pypi" +URL = "https://us-west1-python.pkg.dev/co-gcs/co-pypi/simple/" +HOST = "us-west1-python.pkg.dev" + + +def _uv() -> dict: + return PYPROJECT.get("tool", {}).get("uv", {}) + + +def _declared() -> set[str]: + """Every name declared here or in a workspace member: what a source binds to.""" + paths = [REPO / "pyproject.toml"] + for pattern in _uv().get("workspace", {}).get("members", []): + paths += sorted(REPO.glob(f"{pattern}/pyproject.toml")) + names = set() + for path in paths: + data = tomllib.loads(path.read_text()) + project = data.get("project", {}) + extras = project.get("optional-dependencies", {}).values() + groups = data.get("dependency-groups", {}).values() + lines = [ + *project.get("dependencies", []), + *(r for extra in extras for r in extra), + *(r for group in groups for r in group), + ] + requirements = [Requirement(r) for r in lines if isinstance(r, str)] + names |= {canonicalize_name(req.name) for req in requirements} + return names + + +def _private() -> list[dict]: + return [p for p in LOCK["package"] if p.get("source", {}).get("registry") == URL] + + +@pytest.mark.parametrize("path", [WRAPPER, MINTER], ids=lambda p: p.name) +def test_the_wrapper_is_unedited(path: Path): + text = path.read_text() + match = re.search(rf"^{re.escape(MARK_SHA)}(\w+)$", text, flags=re.M) + assert match, f"{path.name} has no co-pypi marker: run install.sh --co-pypi" + zeroed = text.replace(match.group(0), MARK_SHA + "0" * 64, 1) + assert hashlib.sha256(zeroed.encode()).hexdigest() == match.group(1), ( + f"{path.name} was edited by hand. Raise the change on gregoryfoster/skills, " + "then reinstall with install.sh --co-pypi" + ) + + +def test_the_wrapper_is_executable(): + assert WRAPPER.stat().st_mode & 0o111 + + +def test_the_repos_ruff_leaves_the_minter_alone(): + """A `ruff --fix` or `ruff format` pass would edit the installed minter and + break its marker, so the repo's ruff excludes it.""" + ruff = PYPROJECT.get("tool", {}).get("ruff") + if ruff is None: + pytest.skip("no [tool.ruff] in pyproject.toml") + excluded = [*ruff.get("exclude", []), *ruff.get("extend-exclude", [])] + assert "scripts/co_pypi_token.py" in excluded, ( + 'add "scripts/co_pypi_token.py" to [tool.ruff] extend-exclude' + ) + + +def test_one_explicit_co_pypi_index_at_the_org_registrys_url(): + indexes = [i for i in _uv().get("index", []) if i.get("name") == INDEX] + assert len(indexes) == 1, f"want one [[tool.uv.index]] named {INDEX!r}" + assert indexes[0]["url"] == URL, "byte for byte, trailing slash included" + assert indexes[0].get("explicit") is True, ( + "an implicit index would serve every name, and a private one would be a" + " lookup away from a PyPI namesake" + ) + + +def test_no_wheelhouse(): + assert "find-links" not in _uv(), "the co-pypi index replaced the wheelhouse" + packages = LOCK["package"] + assert not [p["name"] for p in packages if "wheelhouse" in str(p.get("source"))] + + +def test_the_sweep_finds_a_private_package(): + """Guard the guard: with nothing locked from the index, the tests below are vacuous.""" + assert _private(), f"nothing in uv.lock resolves from {URL}" + + +@pytest.mark.parametrize("package", _private(), ids=lambda p: p["name"]) +def test_is_declared_directly_and_pinned_to_the_index(package: dict): + assert canonicalize_name(package["name"]) in _declared(), ( + f"{package['name']} is locked but declared nowhere: uv binds sources to" + " direct dependencies only, so a transitive one resolves from PyPI" + ) + sources = {canonicalize_name(k): v for k, v in _uv().get("sources", {}).items()} + assert sources.get(canonicalize_name(package["name"])) == {"index": INDEX} + + +@pytest.mark.parametrize("package", _private(), ids=lambda p: p["name"]) +def test_locks_a_wheel_with_a_sha256_per_file(package: dict): + """A wheel, not an sdist: CI's ``prune-cache`` keeps wheels built from + source, so a private sdist would land in a cache a fork's PR can restore.""" + assert package.get("wheels"), f"{package['name']} would build from its sdist" + for dist in [ + *([package["sdist"]] if "sdist" in package else []), + *package["wheels"], + ]: + assert dist["url"].startswith(f"https://{HOST}/"), dist["url"] + assert dist["hash"].startswith("sha256:") and len(dist["hash"]) == 7 + 64 + + +def test_the_wrappers_group_needs_nothing_from_the_index(): + assert any( + canonicalize_name(Requirement(r).name) == "google-auth" + for r in PYPROJECT.get("dependency-groups", {}).get(INDEX, []) + if isinstance(r, str) + ), ( + f"declare [dependency-groups] {INDEX} = ['google-auth[requests]>=2,<3'], " + "the minter's own dependency" + ) + by_name: dict[str, list[dict]] = {} + for package in LOCK["package"]: + by_name.setdefault(package["name"], []).append(package) + closure, todo = set(), ["google-auth"] + while todo: + name = todo.pop() + if name in closure: + continue + closure.add(name) + for package in by_name.get(name, []): + todo += [d["name"] for d in package.get("dependencies", [])] + for extra in package.get("optional-dependencies", {}).values(): + todo += [d["name"] for d in extra] + assert "requests" in closure, "the extra the mint's transport needs" + private = {p["name"] for p in _private()} + assert not closure & private, f"the {INDEX} group would need a token to sync" diff --git a/tests/test_co_pypi_token.py b/tests/test_co_pypi_token.py deleted file mode 100644 index 8de7d954..00000000 --- a/tests/test_co_pypi_token.py +++ /dev/null @@ -1,426 +0,0 @@ -"""The co-pypi wrapper (#183): one key variable, an allowlisted environment, and -a token that never reaches output. - -Design: docs/plans/2026-11-10-co-pypi-index-design.md, items 1 and 4. The -host's ``GOOGLE_APPLICATION_CREDENTIALS`` is the worker's *writer*, and the -deploy shell also carries org-wide ``GH_TOKEN_*`` PATs, so neither may reach the -wrapped ``uv`` — not merely "not be read". -""" - -import hashlib -import http.client -import io -import json -import os -import re -import stat -import subprocess -import urllib.error -import urllib.request -from pathlib import Path - -import pytest - -from scripts import co_pypi_token as cp - -REPO = Path(__file__).resolve().parent.parent -WRAPPER = REPO / "scripts" / "co-pypi.sh" -TOKEN = "ya29.SECRET-TOKEN-VALUE" -READER = "/etc/replicator/co-pypi-reader.json" -WRITER = "/etc/replicator/the-writer.json" # a placeholder: test_destinations.py - -# What `. scripts/load_env.sh` leaves in a deploy shell, abridged. -LOADED = { - "PATH": "/usr/local/bin:/usr/bin:/bin", - "HOME": "/home/exedev", - "LANG": "C.UTF-8", - "LC_ALL": "C.UTF-8", - "XDG_CACHE_HOME": "/home/exedev/.cache", - "UV_CACHE_DIR": "/home/exedev/.cache/uv", - "SSL_CERT_FILE": "/etc/ssl/certs/ca-certificates.crt", - "https_proxy": "http://proxy:3128", - "VIRTUAL_ENV": "/tmp/ephemeral-env", - cp.KEY_VAR: READER, - "GOOGLE_APPLICATION_CREDENTIALS": WRITER, - "GH_TOKEN": "ghp_replicator", - "GH_TOKEN_BROKER": "ghp_broker", - "ANTHROPIC_API_KEY": "sk-ant-x", - "REPLICATOR_NOTIFY_TOKEN": "notify", -} - - -class Exec: - """Records the exec instead of replacing the process.""" - - def __init__(self, error: OSError | None = None): - self.calls: list[tuple[str, list[str], dict[str, str]]] = [] - self.error = error - - def __call__(self, file, args, env): - self.calls.append((file, args, env)) - if self.error: - raise self.error - - -def _mint_from(seen: list[str]): - def mint(key_path): - seen.append(key_path) - return TOKEN - - return mint - - -# --- the environment ------------------------------------------------------- - - -def test_the_child_environment_is_the_allowlist(): - env = cp.child_environment(LOADED) - assert env == { - "PATH": LOADED["PATH"], - "HOME": LOADED["HOME"], - "LANG": "C.UTF-8", - "LC_ALL": "C.UTF-8", - "XDG_CACHE_HOME": LOADED["XDG_CACHE_HOME"], - "UV_CACHE_DIR": LOADED["UV_CACHE_DIR"], - "SSL_CERT_FILE": LOADED["SSL_CERT_FILE"], - "https_proxy": LOADED["https_proxy"], - } - - -def test_the_writer_and_the_pats_never_reach_uv(): - execvpe = Exec() - cp.main(["uv", "sync", "--frozen"], LOADED, mint=lambda _: TOKEN, execvpe=execvpe) - (_, _, env) = execvpe.calls[0] - assert "GOOGLE_APPLICATION_CREDENTIALS" not in env - assert not [k for k in env if k.startswith("GH_TOKEN")] - assert "ANTHROPIC_API_KEY" not in env and "REPLICATOR_NOTIFY_TOKEN" not in env - assert "VIRTUAL_ENV" not in env, "the outer `uv run`'s env would be synced instead" - # The reader key's path is the mint's input, not uv's. - assert cp.KEY_VAR not in env - - -def test_uv_gets_the_pair_and_its_arguments_unchanged(): - execvpe = Exec() - assert cp.main(["uv", "lock"], LOADED, mint=lambda _: TOKEN, execvpe=execvpe) is None - (file, args, env) = execvpe.calls[0] - assert (file, args) == ("uv", ["uv", "lock"]) - assert env[cp.USERNAME_VAR] == "oauth2accesstoken" - assert env[cp.PASSWORD_VAR] == TOKEN - assert TOKEN not in " ".join(args) - - -# --- the key --------------------------------------------------------------- - - -def test_the_reader_key_is_minted_never_the_writer(): - seen: list[str] = [] - cp.main(["uv", "lock"], LOADED, mint=_mint_from(seen), execvpe=Exec()) - assert seen == [READER] - - -def test_no_fallback_to_the_writer_when_the_key_variable_is_unset(capsys): - seen: list[str] = [] - environ = {k: v for k, v in LOADED.items() if k != cp.KEY_VAR} - execvpe = Exec() - assert cp.main(["uv", "lock"], environ, mint=_mint_from(seen), execvpe=execvpe) == 2 - assert seen == [] and execvpe.calls == [] - assert cp.KEY_VAR in capsys.readouterr().err - - -def test_a_blank_key_variable_is_unset(capsys): - environ = {**LOADED, cp.KEY_VAR: " "} - assert cp.main(["uv", "lock"], environ, mint=_mint_from([]), execvpe=Exec()) == 2 - - -def _service_account_key(path: Path, email: str) -> str: - """A throwaway service-account key file, real enough for google-auth to load.""" - from cryptography.hazmat.primitives import serialization - from cryptography.hazmat.primitives.asymmetric import rsa - - pem = ( - rsa.generate_private_key(public_exponent=65537, key_size=2048) - .private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.PKCS8, - serialization.NoEncryption(), - ) - .decode() - ) - key = { - "type": "service_account", - "project_id": "test", - "private_key_id": "0", - "private_key": pem, - "client_email": email, - "client_id": "0", - "token_uri": "https://oauth2.googleapis.com/token", - } - path.write_text(json.dumps(key)) - return str(path) - - -def test_the_mint_loads_the_named_key_even_with_a_real_writer_key_in_adc(monkeypatch, tmp_path): - """The real loader runs, and only the token exchange is faked (CR 7). A real - writer key sits where ADC would look: the token is the reader's whatever - GOOGLE_APPLICATION_CREDENTIALS holds, and google.auth.default never runs.""" - reader = _service_account_key(tmp_path / "reader.json", "co-pypi-reader@test.iam") - writer = _service_account_key(tmp_path / "writer.json", "the-writer@test.iam") - monkeypatch.setenv("GOOGLE_APPLICATION_CREDENTIALS", writer) - minted_for: list[tuple[str, tuple]] = [] - - def refresh(self, request): - minted_for.append((self.service_account_email, tuple(self.scopes))) - self.token = TOKEN - - def adc(*a, **k): - raise AssertionError("google.auth.default() must not run") - - monkeypatch.setattr(cp.service_account.Credentials, "refresh", refresh) - monkeypatch.setattr("google.auth.default", adc) - assert cp.mint_token(reader) == TOKEN - assert minted_for == [("co-pypi-reader@test.iam", tuple(cp.SCOPES))] - - -def test_a_key_that_is_not_a_key_fails_the_mint(tmp_path, capsys): - bad = tmp_path / "not-a-key.json" - bad.write_text("{}") - environ = {**LOADED, cp.KEY_VAR: str(bad)} - execvpe = Exec() - assert cp.main(["uv", "lock"], environ, execvpe=execvpe) == 1 - assert execvpe.calls == [] - assert str(bad) in capsys.readouterr().err - - -# --- refusals and failures ------------------------------------------------- - - -@pytest.mark.parametrize("argv", [[], ["pip", "install"], ["bash", "-c", "env"], ["/bin/uvx"]]) -def test_anything_but_uv_or_wheels_is_refused_before_a_mint(argv, capsys): - seen: list[str] = [] - assert cp.main(argv, LOADED, mint=_mint_from(seen), execvpe=Exec()) == 2 - assert seen == [] - assert "usage" in capsys.readouterr().err - - -def test_a_uv_by_path_is_still_uv(): - execvpe = Exec() - cp.main(["/usr/local/bin/uv", "sync"], LOADED, mint=lambda _: TOKEN, execvpe=execvpe) - assert execvpe.calls[0][0] == "/usr/local/bin/uv" - - -def test_a_failed_mint_runs_nothing_and_prints_no_token(capsys): - def mint(_): - raise RuntimeError("invalid_grant: account not found") - - execvpe = Exec() - assert cp.main(["uv", "lock"], LOADED, mint=mint, execvpe=execvpe) == 1 - out = capsys.readouterr() - assert execvpe.calls == [] - assert "invalid_grant" in out.err and READER in out.err - assert TOKEN not in out.out + out.err - - -def test_a_failed_exec_names_the_command_and_prints_no_token(capsys): - execvpe = Exec(FileNotFoundError(2, "No such file or directory")) - assert cp.main(["uv", "lock"], LOADED, mint=lambda _: TOKEN, execvpe=execvpe) == 127 - out = capsys.readouterr() - assert "uv" in out.err - assert TOKEN not in out.out + out.err - - -# --- the wheels verb (item 4) ---------------------------------------------- - - -def _wheel(name: str) -> bytes: - return f"wheel bytes of {name}".encode() - - -def _page(package: str, files: dict[str, bytes], *, host=cp.INDEX_HOST, bad: str = "") -> bytes: - links = [] - for name, data in files.items(): - digest = hashlib.sha256(b"tampered" if name == bad else data).hexdigest() - links.append( - f'{name}' - ) - return ("" + "\n".join(links) + "").encode() - - -def _index(versions=("0.40.1", "0.41.0"), **page_kw): - """A fake opener over the three packages' pages and their files.""" - files: dict[str, bytes] = {} - pages: dict[str, bytes] = {} - for package in cp.PACKAGES: - mine = {} - for version in versions: - for suffix in ("-py3-none-any.whl", ".tar.gz"): - name = f"{package.replace('-', '_')}-{version}{suffix}" - mine[name] = _wheel(name) - files.update(mine) - pages[f"{cp.INDEX_URL}{package}/"] = _page(package, mine, **page_kw) - requests: list[urllib.request.Request] = [] - - def opener(request): - requests.append(request) - url = request.full_url - if url in pages: - return io.BytesIO(pages[url]) - name = url.rsplit("/", 1)[-1] - if name in files: - return io.BytesIO(files[name]) - raise urllib.error.HTTPError(url, 404, "Not Found", {}, None) - - return opener, requests, files - - -def test_wheels_downloads_the_six_wheels_verified(tmp_path): - opener, requests, files = _index() - got = cp.fetch_wheels(TOKEN, ["0.40.1", "0.41.0"], tmp_path, opener=opener) - names = sorted(p.name for p in got) - assert names == sorted(n for n in files if n.endswith(".whl")) - assert sorted(p.name for p in tmp_path.iterdir()) == names, "no sdist, no temp file" - for path in got: - assert path.read_bytes() == files[path.name] - assert all(r.get_header("Authorization", "").startswith("Basic ") for r in requests) - - -def test_wheels_refuses_a_digest_mismatch_and_leaves_nothing(tmp_path): - bad = "co_core_aio-0.41.0-py3-none-any.whl" - opener, _, _ = _index(bad=bad) - with pytest.raises(cp.FetchError, match=bad): - cp.fetch_wheels(TOKEN, ["0.40.1", "0.41.0"], tmp_path, opener=opener) - assert not (tmp_path / bad).exists() - assert not [p for p in tmp_path.iterdir() if p.name.startswith(".")] - - -def test_wheels_refuses_a_version_the_index_lacks(tmp_path): - opener, _, _ = _index(versions=("0.41.0",)) - with pytest.raises(cp.FetchError, match="co_core-0.40.1-py3-none-any.whl"): - cp.fetch_wheels(TOKEN, ["0.40.1", "0.41.0"], tmp_path, opener=opener) - - -def test_wheels_never_sends_the_token_to_another_host(tmp_path): - opener, requests, _ = _index(host="evil.example") - with pytest.raises(cp.FetchError, match="evil.example"): - cp.fetch_wheels(TOKEN, ["0.41.0"], tmp_path, opener=opener) - assert all(r.host == cp.INDEX_HOST for r in requests) - - -@pytest.mark.parametrize( - ("newurl", "followed"), - [ - (f"https://{cp.INDEX_HOST}/artifacts-downloads/x", True), - ("https://storage.googleapis.com/x", False), - (f"http://{cp.INDEX_HOST}/x", False), - ], -) -def test_a_redirect_is_followed_only_on_the_index_host(newurl, followed): - """The index answers a file with a same-host 307 [measured 2026-11-10], and - urllib forwards the Authorization header across any redirect.""" - handler = cp.IndexHostRedirect() - request = urllib.request.Request(f"{cp.INDEX_URL}co-core/", headers={"Authorization": "x"}) - if followed: - new = handler.redirect_request(request, None, 307, "", {}, newurl) - assert new is not None and new.full_url == newurl - else: - with pytest.raises(urllib.error.HTTPError): - handler.redirect_request(request, None, 307, "", {}, newurl) - - -def test_the_wheels_verb_mints_then_fetches(tmp_path): - calls = [] - - def fetch(token, versions, dest): - calls.append((token, versions, dest)) - return [] - - rc = cp.main( - ["wheels", "0.40.1", "0.41.0", str(tmp_path)], LOADED, mint=lambda _: TOKEN, fetch=fetch - ) - assert rc == 0 - assert calls == [(TOKEN, ["0.40.1", "0.41.0"], tmp_path)] - - -@pytest.mark.parametrize( - "argv", [["wheels"], ["wheels", "0.41.0", "dir"], ["wheels", "a", "b", "c", "d"]] -) -def test_the_wheels_verb_takes_two_versions_and_a_directory(argv, capsys): - seen: list[str] = [] - assert cp.main(argv, LOADED, mint=_mint_from(seen), fetch=lambda *a: []) == 2 - assert seen == [] - - -def test_a_failed_fetch_prints_no_token(tmp_path, capsys): - def fetch(token, versions, dest): - raise cp.FetchError(f"sha256 mismatch for x (url had {cp.INDEX_HOST})") - - rc = cp.main( - ["wheels", "0.40.1", "0.41.0", str(tmp_path)], LOADED, mint=lambda _: TOKEN, fetch=fetch - ) - out = capsys.readouterr() - assert rc == 1 - assert TOKEN not in out.out + out.err - - -# --- the shell wrapper ------------------------------------------------------ - - -@pytest.fixture -def fake_uv(tmp_path): - """A `uv` first on PATH that records its argv and environment.""" - bindir = tmp_path / "bin" - bindir.mkdir() - record = tmp_path / "record" - uv = bindir / "uv" - uv.write_text(f'#!/bin/bash\nprintf "%s\\n" "$@" > {record}.argv\nenv > {record}.env\n') - uv.chmod(uv.stat().st_mode | stat.S_IXUSR) - return bindir, record - - -def test_the_shell_wrapper_starts_the_outer_uv_from_the_allowlist(fake_uv): - bindir, record = fake_uv - environ = {**LOADED, "PATH": f"{bindir}:{os.environ['PATH']}"} - subprocess.run(["bash", str(WRAPPER), "uv", "sync", "--frozen"], env=environ, check=True) - env = dict( - line.split("=", 1) for line in Path(f"{record}.env").read_text().splitlines() if "=" in line - ) - argv = Path(f"{record}.argv").read_text().splitlines() - assert "GOOGLE_APPLICATION_CREDENTIALS" not in env - assert not [k for k in env if k.startswith("GH_TOKEN")] - assert "ANTHROPIC_API_KEY" not in env and "VIRTUAL_ENV" not in env - assert env[cp.KEY_VAR] == READER, "the mint's one input" - assert argv[:3] == ["run", "--no-project", "--with"] - assert argv[-4:] == [str(REPO / "scripts" / "co_pypi_token.py"), "uv", "sync", "--frozen"] - - -def test_the_shell_wrapper_allowlist_matches_the_python_one(): - """Two layers, one list: a variable added to one and not the other would - reach uv through the gap.""" - text = WRAPPER.read_text() - - def array(name: str) -> set[str]: - m = re.search(rf"^{name}=\(([^)]*)\)", text, flags=re.MULTILINE) - assert m, f"{name}=(…) not found in {WRAPPER.name}: the derivation broke" - return set(m.group(1).split()) - - # The shell layer also carries the key variable: it is the mint's input. - assert array("ALLOWED") == set(cp.ALLOWED) | {cp.KEY_VAR} - assert array("PREFIXES") == set(cp.ALLOWED_PREFIXES) - - -@pytest.mark.parametrize( - "error", - [ - TimeoutError("The read operation timed out"), - http.client.RemoteDisconnected("Remote end closed connection without response"), - http.client.IncompleteRead(b"partial"), - ], - ids=["timeout", "disconnected", "incomplete-read"], -) -def test_a_dropped_connection_is_a_fetch_error_not_a_traceback(tmp_path, error): - """CR 4: only URLError was caught, so a flaky network escaped as a traceback.""" - - def opener(request): - raise error - - with pytest.raises(cp.FetchError, match=cp.INDEX_HOST): - cp.fetch_wheels(TOKEN, ["0.41.0"], tmp_path, opener=opener) diff --git a/tests/test_co_pypi_wheels.py b/tests/test_co_pypi_wheels.py new file mode 100644 index 00000000..b4175f20 --- /dev/null +++ b/tests/test_co_pypi_wheels.py @@ -0,0 +1,354 @@ +"""The co-pypi wheel fetcher: wheels on disk, each sha256-checked against the index. + +``scripts/co_pypi_wheels.py`` runs under the co-pypi wrapper, as +``bash scripts/co-pypi.sh uv run --no-project python scripts/co_pypi_wheels.py``, +and reads the credential pair the wrapper hands uv. These tests need nothing +from the repo they sit in: the fetcher is loaded by path, the index is a fake +opener, and each ``pyproject.toml`` is written here. No network, no key. +""" + +from __future__ import annotations + +import ast +import base64 +import hashlib +import http.client +import importlib.util +import io +import os +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + +NAME = "co_pypi_wheels.py" +HERE = Path(__file__).resolve().parent +# Beside this test (the skill's assets) or in the repo's scripts/ (installed). +SCRIPT = next( + (p for p in (HERE / NAME, HERE.parent / "scripts" / NAME) if p.is_file()), + HERE.parent / "scripts" / NAME, +) +HOST = "pkg.example" +INDEX = f"https://{HOST}/acme/co-pypi/simple/" +USER = "oauth2accesstoken" +TOKEN = "ya29.sentinel-never-printed" +PAIR = {"UV_INDEX_CO_PYPI_USERNAME": USER, "UV_INDEX_CO_PYPI_PASSWORD": TOKEN} + + +@pytest.fixture(scope="module") +def fw(): + spec = importlib.util.spec_from_file_location("co_pypi_wheels_under_test", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _pyproject(tmp_path: Path, index: str | None = INDEX) -> Path: + path = tmp_path / "pyproject.toml" + body = '[project]\nname = "acme"\n' + if index is not None: + body += f'\n[[tool.uv.index]]\nname = "co-pypi"\nurl = "{index}"\nexplicit = true\n' + path.write_text(body) + return path + + +def _index(packages, versions, *, host=HOST, bad="", no_digest=""): + """A fake opener over each package's PEP 503 page and its files.""" + files: dict[str, bytes] = {} + pages: dict[str, bytes] = {} + for page, prefix in packages: + links = [] + for version in versions: + for suffix in ("-py3-none-any.whl", ".tar.gz"): + name = f"{prefix}-{version}{suffix}" + data = f"bytes of {name}".encode() + files[name] = data + digest = hashlib.sha256(b"tampered" if name == bad else data).hexdigest() + fragment = "" if name == no_digest else f"#sha256={digest}" + links.append( + f'{name}' + ) + pages[f"{INDEX}{page}/"] = ("" + "\n".join(links) + "").encode() + requests: list[urllib.request.Request] = [] + + def opener(request): + requests.append(request) + url = request.full_url + if url in pages: + return io.BytesIO(pages[url]) + name = url.rsplit("/", 1)[-1] + if name in files: + return io.BytesIO(files[name]) + raise urllib.error.HTTPError(url, 404, "Not Found", {}, None) + + return opener, requests, files + + +CO_CORE = [("co-core", "co_core"), ("co-core-aio", "co_core_aio")] + + +def _fetch(fw, tmp_path, opener, packages=("co-core", "co-core-aio"), versions=("0.1.0", "0.2.0")): + return fw.fetch_wheels( + INDEX, (USER, TOKEN), list(packages), list(versions), tmp_path / "wheels", opener=opener + ) + + +# --- the fetch --------------------------------------------------------------- + + +def test_fetches_each_package_at_each_version_verified(fw, tmp_path): + opener, requests, files = _index(CO_CORE, ["0.1.0", "0.2.0"]) + got = _fetch(fw, tmp_path, opener) + want = sorted(n for n in files if n.endswith(".whl")) + assert sorted(p.name for p in got) == want + assert sorted(p.name for p in (tmp_path / "wheels").iterdir()) == want, "no sdist, no temp file" + for path in got: + assert path.read_bytes() == files[path.name] + basic = "Basic " + base64.b64encode(f"{USER}:{TOKEN}".encode()).decode() + assert {r.get_header("Authorization") for r in requests} == {basic} + + +def test_a_digest_mismatch_names_the_file_and_leaves_nothing(fw, tmp_path): + bad = "co_core_aio-0.2.0-py3-none-any.whl" + opener, _, _ = _index(CO_CORE, ["0.1.0", "0.2.0"], bad=bad) + with pytest.raises(fw.FetchError, match=f"sha256 mismatch for {bad}"): + _fetch(fw, tmp_path, opener) + assert not (tmp_path / "wheels" / bad).exists() + assert not [p for p in (tmp_path / "wheels").iterdir() if p.name.startswith(".")] + + +def test_a_link_without_a_digest_is_refused(fw, tmp_path): + bare = "co_core-0.1.0-py3-none-any.whl" + opener, _, _ = _index(CO_CORE, ["0.1.0"], no_digest=bare) + with pytest.raises(fw.FetchError, match=bare): + _fetch(fw, tmp_path, opener, versions=["0.1.0"]) + assert not (tmp_path / "wheels" / bare).exists() + + +def test_a_version_the_index_lacks_names_the_file(fw, tmp_path): + opener, _, _ = _index(CO_CORE, ["0.2.0"]) + with pytest.raises(fw.FetchError, match="co_core-0.1.0-py3-none-any.whl"): + _fetch(fw, tmp_path, opener) + + +def test_a_package_the_index_lacks_names_its_page(fw, tmp_path): + opener, _, _ = _index(CO_CORE, ["0.1.0"]) + with pytest.raises(fw.FetchError, match=f"{INDEX}absent/"): + _fetch(fw, tmp_path, opener, packages=["absent"], versions=["0.1.0"]) + + +@pytest.mark.parametrize( + ("package", "page", "prefix"), + [ + ("co-core", "co-core", "co_core"), + ("co_core_aio", "co-core-aio", "co_core_aio"), + ("Co.Core--Sync", "co-core-sync", "co_core_sync"), + ], +) +def test_names_are_normalized(fw, tmp_path, package, page, prefix): + """The page by PEP 503, the file by the wheel spec: what the closure diff reads.""" + opener, requests, _ = _index([(page, prefix)], ["1.0.0"]) + [got] = _fetch(fw, tmp_path, opener, packages=[package], versions=["1.0.0"]) + assert got == tmp_path / "wheels" / f"{prefix}-1.0.0-py3-none-any.whl" + assert requests[0].full_url == f"{INDEX}{page}/" + + +def test_the_token_never_goes_to_another_host(fw, tmp_path): + opener, requests, _ = _index(CO_CORE, ["0.1.0"], host="evil.example") + with pytest.raises(fw.FetchError, match="evil.example"): + _fetch(fw, tmp_path, opener, versions=["0.1.0"]) + assert all(r.host == HOST for r in requests) + + +@pytest.mark.parametrize( + ("newurl", "followed"), + [ + (f"https://{HOST}/artifacts-downloads/x", True), + ("https://storage.googleapis.com/x", False), + (f"http://{HOST}/x", False), + ], +) +def test_a_redirect_is_followed_only_on_the_index_host(fw, newurl, followed): + """Artifact Registry answers a file with a same-host 307, and urllib copies + the Authorization header onto any redirect.""" + handler = fw.IndexHostRedirect(HOST) + request = urllib.request.Request(f"{INDEX}co-core/", headers={"Authorization": "x"}) + if followed: + new = handler.redirect_request(request, None, 307, "", {}, newurl) + assert new is not None and new.full_url == newurl + else: + with pytest.raises(urllib.error.HTTPError): + handler.redirect_request(request, None, 307, "", {}, newurl) + + +@pytest.mark.parametrize( + "error", + [ + TimeoutError("The read operation timed out"), + http.client.RemoteDisconnected("Remote end closed connection without response"), + http.client.IncompleteRead(b"partial"), + urllib.error.URLError("Name or service not known"), + ], + ids=["timeout", "disconnected", "incomplete-read", "dns"], +) +def test_a_dropped_connection_is_a_fetch_error_not_a_traceback(fw, tmp_path, error): + def opener(request): + raise error + + with pytest.raises(fw.FetchError, match=HOST): + _fetch(fw, tmp_path, opener, versions=["0.1.0"]) + + +def test_an_unusable_destination_is_a_fetch_error_naming_it(fw, tmp_path): + """A file where the directory should be: the error names it, no traceback.""" + opener, _, _ = _index(CO_CORE, ["0.1.0"]) + (tmp_path / "wheels").write_text("not a directory") + with pytest.raises(fw.FetchError, match=str(tmp_path / "wheels")): + _fetch(fw, tmp_path, opener, versions=["0.1.0"]) + + +def test_a_page_that_is_not_utf8_is_a_fetch_error_naming_it(fw, tmp_path): + def opener(request): + return io.BytesIO(b"\xff\xfe") + + with pytest.raises(fw.FetchError, match=f"{INDEX}co-core/"): + _fetch(fw, tmp_path, opener, packages=["co-core"], versions=["0.1.0"]) + + +# --- the index URL ----------------------------------------------------------- + + +def test_the_index_is_the_pyprojects_co_pypi_index(fw, tmp_path): + assert fw.index_url(_pyproject(tmp_path)) == INDEX + + +def test_a_pyproject_without_the_index_is_a_lookup_error(fw, tmp_path): + with pytest.raises(LookupError, match="co-pypi"): + fw.index_url(_pyproject(tmp_path, index=None)) + with pytest.raises(LookupError): + fw.index_url(tmp_path / "absent.toml") + + +def test_the_default_pyproject_is_the_checkouts_own(fw): + assert fw.PYPROJECT == SCRIPT.resolve().parent.parent / "pyproject.toml" + + +# --- the command ------------------------------------------------------------- + +ARGV = ["--dest", "DIR", "--version", "0.1.0", "--version", "0.2.0", "co-core", "co-core-aio"] + + +class Fetch: + def __init__(self, result=(), error: Exception | None = None): + self.calls: list[tuple] = [] + self.result, self.error = list(result), error + + def __call__(self, index, auth, packages, versions, dest): + self.calls.append((index, auth, packages, versions, dest)) + if self.error: + raise self.error + return self.result + + +def test_main_fetches_with_the_wrappers_pair_and_prints_each_path(fw, tmp_path, capsys): + paths = [tmp_path / "a.whl", tmp_path / "b.whl"] + fetch = Fetch(result=paths) + rc = fw.main(ARGV, {**PAIR, "PATH": "/bin"}, fetch=fetch, pyproject=_pyproject(tmp_path)) + out = capsys.readouterr() + assert rc == 0 + assert fetch.calls == [ + (INDEX, (USER, TOKEN), ["co-core", "co-core-aio"], ["0.1.0", "0.2.0"], Path("DIR")) + ] + assert out.out.splitlines() == [str(p) for p in paths] + assert TOKEN not in out.out + out.err + + +@pytest.mark.parametrize("missing", sorted(PAIR)) +def test_outside_the_wrapper_it_exits_2_and_names_the_wrapper(fw, tmp_path, capsys, missing): + environ = {k: v for k, v in PAIR.items() if k != missing} + fetch = Fetch() + assert fw.main(ARGV, environ, fetch=fetch, pyproject=_pyproject(tmp_path)) == 2 + assert fetch.calls == [] + err = capsys.readouterr().err + assert "bash scripts/co-pypi.sh uv run --no-project python" in err + assert TOKEN not in err + + +@pytest.mark.parametrize( + "argv", + [ + [], + ["--dest", "DIR", "--version", "0.1.0"], # no package + ["--dest", "DIR", "co-core"], # no version + ["--version", "0.1.0", "co-core"], # no dest + ], + ids=["nothing", "no-package", "no-version", "no-dest"], +) +def test_usage_errors_exit_2_before_a_fetch(fw, tmp_path, argv): + fetch = Fetch() + with pytest.raises(SystemExit) as exc: + fw.main(argv, PAIR, fetch=fetch, pyproject=_pyproject(tmp_path)) + assert exc.value.code == 2 and fetch.calls == [] + + +def test_no_co_pypi_index_exits_2(fw, tmp_path, capsys): + fetch = Fetch() + assert fw.main(ARGV, PAIR, fetch=fetch, pyproject=_pyproject(tmp_path, index=None)) == 2 + assert fetch.calls == [] and "co-pypi" in capsys.readouterr().err + + +def test_a_failed_fetch_exits_1_names_the_file_and_prints_no_token(fw, tmp_path, capsys): + fetch = Fetch(error=fw.FetchError("sha256 mismatch for co_core-0.2.0-py3-none-any.whl")) + rc = fw.main(ARGV, PAIR, fetch=fetch, pyproject=_pyproject(tmp_path)) + out = capsys.readouterr() + assert rc == 1 + assert "co_core-0.2.0-py3-none-any.whl" in out.err + assert TOKEN not in out.out + out.err + + +def test_end_to_end_through_main_with_a_fake_index(fw, tmp_path, capsys): + """main's own fetch, the opener aside: what a run under the wrapper does.""" + opener, _, files = _index(CO_CORE, ["0.1.0"]) + + def fetch(index, auth, packages, versions, dest): + return fw.fetch_wheels(index, auth, packages, versions, dest, opener=opener) + + argv = ["--dest", str(tmp_path / "w"), "--version", "0.1.0", "co-core", "co-core-aio"] + assert fw.main(argv, PAIR, fetch=fetch, pyproject=_pyproject(tmp_path)) == 0 + assert sorted(p.name for p in (tmp_path / "w").iterdir()) == sorted( + n for n in files if n.endswith(".whl") + ) + + +def test_the_script_runs_as_a_command_and_refuses_without_the_pair(tmp_path): + env = {"PATH": os.environ["PATH"], "HOME": str(tmp_path)} + script = [sys.executable, "-I", str(SCRIPT)] + assert subprocess.run([*script, "--help"], env=env, capture_output=True).returncode == 0 + result = subprocess.run([*script, *ARGV], env=env, capture_output=True, text=True) + assert result.returncode == 2 + assert "co-pypi.sh" in result.stderr + + +# --- built to travel --------------------------------------------------------- + + +def test_the_fetcher_is_no_one_repos(fw): + """No repo's name, no repo variable, no repo path: it drops in anywhere.""" + text = SCRIPT.read_text().lower() + for word in ("replicator", "co_core_closure_diff", "src/"): + assert word not in text, word + + +def test_the_fetcher_imports_the_standard_library_alone(): + """It runs under `uv run --no-project`: nothing to install, nothing resolved.""" + tree = ast.parse(SCRIPT.read_text()) + tops = { + (alias.name if isinstance(node, ast.Import) else node.module or "").split(".")[0] + for node in ast.walk(tree) + if isinstance(node, ast.Import | ast.ImportFrom) + for alias in node.names + } - {"__future__", ""} + assert tops <= set(sys.stdlib_module_names), tops - set(sys.stdlib_module_names) diff --git a/uv.lock b/uv.lock index 5970dc9b..4bce036f 100644 --- a/uv.lock +++ b/uv.lock @@ -557,6 +557,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/b3/6117b2f24065cd7e2c4f140e9a193e215f089ca8ba314cf91eb9d0b7fe0a/google_auth-2.56.3-py3-none-any.whl", hash = "sha256:8ec438808f813ad034535000261eed1067475d229d05bbf4216e78c3f2362e53", size = 259116, upload-time = "2026-08-06T06:22:51.788Z" }, ] +[package.optional-dependencies] +requests = [ + { name = "requests" }, +] + [[package]] name = "google-cloud-core" version = "2.6.1" @@ -1330,6 +1335,9 @@ dependencies = [ ] [package.dev-dependencies] +co-pypi = [ + { name = "google-auth", extra = ["requests"] }, +] dev = [ { name = "anyio" }, { name = "fakeredis" }, @@ -1359,6 +1367,7 @@ requires-dist = [ ] [package.metadata.requires-dev] +co-pypi = [{ name = "google-auth", extras = ["requests"], specifier = ">=2,<3" }] dev = [ { name = "anyio", specifier = ">=4.0,<5" }, { name = "fakeredis", specifier = ">=2.26,<3" },