diff --git a/.github/actions/deploy-prod/action.yml b/.github/actions/deploy-prod/action.yml index 59f8e198e9..5512885f5c 100644 --- a/.github/actions/deploy-prod/action.yml +++ b/.github/actions/deploy-prod/action.yml @@ -188,6 +188,14 @@ runs: shell: bash run: bash scripts/guard-coroot-baseline-context.sh before-publish + - name: 🧭 Validate GitHub bootstrap isolation + shell: bash + run: | + bootstrap_render="$(mktemp)" + trap 'rm -f "$bootstrap_render"' EXIT + kubectl kustomize k8s/providers/hetzner/apps > "$bootstrap_render" + bash scripts/guard-github-bootstrap-isolation.sh "$bootstrap_render" + - name: 💾 Validate live persistence retirement safety # A commit parent is not evidence that its protection reached production: # two direct pushes can be waiting behind one manual dispatch. Compare the diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 65f32017c8..cf42cd9b16 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -35,6 +35,12 @@ jobs: go-version-file: go.mod cache: false + - name: 🧭 Validate GitHub bootstrap isolation + run: | + shellcheck scripts/guard-github-bootstrap-isolation.sh scripts/tests/test-github-bootstrap-isolation.sh + bash scripts/tests/test-github-bootstrap-isolation.sh + bash scripts/tests/test-crossplane-sync-alerter.sh + - name: 🛡️ Validate preserved policy failure actions run: | go test ./scripts/validate-policy-actions diff --git a/docs/github-management.md b/docs/github-management.md index 81e4b1c3ff..8df1818e87 100644 --- a/docs/github-management.md +++ b/docs/github-management.md @@ -44,6 +44,15 @@ time the app reconciles, the provider (infrastructure tier) has installed the namespaced github CRDs the `ProviderConfig` and managed resources need. On a fresh install the app's `Kustomization` retries benignly until those CRDs exist. +GitHub API reconciliation is asynchronous: the tenant Kustomization uses +`wait: false` and has no explicit managed-resource health checks. A repository +awaiting manual bootstrap cannot freeze unrelated application delivery. Artifact +verification, admission and apply failures still fail the tenant reconciliation; +the parent continues checking application health. Crossplane's separate sync +exporter and `crossplane-sync-alert` report unsynced repositories and failed +alert delivery rather than hiding them behind Flux readiness. The bootstrap +isolation guard checks the rendered tenant before publication. + The GitHub App credentials are **not** SOPS-managed. The vault-config bootstrap Job seeds **placeholder** values into OpenBao (`secret/infrastructure/github/app`) **only when the secret is absent**, and the maintainer overwrites them in place @@ -61,8 +70,8 @@ Velero), so the manually-set values are durable without a GitOps source of truth required for the `maintainers`-team resources**: `Team` observe needs `read`, and `TeamMembership`/`TeamRepository` create needs `write` — without it they stay stuck on `CannotObserveExternalResource` / - `403 must be an organization owner or team maintainer` and flap the - `github-config` Flux health check. Install it on **all repositories** + `403 must be an organization owner or team maintainer` and raise the + Crossplane sync alert. Install it on **all repositories** of the org. No webhook. 2. **Overwrite the placeholders in OpenBao** with the App's real values — the keys already exist (seeded by the vault-config Job), so just set them. The diff --git a/k8s/bases/apps/github-config/flux-kustomization.yaml b/k8s/bases/apps/github-config/flux-kustomization.yaml index 520756c965..7a4b655f57 100644 --- a/k8s/bases/apps/github-config/flux-kustomization.yaml +++ b/k8s/bases/apps/github-config/flux-kustomization.yaml @@ -14,7 +14,10 @@ spec: retryInterval: 2m path: . prune: true - wait: true + # Applying the signed artifact must succeed, but asynchronous GitHub API + # operations (including manual repository bootstrap) cannot gate app delivery. + # Crossplane's Synced condition is monitored by crossplane-sync-alert instead. + wait: false force: true # Runs as the app SA: the GitHub managed resources are applied with only the # namespace-scoped authority in rbac.yaml. This Kustomization is itself applied diff --git a/k8s/bases/infrastructure/cluster-policies/flux/enforce-flux-best-practices.yaml b/k8s/bases/infrastructure/cluster-policies/flux/enforce-flux-best-practices.yaml index efc47ea2a1..96958c6d97 100644 --- a/k8s/bases/infrastructure/cluster-policies/flux/enforce-flux-best-practices.yaml +++ b/k8s/bases/infrastructure/cluster-policies/flux/enforce-flux-best-practices.yaml @@ -49,19 +49,34 @@ spec: message: >- Flux Kustomization must set the recommended reliability fields: `spec.interval`, `spec.timeout`, `spec.retryInterval`, - `spec.prune: true` (garbage collection), `spec.wait: true` - (health-gated reconciliation), and `spec.sourceRef`. + `spec.prune: true` (garbage collection), and `spec.sourceRef`. + `spec.wait: true` is required except for the platform-owned GitHub + configuration, whose asynchronous API operations do not gate delivery. See https://fluxcd.io/flux/components/kustomize/kustomizations/#recommended-settings. - pattern: - spec: - interval: "?*" - timeout: "?*" - retryInterval: "?*" - prune: true - wait: true - sourceRef: - kind: "?*" - name: "?*" + anyPattern: + - spec: + interval: "?*" + timeout: "?*" + retryInterval: "?*" + prune: true + wait: true + sourceRef: + kind: "?*" + name: "?*" + # Bound the alternative to this exact platform-owned identity. Keep + # every other reliability requirement; this is not a policy exclusion. + - metadata: + name: github-config + namespace: github-config + spec: + interval: "?*" + timeout: "?*" + retryInterval: "?*" + prune: true + wait: false + sourceRef: + kind: "?*" + name: "?*" - name: helmrelease-recommended-settings match: any: diff --git a/scripts/guard-github-bootstrap-isolation.sh b/scripts/guard-github-bootstrap-isolation.sh new file mode 100755 index 0000000000..cef75dd2a8 --- /dev/null +++ b/scripts/guard-github-bootstrap-isolation.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +manifest="${1:-k8s/bases/apps/github-config/flux-kustomization.yaml}" +if [[ ! -r "${manifest}" ]]; then + printf 'UNKNOWN: cannot read GitHub bootstrap configuration\n' >&2 + exit 2 +fi +if ! config="$(yq eval-all -o=json -I=0 '[select(.apiVersion == "kustomize.toolkit.fluxcd.io/v1" and .kind == "Kustomization" and .metadata.name == "github-config")]' "${manifest}")"; then + printf 'UNKNOWN: cannot parse GitHub bootstrap configuration\n' >&2 + exit 2 +fi +if ! jq -e 'length == 1' <<<"${config}" >/dev/null; then + printf 'UNKNOWN: expected exactly one GitHub Kustomization\n' >&2 + exit 2 +fi +if ! jq -e '.[0].spec | .wait == false and ((.healthChecks // []) | length) == 0' <<<"${config}" >/dev/null; then + printf 'FAIL: pending GitHub resources must not gate application delivery; disable wait and omit explicit health checks\n' >&2 + exit 1 +fi +printf 'PASS: GitHub resource readiness is isolated from application delivery\n' diff --git a/scripts/tests/test-github-bootstrap-isolation.sh b/scripts/tests/test-github-bootstrap-isolation.sh new file mode 100755 index 0000000000..d3b3844e69 --- /dev/null +++ b/scripts/tests/test-github-bootstrap-isolation.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +readonly root_dir +work="$(mktemp -d)" +readonly work +trap 'rm -rf "${work}"' EXIT +cd "${root_dir}" + +kubectl kustomize k8s/providers/hetzner/apps >"${work}/apps.yaml" +bash scripts/guard-github-bootstrap-isolation.sh "${work}/apps.yaml" +# A parent that checks application health must retain that check. Only the +# asynchronous GitHub API operations are detached from the delivery gate. +kubectl kustomize k8s/clusters/prod >"${work}/cluster.yaml" +yq -e 'select(.kind == "Kustomization" and .metadata.name == "apps") | .spec.wait == true' "${work}/cluster.yaml" >/dev/null + +yq 'select(.kind == "Kustomization" and .metadata.name == "github-config")' "${work}/apps.yaml" >"${work}/github.yaml" +for mutation in '.spec.wait = true' '.spec.healthChecks = [{"apiVersion": "repo.github.m.upbound.io/v1alpha1", "kind": "Repository", "name": "pending", "namespace": "github-config"}]'; do + yq "${mutation}" "${work}/github.yaml" >"${work}/regression.yaml" + if bash scripts/guard-github-bootstrap-isolation.sh "${work}/regression.yaml" >"${work}/result" 2>&1; then + printf 'FAIL: a bootstrap-pending resource could gate delivery\n' >&2 + exit 1 + fi + grep -q 'pending GitHub resources must not gate' "${work}/result" +done + +for fixture in empty duplicate malformed; do + case "${fixture}" in + empty) printf 'kind: ConfigMap\n' >"${work}/unknown.yaml" ;; + duplicate) cat "${work}/github.yaml" >"${work}/unknown.yaml"; printf '\n---\n' >>"${work}/unknown.yaml"; cat "${work}/github.yaml" >>"${work}/unknown.yaml" ;; + malformed) printf 'kind: [\n' >"${work}/unknown.yaml" ;; + esac + status=0 + bash scripts/guard-github-bootstrap-isolation.sh "${work}/unknown.yaml" >"${work}/result" 2>&1 || status=$? + if [[ "${status}" != 2 ]]; then + printf 'FAIL: incomplete bootstrap census must be UNKNOWN\n' >&2 + exit 1 + fi +done +printf 'PASS: bootstrap regression and incomplete-census controls\n' diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 6f4db26094..417705c1fe 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -116,7 +116,7 @@ kustomize.toolkit.fluxcd.io/v1|Kustomization|flux-system|infrastructure 9bcbbc41 kustomize.toolkit.fluxcd.io/v1|Kustomization|flux-system|infrastructure-controllers e2c18185ed7c0082c075000dbcd7f9eb8354b7566823a6a32554fb1c3e00d937 -kustomize.toolkit.fluxcd.io/v1|Kustomization|github-config|github-config 72a37b4f3edda8d7f432f5d5afc456e8bfef0061eecab0cdac31e829a5ab5d31 +kustomize.toolkit.fluxcd.io/v1|Kustomization|github-config|github-config 5b4f05a9a4b845f91264751628d2c653459fca822c1f23c6d860441c08d20b32 kustomize.toolkit.fluxcd.io/v1|Kustomization|unifi|unifi e3b3caea1b9c5a3fc994d9f68c9609d7f77843f24dae91c567e4dd7b5051868a diff --git a/scripts/validate-eks-ci-role-policy/main.go b/scripts/validate-eks-ci-role-policy/main.go index 71a39b831e..e2f4a5fdbe 100644 --- a/scripts/validate-eks-ci-role-policy/main.go +++ b/scripts/validate-eks-ci-role-policy/main.go @@ -272,7 +272,7 @@ var expectedRenderedHashes = map[resourceIdentity]string{ {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "bootstrap"}: "7f674a1762f298330c7c9e4d9d4e8bf46108b10727e02a25ca5096d7913cc0a7", {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "infrastructure"}: "d1bc403b6458bd22cf967bd570e24718341cbd584f58e7f0069aaffe1e187945", {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "infrastructure-controllers"}: "9d9b62d3221442d6355d16a34d31c198619fb3b3728df960fd67222a531ece7b", - {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "github-config", name: "github-config"}: "8e9f72b0f4f982d050aff0b97d246c68b538cbc397cdd45d031c95cfae981e7c", + {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "github-config", name: "github-config"}: "785aec7ef00ace6439055d5338cb6d15ef9e3fc56d5b3b13d2f4eccff1d10b09", {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "unifi", name: "unifi"}: "5af50c107c3ca59ea39bf2fa334fb99b5372acc720e469d0a07f52ec97242440", {apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "wedding-app", name: "wedding-app"}: "bf8c003eae3ed35d44b2a54aa73065e429b0c7b463af2efabb722fa709bc3365", } diff --git a/tests/flux-bootstrap-health-gating/kyverno-test.yaml b/tests/flux-bootstrap-health-gating/kyverno-test.yaml new file mode 100644 index 0000000000..c7e1a445b1 --- /dev/null +++ b/tests/flux-bootstrap-health-gating/kyverno-test.yaml @@ -0,0 +1,30 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: flux-bootstrap-health-gating +policies: + - ../../k8s/bases/infrastructure/cluster-policies/flux/enforce-flux-best-practices.yaml +resources: + - resources.yaml +results: + - policy: enforce-flux-best-practices + rule: kustomization-recommended-settings + resources: + - github-config/github-config + - tenant/healthy + kind: Kustomization + result: pass + - policy: enforce-flux-best-practices + rule: kustomization-recommended-settings + resources: + - tenant/github-config + - github-config/sibling + - flux-system/apps + kind: Kustomization + result: fail + - policy: enforce-flux-best-practices + rule: kustomization-recommended-settings + resources: + - flux-system/flux-system + kind: Kustomization + result: skip diff --git a/tests/flux-bootstrap-health-gating/missing-reliability/kyverno-test.yaml b/tests/flux-bootstrap-health-gating/missing-reliability/kyverno-test.yaml new file mode 100644 index 0000000000..334b156b91 --- /dev/null +++ b/tests/flux-bootstrap-health-gating/missing-reliability/kyverno-test.yaml @@ -0,0 +1,15 @@ +apiVersion: cli.kyverno.io/v1alpha1 +kind: Test +metadata: + name: flux-bootstrap-health-gating-keeps-reliability +policies: + - ../../../k8s/bases/infrastructure/cluster-policies/flux/enforce-flux-best-practices.yaml +resources: + - resources.yaml +results: + - policy: enforce-flux-best-practices + rule: kustomization-recommended-settings + resources: + - github-config/github-config + kind: Kustomization + result: fail diff --git a/tests/flux-bootstrap-health-gating/missing-reliability/resources.yaml b/tests/flux-bootstrap-health-gating/missing-reliability/resources.yaml new file mode 100644 index 0000000000..f35890debc --- /dev/null +++ b/tests/flux-bootstrap-health-gating/missing-reliability/resources.yaml @@ -0,0 +1,14 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: github-config + namespace: github-config +spec: + interval: 1m + timeout: 5m + retryInterval: 2m + prune: false + wait: false + sourceRef: + kind: OCIRepository + name: platform diff --git a/tests/flux-bootstrap-health-gating/resources.yaml b/tests/flux-bootstrap-health-gating/resources.yaml new file mode 100644 index 0000000000..91644a572d --- /dev/null +++ b/tests/flux-bootstrap-health-gating/resources.yaml @@ -0,0 +1,87 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: github-config + namespace: github-config +spec: + interval: 1m + timeout: 5m + retryInterval: 2m + prune: true + wait: false + sourceRef: + kind: OCIRepository + name: platform +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: healthy + namespace: tenant +spec: + interval: 1m + timeout: 5m + retryInterval: 2m + prune: true + wait: true + sourceRef: + kind: OCIRepository + name: platform +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: github-config + namespace: tenant +spec: + interval: 1m + timeout: 5m + retryInterval: 2m + prune: true + wait: false + sourceRef: + kind: OCIRepository + name: platform +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: sibling + namespace: github-config +spec: + interval: 1m + timeout: 5m + retryInterval: 2m + prune: true + wait: false + sourceRef: + kind: OCIRepository + name: platform +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: apps + namespace: flux-system +spec: + interval: 1m + timeout: 5m + retryInterval: 2m + prune: true + wait: false + sourceRef: + kind: OCIRepository + name: platform +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: flux-system + namespace: flux-system +spec: + interval: 1m + path: ./bootstrap + prune: true + sourceRef: + kind: GitRepository + name: flux-system