fix(github): isolate bootstrap readiness from application delivery

.github/actions/deploy-prod/action.yml

@@ -188,6 +188,14 @@ runs:
       shell: bash
       run: bash scripts/guard-coroot-baseline-context.sh before-publish
 
+    - name: 🧭 Validate GitHub bootstrap isolation
+      shell: bash
+      run: |
+        bootstrap_render="$(mktemp)"
+        trap 'rm -f "$bootstrap_render"' EXIT
+        kubectl kustomize k8s/providers/hetzner/apps > "$bootstrap_render"
+        bash scripts/guard-github-bootstrap-isolation.sh "$bootstrap_render"
+
     - name: 💾 Validate live persistence retirement safety
       # A commit parent is not evidence that its protection reached production:
       # two direct pushes can be waiting behind one manual dispatch. Compare the

.github/workflows/ci.yaml

@@ -35,6 +35,12 @@ jobs:
           go-version-file: go.mod
           cache: false
 
+      - name: 🧭 Validate GitHub bootstrap isolation
+        run: |
+          shellcheck scripts/guard-github-bootstrap-isolation.sh scripts/tests/test-github-bootstrap-isolation.sh
+          bash scripts/tests/test-github-bootstrap-isolation.sh
+          bash scripts/tests/test-crossplane-sync-alerter.sh
+
       - name: 🛡️ Validate preserved policy failure actions
         run: |
           go test ./scripts/validate-policy-actions

docs/github-management.md

@@ -44,6 +44,15 @@ time the app reconciles, the provider (infrastructure tier) has installed the
 namespaced github CRDs the `ProviderConfig` and managed resources need. On a
 fresh install the app's `Kustomization` retries benignly until those CRDs exist.
 
+GitHub API reconciliation is asynchronous: the tenant Kustomization uses
+`wait: false` and has no explicit managed-resource health checks. A repository
+awaiting manual bootstrap cannot freeze unrelated application delivery. Artifact
+verification, admission and apply failures still fail the tenant reconciliation;
+the parent continues checking application health. Crossplane's separate sync
+exporter and `crossplane-sync-alert` report unsynced repositories and failed
+alert delivery rather than hiding them behind Flux readiness. The bootstrap
+isolation guard checks the rendered tenant before publication.
+
 The GitHub App credentials are **not** SOPS-managed. The vault-config bootstrap
 Job seeds **placeholder** values into OpenBao (`secret/infrastructure/github/app`)
 **only when the secret is absent**, and the maintainer overwrites them in place
@@ -61,8 +70,8 @@ Velero), so the manually-set values are durable without a GitOps source of truth
    required for the `maintainers`-team resources**: `Team` observe needs
    `read`, and `TeamMembership`/`TeamRepository` create needs `write` —
    without it they stay stuck on `CannotObserveExternalResource` /
-   `403 must be an organization owner or team maintainer` and flap the
-   `github-config` Flux health check. Install it on **all repositories**
+   `403 must be an organization owner or team maintainer` and raise the
+   Crossplane sync alert. Install it on **all repositories**
    of the org. No webhook.
 2. **Overwrite the placeholders in OpenBao** with the App's real values — the
    keys already exist (seeded by the vault-config Job), so just set them. The

k8s/bases/apps/github-config/flux-kustomization.yaml

@@ -14,7 +14,10 @@ spec:
   retryInterval: 2m
   path: .
   prune: true
-  wait: true
+  # Applying the signed artifact must succeed, but asynchronous GitHub API
+  # operations (including manual repository bootstrap) cannot gate app delivery.
+  # Crossplane's Synced condition is monitored by crossplane-sync-alert instead.
+  wait: false
   force: true
   # Runs as the app SA: the GitHub managed resources are applied with only the
   # namespace-scoped authority in rbac.yaml. This Kustomization is itself applied

k8s/bases/infrastructure/cluster-policies/flux/enforce-flux-best-practices.yaml

@@ -49,19 +49,34 @@ spec:
         message: >-
           Flux Kustomization must set the recommended reliability fields:
           `spec.interval`, `spec.timeout`, `spec.retryInterval`,
-          `spec.prune: true` (garbage collection), `spec.wait: true`
-          (health-gated reconciliation), and `spec.sourceRef`.
+          `spec.prune: true` (garbage collection), and `spec.sourceRef`.
+          `spec.wait: true` is required except for the platform-owned GitHub
+          configuration, whose asynchronous API operations do not gate delivery.
           See https://fluxcd.io/flux/components/kustomize/kustomizations/#recommended-settings.
-        pattern:
-          spec:
-            interval: "?*"
-            timeout: "?*"
-            retryInterval: "?*"
-            prune: true
-            wait: true
-            sourceRef:
-              kind: "?*"
-              name: "?*"
+        anyPattern:
+          - spec:
+              interval: "?*"
+              timeout: "?*"
+              retryInterval: "?*"
+              prune: true
+              wait: true
+              sourceRef:
+                kind: "?*"
+                name: "?*"
+          # Bound the alternative to this exact platform-owned identity. Keep
+          # every other reliability requirement; this is not a policy exclusion.
+          - metadata:
+              name: github-config
+              namespace: github-config
+            spec:
+              interval: "?*"
+              timeout: "?*"
+              retryInterval: "?*"
+              prune: true
+              wait: false
+              sourceRef:
+                kind: "?*"
+                name: "?*"
     - name: helmrelease-recommended-settings
       match:
         any:

scripts/guard-github-bootstrap-isolation.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+manifest="${1:-k8s/bases/apps/github-config/flux-kustomization.yaml}"
+if [[ ! -r "${manifest}" ]]; then
+  printf 'UNKNOWN: cannot read GitHub bootstrap configuration\n' >&2
+  exit 2
+fi
+if ! config="$(yq eval-all -o=json -I=0 '[select(.apiVersion == "kustomize.toolkit.fluxcd.io/v1" and .kind == "Kustomization" and .metadata.name == "github-config")]' "${manifest}")"; then
+  printf 'UNKNOWN: cannot parse GitHub bootstrap configuration\n' >&2
+  exit 2
+fi
+if ! jq -e 'length == 1' <<<"${config}" >/dev/null; then
+  printf 'UNKNOWN: expected exactly one GitHub Kustomization\n' >&2
+  exit 2
+fi
+if ! jq -e '.[0].spec | .wait == false and ((.healthChecks // []) | length) == 0' <<<"${config}" >/dev/null; then
+  printf 'FAIL: pending GitHub resources must not gate application delivery; disable wait and omit explicit health checks\n' >&2
+  exit 1
+fi
+printf 'PASS: GitHub resource readiness is isolated from application delivery\n'

scripts/tests/test-github-bootstrap-isolation.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+readonly root_dir
+work="$(mktemp -d)"
+readonly work
+trap 'rm -rf "${work}"' EXIT
+cd "${root_dir}"
+
+kubectl kustomize k8s/providers/hetzner/apps >"${work}/apps.yaml"
+bash scripts/guard-github-bootstrap-isolation.sh "${work}/apps.yaml"
+# A parent that checks application health must retain that check. Only the
+# asynchronous GitHub API operations are detached from the delivery gate.
+kubectl kustomize k8s/clusters/prod >"${work}/cluster.yaml"
+yq -e 'select(.kind == "Kustomization" and .metadata.name == "apps") | .spec.wait == true' "${work}/cluster.yaml" >/dev/null
+
+yq 'select(.kind == "Kustomization" and .metadata.name == "github-config")' "${work}/apps.yaml" >"${work}/github.yaml"
+for mutation in '.spec.wait = true' '.spec.healthChecks = [{"apiVersion": "repo.github.m.upbound.io/v1alpha1", "kind": "Repository", "name": "pending", "namespace": "github-config"}]'; do
+  yq "${mutation}" "${work}/github.yaml" >"${work}/regression.yaml"
+  if bash scripts/guard-github-bootstrap-isolation.sh "${work}/regression.yaml" >"${work}/result" 2>&1; then
+    printf 'FAIL: a bootstrap-pending resource could gate delivery\n' >&2
+    exit 1
+  fi
+  grep -q 'pending GitHub resources must not gate' "${work}/result"
+done
+
+for fixture in empty duplicate malformed; do
+  case "${fixture}" in
+    empty) printf 'kind: ConfigMap\n' >"${work}/unknown.yaml" ;;
+    duplicate) cat "${work}/github.yaml" >"${work}/unknown.yaml"; printf '\n---\n' >>"${work}/unknown.yaml"; cat "${work}/github.yaml" >>"${work}/unknown.yaml" ;;
+    malformed) printf 'kind: [\n' >"${work}/unknown.yaml" ;;
+  esac
+  status=0
+  bash scripts/guard-github-bootstrap-isolation.sh "${work}/unknown.yaml" >"${work}/result" 2>&1 || status=$?
+  if [[ "${status}" != 2 ]]; then
+    printf 'FAIL: incomplete bootstrap census must be UNKNOWN\n' >&2
+    exit 1
+  fi
+done
+printf 'PASS: bootstrap regression and incomplete-census controls\n'

scripts/validate-eks-ci-role-policy/approved-surface.txt

@@ -116,7 +116,7 @@ kustomize.toolkit.fluxcd.io/v1|Kustomization|flux-system|infrastructure 9bcbbc41
 
 kustomize.toolkit.fluxcd.io/v1|Kustomization|flux-system|infrastructure-controllers e2c18185ed7c0082c075000dbcd7f9eb8354b7566823a6a32554fb1c3e00d937
 
-kustomize.toolkit.fluxcd.io/v1|Kustomization|github-config|github-config 72a37b4f3edda8d7f432f5d5afc456e8bfef0061eecab0cdac31e829a5ab5d31
+kustomize.toolkit.fluxcd.io/v1|Kustomization|github-config|github-config 5b4f05a9a4b845f91264751628d2c653459fca822c1f23c6d860441c08d20b32
 
 kustomize.toolkit.fluxcd.io/v1|Kustomization|unifi|unifi e3b3caea1b9c5a3fc994d9f68c9609d7f77843f24dae91c567e4dd7b5051868a
 

scripts/validate-eks-ci-role-policy/main.go

@@ -272,7 +272,7 @@ var expectedRenderedHashes = map[resourceIdentity]string{
 	{apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "bootstrap"}:                  "7f674a1762f298330c7c9e4d9d4e8bf46108b10727e02a25ca5096d7913cc0a7",
 	{apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "infrastructure"}:             "d1bc403b6458bd22cf967bd570e24718341cbd584f58e7f0069aaffe1e187945",
 	{apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "flux-system", name: "infrastructure-controllers"}: "9d9b62d3221442d6355d16a34d31c198619fb3b3728df960fd67222a531ece7b",
-	{apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "github-config", name: "github-config"}:            "8e9f72b0f4f982d050aff0b97d246c68b538cbc397cdd45d031c95cfae981e7c",
+	{apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "github-config", name: "github-config"}:            "785aec7ef00ace6439055d5338cb6d15ef9e3fc56d5b3b13d2f4eccff1d10b09",
 	{apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "unifi", name: "unifi"}:                            "5af50c107c3ca59ea39bf2fa334fb99b5372acc720e469d0a07f52ec97242440",
 	{apiVersion: "kustomize.toolkit.fluxcd.io/v1", kind: "Kustomization", namespace: "wedding-app", name: "wedding-app"}:                "bf8c003eae3ed35d44b2a54aa73065e429b0c7b463af2efabb722fa709bc3365",
 }

tests/flux-bootstrap-health-gating/kyverno-test.yaml

@@ -0,0 +1,30 @@
+apiVersion: cli.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: flux-bootstrap-health-gating
+policies:
+  - ../../k8s/bases/infrastructure/cluster-policies/flux/enforce-flux-best-practices.yaml
+resources:
+  - resources.yaml
+results:
+  - policy: enforce-flux-best-practices
+    rule: kustomization-recommended-settings
+    resources:
+      - github-config/github-config
+      - tenant/healthy
+    kind: Kustomization
+    result: pass
+  - policy: enforce-flux-best-practices
+    rule: kustomization-recommended-settings
+    resources:
+      - tenant/github-config
+      - github-config/sibling
+      - flux-system/apps
+    kind: Kustomization
+    result: fail
+  - policy: enforce-flux-best-practices
+    rule: kustomization-recommended-settings
+    resources:
+      - flux-system/flux-system
+    kind: Kustomization
+    result: skip

tests/flux-bootstrap-health-gating/missing-reliability/kyverno-test.yaml

@@ -0,0 +1,15 @@
+apiVersion: cli.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: flux-bootstrap-health-gating-keeps-reliability
+policies:
+  - ../../../k8s/bases/infrastructure/cluster-policies/flux/enforce-flux-best-practices.yaml
+resources:
+  - resources.yaml
+results:
+  - policy: enforce-flux-best-practices
+    rule: kustomization-recommended-settings
+    resources:
+      - github-config/github-config
+    kind: Kustomization
+    result: fail

tests/flux-bootstrap-health-gating/missing-reliability/resources.yaml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: github-config
+  namespace: github-config
+spec:
+  interval: 1m
+  timeout: 5m
+  retryInterval: 2m
+  prune: false
+  wait: false
+  sourceRef:
+    kind: OCIRepository
+    name: platform

tests/flux-bootstrap-health-gating/resources.yaml

@@ -0,0 +1,87 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: github-config
+  namespace: github-config
+spec:
+  interval: 1m
+  timeout: 5m
+  retryInterval: 2m
+  prune: true
+  wait: false
+  sourceRef:
+    kind: OCIRepository
+    name: platform
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: healthy
+  namespace: tenant
+spec:
+  interval: 1m
+  timeout: 5m
+  retryInterval: 2m
+  prune: true
+  wait: true
+  sourceRef:
+    kind: OCIRepository
+    name: platform
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: github-config
+  namespace: tenant
+spec:
+  interval: 1m
+  timeout: 5m
+  retryInterval: 2m
+  prune: true
+  wait: false
+  sourceRef:
+    kind: OCIRepository
+    name: platform
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: sibling
+  namespace: github-config
+spec:
+  interval: 1m
+  timeout: 5m
+  retryInterval: 2m
+  prune: true
+  wait: false
+  sourceRef:
+    kind: OCIRepository
+    name: platform
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: apps
+  namespace: flux-system
+spec:
+  interval: 1m
+  timeout: 5m
+  retryInterval: 2m
+  prune: true
+  wait: false
+  sourceRef:
+    kind: OCIRepository
+    name: platform
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 1m
+  path: ./bootstrap
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system