fix: avoid SSA caBundle conflict installing opentelemetry-operator over pre-existing CRDs

.github/workflows/integration.yml

@@ -35,19 +35,23 @@ jobs:
         run: bats tests/unit.bats
 
   # Layer 2 — real kind cluster, one mode per matrix leg (parallel, isolated).
+  # CRD-conflict legs run on Helm v4 so the server-side-apply default codepath
+  # (the version where the cert-manager-cainjector caBundle conflict surfaces)
+  # is actually exercised; the rest stay on v3 for broad version coverage.
   layer2-kind:
     runs-on: ubuntu-latest
     needs: layer1-orchestration
     strategy:
       fail-fast: false
       matrix:
-        mode:
-          - operator-only
-          - logs-only
-          - monitoring-only
-          - events-only
-          - crd-conflict
-          - context
+        include:
+          - { mode: operator-only,          helm: v3.14.0 }
+          - { mode: logs-only,              helm: v3.14.0 }
+          - { mode: monitoring-only,        helm: v3.14.0 }
+          - { mode: events-only,            helm: v3.14.0 }
+          - { mode: crd-conflict,           helm: v4.1.3 }
+          - { mode: operator-crd-conflict,  helm: v4.1.3 }
+          - { mode: context,                helm: v3.14.0 }
     steps:
       - uses: actions/checkout@v4
         with:
@@ -61,9 +65,9 @@ jobs:
       - name: Install helm
         uses: azure/setup-helm@v4
         with:
-          version: v3.14.0
+          version: ${{ matrix.helm }}
 
-      - name: Run kind e2e (${{ matrix.mode }})
+      - name: Run kind e2e (${{ matrix.mode }}, helm ${{ matrix.helm }})
         run: bash tests/integration/run.sh layer2 ${{ matrix.mode }}
 
       - name: Dump diagnostics on failure

last9-otel-setup.sh

@@ -1738,6 +1738,59 @@ install_operator() {
     # Adopt any pre-existing OTel CRDs before Helm tries to manage them
     adopt_otel_crds
 
+    # When OTel CRDs already exist (e.g. a prior cert-manager-based install),
+    # cert-manager-cainjector owns .spec.conversion.webhook.clientConfig.caBundle
+    # via server-side apply and re-injects it continuously. Helm v4's SSA does
+    # NOT pass --force-conflicts, so its CRD apply fails fatally:
+    #   conflict with "cert-manager-cainjector" ... .caBundle
+    # The adoption migration above cannot win that race against an active
+    # controller.
+    #
+    # Unlike kube-prometheus-stack (CRDs in the chart's crds/ dir, so
+    # `helm show crds` + --skip-crds work), the opentelemetry-operator chart
+    # ships its CRDs as TEMPLATES gated by crds.create — `helm show crds` is
+    # empty and --skip-crds has NO effect. So instead: render the CRDs from the
+    # chart template and force-apply them out-of-band (upgrades schema to the
+    # chart version and forcibly takes field ownership from cainjector), then
+    # disable Helm's own CRD rendering with crds.create=false so Helm never
+    # applies — and never re-conflicts on — the CRDs.
+    # Probe for ANY pre-existing *.opentelemetry.io CRD — a partial set (e.g.
+    # instrumentations present but collectors absent) hits the same cainjector
+    # conflict, so gate on any match rather than one well-known name. The
+    # out-of-band force-apply below is idempotent and safe to run whenever any
+    # OTel CRD exists. This mirrors what adopt_otel_crds already iterates.
+    local disable_crds=""
+    if kubectl get crd -o name 2>/dev/null | grep -q '\.opentelemetry\.io$'; then
+        log_info "Pre-existing OpenTelemetry CRDs detected — force-applying chart CRDs and installing with crds.create=false"
+
+        # NOTE: this script runs under `set -e` but NOT `set -o pipefail`, so a
+        # piped `helm template | awk | kubectl apply` would mask an upstream
+        # failure (a broken render or a failed apply) and still proceed to
+        # crds.create=false — leaving stale/missing CRDs with no error. Capture
+        # each stage and check it explicitly instead.
+        local rendered_crds
+        if ! rendered_crds=$(helm template opentelemetry-operator open-telemetry/opentelemetry-operator \
+            --version "$OPERATOR_VERSION" --include-crds --set crds.create=true \
+            $HELM_SCHEMA_FLAG 2>&1); then
+            log_error "Failed to render opentelemetry-operator CRDs via 'helm template'. Output: $rendered_crds"
+        fi
+
+        # Isolate only the CustomResourceDefinition documents from the chart.
+        local crd_manifests
+        crd_manifests=$(printf '%s\n' "$rendered_crds" \
+            | awk 'BEGIN{RS="\n---\n"} /(^|\n)kind: CustomResourceDefinition/ {print "---"; print $0}')
+
+        if ! printf '%s' "$crd_manifests" | grep -q 'kind: CustomResourceDefinition'; then
+            log_error "Rendered opentelemetry-operator chart contained no CustomResourceDefinition documents; refusing to install with crds.create=false."
+        fi
+
+        if ! printf '%s\n' "$crd_manifests" | kubectl apply --server-side --force-conflicts \
+            --field-manager=helm -f -; then
+            log_error "Failed to force-apply opentelemetry-operator CRDs (kubectl apply --server-side --force-conflicts)."
+        fi
+        disable_crds="yes"
+    fi
+
     # Build helm command as array for proper argument handling
     local helm_args=(
         "upgrade" "--install"
@@ -1751,6 +1804,11 @@ install_operator() {
         "--set" "admissionWebhooks.autoGenerateCert.enabled=true"
     )
 
+    # Disable Helm's CRD rendering when CRDs were pre-existing (handled above)
+    if [ -n "$disable_crds" ]; then
+        helm_args+=("--set" "crds.create=false")
+    fi
+
     # Add tolerations and nodeSelector if provided
     if [ -n "$TOLERATIONS_FILE_PATH" ]; then
         log_info "Adding tolerations to operator installation..."

tests/integration/kind-e2e.sh

@@ -16,6 +16,7 @@
 #   monitoring-only  kube-prometheus-stack (metrics)
 #   events-only      Kubernetes events agent
 #   crd-conflict     Pre-seed Terraform-owned Prometheus CRDs, then monitoring-only
+#   operator-crd-conflict  Pre-seed foreign-owned OTel CRDs, then operator-only
 #   context          monitoring-only pinned to an explicit kubectl context
 #   all              Run every mode above, each in its own cluster
 #
@@ -188,6 +189,50 @@ test_crd_conflict() {
     info "✓ crd-conflict passed"
 }
 
+# Reproduce the customer failure on the OTel operator path: OTel CRDs already on
+# the cluster, owned by a non-Helm field manager (cert-manager-cainjector, which
+# in a live cluster owns .spec.conversion.webhook.clientConfig.caBundle). The
+# opentelemetry-operator chart ships CRDs as templates gated by crds.create (NOT
+# the chart's crds/ dir), so the script must render them with
+# `helm template --include-crds`, force-apply them out-of-band, and install with
+# crds.create=false so Helm never re-applies — and never conflicts on — them.
+#
+# Note: this is a PATH-ASSERTION test, not a live-race test. With no running
+# cainjector re-asserting ownership, adopt_otel_crds would itself resolve a
+# static conflict — so the guard here is that the crds.create=false mitigation
+# path is taken (catches removal of the fix), not that an active-controller race
+# is won. The live race only manifests with real cert-manager + Helm v4; that is
+# verified by a manual repro (cert-manager + seed operator -> cainjector owns
+# caBundle -> the unpatched script reproduces the exact customer conflict and the
+# patched script installs cleanly).
+test_operator_crd_conflict() {
+    create_cluster "${CLUSTER_PREFIX}-operator-crdconflict"
+
+    info "Pre-seeding OTel CRDs as field-manager cert-manager-cainjector"
+    # Pin to the operator tag matching the chart the script installs, so the
+    # seeded CRD schema is the one the mitigation upgrades from — not a moving
+    # `main` that can drift. Chart OPERATOR_VERSION=0.92.1 (last9-otel-setup.sh)
+    # has appVersion 0.129.1, i.e. operator tag v0.129.1. Bump both together.
+    local crd_base="https://raw.githubusercontent.com/open-telemetry/opentelemetry-operator/v0.129.1/config/crd/bases"
+    for crd in \
+        opentelemetry.io_opentelemetrycollectors \
+        opentelemetry.io_instrumentations \
+        opentelemetry.io_opampbridges; do
+        kubectl apply --server-side --field-manager=cert-manager-cainjector \
+            -f "${crd_base}/${crd}.yaml" \
+            || warn "could not pre-seed $crd (continuing)"
+    done
+
+    local out
+    out=$(run_setup operator-only token="$DUMMY_TOKEN" endpoint="$DUMMY_OTLP" 2>&1) \
+        || { echo "$out"; fail "operator-only failed with pre-existing CRDs"; }
+
+    echo "$out" | grep -q -- "crds.create=false\|Pre-existing OpenTelemetry CRDs detected" \
+        || fail "script did not take the crds.create=false path with pre-existing OTel CRDs"
+    assert_rollout deployment opentelemetry-operator
+    info "✓ operator-crd-conflict passed"
+}
+
 test_context() {
     create_cluster "${CLUSTER_PREFIX}-context"
     # Use the existing context when provided, else the kind-<name> context
@@ -226,17 +271,19 @@ main() {
         monitoring-only) test_monitoring_only ;;
         events-only)     test_events_only ;;
         crd-conflict)    test_crd_conflict ;;
+        operator-crd-conflict) test_operator_crd_conflict ;;
         context)         test_context ;;
         all)
-            test_operator_only;   cleanup_cluster
-            test_logs_only;       cleanup_cluster
-            test_monitoring_only; cleanup_cluster
-            test_events_only;     cleanup_cluster
-            test_crd_conflict;    cleanup_cluster
+            test_operator_only;          cleanup_cluster
+            test_logs_only;              cleanup_cluster
+            test_monitoring_only;        cleanup_cluster
+            test_events_only;            cleanup_cluster
+            test_crd_conflict;           cleanup_cluster
+            test_operator_crd_conflict;  cleanup_cluster
             test_context
             ;;
         *)
-            echo "Usage: $0 <operator-only|logs-only|monitoring-only|events-only|crd-conflict|context|all>" >&2
+            echo "Usage: $0 <operator-only|logs-only|monitoring-only|events-only|crd-conflict|operator-crd-conflict|context|all>" >&2
             exit 1
             ;;
     esac