Skip to content

Commit 1194b97

Browse files
ecvecv
authored
Merge pull request #280 from datum-cloud/ci/federated-e2e-rebased
ci: run federated edge e2e against real Karmada
2 parents 24a8f0e + 61b26a3 commit 1194b97

2 files changed

Lines changed: 165 additions & 0 deletions

File tree

‎.github/workflows/test-e2e-federated.yml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
name: Federated E2E
2+
3+
# Exercises the production-fidelity edge: two kind clusters (control plane + edge)
4+
# wired with real Karmada federation, the production Envoy Gateway version, the
5+
# Coraza WAF data plane, and the NSO extension server — then runs the edge e2e
6+
# scenarios against real traffic. This is the path the non-federated test/e2e
7+
# suite (test-e2e.yml) never covers.
8+
on:
9+
pull_request:
10+
branches:
11+
- main
12+
paths:
13+
- 'Taskfile.test-infra.yml'
14+
- 'config/e2e-downstream/**'
15+
- 'config/federation/**'
16+
- 'config/tools/**'
17+
- 'config/extension-server/**'
18+
- 'internal/extensionserver/**'
19+
- 'cmd/**'
20+
- 'test/e2e-edge/**'
21+
- 'test/parity/**'
22+
- '.github/workflows/test-e2e-federated.yml'
23+
push:
24+
branches:
25+
- main
26+
workflow_dispatch:
27+
28+
# One federated run per ref; a newer push cancels an in-flight one.
29+
concurrency:
30+
group: federated-e2e-${{ github.ref }}
31+
cancel-in-progress: true
32+
33+
jobs:
34+
federated-e2e:
35+
name: Two-cluster federated edge (Karmada)
36+
runs-on: ubuntu-latest
37+
timeout-minutes: 60
38+
env:
39+
TMPDIR: /tmp
40+
steps:
41+
- name: Clone the code
42+
uses: actions/checkout@v4
43+
44+
- name: Setup Go
45+
uses: actions/setup-go@v5
46+
with:
47+
go-version: '~1.26'
48+
49+
# Two kind clusters plus a full Karmada control plane and the edge data
50+
# plane need more room than the runner ships with. Reclaim disk from
51+
# toolchains this job never uses.
52+
- name: Free up disk space
53+
run: |
54+
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL
55+
docker system prune -af || true
56+
df -h /
57+
58+
# Karmada plus two clusters open a large number of file watches; the
59+
# default runner limits crashloop a component with "too many open files".
60+
- name: Raise inotify limits
61+
run: |
62+
sudo sysctl -w fs.inotify.max_user_instances=8192
63+
sudo sysctl -w fs.inotify.max_user_watches=1048576
64+
65+
- name: Install Task
66+
uses: arduino/setup-task@v2
67+
with:
68+
version: 3.x
69+
repo-token: ${{ secrets.GITHUB_TOKEN }}
70+
71+
- name: Install Helm
72+
uses: azure/setup-helm@v4
73+
with:
74+
# kustomize v5.5.0 probes helm with `helm version -c`; the -c shorthand
75+
# was removed in helm 3.13, so pin to the last release that accepts it.
76+
version: v3.12.3
77+
78+
- name: Install pinned tools (kind, karmadactl, kustomize, cmctl, chainsaw)
79+
run: task test-infra:tools
80+
81+
- name: Bring up the two-cluster prod-fidelity env
82+
run: task test-infra:up
83+
84+
- name: Stand up Karmada federation
85+
run: task test-infra:karmada-up
86+
87+
- name: Run edge e2e scenarios
88+
run: task test-infra:e2e
89+
90+
# When something fails, capture enough cross-cluster state to diagnose it
91+
# from the logs alone — the runner is torn down immediately after.
92+
- name: Diagnostics on failure
93+
if: failure()
94+
run: |
95+
set +e
96+
echo "::group::upstream pods"
97+
kubectl --context kind-nso-upstream get pods -A -o wide
98+
echo "::endgroup::"
99+
echo "::group::downstream pods"
100+
kubectl --context kind-nso-downstream get pods -A -o wide
101+
echo "::endgroup::"
102+
echo "::group::not-ready pods — describe (both clusters)"
103+
for ctx in kind-nso-upstream kind-nso-downstream; do
104+
echo "### $ctx"
105+
kubectl --context "$ctx" get pods -A \
106+
--field-selector=status.phase!=Running,status.phase!=Succeeded \
107+
-o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' \
108+
| while read -r ns name; do
109+
[ -n "$name" ] || continue
110+
kubectl --context "$ctx" -n "$ns" describe pod "$name" 2>/dev/null | tail -30
111+
done
112+
done
113+
echo "::endgroup::"
114+
echo "::group::cert-manager health (upstream) — restarts + issuance"
115+
kubectl --context kind-nso-upstream -n cert-manager get pods -o wide
116+
kubectl --context kind-nso-upstream -n cert-manager \
117+
logs -l app.kubernetes.io/name=cert-manager --tail=80 --previous 2>/dev/null || true
118+
kubectl --context kind-nso-upstream get certificate,certificaterequest,issuer,clusterissuer -A
119+
echo "::endgroup::"
120+
echo "::group::karmada member clusters"
121+
KCFG=/tmp/karmada-nso-upstream/karmada-apiserver.config
122+
kubectl --kubeconfig "$KCFG" --context karmada-apiserver get clusters -o wide
123+
kubectl --kubeconfig "$KCFG" --context karmada-apiserver get clusterpropagationpolicy,work -A
124+
echo "::endgroup::"
125+
echo "::group::NSO manager logs (upstream)"
126+
kubectl --context kind-nso-upstream -n network-services-operator-system \
127+
logs deploy/network-services-operator-controller-manager --tail=200
128+
echo "::endgroup::"
129+
echo "::group::extension server logs (downstream)"
130+
kubectl --context kind-nso-downstream -n network-services-operator-system \
131+
logs deploy/network-services-operator-envoy-gateway-extension-server --tail=200
132+
echo "::endgroup::"
133+
echo "::group::downstream Envoy Gateway logs"
134+
kubectl --context kind-nso-downstream -n datum-downstream-gateway \
135+
logs deploy/envoy-gateway --tail=200
136+
echo "::endgroup::"

‎Taskfile.test-infra.yml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,35 @@ vars:
5656

5757
tasks:
5858

59+
tools:
60+
desc: "Install the pinned tool binaries this env needs into bin/ (kind v0.32.0, karmadactl, kustomize, cmctl, chainsaw). Idempotent — skips any already present. Lets CI and a clean checkout bring up the env without hand-placed binaries."
61+
vars:
62+
KIND_TOOL_VERSION: v0.32.0
63+
KARMADACTL_VERSION: v1.15.2
64+
KUSTOMIZE_VERSION: v5.5.0
65+
CMCTL_VERSION: v2.1.1
66+
CHAINSAW_VERSION: v0.2.15
67+
cmds:
68+
- mkdir -p {{.REPO}}/bin
69+
- |
70+
if [ ! -x {{.REPO}}/bin/kind-{{.KIND_TOOL_VERSION}} ]; then
71+
echo "installing kind {{.KIND_TOOL_VERSION}}"
72+
GOBIN={{.REPO}}/bin go install sigs.k8s.io/kind@{{.KIND_TOOL_VERSION}}
73+
mv {{.REPO}}/bin/kind {{.REPO}}/bin/kind-{{.KIND_TOOL_VERSION}}
74+
fi
75+
- '[ -x {{.REPO}}/bin/kustomize ] || GOBIN={{.REPO}}/bin go install sigs.k8s.io/kustomize/kustomize/v5@{{.KUSTOMIZE_VERSION}}'
76+
- '[ -x {{.REPO}}/bin/cmctl ] || GOBIN={{.REPO}}/bin go install github.com/cert-manager/cmctl/v2@{{.CMCTL_VERSION}}'
77+
- '[ -x {{.REPO}}/bin/chainsaw ] || GOBIN={{.REPO}}/bin go install github.com/kyverno/chainsaw@{{.CHAINSAW_VERSION}}'
78+
# karmadactl carries replace directives in its go.mod, so `go install` can't
79+
# build it; pull the published release binary for this OS/arch instead.
80+
- |
81+
if [ ! -x {{.REPO}}/bin/karmadactl ]; then
82+
url="https://github.com/karmada-io/karmada/releases/download/{{.KARMADACTL_VERSION}}/karmadactl-{{OS}}-{{ARCH}}.tgz"
83+
echo "downloading karmadactl {{.KARMADACTL_VERSION}} ({{OS}}/{{ARCH}})"
84+
curl -fsSL "$url" | tar -xz -C {{.REPO}}/bin karmadactl
85+
fi
86+
- echo "tools ready in {{.REPO}}/bin"
87+
5988
up:
6089
desc: "Bring the canonical two-cluster prod-fidelity test env ONLINE (clusters + EG v1.7.4 + ext-server + Coraza data plane + NSO manager)."
6190
cmds:

0 commit comments

Comments
 (0)