Skip to content

Commit 7639a85

Browse files
committed
ci: harden H100 GPU qualification
1 parent 3c9f6ec commit 7639a85

11 files changed

Lines changed: 391 additions & 29 deletions

File tree

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
name: 'Check Control Plane Health'
16+
description: 'Fails if Kind control-plane static pods are missing, unready, or restarted.'
17+
18+
inputs:
19+
cluster_name:
20+
description: 'Kind cluster name'
21+
required: true
22+
namespace:
23+
description: 'Namespace that contains the control-plane pods'
24+
required: false
25+
default: kube-system
26+
components:
27+
description: 'Space-separated component label values to check'
28+
required: false
29+
default: kube-apiserver kube-controller-manager kube-scheduler etcd
30+
wait_timeout:
31+
description: 'Timeout for each component readiness wait'
32+
required: false
33+
default: 60s
34+
max_restarts:
35+
description: 'Maximum tolerated restart count for each control-plane container'
36+
required: false
37+
default: '1'
38+
39+
runs:
40+
using: 'composite'
41+
steps:
42+
- name: Check control-plane pods
43+
shell: bash
44+
env:
45+
KIND_CLUSTER_NAME: ${{ inputs.cluster_name }}
46+
NAMESPACE: ${{ inputs.namespace }}
47+
COMPONENTS: ${{ inputs.components }}
48+
WAIT_TIMEOUT: ${{ inputs.wait_timeout }}
49+
MAX_RESTARTS: ${{ inputs.max_restarts }}
50+
run: |
51+
set -euo pipefail
52+
53+
kubectl --context="kind-${KIND_CLUSTER_NAME}" get --raw='/readyz' || true
54+
55+
check_component() {
56+
local component="$1"
57+
local selector="component=${component}"
58+
local pods
59+
60+
pods=$(kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
61+
get pod -l "${selector}" -o name)
62+
if [[ -z "${pods}" ]]; then
63+
echo "::error::no ${component} pods found in ${NAMESPACE} with selector ${selector}"
64+
kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
65+
get pods -o wide || true
66+
exit 1
67+
fi
68+
69+
if ! kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
70+
wait --for=condition=Ready pod -l "${selector}" --timeout="${WAIT_TIMEOUT}"; then
71+
echo "::error::${component} pods did not become Ready within ${WAIT_TIMEOUT}"
72+
kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
73+
get pod -l "${selector}" -o wide || true
74+
kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
75+
describe pod -l "${selector}" || true
76+
kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
77+
get events --sort-by='.lastTimestamp' 2>/dev/null | tail -30 || true
78+
kubectl --context="kind-${KIND_CLUSTER_NAME}" get --raw='/readyz' || true
79+
exit 1
80+
fi
81+
82+
local restart_counts
83+
restart_counts=$(kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
84+
get pod -l "${selector}" \
85+
-o jsonpath='{range .items[*]}{range .status.containerStatuses[*]}{.restartCount}{"\n"}{end}{end}')
86+
if [[ -z "${restart_counts}" ]]; then
87+
echo "::error::no container statuses found for ${component} pods"
88+
kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
89+
describe pod -l "${selector}" || true
90+
exit 1
91+
fi
92+
93+
while IFS= read -r restart_count; do
94+
[[ -z "${restart_count}" ]] && continue
95+
if (( restart_count > MAX_RESTARTS )); then
96+
echo "::error::${component} restartCount=${restart_count}"
97+
kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
98+
get pod -l "${selector}" -o wide || true
99+
kubectl --context="kind-${KIND_CLUSTER_NAME}" -n "${NAMESPACE}" \
100+
describe pod -l "${selector}" || true
101+
exit 1
102+
fi
103+
done <<< "${restart_counts}"
104+
}
105+
106+
for component in ${COMPONENTS}; do
107+
check_component "${component}"
108+
done
109+
kubectl --context="kind-${KIND_CLUSTER_NAME}" get --raw='/readyz'

.github/actions/gpu-operator-install/action.yml

Lines changed: 25 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@ inputs:
3131
description: 'Platform for recipe generation (bundle mode only, e.g. dynamo)'
3232
required: false
3333
default: ''
34+
wait:
35+
description: 'Wait for bundle Helm resources during deploy'
36+
required: false
37+
default: 'false'
38+
best_effort:
39+
description: 'Continue deploying remaining bundle components after a component failure'
40+
required: false
41+
default: 'true'
3442

3543
runs:
3644
using: 'composite'
@@ -82,8 +90,7 @@ runs:
8290
--intent ${{ inputs.intent }} \
8391
${PLATFORM_FLAG} \
8492
--output recipe.yaml
85-
echo "--- Recipe ---"
86-
cat recipe.yaml
93+
echo "Recipe written to recipe.yaml"
8794
8895
- name: Generate deployment bundle
8996
if: inputs.method == 'bundle'
@@ -101,19 +108,23 @@ runs:
101108
shell: bash
102109
run: |
103110
cd bundle
104-
# Use --no-wait: several components (gpu-operator ClusterPolicy,
105-
# kai-scheduler SchedulingShard, nvidia-dra-driver-gpu kubelet plugin)
106-
# stay InProgress in kind because their CRs/DaemonSets require
107-
# features not available in kind (DRA feature gates, driver modules).
108-
# The explicit "Wait for GPU operands" step below gates on what
109-
# actually matters (device plugin readiness).
110-
# --best-effort: some components (e.g. network-operator) have Helm
111-
# hooks that may time out in Kind; continue deploying remaining
112-
# components so the overall stack is functional.
111+
# The default keeps legacy bundle-mode behavior: do not wait on every
112+
# Helm resource and keep deploying after component failures. H100
113+
# qualification jobs override these inputs to hard-fail and wait.
113114
chmod +x deploy.sh
114-
echo "--- deploy.sh ---"
115-
cat deploy.sh
116-
./deploy.sh --no-wait --best-effort
115+
DEPLOY_ARGS=()
116+
if [[ "${{ inputs.wait }}" != "true" ]]; then
117+
DEPLOY_ARGS+=(--no-wait)
118+
fi
119+
if [[ "${{ inputs.best_effort }}" == "true" ]]; then
120+
DEPLOY_ARGS+=(--best-effort)
121+
fi
122+
if [[ "${#DEPLOY_ARGS[@]}" -gt 0 ]]; then
123+
echo "Deploying bundle with args: ${DEPLOY_ARGS[*]}"
124+
else
125+
echo "Deploying bundle with default args"
126+
fi
127+
./deploy.sh "${DEPLOY_ARGS[@]}"
117128
118129
- name: Wait for GPU operands (bundle)
119130
if: inputs.method == 'bundle'

.github/actions/gpu-snapshot-validate/action.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ inputs:
2626
cluster_name:
2727
description: 'Kind cluster name (for kubectl context)'
2828
required: true
29+
snapshot_timeout:
30+
description: 'Timeout for aicr snapshot'
31+
required: false
32+
default: '5m'
2933

3034
runs:
3135
using: composite
@@ -38,6 +42,7 @@ runs:
3842
--namespace=default \
3943
--image=ko.local:smoke-test \
4044
--require-gpu \
45+
--timeout="${{ inputs.snapshot_timeout }}" \
4146
--output=snapshot.yaml
4247
echo "--- Snapshot output ---"
4348
cat snapshot.yaml

.github/actions/gpu-test-cleanup/action.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,15 @@ runs:
4040
kubectl --context="kind-${KIND_CLUSTER_NAME}" -n gpu-operator logs -l app=nvidia-device-plugin-daemonset --tail=100 > /tmp/debug-artifacts/device-plugin-logs.txt || true
4141
kubectl --context="kind-${KIND_CLUSTER_NAME}" -n gpu-operator logs -l app.kubernetes.io/component=gpu-operator --tail=100 > /tmp/debug-artifacts/gpu-operator-logs.txt || true
4242
kubectl --context="kind-${KIND_CLUSTER_NAME}" get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded > /tmp/debug-artifacts/non-running-pods.txt || true
43+
tar_inputs=()
44+
[[ -f recipe.yaml ]] && tar_inputs+=(recipe.yaml)
45+
[[ -d bundle ]] && tar_inputs+=(bundle)
46+
if [[ "${#tar_inputs[@]}" -gt 0 ]]; then
47+
echo "Archiving runtime bundle inputs: ${tar_inputs[*]}"
48+
tar -czf /tmp/debug-artifacts/aicr-runtime-bundle.tar.gz "${tar_inputs[@]}" || true
49+
else
50+
echo "No recipe.yaml or bundle directory found; skipping runtime bundle archive"
51+
fi
4352
4453
- name: Export kind logs
4554
if: failure()

.github/actions/install-karpenter-kwok/action.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,18 @@ inputs:
1919
cluster_name:
2020
description: 'Kind cluster name (used for kubectl context)'
2121
required: true
22+
kwok_helm_timeout:
23+
description: 'Timeout for KWOK controller Helm install'
24+
required: false
25+
default: '300s'
26+
ko_build_timeout:
27+
description: 'Timeout in seconds for Karpenter KWOK provider ko build'
28+
required: false
29+
default: '900'
30+
karpenter_helm_timeout:
31+
description: 'Timeout for Karpenter Helm install'
32+
required: false
33+
default: '300s'
2234

2335
runs:
2436
using: 'composite'
@@ -46,6 +58,9 @@ runs:
4658
env:
4759
KIND_CLUSTER_NAME: ${{ inputs.cluster_name }}
4860
KARPENTER_VERSION: ${{ steps.versions.outputs.karpenter }}
61+
KWOK_HELM_TIMEOUT: ${{ inputs.kwok_helm_timeout }}
62+
KO_BUILD_TIMEOUT: ${{ inputs.ko_build_timeout }}
63+
KARPENTER_HELM_TIMEOUT: ${{ inputs.karpenter_helm_timeout }}
4964
run: |
5065
set -euo pipefail
5166
bash kwok/scripts/install-karpenter-kwok.sh

.github/workflows/gpu-h100-inference-test.yaml

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ jobs:
5050
- '.settings.yaml'
5151
- '.github/actions/gpu-cluster-setup/**'
5252
- '.github/actions/gpu-operator-install/**'
53+
- '.github/actions/check-control-plane-health/**'
5354
- '.github/actions/aicr-build/**'
5455
- '.github/actions/setup-build-tools/**'
5556
- '.github/actions/install-karpenter-kwok/**'
@@ -96,7 +97,9 @@ jobs:
9697
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}
9798
cancel-in-progress: true
9899
runs-on: linux-amd64-gpu-h100-latest-2
99-
timeout-minutes: 120
100+
# Cold self-hosted H100 runners can spend most of the old budget pulling
101+
# images and loading Kind nodes before validation starts.
102+
timeout-minutes: 180
100103

101104
env:
102105
KIND_CLUSTER_NAME: gpu-inference-test
@@ -116,13 +119,27 @@ jobs:
116119
with:
117120
validator_phases: 'conformance'
118121

122+
- name: Check control plane health
123+
uses: ./.github/actions/check-control-plane-health
124+
with:
125+
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
126+
wait_timeout: 120s
127+
119128
- name: Install runtime bundle
120129
id: bundle-install
121130
uses: ./.github/actions/gpu-operator-install
122131
with:
123132
method: bundle
124133
accelerator: h100
125134
platform: dynamo
135+
wait: 'true'
136+
best_effort: 'false'
137+
138+
- name: Check control plane health after runtime install
139+
uses: ./.github/actions/check-control-plane-health
140+
with:
141+
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
142+
wait_timeout: 120s
126143

127144
# --- Snapshot and GPU validation ---
128145

@@ -132,16 +149,32 @@ jobs:
132149
gpu_model: H100
133150
min_gpu_count: '2'
134151
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
152+
snapshot_timeout: 10m
135153

136154
# --- Install Karpenter + KWOK early to give monitoring stack settle time ---
137155

156+
- name: Check control plane health before Karpenter
157+
uses: ./.github/actions/check-control-plane-health
158+
with:
159+
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
160+
wait_timeout: 120s
161+
138162
- name: Install Karpenter + KWOK
139163
uses: ./.github/actions/install-karpenter-kwok
140164
with:
141165
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
166+
kwok_helm_timeout: 600s
167+
ko_build_timeout: '1200'
168+
karpenter_helm_timeout: 600s
142169

143170
# --- Health checks ---
144171

172+
- name: Check control plane health after Karpenter
173+
uses: ./.github/actions/check-control-plane-health
174+
with:
175+
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
176+
wait_timeout: 120s
177+
145178
- name: Prepare chainsaw
146179
id: versions
147180
uses: ./.github/actions/load-versions
@@ -171,6 +204,12 @@ jobs:
171204
--kubeconfig="${HOME}/.kube/config" \
172205
--debug
173206
207+
- name: Check control plane health before conformance validation
208+
uses: ./.github/actions/check-control-plane-health
209+
with:
210+
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
211+
wait_timeout: 120s
212+
174213
- name: Validate CNCF AI Conformance
175214
id: validate-conformance
176215
run: |
@@ -193,8 +232,9 @@ jobs:
193232
# training smoke test if needed.
194233
# --- Validation artifacts ---
195234

196-
# Collect a post-run resource snapshot regardless of whether conformance
197-
# validation ran, so triage always has a cluster-state artifact.
235+
# Collect a post-run resource snapshot whenever the runtime bundle
236+
# installed. This preserves triage data for snapshot/chainsaw/conformance
237+
# failures; continue-on-error keeps the original failure intact.
198238
- name: Collect validation artifacts
199239
if: >-
200240
always()

0 commit comments

Comments
 (0)