Skip to content

Commit f9f26da

Browse files
yzeyu71yzeyu71
committed
fix/connect-docker.io-fail
Signed-off-by: yangzeyu <532183776@qq.com>
1 parent db8db6f commit f9f26da

2 files changed

Lines changed: 102 additions & 68 deletions

File tree

‎.buildkite/pipeline-npu.yaml‎

Lines changed: 98 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,93 @@ steps:
55
- group: ":pipeline: Run NPU test"
66
key: npu-tests
77
steps:
8-
- label: ":buildkit: Build and Push NPU Test Image - A3"
9-
key: image-build-npu
8+
# - label: ":buildkit: Build and Push NPU Test Image - A3"
9+
# key: image-build-npu
10+
# agents:
11+
# queue: "ascend-a3"
12+
# resource_class: "npu-2"
13+
# plugins:
14+
# - kubernetes:
15+
# metadata:
16+
# annotations:
17+
# vault.hashicorp.com/agent-init-first: "true"
18+
# vault.hashicorp.com/agent-inject: "true"
19+
# vault.hashicorp.com/agent-inject-perms-ca.pem: "0400"
20+
# vault.hashicorp.com/agent-inject-perms-cert.pem: "0400"
21+
# vault.hashicorp.com/agent-inject-perms-config.json: "0400"
22+
# vault.hashicorp.com/agent-inject-perms-key.pem: "0400"
23+
# vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd
24+
# vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd
25+
# vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd
26+
# vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd
27+
# vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}"
28+
# vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}"
29+
# vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}"
30+
# vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}"
31+
# vault.hashicorp.com/agent-pre-populate-only: "true"
32+
# vault.hashicorp.com/agent-run-as-group: "1000"
33+
# vault.hashicorp.com/agent-run-as-user: "1000"
34+
# vault.hashicorp.com/agent-service-account-token-volume-name: token-vol
35+
# vault.hashicorp.com/role: ascend-gha-runners
36+
# vault.hashicorp.com/secret-volume-path: /home/user/.docker/
37+
# vault.hashicorp.com/tls-skip-verify: "true"
38+
# podSpecPatch:
39+
# volumes:
40+
# - name: token-vol
41+
# projected:
42+
# defaultMode: 420
43+
# sources:
44+
# - serviceAccountToken:
45+
# audience: api
46+
# expirationSeconds: 600
47+
# path: token
48+
# env:
49+
# VIME_IMAGE_TAG: "${BUILDKITE_COMMIT}"
50+
# IMAGE_NAME: "vime-ci-npu"
51+
# IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
52+
# BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
53+
# command: |
54+
# set -ex
55+
#
56+
# echo "--- Building and pushing NPU Test Image"
57+
# echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}"
58+
# echo "buildkitd address: $${BUILDKITD_ADDR}"
59+
#
60+
# if ! command -v buildctl &> /dev/null; then
61+
# echo "Installing buildctl..."
62+
# mkdir -p /tmp/buildkit
63+
# BUILDKIT_VERSION="v0.29.0"
64+
# wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
65+
# tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
66+
# cp /tmp/buildkit/bin/buildctl /usr/local/bin/
67+
# fi
68+
#
69+
# export DOCKER_CONFIG=/home/user/.docker
70+
# buildctl \
71+
# --addr="$${BUILDKITD_ADDR}" \
72+
# --tlscacert=/home/user/.docker/ca.pem \
73+
# --tlscert=/home/user/.docker/cert.pem \
74+
# --tlskey=/home/user/.docker/key.pem \
75+
# build \
76+
# --frontend dockerfile.v0 \
77+
# --local context=. \
78+
# --local dockerfile=./docker \
79+
# --opt filename=Dockerfile.npu \
80+
# --secret id=dockerconfig,src=/home/user/.docker/config.json \
81+
# --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG},push=true \
82+
# --progress=plain
83+
#
84+
# echo "--- Image pushed successfully"
85+
# echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}"
86+
87+
- label: "qwen3-4B-npu test"
88+
key: qwen3-4B-npu
89+
# depends_on: image-build-npu
1090
agents:
1191
queue: "ascend-a3"
1292
resource_class: "npu-2"
93+
image: "quay.io/ascend/vime:vime-latest"
94+
timeout_in_minutes: 60
1395
plugins:
1496
- kubernetes:
1597
metadata:
@@ -45,76 +127,28 @@ steps:
45127
audience: api
46128
expirationSeconds: 600
47129
path: token
48-
env:
49-
VIME_IMAGE_TAG: "${BUILDKITE_COMMIT}"
50-
IMAGE_NAME: "vime-ci-npu"
51-
IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry"
52-
BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234"
53-
command: |
54-
set -ex
55-
56-
echo "--- Building and pushing NPU Test Image"
57-
echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}"
58-
echo "buildkitd address: $${BUILDKITD_ADDR}"
59-
60-
if ! command -v buildctl &> /dev/null; then
61-
echo "Installing buildctl..."
62-
mkdir -p /tmp/buildkit
63-
BUILDKIT_VERSION="v0.29.0"
64-
wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz
65-
tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit
66-
cp /tmp/buildkit/bin/buildctl /usr/local/bin/
67-
fi
68-
69-
export DOCKER_CONFIG=/home/user/.docker
70-
buildctl \
71-
--addr="$${BUILDKITD_ADDR}" \
72-
--tlscacert=/home/user/.docker/ca.pem \
73-
--tlscert=/home/user/.docker/cert.pem \
74-
--tlskey=/home/user/.docker/key.pem \
75-
build \
76-
--frontend dockerfile.v0 \
77-
--local context=. \
78-
--local dockerfile=./docker \
79-
--opt filename=Dockerfile.npu \
80-
--secret id=dockerconfig,src=/home/user/.docker/config.json \
81-
--output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG},push=true \
82-
--progress=plain
83-
84-
echo "--- Image pushed successfully"
85-
echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}"
86-
87-
- label: "qwen3-4B-npu test"
88-
key: qwen3-4B-npu
89-
depends_on: image-build-npu
90-
agents:
91-
queue: "ascend-a3"
92-
resource_class: "npu-2"
93-
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vime-ci-npu:${BUILDKITE_COMMIT}"
94-
timeout_in_minutes: 60
95-
plugins:
96-
- kubernetes:
97-
working_dir: /root
98-
podSpecPatch:
99130
imagePullSecrets:
100131
- name: swr-secret
101132
env:
102-
export WORKSPACE: /root
103133
HF_HOME: /root/.cache/huggingface
104134
VIME_TEST_ENABLE_INFINITE_RUN: false
105135
BUILDKITE_PULL_REQUEST: "${BUILDKITE_PULL_REQUEST:-false}"
106136
BUILDKITE_COMMIT: "${BUILDKITE_COMMIT}"
107137
GITHUB_COMMIT_NAME: "${BUILDKITE_COMMIT}_${BUILDKITE_PULL_REQUEST:-non-pr}"
108138
HF_TOKEN: "${HF_TOKEN}"
139+
HF_ENDPOINT: https://hf-mirror.com
109140
command: |
110-
set -euo pipefail
111-
cd /root/vime
112-
if [ -n \"${BUILDKITE_COMMIT}\" ]; then
113-
git fetch origin ${BUILDKITE_COMMIT}
114-
git checkout ${BUILDKITE_COMMIT}
115-
pip install -e . --no-deps --break-system-packages || pip install -e . --no-deps
116-
fi
117-
hf download Qwen/Qwen3-4B
118-
hf download --repo-type dataset zhuzilin/dapo-math-17k
119-
bash scripts/run-qwen3-4B-npu.sh
141+
bash -c "
142+
set -euo pipefail
143+
cd /root/vime
144+
if [ -n \"${BUILDKITE_COMMIT}\" ]; then
145+
git fetch origin ${BUILDKITE_COMMIT}
146+
git checkout ${BUILDKITE_COMMIT}
147+
pip install -e . --no-deps --break-system-packages || pip install -e . --no-deps
148+
fi
149+
pip install modelscope -q
150+
modelscope download --model 'Qwen/Qwen3-4B'
151+
hf download --repo-type dataset zhuzilin/dapo-math-17k
152+
bash scripts/run-qwen3-4B-npu.sh
153+
"
120154

‎docker/Dockerfile.npu‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
# syntax=docker/dockerfile:1.7
2-
31
ARG BASE_IMAGE=quay.io/ascend/vllm-ascend:nightly-main-a3
42
FROM ${BASE_IMAGE}
53

@@ -24,7 +22,8 @@ ENV DEBIAN_FRONTEND=noninteractive \
2422
HCCL_NPU_SOCKET_PORT_RANGE=61000-61050 \
2523
PYTORCH_NPU_ALLOC_CONF=expandable_segments:True \
2624
HYDRA_FULL_ERROR=1 \
27-
PYTHONPATH=/root/Megatron-Bridge/src:/root/Megatron-LM:/root/vime
25+
PYTHONPATH=/root/Megatron-Bridge/src:/root/Megatron-LM:/root/vime \
26+
LD_LIBRARY_PATH=/usr/local/Ascend/ascend-toolkit/latest/lib64:/usr/local/Ascend/ascend-toolkit/latest/lib:$LD_LIBRARY_PATH
2827

2928
# Patches copied from build context; always apply from /tmp/npu_patch so local/PR
3029
# builds use the same files as the build context (not the later git-cloned vime tree).
@@ -41,7 +40,8 @@ RUN sed -i 's|ports.ubuntu.com|mirrors.tuna.tsinghua.edu.cn|g' \
4140
https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple && \
4241
pip config set global.extra-index-url \
4342
"https://download.pytorch.org/whl/cpu/ https://mirrors.huaweicloud.com/ascend/repos/pypi" && \
44-
git config --global http.version HTTP/1.1
43+
git config --global http.version HTTP/1.1 && \
44+
git config --global url."https://ghfast.top/https://github.com/".insteadOf "https://github.com/"
4545

4646
# vLLM: fixed, verified commit.
4747
RUN pip uninstall -y \

0 commit comments

Comments
 (0)