Skip to content

Commit 8b3f9f3

Browse files
chensuyuechensuyue
authored
[CICD enhance] run CI with latest base image, group logs in GHA outputs. (opea-project#2071)
Signed-off-by: chensuyue <suyue.chen@intel.com>
1 parent c873c58 commit 8b3f9f3

10 files changed

Lines changed: 209 additions & 107 deletions

File tree

‎InstructionTuning/tests/test_compose_on_xeon.sh‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -98,15 +98,27 @@ function stop_docker() {
9898

9999
function main() {
100100

101+
echo "::group::stop_docker"
101102
stop_docker
103+
echo "::endgroup::"
102104

103-
build_docker_images
104-
start_service
105+
echo "::group::build_docker_images"
106+
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
107+
echo "::endgroup::"
105108

106-
validate_microservice
109+
echo "::group::start_services"
110+
start_services
111+
echo "::endgroup::"
107112

113+
echo "::group::validate_microservices"
114+
validate_microservices
115+
echo "::endgroup::"
116+
117+
echo "::group::stop_docker"
108118
stop_docker
109-
echo y | docker system prune
119+
echo "::endgroup::"
120+
121+
docker system prune -f
110122

111123
}
112124

‎MultimodalQnA/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
# Copyright (C) 2024 Intel Corporation
22
# SPDX-License-Identifier: Apache-2.0
33

4+
ARG IMAGE_REPO=opea
45
ARG BASE_TAG=latest
5-
FROM opea/comps-base:$BASE_TAG
6+
FROM $IMAGE_REPO/comps-base:$BASE_TAG
67

78
COPY ./multimodalqna.py $HOME/multimodalqna.py
89

‎MultimodalQnA/docker_image_build/build.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ services:
55
multimodalqna:
66
build:
77
args:
8+
IMAGE_REPO: ${REGISTRY}
9+
BASE_TAG: ${TAG}
810
http_proxy: ${http_proxy}
911
https_proxy: ${https_proxy}
1012
no_proxy: ${no_proxy}

‎MultimodalQnA/tests/test_compose_milvus_on_xeon.sh‎

Lines changed: 31 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -44,19 +44,13 @@ function check_service_ready() {
4444

4545
function build_docker_images() {
4646
opea_branch=${opea_branch:-"main"}
47-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
48-
if [[ "${opea_branch}" != "main" ]]; then
49-
cd $WORKPATH
50-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
51-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
52-
find . -type f -name "Dockerfile*" | while read -r file; do
53-
echo "Processing file: $file"
54-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
55-
done
56-
fi
5747

5848
cd $WORKPATH/docker_image_build
5949
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
50+
pushd GenAIComps
51+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
52+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
53+
popd && sleep 1s
6054

6155
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
6256
service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding retriever lvm-llava lvm dataprep whisper"
@@ -340,25 +334,44 @@ function stop_docker() {
340334
function main() {
341335

342336
setup_env
337+
338+
echo "::group::stop_docker"
343339
stop_docker
340+
echo "::endgroup::"
341+
342+
echo "::group::build_docker_images"
344343
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
345-
start_time=$(date +%s)
344+
echo "::endgroup::"
345+
346+
echo "::group::start_services"
346347
start_services
347-
end_time=$(date +%s)
348-
duration=$((end_time-start_time))
349-
echo "Mega service start duration is $duration s" && sleep 1s
348+
echo "::endgroup::"
349+
350+
echo "::group::prepare_data"
350351
prepare_data
352+
echo "::endgroup::"
351353

354+
echo "::group::validate_microservices"
352355
validate_microservices
353-
echo "==== microservices validated ===="
356+
echo "::endgroup::"
357+
358+
echo "::group::validate_megaservice"
354359
validate_megaservice
355-
echo "==== megaservice validated ===="
360+
echo "::endgroup::"
361+
362+
echo "::group::validate_delete"
356363
validate_delete
357-
echo "==== delete validated ===="
364+
echo "::endgroup::"
358365

366+
echo "::group::delete_data"
359367
delete_data
368+
echo "::endgroup::"
369+
370+
echo "::group::stop_docker"
360371
stop_docker
361-
echo y | docker system prune
372+
echo "::endgroup::"
373+
374+
docker system prune -f
362375

363376
}
364377

‎MultimodalQnA/tests/test_compose_on_gaudi.sh‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -45,26 +45,18 @@ function check_service_ready() {
4545

4646
function build_docker_images() {
4747
opea_branch=${opea_branch:-"main"}
48-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
49-
if [[ "${opea_branch}" != "main" ]]; then
50-
cd $WORKPATH
51-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
52-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
53-
find . -type f -name "Dockerfile*" | while read -r file; do
54-
echo "Processing file: $file"
55-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
56-
done
57-
fi
5848

5949
cd $WORKPATH/docker_image_build
6050
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
51+
pushd GenAIComps
52+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
53+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
54+
popd && sleep 1s
6155

6256
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
6357
service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower-gaudi embedding retriever speecht5 lvm dataprep whisper"
6458
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
6559

66-
docker pull ghcr.io/huggingface/tgi-gaudi:2.3.1
67-
6860
docker images && sleep 1s
6961
}
7062

@@ -366,25 +358,44 @@ function stop_docker() {
366358
function main() {
367359

368360
setup_env
361+
362+
echo "::group::stop_docker"
369363
stop_docker
364+
echo "::endgroup::"
365+
366+
echo "::group::build_docker_images"
370367
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
371-
start_time=$(date +%s)
368+
echo "::endgroup::"
369+
370+
echo "::group::start_services"
372371
start_services
373-
end_time=$(date +%s)
374-
duration=$((end_time-start_time))
375-
echo "Mega service start duration is $duration s" && sleep 1s
372+
echo "::endgroup::"
373+
374+
echo "::group::prepare_data"
376375
prepare_data
376+
echo "::endgroup::"
377377

378+
echo "::group::validate_microservices"
378379
validate_microservices
379-
echo "==== microservices validated ===="
380+
echo "::endgroup::"
381+
382+
echo "::group::validate_megaservice"
380383
validate_megaservice
381-
echo "==== megaservice validated ===="
384+
echo "::endgroup::"
385+
386+
echo "::group::validate_delete"
382387
validate_delete
383-
echo "==== delete validated ===="
388+
echo "::endgroup::"
384389

390+
echo "::group::delete_data"
385391
delete_data
392+
echo "::endgroup::"
393+
394+
echo "::group::stop_docker"
386395
stop_docker
387-
echo y | docker system prune
396+
echo "::endgroup::"
397+
398+
docker system prune -f
388399

389400
}
390401

‎MultimodalQnA/tests/test_compose_on_rocm.sh‎

Lines changed: 33 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,14 @@ export caption_fn="apple.txt"
2121

2222
function build_docker_images() {
2323
opea_branch=${opea_branch:-"main"}
24-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
25-
if [[ "${opea_branch}" != "main" ]]; then
26-
cd $WORKPATH
27-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
28-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
29-
find . -type f -name "Dockerfile*" | while read -r file; do
30-
echo "Processing file: $file"
31-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
32-
done
33-
fi
3424

3525
cd $WORKPATH/docker_image_build
36-
git clone https://github.com/opea-project/GenAIComps.git && cd GenAIComps && git checkout "${opea_branch:-"main"}" && cd ../
26+
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
27+
pushd GenAIComps
28+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
29+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
30+
popd && sleep 1s
31+
3732
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
3833
service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding retriever lvm dataprep whisper"
3934
docker compose -f build.yaml build ${service_list} --no-cache > ${LOG_PATH}/docker_image_build.log
@@ -289,25 +284,44 @@ function stop_docker() {
289284
function main() {
290285

291286
setup_env
287+
288+
echo "::group::stop_docker"
292289
stop_docker
290+
echo "::endgroup::"
291+
292+
echo "::group::build_docker_images"
293293
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
294-
start_time=$(date +%s)
294+
echo "::endgroup::"
295+
296+
echo "::group::start_services"
295297
start_services
296-
end_time=$(date +%s)
297-
duration=$((end_time-start_time))
298-
echo "Mega service start duration is $duration s" && sleep 1s
298+
echo "::endgroup::"
299+
300+
echo "::group::prepare_data"
299301
prepare_data
302+
echo "::endgroup::"
300303

304+
echo "::group::validate_microservices"
301305
validate_microservices
302-
echo "==== microservices validated ===="
306+
echo "::endgroup::"
307+
308+
echo "::group::validate_megaservice"
303309
validate_megaservice
304-
echo "==== megaservice validated ===="
310+
echo "::endgroup::"
311+
312+
echo "::group::validate_delete"
305313
validate_delete
306-
echo "==== delete validated ===="
314+
echo "::endgroup::"
307315

316+
echo "::group::delete_data"
308317
delete_data
318+
echo "::endgroup::"
319+
320+
echo "::group::stop_docker"
309321
stop_docker
310-
echo y | docker system prune
322+
echo "::endgroup::"
323+
324+
docker system prune -f
311325

312326
}
313327

‎MultimodalQnA/tests/test_compose_on_xeon.sh‎

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -45,19 +45,13 @@ function check_service_ready() {
4545

4646
function build_docker_images() {
4747
opea_branch=${opea_branch:-"main"}
48-
# If the opea_branch isn't main, replace the git clone branch in Dockerfile.
49-
if [[ "${opea_branch}" != "main" ]]; then
50-
cd $WORKPATH
51-
OLD_STRING="RUN git clone --depth 1 https://github.com/opea-project/GenAIComps.git"
52-
NEW_STRING="RUN git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git"
53-
find . -type f -name "Dockerfile*" | while read -r file; do
54-
echo "Processing file: $file"
55-
sed -i "s|$OLD_STRING|$NEW_STRING|g" "$file"
56-
done
57-
fi
5848

5949
cd $WORKPATH/docker_image_build
6050
git clone --depth 1 --branch ${opea_branch} https://github.com/opea-project/GenAIComps.git
51+
pushd GenAIComps
52+
echo "GenAIComps test commit is $(git rev-parse HEAD)"
53+
docker build --no-cache -t ${REGISTRY}/comps-base:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f Dockerfile .
54+
popd && sleep 1s
6155

6256
echo "Build all the images with --no-cache, check docker_image_build.log for details..."
6357
service_list="multimodalqna multimodalqna-ui embedding-multimodal-bridgetower embedding retriever speecht5 lvm-llava lvm dataprep whisper"
@@ -367,26 +361,44 @@ function stop_docker() {
367361
function main() {
368362

369363
setup_env
364+
365+
echo "::group::stop_docker"
370366
stop_docker
367+
echo "::endgroup::"
368+
369+
echo "::group::build_docker_images"
371370
if [[ "$IMAGE_REPO" == "opea" ]]; then build_docker_images; fi
372-
start_time=$(date +%s)
371+
echo "::endgroup::"
372+
373+
echo "::group::start_services"
373374
start_services
374-
end_time=$(date +%s)
375-
duration=$((end_time-start_time))
376-
echo "Mega service start duration is $duration s" && sleep 1s
375+
echo "::endgroup::"
376+
377+
echo "::group::prepare_data"
377378
prepare_data
379+
echo "::endgroup::"
378380

381+
echo "::group::validate_microservices"
379382
validate_microservices
380-
echo "==== microservices validated ===="
383+
echo "::endgroup::"
384+
385+
echo "::group::validate_megaservice"
381386
validate_megaservice
382-
echo "==== megaservice validated ===="
387+
echo "::endgroup::"
388+
389+
echo "::group::validate_delete"
383390
validate_delete
384-
echo "==== delete validated ===="
391+
echo "::endgroup::"
385392

393+
echo "::group::delete_data"
386394
delete_data
395+
echo "::endgroup::"
396+
397+
echo "::group::stop_docker"
387398
stop_docker
388-
echo y | docker system prune
399+
echo "::endgroup::"
389400

401+
docker system prune -f
390402
}
391403

392404
main

0 commit comments

Comments
 (0)