From 06cd30f82b98e49b07012e108d19b2d378f6783c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 31 Mar 2026 17:15:29 +0200 Subject: [PATCH 001/229] Test --- .gitlab-ci.yml | 5 +++++ .gitlab/mock.yml | 11 +++++++++++ 2 files changed, 16 insertions(+) create mode 100644 .gitlab/mock.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57dcbcee17d..8105564088b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,11 @@ include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml + - local: "/.gitlab/mock.yml" + inputs: + test: "Hello world!" + stage: "test" stages: + - test - configure - nodejs - java diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml new file mode 100644 index 00000000000..32f11897631 --- /dev/null +++ b/.gitlab/mock.yml @@ -0,0 +1,11 @@ +spec: + inputs: + test: + stage: +--- + +echo: + tags: + - arch:amd64 + stage: $[[ inputs.stage ]] + script: echo $[[ inputs.test ]] From 2934c37766ab556ff6236f8d4480b5c67974d239 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 31 Mar 2026 17:44:48 +0200 Subject: [PATCH 002/229] Clean --- .gitlab-ci.yml | 303 ------------------------------------------------- 1 file changed, 303 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8105564088b..2c4b56589b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,310 +1,7 @@ include: - - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml - local: "/.gitlab/mock.yml" inputs: test: "Hello world!" stage: "test" stages: - test - - configure - - nodejs - - java - - dotnet - - python - - php - - ruby - - pipeline-status - - system-tests-utils - -variables: - TEST: 1 - -compute_pipeline: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 - tags: ["arch:amd64"] - stage: configure - variables: - CI_ENVIRONMENT: "prod" - script: - - | - if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then - echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined.Computing changes..." - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - source venv/bin/activate - ./run.sh MOCK_THE_TEST --collect-only --scenario-report - git clone https://github.com/DataDog/system-tests.git original - git fetch --all # Ensure all branches are available - git branch --track $CI_COMMIT_REF_NAME origin/$CI_COMMIT_REF_NAME || true # Track branch if not tracked - git checkout $CI_COMMIT_REF_NAME # Ensure the branch is checked out - BASE_COMMIT=$(git merge-base origin/main $CI_COMMIT_REF_NAME) # Get the base commit - echo "Branch was created from commit--> $BASE_COMMIT" - git diff --name-only $BASE_COMMIT $CI_COMMIT_SHA >> modified_files.txt # List modified files - cat modified_files.txt - python utils/scripts/compute_libraries_and_scenarios.py >> impacted_scenarios.txt - cat impacted_scenarios.txt - source impacted_scenarios.txt - else - echo "✅ SYSTEM_TESTS_SCENARIOS OR SYSTEM_TESTS_SCENARIOS_GROUPS are set." - export scenarios=$SYSTEM_TESTS_SCENARIOS - export scenarios_groups=$SYSTEM_TESTS_SCENARIOS_GROUPS - cd /system-tests - git pull - if [ -n "$SYSTEM_TESTS_REF" ]; then - git checkout $SYSTEM_TESTS_REF - else - echo "⚠️ SYSTEM_TESTS_REF variable is not set, skipping git checkout" - fi - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - source venv/bin/activate - fi - - python utils/scripts/compute-workflow-parameters.py nodejs -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py java -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py dotnet -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py python -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py php -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py ruby -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - | - if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then - cp *.yml "$CI_PROJECT_DIR" - fi - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - artifacts: - paths: - - nodejs_ssi_gitlab_pipeline.yml - - java_ssi_gitlab_pipeline.yml - - dotnet_ssi_gitlab_pipeline.yml - - python_ssi_gitlab_pipeline.yml - - php_ssi_gitlab_pipeline.yml - - ruby_ssi_gitlab_pipeline.yml - -nodejs_ssi_pipeline: - stage: nodejs - needs: ["compute_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: nodejs_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -java_ssi_pipeline: - stage: java - needs: ["compute_pipeline", "nodejs_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: java_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -dotnet_ssi_pipeline: - stage: dotnet - needs: ["compute_pipeline", "java_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: dotnet_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -python_ssi_pipeline: - stage: python - needs: ["compute_pipeline", "dotnet_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: python_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -php_ssi_pipeline: - stage: php - needs: ["compute_pipeline", "python_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: php_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -ruby_ssi_pipeline: - stage: ruby - needs: ["compute_pipeline", "php_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: ruby_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -delete_amis: - extends: .base_job_onboarding - stage: system-tests-utils - allow_failure: false - variables: - AMI_RETENTION_DAYS: 10 - AMI_LAST_LAUNCHED_DAYS: 10 - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - python utils/scripts/pulumi_clean_up.py --component amis --ami-retention-days $AMI_RETENTION_DAYS --ami-last-launched-days $AMI_LAST_LAUNCHED_DAYS - rules: - - if: '$SCHEDULED_JOB == "delete_amis"' - after_script: echo "Finish" - timeout: 3h - -delete_amis_by_name_or_lang: - extends: .base_job_onboarding - stage: system-tests-utils - allow_failure: false - script: - - | - if [ -z "$AMI_NAME" ] && [ -z "$AMI_LANG" ]; then - echo "❌ ERROR: Either AMI_NAME or AMI_LANG must be set." - exit 1 - fi - echo "✅ Proceeding with AMI_NAME=$AMI_NAME, AMI_LANG=$AMI_LANG" - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - | - CMD="python utils/scripts/pulumi_clean_up.py --component amis_by_name" - - if [ -n "$AMI_NAME" ]; then - CMD="$CMD --ami-name $AMI_NAME" - fi - - if [ -n "$AMI_LANG" ]; then - CMD="$CMD --ami-lang $AMI_LANG" - fi - - echo "Running: $CMD" - eval $CMD - rules: - - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' - after_script: echo "Finish" - -delete_ec2_instances: - extends: .base_job_onboarding - stage: system-tests-utils - allow_failure: false - variables: - EC2_AGE_MINUTES: 45 - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - python utils/scripts/pulumi_clean_up.py --component ec2 --ec2-age-minutes $EC2_AGE_MINUTES - rules: - - if: '$SCHEDULED_JOB == "delete_ec2_instances"' - after_script: echo "Finish" - -count_amis: - extends: .base_job_onboarding - stage: system-tests-utils - allow_failure: false - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - python utils/scripts/pulumi_clean_up.py --component amis_count - rules: - - if: '$SCHEDULED_JOB == "count_amis"' - after_script: echo "Finish" - -check_merge_labels: - #Build docker images if it's needed. Check if the PR has the labels associated with the image build. - image: registry.ddbuild.io/system-tests/base-image-builder - tags: - - arch:amd64 - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - needs: [] - stage: system-tests-utils - before_script: - # Get a token - - dd-octo-sts version - - dd-octo-sts debug --scope DataDog/system-tests --policy self.gitlab-read - - dd-octo-sts token --scope DataDog/system-tests --policy self.gitlab-read > token.txt - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - script: - - export GITHUB_TOKEN=$(cat token.txt) - - ./utils/scripts/get_pr_merged_labels.sh - after_script: - # Revoke the token after usage - - dd-octo-sts revoke -t $(cat token.txt) - rules: - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" - -generate_system_tests_lambda_proxy_image: - image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 - tags: ["docker-in-docker:amd64"] - needs: [] - stage: system-tests-utils - allow_failure: false - before_script: - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i lambda-proxy - - docker push datadog/system-tests:lambda-proxy-v1 - rules: - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" - changes: - - utils/build/docker/lambda_proxy/pyproject.toml - - utils/build/docker/lambda-proxy.Dockerfile - -generate_system_tests_lib_injection_images: - extends: .base_job_k8s_docker_ssi - needs: [] - stage: system-tests-utils - allow_failure: true - variables: - PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com - PRIVATE_DOCKER_REGISTRY_USER: AWS - script: - - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} - - ./lib-injection/build/build_lib_injection_images.sh $PRIVATE_DOCKER_REGISTRY - rules: - - if: '$SCHEDULED_JOB == "generate_system_tests_lib_injection_images"' - - when: manual - -.delayed_base_job: - image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 - tags: ["arch:amd64"] - script: - - echo "⏳ Waiting before triggering the child pipeline..." - when: delayed - start_in: 5 minutes From 81f74d6c144965a3f3d0a0902b7cf19e8320d11b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 31 Mar 2026 18:30:03 +0200 Subject: [PATCH 003/229] Test matrix --- .gitlab-ci.yml | 1 - .gitlab/mock.yml | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c4b56589b2..b820bcf31e3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,6 @@ include: - local: "/.gitlab/mock.yml" inputs: - test: "Hello world!" stage: "test" stages: - test diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index 32f11897631..b408e0469c6 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -1,11 +1,14 @@ spec: inputs: - test: stage: --- -echo: +load: tags: - arch:amd64 stage: $[[ inputs.stage ]] - script: echo $[[ inputs.test ]] + parallel: + matrix: + - A: [0, 1, 2] + - B: [0, 1, 2] + script: echo "A $A, B $B" From 314ca4ad8889e3bc4091f822dce7c0bf90b1fca3 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 31 Mar 2026 18:43:21 +0200 Subject: [PATCH 004/229] Go big --- .gitlab/mock.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index b408e0469c6..b7752ee9a0a 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -9,6 +9,7 @@ load: stage: $[[ inputs.stage ]] parallel: matrix: - - A: [0, 1, 2] - - B: [0, 1, 2] - script: echo "A $A, B $B" + - A: [0, 1, 2, 3, 4, 5, 6] + B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + script: echo "A $A, B $B, C $C" From 624134d5f716dfacf6a04ba459a424fc28ec1688 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 10:53:55 +0200 Subject: [PATCH 005/229] Job split --- .gitlab/mock.yml | 46 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index b7752ee9a0a..a0e217fe42b 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -9,7 +9,51 @@ load: stage: $[[ inputs.stage ]] parallel: matrix: - - A: [0, 1, 2, 3, 4, 5, 6] + - A: [0, 1] + B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + script: echo "A $A, B $B, C $C" + +load1: + tags: + - arch:amd64 + stage: $[[ inputs.stage ]] + parallel: + matrix: + - A: [0, 1] + B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + script: echo "A $A, B $B, C $C" + +load2: + tags: + - arch:amd64 + stage: $[[ inputs.stage ]] + parallel: + matrix: + - A: [0, 1] + B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + script: echo "A $A, B $B, C $C" + +load3: + tags: + - arch:amd64 + stage: $[[ inputs.stage ]] + parallel: + matrix: + - A: [0, 1] + B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + script: echo "A $A, B $B, C $C" + +load4: + tags: + - arch:amd64 + stage: $[[ inputs.stage ]] + parallel: + matrix: + - A: [0, 1] B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] script: echo "A $A, B $B, C $C" From 1a062cdc4f737b2a7038d88c4ddf694b82537d28 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 11:09:30 +0200 Subject: [PATCH 006/229] Factorizing --- .gitlab/mock.yml | 47 +++++++++-------------------------------------- 1 file changed, 9 insertions(+), 38 deletions(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index a0e217fe42b..b34f9a61909 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -3,7 +3,7 @@ spec: stage: --- -load: +.load: tags: - arch:amd64 stage: $[[ inputs.stage ]] @@ -12,48 +12,19 @@ load: - A: [0, 1] B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - script: echo "A $A, B $B, C $C" + script: sleep 120; echo "A $A, B $B, C $C" + +load: + extends: .load load1: - tags: - - arch:amd64 - stage: $[[ inputs.stage ]] - parallel: - matrix: - - A: [0, 1] - B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - script: echo "A $A, B $B, C $C" + extends: .load load2: - tags: - - arch:amd64 - stage: $[[ inputs.stage ]] - parallel: - matrix: - - A: [0, 1] - B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - script: echo "A $A, B $B, C $C" + extends: .load load3: - tags: - - arch:amd64 - stage: $[[ inputs.stage ]] - parallel: - matrix: - - A: [0, 1] - B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - script: echo "A $A, B $B, C $C" + extends: .load load4: - tags: - - arch:amd64 - stage: $[[ inputs.stage ]] - parallel: - matrix: - - A: [0, 1] - B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - script: echo "A $A, B $B, C $C" + extends: .load From 5289a675e2bc83e6be7adbd0724f6b0f46d254b5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 11:26:04 +0200 Subject: [PATCH 007/229] Docker pull --- .gitlab/mock.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index b34f9a61909..785531749eb 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -4,15 +4,16 @@ spec: --- .load: + image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 tags: - - arch:amd64 + - docker-in-docker:amd64 stage: $[[ inputs.stage ]] parallel: matrix: - A: [0, 1] B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - script: sleep 120; echo "A $A, B $B, C $C" + script: docker pull datadog/system-tests:flask-poc.base-v13; docker pull registry.ddbuild.io/system-tests/base-image-builder:latest; echo "A $A, B $B, C $C" load: extends: .load From 17aca819717c5bd2eca935883e42c07c6bc0473b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 12:43:55 +0200 Subject: [PATCH 008/229] pip --- .gitlab/mock.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index 785531749eb..80bded77b20 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -4,7 +4,7 @@ spec: --- .load: - image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 tags: - docker-in-docker:amd64 stage: $[[ inputs.stage ]] @@ -13,7 +13,11 @@ spec: - A: [0, 1] B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - script: docker pull datadog/system-tests:flask-poc.base-v13; docker pull registry.ddbuild.io/system-tests/base-image-builder:latest; echo "A $A, B $B, C $C" + script: + - pip install pytest + - docker pull datadog/system-tests:flask-poc.base-v13 + - docker pull registry.ddbuild.io/system-tests/base-image-builder:latest + - echo "A $A, B $B, C $C" load: extends: .load From e16695b00842ecc6756fbe1db349c7ad81ae28f6 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 13:10:52 +0200 Subject: [PATCH 009/229] More complex example --- .gitlab/mock.yml | 5 ++--- test/docker-compose.yml | 9 +++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 test/docker-compose.yml diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index 80bded77b20..ac975fe8d4e 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -14,9 +14,8 @@ spec: B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] script: - - pip install pytest - - docker pull datadog/system-tests:flask-poc.base-v13 - - docker pull registry.ddbuild.io/system-tests/base-image-builder:latest + - ./build.sh -i runner + - cd test && docker compose up - echo "A $A, B $B, C $C" load: diff --git a/test/docker-compose.yml b/test/docker-compose.yml new file mode 100644 index 00000000000..b75e70859b4 --- /dev/null +++ b/test/docker-compose.yml @@ -0,0 +1,9 @@ +services: + ubuntu: + image: ubuntu + postgres: + image: postgres + python: + image: python + node: + image: node From 23d959e41c4641ce0378bf89639c50dff42a5a1b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 13:20:11 +0200 Subject: [PATCH 010/229] Docker auth --- .gitlab/mock.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index ac975fe8d4e..37d324e9612 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -17,6 +17,10 @@ spec: - ./build.sh -i runner - cd test && docker compose up - echo "A $A, B $B, C $C" + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin load: extends: .load From adb929a6f0f45b6c3dddfe7aaa90b4428206d91f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 13:48:14 +0200 Subject: [PATCH 011/229] No github CI --- utils/scripts/libraries_and_scenarios_rules.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/utils/scripts/libraries_and_scenarios_rules.yml b/utils/scripts/libraries_and_scenarios_rules.yml index b522c268bc2..414d3262792 100644 --- a/utils/scripts/libraries_and_scenarios_rules.yml +++ b/utils/scripts/libraries_and_scenarios_rules.yml @@ -112,6 +112,7 @@ patterns: scenario_groups: null .gitlab-ci.yml: scenario_groups: null + libraries: null .shellcheck: scenario_groups: null .shellcheckrc: @@ -152,6 +153,9 @@ patterns: run.sh: # run all by default .gitlab/ssi_gitlab-ci.yml: scenario_groups: [onboarding, lib_injection, docker_ssi] + .gitlab/mock.yml: + scenario_groups: null + libraries: null ################################ GitHub Actions ################################ # This section contains rules to apply to files related to GitHub Actions. @@ -550,3 +554,8 @@ patterns: scenario_groups: null # already handled by the manifest comparison tests/*: scenario_groups: null # dynamic selection + + + test/*: + libraries: null + scenario_groups: null From d6ca990879758883078f9e6d780ee7d4b4cae655 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 15:41:51 +0200 Subject: [PATCH 012/229] Running scenarios --- .gitlab/mock.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index 37d324e9612..e59a3b4e3db 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -14,9 +14,8 @@ spec: B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] script: - - ./build.sh -i runner - - cd test && docker compose up - - echo "A $A, B $B, C $C" + - ./build.sh python + - ./run.sh before_script: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) From 7818c06b32774f8369239cfda77d1caf763ac962 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 16:40:59 +0200 Subject: [PATCH 013/229] Test real test pipeline --- .gitlab-ci.yml | 2 + .gitlab/mock.yml | 47 ++++++++----------- .../ci_orchestrators/build_pipeline.py | 24 ++++++++++ .../scripts/ci_orchestrators/system-tests.yml | 16 +++++++ 4 files changed, 62 insertions(+), 27 deletions(-) create mode 100644 utils/scripts/ci_orchestrators/build_pipeline.py create mode 100644 utils/scripts/ci_orchestrators/system-tests.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b820bcf31e3..a6797b4e639 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,4 +3,6 @@ include: inputs: stage: "test" stages: + - build - test + - run diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index e59a3b4e3db..f60f07e228a 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -3,35 +3,28 @@ spec: stage: --- -.load: +build_test_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 tags: - - docker-in-docker:amd64 - stage: $[[ inputs.stage ]] - parallel: - matrix: - - A: [0, 1] - B: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - C: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + - arch:amd64 + stage: build script: - - ./build.sh python - - ./run.sh - before_script: - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - ./build.sh -i runner + - source venv/bin/activate + - pip install Jinja2 + - python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] > generated-pipeline.yml + artifacts: + paths: + - generated-pipeline.yml -load: - extends: .load +run_test_pipeline: + stage: run + needs: + - job: build_test_pipeline + artifacts: true + trigger: + include: + - artifact: generated-pipeline.yml + job: build_test_pipeline + strategy: depend -load1: - extends: .load - -load2: - extends: .load - -load3: - extends: .load - -load4: - extends: .load diff --git a/utils/scripts/ci_orchestrators/build_pipeline.py b/utils/scripts/ci_orchestrators/build_pipeline.py new file mode 100644 index 00000000000..415ad1bab00 --- /dev/null +++ b/utils/scripts/ci_orchestrators/build_pipeline.py @@ -0,0 +1,24 @@ +import argparse +import os + +from jinja2 import Environment, FileSystemLoader, select_autoescape +from utils._context._scenarios import _Scenarios +from utils._context._scenarios.core import Scenario + +parser = argparse.ArgumentParser() +parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") +args = parser.parse_args() + +scenario_list = [] +for var in vars(_Scenarios).values(): + if not isinstance(var, Scenario): + continue + scenario_list.append(var.name) + +scenario_list.sort() + +env = Environment(loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__))), autoescape=select_autoescape()) + +template = env.get_template("system-tests.yml") + +print(template.render(scenarios=scenario_list, stage=args.stage)) diff --git a/utils/scripts/ci_orchestrators/system-tests.yml b/utils/scripts/ci_orchestrators/system-tests.yml new file mode 100644 index 00000000000..ee6a0ded15c --- /dev/null +++ b/utils/scripts/ci_orchestrators/system-tests.yml @@ -0,0 +1,16 @@ +{% for scenario in scenarios %} + +run_{{scenario}}: + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + tags: + - docker-in-docker:amd64 + stage: {{stage}} + script: + - ./build.sh python + - ./run.sh {{scenario}} + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + +{% endfor %} From 0538cc1a6a38a5cd01fb72e0e9986148a353506a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 17:22:33 +0200 Subject: [PATCH 014/229] All variants --- utils/scripts/ci_orchestrators/build_pipeline.py | 9 ++++++++- utils/scripts/ci_orchestrators/system-tests.yml | 8 +++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/utils/scripts/ci_orchestrators/build_pipeline.py b/utils/scripts/ci_orchestrators/build_pipeline.py index 415ad1bab00..cf71d1464f3 100644 --- a/utils/scripts/ci_orchestrators/build_pipeline.py +++ b/utils/scripts/ci_orchestrators/build_pipeline.py @@ -17,8 +17,15 @@ scenario_list.sort() +python_weblog_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../utils/build/docker/python") +python_variants = sorted( + f[: -len(".Dockerfile")] + for f in os.listdir(python_weblog_dir) + if f.endswith(".Dockerfile") and not f.endswith(".base.Dockerfile") +) + env = Environment(loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__))), autoescape=select_autoescape()) template = env.get_template("system-tests.yml") -print(template.render(scenarios=scenario_list, stage=args.stage)) +print(template.render(scenarios=scenario_list, stage=args.stage, python_variants=python_variants)) diff --git a/utils/scripts/ci_orchestrators/system-tests.yml b/utils/scripts/ci_orchestrators/system-tests.yml index ee6a0ded15c..f56dd6d14ba 100644 --- a/utils/scripts/ci_orchestrators/system-tests.yml +++ b/utils/scripts/ci_orchestrators/system-tests.yml @@ -1,10 +1,12 @@ -{% for scenario in scenarios %} +{% for scenario in scenarios %}{% for variant in python_variants %} -run_{{scenario}}: +run_{{scenario}}_{{variant}}: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 tags: - docker-in-docker:amd64 stage: {{stage}} + variables: + WEBLOG_VARIANT: {{variant}} script: - ./build.sh python - ./run.sh {{scenario}} @@ -13,4 +15,4 @@ run_{{scenario}}: - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin -{% endfor %} +{% endfor %}{% endfor %} From 058047f8483648e40b4b029b4fa857490006a864 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 1 Apr 2026 18:36:08 +0200 Subject: [PATCH 015/229] Compute parameters --- .gitlab/mock.yml | 3 +- .../ci_orchestrators/build_pipeline.py | 30 ++++++++++--------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index f60f07e228a..336ed1e1d90 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -12,7 +12,8 @@ build_test_pipeline: - ./build.sh -i runner - source venv/bin/activate - pip install Jinja2 - - python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] > generated-pipeline.yml + - python3 utils/scripts/compute-workflow-parameters.py python --format json -g all --output params.json + - python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] --params params.json > generated-pipeline.yml artifacts: paths: - generated-pipeline.yml diff --git a/utils/scripts/ci_orchestrators/build_pipeline.py b/utils/scripts/ci_orchestrators/build_pipeline.py index cf71d1464f3..d18fd1b2bff 100644 --- a/utils/scripts/ci_orchestrators/build_pipeline.py +++ b/utils/scripts/ci_orchestrators/build_pipeline.py @@ -1,4 +1,5 @@ import argparse +import json import os from jinja2 import Environment, FileSystemLoader, select_autoescape @@ -7,22 +8,23 @@ parser = argparse.ArgumentParser() parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") +parser.add_argument("--params", help="Path to JSON output from compute-workflow-parameters.py") args = parser.parse_args() -scenario_list = [] -for var in vars(_Scenarios).values(): - if not isinstance(var, Scenario): - continue - scenario_list.append(var.name) - -scenario_list.sort() - -python_weblog_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../utils/build/docker/python") -python_variants = sorted( - f[: -len(".Dockerfile")] - for f in os.listdir(python_weblog_dir) - if f.endswith(".Dockerfile") and not f.endswith(".base.Dockerfile") -) +if args.params: + with open(args.params) as f: + params = json.load(f) + scenario_list = params["endtoend"]["scenarios"] + python_variants = params["endtoend"]["weblogs"] +else: + scenario_list = sorted(var.name for var in vars(_Scenarios).values() if isinstance(var, Scenario)) + + python_weblog_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../utils/build/docker/python") + python_variants = sorted( + f[: -len(".Dockerfile")] + for f in os.listdir(python_weblog_dir) + if f.endswith(".Dockerfile") and not f.endswith(".base.Dockerfile") + ) env = Environment(loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__))), autoescape=select_autoescape()) From d0ff6e123f872df41e1f4e9f898b5d40c939aa57 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 2 Apr 2026 10:27:01 +0200 Subject: [PATCH 016/229] debug --- .gitlab/mock.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index 336ed1e1d90..ad961bbc4e7 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -13,6 +13,7 @@ build_test_pipeline: - source venv/bin/activate - pip install Jinja2 - python3 utils/scripts/compute-workflow-parameters.py python --format json -g all --output params.json + - cat params.json - python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] --params params.json > generated-pipeline.yml artifacts: paths: From ca5783821018cbf1da983c77c06ef0c157ae3ccf Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 2 Apr 2026 11:09:43 +0200 Subject: [PATCH 017/229] Java --- .gitlab/mock.yml | 2 +- utils/scripts/ci_orchestrators/system-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index ad961bbc4e7..d1c8c2866a0 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -12,7 +12,7 @@ build_test_pipeline: - ./build.sh -i runner - source venv/bin/activate - pip install Jinja2 - - python3 utils/scripts/compute-workflow-parameters.py python --format json -g all --output params.json + - python3 utils/scripts/compute-workflow-parameters.py java --format json -g all --output params.json - cat params.json - python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] --params params.json > generated-pipeline.yml artifacts: diff --git a/utils/scripts/ci_orchestrators/system-tests.yml b/utils/scripts/ci_orchestrators/system-tests.yml index f56dd6d14ba..42ee845ee32 100644 --- a/utils/scripts/ci_orchestrators/system-tests.yml +++ b/utils/scripts/ci_orchestrators/system-tests.yml @@ -8,7 +8,7 @@ run_{{scenario}}_{{variant}}: variables: WEBLOG_VARIANT: {{variant}} script: - - ./build.sh python + - ./build.sh - ./run.sh {{scenario}} before_script: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) From 0cb88a6eb12e64583003099873a59916af8b28f1 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 2 Apr 2026 11:17:05 +0200 Subject: [PATCH 018/229] Multi library --- .gitlab/mock.yml | 10 +++++++--- utils/scripts/ci_orchestrators/build_pipeline.py | 7 ++++--- utils/scripts/ci_orchestrators/system-tests.yml | 6 +++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index d1c8c2866a0..a1f04bc093e 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -1,6 +1,8 @@ spec: inputs: stage: + libraries: + default: "java python nodejs" --- build_test_pipeline: @@ -12,9 +14,11 @@ build_test_pipeline: - ./build.sh -i runner - source venv/bin/activate - pip install Jinja2 - - python3 utils/scripts/compute-workflow-parameters.py java --format json -g all --output params.json - - cat params.json - - python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] --params params.json > generated-pipeline.yml + - > + for library in $[[ inputs.libraries ]]; do + python3 utils/scripts/compute-workflow-parameters.py $library --format json -g all --output params_${library}.json; + python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; + done artifacts: paths: - generated-pipeline.yml diff --git a/utils/scripts/ci_orchestrators/build_pipeline.py b/utils/scripts/ci_orchestrators/build_pipeline.py index d18fd1b2bff..4a14026d255 100644 --- a/utils/scripts/ci_orchestrators/build_pipeline.py +++ b/utils/scripts/ci_orchestrators/build_pipeline.py @@ -8,6 +8,7 @@ parser = argparse.ArgumentParser() parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") +parser.add_argument("--library", default="", help="Library name, used to prefix job names") parser.add_argument("--params", help="Path to JSON output from compute-workflow-parameters.py") args = parser.parse_args() @@ -15,12 +16,12 @@ with open(args.params) as f: params = json.load(f) scenario_list = params["endtoend"]["scenarios"] - python_variants = params["endtoend"]["weblogs"] + weblog_variants = params["endtoend"]["weblogs"] else: scenario_list = sorted(var.name for var in vars(_Scenarios).values() if isinstance(var, Scenario)) python_weblog_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../utils/build/docker/python") - python_variants = sorted( + weblog_variants = sorted( f[: -len(".Dockerfile")] for f in os.listdir(python_weblog_dir) if f.endswith(".Dockerfile") and not f.endswith(".base.Dockerfile") @@ -30,4 +31,4 @@ template = env.get_template("system-tests.yml") -print(template.render(scenarios=scenario_list, stage=args.stage, python_variants=python_variants)) +print(template.render(scenarios=scenario_list, stage=args.stage, library=args.library, weblog_variants=weblog_variants)) diff --git a/utils/scripts/ci_orchestrators/system-tests.yml b/utils/scripts/ci_orchestrators/system-tests.yml index 42ee845ee32..1092d9981df 100644 --- a/utils/scripts/ci_orchestrators/system-tests.yml +++ b/utils/scripts/ci_orchestrators/system-tests.yml @@ -1,6 +1,6 @@ -{% for scenario in scenarios %}{% for variant in python_variants %} +{% for scenario in scenarios %}{% for variant in weblog_variants %} -run_{{scenario}}_{{variant}}: +run_{{library}}_{{scenario}}_{{variant}}: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 tags: - docker-in-docker:amd64 @@ -8,7 +8,7 @@ run_{{scenario}}_{{variant}}: variables: WEBLOG_VARIANT: {{variant}} script: - - ./build.sh + - ./build.sh {{library}} - ./run.sh {{scenario}} before_script: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) From 2b1087fc934ad2b076973eb91611c21676d9931d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 2 Apr 2026 13:41:08 +0200 Subject: [PATCH 019/229] Remove scenarios that require an API key --- .gitlab/mock.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index a1f04bc093e..cbd1a395d1a 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -15,8 +15,9 @@ build_test_pipeline: - source venv/bin/activate - pip install Jinja2 - > + EXCLUDED_SCENARIOS="APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E"; for library in $[[ inputs.libraries ]]; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json -g all --output params_${library}.json; + python3 utils/scripts/compute-workflow-parameters.py $library --format json -g all --excluded-scenarios $EXCLUDED_SCENARIOS --output params_${library}.json; python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; done artifacts: From 3eed24be9209f5e3f1b58f219ddae2a9cb26e322 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 2 Apr 2026 14:46:58 +0200 Subject: [PATCH 020/229] Excluding scenario needing -L --- .gitlab/mock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/mock.yml b/.gitlab/mock.yml index cbd1a395d1a..26aca40a7d0 100644 --- a/.gitlab/mock.yml +++ b/.gitlab/mock.yml @@ -15,7 +15,7 @@ build_test_pipeline: - source venv/bin/activate - pip install Jinja2 - > - EXCLUDED_SCENARIOS="APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E"; + EXCLUDED_SCENARIOS="APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS"; for library in $[[ inputs.libraries ]]; do python3 utils/scripts/compute-workflow-parameters.py $library --format json -g all --excluded-scenarios $EXCLUDED_SCENARIOS --output params_${library}.json; python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; From 43ef6a62cebaede18c6b308bbf13fb26a309069b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 15:29:54 +0200 Subject: [PATCH 021/229] Moving test pipeline to final dir --- .gitlab-ci.yml | 3 ++- .gitlab/mock.yml => utils/ci/gitlab/main.yml | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename .gitlab/mock.yml => utils/ci/gitlab/main.yml (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a6797b4e639..10a5f21688f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,8 @@ include: - - local: "/.gitlab/mock.yml" + - local: "/utils/ci/gitlab/main.yml" inputs: stage: "test" + libraries: "python" stages: - build - test diff --git a/.gitlab/mock.yml b/utils/ci/gitlab/main.yml similarity index 100% rename from .gitlab/mock.yml rename to utils/ci/gitlab/main.yml From 226937fe3cbd5be699530dd324059acb12088e44 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 16:01:21 +0200 Subject: [PATCH 022/229] Refactor --- utils/ci/gitlab/build_pipeline.py | 23 +++++++++++++ utils/ci/gitlab/main.yml | 2 +- .../gitlab}/system-tests.yml | 0 .../ci_orchestrators/build_pipeline.py | 34 ------------------- 4 files changed, 24 insertions(+), 35 deletions(-) create mode 100644 utils/ci/gitlab/build_pipeline.py rename utils/{scripts/ci_orchestrators => ci/gitlab}/system-tests.yml (100%) delete mode 100644 utils/scripts/ci_orchestrators/build_pipeline.py diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py new file mode 100644 index 00000000000..32904dc9e28 --- /dev/null +++ b/utils/ci/gitlab/build_pipeline.py @@ -0,0 +1,23 @@ +import argparse +import json +import os +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +parser = argparse.ArgumentParser() +parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") +parser.add_argument("--library", required=True, default="", help="Library name, used to prefix job names") +parser.add_argument("--params", required=True, help="Path to JSON output from compute-workflow-parameters.py") +args = parser.parse_args() + +with open(args.params) as f: + params = json.load(f) +scenario_list = params["endtoend"]["scenarios"] +weblog_variants = params["endtoend"]["weblogs"] + +env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) + +template = env.get_template("system-tests.yml") + +print(template.render(scenarios=scenario_list, stage=args.stage, library=args.library, weblog_variants=weblog_variants)) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 26aca40a7d0..a9e9b79f580 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -18,7 +18,7 @@ build_test_pipeline: EXCLUDED_SCENARIOS="APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS"; for library in $[[ inputs.libraries ]]; do python3 utils/scripts/compute-workflow-parameters.py $library --format json -g all --excluded-scenarios $EXCLUDED_SCENARIOS --output params_${library}.json; - python3 utils/scripts/ci_orchestrators/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; done artifacts: paths: diff --git a/utils/scripts/ci_orchestrators/system-tests.yml b/utils/ci/gitlab/system-tests.yml similarity index 100% rename from utils/scripts/ci_orchestrators/system-tests.yml rename to utils/ci/gitlab/system-tests.yml diff --git a/utils/scripts/ci_orchestrators/build_pipeline.py b/utils/scripts/ci_orchestrators/build_pipeline.py deleted file mode 100644 index 4a14026d255..00000000000 --- a/utils/scripts/ci_orchestrators/build_pipeline.py +++ /dev/null @@ -1,34 +0,0 @@ -import argparse -import json -import os - -from jinja2 import Environment, FileSystemLoader, select_autoescape -from utils._context._scenarios import _Scenarios -from utils._context._scenarios.core import Scenario - -parser = argparse.ArgumentParser() -parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") -parser.add_argument("--library", default="", help="Library name, used to prefix job names") -parser.add_argument("--params", help="Path to JSON output from compute-workflow-parameters.py") -args = parser.parse_args() - -if args.params: - with open(args.params) as f: - params = json.load(f) - scenario_list = params["endtoend"]["scenarios"] - weblog_variants = params["endtoend"]["weblogs"] -else: - scenario_list = sorted(var.name for var in vars(_Scenarios).values() if isinstance(var, Scenario)) - - python_weblog_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../../utils/build/docker/python") - weblog_variants = sorted( - f[: -len(".Dockerfile")] - for f in os.listdir(python_weblog_dir) - if f.endswith(".Dockerfile") and not f.endswith(".base.Dockerfile") - ) - -env = Environment(loader=FileSystemLoader(os.path.dirname(os.path.abspath(__file__))), autoescape=select_autoescape()) - -template = env.get_template("system-tests.yml") - -print(template.render(scenarios=scenario_list, stage=args.stage, library=args.library, weblog_variants=weblog_variants)) From b908a594c44491b2a312720de1079d69ced939f2 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 16:54:17 +0200 Subject: [PATCH 023/229] Split build and run into separate CI jobs per variant --- utils/ci/gitlab/system-tests.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 1092d9981df..5ee46e21d84 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -1,10 +1,31 @@ -{% for scenario in scenarios %}{% for variant in weblog_variants %} +{% for variant in weblog_variants %} +build_{{library}}_{{variant}}: + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + tags: + - docker-in-docker:amd64 + stage: {{stage}} + variables: + WEBLOG_VARIANT: {{variant}} + script: + - ./build.sh {{library}} --save-to-binaries + artifacts: + paths: + - binaries/ + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin +{% endfor %} +{% for scenario in scenarios %}{% for variant in weblog_variants %} run_{{library}}_{{scenario}}_{{variant}}: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 tags: - docker-in-docker:amd64 stage: {{stage}} + needs: + - job: build_{{library}}_{{variant}} + artifacts: true variables: WEBLOG_VARIANT: {{variant}} script: From 6cea00fb59e9c6dbb28b8a8215f807a14f9a3f12 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 17:27:37 +0200 Subject: [PATCH 024/229] Add scenarios input to GitLab pipeline --- utils/ci/gitlab/main.yml | 5 ++++- utils/ci/gitlab/plans/01-scenarios.md | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 utils/ci/gitlab/plans/01-scenarios.md diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index a9e9b79f580..429cee5597c 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -3,6 +3,9 @@ spec: stage: libraries: default: "java python nodejs" + scenarios: + description: "Comma-separated list of scenarios to run" + default: "" --- build_test_pipeline: @@ -17,7 +20,7 @@ build_test_pipeline: - > EXCLUDED_SCENARIOS="APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS"; for library in $[[ inputs.libraries ]]; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json -g all --excluded-scenarios $EXCLUDED_SCENARIOS --output params_${library}.json; + python3 utils/scripts/compute-workflow-parameters.py $library --format json -g all --excluded-scenarios $EXCLUDED_SCENARIOS --scenarios "$[[ inputs.scenarios ]]" --output params_${library}.json; python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; done artifacts: diff --git a/utils/ci/gitlab/plans/01-scenarios.md b/utils/ci/gitlab/plans/01-scenarios.md new file mode 100644 index 00000000000..20b9a6b7b96 --- /dev/null +++ b/utils/ci/gitlab/plans/01-scenarios.md @@ -0,0 +1,27 @@ +# Feature: `scenarios` input + +## Context + +Reference files: +- `utils/ci/gitlab/main.yml` — GitLab entry point +- `utils/scripts/compute-workflow-parameters.py` — accepts `--scenarios` + +Currently, no `scenarios` input exists in the GitLab pipeline. The set of scenarios +to run is determined entirely by `scenarios_groups` and `excluded_scenarios`. + +In the GitHub workflow (`system-tests.yml` input `scenarios`), a caller can pass a +comma-separated list of scenario names to run directly, bypassing group-based +selection. Default is `DEFAULT`. + +## Changes + +### `utils/ci/gitlab/main.yml` + +1. Add to `spec.inputs`: + ```yaml + scenarios: + description: "Comma-separated list of scenarios to run" + default: "" + ``` +2. In the `build_test_pipeline` script, add `--scenarios "$[[ inputs.scenarios ]]"` to + the `compute-workflow-parameters.py` invocation. From 4966def951ad23d00f9a920dad7270d6184e7f93 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 17:28:12 +0200 Subject: [PATCH 025/229] Add scenarios_groups input to GitLab pipeline --- utils/ci/gitlab/main.yml | 5 +++- utils/ci/gitlab/plans/02-scenarios-groups.md | 25 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 utils/ci/gitlab/plans/02-scenarios-groups.md diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 429cee5597c..ca2f3c2e8ba 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -6,6 +6,9 @@ spec: scenarios: description: "Comma-separated list of scenarios to run" default: "" + scenarios_groups: + description: "Comma-separated list of scenario groups to run" + default: "all" --- build_test_pipeline: @@ -20,7 +23,7 @@ build_test_pipeline: - > EXCLUDED_SCENARIOS="APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS"; for library in $[[ inputs.libraries ]]; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json -g all --excluded-scenarios $EXCLUDED_SCENARIOS --scenarios "$[[ inputs.scenarios ]]" --output params_${library}.json; + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios $EXCLUDED_SCENARIOS --scenarios "$[[ inputs.scenarios ]]" --output params_${library}.json; python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; done artifacts: diff --git a/utils/ci/gitlab/plans/02-scenarios-groups.md b/utils/ci/gitlab/plans/02-scenarios-groups.md new file mode 100644 index 00000000000..c32f55f6157 --- /dev/null +++ b/utils/ci/gitlab/plans/02-scenarios-groups.md @@ -0,0 +1,25 @@ +# Feature: `scenarios_groups` input + +## Context + +Reference files: +- `utils/ci/gitlab/main.yml` — GitLab entry point +- `utils/scripts/compute-workflow-parameters.py` — accepts `--groups` + +Currently, the `build_test_pipeline` script hardcodes `-g all`, meaning all scenario +groups are always selected. The GitHub workflow exposes this as a `scenarios_groups` +input (comma-separated, default `""`). + +## Changes + +### `utils/ci/gitlab/main.yml` + +1. Add to `spec.inputs`: + ```yaml + scenarios_groups: + description: "Comma-separated list of scenario groups to run" + default: "all" + ``` + Default is `"all"` (not `""`) to preserve current GitLab behaviour. +2. In the `build_test_pipeline` script, replace the hardcoded `-g all` with + `--groups "$[[ inputs.scenarios_groups ]]"`. From e7b71fded918341f01eab7570242dffbe41444f5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 17:29:08 +0200 Subject: [PATCH 026/229] Add excluded_scenarios input to GitLab pipeline --- utils/ci/gitlab/main.yml | 6 ++-- .../ci/gitlab/plans/03-excluded-scenarios.md | 32 +++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 utils/ci/gitlab/plans/03-excluded-scenarios.md diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index ca2f3c2e8ba..ffbe520ab67 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -9,6 +9,9 @@ spec: scenarios_groups: description: "Comma-separated list of scenario groups to run" default: "all" + excluded_scenarios: + description: "Comma-separated list of scenarios not to run" + default: "APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS" --- build_test_pipeline: @@ -21,9 +24,8 @@ build_test_pipeline: - source venv/bin/activate - pip install Jinja2 - > - EXCLUDED_SCENARIOS="APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS"; for library in $[[ inputs.libraries ]]; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios $EXCLUDED_SCENARIOS --scenarios "$[[ inputs.scenarios ]]" --output params_${library}.json; + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --output params_${library}.json; python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; done artifacts: diff --git a/utils/ci/gitlab/plans/03-excluded-scenarios.md b/utils/ci/gitlab/plans/03-excluded-scenarios.md new file mode 100644 index 00000000000..bc3f42c2a7b --- /dev/null +++ b/utils/ci/gitlab/plans/03-excluded-scenarios.md @@ -0,0 +1,32 @@ +# Feature: `excluded_scenarios` input + +## Context + +Reference files: +- `utils/ci/gitlab/main.yml` — GitLab entry point +- `utils/scripts/compute-workflow-parameters.py` — accepts `--excluded-scenarios` + +Currently, the `build_test_pipeline` script hardcodes the following exclusion list +inline in a shell variable: + +``` +APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS +``` + +The GitHub workflow exposes this as an `excluded_scenarios` input (comma-separated, +default `""`). + +## Changes + +### `utils/ci/gitlab/main.yml` + +1. Add to `spec.inputs`: + ```yaml + excluded_scenarios: + description: "Comma-separated list of scenarios not to run" + default: "APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS" + ``` + The default preserves the list currently hardcoded in the script. +2. In the `build_test_pipeline` script, remove the hardcoded `EXCLUDED_SCENARIOS` + shell variable and replace `--excluded-scenarios $EXCLUDED_SCENARIOS` with + `--excluded-scenarios "$[[ inputs.excluded_scenarios ]]"`. From b7ed4ed24a42b20adf2b8916718ca27cb40ca4ed Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 18:38:01 +0200 Subject: [PATCH 027/229] Add weblogs input to GitLab pipeline --- utils/ci/gitlab/main.yml | 5 ++++- utils/ci/gitlab/plans/04-weblogs.md | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 utils/ci/gitlab/plans/04-weblogs.md diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index ffbe520ab67..1bd815ffad4 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -12,6 +12,9 @@ spec: excluded_scenarios: description: "Comma-separated list of scenarios not to run" default: "APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS" + weblogs: + description: "Comma-separated list of weblogs to run (all weblogs if empty)" + default: "" --- build_test_pipeline: @@ -25,7 +28,7 @@ build_test_pipeline: - pip install Jinja2 - > for library in $[[ inputs.libraries ]]; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --output params_${library}.json; + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --output params_${library}.json; python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; done artifacts: diff --git a/utils/ci/gitlab/plans/04-weblogs.md b/utils/ci/gitlab/plans/04-weblogs.md new file mode 100644 index 00000000000..561337ecd44 --- /dev/null +++ b/utils/ci/gitlab/plans/04-weblogs.md @@ -0,0 +1,24 @@ +# Feature: `weblogs` input + +## Context + +Reference files: +- `utils/ci/gitlab/main.yml` — GitLab entry point +- `utils/scripts/compute-workflow-parameters.py` — accepts `--weblogs` + +Currently, no `weblogs` input exists in the GitLab pipeline, so all weblogs are always +run. The GitHub workflow exposes this as a `weblogs` input (comma-separated, default +`""`), allowing callers to restrict the run to a subset of weblogs. + +## Changes + +### `utils/ci/gitlab/main.yml` + +1. Add to `spec.inputs`: + ```yaml + weblogs: + description: "Comma-separated list of weblogs to run (all weblogs if empty)" + default: "" + ``` +2. In the `build_test_pipeline` script, add `--weblogs "$[[ inputs.weblogs ]]"` to + the `compute-workflow-parameters.py` invocation. From 1f0834172fe42065c873c93e75ba49464ce3d83d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 18:39:49 +0200 Subject: [PATCH 028/229] Add binaries_artifact input to GitLab pipeline --- utils/ci/gitlab/build_pipeline.py | 3 +- utils/ci/gitlab/main.yml | 5 +- utils/ci/gitlab/plans/05-binaries-artifact.md | 60 +++++++++++++++++++ utils/ci/gitlab/system-tests.yml | 5 ++ 4 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 utils/ci/gitlab/plans/05-binaries-artifact.md diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 32904dc9e28..1f7617b2079 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -15,9 +15,10 @@ params = json.load(f) scenario_list = params["endtoend"]["scenarios"] weblog_variants = params["endtoend"]["weblogs"] +binaries_artifact = params["miscs"]["binaries_artifact"] env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) template = env.get_template("system-tests.yml") -print(template.render(scenarios=scenario_list, stage=args.stage, library=args.library, weblog_variants=weblog_variants)) +print(template.render(scenarios=scenario_list, stage=args.stage, library=args.library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact)) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 1bd815ffad4..9b443864d11 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -15,6 +15,9 @@ spec: weblogs: description: "Comma-separated list of weblogs to run (all weblogs if empty)" default: "" + binaries_artifact: + description: "Name of the upstream job whose artifacts contain the pre-built binaries to test" + default: "" --- build_test_pipeline: @@ -28,7 +31,7 @@ build_test_pipeline: - pip install Jinja2 - > for library in $[[ inputs.libraries ]]; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --output params_${library}.json; + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --output params_${library}.json; python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; done artifacts: diff --git a/utils/ci/gitlab/plans/05-binaries-artifact.md b/utils/ci/gitlab/plans/05-binaries-artifact.md new file mode 100644 index 00000000000..55de6d6d336 --- /dev/null +++ b/utils/ci/gitlab/plans/05-binaries-artifact.md @@ -0,0 +1,60 @@ +# Feature: `binaries_artifact` input + +## Context + +Reference files: +- `utils/ci/gitlab/main.yml` — GitLab entry point +- `utils/ci/gitlab/build_pipeline.py` — generates child pipeline YAML +- `utils/ci/gitlab/system-tests.yml` — Jinja2 job template +- `utils/scripts/compute-workflow-parameters.py` — accepts `--explicit-binaries-artifact` + +When a caller provides pre-built binaries (e.g. a tracer under test), they upload them +as a named CI artifact and pass the name via `binaries_artifact`. The parameter +computation script records the artifact name in `params["miscs"]["binaries_artifact"]` +and sets `ci_environment` to `"custom"`. Downstream build/run jobs must then download +that artifact before executing. + +Currently this is not wired up in the GitLab pipeline. + +## Changes + +### `utils/ci/gitlab/main.yml` + +1. Add to `spec.inputs`: + ```yaml + binaries_artifact: + description: "Artifact name containing the binaries to test" + default: "" + ``` +2. In the `build_test_pipeline` script, add + `--explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]"` to the + `compute-workflow-parameters.py` invocation. + +### `utils/ci/gitlab/build_pipeline.py` + +3. Pass `binaries_artifact` (from `params["miscs"]["binaries_artifact"]`) to the + Jinja template: + ```python + template.render( + ..., + binaries_artifact=params["miscs"]["binaries_artifact"], + ) + ``` + +### `utils/ci/gitlab/system-tests.yml` + +4. In the build job, add a conditional step to download the artifact when + `binaries_artifact` is non-empty: + ```yaml + {% if binaries_artifact %} + - > + curl --fail -o binaries.zip + "${CI_API_V4_URL}/projects/.../jobs/artifacts/..." + # or use GitLab's `needs: [job: ..., artifacts: true]` if the artifact + # comes from an upstream job in the same pipeline + {% endif %} + ``` + The exact mechanism depends on whether the artifact is produced by a job in the + same pipeline (use `needs: artifacts: true`) or uploaded externally (use the + GitLab artifacts API or a pre-populated `binaries/` directory via a pipeline + variable pointing to a download URL). diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 5ee46e21d84..51909e343de 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -4,6 +4,11 @@ build_{{library}}_{{variant}}: tags: - docker-in-docker:amd64 stage: {{stage}} + {% if binaries_artifact %} + needs: + - job: {{binaries_artifact}} + artifacts: true + {% endif %} variables: WEBLOG_VARIANT: {{variant}} script: From c550d567e9c5cc4872ec6d22b2a428366a24b485 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 18:48:17 +0200 Subject: [PATCH 029/229] Add force_execute input to GitLab pipeline --- utils/ci/gitlab/main.yml | 5 ++++ utils/ci/gitlab/plans/07-force-execute.md | 35 +++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 utils/ci/gitlab/plans/07-force-execute.md diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 9b443864d11..9377d173e33 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -18,6 +18,9 @@ spec: binaries_artifact: description: "Name of the upstream job whose artifacts contain the pre-built binaries to test" default: "" + force_execute: + description: "Comma-separated test node IDs to force-execute" + default: "" --- build_test_pipeline: @@ -43,6 +46,8 @@ run_test_pipeline: needs: - job: build_test_pipeline artifacts: true + variables: + SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" trigger: include: - artifact: generated-pipeline.yml diff --git a/utils/ci/gitlab/plans/07-force-execute.md b/utils/ci/gitlab/plans/07-force-execute.md new file mode 100644 index 00000000000..d1c936e6f81 --- /dev/null +++ b/utils/ci/gitlab/plans/07-force-execute.md @@ -0,0 +1,35 @@ +# Feature: `force_execute` input + +## Context + +Reference files: +- `utils/ci/gitlab/main.yml` — GitLab entry point +- `.github/workflows/run-end-to-end.yml` line 113 — sets `SYSTEM_TESTS_FORCE_EXECUTE` +- `.github/workflows/run-parametric.yml` line 97 — same + +`force_execute` is a comma-separated list of test node IDs that are forced to run +regardless of xfail or skip markers. The run scripts consume it via the +`SYSTEM_TESTS_FORCE_EXECUTE` environment variable. + +## Changes + +### `utils/ci/gitlab/main.yml` + +1. Add to `spec.inputs`: + ```yaml + force_execute: + description: "Comma-separated test node IDs to force-execute" + default: "" + ``` +2. Add a `variables` block to `run_test_pipeline` to forward the value to the child + pipeline: + ```yaml + run_test_pipeline: + variables: + SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" + trigger: + ... + ``` + The existing run job template (`system-tests.yml`) already calls `./run.sh`, which + reads `SYSTEM_TESTS_FORCE_EXECUTE` from the environment, so no template changes are + needed. From 65e3d607810a4931ff593ee45b78539456aba8f2 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 15 May 2026 18:48:43 +0200 Subject: [PATCH 030/229] Add skip_empty_scenarios input to GitLab pipeline --- utils/ci/gitlab/main.yml | 5 +++ .../gitlab/plans/08-skip-empty-scenarios.md | 32 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 utils/ci/gitlab/plans/08-skip-empty-scenarios.md diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 9377d173e33..6e37c406a9f 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -21,6 +21,10 @@ spec: force_execute: description: "Comma-separated test node IDs to force-execute" default: "" + skip_empty_scenarios: + description: "Skip scenarios that contain only xfail or irrelevant tests" + type: boolean + default: false --- build_test_pipeline: @@ -48,6 +52,7 @@ run_test_pipeline: artifacts: true variables: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" + SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" trigger: include: - artifact: generated-pipeline.yml diff --git a/utils/ci/gitlab/plans/08-skip-empty-scenarios.md b/utils/ci/gitlab/plans/08-skip-empty-scenarios.md new file mode 100644 index 00000000000..bf3717b7629 --- /dev/null +++ b/utils/ci/gitlab/plans/08-skip-empty-scenarios.md @@ -0,0 +1,32 @@ +# Feature: `skip_empty_scenarios` input + +## Context + +Reference files: +- `utils/ci/gitlab/main.yml` — GitLab entry point +- `.github/workflows/run-end-to-end.yml` line 112 — sets `SYSTEM_TESTS_SKIP_EMPTY_SCENARIO` + +When `true`, scenarios whose tests are all xfail or irrelevant are skipped entirely. +The run scripts consume this via the `SYSTEM_TESTS_SKIP_EMPTY_SCENARIO` environment +variable. + +## Changes + +### `utils/ci/gitlab/main.yml` + +1. Add to `spec.inputs`: + ```yaml + skip_empty_scenarios: + description: "Skip scenarios that contain only xfail or irrelevant tests" + type: boolean + default: false + ``` +2. Add to the `variables` block of `run_test_pipeline`: + ```yaml + run_test_pipeline: + variables: + SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" + trigger: + ... + ``` + No template changes are needed; `./run.sh` already reads this variable. From 15647629b1e5d3d3aea763d16b29ba0e3d5ed044 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 11:40:21 +0200 Subject: [PATCH 031/229] Add parametric_job_count input to GitLab pipeline --- utils/ci/gitlab/build_pipeline.py | 3 +- utils/ci/gitlab/main.yml | 6 +- .../gitlab/plans/06-parametric-job-count.md | 64 +++++++++++++++++++ utils/ci/gitlab/system-tests.yml | 23 +++++++ 4 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 utils/ci/gitlab/plans/06-parametric-job-count.md diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 1f7617b2079..0ed92f47375 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -16,9 +16,10 @@ scenario_list = params["endtoend"]["scenarios"] weblog_variants = params["endtoend"]["weblogs"] binaries_artifact = params["miscs"]["binaries_artifact"] +parametric = params["parametric"] env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) template = env.get_template("system-tests.yml") -print(template.render(scenarios=scenario_list, stage=args.stage, library=args.library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact)) +print(template.render(scenarios=scenario_list, stage=args.stage, library=args.library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact, parametric=parametric)) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 6e37c406a9f..871909742be 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -25,6 +25,10 @@ spec: description: "Skip scenarios that contain only xfail or irrelevant tests" type: boolean default: false + parametric_job_count: + description: "Number of parallel jobs for the PARAMETRIC scenario" + type: number + default: 1 --- build_test_pipeline: @@ -38,7 +42,7 @@ build_test_pipeline: - pip install Jinja2 - > for library in $[[ inputs.libraries ]]; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --output params_${library}.json; + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; done artifacts: diff --git a/utils/ci/gitlab/plans/06-parametric-job-count.md b/utils/ci/gitlab/plans/06-parametric-job-count.md new file mode 100644 index 00000000000..6ef0578a021 --- /dev/null +++ b/utils/ci/gitlab/plans/06-parametric-job-count.md @@ -0,0 +1,64 @@ +# Feature: `parametric_job_count` input + +## Context + +Reference files: +- `utils/ci/gitlab/main.yml` — GitLab entry point +- `utils/ci/gitlab/build_pipeline.py` — generates child pipeline YAML +- `utils/ci/gitlab/system-tests.yml` — Jinja2 job template +- `utils/scripts/compute-workflow-parameters.py` — accepts `--parametric-job-count`; + produces `params["parametric"]["job_count"]`, `params["parametric"]["job_matrix"]`, + and `params["parametric"]["enable"]` + +The `PARAMETRIC` scenario can be split across N parallel jobs to reduce wall-clock +time. In the GitHub workflow this is controlled by `parametric_job_count` (default 1). +Currently the GitLab pipeline does not generate parametric jobs at all. + +## Changes + +### `utils/ci/gitlab/main.yml` + +1. Add to `spec.inputs`: + ```yaml + parametric_job_count: + description: "Number of parallel jobs for the PARAMETRIC scenario" + type: number + default: 1 + ``` +2. In the `build_test_pipeline` script, add + `--parametric-job-count $[[ inputs.parametric_job_count ]]` to the + `compute-workflow-parameters.py` invocation. + +### `utils/ci/gitlab/build_pipeline.py` + +3. Read the parametric section from params and pass it to the template: + ```python + template.render( + ..., + parametric=params["parametric"], + ) + ``` + +### `utils/ci/gitlab/system-tests.yml` + +4. Add a parametric block to the template that renders one job per matrix entry when + `parametric["enable"]` is true: + ```yaml + {% if parametric.enable %} + {% for job_index in parametric.job_matrix %} + run_{{ library }}_PARAMETRIC_{{ job_index }}: + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + tags: + - docker-in-docker:amd64 + stage: {{ stage }} + variables: + SYSTEM_TESTS_PARAMETRIC_JOB_COUNT: "{{ parametric.job_count }}" + SYSTEM_TESTS_PARAMETRIC_JOB_INDEX: "{{ job_index }}" + script: + - ./build.sh {{ library }} -i runner + - ./run.sh PARAMETRIC + before_script: + - ... # same docker login as other jobs + {% endfor %} + {% endif %} + ``` diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 51909e343de..a834bcd84e4 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -42,3 +42,26 @@ run_{{library}}_{{scenario}}_{{variant}}: - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin {% endfor %}{% endfor %} +{% if parametric.enable %} +{% for job_index in parametric.job_matrix %} +run_{{library}}_PARAMETRIC_{{job_index}}: + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + tags: + - docker-in-docker:amd64 + stage: {{stage}} + {% if binaries_artifact %} + needs: + - job: {{binaries_artifact}} + artifacts: true + {% endif %} + script: + - ./build.sh -i runner + - source venv/bin/activate + - ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + +{% endfor %} +{% endif %} From 247c5a2b16efdeda735de996efa525e1b714a0ca Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 11:53:17 +0200 Subject: [PATCH 032/229] Add push_to_test_optimization input to GitLab pipeline --- utils/ci/gitlab/build_pipeline.py | 13 ++- utils/ci/gitlab/main.yml | 9 ++- .../plans/10-push-to-test-optimization.md | 79 +++++++++++++++++++ utils/ci/gitlab/system-tests.yml | 35 ++++++++ 4 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 utils/ci/gitlab/plans/10-push-to-test-optimization.md diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 0ed92f47375..e58b2f18ea8 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -9,6 +9,8 @@ parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") parser.add_argument("--library", required=True, default="", help="Library name, used to prefix job names") parser.add_argument("--params", required=True, help="Path to JSON output from compute-workflow-parameters.py") +parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") +parser.add_argument("--test-optimization-datadog-site", default="datadoghq.com", help="Datadog site for Test Optimization") args = parser.parse_args() with open(args.params) as f: @@ -22,4 +24,13 @@ template = env.get_template("system-tests.yml") -print(template.render(scenarios=scenario_list, stage=args.stage, library=args.library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact, parametric=parametric)) +print(template.render( + scenarios=scenario_list, + stage=args.stage, + library=args.library, + weblog_variants=weblog_variants, + binaries_artifact=binaries_artifact, + parametric=parametric, + push_to_test_optimization=args.push_to_test_optimization == "true", + test_optimization_datadog_site=args.test_optimization_datadog_site, +)) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 871909742be..12fc0251eae 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -29,6 +29,13 @@ spec: description: "Number of parallel jobs for the PARAMETRIC scenario" type: number default: 1 + push_to_test_optimization: + description: "Push test results to Datadog Test Optimization (requires TEST_OPTIMIZATION_API_KEY CI variable)" + type: boolean + default: false + test_optimization_datadog_site: + description: "Datadog site to use for Test Optimization" + default: "datadoghq.com" --- build_test_pipeline: @@ -43,7 +50,7 @@ build_test_pipeline: - > for library in $[[ inputs.libraries ]]; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json >> generated-pipeline.yml; + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" >> generated-pipeline.yml; done artifacts: paths: diff --git a/utils/ci/gitlab/plans/10-push-to-test-optimization.md b/utils/ci/gitlab/plans/10-push-to-test-optimization.md new file mode 100644 index 00000000000..7c69f21b208 --- /dev/null +++ b/utils/ci/gitlab/plans/10-push-to-test-optimization.md @@ -0,0 +1,79 @@ +# Feature: `push_to_test_optimization` + `test_optimization_datadog_site` inputs + +## Context + +Reference files: +- `utils/ci/gitlab/main.yml` — GitLab entry point +- `utils/ci/gitlab/system-tests.yml` — Jinja2 job template +- `.github/actions/push_to_test_optim/action.yml` — GitHub equivalent +- `.github/workflows/run-end-to-end.yml` lines 546-550 + +After all run jobs complete, the GitHub workflow runs `datadog-ci junit upload` to +push `reportJunit.xml` files to Datadog Test Optimization, tagged with the service, +environment, and codeowners. It requires a `TEST_OPTIMIZATION_API_KEY` secret and an +optional `DATADOG_SITE` setting. + +## Changes + +### `utils/ci/gitlab/main.yml` + +1. Add to `spec.inputs`: + ```yaml + push_to_test_optimization: + description: "Push test results to Datadog Test Optimization (requires TEST_OPTIMIZATION_API_KEY CI variable)" + type: boolean + default: false + test_optimization_datadog_site: + description: "Datadog site to use for Test Optimization" + default: "datadoghq.com" + ``` + +2. Add a new `push_test_optimization` job: + ```yaml + push_test_optimization: + image: node:lts-slim + stage: report # new stage after `run`, see note below + rules: + - if: '$[[ inputs.push_to_test_optimization ]] == "true"' + needs: + - job: run_test_pipeline + artifacts: true + script: + - npm install -g @datadog/datadog-ci + - | + datadog-ci junit upload logs*/reportJunit.xml \ + --service system-tests \ + --env ci \ + --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" + variables: + DATADOG_SITE: "$[[ inputs.test_optimization_datadog_site ]]" + DATADOG_API_KEY: "$TEST_OPTIMIZATION_API_KEY" + ``` + +### `utils/ci/gitlab/system-tests.yml` + +3. Run jobs must export their JUnit XML so the `push_test_optimization` job can + access them. Add to the run job template: + ```yaml + artifacts: + when: always + paths: + - logs/reportJunit.xml + ``` + (This can be merged with the `artifacts: reports: junit` block introduced by the + `display_summary` feature — plan `09-display-summary.md`.) + +### GitLab project/group settings + +4. Add `TEST_OPTIMIZATION_API_KEY` as a masked CI/CD variable in the GitLab + project or group settings. This is the equivalent of the GitHub + `TEST_OPTIMIZATION_API_KEY` secret. + +## Notes + +- The `report` stage must be declared in whatever top-level pipeline configuration + defines the stages list, after the `run` stage. +- The GitHub action also attempts to get a short-lived API key via `dd-sts-action` + as a fallback. This can be added later via a `before_script` that calls the + appropriate AWS SSM parameter (consistent with the existing `before_script` pattern + already used in the GitLab jobs). diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index a834bcd84e4..b18d990d036 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -36,6 +36,10 @@ run_{{library}}_{{scenario}}_{{variant}}: script: - ./build.sh {{library}} - ./run.sh {{scenario}} + artifacts: + when: always + paths: + - logs*/reportJunit.xml before_script: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) @@ -58,6 +62,10 @@ run_{{library}}_PARAMETRIC_{{job_index}}: - ./build.sh -i runner - source venv/bin/activate - ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} + artifacts: + when: always + paths: + - logs*/reportJunit.xml before_script: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) @@ -65,3 +73,30 @@ run_{{library}}_PARAMETRIC_{{job_index}}: {% endfor %} {% endif %} +{% if push_to_test_optimization %} +push_{{library}}_test_optimization: + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + tags: + - docker-in-docker:amd64 + stage: {{stage}} + when: always + needs: + {% for scenario in scenarios %}{% for variant in weblog_variants %} + - job: run_{{library}}_{{scenario}}_{{variant}} + artifacts: true + {% endfor %}{% endfor %} + {% if parametric.enable %}{% for job_index in parametric.job_matrix %} + - job: run_{{library}}_PARAMETRIC_{{job_index}} + artifacts: true + {% endfor %}{% endif %} + script: + - npm install -g @datadog/datadog-ci + - > + datadog-ci junit upload logs*/reportJunit.xml + --service system-tests + --env ci + --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" + variables: + DATADOG_SITE: {{test_optimization_datadog_site}} + DATADOG_API_KEY: "$TEST_OPTIMIZATION_API_KEY" +{% endif %} From c91d50b0d215c48d39a5c86fa8742b9e845d8a28 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 12:01:55 +0200 Subject: [PATCH 033/229] Remove implementation plans --- utils/ci/gitlab/plans/01-scenarios.md | 27 ------- utils/ci/gitlab/plans/02-scenarios-groups.md | 25 ------ .../ci/gitlab/plans/03-excluded-scenarios.md | 32 -------- utils/ci/gitlab/plans/04-weblogs.md | 24 ------ utils/ci/gitlab/plans/05-binaries-artifact.md | 60 -------------- .../gitlab/plans/06-parametric-job-count.md | 64 --------------- utils/ci/gitlab/plans/07-force-execute.md | 35 -------- .../gitlab/plans/08-skip-empty-scenarios.md | 32 -------- .../plans/10-push-to-test-optimization.md | 79 ------------------- 9 files changed, 378 deletions(-) delete mode 100644 utils/ci/gitlab/plans/01-scenarios.md delete mode 100644 utils/ci/gitlab/plans/02-scenarios-groups.md delete mode 100644 utils/ci/gitlab/plans/03-excluded-scenarios.md delete mode 100644 utils/ci/gitlab/plans/04-weblogs.md delete mode 100644 utils/ci/gitlab/plans/05-binaries-artifact.md delete mode 100644 utils/ci/gitlab/plans/06-parametric-job-count.md delete mode 100644 utils/ci/gitlab/plans/07-force-execute.md delete mode 100644 utils/ci/gitlab/plans/08-skip-empty-scenarios.md delete mode 100644 utils/ci/gitlab/plans/10-push-to-test-optimization.md diff --git a/utils/ci/gitlab/plans/01-scenarios.md b/utils/ci/gitlab/plans/01-scenarios.md deleted file mode 100644 index 20b9a6b7b96..00000000000 --- a/utils/ci/gitlab/plans/01-scenarios.md +++ /dev/null @@ -1,27 +0,0 @@ -# Feature: `scenarios` input - -## Context - -Reference files: -- `utils/ci/gitlab/main.yml` — GitLab entry point -- `utils/scripts/compute-workflow-parameters.py` — accepts `--scenarios` - -Currently, no `scenarios` input exists in the GitLab pipeline. The set of scenarios -to run is determined entirely by `scenarios_groups` and `excluded_scenarios`. - -In the GitHub workflow (`system-tests.yml` input `scenarios`), a caller can pass a -comma-separated list of scenario names to run directly, bypassing group-based -selection. Default is `DEFAULT`. - -## Changes - -### `utils/ci/gitlab/main.yml` - -1. Add to `spec.inputs`: - ```yaml - scenarios: - description: "Comma-separated list of scenarios to run" - default: "" - ``` -2. In the `build_test_pipeline` script, add `--scenarios "$[[ inputs.scenarios ]]"` to - the `compute-workflow-parameters.py` invocation. diff --git a/utils/ci/gitlab/plans/02-scenarios-groups.md b/utils/ci/gitlab/plans/02-scenarios-groups.md deleted file mode 100644 index c32f55f6157..00000000000 --- a/utils/ci/gitlab/plans/02-scenarios-groups.md +++ /dev/null @@ -1,25 +0,0 @@ -# Feature: `scenarios_groups` input - -## Context - -Reference files: -- `utils/ci/gitlab/main.yml` — GitLab entry point -- `utils/scripts/compute-workflow-parameters.py` — accepts `--groups` - -Currently, the `build_test_pipeline` script hardcodes `-g all`, meaning all scenario -groups are always selected. The GitHub workflow exposes this as a `scenarios_groups` -input (comma-separated, default `""`). - -## Changes - -### `utils/ci/gitlab/main.yml` - -1. Add to `spec.inputs`: - ```yaml - scenarios_groups: - description: "Comma-separated list of scenario groups to run" - default: "all" - ``` - Default is `"all"` (not `""`) to preserve current GitLab behaviour. -2. In the `build_test_pipeline` script, replace the hardcoded `-g all` with - `--groups "$[[ inputs.scenarios_groups ]]"`. diff --git a/utils/ci/gitlab/plans/03-excluded-scenarios.md b/utils/ci/gitlab/plans/03-excluded-scenarios.md deleted file mode 100644 index bc3f42c2a7b..00000000000 --- a/utils/ci/gitlab/plans/03-excluded-scenarios.md +++ /dev/null @@ -1,32 +0,0 @@ -# Feature: `excluded_scenarios` input - -## Context - -Reference files: -- `utils/ci/gitlab/main.yml` — GitLab entry point -- `utils/scripts/compute-workflow-parameters.py` — accepts `--excluded-scenarios` - -Currently, the `build_test_pipeline` script hardcodes the following exclusion list -inline in a shell variable: - -``` -APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS -``` - -The GitHub workflow exposes this as an `excluded_scenarios` input (comma-separated, -default `""`). - -## Changes - -### `utils/ci/gitlab/main.yml` - -1. Add to `spec.inputs`: - ```yaml - excluded_scenarios: - description: "Comma-separated list of scenarios not to run" - default: "APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS" - ``` - The default preserves the list currently hardcoded in the script. -2. In the `build_test_pipeline` script, remove the hardcoded `EXCLUDED_SCENARIOS` - shell variable and replace `--excluded-scenarios $EXCLUDED_SCENARIOS` with - `--excluded-scenarios "$[[ inputs.excluded_scenarios ]]"`. diff --git a/utils/ci/gitlab/plans/04-weblogs.md b/utils/ci/gitlab/plans/04-weblogs.md deleted file mode 100644 index 561337ecd44..00000000000 --- a/utils/ci/gitlab/plans/04-weblogs.md +++ /dev/null @@ -1,24 +0,0 @@ -# Feature: `weblogs` input - -## Context - -Reference files: -- `utils/ci/gitlab/main.yml` — GitLab entry point -- `utils/scripts/compute-workflow-parameters.py` — accepts `--weblogs` - -Currently, no `weblogs` input exists in the GitLab pipeline, so all weblogs are always -run. The GitHub workflow exposes this as a `weblogs` input (comma-separated, default -`""`), allowing callers to restrict the run to a subset of weblogs. - -## Changes - -### `utils/ci/gitlab/main.yml` - -1. Add to `spec.inputs`: - ```yaml - weblogs: - description: "Comma-separated list of weblogs to run (all weblogs if empty)" - default: "" - ``` -2. In the `build_test_pipeline` script, add `--weblogs "$[[ inputs.weblogs ]]"` to - the `compute-workflow-parameters.py` invocation. diff --git a/utils/ci/gitlab/plans/05-binaries-artifact.md b/utils/ci/gitlab/plans/05-binaries-artifact.md deleted file mode 100644 index 55de6d6d336..00000000000 --- a/utils/ci/gitlab/plans/05-binaries-artifact.md +++ /dev/null @@ -1,60 +0,0 @@ -# Feature: `binaries_artifact` input - -## Context - -Reference files: -- `utils/ci/gitlab/main.yml` — GitLab entry point -- `utils/ci/gitlab/build_pipeline.py` — generates child pipeline YAML -- `utils/ci/gitlab/system-tests.yml` — Jinja2 job template -- `utils/scripts/compute-workflow-parameters.py` — accepts `--explicit-binaries-artifact` - -When a caller provides pre-built binaries (e.g. a tracer under test), they upload them -as a named CI artifact and pass the name via `binaries_artifact`. The parameter -computation script records the artifact name in `params["miscs"]["binaries_artifact"]` -and sets `ci_environment` to `"custom"`. Downstream build/run jobs must then download -that artifact before executing. - -Currently this is not wired up in the GitLab pipeline. - -## Changes - -### `utils/ci/gitlab/main.yml` - -1. Add to `spec.inputs`: - ```yaml - binaries_artifact: - description: "Artifact name containing the binaries to test" - default: "" - ``` -2. In the `build_test_pipeline` script, add - `--explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]"` to the - `compute-workflow-parameters.py` invocation. - -### `utils/ci/gitlab/build_pipeline.py` - -3. Pass `binaries_artifact` (from `params["miscs"]["binaries_artifact"]`) to the - Jinja template: - ```python - template.render( - ..., - binaries_artifact=params["miscs"]["binaries_artifact"], - ) - ``` - -### `utils/ci/gitlab/system-tests.yml` - -4. In the build job, add a conditional step to download the artifact when - `binaries_artifact` is non-empty: - ```yaml - {% if binaries_artifact %} - - > - curl --fail -o binaries.zip - "${CI_API_V4_URL}/projects/.../jobs/artifacts/..." - # or use GitLab's `needs: [job: ..., artifacts: true]` if the artifact - # comes from an upstream job in the same pipeline - {% endif %} - ``` - The exact mechanism depends on whether the artifact is produced by a job in the - same pipeline (use `needs: artifacts: true`) or uploaded externally (use the - GitLab artifacts API or a pre-populated `binaries/` directory via a pipeline - variable pointing to a download URL). diff --git a/utils/ci/gitlab/plans/06-parametric-job-count.md b/utils/ci/gitlab/plans/06-parametric-job-count.md deleted file mode 100644 index 6ef0578a021..00000000000 --- a/utils/ci/gitlab/plans/06-parametric-job-count.md +++ /dev/null @@ -1,64 +0,0 @@ -# Feature: `parametric_job_count` input - -## Context - -Reference files: -- `utils/ci/gitlab/main.yml` — GitLab entry point -- `utils/ci/gitlab/build_pipeline.py` — generates child pipeline YAML -- `utils/ci/gitlab/system-tests.yml` — Jinja2 job template -- `utils/scripts/compute-workflow-parameters.py` — accepts `--parametric-job-count`; - produces `params["parametric"]["job_count"]`, `params["parametric"]["job_matrix"]`, - and `params["parametric"]["enable"]` - -The `PARAMETRIC` scenario can be split across N parallel jobs to reduce wall-clock -time. In the GitHub workflow this is controlled by `parametric_job_count` (default 1). -Currently the GitLab pipeline does not generate parametric jobs at all. - -## Changes - -### `utils/ci/gitlab/main.yml` - -1. Add to `spec.inputs`: - ```yaml - parametric_job_count: - description: "Number of parallel jobs for the PARAMETRIC scenario" - type: number - default: 1 - ``` -2. In the `build_test_pipeline` script, add - `--parametric-job-count $[[ inputs.parametric_job_count ]]` to the - `compute-workflow-parameters.py` invocation. - -### `utils/ci/gitlab/build_pipeline.py` - -3. Read the parametric section from params and pass it to the template: - ```python - template.render( - ..., - parametric=params["parametric"], - ) - ``` - -### `utils/ci/gitlab/system-tests.yml` - -4. Add a parametric block to the template that renders one job per matrix entry when - `parametric["enable"]` is true: - ```yaml - {% if parametric.enable %} - {% for job_index in parametric.job_matrix %} - run_{{ library }}_PARAMETRIC_{{ job_index }}: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 - tags: - - docker-in-docker:amd64 - stage: {{ stage }} - variables: - SYSTEM_TESTS_PARAMETRIC_JOB_COUNT: "{{ parametric.job_count }}" - SYSTEM_TESTS_PARAMETRIC_JOB_INDEX: "{{ job_index }}" - script: - - ./build.sh {{ library }} -i runner - - ./run.sh PARAMETRIC - before_script: - - ... # same docker login as other jobs - {% endfor %} - {% endif %} - ``` diff --git a/utils/ci/gitlab/plans/07-force-execute.md b/utils/ci/gitlab/plans/07-force-execute.md deleted file mode 100644 index d1c936e6f81..00000000000 --- a/utils/ci/gitlab/plans/07-force-execute.md +++ /dev/null @@ -1,35 +0,0 @@ -# Feature: `force_execute` input - -## Context - -Reference files: -- `utils/ci/gitlab/main.yml` — GitLab entry point -- `.github/workflows/run-end-to-end.yml` line 113 — sets `SYSTEM_TESTS_FORCE_EXECUTE` -- `.github/workflows/run-parametric.yml` line 97 — same - -`force_execute` is a comma-separated list of test node IDs that are forced to run -regardless of xfail or skip markers. The run scripts consume it via the -`SYSTEM_TESTS_FORCE_EXECUTE` environment variable. - -## Changes - -### `utils/ci/gitlab/main.yml` - -1. Add to `spec.inputs`: - ```yaml - force_execute: - description: "Comma-separated test node IDs to force-execute" - default: "" - ``` -2. Add a `variables` block to `run_test_pipeline` to forward the value to the child - pipeline: - ```yaml - run_test_pipeline: - variables: - SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" - trigger: - ... - ``` - The existing run job template (`system-tests.yml`) already calls `./run.sh`, which - reads `SYSTEM_TESTS_FORCE_EXECUTE` from the environment, so no template changes are - needed. diff --git a/utils/ci/gitlab/plans/08-skip-empty-scenarios.md b/utils/ci/gitlab/plans/08-skip-empty-scenarios.md deleted file mode 100644 index bf3717b7629..00000000000 --- a/utils/ci/gitlab/plans/08-skip-empty-scenarios.md +++ /dev/null @@ -1,32 +0,0 @@ -# Feature: `skip_empty_scenarios` input - -## Context - -Reference files: -- `utils/ci/gitlab/main.yml` — GitLab entry point -- `.github/workflows/run-end-to-end.yml` line 112 — sets `SYSTEM_TESTS_SKIP_EMPTY_SCENARIO` - -When `true`, scenarios whose tests are all xfail or irrelevant are skipped entirely. -The run scripts consume this via the `SYSTEM_TESTS_SKIP_EMPTY_SCENARIO` environment -variable. - -## Changes - -### `utils/ci/gitlab/main.yml` - -1. Add to `spec.inputs`: - ```yaml - skip_empty_scenarios: - description: "Skip scenarios that contain only xfail or irrelevant tests" - type: boolean - default: false - ``` -2. Add to the `variables` block of `run_test_pipeline`: - ```yaml - run_test_pipeline: - variables: - SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" - trigger: - ... - ``` - No template changes are needed; `./run.sh` already reads this variable. diff --git a/utils/ci/gitlab/plans/10-push-to-test-optimization.md b/utils/ci/gitlab/plans/10-push-to-test-optimization.md deleted file mode 100644 index 7c69f21b208..00000000000 --- a/utils/ci/gitlab/plans/10-push-to-test-optimization.md +++ /dev/null @@ -1,79 +0,0 @@ -# Feature: `push_to_test_optimization` + `test_optimization_datadog_site` inputs - -## Context - -Reference files: -- `utils/ci/gitlab/main.yml` — GitLab entry point -- `utils/ci/gitlab/system-tests.yml` — Jinja2 job template -- `.github/actions/push_to_test_optim/action.yml` — GitHub equivalent -- `.github/workflows/run-end-to-end.yml` lines 546-550 - -After all run jobs complete, the GitHub workflow runs `datadog-ci junit upload` to -push `reportJunit.xml` files to Datadog Test Optimization, tagged with the service, -environment, and codeowners. It requires a `TEST_OPTIMIZATION_API_KEY` secret and an -optional `DATADOG_SITE` setting. - -## Changes - -### `utils/ci/gitlab/main.yml` - -1. Add to `spec.inputs`: - ```yaml - push_to_test_optimization: - description: "Push test results to Datadog Test Optimization (requires TEST_OPTIMIZATION_API_KEY CI variable)" - type: boolean - default: false - test_optimization_datadog_site: - description: "Datadog site to use for Test Optimization" - default: "datadoghq.com" - ``` - -2. Add a new `push_test_optimization` job: - ```yaml - push_test_optimization: - image: node:lts-slim - stage: report # new stage after `run`, see note below - rules: - - if: '$[[ inputs.push_to_test_optimization ]] == "true"' - needs: - - job: run_test_pipeline - artifacts: true - script: - - npm install -g @datadog/datadog-ci - - | - datadog-ci junit upload logs*/reportJunit.xml \ - --service system-tests \ - --env ci \ - --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" - variables: - DATADOG_SITE: "$[[ inputs.test_optimization_datadog_site ]]" - DATADOG_API_KEY: "$TEST_OPTIMIZATION_API_KEY" - ``` - -### `utils/ci/gitlab/system-tests.yml` - -3. Run jobs must export their JUnit XML so the `push_test_optimization` job can - access them. Add to the run job template: - ```yaml - artifacts: - when: always - paths: - - logs/reportJunit.xml - ``` - (This can be merged with the `artifacts: reports: junit` block introduced by the - `display_summary` feature — plan `09-display-summary.md`.) - -### GitLab project/group settings - -4. Add `TEST_OPTIMIZATION_API_KEY` as a masked CI/CD variable in the GitLab - project or group settings. This is the equivalent of the GitHub - `TEST_OPTIMIZATION_API_KEY` secret. - -## Notes - -- The `report` stage must be declared in whatever top-level pipeline configuration - defines the stages list, after the `run` stage. -- The GitHub action also attempts to get a short-lived API key via `dd-sts-action` - as a fallback. This can be added later via a `before_script` that calls the - appropriate AWS SSM parameter (consistent with the existing `before_script` pattern - already used in the GitLab jobs). From 4bf58f7d3a1d6a67f87629bfd86c9fe04ada57bc Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 13:27:14 +0200 Subject: [PATCH 034/229] Add log sections for build and run steps in generated jobs --- utils/ci/gitlab/system-tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index b18d990d036..4feef321aac 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -12,7 +12,9 @@ build_{{library}}_{{variant}}: variables: WEBLOG_VARIANT: {{variant}} script: + - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - ./build.sh {{library}} --save-to-binaries + - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" artifacts: paths: - binaries/ @@ -34,8 +36,12 @@ run_{{library}}_{{scenario}}_{{variant}}: variables: WEBLOG_VARIANT: {{variant}} script: + - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - ./build.sh {{library}} + - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" + - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning {{scenario}}" - ./run.sh {{scenario}} + - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" artifacts: when: always paths: @@ -59,9 +65,13 @@ run_{{library}}_PARAMETRIC_{{job_index}}: artifacts: true {% endif %} script: + - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding runner" - ./build.sh -i runner + - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - source venv/bin/activate + - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" - ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} + - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" artifacts: when: always paths: From 9620625d463b9964f914262864a99188eba98d00 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 13:36:18 +0200 Subject: [PATCH 035/229] Add Datadog custom spans to build and run steps --- utils/ci/gitlab/system-tests.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 4feef321aac..4ebcbc575c4 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -13,12 +13,13 @@ build_{{library}}_{{variant}}: WEBLOG_VARIANT: {{variant}} script: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - - ./build.sh {{library}} --save-to-binaries + - datadog-ci trace --name "Build {{library}} weblog {{variant}}" --no-fail -- ./build.sh {{library}} --save-to-binaries - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" artifacts: paths: - binaries/ before_script: + - npm install -g @datadog/datadog-ci - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin @@ -37,16 +38,17 @@ run_{{library}}_{{scenario}}_{{variant}}: WEBLOG_VARIANT: {{variant}} script: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - - ./build.sh {{library}} + - datadog-ci trace --name "Build {{library}} weblog {{variant}}" --no-fail -- ./build.sh {{library}} - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning {{scenario}}" - - ./run.sh {{scenario}} + - datadog-ci trace --name "Run {{scenario}}" --no-fail -- ./run.sh {{scenario}} - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" artifacts: when: always paths: - logs*/reportJunit.xml before_script: + - npm install -g @datadog/datadog-ci - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin @@ -66,17 +68,18 @@ run_{{library}}_PARAMETRIC_{{job_index}}: {% endif %} script: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding runner" - - ./build.sh -i runner + - datadog-ci trace --name "Build runner" --no-fail -- ./build.sh -i runner - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - source venv/bin/activate - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" - - ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} + - datadog-ci trace --name "Run PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" --no-fail -- ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" artifacts: when: always paths: - logs*/reportJunit.xml before_script: + - npm install -g @datadog/datadog-ci - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin From 61395cb3806b40d6ea4ad8393c3426b2be375117 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 13:47:55 +0200 Subject: [PATCH 036/229] Revert "Add Datadog custom spans to build and run steps" This reverts commit 9620625d463b9964f914262864a99188eba98d00. --- utils/ci/gitlab/system-tests.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 4ebcbc575c4..4feef321aac 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -13,13 +13,12 @@ build_{{library}}_{{variant}}: WEBLOG_VARIANT: {{variant}} script: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - - datadog-ci trace --name "Build {{library}} weblog {{variant}}" --no-fail -- ./build.sh {{library}} --save-to-binaries + - ./build.sh {{library}} --save-to-binaries - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" artifacts: paths: - binaries/ before_script: - - npm install -g @datadog/datadog-ci - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin @@ -38,17 +37,16 @@ run_{{library}}_{{scenario}}_{{variant}}: WEBLOG_VARIANT: {{variant}} script: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - - datadog-ci trace --name "Build {{library}} weblog {{variant}}" --no-fail -- ./build.sh {{library}} + - ./build.sh {{library}} - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning {{scenario}}" - - datadog-ci trace --name "Run {{scenario}}" --no-fail -- ./run.sh {{scenario}} + - ./run.sh {{scenario}} - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" artifacts: when: always paths: - logs*/reportJunit.xml before_script: - - npm install -g @datadog/datadog-ci - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin @@ -68,18 +66,17 @@ run_{{library}}_PARAMETRIC_{{job_index}}: {% endif %} script: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding runner" - - datadog-ci trace --name "Build runner" --no-fail -- ./build.sh -i runner + - ./build.sh -i runner - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - source venv/bin/activate - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" - - datadog-ci trace --name "Run PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" --no-fail -- ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} + - ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" artifacts: when: always paths: - logs*/reportJunit.xml before_script: - - npm install -g @datadog/datadog-ci - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin From 8ed9080fef515d9a8322904721ca00a3b9a95609 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 13:48:34 +0200 Subject: [PATCH 037/229] Small cleanup --- .gitlab-ci.yml | 1 + utils/ci/gitlab/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 10a5f21688f..60abffce950 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ include: inputs: stage: "test" libraries: "python" + scenarios_groups: "all" stages: - build - test diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 12fc0251eae..fc6cff78d01 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -8,7 +8,7 @@ spec: default: "" scenarios_groups: description: "Comma-separated list of scenario groups to run" - default: "all" + default: "" excluded_scenarios: description: "Comma-separated list of scenarios not to run" default: "APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS" @@ -30,7 +30,7 @@ spec: type: number default: 1 push_to_test_optimization: - description: "Push test results to Datadog Test Optimization (requires TEST_OPTIMIZATION_API_KEY CI variable)" + description: "Push test results to Datadog Test Optimization" type: boolean default: false test_optimization_datadog_site: From 361fbf782d2b7c495711b73a3918b93eb9f85667 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 14:12:39 +0200 Subject: [PATCH 038/229] ln the venv --- requirements.txt | 1 + utils/ci/gitlab/main.yml | 3 ++- utils/ci/gitlab/system-tests.yml | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 79c008d855c..0142bd94845 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,6 +46,7 @@ types-protobuf==5.29.1.20241207 types-python-dateutil==2.9.0.20241206 types-PyYAML==6.0.12.20241230 types-requests==2.31.0 +Jinja2==3.1.6 types-retry==0.9.9.20250322 watchdog==3.0.0 yamllint==1.35.1 diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index fc6cff78d01..86d3ef6901f 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -46,7 +46,6 @@ build_test_pipeline: script: - ./build.sh -i runner - source venv/bin/activate - - pip install Jinja2 - > for library in $[[ inputs.libraries ]]; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; @@ -55,6 +54,8 @@ build_test_pipeline: artifacts: paths: - generated-pipeline.yml + before_script: + - ln -sf /system-tests/venv venv run_test_pipeline: stage: run diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 4feef321aac..ee2b934a9cd 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -22,6 +22,7 @@ build_{{library}}_{{variant}}: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - ln -sf /system-tests/venv venv {% endfor %} {% for scenario in scenarios %}{% for variant in weblog_variants %} @@ -50,6 +51,7 @@ run_{{library}}_{{scenario}}_{{variant}}: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - ln -sf /system-tests/venv venv {% endfor %}{% endfor %} {% if parametric.enable %} @@ -80,6 +82,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - ln -sf /system-tests/venv venv {% endfor %} {% endif %} From 7e3d36cf943f281bbf07e4fdc9afa6b3317a5a29 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 14:23:22 +0200 Subject: [PATCH 039/229] Runner build optimization --- utils/ci/gitlab/main.yml | 3 +-- utils/ci/gitlab/system-tests.yml | 9 +++------ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 86d3ef6901f..61da41f3dae 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -44,8 +44,6 @@ build_test_pipeline: - arch:amd64 stage: build script: - - ./build.sh -i runner - - source venv/bin/activate - > for library in $[[ inputs.libraries ]]; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; @@ -56,6 +54,7 @@ build_test_pipeline: - generated-pipeline.yml before_script: - ln -sf /system-tests/venv venv + - source venv/bin/activate run_test_pipeline: stage: run diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index ee2b934a9cd..9cfef8b0c4d 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -13,7 +13,7 @@ build_{{library}}_{{variant}}: WEBLOG_VARIANT: {{variant}} script: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - - ./build.sh {{library}} --save-to-binaries + - ./build.sh {{library}} -i weblog --save-to-binaries - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" artifacts: paths: @@ -38,7 +38,7 @@ run_{{library}}_{{scenario}}_{{variant}}: WEBLOG_VARIANT: {{variant}} script: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - - ./build.sh {{library}} + - ./build.sh {{library}} -i weblog - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning {{scenario}}" - ./run.sh {{scenario}} @@ -67,10 +67,6 @@ run_{{library}}_PARAMETRIC_{{job_index}}: artifacts: true {% endif %} script: - - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding runner" - - ./build.sh -i runner - - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - - source venv/bin/activate - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" - ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" @@ -83,6 +79,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - ln -sf /system-tests/venv venv + - source venv/bin/activate {% endfor %} {% endif %} From 656688bc0f5405dfd5900f308bb7fadf8891fd9c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 14:42:33 +0200 Subject: [PATCH 040/229] Fix import ref path --- utils/ci/gitlab/main.yml | 1 + utils/ci/gitlab/system-tests.yml | 3 +++ 2 files changed, 4 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 61da41f3dae..ab86b604392 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -55,6 +55,7 @@ build_test_pipeline: before_script: - ln -sf /system-tests/venv venv - source venv/bin/activate + - export PYTHONPATH=$CI_PROJECT_DIR run_test_pipeline: stage: run diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 9cfef8b0c4d..eaf5fc522c3 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -23,6 +23,7 @@ build_{{library}}_{{variant}}: - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - ln -sf /system-tests/venv venv + - export PYTHONPATH=$CI_PROJECT_DIR {% endfor %} {% for scenario in scenarios %}{% for variant in weblog_variants %} @@ -52,6 +53,7 @@ run_{{library}}_{{scenario}}_{{variant}}: - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - ln -sf /system-tests/venv venv + - export PYTHONPATH=$CI_PROJECT_DIR {% endfor %}{% endfor %} {% if parametric.enable %} @@ -80,6 +82,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - ln -sf /system-tests/venv venv - source venv/bin/activate + - export PYTHONPATH=$CI_PROJECT_DIR {% endfor %} {% endif %} From c6f40a8c79b6e933ff370b613c6974ccc6cbcd29 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 15:04:26 +0200 Subject: [PATCH 041/229] Cancel pipeline on new commit on same branch --- utils/ci/gitlab/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index ab86b604392..1d7eeb69e10 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -38,6 +38,10 @@ spec: default: "datadoghq.com" --- +workflow: + auto_cancel: + on_new_commit: interruptible + build_test_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 tags: From d0b22e2017c5c2907c3a73c7920cbeee1b9618bb Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 15:06:11 +0200 Subject: [PATCH 042/229] Mark all build and run jobs as interruptible --- utils/ci/gitlab/main.yml | 2 ++ utils/ci/gitlab/system-tests.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 1d7eeb69e10..dfc828b8c94 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -44,6 +44,7 @@ workflow: build_test_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + interruptible: true tags: - arch:amd64 stage: build @@ -62,6 +63,7 @@ build_test_pipeline: - export PYTHONPATH=$CI_PROJECT_DIR run_test_pipeline: + interruptible: true stage: run needs: - job: build_test_pipeline diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index eaf5fc522c3..001469f3524 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -1,6 +1,7 @@ {% for variant in weblog_variants %} build_{{library}}_{{variant}}: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + interruptible: true tags: - docker-in-docker:amd64 stage: {{stage}} @@ -29,6 +30,7 @@ build_{{library}}_{{variant}}: {% for scenario in scenarios %}{% for variant in weblog_variants %} run_{{library}}_{{scenario}}_{{variant}}: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + interruptible: true tags: - docker-in-docker:amd64 stage: {{stage}} @@ -60,6 +62,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% for job_index in parametric.job_matrix %} run_{{library}}_PARAMETRIC_{{job_index}}: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + interruptible: true tags: - docker-in-docker:amd64 stage: {{stage}} From 2f8ff6bedb1c71dde4f079c5f260a06218313784 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 15:08:59 +0200 Subject: [PATCH 043/229] Make auto-cancel configurable and disable it on default branch --- utils/ci/gitlab/main.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index dfc828b8c94..5a51578d1e2 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -36,11 +36,17 @@ spec: test_optimization_datadog_site: description: "Datadog site to use for Test Optimization" default: "datadoghq.com" + auto_cancel_on_new_commit: + description: "Auto-cancel strategy when a new commit is pushed (interruptible, conservative, disabled)" + default: "interruptible" --- workflow: - auto_cancel: - on_new_commit: interruptible + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - when: always + auto_cancel: + on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] build_test_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 From 90955d1400718e67bd7e5ddaaa5debd2b8638900 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 15:46:32 +0200 Subject: [PATCH 044/229] Add plan for moving CI image to system-tests repo --- utils/ci/gitlab/plans/11-ci-image.md | 198 +++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) create mode 100644 utils/ci/gitlab/plans/11-ci-image.md diff --git a/utils/ci/gitlab/plans/11-ci-image.md b/utils/ci/gitlab/plans/11-ci-image.md new file mode 100644 index 00000000000..08b7e4ff0a1 --- /dev/null +++ b/utils/ci/gitlab/plans/11-ci-image.md @@ -0,0 +1,198 @@ +# Feature: move the CI image to the system-tests repo + +## Context + +The CI image (`registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777`) is +currently defined and built in the `libdatadog-build` repo. Its tag is a manually +bumped integer which gives no indication of what changed. This plan moves the image +definition into the system-tests repo, and introduces a CI job that rebuilds and +pushes it automatically whenever its inputs change, using a content-hash as the tag. + +Reference files: +- `libdatadog-build/docker/system-tests.Dockerfile` — current Dockerfile (to be moved) +- `libdatadog-build/docker/install_kube_dependencies.sh` — helper script (to be moved) +- `requirements.txt` — Python dependencies baked into the image +- `utils/ci/gitlab/main.yml` — references the image tag in every job +- `utils/ci/gitlab/system-tests.yml` — Jinja template: references the image tag in every generated job +- `utils/ci/gitlab/build_pipeline.py` — generates the child pipeline YAML + +--- + +## 1. Move the Dockerfile and helper script into the system-tests repo + +Create `utils/ci/docker/` and add two files: + +### `utils/ci/docker/install_kube_dependencies.sh` +Copy verbatim from `libdatadog-build/docker/install_kube_dependencies.sh`. + +### `utils/ci/docker/system-tests.Dockerfile` +Based on the existing Dockerfile with two changes: + +1. **Replace the git clone with a `COPY`** of only the files needed to build the + venv (`requirements.txt`). This removes the dependency on GitHub and makes the + image hermetically tied to the repo's own `requirements.txt`: + + ```dockerfile + # instead of: + RUN git clone https://github.com/DataDog/system-tests.git /system-tests + RUN ./build.sh -i runner + + # use: + COPY requirements.txt /system-tests/requirements.txt + WORKDIR /system-tests + RUN python3.12 -m venv venv && \ + venv/bin/pip install --upgrade pip setuptools==75.8.0 && \ + venv/bin/pip install -r requirements.txt + ``` + + The editable `pip install -e .` is intentionally dropped: CI jobs already + handle package resolution via `PYTHONPATH=$CI_PROJECT_DIR`. + +2. **Replace the `COPY docker/install_kube_dependencies.sh`** path with the new + location inside the repo: + ```dockerfile + COPY utils/ci/docker/install_kube_dependencies.sh . + ``` + +--- + +## 2. Define the hash + +The image tag is the first 12 hex characters of the SHA-256 hash of the files that +fully determine the image content: + +- `utils/ci/docker/system-tests.Dockerfile` +- `requirements.txt` + +```bash +IMAGE_TAG=$(cat utils/ci/docker/system-tests.Dockerfile requirements.txt \ + | sha256sum | cut -c1-12) +``` + +Sorting the inputs by filename is not required here since the order is fixed by the +script, but it should be kept stable across changes. + +--- + +## 3. Define the registry target + +Use the GitLab project's built-in registry: + +``` +$CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG +``` + +`$CI_REGISTRY_IMAGE` expands to the project's registry prefix automatically in +GitLab CI (e.g. `registry.ddbuild.io/datadog/system-tests`). + +--- + +## 4. Add the `build_ci_image` job to `main.yml` + +The job: +1. Computes `IMAGE_TAG` +2. Checks whether `$CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG` already exists in the + registry (`docker manifest inspect`). If it does, skips the build entirely. +3. If the image is absent, builds and pushes it. +4. Writes `IMAGE_TAG` to a dotenv artifact so downstream jobs receive it. + +```yaml +build_ci_image: + image: registry.ddbuild.io/images/docker:20.10.13-jammy + interruptible: true + tags: + - docker-in-docker:amd64 + stage: $[[ inputs.stage ]] + script: + - > + IMAGE_TAG=$(cat utils/ci/docker/system-tests.Dockerfile requirements.txt + | sha256sum | cut -c1-12) + - echo "CI_IMAGE=$CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG" >> build.env + - > + if docker manifest inspect $CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG > /dev/null 2>&1; then + echo "Image already exists, skipping build"; + else + docker build + -f utils/ci/docker/system-tests.Dockerfile + -t $CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG + . && + docker push $CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG; + fi + artifacts: + reports: + dotenv: build.env + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 + --name ci.system-tests.docker-login-write --with-decryption + --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 + --name ci.system-tests.docker-login-pass-write --with-decryption + --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin +``` + +`build_test_pipeline` gets `needs: [job: build_ci_image, artifacts: true]` so it +receives `CI_IMAGE` from the dotenv artifact. No new stage is needed — GitLab's DAG +scheduling runs `build_ci_image` before `build_test_pipeline` even within the same +stage. + +--- + +## 5. Thread `CI_IMAGE` through to generated jobs + +### `main.yml` — `build_test_pipeline` + +Pass `CI_IMAGE` (received from dotenv artifact) as a new `--ci-image` CLI argument +to `build_pipeline.py`: + +```bash +python3 utils/ci/gitlab/build_pipeline.py \ + --stage $[[ inputs.stage ]] \ + --library $library \ + --params params_${library}.json \ + --ci-image "$CI_IMAGE" \ + ... +``` + +Replace the hardcoded image in `build_test_pipeline` itself: +```yaml +build_test_pipeline: + image: $CI_IMAGE # instead of hardcoded tag +``` + +### `utils/ci/gitlab/build_pipeline.py` + +Add `--ci-image` argument and pass it to the Jinja template: + +```python +parser.add_argument("--ci-image", required=True, help="Full CI image reference") +... +template.render(..., ci_image=args.ci_image) +``` + +### `utils/ci/gitlab/system-tests.yml` + +Replace every hardcoded `image: registry.ddbuild.io/ci/libdatadog-build/system-tests:...` +with `image: {{ci_image}}`. + +--- + +## 6. Remove the image from `libdatadog-build` + +Once the above is in place and the new image is in use: +- Delete `docker/system-tests.Dockerfile` and `docker/install_kube_dependencies.sh` + from the `libdatadog-build` repo (separate PR/MR). + +--- + +## Summary of file changes + +| File | Change | +|---|---| +| `utils/ci/docker/system-tests.Dockerfile` | New — moved + adapted from `libdatadog-build` | +| `utils/ci/docker/install_kube_dependencies.sh` | New — moved verbatim | +| `utils/ci/gitlab/main.yml` | Add `build_ci_image` job; `build_test_pipeline` needs it and uses `$CI_IMAGE`; pass `--ci-image` to `build_pipeline.py` | +| `utils/ci/gitlab/build_pipeline.py` | Add `--ci-image` arg, pass to template | +| `utils/ci/gitlab/system-tests.yml` | Replace hardcoded image with `{{ci_image}}` | +| `libdatadog-build/docker/system-tests.Dockerfile` | Delete (separate MR) | +| `libdatadog-build/docker/install_kube_dependencies.sh` | Delete (separate MR) | From decc17f3100eb46c70d5750f88a31567f9641f39 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 15:49:44 +0200 Subject: [PATCH 045/229] Update CI image plan: place Dockerfile under utils/ci/gitlab/docker/ --- utils/ci/gitlab/plans/11-ci-image.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/utils/ci/gitlab/plans/11-ci-image.md b/utils/ci/gitlab/plans/11-ci-image.md index 08b7e4ff0a1..05dca467dc2 100644 --- a/utils/ci/gitlab/plans/11-ci-image.md +++ b/utils/ci/gitlab/plans/11-ci-image.md @@ -20,12 +20,12 @@ Reference files: ## 1. Move the Dockerfile and helper script into the system-tests repo -Create `utils/ci/docker/` and add two files: +Create `utils/ci/gitlab/docker/` and add two files: -### `utils/ci/docker/install_kube_dependencies.sh` +### `utils/ci/gitlab/docker/install_kube_dependencies.sh` Copy verbatim from `libdatadog-build/docker/install_kube_dependencies.sh`. -### `utils/ci/docker/system-tests.Dockerfile` +### `utils/ci/gitlab/docker/system-tests.Dockerfile` Based on the existing Dockerfile with two changes: 1. **Replace the git clone with a `COPY`** of only the files needed to build the @@ -51,7 +51,7 @@ Based on the existing Dockerfile with two changes: 2. **Replace the `COPY docker/install_kube_dependencies.sh`** path with the new location inside the repo: ```dockerfile - COPY utils/ci/docker/install_kube_dependencies.sh . + COPY utils/ci/gitlab/docker/install_kube_dependencies.sh . ``` --- @@ -61,11 +61,11 @@ Based on the existing Dockerfile with two changes: The image tag is the first 12 hex characters of the SHA-256 hash of the files that fully determine the image content: -- `utils/ci/docker/system-tests.Dockerfile` +- `utils/ci/gitlab/docker/system-tests.Dockerfile` - `requirements.txt` ```bash -IMAGE_TAG=$(cat utils/ci/docker/system-tests.Dockerfile requirements.txt \ +IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt \ | sha256sum | cut -c1-12) ``` @@ -105,7 +105,7 @@ build_ci_image: stage: $[[ inputs.stage ]] script: - > - IMAGE_TAG=$(cat utils/ci/docker/system-tests.Dockerfile requirements.txt + IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) - echo "CI_IMAGE=$CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG" >> build.env - > @@ -113,7 +113,7 @@ build_ci_image: echo "Image already exists, skipping build"; else docker build - -f utils/ci/docker/system-tests.Dockerfile + -f utils/ci/gitlab/docker/system-tests.Dockerfile -t $CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG . && docker push $CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG; @@ -189,8 +189,8 @@ Once the above is in place and the new image is in use: | File | Change | |---|---| -| `utils/ci/docker/system-tests.Dockerfile` | New — moved + adapted from `libdatadog-build` | -| `utils/ci/docker/install_kube_dependencies.sh` | New — moved verbatim | +| `utils/ci/gitlab/docker/system-tests.Dockerfile` | New — moved + adapted from `libdatadog-build` | +| `utils/ci/gitlab/docker/install_kube_dependencies.sh` | New — moved verbatim | | `utils/ci/gitlab/main.yml` | Add `build_ci_image` job; `build_test_pipeline` needs it and uses `$CI_IMAGE`; pass `--ci-image` to `build_pipeline.py` | | `utils/ci/gitlab/build_pipeline.py` | Add `--ci-image` arg, pass to template | | `utils/ci/gitlab/system-tests.yml` | Replace hardcoded image with `{{ci_image}}` | From 47d4291e84240d70f7a213137645be2f961c3d96 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 15:52:24 +0200 Subject: [PATCH 046/229] Update CI image plan: keep libdatadog-build image for backward compatibility --- utils/ci/gitlab/plans/11-ci-image.md | 113 ++++++++++++--------------- 1 file changed, 49 insertions(+), 64 deletions(-) diff --git a/utils/ci/gitlab/plans/11-ci-image.md b/utils/ci/gitlab/plans/11-ci-image.md index 05dca467dc2..ba9fd45f174 100644 --- a/utils/ci/gitlab/plans/11-ci-image.md +++ b/utils/ci/gitlab/plans/11-ci-image.md @@ -10,7 +10,7 @@ pushes it automatically whenever its inputs change, using a content-hash as the Reference files: - `libdatadog-build/docker/system-tests.Dockerfile` — current Dockerfile (to be moved) -- `libdatadog-build/docker/install_kube_dependencies.sh` — helper script (to be moved) +- `libdatadog-build/docker/install_kube_dependencies.sh` — helper script (not moved, kept for backward compatibility) - `requirements.txt` — Python dependencies baked into the image - `utils/ci/gitlab/main.yml` — references the image tag in every job - `utils/ci/gitlab/system-tests.yml` — Jinja template: references the image tag in every generated job @@ -18,41 +18,47 @@ Reference files: --- -## 1. Move the Dockerfile and helper script into the system-tests repo +## 1. Create the Dockerfile in the system-tests repo -Create `utils/ci/gitlab/docker/` and add two files: +Create `utils/ci/gitlab/docker/system-tests.Dockerfile` based on the existing +Dockerfile with the following changes: -### `utils/ci/gitlab/docker/install_kube_dependencies.sh` -Copy verbatim from `libdatadog-build/docker/install_kube_dependencies.sh`. +### 1a. Remove unused dependencies -### `utils/ci/gitlab/docker/system-tests.Dockerfile` -Based on the existing Dockerfile with two changes: +None of the following are called by any CI pipeline script: -1. **Replace the git clone with a `COPY`** of only the files needed to build the - venv (`requirements.txt`). This removes the dependency on GitHub and makes the - image hermetically tied to the repo's own `requirements.txt`: +| Removed | Reason | +|---|---| +| `binutils` | Redundant — already installed transitively by `build-essential` | +| `vault`, `libcap2-bin` | Secrets are fetched via AWS SSM, not Vault | +| Hashicorp GPG key, apt source, `setcap` call | Only present to support vault | +| `install_kube_dependencies.sh` (kind, kubectl, helm) | Only needed for K8s scenarios which run on dedicated infrastructure | + +`jq` is kept: it is used by `utils/scripts/load-binary.sh` for GitHub API calls. + +`install_kube_dependencies.sh` is **not** moved to the system-tests repo — it is +deleted entirely from `libdatadog-build`. + +### 1b. Replace the git clone with a COPY - ```dockerfile - # instead of: - RUN git clone https://github.com/DataDog/system-tests.git /system-tests - RUN ./build.sh -i runner +Instead of cloning system-tests at image build time (stale, slow, depends on +GitHub), copy only `requirements.txt` and build the venv directly from it: - # use: - COPY requirements.txt /system-tests/requirements.txt - WORKDIR /system-tests - RUN python3.12 -m venv venv && \ - venv/bin/pip install --upgrade pip setuptools==75.8.0 && \ - venv/bin/pip install -r requirements.txt - ``` +```dockerfile +# Remove: +RUN git clone https://github.com/DataDog/system-tests.git /system-tests +RUN ./build.sh -i runner - The editable `pip install -e .` is intentionally dropped: CI jobs already - handle package resolution via `PYTHONPATH=$CI_PROJECT_DIR`. +# Replace with: +COPY requirements.txt /system-tests/requirements.txt +WORKDIR /system-tests +RUN python3.12 -m venv venv && \ + venv/bin/pip install --upgrade pip setuptools==75.8.0 && \ + venv/bin/pip install -r requirements.txt +``` -2. **Replace the `COPY docker/install_kube_dependencies.sh`** path with the new - location inside the repo: - ```dockerfile - COPY utils/ci/gitlab/docker/install_kube_dependencies.sh . - ``` +The editable `pip install -e .` is intentionally dropped: CI jobs already handle +package resolution via `PYTHONPATH=$CI_PROJECT_DIR`. --- @@ -69,8 +75,7 @@ IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) ``` -Sorting the inputs by filename is not required here since the order is fixed by the -script, but it should be kept stable across changes. +The concatenation order must be kept stable across changes. --- @@ -91,10 +96,10 @@ GitLab CI (e.g. `registry.ddbuild.io/datadog/system-tests`). The job: 1. Computes `IMAGE_TAG` -2. Checks whether `$CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG` already exists in the - registry (`docker manifest inspect`). If it does, skips the build entirely. -3. If the image is absent, builds and pushes it. -4. Writes `IMAGE_TAG` to a dotenv artifact so downstream jobs receive it. +2. Checks whether the image already exists (`docker manifest inspect`). If so, + skips the build entirely — common case has zero overhead. +3. If absent, builds and pushes. +4. Writes `CI_IMAGE` to a dotenv artifact so downstream jobs receive it. ```yaml build_ci_image: @@ -142,23 +147,9 @@ stage. ### `main.yml` — `build_test_pipeline` -Pass `CI_IMAGE` (received from dotenv artifact) as a new `--ci-image` CLI argument -to `build_pipeline.py`: - -```bash -python3 utils/ci/gitlab/build_pipeline.py \ - --stage $[[ inputs.stage ]] \ - --library $library \ - --params params_${library}.json \ - --ci-image "$CI_IMAGE" \ - ... -``` - -Replace the hardcoded image in `build_test_pipeline` itself: -```yaml -build_test_pipeline: - image: $CI_IMAGE # instead of hardcoded tag -``` +- Replace the hardcoded `image:` with `$CI_IMAGE` +- Add `needs: [job: build_ci_image, artifacts: true]` +- Pass `--ci-image "$CI_IMAGE"` to `build_pipeline.py` ### `utils/ci/gitlab/build_pipeline.py` @@ -172,27 +163,21 @@ template.render(..., ci_image=args.ci_image) ### `utils/ci/gitlab/system-tests.yml` -Replace every hardcoded `image: registry.ddbuild.io/ci/libdatadog-build/system-tests:...` +Replace every hardcoded +`image: registry.ddbuild.io/ci/libdatadog-build/system-tests:...` with `image: {{ci_image}}`. --- -## 6. Remove the image from `libdatadog-build` - -Once the above is in place and the new image is in use: -- Delete `docker/system-tests.Dockerfile` and `docker/install_kube_dependencies.sh` - from the `libdatadog-build` repo (separate PR/MR). - ---- - ## Summary of file changes | File | Change | |---|---| -| `utils/ci/gitlab/docker/system-tests.Dockerfile` | New — moved + adapted from `libdatadog-build` | -| `utils/ci/gitlab/docker/install_kube_dependencies.sh` | New — moved verbatim | +| `utils/ci/gitlab/docker/system-tests.Dockerfile` | New — adapted from `libdatadog-build` (unused deps removed, git clone replaced with COPY) | | `utils/ci/gitlab/main.yml` | Add `build_ci_image` job; `build_test_pipeline` needs it and uses `$CI_IMAGE`; pass `--ci-image` to `build_pipeline.py` | | `utils/ci/gitlab/build_pipeline.py` | Add `--ci-image` arg, pass to template | | `utils/ci/gitlab/system-tests.yml` | Replace hardcoded image with `{{ci_image}}` | -| `libdatadog-build/docker/system-tests.Dockerfile` | Delete (separate MR) | -| `libdatadog-build/docker/install_kube_dependencies.sh` | Delete (separate MR) | + +`libdatadog-build/docker/system-tests.Dockerfile` and +`libdatadog-build/docker/install_kube_dependencies.sh` are kept as-is for +backward compatibility. From 82f44c0a9207372dc67d3c69417358a063254201 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 15:54:20 +0200 Subject: [PATCH 047/229] Update CI image plan: use ./build.sh -i runner instead of manual pip setup --- utils/ci/gitlab/plans/11-ci-image.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/utils/ci/gitlab/plans/11-ci-image.md b/utils/ci/gitlab/plans/11-ci-image.md index ba9fd45f174..a2622267403 100644 --- a/utils/ci/gitlab/plans/11-ci-image.md +++ b/utils/ci/gitlab/plans/11-ci-image.md @@ -39,10 +39,11 @@ None of the following are called by any CI pipeline script: `install_kube_dependencies.sh` is **not** moved to the system-tests repo — it is deleted entirely from `libdatadog-build`. -### 1b. Replace the git clone with a COPY +### 1b. Replace the git clone with a COPY + `./build.sh -i runner` -Instead of cloning system-tests at image build time (stale, slow, depends on -GitHub), copy only `requirements.txt` and build the venv directly from it: +Since the Dockerfile now lives inside the system-tests repo, the Docker build +context is the repo itself. Replace the git clone with a `COPY` of the files +needed by `./build.sh -i runner`, then run it directly: ```dockerfile # Remove: @@ -50,15 +51,16 @@ RUN git clone https://github.com/DataDog/system-tests.git /system-tests RUN ./build.sh -i runner # Replace with: -COPY requirements.txt /system-tests/requirements.txt +COPY build.sh build.sh +COPY utils/build/build.sh utils/build/build.sh +COPY requirements.txt requirements.txt +COPY pyproject.toml pyproject.toml WORKDIR /system-tests -RUN python3.12 -m venv venv && \ - venv/bin/pip install --upgrade pip setuptools==75.8.0 && \ - venv/bin/pip install -r requirements.txt +RUN ./build.sh -i runner ``` -The editable `pip install -e .` is intentionally dropped: CI jobs already handle -package resolution via `PYTHONPATH=$CI_PROJECT_DIR`. +The `WORKDIR` and `COPY` destination paths mirror the original clone layout so +`./build.sh -i runner` runs identically to before. --- From 340e47c7fc0608f183c5e7c8383c3b8d7ec434a1 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 15:56:40 +0200 Subject: [PATCH 048/229] Update CI image plan: use multi-stage build to keep only the venv --- utils/ci/gitlab/plans/11-ci-image.md | 35 +++++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/utils/ci/gitlab/plans/11-ci-image.md b/utils/ci/gitlab/plans/11-ci-image.md index a2622267403..96bc76163f2 100644 --- a/utils/ci/gitlab/plans/11-ci-image.md +++ b/utils/ci/gitlab/plans/11-ci-image.md @@ -39,28 +39,35 @@ None of the following are called by any CI pipeline script: `install_kube_dependencies.sh` is **not** moved to the system-tests repo — it is deleted entirely from `libdatadog-build`. -### 1b. Replace the git clone with a COPY + `./build.sh -i runner` +### 1b. Use a multi-stage build Since the Dockerfile now lives inside the system-tests repo, the Docker build -context is the repo itself. Replace the git clone with a `COPY` of the files -needed by `./build.sh -i runner`, then run it directly: +context is the repo itself. Use a two-stage build: -```dockerfile -# Remove: -RUN git clone https://github.com/DataDog/system-tests.git /system-tests -RUN ./build.sh -i runner +- **Builder stage**: copies the full repo and runs `./build.sh -i runner` to + produce the venv. Copying everything keeps the Dockerfile simple and ensures + `./build.sh` has access to any file it may need. +- **Final stage**: copies only `/system-tests/venv/` from the builder. All other + repo content is discarded, keeping the image lean. -# Replace with: -COPY build.sh build.sh -COPY utils/build/build.sh utils/build/build.sh -COPY requirements.txt requirements.txt -COPY pyproject.toml pyproject.toml +```dockerfile +# ---- builder ---- +FROM AS builder +# ... system package installation ... +COPY . /system-tests WORKDIR /system-tests RUN ./build.sh -i runner + +# ---- final ---- +FROM +# ... system package installation ... +COPY --from=builder /system-tests/venv /system-tests/venv +WORKDIR / ``` -The `WORKDIR` and `COPY` destination paths mirror the original clone layout so -`./build.sh -i runner` runs identically to before. +Both stages share the same base image and system package installation. The +`COPY --from=builder` line is the only addition over the original single-stage +Dockerfile. --- From e74440c3810c287b34b76ca123c841d8f946c62f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 16:50:11 +0200 Subject: [PATCH 049/229] Print CI_REGISTRY_IMAGE to verify registry path --- utils/ci/gitlab/main.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 5a51578d1e2..393a7d49557 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -48,6 +48,15 @@ workflow: auto_cancel: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] +print_registry_path: + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + interruptible: true + tags: + - arch:amd64 + stage: $[[ inputs.stage ]] + script: + - echo "CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE" + build_test_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 interruptible: true From a58009564b3496cc1888ed864a73aa1439b29b1b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 16:54:03 +0200 Subject: [PATCH 050/229] Print registry-related CI variables to determine correct image path --- utils/ci/gitlab/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 393a7d49557..0ef39f7bacf 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -56,6 +56,10 @@ print_registry_path: stage: $[[ inputs.stage ]] script: - echo "CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE" + - echo "CI_REGISTRY=$CI_REGISTRY" + - echo "CI_PROJECT_PATH=$CI_PROJECT_PATH" + - echo "CI_PROJECT_NAME=$CI_PROJECT_NAME" + - echo "CI_SERVER_HOST=$CI_SERVER_HOST" build_test_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 From 6301c6805111c25d9489a4a572a686a4d56bb435 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 16:57:30 +0200 Subject: [PATCH 051/229] Remove registry debug job, hardcode registry path to registry.ddbuild.io/system-tests/ci-runner --- utils/ci/gitlab/main.yml | 13 ------------- utils/ci/gitlab/plans/11-ci-image.md | 15 ++++++--------- 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 0ef39f7bacf..5a51578d1e2 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -48,19 +48,6 @@ workflow: auto_cancel: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] -print_registry_path: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 - interruptible: true - tags: - - arch:amd64 - stage: $[[ inputs.stage ]] - script: - - echo "CI_REGISTRY_IMAGE=$CI_REGISTRY_IMAGE" - - echo "CI_REGISTRY=$CI_REGISTRY" - - echo "CI_PROJECT_PATH=$CI_PROJECT_PATH" - - echo "CI_PROJECT_NAME=$CI_PROJECT_NAME" - - echo "CI_SERVER_HOST=$CI_SERVER_HOST" - build_test_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 interruptible: true diff --git a/utils/ci/gitlab/plans/11-ci-image.md b/utils/ci/gitlab/plans/11-ci-image.md index 96bc76163f2..3d5c7db2857 100644 --- a/utils/ci/gitlab/plans/11-ci-image.md +++ b/utils/ci/gitlab/plans/11-ci-image.md @@ -90,15 +90,12 @@ The concatenation order must be kept stable across changes. ## 3. Define the registry target -Use the GitLab project's built-in registry: +The image is pushed to a hardcoded path: ``` -$CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG +registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG ``` -`$CI_REGISTRY_IMAGE` expands to the project's registry prefix automatically in -GitLab CI (e.g. `registry.ddbuild.io/datadog/system-tests`). - --- ## 4. Add the `build_ci_image` job to `main.yml` @@ -121,16 +118,16 @@ build_ci_image: - > IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) - - echo "CI_IMAGE=$CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG" >> build.env + - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env - > - if docker manifest inspect $CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG > /dev/null 2>&1; then + if docker manifest inspect registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG > /dev/null 2>&1; then echo "Image already exists, skipping build"; else docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile - -t $CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG + -t registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG . && - docker push $CI_REGISTRY_IMAGE/ci-image:$IMAGE_TAG; + docker push registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG; fi artifacts: reports: From c2c631e6bf6cde173e1f97e4ec2519bfb1dd970d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 16:59:18 +0200 Subject: [PATCH 052/229] Add CI image Dockerfile --- .../ci/gitlab/docker/system-tests.Dockerfile | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 utils/ci/gitlab/docker/system-tests.Dockerfile diff --git a/utils/ci/gitlab/docker/system-tests.Dockerfile b/utils/ci/gitlab/docker/system-tests.Dockerfile new file mode 100644 index 00000000000..bcef493b5ec --- /dev/null +++ b/utils/ci/gitlab/docker/system-tests.Dockerfile @@ -0,0 +1,47 @@ +FROM registry.ddbuild.io/images/docker:20.10.13-jammy AS builder +USER root + +RUN apt update && apt upgrade -y +RUN apt install software-properties-common -y +RUN add-apt-repository ppa:deadsnakes/ppa -y + +RUN clean-apt install \ + build-essential \ + ca-certificates \ + git \ + python3.12 \ + python3-pip \ + python3.12-venv \ + python3.12-dev + +COPY . /system-tests +WORKDIR /system-tests +RUN ./build.sh -i runner + +FROM registry.ddbuild.io/images/docker:20.10.13-jammy +USER root + +RUN apt update && apt upgrade -y +RUN apt install software-properties-common -y +RUN add-apt-repository ppa:deadsnakes/ppa -y + +RUN clean-apt install \ + jq \ + ca-certificates \ + git \ + python3.12 \ + python3.12-venv + +ARG TARGETARCH +RUN if [ "${TARGETARCH}" = "arm64" ]; then \ + curl --retry 10 -fsSLo awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip; \ + else \ + curl --retry 10 -fsSLo awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip; \ + fi && \ + unzip -q awscliv2.zip && \ + ./aws/install && \ + apt-get clean + +COPY --from=builder /system-tests/venv /system-tests/venv + +WORKDIR / From 077da023b565fd7c6bc58073c0d31ce1afb717bd Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 17:00:08 +0200 Subject: [PATCH 053/229] Build and use content-addressed CI image from system-tests repo --- utils/ci/gitlab/build_pipeline.py | 2 ++ utils/ci/gitlab/main.yml | 33 ++++++++++++++++++++++++++++--- utils/ci/gitlab/system-tests.yml | 8 ++++---- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index e58b2f18ea8..a221be91104 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -9,6 +9,7 @@ parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") parser.add_argument("--library", required=True, default="", help="Library name, used to prefix job names") parser.add_argument("--params", required=True, help="Path to JSON output from compute-workflow-parameters.py") +parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") parser.add_argument("--test-optimization-datadog-site", default="datadoghq.com", help="Datadog site for Test Optimization") args = parser.parse_args() @@ -31,6 +32,7 @@ weblog_variants=weblog_variants, binaries_artifact=binaries_artifact, parametric=parametric, + ci_image=args.ci_image, push_to_test_optimization=args.push_to_test_optimization == "true", test_optimization_datadog_site=args.test_optimization_datadog_site, )) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 5a51578d1e2..f6de0dbd858 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -48,17 +48,44 @@ workflow: auto_cancel: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] +build_ci_image: + image: registry.ddbuild.io/images/docker:20.10.13-jammy + interruptible: true + tags: + - docker-in-docker:amd64 + stage: $[[ inputs.stage ]] + script: + - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) + - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env + - > + if docker manifest inspect registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG > /dev/null 2>&1; then + echo "Image already exists, skipping build"; + else + docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile -t registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG . && + docker push registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG; + fi + artifacts: + reports: + dotenv: build.env + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + build_test_pipeline: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + image: $CI_IMAGE interruptible: true tags: - arch:amd64 - stage: build + stage: $[[ inputs.stage ]] + needs: + - job: build_ci_image + artifacts: true script: - > for library in $[[ inputs.libraries ]]; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" >> generated-pipeline.yml; + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" >> generated-pipeline.yml; done artifacts: paths: diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 001469f3524..78c29c2851b 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -1,6 +1,6 @@ {% for variant in weblog_variants %} build_{{library}}_{{variant}}: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + image: {{ci_image}} interruptible: true tags: - docker-in-docker:amd64 @@ -29,7 +29,7 @@ build_{{library}}_{{variant}}: {% endfor %} {% for scenario in scenarios %}{% for variant in weblog_variants %} run_{{library}}_{{scenario}}_{{variant}}: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + image: {{ci_image}} interruptible: true tags: - docker-in-docker:amd64 @@ -61,7 +61,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% if parametric.enable %} {% for job_index in parametric.job_matrix %} run_{{library}}_PARAMETRIC_{{job_index}}: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + image: {{ci_image}} interruptible: true tags: - docker-in-docker:amd64 @@ -91,7 +91,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: {% endif %} {% if push_to_test_optimization %} push_{{library}}_test_optimization: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + image: {{ci_image}} tags: - docker-in-docker:amd64 stage: {{stage}} From e15c704bb55b84d1cf4a648f3617abe98359b12c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 17:10:14 +0200 Subject: [PATCH 054/229] Move build_ci_image to .gitlab-ci.yml --- .gitlab-ci.yml | 25 +++++++++++++++++++++++++ utils/ci/gitlab/main.yml | 24 ------------------------ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 60abffce950..e967fc24d1f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,32 @@ include: stage: "test" libraries: "python" scenarios_groups: "all" + stages: - build - test - run + +build_ci_image: + image: registry.ddbuild.io/images/docker:20.10.13-jammy + interruptible: true + tags: + - docker-in-docker:amd64 + stage: build + script: + - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) + - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env + - > + if docker manifest inspect registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG > /dev/null 2>&1; then + echo "Image already exists, skipping build"; + else + docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile -t registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG . && + docker push registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG; + fi + artifacts: + reports: + dotenv: build.env + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index f6de0dbd858..89710655e6c 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -48,30 +48,6 @@ workflow: auto_cancel: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] -build_ci_image: - image: registry.ddbuild.io/images/docker:20.10.13-jammy - interruptible: true - tags: - - docker-in-docker:amd64 - stage: $[[ inputs.stage ]] - script: - - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) - - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env - - > - if docker manifest inspect registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG > /dev/null 2>&1; then - echo "Image already exists, skipping build"; - else - docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile -t registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG . && - docker push registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG; - fi - artifacts: - reports: - dotenv: build.env - before_script: - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - build_test_pipeline: image: $CI_IMAGE interruptible: true From 22d53a9dd256f57163e2795f159ffb7685330b3c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 18 May 2026 17:16:29 +0200 Subject: [PATCH 055/229] Clone system-tests when pipeline runs from another repository --- utils/ci/gitlab/build_pipeline.py | 2 ++ utils/ci/gitlab/main.yml | 11 ++++++++++- utils/ci/gitlab/system-tests.yml | 15 +++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index a221be91104..dba8e4afa17 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -10,6 +10,7 @@ parser.add_argument("--library", required=True, default="", help="Library name, used to prefix job names") parser.add_argument("--params", required=True, help="Path to JSON output from compute-workflow-parameters.py") parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") +parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") parser.add_argument("--test-optimization-datadog-site", default="datadoghq.com", help="Datadog site for Test Optimization") args = parser.parse_args() @@ -33,6 +34,7 @@ binaries_artifact=binaries_artifact, parametric=parametric, ci_image=args.ci_image, + ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", test_optimization_datadog_site=args.test_optimization_datadog_site, )) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 89710655e6c..17e7e3762a1 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -39,6 +39,9 @@ spec: auto_cancel_on_new_commit: description: "Auto-cancel strategy when a new commit is pushed (interruptible, conservative, disabled)" default: "interruptible" + ref: + description: "system-tests ref to use when called from another repository (branch, tag or SHA)" + default: "" --- workflow: @@ -61,12 +64,18 @@ build_test_pipeline: - > for library in $[[ inputs.libraries ]]; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" >> generated-pipeline.yml; + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" >> generated-pipeline.yml; done artifacts: paths: - generated-pipeline.yml before_script: + - | + if [ "$CI_PROJECT_NAME" != "system-tests" ]; then + REF="$[[ inputs.ref ]]" + git clone --depth 1 ${REF:+--branch "$REF"} https://github.com/DataDog/system-tests.git /tmp/system-tests + cp -a /tmp/system-tests/. $CI_PROJECT_DIR/ + fi - ln -sf /system-tests/venv venv - source venv/bin/activate - export PYTHONPATH=$CI_PROJECT_DIR diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 78c29c2851b..575f737f0eb 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -20,6 +20,11 @@ build_{{library}}_{{variant}}: paths: - binaries/ before_script: + - | + if [ "$CI_PROJECT_NAME" != "system-tests" ]; then + git clone --depth 1 {% if ref %}--branch "{{ref}}" {% endif %}https://github.com/DataDog/system-tests.git /tmp/system-tests + cp -a /tmp/system-tests/. $CI_PROJECT_DIR/ + fi - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin @@ -51,6 +56,11 @@ run_{{library}}_{{scenario}}_{{variant}}: paths: - logs*/reportJunit.xml before_script: + - | + if [ "$CI_PROJECT_NAME" != "system-tests" ]; then + git clone --depth 1 {% if ref %}--branch "{{ref}}" {% endif %}https://github.com/DataDog/system-tests.git /tmp/system-tests + cp -a /tmp/system-tests/. $CI_PROJECT_DIR/ + fi - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin @@ -80,6 +90,11 @@ run_{{library}}_PARAMETRIC_{{job_index}}: paths: - logs*/reportJunit.xml before_script: + - | + if [ "$CI_PROJECT_NAME" != "system-tests" ]; then + git clone --depth 1 {% if ref %}--branch "{{ref}}" {% endif %}https://github.com/DataDog/system-tests.git /tmp/system-tests + cp -a /tmp/system-tests/. $CI_PROJECT_DIR/ + fi - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin From 8813747318538abcf1c550c35282307172b5c028 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 19 May 2026 11:08:16 +0200 Subject: [PATCH 056/229] Removing docker login steps from image builder job --- .gitlab-ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e967fc24d1f..3ff4a98f768 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,7 +29,3 @@ build_ci_image: artifacts: reports: dotenv: build.env - before_script: - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin From 0501629d29cc6144c83984dcde2c618e26e553f4 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 19 May 2026 11:41:09 +0200 Subject: [PATCH 057/229] Using template workflow for system-tests --- .gitlab-ci.yml | 1 + utils/ci/gitlab/main.yml | 27 +++++------ utils/ci/gitlab/system-tests.yml | 79 ++++++++------------------------ utils/ci/gitlab/templates.yml | 23 ++++++++++ 4 files changed, 55 insertions(+), 75 deletions(-) create mode 100644 utils/ci/gitlab/templates.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3ff4a98f768..ce8df5f6a80 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ include: stage: "test" libraries: "python" scenarios_groups: "all" + ref: "$CI_COMMIT_SHA" stages: - build diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 17e7e3762a1..a0fb3dfe095 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -1,6 +1,7 @@ spec: inputs: stage: + description: "CI stage for the jobs" libraries: default: "java python nodejs" scenarios: @@ -41,9 +42,15 @@ spec: default: "interruptible" ref: description: "system-tests ref to use when called from another repository (branch, tag or SHA)" - default: "" + default: "main" --- +include: + - local: /utils/ci/gitlab/templates.yml + inputs: + stage: $[[ inputs.stage ]] + ref: $[[ inputs.ref ]] + workflow: rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -52,11 +59,9 @@ workflow: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] build_test_pipeline: - image: $CI_IMAGE - interruptible: true + extends: .system_tests_base tags: - arch:amd64 - stage: $[[ inputs.stage ]] needs: - job: build_ci_image artifacts: true @@ -68,17 +73,7 @@ build_test_pipeline: done artifacts: paths: - - generated-pipeline.yml - before_script: - - | - if [ "$CI_PROJECT_NAME" != "system-tests" ]; then - REF="$[[ inputs.ref ]]" - git clone --depth 1 ${REF:+--branch "$REF"} https://github.com/DataDog/system-tests.git /tmp/system-tests - cp -a /tmp/system-tests/. $CI_PROJECT_DIR/ - fi - - ln -sf /system-tests/venv venv - - source venv/bin/activate - - export PYTHONPATH=$CI_PROJECT_DIR + - system-tests/generated-pipeline.yml run_test_pipeline: interruptible: true @@ -91,7 +86,7 @@ run_test_pipeline: SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" trigger: include: - - artifact: generated-pipeline.yml + - artifact: system-tests/generated-pipeline.yml job: build_test_pipeline strategy: depend diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 575f737f0eb..25af313d9eb 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -1,52 +1,40 @@ +include: + - local: /utils/ci/gitlab/templates.yml + inputs: + stage: {{stage}} + ref: {{ref}} + {% for variant in weblog_variants %} build_{{library}}_{{variant}}: - image: {{ci_image}} - interruptible: true - tags: - - docker-in-docker:amd64 - stage: {{stage}} + extends: .system_tests_base {% if binaries_artifact %} needs: - job: {{binaries_artifact}} artifacts: true {% endif %} - variables: - WEBLOG_VARIANT: {{variant}} script: + - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KSetting up docker hub" + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - - ./build.sh {{library}} -i weblog --save-to-binaries + - ./build.sh {{library}} -i weblog --save-to-binaries --weblog-variant {{variant}} - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" artifacts: paths: - - binaries/ - before_script: - - | - if [ "$CI_PROJECT_NAME" != "system-tests" ]; then - git clone --depth 1 {% if ref %}--branch "{{ref}}" {% endif %}https://github.com/DataDog/system-tests.git /tmp/system-tests - cp -a /tmp/system-tests/. $CI_PROJECT_DIR/ - fi - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - - ln -sf /system-tests/venv venv - - export PYTHONPATH=$CI_PROJECT_DIR + - system-tests/binaries/ {% endfor %} {% for scenario in scenarios %}{% for variant in weblog_variants %} run_{{library}}_{{scenario}}_{{variant}}: - image: {{ci_image}} - interruptible: true - tags: - - docker-in-docker:amd64 - stage: {{stage}} + extends: .system_tests_base needs: - job: build_{{library}}_{{variant}} artifacts: true - variables: - WEBLOG_VARIANT: {{variant}} script: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - - ./build.sh {{library}} -i weblog + - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning {{scenario}}" - ./run.sh {{scenario}} @@ -54,28 +42,13 @@ run_{{library}}_{{scenario}}_{{variant}}: artifacts: when: always paths: - - logs*/reportJunit.xml - before_script: - - | - if [ "$CI_PROJECT_NAME" != "system-tests" ]; then - git clone --depth 1 {% if ref %}--branch "{{ref}}" {% endif %}https://github.com/DataDog/system-tests.git /tmp/system-tests - cp -a /tmp/system-tests/. $CI_PROJECT_DIR/ - fi - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - - ln -sf /system-tests/venv venv - - export PYTHONPATH=$CI_PROJECT_DIR + - system-tests/logs*/reportJunit.xml {% endfor %}{% endfor %} {% if parametric.enable %} {% for job_index in parametric.job_matrix %} run_{{library}}_PARAMETRIC_{{job_index}}: - image: {{ci_image}} - interruptible: true - tags: - - docker-in-docker:amd64 - stage: {{stage}} + extends: .system_tests_base {% if binaries_artifact %} needs: - job: {{binaries_artifact}} @@ -88,19 +61,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: artifacts: when: always paths: - - logs*/reportJunit.xml - before_script: - - | - if [ "$CI_PROJECT_NAME" != "system-tests" ]; then - git clone --depth 1 {% if ref %}--branch "{{ref}}" {% endif %}https://github.com/DataDog/system-tests.git /tmp/system-tests - cp -a /tmp/system-tests/. $CI_PROJECT_DIR/ - fi - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - - ln -sf /system-tests/venv venv - - source venv/bin/activate - - export PYTHONPATH=$CI_PROJECT_DIR + - system-tests/logs*/reportJunit.xml {% endfor %} {% endif %} @@ -123,7 +84,7 @@ push_{{library}}_test_optimization: script: - npm install -g @datadog/datadog-ci - > - datadog-ci junit upload logs*/reportJunit.xml + datadog-ci junit upload system-tests/logs*/reportJunit.xml --service system-tests --env ci --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" diff --git a/utils/ci/gitlab/templates.yml b/utils/ci/gitlab/templates.yml new file mode 100644 index 00000000000..b56a08f8f8a --- /dev/null +++ b/utils/ci/gitlab/templates.yml @@ -0,0 +1,23 @@ +spec: + inputs: + stage: + ref: + default: main + +--- + +.system_tests_base: + image: $CI_IMAGE + tags: + - docker-in-docker:amd64 + variables: + GIT_STRATEGY: none + stage: $[[ inputs.stage ]] + interruptible: true + before_script: + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git + - cd system-tests + - git checkout $[[ inputs.ref ]] + - ln -sf /system-tests/venv venv + - source venv/bin/activate + - export PYTHONPATH="$CI_PROJECT_DIR/system-tests" From a7b3e295832d8dee9629bbeacfa33f29a3f6e772 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 19 May 2026 13:52:35 +0200 Subject: [PATCH 058/229] Separate image tag resolution --- utils/ci/gitlab/main.yml | 23 ++++++++++++++++++++++- utils/ci/gitlab/system-tests.yml | 3 +++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index a0fb3dfe095..1b49f26ee2b 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -58,12 +58,33 @@ workflow: auto_cancel: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] +resolve_ci_image: + image: alpine/git + tags: + - arch:amd64 + stage: $[[ inputs.stage ]] + interruptible: true + variables: + GIT_STRATEGY: none + needs: + - job: build_ci_image + optional: true + artifacts: false + script: + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git + - git -C system-tests checkout $[[ inputs.ref ]] + - IMAGE_TAG=$(cat system-tests/utils/ci/gitlab/docker/system-tests.Dockerfile system-tests/requirements.txt | sha256sum | cut -c1-12) + - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env + artifacts: + reports: + dotenv: build.env + build_test_pipeline: extends: .system_tests_base tags: - arch:amd64 needs: - - job: build_ci_image + - job: resolve_ci_image artifacts: true script: - > diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 25af313d9eb..0d9ebaa00f2 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -4,6 +4,9 @@ include: stage: {{stage}} ref: {{ref}} +variables: + CI_IMAGE: {{ci_image}} + {% for variant in weblog_variants %} build_{{library}}_{{variant}}: extends: .system_tests_base From f3f7edbd21bf72173d50f07126d509c03add1f09 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 19 May 2026 17:33:58 +0200 Subject: [PATCH 059/229] Using custom git image --- utils/ci/gitlab/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 1b49f26ee2b..17388417bc9 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -59,7 +59,7 @@ workflow: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] resolve_ci_image: - image: alpine/git + image: registry.ddbuild.io/system-tests/git tags: - arch:amd64 stage: $[[ inputs.stage ]] From 45fbdcd5db3d3a6b766f7524e7bd9cc46da056f1 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 19 May 2026 17:43:39 +0200 Subject: [PATCH 060/229] Simple image with git for the resolve_ci_image job --- utils/ci/gitlab/docker/git.Dockerfile | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 utils/ci/gitlab/docker/git.Dockerfile diff --git a/utils/ci/gitlab/docker/git.Dockerfile b/utils/ci/gitlab/docker/git.Dockerfile new file mode 100644 index 00000000000..b0b33abb390 --- /dev/null +++ b/utils/ci/gitlab/docker/git.Dockerfile @@ -0,0 +1,3 @@ +FROM alpine/git + +# docker build --push --platform linux/amd64 -t registry.ddbuild.io/system-tests/git -f utils/ci/gitlab/docker/git.Dockerfile . From 770f80686e97988d121e69366642ecf1c39f622c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 20 May 2026 11:15:08 +0200 Subject: [PATCH 061/229] Moving binaries out of the clone way --- utils/ci/gitlab/system-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 0d9ebaa00f2..d411ef3063f 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -24,9 +24,10 @@ build_{{library}}_{{variant}}: - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - ./build.sh {{library}} -i weblog --save-to-binaries --weblog-variant {{variant}} - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" + - mv binaries .. artifacts: paths: - - system-tests/binaries/ + - binaries/ {% endfor %} {% for scenario in scenarios %}{% for variant in weblog_variants %} @@ -36,6 +37,7 @@ run_{{library}}_{{scenario}}_{{variant}}: - job: build_{{library}}_{{variant}} artifacts: true script: + - mv ../binaries/* binaries/ - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" From 8d058c73a7ec9b8f87ed848efc4a6f09418d3e54 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 20 May 2026 11:38:57 +0200 Subject: [PATCH 062/229] Exclude the slowest scenario --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ce8df5f6a80..7e75a82d8ba 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ include: stage: "test" libraries: "python" scenarios_groups: "all" + excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE" ref: "$CI_COMMIT_SHA" stages: From bad39ebb35fb98aff58de21882ee7da58c27f5f4 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 20 May 2026 11:44:46 +0200 Subject: [PATCH 063/229] Adding dd-sts --- .gitlab-ci.yml | 1 + .../ci/gitlab/docker/system-tests.Dockerfile | 8 ++- utils/ci/gitlab/system-tests.yml | 51 ++++++++----------- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e75a82d8ba..283f1e775c9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ include: scenarios_groups: "all" excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE" ref: "$CI_COMMIT_SHA" + push_to_test_optimization: true stages: - build diff --git a/utils/ci/gitlab/docker/system-tests.Dockerfile b/utils/ci/gitlab/docker/system-tests.Dockerfile index bcef493b5ec..cdaf0f935d2 100644 --- a/utils/ci/gitlab/docker/system-tests.Dockerfile +++ b/utils/ci/gitlab/docker/system-tests.Dockerfile @@ -30,7 +30,10 @@ RUN clean-apt install \ ca-certificates \ git \ python3.12 \ - python3.12-venv + python3.12-venv \ + npm + +RUN npm install -g @datadog/datadog-ci ARG TARGETARCH RUN if [ "${TARGETARCH}" = "arm64" ]; then \ @@ -42,6 +45,9 @@ RUN if [ "${TARGETARCH}" = "arm64" ]; then \ ./aws/install && \ apt-get clean +COPY --from=registry.ddbuild.io/dd-sts:v0.1.4@sha256:1f4bc8861cca86b0c977ae70843990f9368f9b69dbfc4979cf5c515a97a3ea15 \ + /usr/local/bin/dd-sts /usr/local/bin/dd-sts + COPY --from=builder /system-tests/venv /system-tests/venv WORKDIR / diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index d411ef3063f..07079b21b2b 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -7,6 +7,18 @@ include: variables: CI_IMAGE: {{ci_image}} +.push_to_test_optimization: + id_tokens: + DD_STS_OIDC_TOKEN: + aud: dd-sts + after_script: + - dd-sts debug + - > + dd-sts exchange --policy system-tests-gitlab -- datadog-ci junit upload system-tests/logs*/reportJunit.xml + --service system-tests + --env ci + --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" + {% for variant in weblog_variants %} build_{{library}}_{{variant}}: extends: .system_tests_base @@ -32,7 +44,11 @@ build_{{library}}_{{variant}}: {% endfor %} {% for scenario in scenarios %}{% for variant in weblog_variants %} run_{{library}}_{{scenario}}_{{variant}}: - extends: .system_tests_base + extends: + - .system_tests_base +{% if push_to_test_optimization %} + - .push_to_test_optimization +{% endif %} needs: - job: build_{{library}}_{{variant}} artifacts: true @@ -53,7 +69,11 @@ run_{{library}}_{{scenario}}_{{variant}}: {% if parametric.enable %} {% for job_index in parametric.job_matrix %} run_{{library}}_PARAMETRIC_{{job_index}}: - extends: .system_tests_base + extends: + - .system_tests_base +{% if push_to_test_optimization %} + - .push_to_test_optimization +{% endif %} {% if binaries_artifact %} needs: - job: {{binaries_artifact}} @@ -70,30 +90,3 @@ run_{{library}}_PARAMETRIC_{{job_index}}: {% endfor %} {% endif %} -{% if push_to_test_optimization %} -push_{{library}}_test_optimization: - image: {{ci_image}} - tags: - - docker-in-docker:amd64 - stage: {{stage}} - when: always - needs: - {% for scenario in scenarios %}{% for variant in weblog_variants %} - - job: run_{{library}}_{{scenario}}_{{variant}} - artifacts: true - {% endfor %}{% endfor %} - {% if parametric.enable %}{% for job_index in parametric.job_matrix %} - - job: run_{{library}}_PARAMETRIC_{{job_index}} - artifacts: true - {% endfor %}{% endif %} - script: - - npm install -g @datadog/datadog-ci - - > - datadog-ci junit upload system-tests/logs*/reportJunit.xml - --service system-tests - --env ci - --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" - variables: - DATADOG_SITE: {{test_optimization_datadog_site}} - DATADOG_API_KEY: "$TEST_OPTIMIZATION_API_KEY" -{% endif %} From 3262bf88b8e520783a431a761df9c6bc8e10fdf7 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 22 May 2026 11:37:44 +0200 Subject: [PATCH 064/229] Improved section management --- utils/ci/gitlab/main.yml | 1 + utils/ci/gitlab/section.sh | 14 ++++++++++++++ utils/ci/gitlab/system-tests.yml | 18 ++++++++++-------- utils/ci/gitlab/templates.yml | 3 +++ 4 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 utils/ci/gitlab/section.sh diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 17388417bc9..f264a531276 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -75,6 +75,7 @@ resolve_ci_image: - git -C system-tests checkout $[[ inputs.ref ]] - IMAGE_TAG=$(cat system-tests/utils/ci/gitlab/docker/system-tests.Dockerfile system-tests/requirements.txt | sha256sum | cut -c1-12) - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env + - cat build.env artifacts: reports: dotenv: build.env diff --git a/utils/ci/gitlab/section.sh b/utils/ci/gitlab/section.sh new file mode 100644 index 00000000000..20fa1f00049 --- /dev/null +++ b/utils/ci/gitlab/section.sh @@ -0,0 +1,14 @@ +# function for starting the section +function section_start () { + local section_title="${1}" + local section_description="${2:-$section_title}" + + echo -e "section_start:`date +%s`:${section_title}[collapsed=true]\r\e[0K${section_description}" +} + +# Function for ending the section +function section_end () { + local section_title="${1}" + + echo -e "section_end:`date +%s`:${section_title}\r\e[0K" +} diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 07079b21b2b..c9b0e3883b4 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -12,12 +12,14 @@ variables: DD_STS_OIDC_TOKEN: aud: dd-sts after_script: + - section_start "test_optim" "Push to tests optimization" - dd-sts debug - > dd-sts exchange --policy system-tests-gitlab -- datadog-ci junit upload system-tests/logs*/reportJunit.xml --service system-tests --env ci --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" + - section_end "test_optim" {% for variant in weblog_variants %} build_{{library}}_{{variant}}: @@ -28,15 +30,15 @@ build_{{library}}_{{variant}}: artifacts: true {% endif %} script: - - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KSetting up docker hub" + - section_start "docker_auth" "Docker hub auth" - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" + - section_end "docker_auth" + - section_start "build" "Building weblog" - ./build.sh {{library}} -i weblog --save-to-binaries --weblog-variant {{variant}} - - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - mv binaries .. + - section_end "build" artifacts: paths: - binaries/ @@ -53,13 +55,13 @@ run_{{library}}_{{scenario}}_{{variant}}: - job: build_{{library}}_{{variant}} artifacts: true script: + - section_start "weblog_setup" "Setting up the weblog" - mv ../binaries/* binaries/ - - echo -e "\e[0Ksection_start:$(date +%s):build\r\e[0KBuilding {{library}} weblog {{variant}}" - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} - - echo -e "\e[0Ksection_end:$(date +%s):build\r\e[0K" - - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning {{scenario}}" + - section_end "weblog_setup" + - section_start "scenario_run" "Running scenario" - ./run.sh {{scenario}} - - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" + - section_end "scenario_run" artifacts: when: always paths: diff --git a/utils/ci/gitlab/templates.yml b/utils/ci/gitlab/templates.yml index b56a08f8f8a..3a91e986ee8 100644 --- a/utils/ci/gitlab/templates.yml +++ b/utils/ci/gitlab/templates.yml @@ -15,9 +15,12 @@ spec: stage: $[[ inputs.stage ]] interruptible: true before_script: + - echo -e "section_start:`date +%s`:setup[collapsed=true]\r\e[0KSystem-tests runner setup" - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git - cd system-tests - git checkout $[[ inputs.ref ]] - ln -sf /system-tests/venv venv - source venv/bin/activate - export PYTHONPATH="$CI_PROJECT_DIR/system-tests" + - source utils/ci/gitlab/section.sh + - echo -e "section_end:`date +%s`:setup\r\e[0K" From ea614fef9d44b99cc25b4622d8eda22e0459ad59 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 22 May 2026 12:55:43 +0200 Subject: [PATCH 065/229] fix section_start not found in after_script --- utils/ci/gitlab/system-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index c9b0e3883b4..dd4e375a1f3 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -12,14 +12,14 @@ variables: DD_STS_OIDC_TOKEN: aud: dd-sts after_script: - - section_start "test_optim" "Push to tests optimization" + - echo -e "section_start:$(date +%s):test_optim[collapsed=true]\r\e[0KPush to tests optimization" - dd-sts debug - > dd-sts exchange --policy system-tests-gitlab -- datadog-ci junit upload system-tests/logs*/reportJunit.xml --service system-tests --env ci --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" - - section_end "test_optim" + - echo -e "section_end:$(date +%s):test_optim\r\e[0K" {% for variant in weblog_variants %} build_{{library}}_{{variant}}: From 2fd001b7ea8fe6426ccb98e5a0e14c0c6414422e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 22 May 2026 14:20:50 +0200 Subject: [PATCH 066/229] Test --- utils/ci/gitlab/system-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index dd4e375a1f3..8cc78f1c649 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -14,6 +14,7 @@ variables: after_script: - echo -e "section_start:$(date +%s):test_optim[collapsed=true]\r\e[0KPush to tests optimization" - dd-sts debug + - dd-sts exchange --policy system-tests-gitlab -- echo "Success" || echo "Failure" - > dd-sts exchange --policy system-tests-gitlab -- datadog-ci junit upload system-tests/logs*/reportJunit.xml --service system-tests From c09da236230f7ec345830e6ad9b23d962505b3af Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 22 May 2026 14:37:08 +0200 Subject: [PATCH 067/229] Update base image version --- utils/ci/gitlab/docker/system-tests.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/ci/gitlab/docker/system-tests.Dockerfile b/utils/ci/gitlab/docker/system-tests.Dockerfile index cdaf0f935d2..1e38584f0ed 100644 --- a/utils/ci/gitlab/docker/system-tests.Dockerfile +++ b/utils/ci/gitlab/docker/system-tests.Dockerfile @@ -1,4 +1,4 @@ -FROM registry.ddbuild.io/images/docker:20.10.13-jammy AS builder +FROM registry.ddbuild.io/images/docker:29.4.0-noble AS builder USER root RUN apt update && apt upgrade -y @@ -18,7 +18,7 @@ COPY . /system-tests WORKDIR /system-tests RUN ./build.sh -i runner -FROM registry.ddbuild.io/images/docker:20.10.13-jammy +FROM registry.ddbuild.io/images/docker:29.4.0-noble USER root RUN apt update && apt upgrade -y From a2c45eacaef15189674c986cd452ecfd22b5c827 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 22 May 2026 14:56:00 +0200 Subject: [PATCH 068/229] Updating node version --- utils/ci/gitlab/docker/system-tests.Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/ci/gitlab/docker/system-tests.Dockerfile b/utils/ci/gitlab/docker/system-tests.Dockerfile index 1e38584f0ed..6c0378d3ed7 100644 --- a/utils/ci/gitlab/docker/system-tests.Dockerfile +++ b/utils/ci/gitlab/docker/system-tests.Dockerfile @@ -28,10 +28,13 @@ RUN add-apt-repository ppa:deadsnakes/ppa -y RUN clean-apt install \ jq \ ca-certificates \ + curl \ git \ python3.12 \ - python3.12-venv \ - npm + python3.12-venv + +RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \ + clean-apt install nodejs RUN npm install -g @datadog/datadog-ci From 3d757995f70e11add9d32495f9808ab1fce4aa06 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 22 May 2026 16:06:32 +0200 Subject: [PATCH 069/229] Cleanup and adding custom tag to the pipeline --- .gitlab-ci.yml | 2 +- utils/ci/gitlab/main.yml | 3 +++ utils/ci/gitlab/system-tests.yml | 9 ++++++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 283f1e775c9..89b127e7f9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,7 @@ include: stage: "test" libraries: "python" scenarios_groups: "all" - excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE" + excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index f264a531276..c6fe0020c2d 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -87,6 +87,9 @@ build_test_pipeline: needs: - job: resolve_ci_image artifacts: true + id_tokens: + DD_STS_OIDC_TOKEN: + aud: dd-sts script: - > for library in $[[ inputs.libraries ]]; do diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 8cc78f1c649..9e7ae5827cf 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -14,7 +14,6 @@ variables: after_script: - echo -e "section_start:$(date +%s):test_optim[collapsed=true]\r\e[0KPush to tests optimization" - dd-sts debug - - dd-sts exchange --policy system-tests-gitlab -- echo "Success" || echo "Failure" - > dd-sts exchange --policy system-tests-gitlab -- datadog-ci junit upload system-tests/logs*/reportJunit.xml --service system-tests @@ -22,6 +21,14 @@ variables: --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" - echo -e "section_end:$(date +%s):test_optim\r\e[0K" +setup: + extends: .system_tests_base + id_tokens: + DD_STS_OIDC_TOKEN: + aud: dd-sts + script: + - dd-sts exchange --policy system-tests-gitlab -- datadog-ci tag --level pipeline --tags "system-tests:true" + {% for variant in weblog_variants %} build_{{library}}_{{variant}}: extends: .system_tests_base From 625b219457eed6a16df1934b8c1b29301e5407de Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 10:26:52 +0200 Subject: [PATCH 070/229] Docker hub auth for end to end jobs --- utils/ci/gitlab/system-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 9e7ae5827cf..9dd028bd68a 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -63,6 +63,11 @@ run_{{library}}_{{scenario}}_{{variant}}: - job: build_{{library}}_{{variant}} artifacts: true script: + - section_start "docker_auth" "Docker hub auth" + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - section_end "docker_auth" - section_start "weblog_setup" "Setting up the weblog" - mv ../binaries/* binaries/ - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} From 9497ed9ded639f2fed6259fd4f238e441ef9c852 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 11:07:30 +0200 Subject: [PATCH 071/229] Adding lib argument to end to end jobs for INTEGRATION_FRAMEWORKS --- utils/ci/gitlab/system-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 9dd028bd68a..45e1698e810 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -73,7 +73,7 @@ run_{{library}}_{{scenario}}_{{variant}}: - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} - section_end "weblog_setup" - section_start "scenario_run" "Running scenario" - - ./run.sh {{scenario}} + - ./run.sh {{scenario}} -L {{library}} - section_end "scenario_run" artifacts: when: always From 93fdca559c15bbaa967d66bdf37068c20cb3637f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 11:11:00 +0200 Subject: [PATCH 072/229] cleanup --- test/docker-compose.yml | 9 - utils/ci/gitlab/plans/11-ci-image.md | 189 ------------------ .../scripts/libraries_and_scenarios_rules.yml | 15 +- 3 files changed, 6 insertions(+), 207 deletions(-) delete mode 100644 test/docker-compose.yml delete mode 100644 utils/ci/gitlab/plans/11-ci-image.md diff --git a/test/docker-compose.yml b/test/docker-compose.yml deleted file mode 100644 index b75e70859b4..00000000000 --- a/test/docker-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -services: - ubuntu: - image: ubuntu - postgres: - image: postgres - python: - image: python - node: - image: node diff --git a/utils/ci/gitlab/plans/11-ci-image.md b/utils/ci/gitlab/plans/11-ci-image.md deleted file mode 100644 index 3d5c7db2857..00000000000 --- a/utils/ci/gitlab/plans/11-ci-image.md +++ /dev/null @@ -1,189 +0,0 @@ -# Feature: move the CI image to the system-tests repo - -## Context - -The CI image (`registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777`) is -currently defined and built in the `libdatadog-build` repo. Its tag is a manually -bumped integer which gives no indication of what changed. This plan moves the image -definition into the system-tests repo, and introduces a CI job that rebuilds and -pushes it automatically whenever its inputs change, using a content-hash as the tag. - -Reference files: -- `libdatadog-build/docker/system-tests.Dockerfile` — current Dockerfile (to be moved) -- `libdatadog-build/docker/install_kube_dependencies.sh` — helper script (not moved, kept for backward compatibility) -- `requirements.txt` — Python dependencies baked into the image -- `utils/ci/gitlab/main.yml` — references the image tag in every job -- `utils/ci/gitlab/system-tests.yml` — Jinja template: references the image tag in every generated job -- `utils/ci/gitlab/build_pipeline.py` — generates the child pipeline YAML - ---- - -## 1. Create the Dockerfile in the system-tests repo - -Create `utils/ci/gitlab/docker/system-tests.Dockerfile` based on the existing -Dockerfile with the following changes: - -### 1a. Remove unused dependencies - -None of the following are called by any CI pipeline script: - -| Removed | Reason | -|---|---| -| `binutils` | Redundant — already installed transitively by `build-essential` | -| `vault`, `libcap2-bin` | Secrets are fetched via AWS SSM, not Vault | -| Hashicorp GPG key, apt source, `setcap` call | Only present to support vault | -| `install_kube_dependencies.sh` (kind, kubectl, helm) | Only needed for K8s scenarios which run on dedicated infrastructure | - -`jq` is kept: it is used by `utils/scripts/load-binary.sh` for GitHub API calls. - -`install_kube_dependencies.sh` is **not** moved to the system-tests repo — it is -deleted entirely from `libdatadog-build`. - -### 1b. Use a multi-stage build - -Since the Dockerfile now lives inside the system-tests repo, the Docker build -context is the repo itself. Use a two-stage build: - -- **Builder stage**: copies the full repo and runs `./build.sh -i runner` to - produce the venv. Copying everything keeps the Dockerfile simple and ensures - `./build.sh` has access to any file it may need. -- **Final stage**: copies only `/system-tests/venv/` from the builder. All other - repo content is discarded, keeping the image lean. - -```dockerfile -# ---- builder ---- -FROM AS builder -# ... system package installation ... -COPY . /system-tests -WORKDIR /system-tests -RUN ./build.sh -i runner - -# ---- final ---- -FROM -# ... system package installation ... -COPY --from=builder /system-tests/venv /system-tests/venv -WORKDIR / -``` - -Both stages share the same base image and system package installation. The -`COPY --from=builder` line is the only addition over the original single-stage -Dockerfile. - ---- - -## 2. Define the hash - -The image tag is the first 12 hex characters of the SHA-256 hash of the files that -fully determine the image content: - -- `utils/ci/gitlab/docker/system-tests.Dockerfile` -- `requirements.txt` - -```bash -IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt \ - | sha256sum | cut -c1-12) -``` - -The concatenation order must be kept stable across changes. - ---- - -## 3. Define the registry target - -The image is pushed to a hardcoded path: - -``` -registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG -``` - ---- - -## 4. Add the `build_ci_image` job to `main.yml` - -The job: -1. Computes `IMAGE_TAG` -2. Checks whether the image already exists (`docker manifest inspect`). If so, - skips the build entirely — common case has zero overhead. -3. If absent, builds and pushes. -4. Writes `CI_IMAGE` to a dotenv artifact so downstream jobs receive it. - -```yaml -build_ci_image: - image: registry.ddbuild.io/images/docker:20.10.13-jammy - interruptible: true - tags: - - docker-in-docker:amd64 - stage: $[[ inputs.stage ]] - script: - - > - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt - | sha256sum | cut -c1-12) - - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env - - > - if docker manifest inspect registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG > /dev/null 2>&1; then - echo "Image already exists, skipping build"; - else - docker build - -f utils/ci/gitlab/docker/system-tests.Dockerfile - -t registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG - . && - docker push registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG; - fi - artifacts: - reports: - dotenv: build.env - before_script: - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 - --name ci.system-tests.docker-login-write --with-decryption - --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 - --name ci.system-tests.docker-login-pass-write --with-decryption - --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin -``` - -`build_test_pipeline` gets `needs: [job: build_ci_image, artifacts: true]` so it -receives `CI_IMAGE` from the dotenv artifact. No new stage is needed — GitLab's DAG -scheduling runs `build_ci_image` before `build_test_pipeline` even within the same -stage. - ---- - -## 5. Thread `CI_IMAGE` through to generated jobs - -### `main.yml` — `build_test_pipeline` - -- Replace the hardcoded `image:` with `$CI_IMAGE` -- Add `needs: [job: build_ci_image, artifacts: true]` -- Pass `--ci-image "$CI_IMAGE"` to `build_pipeline.py` - -### `utils/ci/gitlab/build_pipeline.py` - -Add `--ci-image` argument and pass it to the Jinja template: - -```python -parser.add_argument("--ci-image", required=True, help="Full CI image reference") -... -template.render(..., ci_image=args.ci_image) -``` - -### `utils/ci/gitlab/system-tests.yml` - -Replace every hardcoded -`image: registry.ddbuild.io/ci/libdatadog-build/system-tests:...` -with `image: {{ci_image}}`. - ---- - -## Summary of file changes - -| File | Change | -|---|---| -| `utils/ci/gitlab/docker/system-tests.Dockerfile` | New — adapted from `libdatadog-build` (unused deps removed, git clone replaced with COPY) | -| `utils/ci/gitlab/main.yml` | Add `build_ci_image` job; `build_test_pipeline` needs it and uses `$CI_IMAGE`; pass `--ci-image` to `build_pipeline.py` | -| `utils/ci/gitlab/build_pipeline.py` | Add `--ci-image` arg, pass to template | -| `utils/ci/gitlab/system-tests.yml` | Replace hardcoded image with `{{ci_image}}` | - -`libdatadog-build/docker/system-tests.Dockerfile` and -`libdatadog-build/docker/install_kube_dependencies.sh` are kept as-is for -backward compatibility. diff --git a/utils/scripts/libraries_and_scenarios_rules.yml b/utils/scripts/libraries_and_scenarios_rules.yml index 093c1f6504d..708730d64e1 100644 --- a/utils/scripts/libraries_and_scenarios_rules.yml +++ b/utils/scripts/libraries_and_scenarios_rules.yml @@ -112,7 +112,6 @@ patterns: scenario_groups: null .gitlab-ci.yml: scenario_groups: null - libraries: null .shellcheck: scenario_groups: null .shellcheckrc: @@ -153,9 +152,6 @@ patterns: run.sh: # run all by default .gitlab/ssi_gitlab-ci.yml: scenario_groups: [onboarding, lib_injection, docker_ssi] - .gitlab/mock.yml: - scenario_groups: null - libraries: null ################################ GitHub Actions ################################ # This section contains rules to apply to files related to GitHub Actions. @@ -196,6 +192,12 @@ patterns: scenarios: PARAMETRIC .github/workflows/run-exotics.yml: scenario_groups: exotics + .github/workflows/stale-prs.yml: + scenario_groups: null + libraries: null + .github/workflows/update-agent-protobuf.yml: + scenario_groups: null + libraries: null .github/*: scenario_groups: null @@ -561,8 +563,3 @@ patterns: scenario_groups: null # already handled by the manifest comparison tests/*: scenario_groups: null # dynamic selection - - - test/*: - libraries: null - scenario_groups: null From f67ba1aa31d6b575bbd1ceeff9cb05404e20c8cc Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 11:46:56 +0200 Subject: [PATCH 073/229] compute-workflow-parameters: add --workflows filter flag --- utils/scripts/compute-workflow-parameters.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index 9048ef66a94..0a08d1548c9 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -43,6 +43,7 @@ def __init__( explicit_binaries_artifact: str, system_tests_dev_mode: bool, ci_environment: str | None, + workflows: str = "", ): # this data struture is a dict where: # the key is the workflow identifier @@ -78,6 +79,10 @@ def __init__( excluded_scenario_names=excluded_scenarios.split(","), ) + if workflows: + allowed = set(_clean_input_value(workflows).split(",")) + scenario_map = {k: v for k, v in scenario_map.items() if k in allowed} + self.data |= get_endtoend_definitions( library, scenario_map, @@ -286,6 +291,13 @@ def _get_workflow_map( ) parser.add_argument("--ci-environment", type=str, help="Explicitly provide CI environment", default=None) + parser.add_argument( + "--workflows", + type=str, + help="Restrict to specific CI workflow types (comma-separated, e.g. 'aws_ssi')", + default="", + ) + args = parser.parse_args() if args.ci_environment is not None: @@ -304,4 +316,5 @@ def _get_workflow_map( explicit_binaries_artifact=args.explicit_binaries_artifact, system_tests_dev_mode=args.system_tests_dev_mode == "true", ci_environment=args.ci_environment, + workflows=args.workflows, ).export(export_format=args.format, output=args.output) From 02eb0f31ed56538cf5e7c5717e1b0b58d137db18 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 11:47:03 +0200 Subject: [PATCH 074/229] ci/gitlab: add aws-ssi.yml for sequential AWS SSI pipeline --- utils/ci/gitlab/aws-ssi.yml | 230 ++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 utils/ci/gitlab/aws-ssi.yml diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml new file mode 100644 index 00000000000..31f9456d1e5 --- /dev/null +++ b/utils/ci/gitlab/aws-ssi.yml @@ -0,0 +1,230 @@ +# AWS SSI pipeline — sequential per-language stages driven by Pulumi/EC2. +# All Docker/K8s SSI scenarios run in main.yml instead. +# This file is included by .gitlab-ci.yml for system-tests own CI, and used +# by tracer repos that run AWS SSI tests. + +include: + - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml + +compute_pipeline: + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + tags: ["arch:amd64"] + stage: configure + variables: + CI_ENVIRONMENT: "prod" + script: + - | + if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then + echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined. Computing changes..." + SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + source venv/bin/activate + ./run.sh MOCK_THE_TEST --collect-only --scenario-report + git clone https://github.com/DataDog/system-tests.git original + git fetch --all + git branch --track $CI_COMMIT_REF_NAME origin/$CI_COMMIT_REF_NAME || true + git checkout $CI_COMMIT_REF_NAME + BASE_COMMIT=$(git merge-base origin/main $CI_COMMIT_REF_NAME) + echo "Branch was created from commit--> $BASE_COMMIT" + git diff --name-only $BASE_COMMIT $CI_COMMIT_SHA >> modified_files.txt + cat modified_files.txt + python utils/scripts/compute_libraries_and_scenarios.py >> impacted_scenarios.txt + cat impacted_scenarios.txt + source impacted_scenarios.txt + else + echo "✅ SYSTEM_TESTS_SCENARIOS OR SYSTEM_TESTS_SCENARIOS_GROUPS are set." + export scenarios=$SYSTEM_TESTS_SCENARIOS + export scenarios_groups=$SYSTEM_TESTS_SCENARIOS_GROUPS + cd /system-tests + git pull + if [ -n "$SYSTEM_TESTS_REF" ]; then + git checkout $SYSTEM_TESTS_REF + else + echo "⚠️ SYSTEM_TESTS_REF variable is not set, skipping git checkout" + fi + SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + source venv/bin/activate + fi + - python utils/scripts/compute-workflow-parameters.py nodejs --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py java --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py dotnet --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py python --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py php --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py ruby --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - | + if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then + cp *.yml "$CI_PROJECT_DIR" + fi + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + artifacts: + paths: + - nodejs_ssi_gitlab_pipeline.yml + - java_ssi_gitlab_pipeline.yml + - dotnet_ssi_gitlab_pipeline.yml + - python_ssi_gitlab_pipeline.yml + - php_ssi_gitlab_pipeline.yml + - ruby_ssi_gitlab_pipeline.yml + +nodejs_ssi_pipeline: + stage: nodejs + needs: ["compute_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: nodejs_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +java_ssi_pipeline: + stage: java + needs: ["compute_pipeline", "nodejs_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: java_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +dotnet_ssi_pipeline: + stage: dotnet + needs: ["compute_pipeline", "java_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: dotnet_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +python_ssi_pipeline: + stage: python + needs: ["compute_pipeline", "dotnet_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: python_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +php_ssi_pipeline: + stage: php + needs: ["compute_pipeline", "python_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: php_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +ruby_ssi_pipeline: + stage: ruby + needs: ["compute_pipeline", "php_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: ruby_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +delete_amis: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + variables: + AMI_RETENTION_DAYS: 10 + AMI_LAST_LAUNCHED_DAYS: 10 + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - python utils/scripts/pulumi_clean_up.py --component amis --ami-retention-days $AMI_RETENTION_DAYS --ami-last-launched-days $AMI_LAST_LAUNCHED_DAYS + rules: + - if: '$SCHEDULED_JOB == "delete_amis"' + after_script: echo "Finish" + timeout: 3h + +delete_amis_by_name_or_lang: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + script: + - | + if [ -z "$AMI_NAME" ] && [ -z "$AMI_LANG" ]; then + echo "❌ ERROR: Either AMI_NAME or AMI_LANG must be set." + exit 1 + fi + echo "✅ Proceeding with AMI_NAME=$AMI_NAME, AMI_LANG=$AMI_LANG" + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - | + CMD="python utils/scripts/pulumi_clean_up.py --component amis_by_name" + if [ -n "$AMI_NAME" ]; then CMD="$CMD --ami-name $AMI_NAME"; fi + if [ -n "$AMI_LANG" ]; then CMD="$CMD --ami-lang $AMI_LANG"; fi + echo "Running: $CMD" + eval $CMD + rules: + - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' + after_script: echo "Finish" + +delete_ec2_instances: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + variables: + EC2_AGE_MINUTES: 45 + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - python utils/scripts/pulumi_clean_up.py --component ec2 --ec2-age-minutes $EC2_AGE_MINUTES + rules: + - if: '$SCHEDULED_JOB == "delete_ec2_instances"' + after_script: echo "Finish" + +count_amis: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - python utils/scripts/pulumi_clean_up.py --component amis_count + rules: + - if: '$SCHEDULED_JOB == "count_amis"' + after_script: echo "Finish" + +.delayed_base_job: + image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 + tags: ["arch:amd64"] + script: + - echo "⏳ Waiting before triggering the child pipeline..." + when: delayed + start_in: 5 minutes From 2a97fd95155a59ecb234e43cd503d499f164a886 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 11:47:10 +0200 Subject: [PATCH 075/229] ci/gitlab: add dockerssi.yml and libinjection.yml Jinja2 templates --- utils/ci/gitlab/dockerssi.yml | 68 ++++++++++++++++++++++++++++++++ utils/ci/gitlab/libinjection.yml | 67 +++++++++++++++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 utils/ci/gitlab/dockerssi.yml create mode 100644 utils/ci/gitlab/libinjection.yml diff --git a/utils/ci/gitlab/dockerssi.yml b/utils/ci/gitlab/dockerssi.yml new file mode 100644 index 00000000000..9f8fdbc2599 --- /dev/null +++ b/utils/ci/gitlab/dockerssi.yml @@ -0,0 +1,68 @@ +{% for scenario, weblogs in dockerssi_scenario_defs.items() %} +{% for weblog, archs in weblogs.items() %} +{% for arch_short, images in archs.items() %} +run_{{library}}_{{scenario}}_{{weblog}}_{{arch_short}}: + extends: .system_tests_base + tags: + - docker-in-docker:{{arch_short}} + variables: + KIND_EXPERIMENTAL_DOCKER_NETWORK: bridge + TEST_LIBRARY: {{library}} + SCENARIO: {{scenario}} + ONBOARDING_FILTER_ENV: {{ci_environment}} + WEBLOG: {{weblog}} + PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com + PRIVATE_DOCKER_REGISTRY_USER: AWS +{% if ssi_library_version %} + DD_INSTALLER_LIBRARY_VERSION: "{{ssi_library_version}}" +{% endif %} +{% if ssi_injector_version %} + DD_INSTALLER_INJECTOR_VERSION: "{{ssi_injector_version}}" +{% endif %} + parallel: + matrix: +{% for img in images %} + - IMAGE: "{{img.image}}" + ARCH: "linux/{{arch_short}}" + RUNTIME: [{% for r in img.runtimes %}"{{r}}"{% if not loop.last %}, {% endif %}{% endfor %}] +{% endfor %} + script: + - section_start "ecr_auth" "ECR auth" + - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} + - section_end "ecr_auth" + - section_start "build" "Building runner" + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - section_end "build" + - section_start "run" "Running {{scenario}}" + - > + timeout 1200s ./run.sh $SCENARIO + --ssi-weblog "$WEBLOG" + --ssi-library "$TEST_LIBRARY" + --ssi-base-image "$IMAGE" + --ssi-arch "$ARCH" + --ssi-installable-runtime "$RUNTIME" + --ssi-env $ONBOARDING_FILTER_ENV + --report-run-url ${CI_JOB_URL} + --report-environment $ONBOARDING_FILTER_ENV + - section_end "run" + after_script: | + SCENARIO_SUFFIX=$(echo "$SCENARIO" | tr "[:upper:]" "[:lower:]") + mkdir -p reports/ + cp -R logs_"${SCENARIO_SUFFIX}" reports/ || true + retry: + max: 2 + when: + - job_execution_timeout + - unknown_failure + - runner_system_failure + exit_codes: + - 3 + - 124 + artifacts: + when: always + paths: + - system-tests/reports/ + +{% endfor %} +{% endfor %} +{% endfor %} diff --git a/utils/ci/gitlab/libinjection.yml b/utils/ci/gitlab/libinjection.yml new file mode 100644 index 00000000000..be1b640c2e5 --- /dev/null +++ b/utils/ci/gitlab/libinjection.yml @@ -0,0 +1,67 @@ +{% for scenario, weblogs in libinjection_scenario_defs.items() %} +run_{{library}}_{{scenario}}: + extends: .system_tests_base + variables: + TEST_LIBRARY: {{library}} + K8S_SCENARIO: {{scenario}} + REPORT_ENVIRONMENT: {{ci_environment}} + PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com + PRIVATE_DOCKER_REGISTRY_USER: AWS +{% if k8s_lib_init_img %} + K8S_LIB_INIT_IMG: "{{k8s_lib_init_img}}" +{% endif %} + parallel: + matrix: +{% for weblog in weblogs %} + - K8S_WEBLOG: {{weblog}} + K8S_WEBLOG_IMG: ${PRIVATE_DOCKER_REGISTRY}/system-tests/{{weblog}} +{% endfor %} + script: + - section_start "ecr_auth" "ECR auth" + - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} + - export PRIVATE_DOCKER_REGISTRY_TOKEN=$(aws ecr get-login-password --region us-east-1) + - section_end "ecr_auth" + - section_start "build" "Building runner" + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - section_end "build" + - section_start "weblog_build" "Building weblog image" + - | + if [ "$CI_PROJECT_NAME" = "system-tests" ] && [ "$CI_PIPELINE_SOURCE" != "schedule" ]; then + TAG_NAME=$([ "$CI_COMMIT_BRANCH" = "main" ] && echo "latest" || echo "$CI_COMMIT_BRANCH" | tr '/' '-') + ./lib-injection/build/build_lib_injection_weblog.sh -w ${K8S_WEBLOG} -l ${TEST_LIBRARY} \ + --push-tag ${K8S_WEBLOG_IMG}:${TAG_NAME} \ + --docker-platform linux/amd64 + else + TAG_NAME="latest" + fi + - section_end "weblog_build" + - section_start "run" "Running {{scenario}}" + - > + ./run.sh ${K8S_SCENARIO} + --k8s-library ${TEST_LIBRARY} + --k8s-weblog ${K8S_WEBLOG} + --k8s-weblog-img ${K8S_WEBLOG_IMG}:${TAG_NAME} + --k8s-lib-init-img ${K8S_LIB_INIT_IMG} + --k8s-injector-img ${K8S_INJECTOR_IMG:-None} + --k8s-cluster-img ${K8S_CLUSTER_IMG:-None} + --report-run-url $CI_JOB_URL + --report-environment ${REPORT_ENVIRONMENT} + - section_end "run" + after_script: | + kind delete clusters --all || true + mkdir -p reports + cp -R logs_*/ reports/ || true + retry: + max: 2 + when: + - job_execution_timeout + - unknown_failure + - runner_system_failure + exit_codes: + - 3 + artifacts: + when: always + paths: + - system-tests/reports/ + +{% endfor %} From a7baba0085693d5dc96122037cf7da27f8702874 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 11:47:16 +0200 Subject: [PATCH 076/229] ci/gitlab: extend build_pipeline.py and system-tests.yml with SSI support --- utils/ci/gitlab/build_pipeline.py | 52 +++++++++++++++++++++++++++++++ utils/ci/gitlab/system-tests.yml | 8 +++++ 2 files changed, 60 insertions(+) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index dba8e4afa17..98a81b83197 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -13,6 +13,9 @@ parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") parser.add_argument("--test-optimization-datadog-site", default="datadoghq.com", help="Datadog site for Test Optimization") +parser.add_argument("--ssi-library-version", default="", help="DD_INSTALLER_LIBRARY_VERSION for Docker/K8s SSI jobs") +parser.add_argument("--k8s-lib-init-img", default="", help="K8S_LIB_INIT_IMG for libinjection jobs") +parser.add_argument("--ssi-injector-version", default="", help="DD_INSTALLER_INJECTOR_VERSION for dockerssi jobs") args = parser.parse_args() with open(args.params) as f: @@ -20,10 +23,37 @@ scenario_list = params["endtoend"]["scenarios"] weblog_variants = params["endtoend"]["weblogs"] binaries_artifact = params["miscs"]["binaries_artifact"] +ci_environment = params["miscs"]["ci_environment"] parametric = params["parametric"] +dockerssi_scenario_defs = params.get("dockerssi_scenario_defs", {}) +libinjection_scenario_defs = params.get("libinjection_scenario_defs", {}) env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) + +def _transform_dockerssi(raw: dict) -> dict: + """Regroup dockerssi images by arch for easier Jinja2 iteration. + + Input: {scenario: {weblog: [{image_name: [runtimes], "arch": arch}, ...]}} + Output: {scenario: {weblog: {arch_short: [{image, runtimes}, ...]}}} + """ + result: dict = {} + for scenario, weblogs in raw.items(): + result[scenario] = {} + for weblog, images in weblogs.items(): + by_arch: dict = {} + for img in images: + arch = img["arch"] + arch_short = arch.replace("linux/", "") + if arch_short not in by_arch: + by_arch[arch_short] = [] + for key, val in img.items(): + if key != "arch": + by_arch[arch_short].append({"image": key, "runtimes": val}) + result[scenario][weblog] = by_arch + return result + + template = env.get_template("system-tests.yml") print(template.render( @@ -32,9 +62,31 @@ library=args.library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact, + ci_environment=ci_environment, parametric=parametric, ci_image=args.ci_image, ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", test_optimization_datadog_site=args.test_optimization_datadog_site, + ssi_library_version=args.ssi_library_version, + k8s_lib_init_img=args.k8s_lib_init_img, )) + +if dockerssi_scenario_defs: + dockerssi_template = env.get_template("dockerssi.yml") + print(dockerssi_template.render( + library=args.library, + ci_environment=ci_environment, + dockerssi_scenario_defs=_transform_dockerssi(dockerssi_scenario_defs), + ssi_library_version=args.ssi_library_version, + ssi_injector_version=args.ssi_injector_version, + )) + +if libinjection_scenario_defs: + libinjection_template = env.get_template("libinjection.yml") + print(libinjection_template.render( + library=args.library, + ci_environment=ci_environment, + libinjection_scenario_defs=libinjection_scenario_defs, + k8s_lib_init_img=args.k8s_lib_init_img, + )) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 45e1698e810..6f72780c643 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -62,6 +62,14 @@ run_{{library}}_{{scenario}}_{{variant}}: needs: - job: build_{{library}}_{{variant}} artifacts: true + variables: + WEBLOG_VARIANT: {{variant}} +{% if ssi_library_version %} + DD_INSTALLER_LIBRARY_VERSION: "{{ssi_library_version}}" +{% endif %} +{% if k8s_lib_init_img %} + K8S_LIB_INIT_IMG: "{{k8s_lib_init_img}}" +{% endif %} script: - section_start "docker_auth" "Docker hub auth" - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) From df52e8e29395959b3f02c20939bc06bc1aa6e86d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 11:47:21 +0200 Subject: [PATCH 077/229] ci/gitlab: add build_stage, run_stage, and SSI inputs to main.yml --- utils/ci/gitlab/main.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index c6fe0020c2d..ac37bf93258 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -1,7 +1,13 @@ spec: inputs: stage: - description: "CI stage for the jobs" + description: "CI stage for the generated test jobs (inside the child pipeline)" + build_stage: + description: "CI stage for build_test_pipeline and resolve_ci_image" + default: "build" + run_stage: + description: "CI stage for run_test_pipeline" + default: "run" libraries: default: "java python nodejs" scenarios: @@ -43,6 +49,15 @@ spec: ref: description: "system-tests ref to use when called from another repository (branch, tag or SHA)" default: "main" + ssi_library_version: + description: "DD_INSTALLER_LIBRARY_VERSION to inject into generated jobs (Docker/K8s SSI)" + default: "" + k8s_lib_init_img: + description: "K8S_LIB_INIT_IMG to inject into generated libinjection jobs" + default: "" + ssi_injector_version: + description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" + default: "" --- include: @@ -62,7 +77,7 @@ resolve_ci_image: image: registry.ddbuild.io/system-tests/git tags: - arch:amd64 - stage: $[[ inputs.stage ]] + stage: $[[ inputs.build_stage ]] interruptible: true variables: GIT_STRATEGY: none @@ -84,6 +99,7 @@ build_test_pipeline: extends: .system_tests_base tags: - arch:amd64 + stage: $[[ inputs.build_stage ]] needs: - job: resolve_ci_image artifacts: true @@ -94,7 +110,7 @@ build_test_pipeline: - > for library in $[[ inputs.libraries ]]; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" >> generated-pipeline.yml; + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" --ssi-library-version "$[[ inputs.ssi_library_version ]]" --k8s-lib-init-img "$[[ inputs.k8s_lib_init_img ]]" --ssi-injector-version "$[[ inputs.ssi_injector_version ]]" >> generated-pipeline.yml; done artifacts: paths: @@ -102,7 +118,7 @@ build_test_pipeline: run_test_pipeline: interruptible: true - stage: run + stage: $[[ inputs.run_stage ]] needs: - job: build_test_pipeline artifacts: true From 97f353815c32a072a0407434d3d868b625a620c7 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 11:47:27 +0200 Subject: [PATCH 078/229] gitlab-ci: include aws-ssi.yml, add publish job, expand stages --- .gitlab-ci.yml | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89b127e7f9a..dafeffbfdc2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,11 +7,22 @@ include: excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true + - local: "/utils/ci/gitlab/aws-ssi.yml" stages: - build - test - run + - configure + - nodejs + - java + - dotnet + - python + - php + - ruby + - pipeline-status + - system-tests-utils + - publish build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy @@ -32,3 +43,75 @@ build_ci_image: artifacts: reports: dotenv: build.env + +check_merge_labels: + image: registry.ddbuild.io/system-tests/base-image-builder + tags: + - arch:amd64 + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + needs: [] + stage: system-tests-utils + before_script: + - dd-octo-sts version + - dd-octo-sts debug --scope DataDog/system-tests --policy self.gitlab-read + - dd-octo-sts token --scope DataDog/system-tests --policy self.gitlab-read > token.txt + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + script: + - export GITHUB_TOKEN=$(cat token.txt) + - ./utils/scripts/get_pr_merged_labels.sh + after_script: + - dd-octo-sts revoke -t $(cat token.txt) + rules: + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" + +generate_system_tests_lambda_proxy_image: + image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 + tags: ["docker-in-docker:amd64"] + needs: [] + stage: system-tests-utils + allow_failure: false + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i lambda-proxy + - docker push datadog/system-tests:lambda-proxy-v1 + rules: + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" + changes: + - utils/build/docker/lambda_proxy/pyproject.toml + - utils/build/docker/lambda-proxy.Dockerfile + +generate_system_tests_lib_injection_images: + extends: .base_job_k8s_docker_ssi + needs: [] + stage: system-tests-utils + allow_failure: true + variables: + PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com + PRIVATE_DOCKER_REGISTRY_USER: AWS + script: + - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} + - ./lib-injection/build/build_lib_injection_images.sh $PRIVATE_DOCKER_REGISTRY + rules: + - if: '$SCHEDULED_JOB == "generate_system_tests_lib_injection_images"' + - when: manual + +publish_gitlab_component: + image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 + tags: ["arch:amd64"] + needs: [] + stage: publish + script: + - aws s3 cp utils/ci/gitlab/main.yml s3://gitlab-templates.ddbuild.io/system-tests/include/main.yml + - aws s3 cp utils/ci/gitlab/system-tests.yml s3://gitlab-templates.ddbuild.io/system-tests/include/system-tests.yml + - aws s3 cp utils/ci/gitlab/dockerssi.yml s3://gitlab-templates.ddbuild.io/system-tests/include/dockerssi.yml + - aws s3 cp utils/ci/gitlab/libinjection.yml s3://gitlab-templates.ddbuild.io/system-tests/include/libinjection.yml + - aws s3 cp utils/ci/gitlab/build_pipeline.py s3://gitlab-templates.ddbuild.io/system-tests/include/build_pipeline.py + - aws s3 cp utils/ci/gitlab/templates.yml s3://gitlab-templates.ddbuild.io/system-tests/include/templates.yml + rules: + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" From 84432eb106f4dc37ca0f2c391e2f957f60af010e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 11:47:33 +0200 Subject: [PATCH 079/229] ci/gitlab: add tests for dockerssi, libinjection, and SSI variable forwarding --- utils/ci/gitlab/test_build_pipeline.py | 333 +++++++++++++++++++++++++ 1 file changed, 333 insertions(+) create mode 100644 utils/ci/gitlab/test_build_pipeline.py diff --git a/utils/ci/gitlab/test_build_pipeline.py b/utils/ci/gitlab/test_build_pipeline.py new file mode 100644 index 00000000000..c4c849facdf --- /dev/null +++ b/utils/ci/gitlab/test_build_pipeline.py @@ -0,0 +1,333 @@ +"""Tests for build_pipeline.py""" +import json +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +SCRIPT = Path(__file__).parent / "build_pipeline.py" +CI_IMAGE = "registry.example.com/ci-runner:abc123" +STAGE = "test" +LIBRARY = "python" + + +def _params( + scenarios=("DEFAULT",), + weblogs=("flask",), + binaries_artifact="", + parametric_enable=False, + parametric_job_count=1, + dockerssi_scenario_defs=None, + libinjection_scenario_defs=None, +): + return { + "endtoend": { + "scenarios": list(scenarios), + "weblogs": list(weblogs), + }, + "miscs": { + "binaries_artifact": binaries_artifact, + "ci_environment": "gitlab", + }, + "parametric": { + "job_count": parametric_job_count, + "job_matrix": list(range(1, parametric_job_count + 1)), + "enable": parametric_enable, + }, + "dockerssi_scenario_defs": dockerssi_scenario_defs or {}, + "libinjection_scenario_defs": libinjection_scenario_defs or {}, + } + + +def _run(params, extra_args=(), ref="abc1234"): + params_file = pytest.tmp_path_factory.mktemp("params") / "params.json" + params_file.write_text(json.dumps(params)) + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--stage", STAGE, + "--library", LIBRARY, + "--params", str(params_file), + "--ci-image", CI_IMAGE, + "--ref", ref, + *extra_args, + ], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + +@pytest.fixture() +def run(tmp_path): + def _inner(params, extra_args=(), ref="abc1234"): + params_file = tmp_path / "params.json" + params_file.write_text(json.dumps(params)) + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--stage", STAGE, + "--library", LIBRARY, + "--params", str(params_file), + "--ci-image", CI_IMAGE, + "--ref", ref, + *extra_args, + ], + capture_output=True, + text=True, + check=True, + ) + return result.stdout + + return _inner + + +class TestOutputIsValidYaml: + def test_basic(self, run): + output = run(_params()) + assert yaml.safe_load(output) is not None + + def test_multiple_scenarios_and_weblogs(self, run): + output = run(_params(scenarios=["DEFAULT", "APPSEC_SCENARIOS"], weblogs=["flask", "django"])) + assert yaml.safe_load(output) is not None + + def test_with_parametric(self, run): + output = run(_params(parametric_enable=True, parametric_job_count=3)) + assert yaml.safe_load(output) is not None + + def test_with_push_to_test_optimization(self, run): + output = run(_params(), extra_args=["--push-to-test-optimization", "true"]) + assert yaml.safe_load(output) is not None + + +class TestBuildJobs: + def test_one_build_job_per_weblog(self, run): + output = run(_params(weblogs=["flask", "django"])) + jobs = yaml.safe_load(output) + assert f"build_{LIBRARY}_flask" in jobs + assert f"build_{LIBRARY}_django" in jobs + + def test_build_job_has_no_needs_without_binaries_artifact(self, run): + output = run(_params(weblogs=["flask"])) + jobs = yaml.safe_load(output) + assert "needs" not in jobs[f"build_{LIBRARY}_flask"] + + def test_build_job_needs_binaries_artifact(self, run): + output = run(_params(weblogs=["flask"], binaries_artifact="upstream_binaries")) + jobs = yaml.safe_load(output) + needs = jobs[f"build_{LIBRARY}_flask"]["needs"] + assert any(n.get("job") == "upstream_binaries" for n in needs) + + +class TestRunJobs: + def test_one_run_job_per_scenario_and_weblog(self, run): + output = run(_params(scenarios=["DEFAULT", "APPSEC_SCENARIOS"], weblogs=["flask", "django"])) + jobs = yaml.safe_load(output) + for scenario in ["DEFAULT", "APPSEC_SCENARIOS"]: + for weblog in ["flask", "django"]: + assert f"run_{LIBRARY}_{scenario}_{weblog}" in jobs + + def test_run_job_needs_build_job(self, run): + output = run(_params(scenarios=["DEFAULT"], weblogs=["flask"])) + jobs = yaml.safe_load(output) + needs = jobs[f"run_{LIBRARY}_DEFAULT_flask"]["needs"] + assert any(n.get("job") == f"build_{LIBRARY}_flask" for n in needs) + + +class TestParametricJobs: + def test_no_parametric_jobs_when_disabled(self, run): + output = run(_params(parametric_enable=False)) + jobs = yaml.safe_load(output) + assert not any(k.startswith(f"run_{LIBRARY}_PARAMETRIC_") for k in jobs) + + def test_parametric_jobs_created_when_enabled(self, run): + output = run(_params(parametric_enable=True, parametric_job_count=2)) + jobs = yaml.safe_load(output) + assert f"run_{LIBRARY}_PARAMETRIC_1" in jobs + assert f"run_{LIBRARY}_PARAMETRIC_2" in jobs + + def test_parametric_job_count(self, run): + output = run(_params(parametric_enable=True, parametric_job_count=4)) + jobs = yaml.safe_load(output) + parametric_jobs = [k for k in jobs if k.startswith(f"run_{LIBRARY}_PARAMETRIC_")] + assert len(parametric_jobs) == 4 + + +class TestPushTestOptimization: + def test_no_push_job_by_default(self, run): + output = run(_params(scenarios=["DEFAULT"], weblogs=["flask"])) + jobs = yaml.safe_load(output) + assert f"push_{LIBRARY}_test_optimization" not in jobs + + def test_push_job_created_when_enabled(self, run): + output = run(_params(scenarios=["DEFAULT"], weblogs=["flask"]), extra_args=["--push-to-test-optimization", "true"]) + jobs = yaml.safe_load(output) + assert f"push_{LIBRARY}_test_optimization" in jobs + + def test_push_job_needs_all_run_jobs(self, run): + output = run( + _params(scenarios=["DEFAULT", "APPSEC_SCENARIOS"], weblogs=["flask"]), + extra_args=["--push-to-test-optimization", "true"], + ) + jobs = yaml.safe_load(output) + push_needs = {n["job"] for n in jobs[f"push_{LIBRARY}_test_optimization"]["needs"]} + assert f"run_{LIBRARY}_DEFAULT_flask" in push_needs + assert f"run_{LIBRARY}_APPSEC_SCENARIOS_flask" in push_needs + + def test_push_job_uses_custom_datadog_site(self, run): + output = run( + _params(scenarios=["DEFAULT"], weblogs=["flask"]), + extra_args=["--push-to-test-optimization", "true", "--test-optimization-datadog-site", "datadoghq.eu"], + ) + jobs = yaml.safe_load(output) + push_job = jobs[f"push_{LIBRARY}_test_optimization"] + assert push_job["variables"]["DATADOG_SITE"] == "datadoghq.eu" + + +class TestCiImageAndRef: + def test_ci_image_used_in_push_job(self, run): + output = run( + _params(scenarios=["DEFAULT"], weblogs=["flask"]), + extra_args=["--push-to-test-optimization", "true"], + ) + jobs = yaml.safe_load(output) + push_job = jobs[f"push_{LIBRARY}_test_optimization"] + assert push_job["image"] == CI_IMAGE + + def test_ref_passed_to_template_include(self, run): + output = run(_params(), ref="deadbeef") + # The include block is at the top of the YAML + parsed = yaml.safe_load(output) + include = parsed.get("include", []) + assert any("deadbeef" in str(item) for item in include) + + +_DOCKERSSI_DEFS = { + "DOCKER_SSI": { + "my-weblog": [ + {"ubuntu:22.04": ["3.8", "3.9"], "arch": "linux/amd64"}, + {"ubuntu:22.04": ["3.8"], "arch": "linux/arm64"}, + ] + } +} + +_LIBINJECTION_DEFS = { + "K8S_LIB_INJECTION": ["weblog1", "weblog2"], + "K8S_LIB_INJECTION_NO_AC": ["weblog1"], +} + + +class TestDockerSSI: + def test_no_dockerssi_jobs_when_empty(self, run): + output = run(_params()) + jobs = yaml.safe_load(output) + assert not any("DOCKER_SSI" in k for k in jobs) + + def test_dockerssi_jobs_created(self, run): + output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) + jobs = yaml.safe_load(output) + assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64" in jobs + assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_arm64" in jobs + + def test_dockerssi_job_has_parallel_matrix(self, run): + output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) + jobs = yaml.safe_load(output) + job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64"] + assert "parallel" in job + matrix = job["parallel"]["matrix"] + assert len(matrix) == 1 + assert matrix[0]["IMAGE"] == "ubuntu:22.04" + assert "3.8" in matrix[0]["RUNTIME"] + assert "3.9" in matrix[0]["RUNTIME"] + + def test_dockerssi_job_sets_ssi_library_version(self, run): + output = run( + _params(dockerssi_scenario_defs=_DOCKERSSI_DEFS), + extra_args=["--ssi-library-version", "1.2.3"], + ) + jobs = yaml.safe_load(output) + job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64"] + assert job["variables"]["DD_INSTALLER_LIBRARY_VERSION"] == "1.2.3" + + def test_dockerssi_job_no_ssi_version_by_default(self, run): + output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) + jobs = yaml.safe_load(output) + job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64"] + assert "DD_INSTALLER_LIBRARY_VERSION" not in job.get("variables", {}) + + def test_output_is_valid_yaml_with_dockerssi(self, run): + output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) + assert yaml.safe_load(output) is not None + + +class TestLibInjection: + def test_no_libinjection_jobs_when_empty(self, run): + output = run(_params()) + jobs = yaml.safe_load(output) + assert not any("K8S_LIB_INJECTION" in k for k in jobs) + + def test_libinjection_jobs_created(self, run): + output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) + jobs = yaml.safe_load(output) + assert f"run_{LIBRARY}_K8S_LIB_INJECTION" in jobs + assert f"run_{LIBRARY}_K8S_LIB_INJECTION_NO_AC" in jobs + + def test_libinjection_job_has_parallel_matrix(self, run): + output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) + jobs = yaml.safe_load(output) + job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION"] + assert "parallel" in job + weblogs = {e["K8S_WEBLOG"] for e in job["parallel"]["matrix"]} + assert weblogs == {"weblog1", "weblog2"} + + def test_libinjection_job_sets_k8s_lib_init_img(self, run): + output = run( + _params(libinjection_scenario_defs=_LIBINJECTION_DEFS), + extra_args=["--k8s-lib-init-img", "my-registry/lib-init:latest"], + ) + jobs = yaml.safe_load(output) + job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION"] + assert job["variables"]["K8S_LIB_INIT_IMG"] == "my-registry/lib-init:latest" + + def test_libinjection_job_no_k8s_img_by_default(self, run): + output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) + jobs = yaml.safe_load(output) + job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION"] + assert "K8S_LIB_INIT_IMG" not in job.get("variables", {}) + + def test_output_is_valid_yaml_with_libinjection(self, run): + output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) + assert yaml.safe_load(output) is not None + + +class TestSsiVariablesInEndToEndJobs: + def test_ssi_library_version_in_run_job(self, run): + output = run( + _params(scenarios=["DEFAULT"], weblogs=["flask"]), + extra_args=["--ssi-library-version", "2.0.0"], + ) + jobs = yaml.safe_load(output) + job = jobs[f"run_{LIBRARY}_DEFAULT_flask"] + assert job["variables"]["DD_INSTALLER_LIBRARY_VERSION"] == "2.0.0" + + def test_k8s_lib_init_img_in_run_job(self, run): + output = run( + _params(scenarios=["DEFAULT"], weblogs=["flask"]), + extra_args=["--k8s-lib-init-img", "my-registry/lib-init:v1"], + ) + jobs = yaml.safe_load(output) + job = jobs[f"run_{LIBRARY}_DEFAULT_flask"] + assert job["variables"]["K8S_LIB_INIT_IMG"] == "my-registry/lib-init:v1" + + def test_no_ssi_variables_by_default(self, run): + output = run(_params(scenarios=["DEFAULT"], weblogs=["flask"])) + jobs = yaml.safe_load(output) + job = jobs[f"run_{LIBRARY}_DEFAULT_flask"] + variables = job.get("variables", {}) + assert "DD_INSTALLER_LIBRARY_VERSION" not in variables + assert "K8S_LIB_INIT_IMG" not in variables From 1bf5a586f0fb78c4b0881e00f1c1bd299591bae3 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 12:19:51 +0200 Subject: [PATCH 080/229] ci/gitlab: use needs: for ordering, single stage input per component --- .gitlab-ci.yml | 29 +++-------------------------- utils/ci/gitlab/aws-ssi.yml | 34 ++++++++++++++++++++-------------- utils/ci/gitlab/main.yml | 13 +++---------- 3 files changed, 26 insertions(+), 50 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dafeffbfdc2..d52acf46c1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,28 +1,19 @@ include: - local: "/utils/ci/gitlab/main.yml" inputs: - stage: "test" + stage: "build" libraries: "python" scenarios_groups: "all" excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true - local: "/utils/ci/gitlab/aws-ssi.yml" + inputs: + stage: "build" stages: - build - - test - - run - - configure - - nodejs - - java - - dotnet - - python - - php - - ruby - - pipeline-status - system-tests-utils - - publish build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy @@ -101,17 +92,3 @@ generate_system_tests_lib_injection_images: - if: '$SCHEDULED_JOB == "generate_system_tests_lib_injection_images"' - when: manual -publish_gitlab_component: - image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 - tags: ["arch:amd64"] - needs: [] - stage: publish - script: - - aws s3 cp utils/ci/gitlab/main.yml s3://gitlab-templates.ddbuild.io/system-tests/include/main.yml - - aws s3 cp utils/ci/gitlab/system-tests.yml s3://gitlab-templates.ddbuild.io/system-tests/include/system-tests.yml - - aws s3 cp utils/ci/gitlab/dockerssi.yml s3://gitlab-templates.ddbuild.io/system-tests/include/dockerssi.yml - - aws s3 cp utils/ci/gitlab/libinjection.yml s3://gitlab-templates.ddbuild.io/system-tests/include/libinjection.yml - - aws s3 cp utils/ci/gitlab/build_pipeline.py s3://gitlab-templates.ddbuild.io/system-tests/include/build_pipeline.py - - aws s3 cp utils/ci/gitlab/templates.yml s3://gitlab-templates.ddbuild.io/system-tests/include/templates.yml - rules: - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml index 31f9456d1e5..9a29489a770 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/aws-ssi.yml @@ -1,7 +1,13 @@ -# AWS SSI pipeline — sequential per-language stages driven by Pulumi/EC2. +spec: + inputs: + stage: + description: "CI stage for all jobs in this component" + +--- + +# AWS SSI pipeline — sequential per-language execution driven by Pulumi/EC2. # All Docker/K8s SSI scenarios run in main.yml instead. -# This file is included by .gitlab-ci.yml for system-tests own CI, and used -# by tracer repos that run AWS SSI tests. +# Execution order is expressed through needs: chains; stages are not used for ordering. include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml @@ -9,7 +15,7 @@ include: compute_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 tags: ["arch:amd64"] - stage: configure + stage: $[[ inputs.stage ]] variables: CI_ENVIRONMENT: "prod" script: @@ -67,7 +73,7 @@ compute_pipeline: - ruby_ssi_gitlab_pipeline.yml nodejs_ssi_pipeline: - stage: nodejs + stage: $[[ inputs.stage ]] needs: ["compute_pipeline"] variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE @@ -82,7 +88,7 @@ nodejs_ssi_pipeline: when: always java_ssi_pipeline: - stage: java + stage: $[[ inputs.stage ]] needs: ["compute_pipeline", "nodejs_ssi_pipeline"] variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE @@ -97,7 +103,7 @@ java_ssi_pipeline: when: always dotnet_ssi_pipeline: - stage: dotnet + stage: $[[ inputs.stage ]] needs: ["compute_pipeline", "java_ssi_pipeline"] variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE @@ -112,7 +118,7 @@ dotnet_ssi_pipeline: when: always python_ssi_pipeline: - stage: python + stage: $[[ inputs.stage ]] needs: ["compute_pipeline", "dotnet_ssi_pipeline"] variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE @@ -127,7 +133,7 @@ python_ssi_pipeline: when: always php_ssi_pipeline: - stage: php + stage: $[[ inputs.stage ]] needs: ["compute_pipeline", "python_ssi_pipeline"] variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE @@ -142,7 +148,7 @@ php_ssi_pipeline: when: always ruby_ssi_pipeline: - stage: ruby + stage: $[[ inputs.stage ]] needs: ["compute_pipeline", "php_ssi_pipeline"] variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE @@ -158,7 +164,7 @@ ruby_ssi_pipeline: delete_amis: extends: .base_job_onboarding - stage: system-tests-utils + stage: $[[ inputs.stage ]] allow_failure: false variables: AMI_RETENTION_DAYS: 10 @@ -174,7 +180,7 @@ delete_amis: delete_amis_by_name_or_lang: extends: .base_job_onboarding - stage: system-tests-utils + stage: $[[ inputs.stage ]] allow_failure: false script: - | @@ -197,7 +203,7 @@ delete_amis_by_name_or_lang: delete_ec2_instances: extends: .base_job_onboarding - stage: system-tests-utils + stage: $[[ inputs.stage ]] allow_failure: false variables: EC2_AGE_MINUTES: 45 @@ -211,7 +217,7 @@ delete_ec2_instances: count_amis: extends: .base_job_onboarding - stage: system-tests-utils + stage: $[[ inputs.stage ]] allow_failure: false script: - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index ac37bf93258..f90d97a6c77 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -1,13 +1,7 @@ spec: inputs: stage: - description: "CI stage for the generated test jobs (inside the child pipeline)" - build_stage: - description: "CI stage for build_test_pipeline and resolve_ci_image" - default: "build" - run_stage: - description: "CI stage for run_test_pipeline" - default: "run" + description: "CI stage for all jobs in this component and the generated child pipeline" libraries: default: "java python nodejs" scenarios: @@ -77,7 +71,7 @@ resolve_ci_image: image: registry.ddbuild.io/system-tests/git tags: - arch:amd64 - stage: $[[ inputs.build_stage ]] + stage: $[[ inputs.stage ]] interruptible: true variables: GIT_STRATEGY: none @@ -99,7 +93,6 @@ build_test_pipeline: extends: .system_tests_base tags: - arch:amd64 - stage: $[[ inputs.build_stage ]] needs: - job: resolve_ci_image artifacts: true @@ -118,7 +111,7 @@ build_test_pipeline: run_test_pipeline: interruptible: true - stage: $[[ inputs.run_stage ]] + stage: $[[ inputs.stage ]] needs: - job: build_test_pipeline artifacts: true From 34b474f66e275a6feb5007149716254f37573097 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 12:27:31 +0200 Subject: [PATCH 081/229] ci/gitlab: remove dead SSI variables from endtoend run jobs --- utils/ci/gitlab/build_pipeline.py | 2 -- utils/ci/gitlab/system-tests.yml | 8 -------- utils/ci/gitlab/test_build_pipeline.py | 23 ++++------------------- 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 98a81b83197..6d1518ec013 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -68,8 +68,6 @@ def _transform_dockerssi(raw: dict) -> dict: ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", test_optimization_datadog_site=args.test_optimization_datadog_site, - ssi_library_version=args.ssi_library_version, - k8s_lib_init_img=args.k8s_lib_init_img, )) if dockerssi_scenario_defs: diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 6f72780c643..45e1698e810 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -62,14 +62,6 @@ run_{{library}}_{{scenario}}_{{variant}}: needs: - job: build_{{library}}_{{variant}} artifacts: true - variables: - WEBLOG_VARIANT: {{variant}} -{% if ssi_library_version %} - DD_INSTALLER_LIBRARY_VERSION: "{{ssi_library_version}}" -{% endif %} -{% if k8s_lib_init_img %} - K8S_LIB_INIT_IMG: "{{k8s_lib_init_img}}" -{% endif %} script: - section_start "docker_auth" "Docker hub auth" - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) diff --git a/utils/ci/gitlab/test_build_pipeline.py b/utils/ci/gitlab/test_build_pipeline.py index c4c849facdf..53c4c99d0b3 100644 --- a/utils/ci/gitlab/test_build_pipeline.py +++ b/utils/ci/gitlab/test_build_pipeline.py @@ -305,29 +305,14 @@ def test_output_is_valid_yaml_with_libinjection(self, run): assert yaml.safe_load(output) is not None -class TestSsiVariablesInEndToEndJobs: - def test_ssi_library_version_in_run_job(self, run): +class TestSsiVariablesNotLeakedIntoEndToEndJobs: + def test_no_ssi_variables_in_endtoend_run_job(self, run): output = run( _params(scenarios=["DEFAULT"], weblogs=["flask"]), - extra_args=["--ssi-library-version", "2.0.0"], + extra_args=["--ssi-library-version", "2.0.0", "--k8s-lib-init-img", "my-registry/lib-init:v1"], ) jobs = yaml.safe_load(output) job = jobs[f"run_{LIBRARY}_DEFAULT_flask"] - assert job["variables"]["DD_INSTALLER_LIBRARY_VERSION"] == "2.0.0" - - def test_k8s_lib_init_img_in_run_job(self, run): - output = run( - _params(scenarios=["DEFAULT"], weblogs=["flask"]), - extra_args=["--k8s-lib-init-img", "my-registry/lib-init:v1"], - ) - jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_DEFAULT_flask"] - assert job["variables"]["K8S_LIB_INIT_IMG"] == "my-registry/lib-init:v1" - - def test_no_ssi_variables_by_default(self, run): - output = run(_params(scenarios=["DEFAULT"], weblogs=["flask"])) - jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_DEFAULT_flask"] - variables = job.get("variables", {}) + variables = job.get("variables") or {} assert "DD_INSTALLER_LIBRARY_VERSION" not in variables assert "K8S_LIB_INIT_IMG" not in variables From 8caf369bf0d3110c24c5d667a822b410231fc502 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 12:32:31 +0200 Subject: [PATCH 082/229] ci/gitlab: replace parallel matrices with explicit Jinja2 loops --- utils/ci/gitlab/dockerssi.yml | 17 +++++----- utils/ci/gitlab/libinjection.yml | 12 +++---- utils/ci/gitlab/test_build_pipeline.py | 43 +++++++++++++------------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/utils/ci/gitlab/dockerssi.yml b/utils/ci/gitlab/dockerssi.yml index 9f8fdbc2599..aee5917424d 100644 --- a/utils/ci/gitlab/dockerssi.yml +++ b/utils/ci/gitlab/dockerssi.yml @@ -1,7 +1,10 @@ {% for scenario, weblogs in dockerssi_scenario_defs.items() %} {% for weblog, archs in weblogs.items() %} {% for arch_short, images in archs.items() %} -run_{{library}}_{{scenario}}_{{weblog}}_{{arch_short}}: +{% for img in images %} +{% for runtime in img.runtimes %} +{% set runtime_suffix = ('_' + runtime | replace('.', '_')) if runtime else '' %} +run_{{library}}_{{scenario}}_{{weblog}}_{{arch_short}}{{runtime_suffix}}: extends: .system_tests_base tags: - docker-in-docker:{{arch_short}} @@ -11,6 +14,9 @@ run_{{library}}_{{scenario}}_{{weblog}}_{{arch_short}}: SCENARIO: {{scenario}} ONBOARDING_FILTER_ENV: {{ci_environment}} WEBLOG: {{weblog}} + IMAGE: {{img.image}} + ARCH: linux/{{arch_short}} + RUNTIME: "{{runtime}}" PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com PRIVATE_DOCKER_REGISTRY_USER: AWS {% if ssi_library_version %} @@ -19,13 +25,6 @@ run_{{library}}_{{scenario}}_{{weblog}}_{{arch_short}}: {% if ssi_injector_version %} DD_INSTALLER_INJECTOR_VERSION: "{{ssi_injector_version}}" {% endif %} - parallel: - matrix: -{% for img in images %} - - IMAGE: "{{img.image}}" - ARCH: "linux/{{arch_short}}" - RUNTIME: [{% for r in img.runtimes %}"{{r}}"{% if not loop.last %}, {% endif %}{% endfor %}] -{% endfor %} script: - section_start "ecr_auth" "ECR auth" - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} @@ -66,3 +65,5 @@ run_{{library}}_{{scenario}}_{{weblog}}_{{arch_short}}: {% endfor %} {% endfor %} {% endfor %} +{% endfor %} +{% endfor %} diff --git a/utils/ci/gitlab/libinjection.yml b/utils/ci/gitlab/libinjection.yml index be1b640c2e5..1b666993080 100644 --- a/utils/ci/gitlab/libinjection.yml +++ b/utils/ci/gitlab/libinjection.yml @@ -1,21 +1,18 @@ {% for scenario, weblogs in libinjection_scenario_defs.items() %} -run_{{library}}_{{scenario}}: +{% for weblog in weblogs %} +run_{{library}}_{{scenario}}_{{weblog}}: extends: .system_tests_base variables: TEST_LIBRARY: {{library}} K8S_SCENARIO: {{scenario}} + K8S_WEBLOG: {{weblog}} + K8S_WEBLOG_IMG: ${PRIVATE_DOCKER_REGISTRY}/system-tests/{{weblog}} REPORT_ENVIRONMENT: {{ci_environment}} PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com PRIVATE_DOCKER_REGISTRY_USER: AWS {% if k8s_lib_init_img %} K8S_LIB_INIT_IMG: "{{k8s_lib_init_img}}" {% endif %} - parallel: - matrix: -{% for weblog in weblogs %} - - K8S_WEBLOG: {{weblog}} - K8S_WEBLOG_IMG: ${PRIVATE_DOCKER_REGISTRY}/system-tests/{{weblog}} -{% endfor %} script: - section_start "ecr_auth" "ECR auth" - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} @@ -65,3 +62,4 @@ run_{{library}}_{{scenario}}: - system-tests/reports/ {% endfor %} +{% endfor %} diff --git a/utils/ci/gitlab/test_build_pipeline.py b/utils/ci/gitlab/test_build_pipeline.py index 53c4c99d0b3..95bd4313fb9 100644 --- a/utils/ci/gitlab/test_build_pipeline.py +++ b/utils/ci/gitlab/test_build_pipeline.py @@ -231,19 +231,19 @@ def test_no_dockerssi_jobs_when_empty(self, run): def test_dockerssi_jobs_created(self, run): output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) jobs = yaml.safe_load(output) - assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64" in jobs - assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_arm64" in jobs + # one job per (weblog, arch, runtime) combination + assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_8" in jobs + assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_9" in jobs + assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_arm64_3_8" in jobs - def test_dockerssi_job_has_parallel_matrix(self, run): + def test_dockerssi_job_has_static_variables(self, run): output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64"] - assert "parallel" in job - matrix = job["parallel"]["matrix"] - assert len(matrix) == 1 - assert matrix[0]["IMAGE"] == "ubuntu:22.04" - assert "3.8" in matrix[0]["RUNTIME"] - assert "3.9" in matrix[0]["RUNTIME"] + job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_8"] + assert job["variables"]["IMAGE"] == "ubuntu:22.04" + assert job["variables"]["RUNTIME"] == "3.8" + assert job["variables"]["ARCH"] == "linux/amd64" + assert "parallel" not in job def test_dockerssi_job_sets_ssi_library_version(self, run): output = run( @@ -251,13 +251,13 @@ def test_dockerssi_job_sets_ssi_library_version(self, run): extra_args=["--ssi-library-version", "1.2.3"], ) jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64"] + job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_8"] assert job["variables"]["DD_INSTALLER_LIBRARY_VERSION"] == "1.2.3" def test_dockerssi_job_no_ssi_version_by_default(self, run): output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64"] + job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_8"] assert "DD_INSTALLER_LIBRARY_VERSION" not in job.get("variables", {}) def test_output_is_valid_yaml_with_dockerssi(self, run): @@ -274,16 +274,17 @@ def test_no_libinjection_jobs_when_empty(self, run): def test_libinjection_jobs_created(self, run): output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) jobs = yaml.safe_load(output) - assert f"run_{LIBRARY}_K8S_LIB_INJECTION" in jobs - assert f"run_{LIBRARY}_K8S_LIB_INJECTION_NO_AC" in jobs + # one job per (scenario, weblog) combination + assert f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog1" in jobs + assert f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog2" in jobs + assert f"run_{LIBRARY}_K8S_LIB_INJECTION_NO_AC_weblog1" in jobs - def test_libinjection_job_has_parallel_matrix(self, run): + def test_libinjection_job_has_static_variables(self, run): output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION"] - assert "parallel" in job - weblogs = {e["K8S_WEBLOG"] for e in job["parallel"]["matrix"]} - assert weblogs == {"weblog1", "weblog2"} + job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog1"] + assert job["variables"]["K8S_WEBLOG"] == "weblog1" + assert "parallel" not in job def test_libinjection_job_sets_k8s_lib_init_img(self, run): output = run( @@ -291,13 +292,13 @@ def test_libinjection_job_sets_k8s_lib_init_img(self, run): extra_args=["--k8s-lib-init-img", "my-registry/lib-init:latest"], ) jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION"] + job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog1"] assert job["variables"]["K8S_LIB_INIT_IMG"] == "my-registry/lib-init:latest" def test_libinjection_job_no_k8s_img_by_default(self, run): output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION"] + job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog1"] assert "K8S_LIB_INIT_IMG" not in job.get("variables", {}) def test_output_is_valid_yaml_with_libinjection(self, run): From e230e8dc75648bc42d5a90277f89a81a0f9db668 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 12:37:50 +0200 Subject: [PATCH 083/229] fix: handle missing workflow keys when --workflows filter is active --- utils/scripts/ci_orchestrators/workflow_data.py | 2 +- utils/scripts/compute-workflow-parameters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 99d8296f2dd..1465f2fd03d 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -344,7 +344,7 @@ def get_endtoend_definitions( unique_id: str, binaries_artifact: str, ) -> dict: - scenarios = scenario_map["endtoend"] + scenarios = scenario_map.get("endtoend", []) # get time stats with open("utils/scripts/ci_orchestrators/time-stats.json", "r") as file: diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index 0a08d1548c9..2b061f3f6f3 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -97,7 +97,7 @@ def __init__( self.data["parametric"] = { "job_count": parametric_job_count, "job_matrix": list(range(1, parametric_job_count + 1)), - "enable": len(scenario_map["parametric"]) > 0 + "enable": len(scenario_map.get("parametric", [])) > 0 and "otel" not in library and library not in ( From 2bf3d64b7c55186e25c80be4985bdec4a1fe5a8a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 14:01:51 +0200 Subject: [PATCH 084/229] ci/gitlab: add workflow names to child pipelines --- utils/ci/gitlab/system-tests.yml | 3 +++ utils/scripts/ci_orchestrators/gitlab_exporter.py | 1 + 2 files changed, 4 insertions(+) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 45e1698e810..9c7c6316d14 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -1,3 +1,6 @@ +workflow: + name: "{{library}}" + include: - local: /utils/ci/gitlab/templates.yml inputs: diff --git a/utils/scripts/ci_orchestrators/gitlab_exporter.py b/utils/scripts/ci_orchestrators/gitlab_exporter.py index a2482a45aaf..cd9176de1ec 100644 --- a/utils/scripts/ci_orchestrators/gitlab_exporter.py +++ b/utils/scripts/ci_orchestrators/gitlab_exporter.py @@ -66,6 +66,7 @@ def print_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_enviro def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_environment: str) -> None: result_pipeline = {} # type: dict + result_pipeline["workflow"] = {"name": f"{language} SSI"} result_pipeline["include"] = [] result_pipeline["stages"] = [] pipeline_file = ".gitlab/ssi_gitlab-ci.yml" From 268f89169329547de140b48f87f8dec5436684c2 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 14:17:43 +0200 Subject: [PATCH 085/229] route aws-ssi.yml through main.yml --- .gitlab-ci.yml | 3 --- utils/ci/gitlab/main.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d52acf46c1e..78f4709d672 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,9 +7,6 @@ include: excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true - - local: "/utils/ci/gitlab/aws-ssi.yml" - inputs: - stage: "build" stages: - build diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index f90d97a6c77..3ff36107031 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -59,6 +59,9 @@ include: inputs: stage: $[[ inputs.stage ]] ref: $[[ inputs.ref ]] + - local: /utils/ci/gitlab/aws-ssi.yml + inputs: + stage: $[[ inputs.stage ]] workflow: rules: From b1f43a49f19e64d5cdda810852cdd999da1b9e55 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 15:06:05 +0200 Subject: [PATCH 086/229] fix: lint and after_script artifact paths --- utils/ci/gitlab/build_pipeline.py | 1 - utils/ci/gitlab/dockerssi.yml | 4 ++-- utils/ci/gitlab/libinjection.yml | 4 ++-- utils/ci/gitlab/test_build_pipeline.py | 25 ++++--------------------- 4 files changed, 8 insertions(+), 26 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 6d1518ec013..b193ff7edd7 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -1,6 +1,5 @@ import argparse import json -import os from pathlib import Path from jinja2 import Environment, FileSystemLoader, select_autoescape diff --git a/utils/ci/gitlab/dockerssi.yml b/utils/ci/gitlab/dockerssi.yml index aee5917424d..eb36146d7ea 100644 --- a/utils/ci/gitlab/dockerssi.yml +++ b/utils/ci/gitlab/dockerssi.yml @@ -46,8 +46,8 @@ run_{{library}}_{{scenario}}_{{weblog}}_{{arch_short}}{{runtime_suffix}}: - section_end "run" after_script: | SCENARIO_SUFFIX=$(echo "$SCENARIO" | tr "[:upper:]" "[:lower:]") - mkdir -p reports/ - cp -R logs_"${SCENARIO_SUFFIX}" reports/ || true + mkdir -p system-tests/reports/ + cp -R system-tests/logs_"${SCENARIO_SUFFIX}" system-tests/reports/ || true retry: max: 2 when: diff --git a/utils/ci/gitlab/libinjection.yml b/utils/ci/gitlab/libinjection.yml index 1b666993080..4502511fc9e 100644 --- a/utils/ci/gitlab/libinjection.yml +++ b/utils/ci/gitlab/libinjection.yml @@ -46,8 +46,8 @@ run_{{library}}_{{scenario}}_{{weblog}}: - section_end "run" after_script: | kind delete clusters --all || true - mkdir -p reports - cp -R logs_*/ reports/ || true + mkdir -p system-tests/reports + cp -R system-tests/logs_*/ system-tests/reports/ || true retry: max: 2 when: diff --git a/utils/ci/gitlab/test_build_pipeline.py b/utils/ci/gitlab/test_build_pipeline.py index 95bd4313fb9..f1aa34df0cb 100644 --- a/utils/ci/gitlab/test_build_pipeline.py +++ b/utils/ci/gitlab/test_build_pipeline.py @@ -41,26 +41,6 @@ def _params( } -def _run(params, extra_args=(), ref="abc1234"): - params_file = pytest.tmp_path_factory.mktemp("params") / "params.json" - params_file.write_text(json.dumps(params)) - result = subprocess.run( - [ - sys.executable, - str(SCRIPT), - "--stage", STAGE, - "--library", LIBRARY, - "--params", str(params_file), - "--ci-image", CI_IMAGE, - "--ref", ref, - *extra_args, - ], - capture_output=True, - text=True, - check=True, - ) - return result.stdout - @pytest.fixture() def run(tmp_path): @@ -165,7 +145,10 @@ def test_no_push_job_by_default(self, run): assert f"push_{LIBRARY}_test_optimization" not in jobs def test_push_job_created_when_enabled(self, run): - output = run(_params(scenarios=["DEFAULT"], weblogs=["flask"]), extra_args=["--push-to-test-optimization", "true"]) + output = run( + _params(scenarios=["DEFAULT"], weblogs=["flask"]), + extra_args=["--push-to-test-optimization", "true"], + ) jobs = yaml.safe_load(output) assert f"push_{LIBRARY}_test_optimization" in jobs From d506afaabcf40c4d7a432161b37a1c93061ba3a9 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 15:46:46 +0200 Subject: [PATCH 087/229] fix: respect libraries input in aws-ssi pipelines --- utils/ci/gitlab/aws-ssi.yml | 41 +++++++++++++++---------------------- utils/ci/gitlab/main.yml | 1 + 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml index 9a29489a770..4c20ae107e9 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/aws-ssi.yml @@ -2,6 +2,9 @@ spec: inputs: stage: description: "CI stage for all jobs in this component" + libraries: + description: "Space-separated list of libraries to test (e.g. 'java python nodejs')" + default: "java python nodejs" --- @@ -50,12 +53,7 @@ compute_pipeline: SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner source venv/bin/activate fi - - python utils/scripts/compute-workflow-parameters.py nodejs --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py java --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py dotnet --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py python --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py php --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py ruby --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - for library in $[[ inputs.libraries ]]; do python utils/scripts/compute-workflow-parameters.py $library --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab; done - | if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then cp *.yml "$CI_PROJECT_DIR" @@ -65,12 +63,7 @@ compute_pipeline: - if: '$SCHEDULED_JOB == null' artifacts: paths: - - nodejs_ssi_gitlab_pipeline.yml - - java_ssi_gitlab_pipeline.yml - - dotnet_ssi_gitlab_pipeline.yml - - python_ssi_gitlab_pipeline.yml - - php_ssi_gitlab_pipeline.yml - - ruby_ssi_gitlab_pipeline.yml + - "*_ssi_gitlab_pipeline.yml" nodejs_ssi_pipeline: stage: $[[ inputs.stage ]] @@ -83,8 +76,8 @@ nodejs_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/' when: always java_ssi_pipeline: @@ -98,8 +91,8 @@ java_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/' when: always dotnet_ssi_pipeline: @@ -113,8 +106,8 @@ dotnet_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/' when: always python_ssi_pipeline: @@ -128,8 +121,8 @@ python_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/' when: always php_ssi_pipeline: @@ -143,8 +136,8 @@ php_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/' when: always ruby_ssi_pipeline: @@ -158,8 +151,8 @@ ruby_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/' when: always delete_amis: diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 3ff36107031..1a4ae7da0f7 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -62,6 +62,7 @@ include: - local: /utils/ci/gitlab/aws-ssi.yml inputs: stage: $[[ inputs.stage ]] + libraries: $[[ inputs.libraries ]] workflow: rules: From 26dd6f6c4e50f3152b7bb76d63f59206a6eb2357 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 15:50:46 +0200 Subject: [PATCH 088/229] fix: make cross-language ssi pipeline needs optional --- utils/ci/gitlab/aws-ssi.yml | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml index 4c20ae107e9..12d89b4c583 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/aws-ssi.yml @@ -82,7 +82,10 @@ nodejs_ssi_pipeline: java_ssi_pipeline: stage: $[[ inputs.stage ]] - needs: ["compute_pipeline", "nodejs_ssi_pipeline"] + needs: + - job: compute_pipeline + - job: nodejs_ssi_pipeline + optional: true variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE trigger: @@ -97,7 +100,10 @@ java_ssi_pipeline: dotnet_ssi_pipeline: stage: $[[ inputs.stage ]] - needs: ["compute_pipeline", "java_ssi_pipeline"] + needs: + - job: compute_pipeline + - job: java_ssi_pipeline + optional: true variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE trigger: @@ -112,7 +118,10 @@ dotnet_ssi_pipeline: python_ssi_pipeline: stage: $[[ inputs.stage ]] - needs: ["compute_pipeline", "dotnet_ssi_pipeline"] + needs: + - job: compute_pipeline + - job: dotnet_ssi_pipeline + optional: true variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE trigger: @@ -127,7 +136,10 @@ python_ssi_pipeline: php_ssi_pipeline: stage: $[[ inputs.stage ]] - needs: ["compute_pipeline", "python_ssi_pipeline"] + needs: + - job: compute_pipeline + - job: python_ssi_pipeline + optional: true variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE trigger: @@ -142,7 +154,10 @@ php_ssi_pipeline: ruby_ssi_pipeline: stage: $[[ inputs.stage ]] - needs: ["compute_pipeline", "php_ssi_pipeline"] + needs: + - job: compute_pipeline + - job: php_ssi_pipeline + optional: true variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE trigger: From 6fc588dce97be2a8d06cf57c28bb0aecc280d9a8 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 16:12:17 +0200 Subject: [PATCH 089/229] fix: skip SSI pipelines when no SSI scenarios match the scenario list --- utils/ci/gitlab/aws-ssi.yml | 46 +++++++++++++------ utils/ci/gitlab/main.yml | 2 + .../ci_orchestrators/gitlab_exporter.py | 4 +- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml index 12d89b4c583..5752429a099 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/aws-ssi.yml @@ -5,6 +5,12 @@ spec: libraries: description: "Space-separated list of libraries to test (e.g. 'java python nodejs')" default: "java python nodejs" + scenarios: + description: "Comma-separated list of scenarios to run (all if empty)" + default: "" + scenarios_groups: + description: "Comma-separated list of scenario groups to run (all if empty)" + default: "" --- @@ -23,6 +29,8 @@ compute_pipeline: CI_ENVIRONMENT: "prod" script: - | + [ -n "$[[ inputs.scenarios ]]" ] && export SYSTEM_TESTS_SCENARIOS="$[[ inputs.scenarios ]]" + [ -n "$[[ inputs.scenarios_groups ]]" ] && export SYSTEM_TESTS_SCENARIOS_GROUPS="$[[ inputs.scenarios_groups ]]" if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined. Computing changes..." SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner @@ -53,7 +61,17 @@ compute_pipeline: SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner source venv/bin/activate fi - - for library in $[[ inputs.libraries ]]; do python utils/scripts/compute-workflow-parameters.py $library --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab; done + - | + touch ssi_check.env + for library in $[[ inputs.libraries ]]; do + python utils/scripts/compute-workflow-parameters.py $library --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + library_upper=$(echo "$library" | tr '[:lower:]' '[:upper:]') + if [ -f "${library}_ssi_gitlab_pipeline.yml" ]; then + echo "${library_upper}_HAS_SSI_JOBS=true" >> ssi_check.env + else + echo "${library_upper}_HAS_SSI_JOBS=false" >> ssi_check.env + fi + done - | if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then cp *.yml "$CI_PROJECT_DIR" @@ -64,6 +82,8 @@ compute_pipeline: artifacts: paths: - "*_ssi_gitlab_pipeline.yml" + reports: + dotenv: ssi_check.env nodejs_ssi_pipeline: stage: $[[ inputs.stage ]] @@ -76,8 +96,8 @@ nodejs_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/ && $NODEJS_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/ && $NODEJS_HAS_SSI_JOBS == "true"' when: always java_ssi_pipeline: @@ -94,8 +114,8 @@ java_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/ && $JAVA_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/ && $JAVA_HAS_SSI_JOBS == "true"' when: always dotnet_ssi_pipeline: @@ -112,8 +132,8 @@ dotnet_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/ && $DOTNET_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/ && $DOTNET_HAS_SSI_JOBS == "true"' when: always python_ssi_pipeline: @@ -130,8 +150,8 @@ python_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/ && $PYTHON_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/ && $PYTHON_HAS_SSI_JOBS == "true"' when: always php_ssi_pipeline: @@ -148,8 +168,8 @@ php_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/ && $PHP_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/ && $PHP_HAS_SSI_JOBS == "true"' when: always ruby_ssi_pipeline: @@ -166,8 +186,8 @@ ruby_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/ && $RUBY_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/ && $RUBY_HAS_SSI_JOBS == "true"' when: always delete_amis: diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 1a4ae7da0f7..65caac98d21 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -63,6 +63,8 @@ include: inputs: stage: $[[ inputs.stage ]] libraries: $[[ inputs.libraries ]] + scenarios: $[[ inputs.scenarios ]] + scenarios_groups: $[[ inputs.scenarios_groups ]] workflow: rules: diff --git a/utils/scripts/ci_orchestrators/gitlab_exporter.py b/utils/scripts/ci_orchestrators/gitlab_exporter.py index cd9176de1ec..2c3a98e5f57 100644 --- a/utils/scripts/ci_orchestrators/gitlab_exporter.py +++ b/utils/scripts/ci_orchestrators/gitlab_exporter.py @@ -81,8 +81,8 @@ def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_en and not matrix_data["dockerssi_scenario_defs"] and not matrix_data["libinjection_scenario_defs"] ): - result_pipeline["stages"].append("SSI_TESTS") - result_pipeline["ssi_tests"] = pipeline_data["ssi_tests"] + print(f"No SSI scenarios to run for {language}, skipping pipeline generation") + return if matrix_data["aws_ssi_scenario_defs"]: # Copy the base job for the onboarding system tests From e9008229be88f8de12e093cb656033a33fbd8ccc Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 16:22:00 +0200 Subject: [PATCH 090/229] fix: remove --workflows flag and correct ssi_check.env path --- utils/ci/gitlab/aws-ssi.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml index 5752429a099..91dcc649064 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/aws-ssi.yml @@ -29,8 +29,6 @@ compute_pipeline: CI_ENVIRONMENT: "prod" script: - | - [ -n "$[[ inputs.scenarios ]]" ] && export SYSTEM_TESTS_SCENARIOS="$[[ inputs.scenarios ]]" - [ -n "$[[ inputs.scenarios_groups ]]" ] && export SYSTEM_TESTS_SCENARIOS_GROUPS="$[[ inputs.scenarios_groups ]]" if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined. Computing changes..." SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner @@ -62,14 +60,16 @@ compute_pipeline: source venv/bin/activate fi - | - touch ssi_check.env + [ -n "$[[ inputs.scenarios ]]" ] && scenarios="$[[ inputs.scenarios ]]" + [ -n "$[[ inputs.scenarios_groups ]]" ] && scenarios_groups="$[[ inputs.scenarios_groups ]]" + touch "$CI_PROJECT_DIR/ssi_check.env" for library in $[[ inputs.libraries ]]; do - python utils/scripts/compute-workflow-parameters.py $library --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + python utils/scripts/compute-workflow-parameters.py $library -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab library_upper=$(echo "$library" | tr '[:lower:]' '[:upper:]') if [ -f "${library}_ssi_gitlab_pipeline.yml" ]; then - echo "${library_upper}_HAS_SSI_JOBS=true" >> ssi_check.env + echo "${library_upper}_HAS_SSI_JOBS=true" >> "$CI_PROJECT_DIR/ssi_check.env" else - echo "${library_upper}_HAS_SSI_JOBS=false" >> ssi_check.env + echo "${library_upper}_HAS_SSI_JOBS=false" >> "$CI_PROJECT_DIR/ssi_check.env" fi done - | From a079581081ba742ba645b7932302b3aa85c119c3 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 16:40:16 +0200 Subject: [PATCH 091/229] fix: rules use runtime dotenv vars - revert to library-only gating --- utils/ci/gitlab/aws-ssi.yml | 33 +++++++------------ .../ci_orchestrators/gitlab_exporter.py | 4 +-- 2 files changed, 14 insertions(+), 23 deletions(-) diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml index 91dcc649064..c6b53ff3641 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/aws-ssi.yml @@ -62,15 +62,8 @@ compute_pipeline: - | [ -n "$[[ inputs.scenarios ]]" ] && scenarios="$[[ inputs.scenarios ]]" [ -n "$[[ inputs.scenarios_groups ]]" ] && scenarios_groups="$[[ inputs.scenarios_groups ]]" - touch "$CI_PROJECT_DIR/ssi_check.env" for library in $[[ inputs.libraries ]]; do python utils/scripts/compute-workflow-parameters.py $library -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - library_upper=$(echo "$library" | tr '[:lower:]' '[:upper:]') - if [ -f "${library}_ssi_gitlab_pipeline.yml" ]; then - echo "${library_upper}_HAS_SSI_JOBS=true" >> "$CI_PROJECT_DIR/ssi_check.env" - else - echo "${library_upper}_HAS_SSI_JOBS=false" >> "$CI_PROJECT_DIR/ssi_check.env" - fi done - | if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then @@ -82,8 +75,6 @@ compute_pipeline: artifacts: paths: - "*_ssi_gitlab_pipeline.yml" - reports: - dotenv: ssi_check.env nodejs_ssi_pipeline: stage: $[[ inputs.stage ]] @@ -96,8 +87,8 @@ nodejs_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/ && $NODEJS_HAS_SSI_JOBS == "true"' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/ && $NODEJS_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/' when: always java_ssi_pipeline: @@ -114,8 +105,8 @@ java_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/ && $JAVA_HAS_SSI_JOBS == "true"' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/ && $JAVA_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/' when: always dotnet_ssi_pipeline: @@ -132,8 +123,8 @@ dotnet_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/ && $DOTNET_HAS_SSI_JOBS == "true"' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/ && $DOTNET_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/' when: always python_ssi_pipeline: @@ -150,8 +141,8 @@ python_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/ && $PYTHON_HAS_SSI_JOBS == "true"' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/ && $PYTHON_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/' when: always php_ssi_pipeline: @@ -168,8 +159,8 @@ php_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/ && $PHP_HAS_SSI_JOBS == "true"' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/ && $PHP_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/' when: always ruby_ssi_pipeline: @@ -186,8 +177,8 @@ ruby_ssi_pipeline: job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/ && $RUBY_HAS_SSI_JOBS == "true"' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/ && $RUBY_HAS_SSI_JOBS == "true"' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/' when: always delete_amis: diff --git a/utils/scripts/ci_orchestrators/gitlab_exporter.py b/utils/scripts/ci_orchestrators/gitlab_exporter.py index 2c3a98e5f57..cd9176de1ec 100644 --- a/utils/scripts/ci_orchestrators/gitlab_exporter.py +++ b/utils/scripts/ci_orchestrators/gitlab_exporter.py @@ -81,8 +81,8 @@ def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_en and not matrix_data["dockerssi_scenario_defs"] and not matrix_data["libinjection_scenario_defs"] ): - print(f"No SSI scenarios to run for {language}, skipping pipeline generation") - return + result_pipeline["stages"].append("SSI_TESTS") + result_pipeline["ssi_tests"] = pipeline_data["ssi_tests"] if matrix_data["aws_ssi_scenario_defs"]: # Copy the base job for the onboarding system tests From fe002191f806d122fb8d95c0c67f9619b945e87a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 16:48:23 +0200 Subject: [PATCH 092/229] refactor: replace 6 ssi trigger jobs with single run_ssi_pipeline --- utils/ci/gitlab/aws-ssi.yml | 106 +++----------------------- utils/ci/gitlab/build_ssi_pipeline.py | 78 +++++++++++++++++++ 2 files changed, 88 insertions(+), 96 deletions(-) create mode 100644 utils/ci/gitlab/build_ssi_pipeline.py diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml index c6b53ff3641..cad58fc2fd9 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/aws-ssi.yml @@ -14,9 +14,9 @@ spec: --- -# AWS SSI pipeline — sequential per-language execution driven by Pulumi/EC2. +# AWS SSI pipeline — mirrors the end-to-end pattern: +# compute_pipeline → generated-ssi-pipeline.yml → run_ssi_pipeline # All Docker/K8s SSI scenarios run in main.yml instead. -# Execution order is expressed through needs: chains; stages are not used for ordering. include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml @@ -65,6 +65,7 @@ compute_pipeline: for library in $[[ inputs.libraries ]]; do python utils/scripts/compute-workflow-parameters.py $library -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab done + python3 utils/ci/gitlab/build_ssi_pipeline.py $[[ inputs.libraries ]] - | if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then cp *.yml "$CI_PROJECT_DIR" @@ -74,111 +75,24 @@ compute_pipeline: - if: '$SCHEDULED_JOB == null' artifacts: paths: - - "*_ssi_gitlab_pipeline.yml" + - generated-ssi-pipeline.yml -nodejs_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: ["compute_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: nodejs_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/' - when: always - -java_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - - job: nodejs_ssi_pipeline - optional: true - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: java_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/' - when: always - -dotnet_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - - job: java_ssi_pipeline - optional: true - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: dotnet_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/' - when: always - -python_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - - job: dotnet_ssi_pipeline - optional: true - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: python_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/' - when: always - -php_ssi_pipeline: +run_ssi_pipeline: + interruptible: true stage: $[[ inputs.stage ]] needs: - job: compute_pipeline - - job: python_ssi_pipeline - optional: true + artifacts: true variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE trigger: include: - - artifact: php_ssi_gitlab_pipeline.yml + - artifact: generated-ssi-pipeline.yml job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/' - when: always - -ruby_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - - job: php_ssi_pipeline - optional: true - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: ruby_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/' + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' when: always delete_amis: diff --git a/utils/ci/gitlab/build_ssi_pipeline.py b/utils/ci/gitlab/build_ssi_pipeline.py new file mode 100644 index 00000000000..c2d244aeceb --- /dev/null +++ b/utils/ci/gitlab/build_ssi_pipeline.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +""" +Merge per-language SSI pipeline YAMLs into a single generated-ssi-pipeline.yml. + +Works the same way as build_pipeline.py does for end-to-end scenarios: +compute_pipeline generates one YAML per language, this script combines them into +a single artifact consumed by a single run_ssi_pipeline trigger job. + +Usage: build_ssi_pipeline.py library1 library2 ... +""" + +import sys +from pathlib import Path + +import yaml + +_PIPELINE_KEYS = {"workflow", "include", "variables", "stages"} + + +def _merge(libraries: list[str]) -> dict: + merged: dict = { + "workflow": {"name": "SSI"}, + "include": [], + "variables": {}, + "stages": [], + } + seen_includes: set[str] = set() + + for lib in libraries: + path = Path(f"{lib}_ssi_gitlab_pipeline.yml") + if not path.exists(): + continue + data: dict = yaml.safe_load(path.read_text()) or {} + + # Deduplicate includes (all languages share the same remote template) + for item in data.get("include", []): + key = yaml.dump(item, default_flow_style=True) + if key not in seen_includes: + merged["include"].append(item) + seen_includes.add(key) + + # Variables are identical across languages — first one wins + for k, v in data.get("variables", {}).items(): + merged["variables"].setdefault(k, v) + + # Prefix stage names with the library to avoid cross-language conflicts + stage_map: dict[str, str] = {} + for stage in data.get("stages", []): + new_stage = f"{lib.upper()}_{stage}" + stage_map[stage] = new_stage + if new_stage not in merged["stages"]: + merged["stages"].append(new_stage) + + for key, value in data.items(): + if key in _PIPELINE_KEYS: + continue + if key.startswith("."): + # Hidden/base job — shared across languages, include only once + merged.setdefault(key, value) + else: + # Regular job — prefix with library to guarantee uniqueness + job = dict(value) + if "stage" in job: + job["stage"] = stage_map.get(job["stage"], job["stage"]) + merged[f"{lib}_{key}"] = job + + return merged + + +if __name__ == "__main__": + libraries = sys.argv[1:] + if not libraries: + print("Usage: build_ssi_pipeline.py lib1 lib2 ...", file=sys.stderr) + sys.exit(1) + + output = yaml.dump(_merge(libraries), sort_keys=False, default_flow_style=False) + Path("generated-ssi-pipeline.yml").write_text(output) + print("Generated: generated-ssi-pipeline.yml") From 5a8b5b7cca3391188cb3cd063e0a56380f7fdb38 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 17:05:15 +0200 Subject: [PATCH 093/229] fix: don't prefix stage names in merged SSI pipeline --- utils/ci/gitlab/build_ssi_pipeline.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/utils/ci/gitlab/build_ssi_pipeline.py b/utils/ci/gitlab/build_ssi_pipeline.py index c2d244aeceb..c532d2cd2d7 100644 --- a/utils/ci/gitlab/build_ssi_pipeline.py +++ b/utils/ci/gitlab/build_ssi_pipeline.py @@ -43,13 +43,13 @@ def _merge(libraries: list[str]) -> dict: for k, v in data.get("variables", {}).items(): merged["variables"].setdefault(k, v) - # Prefix stage names with the library to avoid cross-language conflicts - stage_map: dict[str, str] = {} + # Stages are shared across languages — jobs from different libraries + # run in parallel within the same stage (e.g. INSTALLER_AUTO_INJECTION). + # Base jobs reference stage names directly via extends:, so stage names + # must not be prefixed. for stage in data.get("stages", []): - new_stage = f"{lib.upper()}_{stage}" - stage_map[stage] = new_stage - if new_stage not in merged["stages"]: - merged["stages"].append(new_stage) + if stage not in merged["stages"]: + merged["stages"].append(stage) for key, value in data.items(): if key in _PIPELINE_KEYS: @@ -59,10 +59,7 @@ def _merge(libraries: list[str]) -> dict: merged.setdefault(key, value) else: # Regular job — prefix with library to guarantee uniqueness - job = dict(value) - if "stage" in job: - job["stage"] = stage_map.get(job["stage"], job["stage"]) - merged[f"{lib}_{key}"] = job + merged[f"{lib}_{key}"] = value return merged From 6ae1387d32cbfcb553164264c0025c2cde23b676 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 18:18:10 +0200 Subject: [PATCH 094/229] fix: restrict aws-ssi pipeline to aws_ssi workflow only --- utils/ci/gitlab/aws-ssi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml index cad58fc2fd9..2d021759d35 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/aws-ssi.yml @@ -63,7 +63,7 @@ compute_pipeline: [ -n "$[[ inputs.scenarios ]]" ] && scenarios="$[[ inputs.scenarios ]]" [ -n "$[[ inputs.scenarios_groups ]]" ] && scenarios_groups="$[[ inputs.scenarios_groups ]]" for library in $[[ inputs.libraries ]]; do - python utils/scripts/compute-workflow-parameters.py $library -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + python utils/scripts/compute-workflow-parameters.py $library --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab done python3 utils/ci/gitlab/build_ssi_pipeline.py $[[ inputs.libraries ]] - | From b97ab0a4d7f2c232ed173db89d050b6fafbd1ca7 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 18:25:30 +0200 Subject: [PATCH 095/229] refactor: move docker/k8s ssi scenarios from e2e pipeline to ssi pipeline --- utils/ci/gitlab/aws-ssi.yml | 14 ++++++++- utils/ci/gitlab/build_pipeline.py | 47 +------------------------------ utils/ci/gitlab/main.yml | 5 +++- 3 files changed, 18 insertions(+), 48 deletions(-) diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/aws-ssi.yml index 2d021759d35..dc270e1708b 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/aws-ssi.yml @@ -11,6 +11,15 @@ spec: scenarios_groups: description: "Comma-separated list of scenario groups to run (all if empty)" default: "" + ssi_library_version: + description: "DD_INSTALLER_LIBRARY_VERSION to inject into generated jobs (Docker/K8s SSI)" + default: "" + k8s_lib_init_img: + description: "K8S_LIB_INIT_IMG to inject into generated libinjection jobs" + default: "" + ssi_injector_version: + description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" + default: "" --- @@ -62,8 +71,11 @@ compute_pipeline: - | [ -n "$[[ inputs.scenarios ]]" ] && scenarios="$[[ inputs.scenarios ]]" [ -n "$[[ inputs.scenarios_groups ]]" ] && scenarios_groups="$[[ inputs.scenarios_groups ]]" + [ -n "$[[ inputs.ssi_library_version ]]" ] && export DD_INSTALLER_LIBRARY_VERSION="$[[ inputs.ssi_library_version ]]" + [ -n "$[[ inputs.k8s_lib_init_img ]]" ] && export K8S_LIB_INIT_IMG="$[[ inputs.k8s_lib_init_img ]]" + [ -n "$[[ inputs.ssi_injector_version ]]" ] && export DD_INSTALLER_INJECTOR_VERSION="$[[ inputs.ssi_injector_version ]]" for library in $[[ inputs.libraries ]]; do - python utils/scripts/compute-workflow-parameters.py $library --workflows aws_ssi -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + python utils/scripts/compute-workflow-parameters.py $library -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab done python3 utils/ci/gitlab/build_ssi_pipeline.py $[[ inputs.libraries ]] - | diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index b193ff7edd7..3047e746ea8 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -12,9 +12,7 @@ parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") parser.add_argument("--test-optimization-datadog-site", default="datadoghq.com", help="Datadog site for Test Optimization") -parser.add_argument("--ssi-library-version", default="", help="DD_INSTALLER_LIBRARY_VERSION for Docker/K8s SSI jobs") -parser.add_argument("--k8s-lib-init-img", default="", help="K8S_LIB_INIT_IMG for libinjection jobs") -parser.add_argument("--ssi-injector-version", default="", help="DD_INSTALLER_INJECTOR_VERSION for dockerssi jobs") + args = parser.parse_args() with open(args.params) as f: @@ -24,35 +22,9 @@ binaries_artifact = params["miscs"]["binaries_artifact"] ci_environment = params["miscs"]["ci_environment"] parametric = params["parametric"] -dockerssi_scenario_defs = params.get("dockerssi_scenario_defs", {}) -libinjection_scenario_defs = params.get("libinjection_scenario_defs", {}) - env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) -def _transform_dockerssi(raw: dict) -> dict: - """Regroup dockerssi images by arch for easier Jinja2 iteration. - - Input: {scenario: {weblog: [{image_name: [runtimes], "arch": arch}, ...]}} - Output: {scenario: {weblog: {arch_short: [{image, runtimes}, ...]}}} - """ - result: dict = {} - for scenario, weblogs in raw.items(): - result[scenario] = {} - for weblog, images in weblogs.items(): - by_arch: dict = {} - for img in images: - arch = img["arch"] - arch_short = arch.replace("linux/", "") - if arch_short not in by_arch: - by_arch[arch_short] = [] - for key, val in img.items(): - if key != "arch": - by_arch[arch_short].append({"image": key, "runtimes": val}) - result[scenario][weblog] = by_arch - return result - - template = env.get_template("system-tests.yml") print(template.render( @@ -69,21 +41,4 @@ def _transform_dockerssi(raw: dict) -> dict: test_optimization_datadog_site=args.test_optimization_datadog_site, )) -if dockerssi_scenario_defs: - dockerssi_template = env.get_template("dockerssi.yml") - print(dockerssi_template.render( - library=args.library, - ci_environment=ci_environment, - dockerssi_scenario_defs=_transform_dockerssi(dockerssi_scenario_defs), - ssi_library_version=args.ssi_library_version, - ssi_injector_version=args.ssi_injector_version, - )) -if libinjection_scenario_defs: - libinjection_template = env.get_template("libinjection.yml") - print(libinjection_template.render( - library=args.library, - ci_environment=ci_environment, - libinjection_scenario_defs=libinjection_scenario_defs, - k8s_lib_init_img=args.k8s_lib_init_img, - )) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 65caac98d21..d0f4c4699c3 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -65,6 +65,9 @@ include: libraries: $[[ inputs.libraries ]] scenarios: $[[ inputs.scenarios ]] scenarios_groups: $[[ inputs.scenarios_groups ]] + ssi_library_version: $[[ inputs.ssi_library_version ]] + k8s_lib_init_img: $[[ inputs.k8s_lib_init_img ]] + ssi_injector_version: $[[ inputs.ssi_injector_version ]] workflow: rules: @@ -109,7 +112,7 @@ build_test_pipeline: - > for library in $[[ inputs.libraries ]]; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" --ssi-library-version "$[[ inputs.ssi_library_version ]]" --k8s-lib-init-img "$[[ inputs.k8s_lib_init_img ]]" --ssi-injector-version "$[[ inputs.ssi_injector_version ]]" >> generated-pipeline.yml; + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" >> generated-pipeline.yml; done artifacts: paths: From 93f8e5d2123e105b129ea951e1d4324ce65c92e2 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 19:03:36 +0200 Subject: [PATCH 096/229] chore: remove orphan SSI templates, stale tests, unused --workflows flag dockerssi.yml and libinjection.yml were rendered by build_pipeline.py until commit b97ab0a4d moved docker/k8s SSI generation back into the SSI pipeline (gitlab_exporter.py). The Jinja2 templates and their tests became dead code, and the tests now fail because the corresponding CLI flags were also removed from build_pipeline.py. The --workflows flag on compute-workflow-parameters.py had a single call site in aws-ssi.yml that was reverted in the same commit, leaving the flag with no callers. --- utils/ci/gitlab/dockerssi.yml | 69 ----- utils/ci/gitlab/libinjection.yml | 65 ---- utils/ci/gitlab/test_build_pipeline.py | 302 ------------------- utils/scripts/compute-workflow-parameters.py | 13 - 4 files changed, 449 deletions(-) delete mode 100644 utils/ci/gitlab/dockerssi.yml delete mode 100644 utils/ci/gitlab/libinjection.yml delete mode 100644 utils/ci/gitlab/test_build_pipeline.py diff --git a/utils/ci/gitlab/dockerssi.yml b/utils/ci/gitlab/dockerssi.yml deleted file mode 100644 index eb36146d7ea..00000000000 --- a/utils/ci/gitlab/dockerssi.yml +++ /dev/null @@ -1,69 +0,0 @@ -{% for scenario, weblogs in dockerssi_scenario_defs.items() %} -{% for weblog, archs in weblogs.items() %} -{% for arch_short, images in archs.items() %} -{% for img in images %} -{% for runtime in img.runtimes %} -{% set runtime_suffix = ('_' + runtime | replace('.', '_')) if runtime else '' %} -run_{{library}}_{{scenario}}_{{weblog}}_{{arch_short}}{{runtime_suffix}}: - extends: .system_tests_base - tags: - - docker-in-docker:{{arch_short}} - variables: - KIND_EXPERIMENTAL_DOCKER_NETWORK: bridge - TEST_LIBRARY: {{library}} - SCENARIO: {{scenario}} - ONBOARDING_FILTER_ENV: {{ci_environment}} - WEBLOG: {{weblog}} - IMAGE: {{img.image}} - ARCH: linux/{{arch_short}} - RUNTIME: "{{runtime}}" - PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com - PRIVATE_DOCKER_REGISTRY_USER: AWS -{% if ssi_library_version %} - DD_INSTALLER_LIBRARY_VERSION: "{{ssi_library_version}}" -{% endif %} -{% if ssi_injector_version %} - DD_INSTALLER_INJECTOR_VERSION: "{{ssi_injector_version}}" -{% endif %} - script: - - section_start "ecr_auth" "ECR auth" - - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} - - section_end "ecr_auth" - - section_start "build" "Building runner" - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - section_end "build" - - section_start "run" "Running {{scenario}}" - - > - timeout 1200s ./run.sh $SCENARIO - --ssi-weblog "$WEBLOG" - --ssi-library "$TEST_LIBRARY" - --ssi-base-image "$IMAGE" - --ssi-arch "$ARCH" - --ssi-installable-runtime "$RUNTIME" - --ssi-env $ONBOARDING_FILTER_ENV - --report-run-url ${CI_JOB_URL} - --report-environment $ONBOARDING_FILTER_ENV - - section_end "run" - after_script: | - SCENARIO_SUFFIX=$(echo "$SCENARIO" | tr "[:upper:]" "[:lower:]") - mkdir -p system-tests/reports/ - cp -R system-tests/logs_"${SCENARIO_SUFFIX}" system-tests/reports/ || true - retry: - max: 2 - when: - - job_execution_timeout - - unknown_failure - - runner_system_failure - exit_codes: - - 3 - - 124 - artifacts: - when: always - paths: - - system-tests/reports/ - -{% endfor %} -{% endfor %} -{% endfor %} -{% endfor %} -{% endfor %} diff --git a/utils/ci/gitlab/libinjection.yml b/utils/ci/gitlab/libinjection.yml deleted file mode 100644 index 4502511fc9e..00000000000 --- a/utils/ci/gitlab/libinjection.yml +++ /dev/null @@ -1,65 +0,0 @@ -{% for scenario, weblogs in libinjection_scenario_defs.items() %} -{% for weblog in weblogs %} -run_{{library}}_{{scenario}}_{{weblog}}: - extends: .system_tests_base - variables: - TEST_LIBRARY: {{library}} - K8S_SCENARIO: {{scenario}} - K8S_WEBLOG: {{weblog}} - K8S_WEBLOG_IMG: ${PRIVATE_DOCKER_REGISTRY}/system-tests/{{weblog}} - REPORT_ENVIRONMENT: {{ci_environment}} - PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com - PRIVATE_DOCKER_REGISTRY_USER: AWS -{% if k8s_lib_init_img %} - K8S_LIB_INIT_IMG: "{{k8s_lib_init_img}}" -{% endif %} - script: - - section_start "ecr_auth" "ECR auth" - - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} - - export PRIVATE_DOCKER_REGISTRY_TOKEN=$(aws ecr get-login-password --region us-east-1) - - section_end "ecr_auth" - - section_start "build" "Building runner" - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - section_end "build" - - section_start "weblog_build" "Building weblog image" - - | - if [ "$CI_PROJECT_NAME" = "system-tests" ] && [ "$CI_PIPELINE_SOURCE" != "schedule" ]; then - TAG_NAME=$([ "$CI_COMMIT_BRANCH" = "main" ] && echo "latest" || echo "$CI_COMMIT_BRANCH" | tr '/' '-') - ./lib-injection/build/build_lib_injection_weblog.sh -w ${K8S_WEBLOG} -l ${TEST_LIBRARY} \ - --push-tag ${K8S_WEBLOG_IMG}:${TAG_NAME} \ - --docker-platform linux/amd64 - else - TAG_NAME="latest" - fi - - section_end "weblog_build" - - section_start "run" "Running {{scenario}}" - - > - ./run.sh ${K8S_SCENARIO} - --k8s-library ${TEST_LIBRARY} - --k8s-weblog ${K8S_WEBLOG} - --k8s-weblog-img ${K8S_WEBLOG_IMG}:${TAG_NAME} - --k8s-lib-init-img ${K8S_LIB_INIT_IMG} - --k8s-injector-img ${K8S_INJECTOR_IMG:-None} - --k8s-cluster-img ${K8S_CLUSTER_IMG:-None} - --report-run-url $CI_JOB_URL - --report-environment ${REPORT_ENVIRONMENT} - - section_end "run" - after_script: | - kind delete clusters --all || true - mkdir -p system-tests/reports - cp -R system-tests/logs_*/ system-tests/reports/ || true - retry: - max: 2 - when: - - job_execution_timeout - - unknown_failure - - runner_system_failure - exit_codes: - - 3 - artifacts: - when: always - paths: - - system-tests/reports/ - -{% endfor %} -{% endfor %} diff --git a/utils/ci/gitlab/test_build_pipeline.py b/utils/ci/gitlab/test_build_pipeline.py deleted file mode 100644 index f1aa34df0cb..00000000000 --- a/utils/ci/gitlab/test_build_pipeline.py +++ /dev/null @@ -1,302 +0,0 @@ -"""Tests for build_pipeline.py""" -import json -import subprocess -import sys -from pathlib import Path - -import pytest -import yaml - -SCRIPT = Path(__file__).parent / "build_pipeline.py" -CI_IMAGE = "registry.example.com/ci-runner:abc123" -STAGE = "test" -LIBRARY = "python" - - -def _params( - scenarios=("DEFAULT",), - weblogs=("flask",), - binaries_artifact="", - parametric_enable=False, - parametric_job_count=1, - dockerssi_scenario_defs=None, - libinjection_scenario_defs=None, -): - return { - "endtoend": { - "scenarios": list(scenarios), - "weblogs": list(weblogs), - }, - "miscs": { - "binaries_artifact": binaries_artifact, - "ci_environment": "gitlab", - }, - "parametric": { - "job_count": parametric_job_count, - "job_matrix": list(range(1, parametric_job_count + 1)), - "enable": parametric_enable, - }, - "dockerssi_scenario_defs": dockerssi_scenario_defs or {}, - "libinjection_scenario_defs": libinjection_scenario_defs or {}, - } - - - -@pytest.fixture() -def run(tmp_path): - def _inner(params, extra_args=(), ref="abc1234"): - params_file = tmp_path / "params.json" - params_file.write_text(json.dumps(params)) - result = subprocess.run( - [ - sys.executable, - str(SCRIPT), - "--stage", STAGE, - "--library", LIBRARY, - "--params", str(params_file), - "--ci-image", CI_IMAGE, - "--ref", ref, - *extra_args, - ], - capture_output=True, - text=True, - check=True, - ) - return result.stdout - - return _inner - - -class TestOutputIsValidYaml: - def test_basic(self, run): - output = run(_params()) - assert yaml.safe_load(output) is not None - - def test_multiple_scenarios_and_weblogs(self, run): - output = run(_params(scenarios=["DEFAULT", "APPSEC_SCENARIOS"], weblogs=["flask", "django"])) - assert yaml.safe_load(output) is not None - - def test_with_parametric(self, run): - output = run(_params(parametric_enable=True, parametric_job_count=3)) - assert yaml.safe_load(output) is not None - - def test_with_push_to_test_optimization(self, run): - output = run(_params(), extra_args=["--push-to-test-optimization", "true"]) - assert yaml.safe_load(output) is not None - - -class TestBuildJobs: - def test_one_build_job_per_weblog(self, run): - output = run(_params(weblogs=["flask", "django"])) - jobs = yaml.safe_load(output) - assert f"build_{LIBRARY}_flask" in jobs - assert f"build_{LIBRARY}_django" in jobs - - def test_build_job_has_no_needs_without_binaries_artifact(self, run): - output = run(_params(weblogs=["flask"])) - jobs = yaml.safe_load(output) - assert "needs" not in jobs[f"build_{LIBRARY}_flask"] - - def test_build_job_needs_binaries_artifact(self, run): - output = run(_params(weblogs=["flask"], binaries_artifact="upstream_binaries")) - jobs = yaml.safe_load(output) - needs = jobs[f"build_{LIBRARY}_flask"]["needs"] - assert any(n.get("job") == "upstream_binaries" for n in needs) - - -class TestRunJobs: - def test_one_run_job_per_scenario_and_weblog(self, run): - output = run(_params(scenarios=["DEFAULT", "APPSEC_SCENARIOS"], weblogs=["flask", "django"])) - jobs = yaml.safe_load(output) - for scenario in ["DEFAULT", "APPSEC_SCENARIOS"]: - for weblog in ["flask", "django"]: - assert f"run_{LIBRARY}_{scenario}_{weblog}" in jobs - - def test_run_job_needs_build_job(self, run): - output = run(_params(scenarios=["DEFAULT"], weblogs=["flask"])) - jobs = yaml.safe_load(output) - needs = jobs[f"run_{LIBRARY}_DEFAULT_flask"]["needs"] - assert any(n.get("job") == f"build_{LIBRARY}_flask" for n in needs) - - -class TestParametricJobs: - def test_no_parametric_jobs_when_disabled(self, run): - output = run(_params(parametric_enable=False)) - jobs = yaml.safe_load(output) - assert not any(k.startswith(f"run_{LIBRARY}_PARAMETRIC_") for k in jobs) - - def test_parametric_jobs_created_when_enabled(self, run): - output = run(_params(parametric_enable=True, parametric_job_count=2)) - jobs = yaml.safe_load(output) - assert f"run_{LIBRARY}_PARAMETRIC_1" in jobs - assert f"run_{LIBRARY}_PARAMETRIC_2" in jobs - - def test_parametric_job_count(self, run): - output = run(_params(parametric_enable=True, parametric_job_count=4)) - jobs = yaml.safe_load(output) - parametric_jobs = [k for k in jobs if k.startswith(f"run_{LIBRARY}_PARAMETRIC_")] - assert len(parametric_jobs) == 4 - - -class TestPushTestOptimization: - def test_no_push_job_by_default(self, run): - output = run(_params(scenarios=["DEFAULT"], weblogs=["flask"])) - jobs = yaml.safe_load(output) - assert f"push_{LIBRARY}_test_optimization" not in jobs - - def test_push_job_created_when_enabled(self, run): - output = run( - _params(scenarios=["DEFAULT"], weblogs=["flask"]), - extra_args=["--push-to-test-optimization", "true"], - ) - jobs = yaml.safe_load(output) - assert f"push_{LIBRARY}_test_optimization" in jobs - - def test_push_job_needs_all_run_jobs(self, run): - output = run( - _params(scenarios=["DEFAULT", "APPSEC_SCENARIOS"], weblogs=["flask"]), - extra_args=["--push-to-test-optimization", "true"], - ) - jobs = yaml.safe_load(output) - push_needs = {n["job"] for n in jobs[f"push_{LIBRARY}_test_optimization"]["needs"]} - assert f"run_{LIBRARY}_DEFAULT_flask" in push_needs - assert f"run_{LIBRARY}_APPSEC_SCENARIOS_flask" in push_needs - - def test_push_job_uses_custom_datadog_site(self, run): - output = run( - _params(scenarios=["DEFAULT"], weblogs=["flask"]), - extra_args=["--push-to-test-optimization", "true", "--test-optimization-datadog-site", "datadoghq.eu"], - ) - jobs = yaml.safe_load(output) - push_job = jobs[f"push_{LIBRARY}_test_optimization"] - assert push_job["variables"]["DATADOG_SITE"] == "datadoghq.eu" - - -class TestCiImageAndRef: - def test_ci_image_used_in_push_job(self, run): - output = run( - _params(scenarios=["DEFAULT"], weblogs=["flask"]), - extra_args=["--push-to-test-optimization", "true"], - ) - jobs = yaml.safe_load(output) - push_job = jobs[f"push_{LIBRARY}_test_optimization"] - assert push_job["image"] == CI_IMAGE - - def test_ref_passed_to_template_include(self, run): - output = run(_params(), ref="deadbeef") - # The include block is at the top of the YAML - parsed = yaml.safe_load(output) - include = parsed.get("include", []) - assert any("deadbeef" in str(item) for item in include) - - -_DOCKERSSI_DEFS = { - "DOCKER_SSI": { - "my-weblog": [ - {"ubuntu:22.04": ["3.8", "3.9"], "arch": "linux/amd64"}, - {"ubuntu:22.04": ["3.8"], "arch": "linux/arm64"}, - ] - } -} - -_LIBINJECTION_DEFS = { - "K8S_LIB_INJECTION": ["weblog1", "weblog2"], - "K8S_LIB_INJECTION_NO_AC": ["weblog1"], -} - - -class TestDockerSSI: - def test_no_dockerssi_jobs_when_empty(self, run): - output = run(_params()) - jobs = yaml.safe_load(output) - assert not any("DOCKER_SSI" in k for k in jobs) - - def test_dockerssi_jobs_created(self, run): - output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) - jobs = yaml.safe_load(output) - # one job per (weblog, arch, runtime) combination - assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_8" in jobs - assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_9" in jobs - assert f"run_{LIBRARY}_DOCKER_SSI_my-weblog_arm64_3_8" in jobs - - def test_dockerssi_job_has_static_variables(self, run): - output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) - jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_8"] - assert job["variables"]["IMAGE"] == "ubuntu:22.04" - assert job["variables"]["RUNTIME"] == "3.8" - assert job["variables"]["ARCH"] == "linux/amd64" - assert "parallel" not in job - - def test_dockerssi_job_sets_ssi_library_version(self, run): - output = run( - _params(dockerssi_scenario_defs=_DOCKERSSI_DEFS), - extra_args=["--ssi-library-version", "1.2.3"], - ) - jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_8"] - assert job["variables"]["DD_INSTALLER_LIBRARY_VERSION"] == "1.2.3" - - def test_dockerssi_job_no_ssi_version_by_default(self, run): - output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) - jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_DOCKER_SSI_my-weblog_amd64_3_8"] - assert "DD_INSTALLER_LIBRARY_VERSION" not in job.get("variables", {}) - - def test_output_is_valid_yaml_with_dockerssi(self, run): - output = run(_params(dockerssi_scenario_defs=_DOCKERSSI_DEFS)) - assert yaml.safe_load(output) is not None - - -class TestLibInjection: - def test_no_libinjection_jobs_when_empty(self, run): - output = run(_params()) - jobs = yaml.safe_load(output) - assert not any("K8S_LIB_INJECTION" in k for k in jobs) - - def test_libinjection_jobs_created(self, run): - output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) - jobs = yaml.safe_load(output) - # one job per (scenario, weblog) combination - assert f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog1" in jobs - assert f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog2" in jobs - assert f"run_{LIBRARY}_K8S_LIB_INJECTION_NO_AC_weblog1" in jobs - - def test_libinjection_job_has_static_variables(self, run): - output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) - jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog1"] - assert job["variables"]["K8S_WEBLOG"] == "weblog1" - assert "parallel" not in job - - def test_libinjection_job_sets_k8s_lib_init_img(self, run): - output = run( - _params(libinjection_scenario_defs=_LIBINJECTION_DEFS), - extra_args=["--k8s-lib-init-img", "my-registry/lib-init:latest"], - ) - jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog1"] - assert job["variables"]["K8S_LIB_INIT_IMG"] == "my-registry/lib-init:latest" - - def test_libinjection_job_no_k8s_img_by_default(self, run): - output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) - jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_K8S_LIB_INJECTION_weblog1"] - assert "K8S_LIB_INIT_IMG" not in job.get("variables", {}) - - def test_output_is_valid_yaml_with_libinjection(self, run): - output = run(_params(libinjection_scenario_defs=_LIBINJECTION_DEFS)) - assert yaml.safe_load(output) is not None - - -class TestSsiVariablesNotLeakedIntoEndToEndJobs: - def test_no_ssi_variables_in_endtoend_run_job(self, run): - output = run( - _params(scenarios=["DEFAULT"], weblogs=["flask"]), - extra_args=["--ssi-library-version", "2.0.0", "--k8s-lib-init-img", "my-registry/lib-init:v1"], - ) - jobs = yaml.safe_load(output) - job = jobs[f"run_{LIBRARY}_DEFAULT_flask"] - variables = job.get("variables") or {} - assert "DD_INSTALLER_LIBRARY_VERSION" not in variables - assert "K8S_LIB_INIT_IMG" not in variables diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index 2b061f3f6f3..0642e8068d0 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -43,7 +43,6 @@ def __init__( explicit_binaries_artifact: str, system_tests_dev_mode: bool, ci_environment: str | None, - workflows: str = "", ): # this data struture is a dict where: # the key is the workflow identifier @@ -79,10 +78,6 @@ def __init__( excluded_scenario_names=excluded_scenarios.split(","), ) - if workflows: - allowed = set(_clean_input_value(workflows).split(",")) - scenario_map = {k: v for k, v in scenario_map.items() if k in allowed} - self.data |= get_endtoend_definitions( library, scenario_map, @@ -291,13 +286,6 @@ def _get_workflow_map( ) parser.add_argument("--ci-environment", type=str, help="Explicitly provide CI environment", default=None) - parser.add_argument( - "--workflows", - type=str, - help="Restrict to specific CI workflow types (comma-separated, e.g. 'aws_ssi')", - default="", - ) - args = parser.parse_args() if args.ci_environment is not None: @@ -316,5 +304,4 @@ def _get_workflow_map( explicit_binaries_artifact=args.explicit_binaries_artifact, system_tests_dev_mode=args.system_tests_dev_mode == "true", ci_environment=args.ci_environment, - workflows=args.workflows, ).export(export_format=args.format, output=args.output) From 22f1e81c56529435c7640c460b14ba2b376651c5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 26 May 2026 19:03:51 +0200 Subject: [PATCH 097/229] refactor: rename aws-ssi.yml to ssi.yml The pipeline covers all SSI workflows (aws_ssi, dockerssi, libinjection), not just AWS. Also update the stale header comment that incorrectly claimed Docker/K8s SSI scenarios run from main.yml. --- utils/ci/gitlab/main.yml | 2 +- utils/ci/gitlab/{aws-ssi.yml => ssi.yml} | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename utils/ci/gitlab/{aws-ssi.yml => ssi.yml} (98%) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index d0f4c4699c3..d9df79c5e72 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -59,7 +59,7 @@ include: inputs: stage: $[[ inputs.stage ]] ref: $[[ inputs.ref ]] - - local: /utils/ci/gitlab/aws-ssi.yml + - local: /utils/ci/gitlab/ssi.yml inputs: stage: $[[ inputs.stage ]] libraries: $[[ inputs.libraries ]] diff --git a/utils/ci/gitlab/aws-ssi.yml b/utils/ci/gitlab/ssi.yml similarity index 98% rename from utils/ci/gitlab/aws-ssi.yml rename to utils/ci/gitlab/ssi.yml index dc270e1708b..54299c459e8 100644 --- a/utils/ci/gitlab/aws-ssi.yml +++ b/utils/ci/gitlab/ssi.yml @@ -23,9 +23,9 @@ spec: --- -# AWS SSI pipeline — mirrors the end-to-end pattern: +# SSI pipeline — mirrors the end-to-end pattern: # compute_pipeline → generated-ssi-pipeline.yml → run_ssi_pipeline -# All Docker/K8s SSI scenarios run in main.yml instead. +# Covers all SSI workflows: aws_ssi, dockerssi, libinjection. include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml From 6332f45ede7ac9a5022bc71c2ddfa27b7f902910 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 11:02:25 +0200 Subject: [PATCH 098/229] fix(ssi): restore sequential per-language child pipelines The single run_ssi_pipeline trigger introduced by fe002191f merged all six per-language SSI pipelines into one parallel child pipeline. That broke the AWS EC2 / subnet quota constraint that the legacy pipeline on main enforced via a sequential needs: chain nodejs -> java -> dotnet -> python -> php -> ruby. Restore the six per-language trigger jobs with the same ordering, gated on inputs.libraries so a caller can request a subset. Each library lists all earlier libraries as optional needs so the chain self-heals under any subset (e.g. 'java python nodejs' still runs nodejs -> java -> python; 'python' alone just runs python). Drop build_ssi_pipeline.py, which was the merger script for the parallel flow; each language artifact is consumed directly by its own trigger job, exactly like main. --- utils/ci/gitlab/build_ssi_pipeline.py | 75 --------------- utils/ci/gitlab/ssi.yml | 129 ++++++++++++++++++++++++-- 2 files changed, 121 insertions(+), 83 deletions(-) delete mode 100644 utils/ci/gitlab/build_ssi_pipeline.py diff --git a/utils/ci/gitlab/build_ssi_pipeline.py b/utils/ci/gitlab/build_ssi_pipeline.py deleted file mode 100644 index c532d2cd2d7..00000000000 --- a/utils/ci/gitlab/build_ssi_pipeline.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -""" -Merge per-language SSI pipeline YAMLs into a single generated-ssi-pipeline.yml. - -Works the same way as build_pipeline.py does for end-to-end scenarios: -compute_pipeline generates one YAML per language, this script combines them into -a single artifact consumed by a single run_ssi_pipeline trigger job. - -Usage: build_ssi_pipeline.py library1 library2 ... -""" - -import sys -from pathlib import Path - -import yaml - -_PIPELINE_KEYS = {"workflow", "include", "variables", "stages"} - - -def _merge(libraries: list[str]) -> dict: - merged: dict = { - "workflow": {"name": "SSI"}, - "include": [], - "variables": {}, - "stages": [], - } - seen_includes: set[str] = set() - - for lib in libraries: - path = Path(f"{lib}_ssi_gitlab_pipeline.yml") - if not path.exists(): - continue - data: dict = yaml.safe_load(path.read_text()) or {} - - # Deduplicate includes (all languages share the same remote template) - for item in data.get("include", []): - key = yaml.dump(item, default_flow_style=True) - if key not in seen_includes: - merged["include"].append(item) - seen_includes.add(key) - - # Variables are identical across languages — first one wins - for k, v in data.get("variables", {}).items(): - merged["variables"].setdefault(k, v) - - # Stages are shared across languages — jobs from different libraries - # run in parallel within the same stage (e.g. INSTALLER_AUTO_INJECTION). - # Base jobs reference stage names directly via extends:, so stage names - # must not be prefixed. - for stage in data.get("stages", []): - if stage not in merged["stages"]: - merged["stages"].append(stage) - - for key, value in data.items(): - if key in _PIPELINE_KEYS: - continue - if key.startswith("."): - # Hidden/base job — shared across languages, include only once - merged.setdefault(key, value) - else: - # Regular job — prefix with library to guarantee uniqueness - merged[f"{lib}_{key}"] = value - - return merged - - -if __name__ == "__main__": - libraries = sys.argv[1:] - if not libraries: - print("Usage: build_ssi_pipeline.py lib1 lib2 ...", file=sys.stderr) - sys.exit(1) - - output = yaml.dump(_merge(libraries), sort_keys=False, default_flow_style=False) - Path("generated-ssi-pipeline.yml").write_text(output) - print("Generated: generated-ssi-pipeline.yml") diff --git a/utils/ci/gitlab/ssi.yml b/utils/ci/gitlab/ssi.yml index 54299c459e8..deb9dab855d 100644 --- a/utils/ci/gitlab/ssi.yml +++ b/utils/ci/gitlab/ssi.yml @@ -77,7 +77,6 @@ compute_pipeline: for library in $[[ inputs.libraries ]]; do python utils/scripts/compute-workflow-parameters.py $library -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab done - python3 utils/ci/gitlab/build_ssi_pipeline.py $[[ inputs.libraries ]] - | if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then cp *.yml "$CI_PROJECT_DIR" @@ -87,24 +86,138 @@ compute_pipeline: - if: '$SCHEDULED_JOB == null' artifacts: paths: - - generated-ssi-pipeline.yml + - "*_ssi_gitlab_pipeline.yml" -run_ssi_pipeline: - interruptible: true +# SSI child pipelines run sequentially per language to respect AWS EC2 / subnet +# quotas. Order mirrors the legacy pipeline on main: +# nodejs → java → dotnet → python → php → ruby. +# Each trigger is gated on whether the language is in `inputs.libraries`; +# `needs.optional: true` on every earlier library makes the chain skip +# unrequested languages while keeping order between the ones that are selected. +nodejs_ssi_pipeline: stage: $[[ inputs.stage ]] needs: - job: compute_pipeline - artifacts: true variables: PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE trigger: include: - - artifact: generated-ssi-pipeline.yml + - artifact: nodejs_ssi_gitlab_pipeline.yml job: compute_pipeline strategy: depend rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/' + when: always + +java_ssi_pipeline: + stage: $[[ inputs.stage ]] + needs: + - job: compute_pipeline + - job: nodejs_ssi_pipeline + optional: true + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: java_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/' + when: always + +dotnet_ssi_pipeline: + stage: $[[ inputs.stage ]] + needs: + - job: compute_pipeline + - job: nodejs_ssi_pipeline + optional: true + - job: java_ssi_pipeline + optional: true + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: dotnet_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/' + when: always + +python_ssi_pipeline: + stage: $[[ inputs.stage ]] + needs: + - job: compute_pipeline + - job: nodejs_ssi_pipeline + optional: true + - job: java_ssi_pipeline + optional: true + - job: dotnet_ssi_pipeline + optional: true + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: python_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/' + when: always + +php_ssi_pipeline: + stage: $[[ inputs.stage ]] + needs: + - job: compute_pipeline + - job: nodejs_ssi_pipeline + optional: true + - job: java_ssi_pipeline + optional: true + - job: dotnet_ssi_pipeline + optional: true + - job: python_ssi_pipeline + optional: true + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: php_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/' + when: always + +ruby_ssi_pipeline: + stage: $[[ inputs.stage ]] + needs: + - job: compute_pipeline + - job: nodejs_ssi_pipeline + optional: true + - job: java_ssi_pipeline + optional: true + - job: dotnet_ssi_pipeline + optional: true + - job: python_ssi_pipeline + optional: true + - job: php_ssi_pipeline + optional: true + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: ruby_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/' + - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/' when: always delete_amis: From fd7f4106110c52e3775acaabb6d69d6413dcad4a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 13:31:54 +0200 Subject: [PATCH 099/229] test --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78f4709d672..07a3cab1a4c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ include: - local: "/utils/ci/gitlab/main.yml" inputs: stage: "build" - libraries: "python" + libraries: "python,java" scenarios_groups: "all" excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" From 8a9ac7bc69bd1b54bb7af47f4c07f521c0cc9037 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 13:36:14 +0200 Subject: [PATCH 100/229] fix: handle comma-separated inputs.libraries in for loops Tracer repos pass libraries as comma-separated (e.g. 'python,java') while the component default uses spaces. The bare 'for library in $[[ inputs.libraries ]]' expansion treated the whole value as a single token, causing compute-workflow-parameters.py to reject it. Pipe through tr ',' ' ' so both formats are accepted. --- utils/ci/gitlab/main.yml | 2 +- utils/ci/gitlab/ssi.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index d9df79c5e72..bd959cafdbf 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -110,7 +110,7 @@ build_test_pipeline: aud: dd-sts script: - > - for library in $[[ inputs.libraries ]]; do + for library in $(echo "$[[ inputs.libraries ]]" | tr ',' ' '); do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" >> generated-pipeline.yml; done diff --git a/utils/ci/gitlab/ssi.yml b/utils/ci/gitlab/ssi.yml index deb9dab855d..b59afdeb651 100644 --- a/utils/ci/gitlab/ssi.yml +++ b/utils/ci/gitlab/ssi.yml @@ -3,7 +3,7 @@ spec: stage: description: "CI stage for all jobs in this component" libraries: - description: "Space-separated list of libraries to test (e.g. 'java python nodejs')" + description: "Space- or comma-separated list of libraries to test (e.g. 'java python nodejs' or 'java,python,nodejs')" default: "java python nodejs" scenarios: description: "Comma-separated list of scenarios to run (all if empty)" @@ -74,7 +74,7 @@ compute_pipeline: [ -n "$[[ inputs.ssi_library_version ]]" ] && export DD_INSTALLER_LIBRARY_VERSION="$[[ inputs.ssi_library_version ]]" [ -n "$[[ inputs.k8s_lib_init_img ]]" ] && export K8S_LIB_INIT_IMG="$[[ inputs.k8s_lib_init_img ]]" [ -n "$[[ inputs.ssi_injector_version ]]" ] && export DD_INSTALLER_INJECTOR_VERSION="$[[ inputs.ssi_injector_version ]]" - for library in $[[ inputs.libraries ]]; do + for library in $(echo "$[[ inputs.libraries ]]" | tr ',' ' '); do python utils/scripts/compute-workflow-parameters.py $library -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab done - | From 02b43bfe0c96e12bc339eeeb46b1d11753ed5530 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 14:08:41 +0200 Subject: [PATCH 101/229] test --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 07a3cab1a4c..85c230e3032 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,7 @@ include: - local: "/utils/ci/gitlab/main.yml" inputs: stage: "build" - libraries: "python,java" + libraries: "python,java,php" scenarios_groups: "all" excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" From f059a4ba1d3699859e13c5a0c214895114245c90 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 14:45:43 +0200 Subject: [PATCH 102/229] Updating end to end pipeline name --- utils/ci/gitlab/system-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 9c7c6316d14..23984bf0a35 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -1,5 +1,5 @@ workflow: - name: "{{library}}" + name: "System-tests end to end" include: - local: /utils/ci/gitlab/templates.yml From 400833ec59cf8a80a19321cd2be1da73b7415ef8 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 16:18:44 +0200 Subject: [PATCH 103/229] gitlab: dynamically select libraries from modified files Replace static include of main.yml with a compute_libraries job that mirrors the GitHub compute_libraries_and_scenarios workflow: - gets modified files via CI_MERGE_REQUEST_DIFF_BASE_SHA git diff - runs MOCK_THE_TEST --collect-only to generate the scenario map - runs compute_libraries_and_scenarios.py to select libraries/scenarios - generates a wrapper pipeline YAML that includes main.yml with the computed values, triggered as a child pipeline Also extends compute_libraries_and_scenarios.py to handle GitLab: - maps merge_request_event -> pull_request for existing branching logic - normalizes CI_COMMIT_REF_NAME to refs/heads/ format - emits a flat 'libraries' string output on GitLab --- .gitlab-ci.yml | 70 ++++++++++++++++--- utils/ci/gitlab/generate_wrapper_pipeline.py | 47 +++++++++++++ .../compute_libraries_and_scenarios.py | 7 +- 3 files changed, 112 insertions(+), 12 deletions(-) create mode 100644 utils/ci/gitlab/generate_wrapper_pipeline.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 85c230e3032..b9a61f7181c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,17 +1,67 @@ -include: - - local: "/utils/ci/gitlab/main.yml" - inputs: - stage: "build" - libraries: "python,java,php" - scenarios_groups: "all" - excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" - ref: "$CI_COMMIT_SHA" - push_to_test_optimization: true - stages: - build - system-tests-utils +compute_libraries: + image: $CI_IMAGE + tags: + - arch:amd64 + stage: build + interruptible: true + needs: + - job: build_ci_image + artifacts: true + variables: + GIT_STRATEGY: none + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + script: + - git clone $CI_REPOSITORY_URL + - cd system-tests + - git checkout $CI_COMMIT_SHA + - mkdir original + - git archive origin/main manifests/ | tar -x -C original/ + - | + if [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then + git diff --name-only "$CI_MERGE_REQUEST_DIFF_BASE_SHA" > modified_files.txt + else + touch modified_files.txt + fi + - ln -sf /system-tests/venv venv + - source venv/bin/activate + - export PYTHONPATH="$PWD" + - ./run.sh MOCK_THE_TEST --collect-only --scenario-report + - python utils/scripts/compute_libraries_and_scenarios.py -o compute_output.env + - source compute_output.env + - | + python utils/ci/gitlab/generate_wrapper_pipeline.py \ + --libraries "$libraries" \ + --scenarios "$scenarios" \ + --scenarios-groups "$scenarios_groups" \ + --excluded-scenarios "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" \ + --ref "$CI_COMMIT_SHA" \ + --push-to-test-optimization "true" \ + -o "$CI_PROJECT_DIR/generated-wrapper.yml" + artifacts: + paths: + - generated-wrapper.yml + +trigger_system_tests: + stage: build + interruptible: true + needs: + - job: compute_libraries + artifacts: true + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + trigger: + include: + - artifact: generated-wrapper.yml + job: compute_libraries + strategy: depend + build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy interruptible: true diff --git a/utils/ci/gitlab/generate_wrapper_pipeline.py b/utils/ci/gitlab/generate_wrapper_pipeline.py new file mode 100644 index 00000000000..2efd94dba36 --- /dev/null +++ b/utils/ci/gitlab/generate_wrapper_pipeline.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Generate a GitLab CI wrapper pipeline that includes main.yml with computed inputs.""" + +import argparse + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate a GitLab CI wrapper pipeline YAML") + parser.add_argument("--libraries", required=True, help="Space-separated list of libraries") + parser.add_argument("--scenarios", default="", help="Comma-separated list of scenarios") + parser.add_argument("--scenarios-groups", default="", help="Comma-separated list of scenario groups") + parser.add_argument( + "--excluded-scenarios", + default="", + help="Comma-separated list of scenarios to exclude", + ) + parser.add_argument("--ref", required=True, help="system-tests ref (branch, tag or SHA)") + parser.add_argument( + "--push-to-test-optimization", + default="false", + choices=["true", "false"], + help="Push results to Datadog Test Optimization", + ) + parser.add_argument("-o", "--output", required=True, help="Output file path") + args = parser.parse_args() + + content = f"""include: + - local: /utils/ci/gitlab/main.yml + inputs: + stage: build + libraries: "{args.libraries}" + scenarios: "{args.scenarios}" + scenarios_groups: "{args.scenarios_groups}" + excluded_scenarios: "{args.excluded_scenarios}" + ref: "{args.ref}" + push_to_test_optimization: {args.push_to_test_optimization} + +stages: + - build +""" + + with open(args.output, "w", encoding="utf-8") as f: + f.write(content) + + +if __name__ == "__main__": + main() diff --git a/utils/scripts/compute_libraries_and_scenarios.py b/utils/scripts/compute_libraries_and_scenarios.py index 1549811f671..a62c922cae1 100644 --- a/utils/scripts/compute_libraries_and_scenarios.py +++ b/utils/scripts/compute_libraries_and_scenarios.py @@ -307,8 +307,10 @@ def __init__( def load_git_info(self) -> None: # Get all relevant environment variables. if "GITLAB_CI" in os.environ: - self.event_name = os.environ.get("CI_PIPELINE_SOURCE", "push") - self.ref = os.environ.get("CI_COMMIT_REF_NAME", "") + source = os.environ.get("CI_PIPELINE_SOURCE", "push") + self.event_name = "pull_request" if source == "merge_request_event" else source + branch = os.environ.get("CI_COMMIT_REF_NAME", "") + self.ref = f"refs/heads/{branch}" if branch else "" self.pr_title = "" self.is_gitlab = True else: @@ -406,6 +408,7 @@ def process(inputs: Inputs) -> list[str]: library_processor.selected |= scenario_processor.impacted_libraries if inputs.is_gitlab: + outputs["libraries"] = " ".join(sorted(lib for lib in library_processor.selected if lib not in ("rust",))) outputs |= scenario_processor.get_outputs() else: outputs |= ( From d2d16382f45b8dfe605ba89815daa5990c0826c2 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 16:29:19 +0200 Subject: [PATCH 104/229] move generate_wrapper_pipeline.py to utils/scripts/ci_orchestrators --- .gitlab-ci.yml | 2 +- .../ci_orchestrators}/generate_wrapper_pipeline.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename utils/{ci/gitlab => scripts/ci_orchestrators}/generate_wrapper_pipeline.py (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b9a61f7181c..827fb1bde5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,7 +35,7 @@ compute_libraries: - python utils/scripts/compute_libraries_and_scenarios.py -o compute_output.env - source compute_output.env - | - python utils/ci/gitlab/generate_wrapper_pipeline.py \ + python utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py \ --libraries "$libraries" \ --scenarios "$scenarios" \ --scenarios-groups "$scenarios_groups" \ diff --git a/utils/ci/gitlab/generate_wrapper_pipeline.py b/utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py similarity index 100% rename from utils/ci/gitlab/generate_wrapper_pipeline.py rename to utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py From 373ed76c33bf165dc3f291b438bf3e090e7acef9 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 16:33:33 +0200 Subject: [PATCH 105/229] fix: include remote SSI template directly in parent pipeline .base_job_k8s_docker_ssi was previously available via main.yml -> ssi.yml. Now that main.yml is only in the child pipeline, the parent needs the remote include directly. --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 827fb1bde5a..8fc264ef9bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,6 @@ +include: + - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml + stages: - build - system-tests-utils From fa04737b474a06e7ef808a02e486670fa7cea810 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 16:45:30 +0200 Subject: [PATCH 106/229] compute_libraries: use default git strategy instead of manual clone --- .gitlab-ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8fc264ef9bd..96dc5c1ce7e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -14,16 +14,12 @@ compute_libraries: needs: - job: build_ci_image artifacts: true - variables: - GIT_STRATEGY: none rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH script: - - git clone $CI_REPOSITORY_URL - - cd system-tests - - git checkout $CI_COMMIT_SHA - mkdir original + - git fetch origin main - git archive origin/main manifests/ | tar -x -C original/ - | if [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then From e0aab2725f7e9e85864c7300a36e6474504efc24 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 16:53:09 +0200 Subject: [PATCH 107/229] fix compute_libraries rules and diff base for GitHub mirror setup CI_PIPELINE_SOURCE is always 'push' on a GitLab mirror of GitHub. Replace merge_request_event rule with a plain push rule and replace CI_MERGE_REQUEST_DIFF_BASE_SHA (never set on mirrors) with git merge-base HEAD origin/CI_DEFAULT_BRANCH. --- .gitlab-ci.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 96dc5c1ce7e..820230c1f4e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,18 +15,14 @@ compute_libraries: - job: build_ci_image artifacts: true rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "push" script: - mkdir original - git fetch origin main - git archive origin/main manifests/ | tar -x -C original/ - | - if [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then - git diff --name-only "$CI_MERGE_REQUEST_DIFF_BASE_SHA" > modified_files.txt - else - touch modified_files.txt - fi + BASE_SHA=$(git merge-base HEAD origin/$CI_DEFAULT_BRANCH) + git diff --name-only "$BASE_SHA" > modified_files.txt - ln -sf /system-tests/venv venv - source venv/bin/activate - export PYTHONPATH="$PWD" @@ -53,8 +49,7 @@ trigger_system_tests: - job: compute_libraries artifacts: true rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + - if: $CI_PIPELINE_SOURCE == "push" trigger: include: - artifact: generated-wrapper.yml From 3ea3b595c3c9316d48cea4d98671cf54db97e189 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 17:01:34 +0200 Subject: [PATCH 108/229] fix: unshallow clone before git merge-base Shallow clone (depth 50) causes merge-base to fail when the branch diverged from main more than 50 commits ago. Unshallow first, then use FETCH_HEAD throughout for consistency. --- .gitlab-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 820230c1f4e..334fd364213 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,11 +17,12 @@ compute_libraries: rules: - if: $CI_PIPELINE_SOURCE == "push" script: + - git fetch --unshallow 2>/dev/null || true + - git fetch origin $CI_DEFAULT_BRANCH - mkdir original - - git fetch origin main - - git archive origin/main manifests/ | tar -x -C original/ + - git archive FETCH_HEAD manifests/ | tar -x -C original/ - | - BASE_SHA=$(git merge-base HEAD origin/$CI_DEFAULT_BRANCH) + BASE_SHA=$(git merge-base HEAD FETCH_HEAD) git diff --name-only "$BASE_SHA" > modified_files.txt - ln -sf /system-tests/venv venv - source venv/bin/activate From 7521ea267c56a7343dc85ce9faf692f6c3a9cff5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 17:07:22 +0200 Subject: [PATCH 109/229] fix: hardcode main branch instead of CI_DEFAULT_BRANCH On the GitLab mirror CI_DEFAULT_BRANCH is not set to main. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 334fd364213..f833c2fcb49 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ compute_libraries: - if: $CI_PIPELINE_SOURCE == "push" script: - git fetch --unshallow 2>/dev/null || true - - git fetch origin $CI_DEFAULT_BRANCH + - git fetch origin main - mkdir original - git archive FETCH_HEAD manifests/ | tar -x -C original/ - | From eb96a20efce37d77230127d9f2392e370594f3dc Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 27 May 2026 17:46:39 +0200 Subject: [PATCH 110/229] gitlab: one child pipeline per library --- docs/CI/gitlab-pipeline-merge.md | 250 +++++++++++++++++++++++++++++++ utils/ci/gitlab/main.yml | 173 ++++++++++++++++++++- 2 files changed, 419 insertions(+), 4 deletions(-) create mode 100644 docs/CI/gitlab-pipeline-merge.md diff --git a/docs/CI/gitlab-pipeline-merge.md b/docs/CI/gitlab-pipeline-merge.md new file mode 100644 index 00000000000..ea8d6e62466 --- /dev/null +++ b/docs/CI/gitlab-pipeline-merge.md @@ -0,0 +1,250 @@ +# Merging the new end-to-end GitLab pipeline with the SSI pipeline + +## Context + +There are currently two separate GitLab CI pipelines for system-tests: + +- **SSI pipeline** — the existing `.gitlab-ci.yml` on `main`, used by all tracer repos via + `one-pipeline` (`libdatadog-build/templates/one-pipeline.yml`) +- **End-to-end prototype** — the new `utils/ci/gitlab/main.yml` on this branch, designed to + replace the GitHub Actions end-to-end workflow + +The goal is to merge them into a single, unified GitLab pipeline. + +--- + +## How the SSI pipeline currently works + +### Distribution chain + +``` +libdatadog-build/templates/one-pipeline.yml + └── includes (remote): gitlab-templates.ddbuild.io/libdatadog/include/single-step-instrumentation-tests.yml + (base job templates: .base_job_onboarding, .base_job_k8s_docker_ssi) + +each tracer repo (.gitlab-ci.yml) + └── includes: .gitlab/one-pipeline.locked.yml + └── includes (remote, content-addressed): gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca//one-pipeline.yml + └── defines: configure_system_tests + system_tests jobs +``` + +### configure_system_tests → system_tests flow + +`configure_system_tests` in `one-pipeline.yml`: +1. Pulls system-tests at `$SYSTEM_TESTS_REF` (default: `main`) +2. Builds the runner +3. Calls `utils/scripts/ci_orchestrators/external_gitlab_pipeline.py`, which **reads + `.gitlab-ci.yml` directly** from the cloned system-tests repo, injects SSI image + variables (`K8S_LIB_INIT_IMG`, `DD_INSTALLER_LIBRARY_VERSION`, etc.), filters by + `$SYSTEM_TESTS_LIBRARY`, and dumps the result as `system_tests_pipeline.yml` +4. `system_tests` triggers `system_tests_pipeline.yml` as a child pipeline + +The system-tests `.gitlab-ci.yml` is therefore **already the source of truth** for what +runs in each tracer repo's SSI pipeline. `one-pipeline` just reads and filters it. + +### gitlab-templates.ddbuild.io is an S3 bucket + +Templates are served from `s3://gitlab-templates.ddbuild.io`. There are two publishing paths: + +| Path | URL pattern | When published | +|---|---|---| +| Content-addressed (immutable) | `libdatadog/one-pipeline/ca//` | Automatically on every libdatadog-build pipeline | +| Mutable "latest stable" | `libdatadog/include/` | Manually via `publish-templates-to-versionless-bucket` job | + +Any repo with S3 write access to that bucket can publish its own templates there. The +mutable path is what `one-pipeline.yml` uses to include `single-step-instrumentation-tests.yml`. + +--- + +## Why the SSI pipeline is complex + +There are three scenario categories in the SSI pipeline: + +| Category | Scenarios | Infrastructure | Complexity | +|---|---|---|---| +| `aws_ssi` | `simple_onboarding`, `*_profiling`, `*_appsec` | Real EC2 VMs via Pulumi | High | +| `dockerssi` | `DOCKER_SSI` | Docker-in-Docker | Low (same as e2e) | +| `libinjection` | `K8S_LIB_INJECTION_*` | kind clusters | Moderate | + +**All the complexity in `.gitlab-ci.yml` comes from `aws_ssi` alone:** + +- **Sequential per-language stages** (`nodejs → java → dotnet → ...`) — AWS EC2 and subnet + quotas prevent running all languages in parallel +- **Staggered `.delayed_base_job`** on releases — avoids exhausting AWS capacity at once +- **Pulumi credentials, SSH keys, keypairs** in `before_script` — required for EC2 provisioning +- **`delete_amis`, `delete_ec2_instances` scheduled jobs** — AWS resource cleanup +- **`check_merge_labels`, image build jobs** — infra maintenance + +`DOCKER_SSI` and `K8S_LIB_INJECTION_*` run on the same `docker-in-docker:amd64` runners as +end-to-end tests and have **none** of these constraints. + +--- + +## The new end-to-end prototype + +`utils/ci/gitlab/main.yml` is a GitLab CI component (`spec: inputs:`) with two jobs: + +1. **`build_test_pipeline`** — for each library: runs + `compute-workflow-parameters.py --format json`, pipes through `build_pipeline.py` + (Jinja2) → writes `generated-pipeline-${library}.yml` (one file per library) +2. **`run_test_pipeline_{library}`** — one trigger job per known library, each triggering + its own `generated-pipeline-${library}.yml` as a child pipeline; gated by a rule that + checks whether the library appears in `inputs.libraries` + +`utils/ci/gitlab/system-tests.yml` is the Jinja2 template that produces one flat job per +`(scenario, weblog_variant)` pair: + +``` +run_{library}_{scenario}_{variant} +``` + +The component exposes two inputs: `stage` (for the generated jobs) and `libraries`. +It currently hardcodes `-g all` with a fixed exclusion list and assumes it runs from within +the system-tests repo directory. + +--- + +## Proposed merge strategy + +### Core insight + +`DOCKER_SSI` and `K8S_LIB_INJECTION_*` are structurally identical to end-to-end scenarios: +they run in Docker/K8s on standard CI runners. They can be expressed in the same Jinja2 +template with no special infrastructure. Only `aws_ssi` scenarios need to remain separate. + +### Step 1 — Split the SSI pipeline by infrastructure type + +Separate `.gitlab-ci.yml` into two independent pipelines: + +- **`utils/ci/gitlab/main.yml`** (this file, extended) — handles `endtoend` + `dockerssi` + + `libinjection`. All Docker/K8s, all parallel, no AWS dependencies. +- **`utils/ci/gitlab/aws-ssi.yml`** (new) — handles `aws_ssi` only. Keeps sequential + language stages, Pulumi infra, delayed starts, and cleanup jobs. This pipeline stays + complex because the infrastructure requires it. + +### Step 2 — Extend main.yml inputs + +Add the inputs needed for one-pipeline to drive it: + +```yaml +spec: + inputs: + stage: + build_stage: + default: "build" # stage for build_test_pipeline itself + run_stage: + default: "run" # stage for run_test_pipeline_{library} jobs + libraries: + default: "java python nodejs" + scenarios_groups: + default: "all" # replaces hardcoded "-g all" + excluded_scenarios: + default: "APM_TRACING_E2E_OTEL,..." +``` + +The `build_stage` and `run_stage` inputs allow one-pipeline to map both jobs onto its +single `shared-pipeline` stage. + +### Step 3 — Extend the Jinja2 template for SSI variables + +`system-tests.yml` needs to forward the SSI image variables into generated jobs so that +tracer repos can inject `K8S_LIB_INIT_IMG`, `DD_INSTALLER_LIBRARY_VERSION`, etc.: + +```jinja +run_{{library}}_{{scenario}}_{{variant}}: + variables: + WEBLOG_VARIANT: {{variant}} + {% if ssi_library_version is defined %} + DD_INSTALLER_LIBRARY_VERSION: "{{ssi_library_version}}" + {% endif %} + {% if k8s_lib_init_img is defined %} + K8S_LIB_INIT_IMG: "{{k8s_lib_init_img}}" + {% endif %} +``` + +These values flow from `build_test_pipeline`'s environment into the template at generation +time, so the child pipeline jobs carry them as static variables. + +### Step 4 — Update configure_system_tests in one-pipeline + +Replace the `external_gitlab_pipeline.py` call with the new generation approach: + +```yaml +configure_system_tests: + script: + - *verify-variables + - cd /system-tests + - git pull && git checkout $SYSTEM_TESTS_REF + - ./build.sh -i runner + - source venv/bin/activate && pip install Jinja2 + - | + python utils/scripts/compute-workflow-parameters.py "$SYSTEM_TESTS_LIBRARY" \ + -g "$SYSTEM_TESTS_SCENARIOS_GROUPS" \ + --format json \ + --output params.json + - | + python utils/ci/gitlab/build_pipeline.py \ + --stage test \ + --library "$SYSTEM_TESTS_LIBRARY" \ + --params params.json \ + --ssi-library-version "pipeline-${CI_PIPELINE_ID}" \ + --k8s-lib-init-img "${APM_ECOSYSTEMS_ECR_BASE}/${LIB_INJECTION_NAME}:glci${CI_PIPELINE_ID}" \ + > system_tests_pipeline.yml + - cp system_tests_pipeline.yml $CI_PROJECT_DIR +``` + +This removes `ALLOW_MULTIPLE_CHILD_LEVELS`, `external_gitlab_pipeline.py`, and the +`--format gitlab` code path entirely. + +### Step 5 — Publish main.yml to S3 (optional, for direct include) + +If one-pipeline should include `main.yml` without cloning system-tests, system-tests CI can +push it to the S3 bucket on merge to main: + +```yaml +publish-gitlab-component: + stage: publish + rules: + - if: $CI_COMMIT_BRANCH == "main" + script: + - aws s3 cp utils/ci/gitlab/main.yml s3://gitlab-templates.ddbuild.io/system-tests/include/main.yml + - aws s3 cp utils/ci/gitlab/system-tests.yml s3://gitlab-templates.ddbuild.io/system-tests/include/system-tests.yml + - aws s3 cp utils/ci/gitlab/build_pipeline.py s3://gitlab-templates.ddbuild.io/system-tests/include/build_pipeline.py +``` + +One-pipeline would then include it via: + +```yaml +include: + - remote: https://gitlab-templates.ddbuild.io/system-tests/include/main.yml +``` + +This makes system-tests the owner of its own pipeline distribution, with no git checkout +needed inside `configure_system_tests`. + +--- + +## What stays, what goes + +| Current artifact | Fate | +|---|---| +| `.gitlab-ci.yml` (all languages, all scenario types) | Split: Docker/K8s → `main.yml`; AWS → `aws-ssi.yml` | +| `external_gitlab_pipeline.py` | Deleted — replaced by `build_pipeline.py` + Jinja2 | +| `ALLOW_MULTIPLE_CHILD_LEVELS` variable | Deleted — single code path | +| `compute-workflow-parameters.py --format gitlab` | Deleted — `--format json` + Jinja2 only | +| Sequential language stages in `.gitlab-ci.yml` | Kept in `aws-ssi.yml` only | +| `.delayed_base_job`, delete_amis, delete_ec2_instances | Kept in `aws-ssi.yml` only | +| `single-step-instrumentation-tests.yml` base job templates | Kept in `libdatadog-build`, referenced by `aws-ssi.yml` | + +--- + +## Constraints and caveats + +- **`include: ref:` cannot use CI variables** — GitLab does not expand `$SYSTEM_TESTS_REF` + in `include: project: ref:`. The S3 publish approach (Step 5) sidesteps this entirely since + `remote:` URLs are fetched at pipeline creation using whatever is currently in S3. +- **S3 write access** — system-tests CI needs credentials to write to + `s3://gitlab-templates.ddbuild.io`. This requires coordination with the libdatadog-build + team to grant access or to establish a separate bucket path. +- **`aws_ssi` pipeline is out of scope** — its complexity is infrastructure-driven and cannot + be simplified without changing the underlying AWS/Pulumi approach. diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index bd959cafdbf..394ec7c8e8e 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -112,13 +112,13 @@ build_test_pipeline: - > for library in $(echo "$[[ inputs.libraries ]]" | tr ',' ' '); do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" >> generated-pipeline.yml; + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" > generated-pipeline-${library}.yml; done artifacts: paths: - - system-tests/generated-pipeline.yml + - system-tests/generated-pipeline-*.yml -run_test_pipeline: +.run_test_pipeline_base: interruptible: true stage: $[[ inputs.stage ]] needs: @@ -127,9 +127,174 @@ run_test_pipeline: variables: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" + +run_test_pipeline_java: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bjava\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-java.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_java_otel: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bjava_otel\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-java_otel.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_java_lambda: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bjava_lambda\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-java_lambda.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_python: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bpython\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-python.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_python_otel: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bpython_otel\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-python_otel.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_python_lambda: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bpython_lambda\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-python_lambda.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_nodejs: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bnodejs\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-nodejs.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_nodejs_otel: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bnodejs_otel\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-nodejs_otel.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_nodejs_lambda: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bnodejs_lambda\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-nodejs_lambda.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_dotnet: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bdotnet\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-dotnet.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_golang: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bgolang\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-golang.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_php: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bphp\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-php.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_ruby: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bruby\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-ruby.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_cpp: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bcpp\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-cpp.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_cpp_nginx: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bcpp_nginx\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-cpp_nginx.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_cpp_kong: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bcpp_kong\b/' + trigger: + include: + - artifact: system-tests/generated-pipeline-cpp_kong.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_cpp_httpd: + extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]]" =~ /\bcpp_httpd\b/' trigger: include: - - artifact: system-tests/generated-pipeline.yml + - artifact: system-tests/generated-pipeline-cpp_httpd.yml job: build_test_pipeline strategy: depend From 10755c4add30dfffd58a4417f94eebc8a040729c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 28 May 2026 11:48:15 +0200 Subject: [PATCH 111/229] Removing -L --- utils/ci/gitlab/system-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 23984bf0a35..4aaa1c7f49c 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -76,7 +76,7 @@ run_{{library}}_{{scenario}}_{{variant}}: - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} - section_end "weblog_setup" - section_start "scenario_run" "Running scenario" - - ./run.sh {{scenario}} -L {{library}} + - ./run.sh {{scenario}} - section_end "scenario_run" artifacts: when: always From 94b3bfd48e4f1083729475e6adf0da93aa26fb57 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 28 May 2026 15:58:31 +0200 Subject: [PATCH 112/229] Test --- .gitlab-ci.yml | 110 ++++++++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 51 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f833c2fcb49..6465c16e5f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,61 +1,69 @@ include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml + - local: "/utils/ci/gitlab/main.yml" + inputs: + stage: "build" + libraries: "python,java,php" + scenarios_groups: "all" + excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" + ref: "$CI_COMMIT_SHA" + push_to_test_optimization: true stages: - build - system-tests-utils -compute_libraries: - image: $CI_IMAGE - tags: - - arch:amd64 - stage: build - interruptible: true - needs: - - job: build_ci_image - artifacts: true - rules: - - if: $CI_PIPELINE_SOURCE == "push" - script: - - git fetch --unshallow 2>/dev/null || true - - git fetch origin main - - mkdir original - - git archive FETCH_HEAD manifests/ | tar -x -C original/ - - | - BASE_SHA=$(git merge-base HEAD FETCH_HEAD) - git diff --name-only "$BASE_SHA" > modified_files.txt - - ln -sf /system-tests/venv venv - - source venv/bin/activate - - export PYTHONPATH="$PWD" - - ./run.sh MOCK_THE_TEST --collect-only --scenario-report - - python utils/scripts/compute_libraries_and_scenarios.py -o compute_output.env - - source compute_output.env - - | - python utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py \ - --libraries "$libraries" \ - --scenarios "$scenarios" \ - --scenarios-groups "$scenarios_groups" \ - --excluded-scenarios "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" \ - --ref "$CI_COMMIT_SHA" \ - --push-to-test-optimization "true" \ - -o "$CI_PROJECT_DIR/generated-wrapper.yml" - artifacts: - paths: - - generated-wrapper.yml - -trigger_system_tests: - stage: build - interruptible: true - needs: - - job: compute_libraries - artifacts: true - rules: - - if: $CI_PIPELINE_SOURCE == "push" - trigger: - include: - - artifact: generated-wrapper.yml - job: compute_libraries - strategy: depend +# compute_libraries: +# image: $CI_IMAGE +# tags: +# - arch:amd64 +# stage: build +# interruptible: true +# needs: +# - job: build_ci_image +# artifacts: true +# rules: +# - if: $CI_PIPELINE_SOURCE == "push" +# script: +# - git fetch --unshallow 2>/dev/null || true +# - git fetch origin main +# - mkdir original +# - git archive FETCH_HEAD manifests/ | tar -x -C original/ +# - | +# BASE_SHA=$(git merge-base HEAD FETCH_HEAD) +# git diff --name-only "$BASE_SHA" > modified_files.txt +# - ln -sf /system-tests/venv venv +# - source venv/bin/activate +# - export PYTHONPATH="$PWD" +# - ./run.sh MOCK_THE_TEST --collect-only --scenario-report +# - python utils/scripts/compute_libraries_and_scenarios.py -o compute_output.env +# - source compute_output.env +# - | +# python utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py \ +# --libraries "$libraries" \ +# --scenarios "$scenarios" \ +# --scenarios-groups "$scenarios_groups" \ +# --excluded-scenarios "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" \ +# --ref "$CI_COMMIT_SHA" \ +# --push-to-test-optimization "true" \ +# -o "$CI_PROJECT_DIR/generated-wrapper.yml" +# artifacts: +# paths: +# - generated-wrapper.yml +# +# trigger_system_tests: +# stage: build +# interruptible: true +# needs: +# - job: compute_libraries +# artifacts: true +# rules: +# - if: $CI_PIPELINE_SOURCE == "push" +# trigger: +# include: +# - artifact: generated-wrapper.yml +# job: compute_libraries +# strategy: depend build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy From d48d856f7bad1f9e938dbf0e1e16c5165637e2a8 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 28 May 2026 16:20:51 +0200 Subject: [PATCH 113/229] gitlab: retry docker hub auth up to 3 times --- utils/ci/gitlab/system-tests.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 4aaa1c7f49c..b456dd098e7 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -44,7 +44,13 @@ build_{{library}}_{{variant}}: - section_start "docker_auth" "Docker hub auth" - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - | + for i in 1 2 3; do + echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin && break + if [ "$i" -eq 3 ]; then echo "docker login failed after 3 attempts"; exit 1; fi + echo "docker login failed (attempt $i), retrying in $((i*5))s..." + sleep $((i*5)) + done - section_end "docker_auth" - section_start "build" "Building weblog" - ./build.sh {{library}} -i weblog --save-to-binaries --weblog-variant {{variant}} @@ -69,7 +75,13 @@ run_{{library}}_{{scenario}}_{{variant}}: - section_start "docker_auth" "Docker hub auth" - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + - | + for i in 1 2 3; do + echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin && break + if [ "$i" -eq 3 ]; then echo "docker login failed after 3 attempts"; exit 1; fi + echo "docker login failed (attempt $i), retrying in $((i*5))s..." + sleep $((i*5)) + done - section_end "docker_auth" - section_start "weblog_setup" "Setting up the weblog" - mv ../binaries/* binaries/ From 937e99a333eb48dae640b0e298fa095e95fd2048 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 28 May 2026 16:48:03 +0200 Subject: [PATCH 114/229] Add -L for scenarios that require it --- utils/ci/gitlab/system-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index b456dd098e7..bdde1bdd1ea 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -88,7 +88,11 @@ run_{{library}}_{{scenario}}_{{variant}}: - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} - section_end "weblog_setup" - section_start "scenario_run" "Running scenario" + {% if scenario in ("INTEGRATION_FRAMEWORKS") %} + - ./run.sh {{scenario}} -L {{library}} + {% else %} - ./run.sh {{scenario}} + {% endif %} - section_end "scenario_run" artifacts: when: always From 106c78c79b11a35d0db341335ce7d8e0a95e3068 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 28 May 2026 17:06:14 +0200 Subject: [PATCH 115/229] Adding weblog flag for weblogs that require it --- utils/ci/gitlab/system-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index bdde1bdd1ea..bd037ced225 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -89,7 +89,7 @@ run_{{library}}_{{scenario}}_{{variant}}: - section_end "weblog_setup" - section_start "scenario_run" "Running scenario" {% if scenario in ("INTEGRATION_FRAMEWORKS") %} - - ./run.sh {{scenario}} -L {{library}} + - ./run.sh {{scenario}} -L {{library}} --weblog {{variant}} {% else %} - ./run.sh {{scenario}} {% endif %} From e4b3ab8e93a1ab71384f357cb029b6097afa975f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 10:45:38 +0200 Subject: [PATCH 116/229] Run only python --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6465c16e5f4..f8c51e68b3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ include: - local: "/utils/ci/gitlab/main.yml" inputs: stage: "build" - libraries: "python,java,php" + libraries: "python" scenarios_groups: "all" excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" From cc899bb77056ebdf36e35ba93e272e9bc742384b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 12:06:13 +0200 Subject: [PATCH 117/229] restore original SSI pipeline in .gitlab-ci.yml, add push-gated end-to-end pipeline The original .gitlab-ci.yml (consumed by tracer repos via one-pipeline) is fully restored from origin/main. The new end-to-end pipeline is appended as a separate YAML document with rules: == 'push', so it only runs on direct pushes to the system-tests repo and never on child pipelines triggered by tracer repos (which get parent_pipeline). --- .gitlab-ci.yml | 354 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 275 insertions(+), 79 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f8c51e68b3f..354a49dded5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,91 +1,244 @@ include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml - - local: "/utils/ci/gitlab/main.yml" - inputs: - stage: "build" - libraries: "python" - scenarios_groups: "all" - excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" - ref: "$CI_COMMIT_SHA" - push_to_test_optimization: true - stages: + - configure + - nodejs + - java + - dotnet + - python + - php + - ruby - build + - pipeline-status - system-tests-utils -# compute_libraries: -# image: $CI_IMAGE -# tags: -# - arch:amd64 -# stage: build -# interruptible: true -# needs: -# - job: build_ci_image -# artifacts: true -# rules: -# - if: $CI_PIPELINE_SOURCE == "push" -# script: -# - git fetch --unshallow 2>/dev/null || true -# - git fetch origin main -# - mkdir original -# - git archive FETCH_HEAD manifests/ | tar -x -C original/ -# - | -# BASE_SHA=$(git merge-base HEAD FETCH_HEAD) -# git diff --name-only "$BASE_SHA" > modified_files.txt -# - ln -sf /system-tests/venv venv -# - source venv/bin/activate -# - export PYTHONPATH="$PWD" -# - ./run.sh MOCK_THE_TEST --collect-only --scenario-report -# - python utils/scripts/compute_libraries_and_scenarios.py -o compute_output.env -# - source compute_output.env -# - | -# python utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py \ -# --libraries "$libraries" \ -# --scenarios "$scenarios" \ -# --scenarios-groups "$scenarios_groups" \ -# --excluded-scenarios "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" \ -# --ref "$CI_COMMIT_SHA" \ -# --push-to-test-optimization "true" \ -# -o "$CI_PROJECT_DIR/generated-wrapper.yml" -# artifacts: -# paths: -# - generated-wrapper.yml -# -# trigger_system_tests: -# stage: build -# interruptible: true -# needs: -# - job: compute_libraries -# artifacts: true -# rules: -# - if: $CI_PIPELINE_SOURCE == "push" -# trigger: -# include: -# - artifact: generated-wrapper.yml -# job: compute_libraries -# strategy: depend +variables: + TEST: 1 -build_ci_image: - image: registry.ddbuild.io/images/docker:20.10.13-jammy - interruptible: true - tags: - - docker-in-docker:amd64 - stage: build +compute_pipeline: + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + tags: ["arch:amd64"] + stage: configure + variables: + CI_ENVIRONMENT: "prod" script: - - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) - - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env - - > - if docker manifest inspect registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG > /dev/null 2>&1; then - echo "Image already exists, skipping build"; + - | + if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then + echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined.Computing changes..." + SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + source venv/bin/activate + ./run.sh MOCK_THE_TEST --collect-only --scenario-report + git clone https://github.com/DataDog/system-tests.git original + git fetch --all # Ensure all branches are available + git branch --track $CI_COMMIT_REF_NAME origin/$CI_COMMIT_REF_NAME || true # Track branch if not tracked + git checkout $CI_COMMIT_REF_NAME # Ensure the branch is checked out + BASE_COMMIT=$(git merge-base origin/main $CI_COMMIT_REF_NAME) # Get the base commit + echo "Branch was created from commit--> $BASE_COMMIT" + git diff --name-only $BASE_COMMIT $CI_COMMIT_SHA >> modified_files.txt # List modified files + cat modified_files.txt + python utils/scripts/compute_libraries_and_scenarios.py >> impacted_scenarios.txt + cat impacted_scenarios.txt + source impacted_scenarios.txt else - docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile -t registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG . && - docker push registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG; + echo "✅ SYSTEM_TESTS_SCENARIOS OR SYSTEM_TESTS_SCENARIOS_GROUPS are set." + export scenarios=$SYSTEM_TESTS_SCENARIOS + export scenarios_groups=$SYSTEM_TESTS_SCENARIOS_GROUPS + cd /system-tests + git pull + if [ -n "$SYSTEM_TESTS_REF" ]; then + git checkout $SYSTEM_TESTS_REF + else + echo "⚠️ SYSTEM_TESTS_REF variable is not set, skipping git checkout" + fi + SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + source venv/bin/activate fi + - python utils/scripts/compute-workflow-parameters.py nodejs -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py java -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py dotnet -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py python -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py php -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py ruby -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - | + if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then + cp *.yml "$CI_PROJECT_DIR" + fi + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' artifacts: - reports: - dotenv: build.env + paths: + - nodejs_ssi_gitlab_pipeline.yml + - java_ssi_gitlab_pipeline.yml + - dotnet_ssi_gitlab_pipeline.yml + - python_ssi_gitlab_pipeline.yml + - php_ssi_gitlab_pipeline.yml + - ruby_ssi_gitlab_pipeline.yml + +nodejs_ssi_pipeline: + stage: nodejs + needs: ["compute_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: nodejs_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +java_ssi_pipeline: + stage: java + needs: ["compute_pipeline", "nodejs_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: java_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +dotnet_ssi_pipeline: + stage: dotnet + needs: ["compute_pipeline", "java_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: dotnet_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +python_ssi_pipeline: + stage: python + needs: ["compute_pipeline", "dotnet_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: python_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +php_ssi_pipeline: + stage: php + needs: ["compute_pipeline", "python_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: php_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +ruby_ssi_pipeline: + stage: ruby + needs: ["compute_pipeline", "php_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: ruby_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +delete_amis: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + variables: + AMI_RETENTION_DAYS: 10 + AMI_LAST_LAUNCHED_DAYS: 10 + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - python utils/scripts/pulumi_clean_up.py --component amis --ami-retention-days $AMI_RETENTION_DAYS --ami-last-launched-days $AMI_LAST_LAUNCHED_DAYS + rules: + - if: '$SCHEDULED_JOB == "delete_amis"' + after_script: echo "Finish" + timeout: 3h + +delete_amis_by_name_or_lang: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + script: + - | + if [ -z "$AMI_NAME" ] && [ -z "$AMI_LANG" ]; then + echo "❌ ERROR: Either AMI_NAME or AMI_LANG must be set." + exit 1 + fi + echo "✅ Proceeding with AMI_NAME=$AMI_NAME, AMI_LANG=$AMI_LANG" + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - | + CMD="python utils/scripts/pulumi_clean_up.py --component amis_by_name" + + if [ -n "$AMI_NAME" ]; then + CMD="$CMD --ami-name $AMI_NAME" + fi + + if [ -n "$AMI_LANG" ]; then + CMD="$CMD --ami-lang $AMI_LANG" + fi + + echo "Running: $CMD" + eval $CMD + rules: + - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' + after_script: echo "Finish" + +delete_ec2_instances: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + variables: + EC2_AGE_MINUTES: 45 + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - python utils/scripts/pulumi_clean_up.py --component ec2 --ec2-age-minutes $EC2_AGE_MINUTES + rules: + - if: '$SCHEDULED_JOB == "delete_ec2_instances"' + after_script: echo "Finish" + +count_amis: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - python utils/scripts/pulumi_clean_up.py --component amis_count + rules: + - if: '$SCHEDULED_JOB == "count_amis"' + after_script: echo "Finish" check_merge_labels: + #Build docker images if it's needed. Check if the PR has the labels associated with the image build. image: registry.ddbuild.io/system-tests/base-image-builder tags: - arch:amd64 @@ -95,6 +248,7 @@ check_merge_labels: needs: [] stage: system-tests-utils before_script: + # Get a token - dd-octo-sts version - dd-octo-sts debug --scope DataDog/system-tests --policy self.gitlab-read - dd-octo-sts token --scope DataDog/system-tests --policy self.gitlab-read > token.txt @@ -104,6 +258,7 @@ check_merge_labels: - export GITHUB_TOKEN=$(cat token.txt) - ./utils/scripts/get_pr_merged_labels.sh after_script: + # Revoke the token after usage - dd-octo-sts revoke -t $(cat token.txt) rules: - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" @@ -119,8 +274,8 @@ generate_system_tests_lambda_proxy_image: - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i lambda-proxy - - docker push datadog/system-tests:lambda-proxy-v1 + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i lambda-proxy + - docker push datadog/system-tests:lambda-proxy-v1 rules: - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" changes: @@ -136,9 +291,50 @@ generate_system_tests_lib_injection_images: PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com PRIVATE_DOCKER_REGISTRY_USER: AWS script: - - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} - - ./lib-injection/build/build_lib_injection_images.sh $PRIVATE_DOCKER_REGISTRY + - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} + - ./lib-injection/build/build_lib_injection_images.sh $PRIVATE_DOCKER_REGISTRY rules: - if: '$SCHEDULED_JOB == "generate_system_tests_lib_injection_images"' - when: manual +.delayed_base_job: + image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 + tags: ["arch:amd64"] + script: + - echo "⏳ Waiting before triggering the child pipeline..." + when: delayed + start_in: 5 minutes +--- +include: + - local: "/utils/ci/gitlab/main.yml" + inputs: + stage: "build" + libraries: "python" + scenarios_groups: "all" + excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" + ref: "$CI_COMMIT_SHA" + push_to_test_optimization: true + rules: + - if: $CI_PIPELINE_SOURCE == "push" + +build_ci_image: + image: registry.ddbuild.io/images/docker:20.10.13-jammy + interruptible: true + tags: + - docker-in-docker:amd64 + stage: build + script: + - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) + - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env + - > + if docker manifest inspect registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG > /dev/null 2>&1; then + echo "Image already exists, skipping build"; + else + docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile -t registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG . && + docker push registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG; + fi + artifacts: + reports: + dotenv: build.env + rules: + - if: $CI_PIPELINE_SOURCE == "push" From 4ebb7b9e26e7e8ba88184a58fb6927337c87188d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 12:18:40 +0200 Subject: [PATCH 118/229] restore original SSI pipeline in .gitlab-ci.yml, add push-gated end-to-end pipeline The original .gitlab-ci.yml (consumed by tracer repos via one-pipeline) is fully restored from origin/main. The end-to-end pipeline jobs are appended with rules: if: $CI_PIPELINE_SOURCE == 'push', so they only run on direct pushes to the system-tests repo and never on child pipelines triggered by tracer repos (which get parent_pipeline). --- .gitlab-ci.yml | 229 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 217 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 354a49dded5..d6c897ea8e0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -304,18 +304,29 @@ generate_system_tests_lib_injection_images: - echo "⏳ Waiting before triggering the child pipeline..." when: delayed start_in: 5 minutes ---- -include: - - local: "/utils/ci/gitlab/main.yml" - inputs: - stage: "build" - libraries: "python" - scenarios_groups: "all" - excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" - ref: "$CI_COMMIT_SHA" - push_to_test_optimization: true - rules: - - if: $CI_PIPELINE_SOURCE == "push" + +# ────────────────────────────────────────────── +# End-to-end pipeline — only on direct pushes +# ────────────────────────────────────────────── + +.system_tests_base: + image: $CI_IMAGE + tags: + - docker-in-docker:amd64 + variables: + GIT_STRATEGY: none + stage: build + interruptible: true + before_script: + - echo -e "section_start:`date +%s`:setup[collapsed=true]\r\e[0KSystem-tests runner setup" + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git + - cd system-tests + - git checkout $CI_COMMIT_SHA + - ln -sf /system-tests/venv venv + - source venv/bin/activate + - export PYTHONPATH="$CI_PROJECT_DIR/system-tests" + - source utils/ci/gitlab/section.sh + - echo -e "section_end:`date +%s`:setup\r\e[0K" build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy @@ -338,3 +349,197 @@ build_ci_image: dotenv: build.env rules: - if: $CI_PIPELINE_SOURCE == "push" + +resolve_ci_image: + image: registry.ddbuild.io/system-tests/git + tags: + - arch:amd64 + stage: build + interruptible: true + variables: + GIT_STRATEGY: none + needs: + - job: build_ci_image + optional: true + artifacts: false + script: + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git + - git -C system-tests checkout $CI_COMMIT_SHA + - IMAGE_TAG=$(cat system-tests/utils/ci/gitlab/docker/system-tests.Dockerfile system-tests/requirements.txt | sha256sum | cut -c1-12) + - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env + - cat build.env + artifacts: + reports: + dotenv: build.env + rules: + - if: $CI_PIPELINE_SOURCE == "push" + +build_test_pipeline: + extends: .system_tests_base + tags: + - arch:amd64 + needs: + - job: resolve_ci_image + artifacts: true + id_tokens: + DD_STS_OIDC_TOKEN: + aud: dd-sts + script: + - > + for library in $(echo "python" | tr ',' ' '); do + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "all" --excluded-scenarios "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" --parametric-job-count 1 --output params_${library}.json; + python3 utils/ci/gitlab/build_pipeline.py --stage build --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$CI_COMMIT_SHA" --push-to-test-optimization "true" > generated-pipeline-${library}.yml; + done + artifacts: + paths: + - system-tests/generated-pipeline-*.yml + rules: + - if: $CI_PIPELINE_SOURCE == "push" + +.run_test_pipeline_base: + interruptible: true + stage: build + needs: + - job: build_test_pipeline + artifacts: true + rules: + - if: $CI_PIPELINE_SOURCE == "push" + variables: + SYSTEM_TESTS_FORCE_EXECUTE: "" + SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "false" + +run_test_pipeline_java: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-java.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_java_otel: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-java_otel.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_java_lambda: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-java_lambda.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_python: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-python.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_python_otel: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-python_otel.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_python_lambda: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-python_lambda.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_nodejs: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-nodejs.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_nodejs_otel: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-nodejs_otel.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_nodejs_lambda: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-nodejs_lambda.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_dotnet: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-dotnet.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_golang: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-golang.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_php: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-php.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_ruby: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-ruby.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_cpp: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-cpp.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_cpp_nginx: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-cpp_nginx.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_cpp_kong: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-cpp_kong.yml + job: build_test_pipeline + strategy: depend + +run_test_pipeline_cpp_httpd: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-cpp_httpd.yml + job: build_test_pipeline + strategy: depend From 9e21fafd9c4d83eb36c2a783f65af9b4764ad5a7 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 13:05:21 +0200 Subject: [PATCH 119/229] restore original SSI pipeline in .gitlab-ci.yml, add push-gated end-to-end pipeline The original .gitlab-ci.yml (consumed by tracer repos via one-pipeline) is fully restored from origin/main. The end-to-end pipeline is included from utils/ci/gitlab/main.yml with only_on_push: true, so all its jobs only run on direct pushes to the system-tests repo and never on child pipelines triggered by tracer repos (which get parent_pipeline). build_ci_image is defined directly in .gitlab-ci.yml with push rules as it is needed by resolve_ci_image from the included main.yml component. --- .gitlab-ci.yml | 222 +++---------------------------------------------- 1 file changed, 10 insertions(+), 212 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d6c897ea8e0..f120b925e0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -309,24 +309,16 @@ generate_system_tests_lib_injection_images: # End-to-end pipeline — only on direct pushes # ────────────────────────────────────────────── -.system_tests_base: - image: $CI_IMAGE - tags: - - docker-in-docker:amd64 - variables: - GIT_STRATEGY: none - stage: build - interruptible: true - before_script: - - echo -e "section_start:`date +%s`:setup[collapsed=true]\r\e[0KSystem-tests runner setup" - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git - - cd system-tests - - git checkout $CI_COMMIT_SHA - - ln -sf /system-tests/venv venv - - source venv/bin/activate - - export PYTHONPATH="$CI_PROJECT_DIR/system-tests" - - source utils/ci/gitlab/section.sh - - echo -e "section_end:`date +%s`:setup\r\e[0K" +include: + - local: /utils/ci/gitlab/main.yml + inputs: + stage: "build" + libraries: "python" + scenarios_groups: "all" + excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" + ref: "$CI_COMMIT_SHA" + push_to_test_optimization: true + only_on_push: true build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy @@ -349,197 +341,3 @@ build_ci_image: dotenv: build.env rules: - if: $CI_PIPELINE_SOURCE == "push" - -resolve_ci_image: - image: registry.ddbuild.io/system-tests/git - tags: - - arch:amd64 - stage: build - interruptible: true - variables: - GIT_STRATEGY: none - needs: - - job: build_ci_image - optional: true - artifacts: false - script: - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git - - git -C system-tests checkout $CI_COMMIT_SHA - - IMAGE_TAG=$(cat system-tests/utils/ci/gitlab/docker/system-tests.Dockerfile system-tests/requirements.txt | sha256sum | cut -c1-12) - - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env - - cat build.env - artifacts: - reports: - dotenv: build.env - rules: - - if: $CI_PIPELINE_SOURCE == "push" - -build_test_pipeline: - extends: .system_tests_base - tags: - - arch:amd64 - needs: - - job: resolve_ci_image - artifacts: true - id_tokens: - DD_STS_OIDC_TOKEN: - aud: dd-sts - script: - - > - for library in $(echo "python" | tr ',' ' '); do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "all" --excluded-scenarios "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" --parametric-job-count 1 --output params_${library}.json; - python3 utils/ci/gitlab/build_pipeline.py --stage build --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$CI_COMMIT_SHA" --push-to-test-optimization "true" > generated-pipeline-${library}.yml; - done - artifacts: - paths: - - system-tests/generated-pipeline-*.yml - rules: - - if: $CI_PIPELINE_SOURCE == "push" - -.run_test_pipeline_base: - interruptible: true - stage: build - needs: - - job: build_test_pipeline - artifacts: true - rules: - - if: $CI_PIPELINE_SOURCE == "push" - variables: - SYSTEM_TESTS_FORCE_EXECUTE: "" - SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "false" - -run_test_pipeline_java: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-java.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_java_otel: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-java_otel.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_java_lambda: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-java_lambda.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_python: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-python.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_python_otel: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-python_otel.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_python_lambda: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-python_lambda.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_nodejs: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-nodejs.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_nodejs_otel: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-nodejs_otel.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_nodejs_lambda: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-nodejs_lambda.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_dotnet: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-dotnet.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_golang: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-golang.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_php: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-php.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_ruby: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-ruby.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_cpp: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-cpp.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_cpp_nginx: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-cpp_nginx.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_cpp_kong: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-cpp_kong.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_cpp_httpd: - extends: .run_test_pipeline_base - trigger: - include: - - artifact: system-tests/generated-pipeline-cpp_httpd.yml - job: build_test_pipeline - strategy: depend From 054d39cc1693a82915ded17b27f6fd734e52d87b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 13:08:26 +0200 Subject: [PATCH 120/229] restore original SSI pipeline in .gitlab-ci.yml, add push-gated end-to-end pipeline The original .gitlab-ci.yml (consumed by tracer repos via one-pipeline) is fully restored from origin/main. The end-to-end pipeline is included from utils/ci/gitlab/main.yml with only_on_push: true, so all its jobs only run on direct pushes to the system-tests repo and never on child pipelines triggered by tracer repos (which get parent_pipeline). build_ci_image is defined directly in .gitlab-ci.yml with push rules as it is needed by resolve_ci_image from the included main.yml component. --- utils/ci/gitlab/main.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 394ec7c8e8e..b7dbd16e20f 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -52,6 +52,10 @@ spec: ssi_injector_version: description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" default: "" + only_on_push: + description: "Only run jobs on direct push events" + type: boolean + default: false --- include: @@ -88,6 +92,11 @@ resolve_ci_image: - job: build_ci_image optional: true artifacts: false + rules: + - if: $[[ inputs.only_on_push ]] && $CI_PIPELINE_SOURCE == "push" + when: always + - if: $[[ inputs.only_on_push ]] + when: never script: - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git - git -C system-tests checkout $[[ inputs.ref ]] @@ -108,6 +117,11 @@ build_test_pipeline: id_tokens: DD_STS_OIDC_TOKEN: aud: dd-sts + rules: + - if: $[[ inputs.only_on_push ]] && $CI_PIPELINE_SOURCE == "push" + when: always + - if: $[[ inputs.only_on_push ]] + when: never script: - > for library in $(echo "$[[ inputs.libraries ]]" | tr ',' ' '); do @@ -124,6 +138,11 @@ build_test_pipeline: needs: - job: build_test_pipeline artifacts: true + rules: + - if: $[[ inputs.only_on_push ]] && $CI_PIPELINE_SOURCE == "push" + when: always + - if: $[[ inputs.only_on_push ]] + when: never variables: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" From 7d52a9741aecd93cc0748401af165bf67e5122d9 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 13:20:42 +0200 Subject: [PATCH 121/229] restore original SSI pipeline in .gitlab-ci.yml, add push-gated end-to-end pipeline The original .gitlab-ci.yml (consumed by tracer repos via one-pipeline) is fully restored from origin/main. The end-to-end pipeline is included from utils/ci/gitlab/main.yml. A SYSTEM_TESTS_E2E_ENABLED variable gates the e2e jobs, and build_ci_image has a direct push rule, so the e2e pipeline only runs on direct pushes to the system-tests repo and never on child pipelines triggered by tracer repos (which get parent_pipeline). --- .gitlab-ci.yml | 4 +++- utils/ci/gitlab/main.yml | 17 ++++------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f120b925e0b..3e1555133c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -309,6 +309,9 @@ generate_system_tests_lib_injection_images: # End-to-end pipeline — only on direct pushes # ────────────────────────────────────────────── +variables: + SYSTEM_TESTS_E2E_ENABLED: "true" + include: - local: /utils/ci/gitlab/main.yml inputs: @@ -318,7 +321,6 @@ include: excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true - only_on_push: true build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index b7dbd16e20f..e0f35af234b 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -52,10 +52,6 @@ spec: ssi_injector_version: description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" default: "" - only_on_push: - description: "Only run jobs on direct push events" - type: boolean - default: false --- include: @@ -93,9 +89,7 @@ resolve_ci_image: optional: true artifacts: false rules: - - if: $[[ inputs.only_on_push ]] && $CI_PIPELINE_SOURCE == "push" - when: always - - if: $[[ inputs.only_on_push ]] + - if: $SYSTEM_TESTS_E2E_ENABLED != "true" when: never script: - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git @@ -118,9 +112,7 @@ build_test_pipeline: DD_STS_OIDC_TOKEN: aud: dd-sts rules: - - if: $[[ inputs.only_on_push ]] && $CI_PIPELINE_SOURCE == "push" - when: always - - if: $[[ inputs.only_on_push ]] + - if: $SYSTEM_TESTS_E2E_ENABLED != "true" when: never script: - > @@ -137,11 +129,10 @@ build_test_pipeline: stage: $[[ inputs.stage ]] needs: - job: build_test_pipeline + optional: true artifacts: true rules: - - if: $[[ inputs.only_on_push ]] && $CI_PIPELINE_SOURCE == "push" - when: always - - if: $[[ inputs.only_on_push ]] + - if: $SYSTEM_TESTS_E2E_ENABLED != "true" when: never variables: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" From d588f946f1037887067ed71595132934ad02bda2 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 13:58:07 +0200 Subject: [PATCH 122/229] restore original SSI pipeline in .gitlab-ci.yml, add push-gated end-to-end pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original .gitlab-ci.yml (consumed by tracer repos via one-pipeline) is fully restored from origin/main. The end-to-end pipeline is included from utils/ci/gitlab/main.yml. Jobs in main.yml have rules gated on the SYSTEM_TESTS_E2E_ENABLED variable (set by .gitlab-ci.yml on push), and build_ci_image has a direct push rule — so the e2e pipeline only runs on system-tests repo pushes. All artifact needs are optional to avoid pipeline creation failures when jobs are excluded by rules. --- utils/ci/gitlab/main.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index e0f35af234b..d7bdca10dd4 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -146,6 +146,7 @@ run_test_pipeline_java: include: - artifact: system-tests/generated-pipeline-java.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_java_otel: @@ -156,6 +157,7 @@ run_test_pipeline_java_otel: include: - artifact: system-tests/generated-pipeline-java_otel.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_java_lambda: @@ -166,6 +168,7 @@ run_test_pipeline_java_lambda: include: - artifact: system-tests/generated-pipeline-java_lambda.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_python: @@ -176,6 +179,7 @@ run_test_pipeline_python: include: - artifact: system-tests/generated-pipeline-python.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_python_otel: @@ -186,6 +190,7 @@ run_test_pipeline_python_otel: include: - artifact: system-tests/generated-pipeline-python_otel.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_python_lambda: @@ -196,6 +201,7 @@ run_test_pipeline_python_lambda: include: - artifact: system-tests/generated-pipeline-python_lambda.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_nodejs: @@ -206,6 +212,7 @@ run_test_pipeline_nodejs: include: - artifact: system-tests/generated-pipeline-nodejs.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_nodejs_otel: @@ -216,6 +223,7 @@ run_test_pipeline_nodejs_otel: include: - artifact: system-tests/generated-pipeline-nodejs_otel.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_nodejs_lambda: @@ -226,6 +234,7 @@ run_test_pipeline_nodejs_lambda: include: - artifact: system-tests/generated-pipeline-nodejs_lambda.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_dotnet: @@ -236,6 +245,7 @@ run_test_pipeline_dotnet: include: - artifact: system-tests/generated-pipeline-dotnet.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_golang: @@ -246,6 +256,7 @@ run_test_pipeline_golang: include: - artifact: system-tests/generated-pipeline-golang.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_php: @@ -256,6 +267,7 @@ run_test_pipeline_php: include: - artifact: system-tests/generated-pipeline-php.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_ruby: @@ -266,6 +278,7 @@ run_test_pipeline_ruby: include: - artifact: system-tests/generated-pipeline-ruby.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_cpp: @@ -276,6 +289,7 @@ run_test_pipeline_cpp: include: - artifact: system-tests/generated-pipeline-cpp.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_cpp_nginx: @@ -286,6 +300,7 @@ run_test_pipeline_cpp_nginx: include: - artifact: system-tests/generated-pipeline-cpp_nginx.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_cpp_kong: @@ -296,6 +311,7 @@ run_test_pipeline_cpp_kong: include: - artifact: system-tests/generated-pipeline-cpp_kong.yml job: build_test_pipeline + optional: true strategy: depend run_test_pipeline_cpp_httpd: @@ -306,5 +322,6 @@ run_test_pipeline_cpp_httpd: include: - artifact: system-tests/generated-pipeline-cpp_httpd.yml job: build_test_pipeline + optional: true strategy: depend From aa147f1d1b0d500ff3d234fb9b7c5bff10d3fada Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 15:46:58 +0200 Subject: [PATCH 123/229] Clean dynamic parameter selection --- .gitlab-ci.yml | 603 ++++++++++++++++++++------------------- utils/ci/gitlab/main.yml | 92 +++--- 2 files changed, 338 insertions(+), 357 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3e1555133c5..d5b41d227f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,12 @@ include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml + - local: /utils/ci/gitlab/main.yml + inputs: + stage: "build" + scenarios_groups: "all" + excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" + ref: "$CI_COMMIT_SHA" + push_to_test_optimization: true stages: - configure - nodejs @@ -15,312 +22,306 @@ stages: variables: TEST: 1 -compute_pipeline: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 - tags: ["arch:amd64"] - stage: configure - variables: - CI_ENVIRONMENT: "prod" - script: - - | - if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then - echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined.Computing changes..." - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - source venv/bin/activate - ./run.sh MOCK_THE_TEST --collect-only --scenario-report - git clone https://github.com/DataDog/system-tests.git original - git fetch --all # Ensure all branches are available - git branch --track $CI_COMMIT_REF_NAME origin/$CI_COMMIT_REF_NAME || true # Track branch if not tracked - git checkout $CI_COMMIT_REF_NAME # Ensure the branch is checked out - BASE_COMMIT=$(git merge-base origin/main $CI_COMMIT_REF_NAME) # Get the base commit - echo "Branch was created from commit--> $BASE_COMMIT" - git diff --name-only $BASE_COMMIT $CI_COMMIT_SHA >> modified_files.txt # List modified files - cat modified_files.txt - python utils/scripts/compute_libraries_and_scenarios.py >> impacted_scenarios.txt - cat impacted_scenarios.txt - source impacted_scenarios.txt - else - echo "✅ SYSTEM_TESTS_SCENARIOS OR SYSTEM_TESTS_SCENARIOS_GROUPS are set." - export scenarios=$SYSTEM_TESTS_SCENARIOS - export scenarios_groups=$SYSTEM_TESTS_SCENARIOS_GROUPS - cd /system-tests - git pull - if [ -n "$SYSTEM_TESTS_REF" ]; then - git checkout $SYSTEM_TESTS_REF - else - echo "⚠️ SYSTEM_TESTS_REF variable is not set, skipping git checkout" - fi - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - source venv/bin/activate - fi - - python utils/scripts/compute-workflow-parameters.py nodejs -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py java -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py dotnet -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py python -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py php -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - python utils/scripts/compute-workflow-parameters.py ruby -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - - | - if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then - cp *.yml "$CI_PROJECT_DIR" - fi - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - artifacts: - paths: - - nodejs_ssi_gitlab_pipeline.yml - - java_ssi_gitlab_pipeline.yml - - dotnet_ssi_gitlab_pipeline.yml - - python_ssi_gitlab_pipeline.yml - - php_ssi_gitlab_pipeline.yml - - ruby_ssi_gitlab_pipeline.yml - -nodejs_ssi_pipeline: - stage: nodejs - needs: ["compute_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: nodejs_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -java_ssi_pipeline: - stage: java - needs: ["compute_pipeline", "nodejs_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: java_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -dotnet_ssi_pipeline: - stage: dotnet - needs: ["compute_pipeline", "java_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: dotnet_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -python_ssi_pipeline: - stage: python - needs: ["compute_pipeline", "dotnet_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: python_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -php_ssi_pipeline: - stage: php - needs: ["compute_pipeline", "python_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: php_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -ruby_ssi_pipeline: - stage: ruby - needs: ["compute_pipeline", "php_ssi_pipeline"] - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: ruby_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - when: always - -delete_amis: - extends: .base_job_onboarding - stage: system-tests-utils - allow_failure: false - variables: - AMI_RETENTION_DAYS: 10 - AMI_LAST_LAUNCHED_DAYS: 10 - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - python utils/scripts/pulumi_clean_up.py --component amis --ami-retention-days $AMI_RETENTION_DAYS --ami-last-launched-days $AMI_LAST_LAUNCHED_DAYS - rules: - - if: '$SCHEDULED_JOB == "delete_amis"' - after_script: echo "Finish" - timeout: 3h - -delete_amis_by_name_or_lang: - extends: .base_job_onboarding - stage: system-tests-utils - allow_failure: false - script: - - | - if [ -z "$AMI_NAME" ] && [ -z "$AMI_LANG" ]; then - echo "❌ ERROR: Either AMI_NAME or AMI_LANG must be set." - exit 1 - fi - echo "✅ Proceeding with AMI_NAME=$AMI_NAME, AMI_LANG=$AMI_LANG" - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - | - CMD="python utils/scripts/pulumi_clean_up.py --component amis_by_name" - - if [ -n "$AMI_NAME" ]; then - CMD="$CMD --ami-name $AMI_NAME" - fi - - if [ -n "$AMI_LANG" ]; then - CMD="$CMD --ami-lang $AMI_LANG" - fi +# compute_pipeline: +# image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 +# tags: ["arch:amd64"] +# stage: configure +# variables: +# CI_ENVIRONMENT: "prod" +# script: +# - | +# if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then +# echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined.Computing changes..." +# SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner +# source venv/bin/activate +# ./run.sh MOCK_THE_TEST --collect-only --scenario-report +# git clone https://github.com/DataDog/system-tests.git original +# git fetch --all # Ensure all branches are available +# git branch --track $CI_COMMIT_REF_NAME origin/$CI_COMMIT_REF_NAME || true # Track branch if not tracked +# git checkout $CI_COMMIT_REF_NAME # Ensure the branch is checked out +# BASE_COMMIT=$(git merge-base origin/main $CI_COMMIT_REF_NAME) # Get the base commit +# echo "Branch was created from commit--> $BASE_COMMIT" +# git diff --name-only $BASE_COMMIT $CI_COMMIT_SHA >> modified_files.txt # List modified files +# cat modified_files.txt +# python utils/scripts/compute_libraries_and_scenarios.py >> impacted_scenarios.txt +# cat impacted_scenarios.txt +# source impacted_scenarios.txt +# else +# echo "✅ SYSTEM_TESTS_SCENARIOS OR SYSTEM_TESTS_SCENARIOS_GROUPS are set." +# export scenarios=$SYSTEM_TESTS_SCENARIOS +# export scenarios_groups=$SYSTEM_TESTS_SCENARIOS_GROUPS +# cd /system-tests +# git pull +# if [ -n "$SYSTEM_TESTS_REF" ]; then +# git checkout $SYSTEM_TESTS_REF +# else +# echo "⚠️ SYSTEM_TESTS_REF variable is not set, skipping git checkout" +# fi +# SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner +# source venv/bin/activate +# fi +# - python utils/scripts/compute-workflow-parameters.py nodejs -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab +# - python utils/scripts/compute-workflow-parameters.py java -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab +# - python utils/scripts/compute-workflow-parameters.py dotnet -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab +# - python utils/scripts/compute-workflow-parameters.py python -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab +# - python utils/scripts/compute-workflow-parameters.py php -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab +# - python utils/scripts/compute-workflow-parameters.py ruby -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab +# - | +# if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then +# cp *.yml "$CI_PROJECT_DIR" +# fi +# rules: +# - if: '$SCHEDULED_JOB == ""' +# - if: '$SCHEDULED_JOB == null' +# artifacts: +# paths: +# - nodejs_ssi_gitlab_pipeline.yml +# - java_ssi_gitlab_pipeline.yml +# - dotnet_ssi_gitlab_pipeline.yml +# - python_ssi_gitlab_pipeline.yml +# - php_ssi_gitlab_pipeline.yml +# - ruby_ssi_gitlab_pipeline.yml +# +# nodejs_ssi_pipeline: +# stage: nodejs +# needs: ["compute_pipeline"] +# variables: +# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE +# trigger: +# include: +# - artifact: nodejs_ssi_gitlab_pipeline.yml +# job: compute_pipeline +# strategy: depend +# rules: +# - if: '$SCHEDULED_JOB == ""' +# - if: '$SCHEDULED_JOB == null' +# when: always +# +# java_ssi_pipeline: +# stage: java +# needs: ["compute_pipeline", "nodejs_ssi_pipeline"] +# variables: +# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE +# trigger: +# include: +# - artifact: java_ssi_gitlab_pipeline.yml +# job: compute_pipeline +# strategy: depend +# rules: +# - if: '$SCHEDULED_JOB == ""' +# - if: '$SCHEDULED_JOB == null' +# when: always +# +# dotnet_ssi_pipeline: +# stage: dotnet +# needs: ["compute_pipeline", "java_ssi_pipeline"] +# variables: +# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE +# trigger: +# include: +# - artifact: dotnet_ssi_gitlab_pipeline.yml +# job: compute_pipeline +# strategy: depend +# rules: +# - if: '$SCHEDULED_JOB == ""' +# - if: '$SCHEDULED_JOB == null' +# when: always +# +# python_ssi_pipeline: +# stage: python +# needs: ["compute_pipeline", "dotnet_ssi_pipeline"] +# variables: +# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE +# trigger: +# include: +# - artifact: python_ssi_gitlab_pipeline.yml +# job: compute_pipeline +# strategy: depend +# rules: +# - if: '$SCHEDULED_JOB == ""' +# - if: '$SCHEDULED_JOB == null' +# when: always +# +# php_ssi_pipeline: +# stage: php +# needs: ["compute_pipeline", "python_ssi_pipeline"] +# variables: +# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE +# trigger: +# include: +# - artifact: php_ssi_gitlab_pipeline.yml +# job: compute_pipeline +# strategy: depend +# rules: +# - if: '$SCHEDULED_JOB == ""' +# - if: '$SCHEDULED_JOB == null' +# when: always +# +# ruby_ssi_pipeline: +# stage: ruby +# needs: ["compute_pipeline", "php_ssi_pipeline"] +# variables: +# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE +# trigger: +# include: +# - artifact: ruby_ssi_gitlab_pipeline.yml +# job: compute_pipeline +# strategy: depend +# rules: +# - if: '$SCHEDULED_JOB == ""' +# - if: '$SCHEDULED_JOB == null' +# when: always +# +# delete_amis: +# extends: .base_job_onboarding +# stage: system-tests-utils +# allow_failure: false +# variables: +# AMI_RETENTION_DAYS: 10 +# AMI_LAST_LAUNCHED_DAYS: 10 +# script: +# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner +# - source venv/bin/activate +# - python utils/scripts/pulumi_clean_up.py --component amis --ami-retention-days $AMI_RETENTION_DAYS --ami-last-launched-days $AMI_LAST_LAUNCHED_DAYS +# rules: +# - if: '$SCHEDULED_JOB == "delete_amis"' +# after_script: echo "Finish" +# timeout: 3h +# +# delete_amis_by_name_or_lang: +# extends: .base_job_onboarding +# stage: system-tests-utils +# allow_failure: false +# script: +# - | +# if [ -z "$AMI_NAME" ] && [ -z "$AMI_LANG" ]; then +# echo "❌ ERROR: Either AMI_NAME or AMI_LANG must be set." +# exit 1 +# fi +# echo "✅ Proceeding with AMI_NAME=$AMI_NAME, AMI_LANG=$AMI_LANG" +# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner +# - source venv/bin/activate +# - | +# CMD="python utils/scripts/pulumi_clean_up.py --component amis_by_name" +# +# if [ -n "$AMI_NAME" ]; then +# CMD="$CMD --ami-name $AMI_NAME" +# fi +# +# if [ -n "$AMI_LANG" ]; then +# CMD="$CMD --ami-lang $AMI_LANG" +# fi +# +# echo "Running: $CMD" +# eval $CMD +# rules: +# - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' +# after_script: echo "Finish" +# +# delete_ec2_instances: +# extends: .base_job_onboarding +# stage: system-tests-utils +# allow_failure: false +# variables: +# EC2_AGE_MINUTES: 45 +# script: +# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner +# - source venv/bin/activate +# - python utils/scripts/pulumi_clean_up.py --component ec2 --ec2-age-minutes $EC2_AGE_MINUTES +# rules: +# - if: '$SCHEDULED_JOB == "delete_ec2_instances"' +# after_script: echo "Finish" +# +# count_amis: +# extends: .base_job_onboarding +# stage: system-tests-utils +# allow_failure: false +# script: +# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner +# - source venv/bin/activate +# - python utils/scripts/pulumi_clean_up.py --component amis_count +# rules: +# - if: '$SCHEDULED_JOB == "count_amis"' +# after_script: echo "Finish" +# +# check_merge_labels: +# #Build docker images if it's needed. Check if the PR has the labels associated with the image build. +# image: registry.ddbuild.io/system-tests/base-image-builder +# tags: +# - arch:amd64 +# id_tokens: +# DDOCTOSTS_ID_TOKEN: +# aud: dd-octo-sts +# needs: [] +# stage: system-tests-utils +# before_script: +# # Get a token +# - dd-octo-sts version +# - dd-octo-sts debug --scope DataDog/system-tests --policy self.gitlab-read +# - dd-octo-sts token --scope DataDog/system-tests --policy self.gitlab-read > token.txt +# - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) +# - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) +# script: +# - export GITHUB_TOKEN=$(cat token.txt) +# - ./utils/scripts/get_pr_merged_labels.sh +# after_script: +# # Revoke the token after usage +# - dd-octo-sts revoke -t $(cat token.txt) +# rules: +# - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" +# +# generate_system_tests_lambda_proxy_image: +# image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 +# tags: ["docker-in-docker:amd64"] +# needs: [] +# stage: system-tests-utils +# allow_failure: false +# before_script: +# - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) +# - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) +# - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin +# script: +# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i lambda-proxy +# - docker push datadog/system-tests:lambda-proxy-v1 +# rules: +# - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" +# changes: +# - utils/build/docker/lambda_proxy/pyproject.toml +# - utils/build/docker/lambda-proxy.Dockerfile +# +# generate_system_tests_lib_injection_images: +# extends: .base_job_k8s_docker_ssi +# needs: [] +# stage: system-tests-utils +# allow_failure: true +# variables: +# PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com +# PRIVATE_DOCKER_REGISTRY_USER: AWS +# script: +# - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} +# - ./lib-injection/build/build_lib_injection_images.sh $PRIVATE_DOCKER_REGISTRY +# rules: +# - if: '$SCHEDULED_JOB == "generate_system_tests_lib_injection_images"' +# - when: manual +# +# .delayed_base_job: +# image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 +# tags: ["arch:amd64"] +# script: +# - echo "⏳ Waiting before triggering the child pipeline..." +# when: delayed +# start_in: 5 minutes - echo "Running: $CMD" - eval $CMD - rules: - - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' - after_script: echo "Finish" - -delete_ec2_instances: - extends: .base_job_onboarding - stage: system-tests-utils - allow_failure: false - variables: - EC2_AGE_MINUTES: 45 - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - python utils/scripts/pulumi_clean_up.py --component ec2 --ec2-age-minutes $EC2_AGE_MINUTES - rules: - - if: '$SCHEDULED_JOB == "delete_ec2_instances"' - after_script: echo "Finish" - -count_amis: - extends: .base_job_onboarding - stage: system-tests-utils - allow_failure: false - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - python utils/scripts/pulumi_clean_up.py --component amis_count - rules: - - if: '$SCHEDULED_JOB == "count_amis"' - after_script: echo "Finish" +# ────────────────────────────────────────────── +# End-to-end pipeline — only for the system-tests repo +# ────────────────────────────────────────────── -check_merge_labels: - #Build docker images if it's needed. Check if the PR has the labels associated with the image build. - image: registry.ddbuild.io/system-tests/base-image-builder +system-tests-param: + stage: build tags: - arch:amd64 - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - needs: [] - stage: system-tests-utils - before_script: - # Get a token - - dd-octo-sts version - - dd-octo-sts debug --scope DataDog/system-tests --policy self.gitlab-read - - dd-octo-sts token --scope DataDog/system-tests --policy self.gitlab-read > token.txt - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - script: - - export GITHUB_TOKEN=$(cat token.txt) - - ./utils/scripts/get_pr_merged_labels.sh - after_script: - # Revoke the token after usage - - dd-octo-sts revoke -t $(cat token.txt) - rules: - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" - -generate_system_tests_lambda_proxy_image: - image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 - tags: ["docker-in-docker:amd64"] - needs: [] - stage: system-tests-utils - allow_failure: false - before_script: - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i lambda-proxy - - docker push datadog/system-tests:lambda-proxy-v1 - rules: - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" - changes: - - utils/build/docker/lambda_proxy/pyproject.toml - - utils/build/docker/lambda-proxy.Dockerfile - -generate_system_tests_lib_injection_images: - extends: .base_job_k8s_docker_ssi - needs: [] - stage: system-tests-utils - allow_failure: true - variables: - PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com - PRIVATE_DOCKER_REGISTRY_USER: AWS script: - - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} - - ./lib-injection/build/build_lib_injection_images.sh $PRIVATE_DOCKER_REGISTRY - rules: - - if: '$SCHEDULED_JOB == "generate_system_tests_lib_injection_images"' - - when: manual - -.delayed_base_job: - image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 - tags: ["arch:amd64"] - script: - - echo "⏳ Waiting before triggering the child pipeline..." - when: delayed - start_in: 5 minutes - -# ────────────────────────────────────────────── -# End-to-end pipeline — only on direct pushes -# ────────────────────────────────────────────── - -variables: - SYSTEM_TESTS_E2E_ENABLED: "true" - -include: - - local: /utils/ci/gitlab/main.yml - inputs: - stage: "build" - libraries: "python" - scenarios_groups: "all" - excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" - ref: "$CI_COMMIT_SHA" - push_to_test_optimization: true + - echo 'LIBRARIES="python"' > param.env build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index d7bdca10dd4..5693b1fb028 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -3,7 +3,7 @@ spec: stage: description: "CI stage for all jobs in this component and the generated child pipeline" libraries: - default: "java python nodejs" + default: "" scenarios: description: "Comma-separated list of scenarios to run" default: "" @@ -59,19 +59,18 @@ include: inputs: stage: $[[ inputs.stage ]] ref: $[[ inputs.ref ]] - - local: /utils/ci/gitlab/ssi.yml - inputs: - stage: $[[ inputs.stage ]] - libraries: $[[ inputs.libraries ]] - scenarios: $[[ inputs.scenarios ]] - scenarios_groups: $[[ inputs.scenarios_groups ]] - ssi_library_version: $[[ inputs.ssi_library_version ]] - k8s_lib_init_img: $[[ inputs.k8s_lib_init_img ]] - ssi_injector_version: $[[ inputs.ssi_injector_version ]] + # - local: /utils/ci/gitlab/ssi.yml + # inputs: + # stage: $[[ inputs.stage ]] + # libraries: $[[ inputs.libraries ]] + # scenarios: $[[ inputs.scenarios ]] + # scenarios_groups: $[[ inputs.scenarios_groups ]] + # ssi_library_version: $[[ inputs.ssi_library_version ]] + # k8s_lib_init_img: $[[ inputs.k8s_lib_init_img ]] + # ssi_injector_version: $[[ inputs.ssi_injector_version ]] workflow: rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - when: always auto_cancel: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] @@ -88,9 +87,6 @@ resolve_ci_image: - job: build_ci_image optional: true artifacts: false - rules: - - if: $SYSTEM_TESTS_E2E_ENABLED != "true" - when: never script: - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git - git -C system-tests checkout $[[ inputs.ref ]] @@ -108,16 +104,16 @@ build_test_pipeline: needs: - job: resolve_ci_image artifacts: true + - job: system-tests-param + optional: true + artifacts: true id_tokens: DD_STS_OIDC_TOKEN: aud: dd-sts - rules: - - if: $SYSTEM_TESTS_E2E_ENABLED != "true" - when: never script: - > - for library in $(echo "$[[ inputs.libraries ]]" | tr ',' ' '); do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]]" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$[[ inputs.scenarios ]]" --weblogs "$[[ inputs.weblogs ]]" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; + for library in $(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' '); do + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" --excluded-scenarios "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" --scenarios "$[[ inputs.scenarios ]],$SCENARIOS" --weblogs "$[[ inputs.weblogs ]],$WEBLOGS" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" > generated-pipeline-${library}.yml; done artifacts: @@ -131,197 +127,181 @@ build_test_pipeline: - job: build_test_pipeline optional: true artifacts: true - rules: - - if: $SYSTEM_TESTS_E2E_ENABLED != "true" - when: never + - job: system-tests-param + optional: true + artifacts: true variables: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" + LIBRARIES: $LIBRARIES run_test_pipeline_java: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bjava\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bjava\b/' trigger: include: - artifact: system-tests/generated-pipeline-java.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_java_otel: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bjava_otel\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bjava_otel\b/' trigger: include: - artifact: system-tests/generated-pipeline-java_otel.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_java_lambda: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bjava_lambda\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bjava_lambda\b/' trigger: include: - artifact: system-tests/generated-pipeline-java_lambda.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_python: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bpython\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bpython\b/' trigger: include: - artifact: system-tests/generated-pipeline-python.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_python_otel: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bpython_otel\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bpython_otel\b/' trigger: include: - artifact: system-tests/generated-pipeline-python_otel.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_python_lambda: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bpython_lambda\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bpython_lambda\b/' trigger: include: - artifact: system-tests/generated-pipeline-python_lambda.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_nodejs: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bnodejs\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bnodejs\b/' trigger: include: - artifact: system-tests/generated-pipeline-nodejs.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_nodejs_otel: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bnodejs_otel\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bnodejs_otel\b/' trigger: include: - artifact: system-tests/generated-pipeline-nodejs_otel.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_nodejs_lambda: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bnodejs_lambda\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bnodejs_lambda\b/' trigger: include: - artifact: system-tests/generated-pipeline-nodejs_lambda.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_dotnet: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bdotnet\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bdotnet\b/' trigger: include: - artifact: system-tests/generated-pipeline-dotnet.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_golang: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bgolang\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bgolang\b/' trigger: include: - artifact: system-tests/generated-pipeline-golang.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_php: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bphp\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bphp\b/' trigger: include: - artifact: system-tests/generated-pipeline-php.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_ruby: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bruby\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bruby\b/' trigger: include: - artifact: system-tests/generated-pipeline-ruby.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_cpp: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bcpp\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bcpp\b/' trigger: include: - artifact: system-tests/generated-pipeline-cpp.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_cpp_nginx: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bcpp_nginx\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bcpp_nginx\b/' trigger: include: - artifact: system-tests/generated-pipeline-cpp_nginx.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_cpp_kong: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bcpp_kong\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bcpp_kong\b/' trigger: include: - artifact: system-tests/generated-pipeline-cpp_kong.yml job: build_test_pipeline - optional: true strategy: depend run_test_pipeline_cpp_httpd: extends: .run_test_pipeline_base rules: - - if: '"$[[ inputs.libraries ]]" =~ /\bcpp_httpd\b/' + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bcpp_httpd\b/' trigger: include: - artifact: system-tests/generated-pipeline-cpp_httpd.yml job: build_test_pipeline - optional: true strategy: depend From 95881123cd1e5e7aba2634859a041de580c5a3f3 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 17:37:07 +0200 Subject: [PATCH 124/229] gitlab: split generated pipeline into 2 chunks, single trigger job --- utils/ci/gitlab/build_pipeline.py | 19 ++- utils/ci/gitlab/main.yml | 190 +++--------------------------- 2 files changed, 35 insertions(+), 174 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 3047e746ea8..d7af50e8286 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -12,6 +12,7 @@ parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") parser.add_argument("--test-optimization-datadog-site", default="datadoghq.com", help="Datadog site for Test Optimization") +parser.add_argument("--skip-header", action="store_true", help="Skip the workflow/include/variables header (for concatenation)") args = parser.parse_args() @@ -27,7 +28,7 @@ template = env.get_template("system-tests.yml") -print(template.render( +output = template.render( scenarios=scenario_list, stage=args.stage, library=args.library, @@ -39,6 +40,20 @@ ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", test_optimization_datadog_site=args.test_optimization_datadog_site, -)) +) + +if args.skip_header: + # Start output from the first generated job (build_ or run_) + lines = output.splitlines(keepends=True) + body_start = 0 + for i, line in enumerate(lines): + if not line.startswith(" ") and ":" in line: + key = line.split(":")[0] + if key.startswith("build_") or key.startswith("run_"): + body_start = i + break + print("".join(lines[body_start:]), end="") +else: + print(output, end="") diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 5693b1fb028..6c2b11d8b37 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -111,16 +111,27 @@ build_test_pipeline: DD_STS_OIDC_TOKEN: aud: dd-sts script: - - > - for library in $(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' '); do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" --excluded-scenarios "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" --scenarios "$[[ inputs.scenarios ]],$SCENARIOS" --weblogs "$[[ inputs.weblogs ]],$WEBLOGS" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json; - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" > generated-pipeline-${library}.yml; + - | + libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ') + count=0 + for lib in $libraries; do count=$((count + 1)); done + half=$(( (count + 1) / 2 )) + idx=0 + for library in $libraries; do + [ $idx -lt $half ] && chunk=0 || chunk=1 + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" --excluded-scenarios "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" --scenarios "$[[ inputs.scenarios ]],$SCENARIOS" --weblogs "$[[ inputs.weblogs ]],$WEBLOGS" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json + if [ ! -f generated-pipeline-chunk-${chunk}.yml ]; then + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" > generated-pipeline-chunk-${chunk}.yml + else + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" --skip-header >> generated-pipeline-chunk-${chunk}.yml + fi + idx=$((idx + 1)) done artifacts: paths: - - system-tests/generated-pipeline-*.yml + - system-tests/generated-pipeline-chunk-*.yml -.run_test_pipeline_base: +run_test_pipeline: interruptible: true stage: $[[ inputs.stage ]] needs: @@ -134,174 +145,9 @@ build_test_pipeline: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" LIBRARIES: $LIBRARIES - -run_test_pipeline_java: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bjava\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-java.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_java_otel: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bjava_otel\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-java_otel.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_java_lambda: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bjava_lambda\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-java_lambda.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_python: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bpython\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-python.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_python_otel: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bpython_otel\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-python_otel.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_python_lambda: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bpython_lambda\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-python_lambda.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_nodejs: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bnodejs\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-nodejs.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_nodejs_otel: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bnodejs_otel\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-nodejs_otel.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_nodejs_lambda: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bnodejs_lambda\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-nodejs_lambda.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_dotnet: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bdotnet\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-dotnet.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_golang: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bgolang\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-golang.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_php: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bphp\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-php.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_ruby: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bruby\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-ruby.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_cpp: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bcpp\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-cpp.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_cpp_nginx: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bcpp_nginx\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-cpp_nginx.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_cpp_kong: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bcpp_kong\b/' - trigger: - include: - - artifact: system-tests/generated-pipeline-cpp_kong.yml - job: build_test_pipeline - strategy: depend - -run_test_pipeline_cpp_httpd: - extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\bcpp_httpd\b/' trigger: include: - - artifact: system-tests/generated-pipeline-cpp_httpd.yml + - artifact: system-tests/generated-pipeline-chunk-*.yml job: build_test_pipeline strategy: depend From 7052113a3c187b81e371e1f82ed2b17bed85fd40 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 18:01:07 +0200 Subject: [PATCH 125/229] gitlab: fix trigger include when only one chunk is needed --- utils/ci/gitlab/main.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 6c2b11d8b37..d7a9f92762c 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -127,6 +127,11 @@ build_test_pipeline: fi idx=$((idx + 1)) done + # Ensure chunk-1 always exists: trigger include requires both files regardless of library count + if [ ! -f generated-pipeline-chunk-1.yml ]; then + first_job=$(grep -nm1 '^build_' generated-pipeline-chunk-0.yml | cut -d: -f1) + head -n $((first_job - 1)) generated-pipeline-chunk-0.yml > generated-pipeline-chunk-1.yml + fi artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml @@ -147,7 +152,9 @@ run_test_pipeline: LIBRARIES: $LIBRARIES trigger: include: - - artifact: system-tests/generated-pipeline-chunk-*.yml + - artifact: system-tests/generated-pipeline-chunk-0.yml + job: build_test_pipeline + - artifact: system-tests/generated-pipeline-chunk-1.yml job: build_test_pipeline strategy: depend From 560ef6296ffc451bee8d14d306c6f5c49253b230 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 18:11:01 +0200 Subject: [PATCH 126/229] gitlab: skip trigger when no libraries, guard chunk-1 fallback --- utils/ci/gitlab/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index d7a9f92762c..b3a7509870b 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -128,7 +128,7 @@ build_test_pipeline: idx=$((idx + 1)) done # Ensure chunk-1 always exists: trigger include requires both files regardless of library count - if [ ! -f generated-pipeline-chunk-1.yml ]; then + if [ ! -f generated-pipeline-chunk-1.yml ] && [ -f generated-pipeline-chunk-0.yml ]; then first_job=$(grep -nm1 '^build_' generated-pipeline-chunk-0.yml | cut -d: -f1) head -n $((first_job - 1)) generated-pipeline-chunk-0.yml > generated-pipeline-chunk-1.yml fi @@ -139,6 +139,9 @@ build_test_pipeline: run_test_pipeline: interruptible: true stage: $[[ inputs.stage ]] + rules: + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/' + - when: never needs: - job: build_test_pipeline optional: true From 5a280ef8418f7e078b5deae15f6d783a10b895a1 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 18:12:07 +0200 Subject: [PATCH 127/229] gitlab: export param.env as dotenv artifact --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5b41d227f5..64eae2e0507 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -322,6 +322,9 @@ system-tests-param: - arch:amd64 script: - echo 'LIBRARIES="python"' > param.env + artifacts: + reports: + dotenv: param.env build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy From 82bf0c7e1a176dcb52e78c18fdd7b7cff1274427 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 29 May 2026 18:16:06 +0200 Subject: [PATCH 128/229] gitlab: remove quotes around LIBRARIES value in param.env --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 64eae2e0507..125f6b6e001 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -321,7 +321,7 @@ system-tests-param: tags: - arch:amd64 script: - - echo 'LIBRARIES="python"' > param.env + - echo 'LIBRARIES=python' > param.env artifacts: reports: dotenv: param.env From 7c8945622bb8679ab5c90df1e9bff2d9efc4745b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 10:37:48 +0200 Subject: [PATCH 129/229] fix --- utils/ci/gitlab/main.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index b3a7509870b..830caf5a798 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -9,7 +9,7 @@ spec: default: "" scenarios_groups: description: "Comma-separated list of scenario groups to run" - default: "" + default: "all" excluded_scenarios: description: "Comma-separated list of scenarios not to run" default: "APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS" @@ -129,8 +129,12 @@ build_test_pipeline: done # Ensure chunk-1 always exists: trigger include requires both files regardless of library count if [ ! -f generated-pipeline-chunk-1.yml ] && [ -f generated-pipeline-chunk-0.yml ]; then - first_job=$(grep -nm1 '^build_' generated-pipeline-chunk-0.yml | cut -d: -f1) - head -n $((first_job - 1)) generated-pipeline-chunk-0.yml > generated-pipeline-chunk-1.yml + first_job=$(grep -nm1 '^build_' generated-pipeline-chunk-0.yml | cut -d: -f1 || true) + if [ -n "$first_job" ]; then + head -n $((first_job - 1)) generated-pipeline-chunk-0.yml > generated-pipeline-chunk-1.yml + else + cp generated-pipeline-chunk-0.yml generated-pipeline-chunk-1.yml + fi fi artifacts: paths: From 824b51c499f9f70714fefed1b6de6ec24f5ff8bc Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 10:49:16 +0200 Subject: [PATCH 130/229] gitlab: add debug prints for libraries, scenarios, scenario_groups --- utils/ci/gitlab/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 830caf5a798..7ffa42df3b1 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -113,13 +113,18 @@ build_test_pipeline: script: - | libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ') + scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS") + scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS") + echo "libraries: $libraries" + echo "scenarios: $scenarios" + echo "scenario_groups: $scenario_groups" count=0 for lib in $libraries; do count=$((count + 1)); done half=$(( (count + 1) / 2 )) idx=0 for library in $libraries; do [ $idx -lt $half ] && chunk=0 || chunk=1 - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" --excluded-scenarios "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" --scenarios "$[[ inputs.scenarios ]],$SCENARIOS" --weblogs "$[[ inputs.weblogs ]],$WEBLOGS" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" --scenarios "$scenarios" --weblogs "$[[ inputs.weblogs ]],$WEBLOGS" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json if [ ! -f generated-pipeline-chunk-${chunk}.yml ]; then python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" > generated-pipeline-chunk-${chunk}.yml else From 210f8d643fa6d285d7de4ded73e269e40d768dd6 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 10:51:55 +0200 Subject: [PATCH 131/229] gitlab: skip git clone in resolve_ci_image when build_ci_image already ran --- utils/ci/gitlab/main.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 7ffa42df3b1..9d8457b4bb9 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -86,13 +86,19 @@ resolve_ci_image: needs: - job: build_ci_image optional: true - artifacts: false + artifacts: true script: - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git - - git -C system-tests checkout $[[ inputs.ref ]] - - IMAGE_TAG=$(cat system-tests/utils/ci/gitlab/docker/system-tests.Dockerfile system-tests/requirements.txt | sha256sum | cut -c1-12) - - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env - - cat build.env + - | + if [ -n "$CI_IMAGE" ]; then + echo "Reusing CI_IMAGE from build_ci_image: $CI_IMAGE" + echo "CI_IMAGE=$CI_IMAGE" >> build.env + else + git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git + git -C system-tests checkout $[[ inputs.ref ]] + IMAGE_TAG=$(cat system-tests/utils/ci/gitlab/docker/system-tests.Dockerfile system-tests/requirements.txt | sha256sum | cut -c1-12) + echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env + fi + cat build.env artifacts: reports: dotenv: build.env From 5f389c3f5434d157a44bbb619eadc7d7a0b3c939 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 10:55:10 +0200 Subject: [PATCH 132/229] gitlab: add skip_resolve_ci_image input to fully skip the job --- .gitlab-ci.yml | 1 + utils/ci/gitlab/main.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 125f6b6e001..a9d2b2142d1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,7 @@ include: - local: /utils/ci/gitlab/main.yml inputs: stage: "build" + skip_resolve_ci_image: true scenarios_groups: "all" excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 9d8457b4bb9..785d3b4de1d 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -52,6 +52,10 @@ spec: ssi_injector_version: description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" default: "" + skip_resolve_ci_image: + description: "Skip resolve_ci_image when build_ci_image already provides CI_IMAGE" + type: boolean + default: false --- include: @@ -76,6 +80,10 @@ workflow: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] resolve_ci_image: + rules: + - if: '$[[ inputs.skip_resolve_ci_image ]]' + when: never + - when: always image: registry.ddbuild.io/system-tests/git tags: - arch:amd64 @@ -109,6 +117,10 @@ build_test_pipeline: - arch:amd64 needs: - job: resolve_ci_image + optional: true + artifacts: true + - job: build_ci_image + optional: true artifacts: true - job: system-tests-param optional: true From b71416a77aa79907cf1fc29bc96d12704d884f66 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 10:56:12 +0200 Subject: [PATCH 133/229] Revert "gitlab: skip git clone in resolve_ci_image when build_ci_image already ran" This reverts commit 210f8d643fa6d285d7de4ded73e269e40d768dd6. --- utils/ci/gitlab/main.yml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 785d3b4de1d..efc59b1ac2e 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -94,19 +94,13 @@ resolve_ci_image: needs: - job: build_ci_image optional: true - artifacts: true + artifacts: false script: - - | - if [ -n "$CI_IMAGE" ]; then - echo "Reusing CI_IMAGE from build_ci_image: $CI_IMAGE" - echo "CI_IMAGE=$CI_IMAGE" >> build.env - else - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git - git -C system-tests checkout $[[ inputs.ref ]] - IMAGE_TAG=$(cat system-tests/utils/ci/gitlab/docker/system-tests.Dockerfile system-tests/requirements.txt | sha256sum | cut -c1-12) - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env - fi - cat build.env + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git + - git -C system-tests checkout $[[ inputs.ref ]] + - IMAGE_TAG=$(cat system-tests/utils/ci/gitlab/docker/system-tests.Dockerfile system-tests/requirements.txt | sha256sum | cut -c1-12) + - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env + - cat build.env artifacts: reports: dotenv: build.env From 204a39f49922764846c14405ba151e219e7f333a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 10:58:05 +0200 Subject: [PATCH 134/229] gitlab: clean up libraries/scenarios/scenario_groups display values --- utils/ci/gitlab/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index efc59b1ac2e..0bf59d48fb7 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -124,9 +124,9 @@ build_test_pipeline: aud: dd-sts script: - | - libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ') - scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS") - scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS") + libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) + scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS" | tr ',' '\n' | grep . | tr '\n' ',' | sed 's/,$//') + scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" | tr ',' '\n' | grep . | tr '\n' ',' | sed 's/,$//') echo "libraries: $libraries" echo "scenarios: $scenarios" echo "scenario_groups: $scenario_groups" From 7a716919f8ac06b9e8000c667d24c0312d8cce31 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 11:05:38 +0200 Subject: [PATCH 135/229] gitlab: fix empty weblogs filter wiping all endtoend jobs --- utils/ci/gitlab/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 0bf59d48fb7..d872d45baac 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -127,16 +127,18 @@ build_test_pipeline: libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS" | tr ',' '\n' | grep . | tr '\n' ',' | sed 's/,$//') scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" | tr ',' '\n' | grep . | tr '\n' ',' | sed 's/,$//') + weblogs=$(echo "$[[ inputs.weblogs ]],$WEBLOGS" | tr ',' '\n' | grep . | tr '\n' ',' | sed 's/,$//') echo "libraries: $libraries" echo "scenarios: $scenarios" echo "scenario_groups: $scenario_groups" + echo "weblogs: $weblogs" count=0 for lib in $libraries; do count=$((count + 1)); done half=$(( (count + 1) / 2 )) idx=0 for library in $libraries; do [ $idx -lt $half ] && chunk=0 || chunk=1 - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" --scenarios "$scenarios" --weblogs "$[[ inputs.weblogs ]],$WEBLOGS" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json if [ ! -f generated-pipeline-chunk-${chunk}.yml ]; then python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" > generated-pipeline-chunk-${chunk}.yml else From 4fe38076b8cde6fb0d751fcd844aab22ecac3087 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 11:09:19 +0200 Subject: [PATCH 136/229] gitlab: use CI variable instead of component input in resolve_ci_image rule --- .gitlab-ci.yml | 2 +- utils/ci/gitlab/main.yml | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a9d2b2142d1..3947bc300f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ include: - local: /utils/ci/gitlab/main.yml inputs: stage: "build" - skip_resolve_ci_image: true scenarios_groups: "all" excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" @@ -22,6 +21,7 @@ stages: variables: TEST: 1 + SKIP_RESOLVE_CI_IMAGE: "true" # compute_pipeline: # image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index d872d45baac..218146eaa68 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -52,10 +52,7 @@ spec: ssi_injector_version: description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" default: "" - skip_resolve_ci_image: - description: "Skip resolve_ci_image when build_ci_image already provides CI_IMAGE" - type: boolean - default: false + --- include: @@ -81,7 +78,7 @@ workflow: resolve_ci_image: rules: - - if: '$[[ inputs.skip_resolve_ci_image ]]' + - if: '$SKIP_RESOLVE_CI_IMAGE == "true"' when: never - when: always image: registry.ddbuild.io/system-tests/git From 2af88f833ee94f587bfdf88009ed399a5fd059be Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 11:13:27 +0200 Subject: [PATCH 137/229] gitlab: replace grep with sed to avoid exit 1 on empty input --- utils/ci/gitlab/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 218146eaa68..5fe8d363079 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -122,9 +122,9 @@ build_test_pipeline: script: - | libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) - scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS" | tr ',' '\n' | grep . | tr '\n' ',' | sed 's/,$//') - scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" | tr ',' '\n' | grep . | tr '\n' ',' | sed 's/,$//') - weblogs=$(echo "$[[ inputs.weblogs ]],$WEBLOGS" | tr ',' '\n' | grep . | tr '\n' ',' | sed 's/,$//') + scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') + scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') + weblogs=$(echo "$[[ inputs.weblogs ]],$WEBLOGS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') echo "libraries: $libraries" echo "scenarios: $scenarios" echo "scenario_groups: $scenario_groups" From 9eebf8f4dabe8d2af37bf277a8b9b432a6072264 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 11:30:03 +0200 Subject: [PATCH 138/229] gitlab: use compute_libraries_and_scenarios in system-tests-param --- .gitlab-ci.yml | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3947bc300f5..196a54042a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ include: - local: /utils/ci/gitlab/main.yml inputs: stage: "build" - scenarios_groups: "all" excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true @@ -318,11 +317,35 @@ variables: # ────────────────────────────────────────────── system-tests-param: + image: $CI_IMAGE stage: build tags: - arch:amd64 + needs: + - job: build_ci_image + artifacts: true script: - - echo 'LIBRARIES=python' > param.env + - source /system-tests/venv/bin/activate + - export PYTHONPATH=$(pwd) + - git fetch origin main + - mkdir -p original && git archive origin/main -- manifests/ | tar -xC original/ + - git diff origin/main...HEAD --name-only > modified_files.txt + - mkdir -p logs_mock_the_test && echo '{}' > logs_mock_the_test/scenarios.json + - python3 utils/scripts/compute_libraries_and_scenarios.py --output raw_params.txt + - | + python3 -c " + import json + with open('raw_params.txt') as f: + for line in f: + line = line.strip() + if '=' not in line: continue + key, val = line.split('=', 1) + val = json.loads(val) + if key == 'libraries': print(f'LIBRARIES={val}') + elif key == 'scenarios': print(f'SCENARIOS={val}') + elif key == 'scenarios_groups': print(f'SCENARIO_GROUPS={val}') + " > param.env + - cat param.env artifacts: reports: dotenv: param.env From 18131aa8bfd0fb1694ae1e7bcdf437000fd45f87 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 11:39:48 +0200 Subject: [PATCH 139/229] gitlab: split into 3 chunks to stay under 5 MiB limit --- utils/ci/gitlab/main.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 5fe8d363079..8a35585eba9 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -9,7 +9,7 @@ spec: default: "" scenarios_groups: description: "Comma-separated list of scenario groups to run" - default: "all" + default: "" excluded_scenarios: description: "Comma-separated list of scenarios not to run" default: "APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS" @@ -131,10 +131,12 @@ build_test_pipeline: echo "weblogs: $weblogs" count=0 for lib in $libraries; do count=$((count + 1)); done - half=$(( (count + 1) / 2 )) + third=$(( (count + 2) / 3 )) + [ $third -eq 0 ] && third=1 idx=0 for library in $libraries; do - [ $idx -lt $half ] && chunk=0 || chunk=1 + chunk=$(( idx / third )) + [ $chunk -gt 2 ] && chunk=2 python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json if [ ! -f generated-pipeline-chunk-${chunk}.yml ]; then python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" > generated-pipeline-chunk-${chunk}.yml @@ -143,15 +145,17 @@ build_test_pipeline: fi idx=$((idx + 1)) done - # Ensure chunk-1 always exists: trigger include requires both files regardless of library count - if [ ! -f generated-pipeline-chunk-1.yml ] && [ -f generated-pipeline-chunk-0.yml ]; then - first_job=$(grep -nm1 '^build_' generated-pipeline-chunk-0.yml | cut -d: -f1 || true) - if [ -n "$first_job" ]; then - head -n $((first_job - 1)) generated-pipeline-chunk-0.yml > generated-pipeline-chunk-1.yml - else - cp generated-pipeline-chunk-0.yml generated-pipeline-chunk-1.yml + # Ensure all 3 chunk files exist: trigger include requires all files regardless of library count + for i in 1 2; do + if [ ! -f generated-pipeline-chunk-${i}.yml ] && [ -f generated-pipeline-chunk-0.yml ]; then + first_job=$(grep -nm1 '^build_' generated-pipeline-chunk-0.yml | cut -d: -f1 || true) + if [ -n "$first_job" ]; then + head -n $((first_job - 1)) generated-pipeline-chunk-0.yml > generated-pipeline-chunk-${i}.yml + else + cp generated-pipeline-chunk-0.yml generated-pipeline-chunk-${i}.yml + fi fi - fi + done artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml @@ -179,5 +183,7 @@ run_test_pipeline: job: build_test_pipeline - artifact: system-tests/generated-pipeline-chunk-1.yml job: build_test_pipeline + - artifact: system-tests/generated-pipeline-chunk-2.yml + job: build_test_pipeline strategy: depend From dab06f0c83238ce5252a204d8a039450399e85e0 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 13:11:02 +0200 Subject: [PATCH 140/229] gitlab: one child pipeline per chunk instead of merged single pipeline --- utils/ci/gitlab/main.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 8a35585eba9..73e73e2953a 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -160,7 +160,7 @@ build_test_pipeline: paths: - system-tests/generated-pipeline-chunk-*.yml -run_test_pipeline: +.run_test_pipeline_base: interruptible: true stage: $[[ inputs.stage ]] rules: @@ -177,12 +177,27 @@ run_test_pipeline: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" LIBRARIES: $LIBRARIES + +run_test_pipeline_0: + extends: .run_test_pipeline_base trigger: include: - artifact: system-tests/generated-pipeline-chunk-0.yml job: build_test_pipeline + strategy: depend + +run_test_pipeline_1: + extends: .run_test_pipeline_base + trigger: + include: - artifact: system-tests/generated-pipeline-chunk-1.yml job: build_test_pipeline + strategy: depend + +run_test_pipeline_2: + extends: .run_test_pipeline_base + trigger: + include: - artifact: system-tests/generated-pipeline-chunk-2.yml job: build_test_pipeline strategy: depend From 2833510636a2fb6212862521ac8173d9af14c2b4 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 13:31:52 +0200 Subject: [PATCH 141/229] gitlab: move header guard into template, remove fragile line-scanning in build_pipeline.py --- utils/ci/gitlab/build_pipeline.py | 21 +++------------------ utils/ci/gitlab/system-tests.yml | 2 ++ 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index d7af50e8286..eea30ed76ce 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -28,7 +28,7 @@ template = env.get_template("system-tests.yml") -output = template.render( +print(template.render( scenarios=scenario_list, stage=args.stage, library=args.library, @@ -40,20 +40,5 @@ ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", test_optimization_datadog_site=args.test_optimization_datadog_site, -) - -if args.skip_header: - # Start output from the first generated job (build_ or run_) - lines = output.splitlines(keepends=True) - body_start = 0 - for i, line in enumerate(lines): - if not line.startswith(" ") and ":" in line: - key = line.split(":")[0] - if key.startswith("build_") or key.startswith("run_"): - body_start = i - break - print("".join(lines[body_start:]), end="") -else: - print(output, end="") - - + skip_header=args.skip_header, +), end="") diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index bd037ced225..54293977729 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -1,3 +1,4 @@ +{% if not skip_header %} workflow: name: "System-tests end to end" @@ -31,6 +32,7 @@ setup: aud: dd-sts script: - dd-sts exchange --policy system-tests-gitlab -- datadog-ci tag --level pipeline --tags "system-tests:true" +{% endif %} {% for variant in weblog_variants %} build_{{library}}_{{variant}}: From 3cd642a8a6352cb0b4bcd51566d3b8c16fd9f239 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 13:33:52 +0200 Subject: [PATCH 142/229] gitlab: run MOCK_THE_TEST collect to generate real scenarios.json --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 196a54042a0..0ea29dfd96b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -330,7 +330,7 @@ system-tests-param: - git fetch origin main - mkdir -p original && git archive origin/main -- manifests/ | tar -xC original/ - git diff origin/main...HEAD --name-only > modified_files.txt - - mkdir -p logs_mock_the_test && echo '{}' > logs_mock_the_test/scenarios.json + - ./run.sh MOCK_THE_TEST --collect-only --scenario-report - python3 utils/scripts/compute_libraries_and_scenarios.py --output raw_params.txt - | python3 -c " @@ -369,5 +369,3 @@ build_ci_image: artifacts: reports: dotenv: build.env - rules: - - if: $CI_PIPELINE_SOURCE == "push" From 6ce2b53f8a0de7c0e00d5b240cca29acd12964eb Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 13:53:22 +0200 Subject: [PATCH 143/229] gitlab: symlink venv before run.sh to prevent runner rebuild --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ea29dfd96b..a86f9387287 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -325,7 +325,8 @@ system-tests-param: - job: build_ci_image artifacts: true script: - - source /system-tests/venv/bin/activate + - ln -sf /system-tests/venv venv + - source venv/bin/activate - export PYTHONPATH=$(pwd) - git fetch origin main - mkdir -p original && git archive origin/main -- manifests/ | tar -xC original/ From fdf6e840084f37e79edecded987b4b9d59ed0a17 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 15:47:29 +0200 Subject: [PATCH 144/229] gitlab: fix weblog host in DinD by setting SYSTEM_TESTS_WEBLOG_HOST --- utils/ci/gitlab/templates.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/ci/gitlab/templates.yml b/utils/ci/gitlab/templates.yml index 3a91e986ee8..4bba683dd11 100644 --- a/utils/ci/gitlab/templates.yml +++ b/utils/ci/gitlab/templates.yml @@ -12,6 +12,10 @@ spec: - docker-in-docker:amd64 variables: GIT_STRATEGY: none + # In DinD, DOCKER_HOST is set to tcp://docker:2376 which causes _Weblog to + # resolve the weblog as docker:7777 instead of localhost:7777, breaking span + # snapshot assertions. Force it back to localhost. + SYSTEM_TESTS_WEBLOG_HOST: "localhost" stage: $[[ inputs.stage ]] interruptible: true before_script: From 37d9ebcc5ccd1b0ce4631402a855db47fe82ac7f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 16:52:22 +0200 Subject: [PATCH 145/229] gitlab: use per-weblog scenario pairs instead of flat cross-product --- utils/ci/gitlab/build_pipeline.py | 13 ++++++++++--- utils/ci/gitlab/system-tests.yml | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index eea30ed76ce..bce320e07e4 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -18,8 +18,13 @@ with open(args.params) as f: params = json.load(f) -scenario_list = params["endtoend"]["scenarios"] -weblog_variants = params["endtoend"]["weblogs"] +# Use per-weblog scenario assignments from endtoend_defs +parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) +parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) +# Build variants to compile are those requiring build +weblog_variants = [w["name"] for w in parallel_weblogs] +# Flatten scenario/job pairs for run jobs +scenario_pairs = [(job["weblog"], scenario) for job in parallel_jobs for scenario in job.get("scenarios", [])] binaries_artifact = params["miscs"]["binaries_artifact"] ci_environment = params["miscs"]["ci_environment"] parametric = params["parametric"] @@ -28,8 +33,10 @@ template = env.get_template("system-tests.yml") +# Render the pipeline with per-weblog scenario pairs print(template.render( - scenarios=scenario_list, + # we're now using precomputed scenario pairs instead of flat scenarios list + scenario_pairs=scenario_pairs, stage=args.stage, library=args.library, weblog_variants=weblog_variants, diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 54293977729..7ac064dc6c3 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -63,7 +63,7 @@ build_{{library}}_{{variant}}: - binaries/ {% endfor %} -{% for scenario in scenarios %}{% for variant in weblog_variants %} +{% for variant, scenario in scenario_pairs %} run_{{library}}_{{scenario}}_{{variant}}: extends: - .system_tests_base @@ -101,7 +101,7 @@ run_{{library}}_{{scenario}}_{{variant}}: paths: - system-tests/logs*/reportJunit.xml -{% endfor %}{% endfor %} +{% endfor %} {% if parametric.enable %} {% for job_index in parametric.job_matrix %} run_{{library}}_PARAMETRIC_{{job_index}}: From ad77a461e7634a227927fa0eda3bb1b17ea2ffad Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 17:00:38 +0200 Subject: [PATCH 146/229] gitlab: fix undefined needs for weblogs that skip the build step --- utils/ci/gitlab/build_pipeline.py | 6 +++++- utils/ci/gitlab/system-tests.yml | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index bce320e07e4..564d4c4b7d6 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -24,7 +24,11 @@ # Build variants to compile are those requiring build weblog_variants = [w["name"] for w in parallel_weblogs] # Flatten scenario/job pairs for run jobs -scenario_pairs = [(job["weblog"], scenario) for job in parallel_jobs for scenario in job.get("scenarios", [])] +scenario_pairs = [ + (job["weblog"], scenario, job.get("weblog_build_required", True)) + for job in parallel_jobs + for scenario in job.get("scenarios", []) +] binaries_artifact = params["miscs"]["binaries_artifact"] ci_environment = params["miscs"]["ci_environment"] parametric = params["parametric"] diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 7ac064dc6c3..dff5be4a4ba 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -63,7 +63,7 @@ build_{{library}}_{{variant}}: - binaries/ {% endfor %} -{% for variant, scenario in scenario_pairs %} +{% for variant, scenario, build_required in scenario_pairs %} run_{{library}}_{{scenario}}_{{variant}}: extends: - .system_tests_base @@ -71,8 +71,13 @@ run_{{library}}_{{scenario}}_{{variant}}: - .push_to_test_optimization {% endif %} needs: + {% if build_required %} - job: build_{{library}}_{{variant}} artifacts: true + {% elif binaries_artifact %} + - job: {{binaries_artifact}} + artifacts: true + {% endif %} script: - section_start "docker_auth" "Docker hub auth" - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) From 5144f70a3463723c04967596f1d3dd53748a6261 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 17:09:18 +0200 Subject: [PATCH 147/229] gitlab: skip binaries mv and run build for weblogs that don't require a build step --- utils/ci/gitlab/system-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index dff5be4a4ba..dd93b11f87f 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -91,8 +91,10 @@ run_{{library}}_{{scenario}}_{{variant}}: done - section_end "docker_auth" - section_start "weblog_setup" "Setting up the weblog" + {% if build_required %} - mv ../binaries/* binaries/ - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} + {% endif %} - section_end "weblog_setup" - section_start "scenario_run" "Running scenario" {% if scenario in ("INTEGRATION_FRAMEWORKS") %} From 6c03b86971faa71a6a539f473fddce88068a7377 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 17:46:40 +0200 Subject: [PATCH 148/229] Fix --- .gitlab-ci.yml | 2 +- utils/ci/gitlab/system-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d5f352e6eea..3527e3e9def 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,7 @@ include: - local: /utils/ci/gitlab/main.yml inputs: stage: "build" - excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL" + excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true stages: diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index dd93b11f87f..11f7571b89e 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -97,7 +97,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% endif %} - section_end "weblog_setup" - section_start "scenario_run" "Running scenario" - {% if scenario in ("INTEGRATION_FRAMEWORKS") %} + {% if scenario in ("INTEGRATION_FRAMEWORKS", "GO_PROXIES_DEFAULT") %} - ./run.sh {{scenario}} -L {{library}} --weblog {{variant}} {% else %} - ./run.sh {{scenario}} From 1b22d291a32ce67a23109b5094ba11a1315ca25f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 8 Jun 2026 17:55:43 +0200 Subject: [PATCH 149/229] fix --- utils/ci/gitlab/system-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 11f7571b89e..cf4dde16980 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -97,8 +97,10 @@ run_{{library}}_{{scenario}}_{{variant}}: {% endif %} - section_end "weblog_setup" - section_start "scenario_run" "Running scenario" - {% if scenario in ("INTEGRATION_FRAMEWORKS", "GO_PROXIES_DEFAULT") %} + {% if scenario in ("INTEGRATION_FRAMEWORKS") %} - ./run.sh {{scenario}} -L {{library}} --weblog {{variant}} + {% elif scneario in ("GO_PROXIES_DEFAULT", "GO_PROXIES_APPSEC_BLOCKING")} + - ./run.sh {{scenario}} --weblog {{variant}} {% else %} - ./run.sh {{scenario}} {% endif %} From 6bd1c7251661335ca06e673c9c0dbb8bd748fdab Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 9 Jun 2026 09:35:47 +0200 Subject: [PATCH 150/229] gitlab: add zstd to ci-runner image --- utils/ci/gitlab/docker/system-tests.Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/ci/gitlab/docker/system-tests.Dockerfile b/utils/ci/gitlab/docker/system-tests.Dockerfile index 6c0378d3ed7..149e9c268ae 100644 --- a/utils/ci/gitlab/docker/system-tests.Dockerfile +++ b/utils/ci/gitlab/docker/system-tests.Dockerfile @@ -27,6 +27,7 @@ RUN add-apt-repository ppa:deadsnakes/ppa -y RUN clean-apt install \ jq \ + zstd \ ca-certificates \ curl \ git \ From bcbdbb611d710e940e3f555d977639a2c0ca3ab9 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 9 Jun 2026 09:37:18 +0200 Subject: [PATCH 151/229] fix --- utils/ci/gitlab/system-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index cf4dde16980..2a19092f4cd 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -99,7 +99,7 @@ run_{{library}}_{{scenario}}_{{variant}}: - section_start "scenario_run" "Running scenario" {% if scenario in ("INTEGRATION_FRAMEWORKS") %} - ./run.sh {{scenario}} -L {{library}} --weblog {{variant}} - {% elif scneario in ("GO_PROXIES_DEFAULT", "GO_PROXIES_APPSEC_BLOCKING")} + {% elif scenario in ("GO_PROXIES_DEFAULT", "GO_PROXIES_APPSEC_BLOCKING") %} - ./run.sh {{scenario}} --weblog {{variant}} {% else %} - ./run.sh {{scenario}} From db736a7155b286572427cd4cc55a64bd2012d224 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 9 Jun 2026 14:42:42 +0200 Subject: [PATCH 152/229] Export more logs --- utils/ci/gitlab/system-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 2a19092f4cd..2b0a2915d29 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -108,7 +108,7 @@ run_{{library}}_{{scenario}}_{{variant}}: artifacts: when: always paths: - - system-tests/logs*/reportJunit.xml + - system-tests/logs* {% endfor %} {% if parametric.enable %} @@ -131,7 +131,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: artifacts: when: always paths: - - system-tests/logs*/reportJunit.xml + - system-tests/logs* {% endfor %} {% endif %} From 4bd9f23e08693de1e183ef50b81ee987fe354818 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 9 Jun 2026 17:10:17 +0200 Subject: [PATCH 153/229] gitlab: capture network/DNS diagnostics for weblog -> postgres --- utils/ci/gitlab/system-tests.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 2b0a2915d29..8f3ed5bf6c2 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -104,6 +104,14 @@ run_{{library}}_{{scenario}}_{{variant}}: {% else %} - ./run.sh {{scenario}} {% endif %} + - docker version || true + - docker info || true + - docker network ls || true + - docker network inspect system-tests-ipv4 || true + - docker ps -a || true + - docker inspect system-tests-postgres || true + - docker inspect system-tests-weblog || true + - docker exec system-tests-weblog sh -c 'cat /etc/resolv.conf; echo ---; getent hosts postgres; getent hosts agent; getent hosts proxy; getent hosts weblog' || true - section_end "scenario_run" artifacts: when: always From 3b3a2a46630dbfe5b55f1c604bb15f69fd464a92 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Tue, 9 Jun 2026 17:41:22 +0200 Subject: [PATCH 154/229] gitlab: move weblog/postgres diagnostics to after_script + log file --- utils/ci/gitlab/system-tests.yml | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 8f3ed5bf6c2..3ebf70e8d72 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -16,6 +16,23 @@ variables: DD_STS_OIDC_TOKEN: aud: dd-sts after_script: + - | + ( + set +e + cd "$CI_PROJECT_DIR/system-tests" 2>/dev/null || cd system-tests 2>/dev/null || true + mkdir -p logs + { + echo "===== docker version ====="; docker version + echo "===== docker info ====="; docker info + echo "===== docker network ls ====="; docker network ls + echo "===== docker network inspect system-tests-ipv4 ====="; docker network inspect system-tests-ipv4 + echo "===== docker ps -a ====="; docker ps -a + echo "===== docker inspect system-tests-postgres ====="; docker inspect system-tests-postgres + echo "===== docker inspect system-tests-weblog ====="; docker inspect system-tests-weblog + echo "===== weblog /etc/resolv.conf and DNS lookups =====" + docker exec system-tests-weblog sh -c 'cat /etc/resolv.conf; echo ---; getent hosts postgres; getent hosts agent; getent hosts proxy; getent hosts weblog' + } > logs/network-diagnostics.txt 2>&1 + ) || true - echo -e "section_start:$(date +%s):test_optim[collapsed=true]\r\e[0KPush to tests optimization" - dd-sts debug - > @@ -104,14 +121,6 @@ run_{{library}}_{{scenario}}_{{variant}}: {% else %} - ./run.sh {{scenario}} {% endif %} - - docker version || true - - docker info || true - - docker network ls || true - - docker network inspect system-tests-ipv4 || true - - docker ps -a || true - - docker inspect system-tests-postgres || true - - docker inspect system-tests-weblog || true - - docker exec system-tests-weblog sh -c 'cat /etc/resolv.conf; echo ---; getent hosts postgres; getent hosts agent; getent hosts proxy; getent hosts weblog' || true - section_end "scenario_run" artifacts: when: always From bb45169b8060c4e7ba229135300d551e1614f327 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 13:46:41 +0200 Subject: [PATCH 155/229] gitlab: filter LIBRARIES via E2E_SUPPORTED_LANGUAGES in system-tests-param --- .gitlab-ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3527e3e9def..4eaf4ea3bd2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -347,6 +347,26 @@ system-tests-param: elif key == 'scenarios_groups': print(f'SCENARIO_GROUPS={val}') " > param.env - cat param.env + # TEMPORARY: filter LIBRARIES via E2E_SUPPORTED_LANGUAGES (space-separated allowlist) + # used to control rollout, to be removed once rollout is complete. + - | + if [ -n "${E2E_SUPPORTED_LANGUAGES:-}" ]; then + echo "Filtering LIBRARIES with E2E_SUPPORTED_LANGUAGES='$E2E_SUPPORTED_LANGUAGES'" + current=$(grep '^LIBRARIES=' param.env | sed 's/^LIBRARIES=//') + filtered="" + for lib in $current; do + for allowed in $E2E_SUPPORTED_LANGUAGES; do + if [ "$lib" = "$allowed" ]; then + filtered="${filtered:+$filtered }$lib" + break + fi + done + done + grep -v '^LIBRARIES=' param.env > param.env.tmp || true + echo "LIBRARIES=$filtered" >> param.env.tmp + mv param.env.tmp param.env + cat param.env + fi artifacts: reports: dotenv: param.env From 99e93c0575bbd03767fbfe006377510eb49215a5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 13:47:20 +0200 Subject: [PATCH 156/229] gitlab: set E2E_SUPPORTED_LANGUAGES to python --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4eaf4ea3bd2..b3daeb4b2da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,8 @@ stages: variables: TEST: 1 SKIP_RESOLVE_CI_IMAGE: "true" + # TEMPORARY: rollout-control allowlist for system-tests-param LIBRARIES filter. + E2E_SUPPORTED_LANGUAGES: "python" # compute_pipeline: # image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 From 882bc64567e97c2094f214be6cbc96c23669e5bf Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 14:23:03 +0200 Subject: [PATCH 157/229] gitlab: remove unused generate_wrapper_pipeline.py --- .../generate_wrapper_pipeline.py | 47 ------------------- 1 file changed, 47 deletions(-) delete mode 100644 utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py diff --git a/utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py b/utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py deleted file mode 100644 index 2efd94dba36..00000000000 --- a/utils/scripts/ci_orchestrators/generate_wrapper_pipeline.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env python3 -"""Generate a GitLab CI wrapper pipeline that includes main.yml with computed inputs.""" - -import argparse - - -def main() -> None: - parser = argparse.ArgumentParser(description="Generate a GitLab CI wrapper pipeline YAML") - parser.add_argument("--libraries", required=True, help="Space-separated list of libraries") - parser.add_argument("--scenarios", default="", help="Comma-separated list of scenarios") - parser.add_argument("--scenarios-groups", default="", help="Comma-separated list of scenario groups") - parser.add_argument( - "--excluded-scenarios", - default="", - help="Comma-separated list of scenarios to exclude", - ) - parser.add_argument("--ref", required=True, help="system-tests ref (branch, tag or SHA)") - parser.add_argument( - "--push-to-test-optimization", - default="false", - choices=["true", "false"], - help="Push results to Datadog Test Optimization", - ) - parser.add_argument("-o", "--output", required=True, help="Output file path") - args = parser.parse_args() - - content = f"""include: - - local: /utils/ci/gitlab/main.yml - inputs: - stage: build - libraries: "{args.libraries}" - scenarios: "{args.scenarios}" - scenarios_groups: "{args.scenarios_groups}" - excluded_scenarios: "{args.excluded_scenarios}" - ref: "{args.ref}" - push_to_test_optimization: {args.push_to_test_optimization} - -stages: - - build -""" - - with open(args.output, "w", encoding="utf-8") as f: - f.write(content) - - -if __name__ == "__main__": - main() From 9ab3f8d6a683b24fd460cae3a65882e6f3c7336a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 14:23:45 +0200 Subject: [PATCH 158/229] gitlab: remove unused ci_environment template variable --- utils/ci/gitlab/build_pipeline.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 564d4c4b7d6..51897f20328 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -11,7 +11,6 @@ parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") -parser.add_argument("--test-optimization-datadog-site", default="datadoghq.com", help="Datadog site for Test Optimization") parser.add_argument("--skip-header", action="store_true", help="Skip the workflow/include/variables header (for concatenation)") args = parser.parse_args() @@ -30,7 +29,6 @@ for scenario in job.get("scenarios", []) ] binaries_artifact = params["miscs"]["binaries_artifact"] -ci_environment = params["miscs"]["ci_environment"] parametric = params["parametric"] env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) @@ -45,11 +43,9 @@ library=args.library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact, - ci_environment=ci_environment, parametric=parametric, ci_image=args.ci_image, ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", - test_optimization_datadog_site=args.test_optimization_datadog_site, skip_header=args.skip_header, ), end="") From 325aa4b6a9df1a8f17f9abea516db85ab0826b26 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 14:24:35 +0200 Subject: [PATCH 159/229] gitlab: remove unused extra_gitlab_output function --- utils/scripts/compute_libraries_and_scenarios.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/utils/scripts/compute_libraries_and_scenarios.py b/utils/scripts/compute_libraries_and_scenarios.py index 9c7413a8b4b..b0690629ac7 100644 --- a/utils/scripts/compute_libraries_and_scenarios.py +++ b/utils/scripts/compute_libraries_and_scenarios.py @@ -347,10 +347,6 @@ def load_scenario_mappings(self) -> None: self.scenario_map = json.load(f) -def extra_gitlab_output(inputs: Inputs) -> dict[str, str]: - return {"CI_PIPELINE_SOURCE": inputs.event_name, "CI_COMMIT_REF_NAME": inputs.ref} - - def stringify_outputs(outputs: dict[str, Any]) -> list[str]: ret = [] for name, value in outputs.items(): @@ -373,7 +369,6 @@ def print_ci_outputs(strings_out: list[str], f: Any) -> None: # noqa: ANN401 def process(inputs: Inputs) -> list[str]: outputs: dict[str, Any] = {} if inputs.is_gitlab: - outputs |= extra_gitlab_output(inputs) logging.disable() rebuild_lambda_proxy = False From 3bb38daba5be3d348484d0c3331b15b4d1154b54 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 14:25:08 +0200 Subject: [PATCH 160/229] gitlab: remove unused id_tokens from build_test_pipeline --- utils/ci/gitlab/main.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 73e73e2953a..4d2ea1b60f6 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -116,15 +116,12 @@ build_test_pipeline: - job: system-tests-param optional: true artifacts: true - id_tokens: - DD_STS_OIDC_TOKEN: - aud: dd-sts script: - | libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') - weblogs=$(echo "$[[ inputs.weblogs ]],$WEBLOGS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') + weblogs=$(echo "$[[ inputs.weblogs ]]" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') echo "libraries: $libraries" echo "scenarios: $scenarios" echo "scenario_groups: $scenario_groups" From edd174cf59c70fc3f2faf9dc7b6aef35d0b3c668 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 14:25:33 +0200 Subject: [PATCH 161/229] gitlab: remove unused test_optimization_datadog_site and EXCLUDED_SCENARIOS --- utils/ci/gitlab/main.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 4d2ea1b60f6..3303e507ed7 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -34,9 +34,6 @@ spec: description: "Push test results to Datadog Test Optimization" type: boolean default: false - test_optimization_datadog_site: - description: "Datadog site to use for Test Optimization" - default: "datadoghq.com" auto_cancel_on_new_commit: description: "Auto-cancel strategy when a new commit is pushed (interruptible, conservative, disabled)" default: "interruptible" @@ -134,11 +131,11 @@ build_test_pipeline: for library in $libraries; do chunk=$(( idx / third )) [ $chunk -gt 2 ] && chunk=2 - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json if [ ! -f generated-pipeline-chunk-${chunk}.yml ]; then - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" > generated-pipeline-chunk-${chunk}.yml + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" > generated-pipeline-chunk-${chunk}.yml else - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --test-optimization-datadog-site "$[[ inputs.test_optimization_datadog_site ]]" --skip-header >> generated-pipeline-chunk-${chunk}.yml + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --skip-header >> generated-pipeline-chunk-${chunk}.yml fi idx=$((idx + 1)) done From dfbbd75139cfc3eb2830b44f6f408a473174373a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 14:26:30 +0200 Subject: [PATCH 162/229] gitlab: factor docker-auth retry block into Jinja2 macro --- utils/ci/gitlab/system-tests.yml | 37 +++++++++++++------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 3ebf70e8d72..1b21c7bad0a 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -1,3 +1,16 @@ +{% macro docker_auth() %} + - section_start "docker_auth" "Docker hub auth" + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - | + for i in 1 2 3; do + echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin && break + if [ "$i" -eq 3 ]; then echo "docker login failed after 3 attempts"; exit 1; fi + echo "docker login failed (attempt $i), retrying in $((i*5))s..." + sleep $((i*5)) + done + - section_end "docker_auth" +{% endmacro %} {% if not skip_header %} workflow: name: "System-tests end to end" @@ -60,17 +73,7 @@ build_{{library}}_{{variant}}: artifacts: true {% endif %} script: - - section_start "docker_auth" "Docker hub auth" - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - | - for i in 1 2 3; do - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin && break - if [ "$i" -eq 3 ]; then echo "docker login failed after 3 attempts"; exit 1; fi - echo "docker login failed (attempt $i), retrying in $((i*5))s..." - sleep $((i*5)) - done - - section_end "docker_auth" + {{ docker_auth() }} - section_start "build" "Building weblog" - ./build.sh {{library}} -i weblog --save-to-binaries --weblog-variant {{variant}} - mv binaries .. @@ -96,17 +99,7 @@ run_{{library}}_{{scenario}}_{{variant}}: artifacts: true {% endif %} script: - - section_start "docker_auth" "Docker hub auth" - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - - | - for i in 1 2 3; do - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin && break - if [ "$i" -eq 3 ]; then echo "docker login failed after 3 attempts"; exit 1; fi - echo "docker login failed (attempt $i), retrying in $((i*5))s..." - sleep $((i*5)) - done - - section_end "docker_auth" + {{ docker_auth() }} - section_start "weblog_setup" "Setting up the weblog" {% if build_required %} - mv ../binaries/* binaries/ From 522655b9308ce9bf8b53ab18a9843183813a1830 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 15:15:45 +0200 Subject: [PATCH 163/229] format --- utils/ci/__init__.py | 0 utils/ci/gitlab/__init__.py | 0 utils/ci/gitlab/build_pipeline.py | 33 ++++++++++++++++++------------- utils/ci/gitlab/section.sh | 5 +++-- 4 files changed, 22 insertions(+), 16 deletions(-) create mode 100644 utils/ci/__init__.py create mode 100644 utils/ci/gitlab/__init__.py diff --git a/utils/ci/__init__.py b/utils/ci/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/utils/ci/gitlab/__init__.py b/utils/ci/gitlab/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 51897f20328..9e5ef408408 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -11,7 +11,9 @@ parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") -parser.add_argument("--skip-header", action="store_true", help="Skip the workflow/include/variables header (for concatenation)") +parser.add_argument( + "--skip-header", action="store_true", help="Skip the workflow/include/variables header (for concatenation)" +) args = parser.parse_args() @@ -36,16 +38,19 @@ template = env.get_template("system-tests.yml") # Render the pipeline with per-weblog scenario pairs -print(template.render( - # we're now using precomputed scenario pairs instead of flat scenarios list - scenario_pairs=scenario_pairs, - stage=args.stage, - library=args.library, - weblog_variants=weblog_variants, - binaries_artifact=binaries_artifact, - parametric=parametric, - ci_image=args.ci_image, - ref=args.ref, - push_to_test_optimization=args.push_to_test_optimization == "true", - skip_header=args.skip_header, -), end="") +print( # noqa: T201 + template.render( + # we're now using precomputed scenario pairs instead of flat scenarios list + scenario_pairs=scenario_pairs, + stage=args.stage, + library=args.library, + weblog_variants=weblog_variants, + binaries_artifact=binaries_artifact, + parametric=parametric, + ci_image=args.ci_image, + ref=args.ref, + push_to_test_optimization=args.push_to_test_optimization == "true", + skip_header=args.skip_header, + ), + end="", +) diff --git a/utils/ci/gitlab/section.sh b/utils/ci/gitlab/section.sh index 20fa1f00049..4bbcd09834f 100644 --- a/utils/ci/gitlab/section.sh +++ b/utils/ci/gitlab/section.sh @@ -1,14 +1,15 @@ +# shellcheck shell=bash # function for starting the section function section_start () { local section_title="${1}" local section_description="${2:-$section_title}" - echo -e "section_start:`date +%s`:${section_title}[collapsed=true]\r\e[0K${section_description}" + echo -e "section_start:$(date +%s):${section_title}[collapsed=true]\r\e[0K${section_description}" } # Function for ending the section function section_end () { local section_title="${1}" - echo -e "section_end:`date +%s`:${section_title}\r\e[0K" + echo -e "section_end:$(date +%s):${section_title}\r\e[0K" } From 7923ccdbf2a1683edfcf3ac7f7592cf3562a608a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 15:18:26 +0200 Subject: [PATCH 164/229] cleanup --- .gitlab-ci.yml | 578 +++++++++++++++---------------- docs/CI/gitlab-pipeline-merge.md | 250 ------------- 2 files changed, 289 insertions(+), 539 deletions(-) delete mode 100644 docs/CI/gitlab-pipeline-merge.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b3daeb4b2da..253e5ef2078 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,295 +24,295 @@ variables: # TEMPORARY: rollout-control allowlist for system-tests-param LIBRARIES filter. E2E_SUPPORTED_LANGUAGES: "python" -# compute_pipeline: -# image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 -# tags: ["arch:amd64"] -# stage: configure -# variables: -# CI_ENVIRONMENT: "prod" -# script: -# - | -# if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then -# echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined.Computing changes..." -# SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner -# source venv/bin/activate -# ./run.sh MOCK_THE_TEST --collect-only --scenario-report -# git clone https://github.com/DataDog/system-tests.git original -# git fetch --all # Ensure all branches are available -# git branch --track $CI_COMMIT_REF_NAME origin/$CI_COMMIT_REF_NAME || true # Track branch if not tracked -# git checkout $CI_COMMIT_REF_NAME # Ensure the branch is checked out -# BASE_COMMIT=$(git merge-base origin/main $CI_COMMIT_REF_NAME) # Get the base commit -# echo "Branch was created from commit--> $BASE_COMMIT" -# git diff --name-only $BASE_COMMIT $CI_COMMIT_SHA >> modified_files.txt # List modified files -# cat modified_files.txt -# python utils/scripts/compute_libraries_and_scenarios.py >> impacted_scenarios.txt -# cat impacted_scenarios.txt -# source impacted_scenarios.txt -# else -# echo "✅ SYSTEM_TESTS_SCENARIOS OR SYSTEM_TESTS_SCENARIOS_GROUPS are set." -# export scenarios=$SYSTEM_TESTS_SCENARIOS -# export scenarios_groups=$SYSTEM_TESTS_SCENARIOS_GROUPS -# cd /system-tests -# git pull -# if [ -n "$SYSTEM_TESTS_REF" ]; then -# git checkout $SYSTEM_TESTS_REF -# else -# echo "⚠️ SYSTEM_TESTS_REF variable is not set, skipping git checkout" -# fi -# SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner -# source venv/bin/activate -# fi -# - python utils/scripts/compute-workflow-parameters.py nodejs -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab -# - python utils/scripts/compute-workflow-parameters.py java -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab -# - python utils/scripts/compute-workflow-parameters.py dotnet -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab -# - python utils/scripts/compute-workflow-parameters.py python -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab -# - python utils/scripts/compute-workflow-parameters.py php -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab -# - python utils/scripts/compute-workflow-parameters.py ruby -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab -# - | -# if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then -# cp *.yml "$CI_PROJECT_DIR" -# fi -# rules: -# - if: '$SCHEDULED_JOB == ""' -# - if: '$SCHEDULED_JOB == null' -# artifacts: -# paths: -# - nodejs_ssi_gitlab_pipeline.yml -# - java_ssi_gitlab_pipeline.yml -# - dotnet_ssi_gitlab_pipeline.yml -# - python_ssi_gitlab_pipeline.yml -# - php_ssi_gitlab_pipeline.yml -# - ruby_ssi_gitlab_pipeline.yml -# -# nodejs_ssi_pipeline: -# stage: nodejs -# needs: ["compute_pipeline"] -# variables: -# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE -# trigger: -# include: -# - artifact: nodejs_ssi_gitlab_pipeline.yml -# job: compute_pipeline -# strategy: depend -# rules: -# - if: '$SCHEDULED_JOB == ""' -# - if: '$SCHEDULED_JOB == null' -# when: always -# -# java_ssi_pipeline: -# stage: java -# needs: ["compute_pipeline", "nodejs_ssi_pipeline"] -# variables: -# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE -# trigger: -# include: -# - artifact: java_ssi_gitlab_pipeline.yml -# job: compute_pipeline -# strategy: depend -# rules: -# - if: '$SCHEDULED_JOB == ""' -# - if: '$SCHEDULED_JOB == null' -# when: always -# -# dotnet_ssi_pipeline: -# stage: dotnet -# needs: ["compute_pipeline", "java_ssi_pipeline"] -# variables: -# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE -# trigger: -# include: -# - artifact: dotnet_ssi_gitlab_pipeline.yml -# job: compute_pipeline -# strategy: depend -# rules: -# - if: '$SCHEDULED_JOB == ""' -# - if: '$SCHEDULED_JOB == null' -# when: always -# -# python_ssi_pipeline: -# stage: python -# needs: ["compute_pipeline", "dotnet_ssi_pipeline"] -# variables: -# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE -# trigger: -# include: -# - artifact: python_ssi_gitlab_pipeline.yml -# job: compute_pipeline -# strategy: depend -# rules: -# - if: '$SCHEDULED_JOB == ""' -# - if: '$SCHEDULED_JOB == null' -# when: always -# -# php_ssi_pipeline: -# stage: php -# needs: ["compute_pipeline", "python_ssi_pipeline"] -# variables: -# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE -# trigger: -# include: -# - artifact: php_ssi_gitlab_pipeline.yml -# job: compute_pipeline -# strategy: depend -# rules: -# - if: '$SCHEDULED_JOB == ""' -# - if: '$SCHEDULED_JOB == null' -# when: always -# -# ruby_ssi_pipeline: -# stage: ruby -# needs: ["compute_pipeline", "php_ssi_pipeline"] -# variables: -# PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE -# trigger: -# include: -# - artifact: ruby_ssi_gitlab_pipeline.yml -# job: compute_pipeline -# strategy: depend -# rules: -# - if: '$SCHEDULED_JOB == ""' -# - if: '$SCHEDULED_JOB == null' -# when: always -# -# delete_amis: -# extends: .base_job_onboarding -# stage: system-tests-utils -# allow_failure: false -# variables: -# AMI_RETENTION_DAYS: 10 -# AMI_LAST_LAUNCHED_DAYS: 10 -# script: -# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner -# - source venv/bin/activate -# - python utils/scripts/pulumi_clean_up.py --component amis --ami-retention-days $AMI_RETENTION_DAYS --ami-last-launched-days $AMI_LAST_LAUNCHED_DAYS -# rules: -# - if: '$SCHEDULED_JOB == "delete_amis"' -# after_script: echo "Finish" -# timeout: 3h -# -# delete_amis_by_name_or_lang: -# extends: .base_job_onboarding -# stage: system-tests-utils -# allow_failure: false -# script: -# - | -# if [ -z "$AMI_NAME" ] && [ -z "$AMI_LANG" ]; then -# echo "❌ ERROR: Either AMI_NAME or AMI_LANG must be set." -# exit 1 -# fi -# echo "✅ Proceeding with AMI_NAME=$AMI_NAME, AMI_LANG=$AMI_LANG" -# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner -# - source venv/bin/activate -# - | -# CMD="python utils/scripts/pulumi_clean_up.py --component amis_by_name" -# -# if [ -n "$AMI_NAME" ]; then -# CMD="$CMD --ami-name $AMI_NAME" -# fi -# -# if [ -n "$AMI_LANG" ]; then -# CMD="$CMD --ami-lang $AMI_LANG" -# fi -# -# echo "Running: $CMD" -# eval $CMD -# rules: -# - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' -# after_script: echo "Finish" -# -# delete_ec2_instances: -# extends: .base_job_onboarding -# stage: system-tests-utils -# allow_failure: false -# variables: -# EC2_AGE_MINUTES: 45 -# script: -# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner -# - source venv/bin/activate -# - python utils/scripts/pulumi_clean_up.py --component ec2 --ec2-age-minutes $EC2_AGE_MINUTES -# rules: -# - if: '$SCHEDULED_JOB == "delete_ec2_instances"' -# after_script: echo "Finish" -# -# count_amis: -# extends: .base_job_onboarding -# stage: system-tests-utils -# allow_failure: false -# script: -# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner -# - source venv/bin/activate -# - python utils/scripts/pulumi_clean_up.py --component amis_count -# rules: -# - if: '$SCHEDULED_JOB == "count_amis"' -# after_script: echo "Finish" -# -# check_merge_labels: -# #Build docker images if it's needed. Check if the PR has the labels associated with the image build. -# image: registry.ddbuild.io/system-tests/base-image-builder -# tags: -# - arch:amd64 -# id_tokens: -# DDOCTOSTS_ID_TOKEN: -# aud: dd-octo-sts -# needs: [] -# stage: system-tests-utils -# before_script: -# # Get a token -# - dd-octo-sts version -# - dd-octo-sts debug --scope DataDog/system-tests --policy self.gitlab-read -# - dd-octo-sts token --scope DataDog/system-tests --policy self.gitlab-read > token.txt -# - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) -# - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) -# script: -# - export GITHUB_TOKEN=$(cat token.txt) -# - ./utils/scripts/get_pr_merged_labels.sh -# after_script: -# # Revoke the token after usage -# - dd-octo-sts revoke -t $(cat token.txt) -# rules: -# - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" -# -# generate_system_tests_lambda_proxy_image: -# image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 -# tags: ["docker-in-docker:amd64"] -# needs: [] -# stage: system-tests-utils -# allow_failure: false -# before_script: -# - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) -# - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) -# - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin -# script: -# - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i lambda-proxy -# - docker push datadog/system-tests:lambda-proxy-v1 -# rules: -# - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" -# changes: -# - utils/build/docker/lambda_proxy/pyproject.toml -# - utils/build/docker/lambda-proxy.Dockerfile -# -# generate_system_tests_lib_injection_images: -# extends: .base_job_k8s_docker_ssi -# needs: [] -# stage: system-tests-utils -# allow_failure: true -# variables: -# PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com -# PRIVATE_DOCKER_REGISTRY_USER: AWS -# script: -# - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} -# - ./lib-injection/build/build_lib_injection_images.sh $PRIVATE_DOCKER_REGISTRY -# rules: -# - if: '$SCHEDULED_JOB == "generate_system_tests_lib_injection_images"' -# - when: manual -# -# .delayed_base_job: -# image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 -# tags: ["arch:amd64"] -# script: -# - echo "⏳ Waiting before triggering the child pipeline..." -# when: delayed -# start_in: 5 minutes +compute_pipeline: + image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 + tags: ["arch:amd64"] + stage: configure + variables: + CI_ENVIRONMENT: "prod" + script: + - | + if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then + echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined.Computing changes..." + SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + source venv/bin/activate + ./run.sh MOCK_THE_TEST --collect-only --scenario-report + git clone https://github.com/DataDog/system-tests.git original + git fetch --all # Ensure all branches are available + git branch --track $CI_COMMIT_REF_NAME origin/$CI_COMMIT_REF_NAME || true # Track branch if not tracked + git checkout $CI_COMMIT_REF_NAME # Ensure the branch is checked out + BASE_COMMIT=$(git merge-base origin/main $CI_COMMIT_REF_NAME) # Get the base commit + echo "Branch was created from commit--> $BASE_COMMIT" + git diff --name-only $BASE_COMMIT $CI_COMMIT_SHA >> modified_files.txt # List modified files + cat modified_files.txt + python utils/scripts/compute_libraries_and_scenarios.py >> impacted_scenarios.txt + cat impacted_scenarios.txt + source impacted_scenarios.txt + else + echo "✅ SYSTEM_TESTS_SCENARIOS OR SYSTEM_TESTS_SCENARIOS_GROUPS are set." + export scenarios=$SYSTEM_TESTS_SCENARIOS + export scenarios_groups=$SYSTEM_TESTS_SCENARIOS_GROUPS + cd /system-tests + git pull + if [ -n "$SYSTEM_TESTS_REF" ]; then + git checkout $SYSTEM_TESTS_REF + else + echo "⚠️ SYSTEM_TESTS_REF variable is not set, skipping git checkout" + fi + SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + source venv/bin/activate + fi + - python utils/scripts/compute-workflow-parameters.py nodejs -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py java -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py dotnet -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py python -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py php -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - python utils/scripts/compute-workflow-parameters.py ruby -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab + - | + if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then + cp *.yml "$CI_PROJECT_DIR" + fi + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + artifacts: + paths: + - nodejs_ssi_gitlab_pipeline.yml + - java_ssi_gitlab_pipeline.yml + - dotnet_ssi_gitlab_pipeline.yml + - python_ssi_gitlab_pipeline.yml + - php_ssi_gitlab_pipeline.yml + - ruby_ssi_gitlab_pipeline.yml + +nodejs_ssi_pipeline: + stage: nodejs + needs: ["compute_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: nodejs_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +java_ssi_pipeline: + stage: java + needs: ["compute_pipeline", "nodejs_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: java_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +dotnet_ssi_pipeline: + stage: dotnet + needs: ["compute_pipeline", "java_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: dotnet_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +python_ssi_pipeline: + stage: python + needs: ["compute_pipeline", "dotnet_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: python_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +php_ssi_pipeline: + stage: php + needs: ["compute_pipeline", "python_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: php_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +ruby_ssi_pipeline: + stage: ruby + needs: ["compute_pipeline", "php_ssi_pipeline"] + variables: + PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE + trigger: + include: + - artifact: ruby_ssi_gitlab_pipeline.yml + job: compute_pipeline + strategy: depend + rules: + - if: '$SCHEDULED_JOB == ""' + - if: '$SCHEDULED_JOB == null' + when: always + +delete_amis: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + variables: + AMI_RETENTION_DAYS: 10 + AMI_LAST_LAUNCHED_DAYS: 10 + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - python utils/scripts/pulumi_clean_up.py --component amis --ami-retention-days $AMI_RETENTION_DAYS --ami-last-launched-days $AMI_LAST_LAUNCHED_DAYS + rules: + - if: '$SCHEDULED_JOB == "delete_amis"' + after_script: echo "Finish" + timeout: 3h + +delete_amis_by_name_or_lang: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + script: + - | + if [ -z "$AMI_NAME" ] && [ -z "$AMI_LANG" ]; then + echo "❌ ERROR: Either AMI_NAME or AMI_LANG must be set." + exit 1 + fi + echo "✅ Proceeding with AMI_NAME=$AMI_NAME, AMI_LANG=$AMI_LANG" + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - | + CMD="python utils/scripts/pulumi_clean_up.py --component amis_by_name" + + if [ -n "$AMI_NAME" ]; then + CMD="$CMD --ami-name $AMI_NAME" + fi + + if [ -n "$AMI_LANG" ]; then + CMD="$CMD --ami-lang $AMI_LANG" + fi + + echo "Running: $CMD" + eval $CMD + rules: + - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' + after_script: echo "Finish" + +delete_ec2_instances: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + variables: + EC2_AGE_MINUTES: 45 + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - python utils/scripts/pulumi_clean_up.py --component ec2 --ec2-age-minutes $EC2_AGE_MINUTES + rules: + - if: '$SCHEDULED_JOB == "delete_ec2_instances"' + after_script: echo "Finish" + +count_amis: + extends: .base_job_onboarding + stage: system-tests-utils + allow_failure: false + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner + - source venv/bin/activate + - python utils/scripts/pulumi_clean_up.py --component amis_count + rules: + - if: '$SCHEDULED_JOB == "count_amis"' + after_script: echo "Finish" + +check_merge_labels: + #Build docker images if it's needed. Check if the PR has the labels associated with the image build. + image: registry.ddbuild.io/system-tests/base-image-builder + tags: + - arch:amd64 + id_tokens: + DDOCTOSTS_ID_TOKEN: + aud: dd-octo-sts + needs: [] + stage: system-tests-utils + before_script: + # Get a token + - dd-octo-sts version + - dd-octo-sts debug --scope DataDog/system-tests --policy self.gitlab-read + - dd-octo-sts token --scope DataDog/system-tests --policy self.gitlab-read > token.txt + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + script: + - export GITHUB_TOKEN=$(cat token.txt) + - ./utils/scripts/get_pr_merged_labels.sh + after_script: + # Revoke the token after usage + - dd-octo-sts revoke -t $(cat token.txt) + rules: + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" + +generate_system_tests_lambda_proxy_image: + image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 + tags: ["docker-in-docker:amd64"] + needs: [] + stage: system-tests-utils + allow_failure: false + before_script: + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin + script: + - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i lambda-proxy + - docker push datadog/system-tests:lambda-proxy-v1 + rules: + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" + changes: + - utils/build/docker/lambda_proxy/pyproject.toml + - utils/build/docker/lambda-proxy.Dockerfile + +generate_system_tests_lib_injection_images: + extends: .base_job_k8s_docker_ssi + needs: [] + stage: system-tests-utils + allow_failure: true + variables: + PRIVATE_DOCKER_REGISTRY: 235494822917.dkr.ecr.us-east-1.amazonaws.com + PRIVATE_DOCKER_REGISTRY_USER: AWS + script: + - aws ecr get-login-password | docker login --username ${PRIVATE_DOCKER_REGISTRY_USER} --password-stdin ${PRIVATE_DOCKER_REGISTRY} + - ./lib-injection/build/build_lib_injection_images.sh $PRIVATE_DOCKER_REGISTRY + rules: + - if: '$SCHEDULED_JOB == "generate_system_tests_lib_injection_images"' + - when: manual + +.delayed_base_job: + image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 + tags: ["arch:amd64"] + script: + - echo "⏳ Waiting before triggering the child pipeline..." + when: delayed + start_in: 5 minutes # ────────────────────────────────────────────── # End-to-end pipeline — only for the system-tests repo diff --git a/docs/CI/gitlab-pipeline-merge.md b/docs/CI/gitlab-pipeline-merge.md deleted file mode 100644 index ea8d6e62466..00000000000 --- a/docs/CI/gitlab-pipeline-merge.md +++ /dev/null @@ -1,250 +0,0 @@ -# Merging the new end-to-end GitLab pipeline with the SSI pipeline - -## Context - -There are currently two separate GitLab CI pipelines for system-tests: - -- **SSI pipeline** — the existing `.gitlab-ci.yml` on `main`, used by all tracer repos via - `one-pipeline` (`libdatadog-build/templates/one-pipeline.yml`) -- **End-to-end prototype** — the new `utils/ci/gitlab/main.yml` on this branch, designed to - replace the GitHub Actions end-to-end workflow - -The goal is to merge them into a single, unified GitLab pipeline. - ---- - -## How the SSI pipeline currently works - -### Distribution chain - -``` -libdatadog-build/templates/one-pipeline.yml - └── includes (remote): gitlab-templates.ddbuild.io/libdatadog/include/single-step-instrumentation-tests.yml - (base job templates: .base_job_onboarding, .base_job_k8s_docker_ssi) - -each tracer repo (.gitlab-ci.yml) - └── includes: .gitlab/one-pipeline.locked.yml - └── includes (remote, content-addressed): gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca//one-pipeline.yml - └── defines: configure_system_tests + system_tests jobs -``` - -### configure_system_tests → system_tests flow - -`configure_system_tests` in `one-pipeline.yml`: -1. Pulls system-tests at `$SYSTEM_TESTS_REF` (default: `main`) -2. Builds the runner -3. Calls `utils/scripts/ci_orchestrators/external_gitlab_pipeline.py`, which **reads - `.gitlab-ci.yml` directly** from the cloned system-tests repo, injects SSI image - variables (`K8S_LIB_INIT_IMG`, `DD_INSTALLER_LIBRARY_VERSION`, etc.), filters by - `$SYSTEM_TESTS_LIBRARY`, and dumps the result as `system_tests_pipeline.yml` -4. `system_tests` triggers `system_tests_pipeline.yml` as a child pipeline - -The system-tests `.gitlab-ci.yml` is therefore **already the source of truth** for what -runs in each tracer repo's SSI pipeline. `one-pipeline` just reads and filters it. - -### gitlab-templates.ddbuild.io is an S3 bucket - -Templates are served from `s3://gitlab-templates.ddbuild.io`. There are two publishing paths: - -| Path | URL pattern | When published | -|---|---|---| -| Content-addressed (immutable) | `libdatadog/one-pipeline/ca//` | Automatically on every libdatadog-build pipeline | -| Mutable "latest stable" | `libdatadog/include/` | Manually via `publish-templates-to-versionless-bucket` job | - -Any repo with S3 write access to that bucket can publish its own templates there. The -mutable path is what `one-pipeline.yml` uses to include `single-step-instrumentation-tests.yml`. - ---- - -## Why the SSI pipeline is complex - -There are three scenario categories in the SSI pipeline: - -| Category | Scenarios | Infrastructure | Complexity | -|---|---|---|---| -| `aws_ssi` | `simple_onboarding`, `*_profiling`, `*_appsec` | Real EC2 VMs via Pulumi | High | -| `dockerssi` | `DOCKER_SSI` | Docker-in-Docker | Low (same as e2e) | -| `libinjection` | `K8S_LIB_INJECTION_*` | kind clusters | Moderate | - -**All the complexity in `.gitlab-ci.yml` comes from `aws_ssi` alone:** - -- **Sequential per-language stages** (`nodejs → java → dotnet → ...`) — AWS EC2 and subnet - quotas prevent running all languages in parallel -- **Staggered `.delayed_base_job`** on releases — avoids exhausting AWS capacity at once -- **Pulumi credentials, SSH keys, keypairs** in `before_script` — required for EC2 provisioning -- **`delete_amis`, `delete_ec2_instances` scheduled jobs** — AWS resource cleanup -- **`check_merge_labels`, image build jobs** — infra maintenance - -`DOCKER_SSI` and `K8S_LIB_INJECTION_*` run on the same `docker-in-docker:amd64` runners as -end-to-end tests and have **none** of these constraints. - ---- - -## The new end-to-end prototype - -`utils/ci/gitlab/main.yml` is a GitLab CI component (`spec: inputs:`) with two jobs: - -1. **`build_test_pipeline`** — for each library: runs - `compute-workflow-parameters.py --format json`, pipes through `build_pipeline.py` - (Jinja2) → writes `generated-pipeline-${library}.yml` (one file per library) -2. **`run_test_pipeline_{library}`** — one trigger job per known library, each triggering - its own `generated-pipeline-${library}.yml` as a child pipeline; gated by a rule that - checks whether the library appears in `inputs.libraries` - -`utils/ci/gitlab/system-tests.yml` is the Jinja2 template that produces one flat job per -`(scenario, weblog_variant)` pair: - -``` -run_{library}_{scenario}_{variant} -``` - -The component exposes two inputs: `stage` (for the generated jobs) and `libraries`. -It currently hardcodes `-g all` with a fixed exclusion list and assumes it runs from within -the system-tests repo directory. - ---- - -## Proposed merge strategy - -### Core insight - -`DOCKER_SSI` and `K8S_LIB_INJECTION_*` are structurally identical to end-to-end scenarios: -they run in Docker/K8s on standard CI runners. They can be expressed in the same Jinja2 -template with no special infrastructure. Only `aws_ssi` scenarios need to remain separate. - -### Step 1 — Split the SSI pipeline by infrastructure type - -Separate `.gitlab-ci.yml` into two independent pipelines: - -- **`utils/ci/gitlab/main.yml`** (this file, extended) — handles `endtoend` + `dockerssi` + - `libinjection`. All Docker/K8s, all parallel, no AWS dependencies. -- **`utils/ci/gitlab/aws-ssi.yml`** (new) — handles `aws_ssi` only. Keeps sequential - language stages, Pulumi infra, delayed starts, and cleanup jobs. This pipeline stays - complex because the infrastructure requires it. - -### Step 2 — Extend main.yml inputs - -Add the inputs needed for one-pipeline to drive it: - -```yaml -spec: - inputs: - stage: - build_stage: - default: "build" # stage for build_test_pipeline itself - run_stage: - default: "run" # stage for run_test_pipeline_{library} jobs - libraries: - default: "java python nodejs" - scenarios_groups: - default: "all" # replaces hardcoded "-g all" - excluded_scenarios: - default: "APM_TRACING_E2E_OTEL,..." -``` - -The `build_stage` and `run_stage` inputs allow one-pipeline to map both jobs onto its -single `shared-pipeline` stage. - -### Step 3 — Extend the Jinja2 template for SSI variables - -`system-tests.yml` needs to forward the SSI image variables into generated jobs so that -tracer repos can inject `K8S_LIB_INIT_IMG`, `DD_INSTALLER_LIBRARY_VERSION`, etc.: - -```jinja -run_{{library}}_{{scenario}}_{{variant}}: - variables: - WEBLOG_VARIANT: {{variant}} - {% if ssi_library_version is defined %} - DD_INSTALLER_LIBRARY_VERSION: "{{ssi_library_version}}" - {% endif %} - {% if k8s_lib_init_img is defined %} - K8S_LIB_INIT_IMG: "{{k8s_lib_init_img}}" - {% endif %} -``` - -These values flow from `build_test_pipeline`'s environment into the template at generation -time, so the child pipeline jobs carry them as static variables. - -### Step 4 — Update configure_system_tests in one-pipeline - -Replace the `external_gitlab_pipeline.py` call with the new generation approach: - -```yaml -configure_system_tests: - script: - - *verify-variables - - cd /system-tests - - git pull && git checkout $SYSTEM_TESTS_REF - - ./build.sh -i runner - - source venv/bin/activate && pip install Jinja2 - - | - python utils/scripts/compute-workflow-parameters.py "$SYSTEM_TESTS_LIBRARY" \ - -g "$SYSTEM_TESTS_SCENARIOS_GROUPS" \ - --format json \ - --output params.json - - | - python utils/ci/gitlab/build_pipeline.py \ - --stage test \ - --library "$SYSTEM_TESTS_LIBRARY" \ - --params params.json \ - --ssi-library-version "pipeline-${CI_PIPELINE_ID}" \ - --k8s-lib-init-img "${APM_ECOSYSTEMS_ECR_BASE}/${LIB_INJECTION_NAME}:glci${CI_PIPELINE_ID}" \ - > system_tests_pipeline.yml - - cp system_tests_pipeline.yml $CI_PROJECT_DIR -``` - -This removes `ALLOW_MULTIPLE_CHILD_LEVELS`, `external_gitlab_pipeline.py`, and the -`--format gitlab` code path entirely. - -### Step 5 — Publish main.yml to S3 (optional, for direct include) - -If one-pipeline should include `main.yml` without cloning system-tests, system-tests CI can -push it to the S3 bucket on merge to main: - -```yaml -publish-gitlab-component: - stage: publish - rules: - - if: $CI_COMMIT_BRANCH == "main" - script: - - aws s3 cp utils/ci/gitlab/main.yml s3://gitlab-templates.ddbuild.io/system-tests/include/main.yml - - aws s3 cp utils/ci/gitlab/system-tests.yml s3://gitlab-templates.ddbuild.io/system-tests/include/system-tests.yml - - aws s3 cp utils/ci/gitlab/build_pipeline.py s3://gitlab-templates.ddbuild.io/system-tests/include/build_pipeline.py -``` - -One-pipeline would then include it via: - -```yaml -include: - - remote: https://gitlab-templates.ddbuild.io/system-tests/include/main.yml -``` - -This makes system-tests the owner of its own pipeline distribution, with no git checkout -needed inside `configure_system_tests`. - ---- - -## What stays, what goes - -| Current artifact | Fate | -|---|---| -| `.gitlab-ci.yml` (all languages, all scenario types) | Split: Docker/K8s → `main.yml`; AWS → `aws-ssi.yml` | -| `external_gitlab_pipeline.py` | Deleted — replaced by `build_pipeline.py` + Jinja2 | -| `ALLOW_MULTIPLE_CHILD_LEVELS` variable | Deleted — single code path | -| `compute-workflow-parameters.py --format gitlab` | Deleted — `--format json` + Jinja2 only | -| Sequential language stages in `.gitlab-ci.yml` | Kept in `aws-ssi.yml` only | -| `.delayed_base_job`, delete_amis, delete_ec2_instances | Kept in `aws-ssi.yml` only | -| `single-step-instrumentation-tests.yml` base job templates | Kept in `libdatadog-build`, referenced by `aws-ssi.yml` | - ---- - -## Constraints and caveats - -- **`include: ref:` cannot use CI variables** — GitLab does not expand `$SYSTEM_TESTS_REF` - in `include: project: ref:`. The S3 publish approach (Step 5) sidesteps this entirely since - `remote:` URLs are fetched at pipeline creation using whatever is currently in S3. -- **S3 write access** — system-tests CI needs credentials to write to - `s3://gitlab-templates.ddbuild.io`. This requires coordination with the libdatadog-build - team to grant access or to establish a separate bucket path. -- **`aws_ssi` pipeline is out of scope** — its complexity is infrastructure-driven and cannot - be simplified without changing the underlying AWS/Pulumi approach. From 75c517afe0b921799ddcc510bc09a2f547538fc5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 16:12:50 +0200 Subject: [PATCH 165/229] gitlab: keep CI_PIPELINE_SOURCE/CI_COMMIT_REF_NAME and omit empty libraries Restore the extra_gitlab_output helper that prepends CI_PIPELINE_SOURCE and CI_COMMIT_REF_NAME entries to the gitlab process() output, and only emit the dynamically-computed libraries entry when it is non-empty so the output stays backward compatible with existing consumers. --- utils/scripts/compute_libraries_and_scenarios.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/utils/scripts/compute_libraries_and_scenarios.py b/utils/scripts/compute_libraries_and_scenarios.py index b0690629ac7..cf286d032b7 100644 --- a/utils/scripts/compute_libraries_and_scenarios.py +++ b/utils/scripts/compute_libraries_and_scenarios.py @@ -366,9 +366,14 @@ def print_ci_outputs(strings_out: list[str], f: Any) -> None: # noqa: ANN401 print_ci_outputs(strings_out, sys.stdout) +def extra_gitlab_output(inputs: Inputs) -> dict[str, str]: + return {"CI_PIPELINE_SOURCE": inputs.event_name, "CI_COMMIT_REF_NAME": inputs.ref} + + def process(inputs: Inputs) -> list[str]: outputs: dict[str, Any] = {} if inputs.is_gitlab: + outputs |= extra_gitlab_output(inputs) logging.disable() rebuild_lambda_proxy = False @@ -402,7 +407,9 @@ def process(inputs: Inputs) -> list[str]: library_processor.selected |= scenario_processor.impacted_libraries if inputs.is_gitlab: - outputs["libraries"] = " ".join(sorted(lib for lib in library_processor.selected if lib not in ("rust",))) + libraries = " ".join(sorted(lib for lib in library_processor.selected if lib not in ("rust",))) + if libraries: + outputs["libraries"] = libraries outputs |= scenario_processor.get_outputs() else: outputs |= ( From 86af4795a4eaddd0f533def9071628b82f0073fe Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 15:57:37 +0200 Subject: [PATCH 166/229] Uncommenting new SSI path --- utils/ci/gitlab/main.yml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 3303e507ed7..794d17f9f13 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -40,6 +40,9 @@ spec: ref: description: "system-tests ref to use when called from another repository (branch, tag or SHA)" default: "main" + ssi_enabled: + description: "Whether to run SSI tests" + default: false ssi_library_version: description: "DD_INSTALLER_LIBRARY_VERSION to inject into generated jobs (Docker/K8s SSI)" default: "" @@ -57,15 +60,16 @@ include: inputs: stage: $[[ inputs.stage ]] ref: $[[ inputs.ref ]] - # - local: /utils/ci/gitlab/ssi.yml - # inputs: - # stage: $[[ inputs.stage ]] - # libraries: $[[ inputs.libraries ]] - # scenarios: $[[ inputs.scenarios ]] - # scenarios_groups: $[[ inputs.scenarios_groups ]] - # ssi_library_version: $[[ inputs.ssi_library_version ]] - # k8s_lib_init_img: $[[ inputs.k8s_lib_init_img ]] - # ssi_injector_version: $[[ inputs.ssi_injector_version ]] + - local: /utils/ci/gitlab/ssi.yml + if: $[[ inputs.ssi_enabled ]] + inputs: + stage: $[[ inputs.stage ]] + libraries: $[[ inputs.libraries ]] + scenarios: $[[ inputs.scenarios ]] + scenarios_groups: $[[ inputs.scenarios_groups ]] + ssi_library_version: $[[ inputs.ssi_library_version ]] + k8s_lib_init_img: $[[ inputs.k8s_lib_init_img ]] + ssi_injector_version: $[[ inputs.ssi_injector_version ]] workflow: rules: From c022969609a891ab016830607d7f96009652b136 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 16:29:04 +0200 Subject: [PATCH 167/229] Fix end to end jobs stage --- .gitlab-ci.yml | 8 ++++---- utils/ci/gitlab/system-tests.yml | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 253e5ef2078..5f39684eaf3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,19 +2,19 @@ include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/0b4d88122bb64a87131eeb9446a3671598790917ec30fc31851ccb6ee8375b8a/single-step-instrumentation-tests.yml - local: /utils/ci/gitlab/main.yml inputs: - stage: "build" + stage: "e2e" excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true stages: - configure + - e2e - nodejs - java - dotnet - python - php - ruby - - build - pipeline-status - system-tests-utils @@ -320,7 +320,7 @@ generate_system_tests_lib_injection_images: system-tests-param: image: $CI_IMAGE - stage: build + stage: e2e tags: - arch:amd64 needs: @@ -378,7 +378,7 @@ build_ci_image: interruptible: true tags: - docker-in-docker:amd64 - stage: build + stage: e2e script: - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml index 1b21c7bad0a..f423adf9093 100644 --- a/utils/ci/gitlab/system-tests.yml +++ b/utils/ci/gitlab/system-tests.yml @@ -21,6 +21,9 @@ include: stage: {{stage}} ref: {{ref}} +stages: + - {{stage}} + variables: CI_IMAGE: {{ci_image}} From 809cbcb05b7fd2d46ee0e251fe0fdb85041870b7 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 17:08:17 +0200 Subject: [PATCH 168/229] gitlab: handle pipeline chunking in build_pipeline.py --- utils/ci/gitlab/build_pipeline.py | 94 +++++++++++++++++++++---------- utils/ci/gitlab/main.yml | 39 ++++--------- 2 files changed, 75 insertions(+), 58 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 9e5ef408408..2c8745fd379 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -1,56 +1,88 @@ import argparse -import json +import sys from pathlib import Path +import json + from jinja2 import Environment, FileSystemLoader, select_autoescape parser = argparse.ArgumentParser() parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") -parser.add_argument("--library", required=True, default="", help="Library name, used to prefix job names") -parser.add_argument("--params", required=True, help="Path to JSON output from compute-workflow-parameters.py") +parser.add_argument("--libraries", required=True, help="Space-separated list of library names") +parser.add_argument("--params-dir", required=True, help="Directory containing params_.json files") parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") -parser.add_argument( - "--skip-header", action="store_true", help="Skip the workflow/include/variables header (for concatenation)" -) +parser.add_argument("--output-dir", required=True, help="Directory where generated-pipeline-chunk-.yml files are written") +parser.add_argument("--chunks", type=int, default=3, help="Number of pipeline chunks (default: 3)") args = parser.parse_args() -with open(args.params) as f: - params = json.load(f) -# Use per-weblog scenario assignments from endtoend_defs -parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) -parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) -# Build variants to compile are those requiring build -weblog_variants = [w["name"] for w in parallel_weblogs] -# Flatten scenario/job pairs for run jobs -scenario_pairs = [ - (job["weblog"], scenario, job.get("weblog_build_required", True)) - for job in parallel_jobs - for scenario in job.get("scenarios", []) -] -binaries_artifact = params["miscs"]["binaries_artifact"] -parametric = params["parametric"] -env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) +libraries = args.libraries.split() +if not libraries: + print("No libraries specified, nothing to generate.", file=sys.stderr) + sys.exit(0) +params_dir = Path(args.params_dir) +output_dir = Path(args.output_dir) +output_dir.mkdir(parents=True, exist_ok=True) +env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) template = env.get_template("system-tests.yml") -# Render the pipeline with per-weblog scenario pairs -print( # noqa: T201 - template.render( - # we're now using precomputed scenario pairs instead of flat scenarios list + +def render_library(library: str, params: dict, skip_header: bool) -> str: + parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) + parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) + weblog_variants = [w["name"] for w in parallel_weblogs] + scenario_pairs = [ + (job["weblog"], scenario, job.get("weblog_build_required", True)) + for job in parallel_jobs + for scenario in job.get("scenarios", []) + ] + binaries_artifact = params["miscs"]["binaries_artifact"] + parametric = params["parametric"] + return template.render( scenario_pairs=scenario_pairs, stage=args.stage, - library=args.library, + library=library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact, parametric=parametric, ci_image=args.ci_image, ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", - skip_header=args.skip_header, - ), - end="", -) + skip_header=skip_header, + ) + + +# Assign libraries to chunks via simple round-robin +chunk_libraries: dict[int, list[str]] = {i: [] for i in range(args.chunks)} +for idx, library in enumerate(libraries): + chunk_libraries[idx % args.chunks].append(library) + +# Render and write non-empty chunks; track which chunks have content +nonempty_chunks = [] + +for chunk_idx, chunk_libs in chunk_libraries.items(): + if not chunk_libs: + continue + + parts = [] + for lib_idx, library in enumerate(chunk_libs): + params_file = params_dir / f"params_{library}.json" + if not params_file.exists(): + print(f"ERROR: params file not found for library '{library}': {params_file}", file=sys.stderr) + sys.exit(1) + with open(params_file) as f: + params = json.load(f) + parts.append(render_library(library, params, skip_header=(lib_idx > 0))) + + chunk_file = output_dir / f"generated-pipeline-chunk-{chunk_idx}.yml" + chunk_file.write_text("\n".join(parts)) + nonempty_chunks.append(chunk_idx) + +# Write chunks.env dotenv so run_test_pipeline_ rules can gate on content +chunks_env = output_dir / "chunks.env" +lines = [f"CHUNK_{i}_NONEMPTY=true" for i in nonempty_chunks] +chunks_env.write_text("\n".join(lines) + ("\n" if lines else "")) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 794d17f9f13..48a687d6893 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -127,43 +127,19 @@ build_test_pipeline: echo "scenarios: $scenarios" echo "scenario_groups: $scenario_groups" echo "weblogs: $weblogs" - count=0 - for lib in $libraries; do count=$((count + 1)); done - third=$(( (count + 2) / 3 )) - [ $third -eq 0 ] && third=1 - idx=0 for library in $libraries; do - chunk=$(( idx / third )) - [ $chunk -gt 2 ] && chunk=2 python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json - if [ ! -f generated-pipeline-chunk-${chunk}.yml ]; then - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" > generated-pipeline-chunk-${chunk}.yml - else - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --library $library --params params_${library}.json --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --skip-header >> generated-pipeline-chunk-${chunk}.yml - fi - idx=$((idx + 1)) - done - # Ensure all 3 chunk files exist: trigger include requires all files regardless of library count - for i in 1 2; do - if [ ! -f generated-pipeline-chunk-${i}.yml ] && [ -f generated-pipeline-chunk-0.yml ]; then - first_job=$(grep -nm1 '^build_' generated-pipeline-chunk-0.yml | cut -d: -f1 || true) - if [ -n "$first_job" ]; then - head -n $((first_job - 1)) generated-pipeline-chunk-0.yml > generated-pipeline-chunk-${i}.yml - else - cp generated-pipeline-chunk-0.yml generated-pipeline-chunk-${i}.yml - fi - fi done + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml + reports: + dotenv: system-tests/chunks.env .run_test_pipeline_base: interruptible: true stage: $[[ inputs.stage ]] - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/' - - when: never needs: - job: build_test_pipeline optional: true @@ -178,6 +154,9 @@ build_test_pipeline: run_test_pipeline_0: extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/ && $CHUNK_0_NONEMPTY == "true"' + - when: never trigger: include: - artifact: system-tests/generated-pipeline-chunk-0.yml @@ -186,6 +165,9 @@ run_test_pipeline_0: run_test_pipeline_1: extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/ && $CHUNK_1_NONEMPTY == "true"' + - when: never trigger: include: - artifact: system-tests/generated-pipeline-chunk-1.yml @@ -194,6 +176,9 @@ run_test_pipeline_1: run_test_pipeline_2: extends: .run_test_pipeline_base + rules: + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/ && $CHUNK_2_NONEMPTY == "true"' + - when: never trigger: include: - artifact: system-tests/generated-pipeline-chunk-2.yml From b9fccc66294c2ac85fd736b90dd1466abe30b456 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 17:26:45 +0200 Subject: [PATCH 169/229] gitlab: emit noop stub for empty chunks instead of dotenv-gated rules GitLab evaluates rules: at pipeline-creation time, so dotenv vars from needs: jobs are not visible there (gitlab-org/gitlab#235812). The CHUNK__NONEMPTY rule was always false, skipping every chunk trigger. Always emit all chunk files; empty chunks get a minimal noop pipeline. --- utils/ci/gitlab/build_pipeline.py | 31 ++++++++++++++++++++++--------- utils/ci/gitlab/main.yml | 14 +++----------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 2c8745fd379..43247b1f822 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -61,11 +61,31 @@ def render_library(library: str, params: dict, skip_header: bool) -> str: for idx, library in enumerate(libraries): chunk_libraries[idx % args.chunks].append(library) -# Render and write non-empty chunks; track which chunks have content -nonempty_chunks = [] +def noop_stub(stage: str) -> str: + """Return a minimal valid GitLab pipeline YAML for an empty chunk.""" + return f"""workflow: + name: "System-tests end to end (empty chunk)" +stages: + - {stage} +noop: + stage: {stage} + image: registry.ddbuild.io/images/mirror/alpine:latest + tags: + - arch:amd64 + script: + - echo "no libraries assigned to this chunk" +""" + + +# Render and write all chunks; empty chunks get a noop stub so trigger jobs +# always have a valid pipeline file — dotenv vars are not available in rules: +# at pipeline-creation time (gitlab-org/gitlab#235812). for chunk_idx, chunk_libs in chunk_libraries.items(): + chunk_file = output_dir / f"generated-pipeline-chunk-{chunk_idx}.yml" + if not chunk_libs: + chunk_file.write_text(noop_stub(args.stage)) continue parts = [] @@ -78,11 +98,4 @@ def render_library(library: str, params: dict, skip_header: bool) -> str: params = json.load(f) parts.append(render_library(library, params, skip_header=(lib_idx > 0))) - chunk_file = output_dir / f"generated-pipeline-chunk-{chunk_idx}.yml" chunk_file.write_text("\n".join(parts)) - nonempty_chunks.append(chunk_idx) - -# Write chunks.env dotenv so run_test_pipeline_ rules can gate on content -chunks_env = output_dir / "chunks.env" -lines = [f"CHUNK_{i}_NONEMPTY=true" for i in nonempty_chunks] -chunks_env.write_text("\n".join(lines) + ("\n" if lines else "")) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 48a687d6893..bf56641eaf5 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -134,12 +134,13 @@ build_test_pipeline: artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml - reports: - dotenv: system-tests/chunks.env .run_test_pipeline_base: interruptible: true stage: $[[ inputs.stage ]] + rules: + - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/' + - when: never needs: - job: build_test_pipeline optional: true @@ -154,9 +155,6 @@ build_test_pipeline: run_test_pipeline_0: extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/ && $CHUNK_0_NONEMPTY == "true"' - - when: never trigger: include: - artifact: system-tests/generated-pipeline-chunk-0.yml @@ -165,9 +163,6 @@ run_test_pipeline_0: run_test_pipeline_1: extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/ && $CHUNK_1_NONEMPTY == "true"' - - when: never trigger: include: - artifact: system-tests/generated-pipeline-chunk-1.yml @@ -176,9 +171,6 @@ run_test_pipeline_1: run_test_pipeline_2: extends: .run_test_pipeline_base - rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/ && $CHUNK_2_NONEMPTY == "true"' - - when: never trigger: include: - artifact: system-tests/generated-pipeline-chunk-2.yml From 1c72100702a0916632dc82b9157b7b0cb176f9c5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 17:28:47 +0200 Subject: [PATCH 170/229] Stage ordering --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f39684eaf3..b0abd5a42f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,8 +7,8 @@ include: ref: "$CI_COMMIT_SHA" push_to_test_optimization: true stages: - - configure - e2e + - configure - nodejs - java - dotnet From 734a520060f4c56323ebcff7f1172caa032309e8 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 17:46:40 +0200 Subject: [PATCH 171/229] format --- utils/ci/gitlab/build_pipeline.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 43247b1f822..fe567ab5514 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -13,14 +13,16 @@ parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") -parser.add_argument("--output-dir", required=True, help="Directory where generated-pipeline-chunk-.yml files are written") +parser.add_argument( + "--output-dir", required=True, help="Directory where generated-pipeline-chunk-.yml files are written" +) parser.add_argument("--chunks", type=int, default=3, help="Number of pipeline chunks (default: 3)") args = parser.parse_args() libraries = args.libraries.split() if not libraries: - print("No libraries specified, nothing to generate.", file=sys.stderr) + print("No libraries specified, nothing to generate.", file=sys.stderr) # noqa: T201 sys.exit(0) params_dir = Path(args.params_dir) @@ -31,7 +33,7 @@ template = env.get_template("system-tests.yml") -def render_library(library: str, params: dict, skip_header: bool) -> str: +def render_library(library: str, params: dict, *, skip_header: bool) -> str: parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) weblog_variants = [w["name"] for w in parallel_weblogs] @@ -92,7 +94,7 @@ def noop_stub(stage: str) -> str: for lib_idx, library in enumerate(chunk_libs): params_file = params_dir / f"params_{library}.json" if not params_file.exists(): - print(f"ERROR: params file not found for library '{library}': {params_file}", file=sys.stderr) + print(f"ERROR: params file not found for library '{library}': {params_file}", file=sys.stderr) # noqa: T201 sys.exit(1) with open(params_file) as f: params = json.load(f) From d72811462047f16f70e7bc7b5ca061c9ea3ac5ae Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 10 Jun 2026 18:03:50 +0200 Subject: [PATCH 172/229] Stage rereordering --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0abd5a42f9..5f39684eaf3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,8 +7,8 @@ include: ref: "$CI_COMMIT_SHA" push_to_test_optimization: true stages: - - e2e - configure + - e2e - nodejs - java - dotnet From 66de5b281102d09769d5d560f39cee248f94db73 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 10:21:50 +0200 Subject: [PATCH 173/229] gitlab: strip local includes when generating tracer child pipeline external_gitlab_pipeline.py runs in tracer repos via one-pipeline's configure_system_tests job. GitLab resolves local: includes against the root project, so any local: include in system-tests' .gitlab-ci.yml breaks tracer child pipelines. Drop them before emitting. --- .../external_gitlab_pipeline.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/utils/scripts/ci_orchestrators/external_gitlab_pipeline.py b/utils/scripts/ci_orchestrators/external_gitlab_pipeline.py index dd7c0e04777..bb10b53adfb 100644 --- a/utils/scripts/ci_orchestrators/external_gitlab_pipeline.py +++ b/utils/scripts/ci_orchestrators/external_gitlab_pipeline.py @@ -18,6 +18,37 @@ LANG_STAGES = sorted(COMPONENT_GROUPS.ssi) +def _is_local_include(entry: object) -> bool: + """Return True if a GitLab include entry refers to a local file path. + + local: includes are resolved against the root project, not the project + that wrote the YAML. When this file's output is used as a child pipeline + in a tracer repo, any local: path from system-tests' .gitlab-ci.yml would + be looked up inside the tracer repo and cause a pipeline compilation error. + """ + if isinstance(entry, str): + return True # bare strings are local file paths + if isinstance(entry, dict): + return "local" in entry + return False + + +def _strip_local_includes(data: dict) -> None: + """Remove local: include entries from *data* in-place. + + Only remote: (and other non-local) entries are kept so that the generated + pipeline is safe to run from any project. + """ + raw = data.get("include") + if raw is None: + return + if isinstance(raw, list): + data["include"] = [e for e in raw if not _is_local_include(e)] + else: + # Single mapping or bare string + data["include"] = [] if _is_local_include(raw) else [raw] + + def main(language: str | None = None) -> None: """Main function to generate the gitlab system-tests pipeline Args: @@ -30,6 +61,11 @@ def main(language: str | None = None) -> None: with open(".gitlab-ci.yml", "r") as file: data = yaml.safe_load(file) + # Drop local: includes — they resolve against the root project (the tracer + # repo) when this output is used as a child pipeline, not against + # system-tests, so any local: path would cause a compilation error there. + _strip_local_includes(data) + # Ensure 'variables' section exists and update with new values data.setdefault("variables", {}).update(new_variables) From c648db73b064715d6d98f7c91a060e7e05ff6251 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 13:47:00 +0200 Subject: [PATCH 174/229] gitlab: rename system-tests.yml jinja template to .yml.j2 --- utils/ci/gitlab/build_pipeline.py | 2 +- utils/ci/gitlab/{system-tests.yml => system-tests.yml.j2} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename utils/ci/gitlab/{system-tests.yml => system-tests.yml.j2} (100%) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index fe567ab5514..b7a2990eee3 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -30,7 +30,7 @@ output_dir.mkdir(parents=True, exist_ok=True) env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) -template = env.get_template("system-tests.yml") +template = env.get_template("system-tests.yml.j2") def render_library(library: str, params: dict, *, skip_header: bool) -> str: diff --git a/utils/ci/gitlab/system-tests.yml b/utils/ci/gitlab/system-tests.yml.j2 similarity index 100% rename from utils/ci/gitlab/system-tests.yml rename to utils/ci/gitlab/system-tests.yml.j2 From 612d10ef183daf14056f9bb3e3df2bac4c2d4cd0 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 13:49:11 +0200 Subject: [PATCH 175/229] format: extend yaml linting to utils/ci/gitlab and .gitlab-ci.yml --- .yamllint | 1 + format.sh | 6 +++--- utils/ci/gitlab/main.yml | 11 +---------- utils/ci/gitlab/ssi.yml | 16 +--------------- utils/ci/gitlab/templates.yml | 3 +-- 5 files changed, 7 insertions(+), 30 deletions(-) diff --git a/.yamllint b/.yamllint index 8b27b985f77..c9b27c90bf3 100644 --- a/.yamllint +++ b/.yamllint @@ -3,3 +3,4 @@ extends: relaxed rules: line-length: disable key-ordering: disable + indentation: disable diff --git a/format.sh b/format.sh index af0d1ba25ee..8aa6f4b935b 100755 --- a/format.sh +++ b/format.sh @@ -146,13 +146,13 @@ fi echo "Running yamlfmt formatter..." if [ "$COMMAND" == "fix" ]; then - yamlfmt manifests/ + yamlfmt manifests/ utils/ci/gitlab/ else - yamlfmt -lint manifests/ + yamlfmt -lint manifests/ utils/ci/gitlab/ fi echo "Running yamllint checks..." -if ! ./venv/bin/yamllint -s manifests/; then +if ! ./venv/bin/yamllint -s manifests/ utils/ci/gitlab/ .gitlab-ci.yml; then echo "yamllint checks failed. Please fix the errors above. 💥 💔 💥" exit 1 fi diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index bf56641eaf5..3509c2ae69f 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -1,3 +1,4 @@ +--- spec: inputs: stage: @@ -52,9 +53,7 @@ spec: ssi_injector_version: description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" default: "" - --- - include: - local: /utils/ci/gitlab/templates.yml inputs: @@ -70,13 +69,11 @@ include: ssi_library_version: $[[ inputs.ssi_library_version ]] k8s_lib_init_img: $[[ inputs.k8s_lib_init_img ]] ssi_injector_version: $[[ inputs.ssi_injector_version ]] - workflow: rules: - when: always auto_cancel: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] - resolve_ci_image: rules: - if: '$SKIP_RESOLVE_CI_IMAGE == "true"' @@ -102,7 +99,6 @@ resolve_ci_image: artifacts: reports: dotenv: build.env - build_test_pipeline: extends: .system_tests_base tags: @@ -134,7 +130,6 @@ build_test_pipeline: artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml - .run_test_pipeline_base: interruptible: true stage: $[[ inputs.stage ]] @@ -152,7 +147,6 @@ build_test_pipeline: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" LIBRARIES: $LIBRARIES - run_test_pipeline_0: extends: .run_test_pipeline_base trigger: @@ -160,7 +154,6 @@ run_test_pipeline_0: - artifact: system-tests/generated-pipeline-chunk-0.yml job: build_test_pipeline strategy: depend - run_test_pipeline_1: extends: .run_test_pipeline_base trigger: @@ -168,7 +161,6 @@ run_test_pipeline_1: - artifact: system-tests/generated-pipeline-chunk-1.yml job: build_test_pipeline strategy: depend - run_test_pipeline_2: extends: .run_test_pipeline_base trigger: @@ -176,4 +168,3 @@ run_test_pipeline_2: - artifact: system-tests/generated-pipeline-chunk-2.yml job: build_test_pipeline strategy: depend - diff --git a/utils/ci/gitlab/ssi.yml b/utils/ci/gitlab/ssi.yml index b59afdeb651..00b56772487 100644 --- a/utils/ci/gitlab/ssi.yml +++ b/utils/ci/gitlab/ssi.yml @@ -1,3 +1,4 @@ +--- spec: inputs: stage: @@ -20,16 +21,12 @@ spec: ssi_injector_version: description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" default: "" - --- - # SSI pipeline — mirrors the end-to-end pattern: # compute_pipeline → generated-ssi-pipeline.yml → run_ssi_pipeline # Covers all SSI workflows: aws_ssi, dockerssi, libinjection. - include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml - compute_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 tags: ["arch:amd64"] @@ -87,7 +84,6 @@ compute_pipeline: artifacts: paths: - "*_ssi_gitlab_pipeline.yml" - # SSI child pipelines run sequentially per language to respect AWS EC2 / subnet # quotas. Order mirrors the legacy pipeline on main: # nodejs → java → dotnet → python → php → ruby. @@ -109,7 +105,6 @@ nodejs_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/' when: always - java_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -127,7 +122,6 @@ java_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/' when: always - dotnet_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -147,7 +141,6 @@ dotnet_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/' when: always - python_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -169,7 +162,6 @@ python_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/' when: always - php_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -193,7 +185,6 @@ php_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/' when: always - ruby_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -219,7 +210,6 @@ ruby_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/' when: always - delete_amis: extends: .base_job_onboarding stage: $[[ inputs.stage ]] @@ -235,7 +225,6 @@ delete_amis: - if: '$SCHEDULED_JOB == "delete_amis"' after_script: echo "Finish" timeout: 3h - delete_amis_by_name_or_lang: extends: .base_job_onboarding stage: $[[ inputs.stage ]] @@ -258,7 +247,6 @@ delete_amis_by_name_or_lang: rules: - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' after_script: echo "Finish" - delete_ec2_instances: extends: .base_job_onboarding stage: $[[ inputs.stage ]] @@ -272,7 +260,6 @@ delete_ec2_instances: rules: - if: '$SCHEDULED_JOB == "delete_ec2_instances"' after_script: echo "Finish" - count_amis: extends: .base_job_onboarding stage: $[[ inputs.stage ]] @@ -284,7 +271,6 @@ count_amis: rules: - if: '$SCHEDULED_JOB == "count_amis"' after_script: echo "Finish" - .delayed_base_job: image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 tags: ["arch:amd64"] diff --git a/utils/ci/gitlab/templates.yml b/utils/ci/gitlab/templates.yml index 4bba683dd11..a6f79d372d1 100644 --- a/utils/ci/gitlab/templates.yml +++ b/utils/ci/gitlab/templates.yml @@ -1,11 +1,10 @@ +--- spec: inputs: stage: ref: default: main - --- - .system_tests_base: image: $CI_IMAGE tags: From ba96c5a8ab46cfcf3036a275b2a95018fb52648e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 13:49:53 +0200 Subject: [PATCH 176/229] gitlab(build_pipeline): refactor into testable main/build functions --- utils/ci/gitlab/build_pipeline.py | 170 +++++++++++++++++------------- 1 file changed, 98 insertions(+), 72 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index b7a2990eee3..cb2bd99a81f 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -1,39 +1,33 @@ +from __future__ import annotations + import argparse +import json import sys from pathlib import Path -import json - from jinja2 import Environment, FileSystemLoader, select_autoescape -parser = argparse.ArgumentParser() -parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") -parser.add_argument("--libraries", required=True, help="Space-separated list of library names") -parser.add_argument("--params-dir", required=True, help="Directory containing params_.json files") -parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") -parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") -parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") -parser.add_argument( - "--output-dir", required=True, help="Directory where generated-pipeline-chunk-.yml files are written" -) -parser.add_argument("--chunks", type=int, default=3, help="Number of pipeline chunks (default: 3)") - -args = parser.parse_args() - -libraries = args.libraries.split() -if not libraries: - print("No libraries specified, nothing to generate.", file=sys.stderr) # noqa: T201 - sys.exit(0) +_env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) +_template = _env.get_template("system-tests.yml.j2") -params_dir = Path(args.params_dir) -output_dir = Path(args.output_dir) -output_dir.mkdir(parents=True, exist_ok=True) -env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) -template = env.get_template("system-tests.yml.j2") +def noop_stub(stage: str) -> str: + """Return a minimal valid GitLab pipeline YAML for an empty chunk.""" + return f"""workflow: + name: "System-tests end to end (empty chunk)" +stages: + - {stage} +noop: + stage: {stage} + image: registry.ddbuild.io/images/mirror/alpine:latest + tags: + - arch:amd64 + script: + - echo "no libraries assigned to this chunk" +""" -def render_library(library: str, params: dict, *, skip_header: bool) -> str: +def render_library(library: str, params: dict, *, skip_header: bool, stage: str, ci_image: str, ref: str, push_to_test_optimization: bool) -> str: # noqa: E501 parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) weblog_variants = [w["name"] for w in parallel_weblogs] @@ -44,60 +38,92 @@ def render_library(library: str, params: dict, *, skip_header: bool) -> str: ] binaries_artifact = params["miscs"]["binaries_artifact"] parametric = params["parametric"] - return template.render( + return _template.render( scenario_pairs=scenario_pairs, - stage=args.stage, + stage=stage, library=library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact, parametric=parametric, - ci_image=args.ci_image, - ref=args.ref, - push_to_test_optimization=args.push_to_test_optimization == "true", + ci_image=ci_image, + ref=ref, + push_to_test_optimization=push_to_test_optimization, skip_header=skip_header, ) -# Assign libraries to chunks via simple round-robin -chunk_libraries: dict[int, list[str]] = {i: [] for i in range(args.chunks)} -for idx, library in enumerate(libraries): - chunk_libraries[idx % args.chunks].append(library) - - -def noop_stub(stage: str) -> str: - """Return a minimal valid GitLab pipeline YAML for an empty chunk.""" - return f"""workflow: - name: "System-tests end to end (empty chunk)" -stages: - - {stage} -noop: - stage: {stage} - image: registry.ddbuild.io/images/mirror/alpine:latest - tags: - - arch:amd64 - script: - - echo "no libraries assigned to this chunk" -""" - - -# Render and write all chunks; empty chunks get a noop stub so trigger jobs -# always have a valid pipeline file — dotenv vars are not available in rules: -# at pipeline-creation time (gitlab-org/gitlab#235812). -for chunk_idx, chunk_libs in chunk_libraries.items(): - chunk_file = output_dir / f"generated-pipeline-chunk-{chunk_idx}.yml" - - if not chunk_libs: - chunk_file.write_text(noop_stub(args.stage)) - continue +def build( + libraries: list[str], + params_dir: Path, + output_dir: Path, + *, + stage: str, + ci_image: str, + ref: str = "", + push_to_test_optimization: bool = False, + chunks: int = 3, +) -> None: + """Render pipeline chunk files into *output_dir*, one per chunk.""" + output_dir.mkdir(parents=True, exist_ok=True) + + # Assign libraries to chunks via simple round-robin + chunk_libraries: dict[int, list[str]] = {i: [] for i in range(chunks)} + for idx, library in enumerate(libraries): + chunk_libraries[idx % chunks].append(library) + + # Render and write all chunks; empty chunks get a noop stub so trigger jobs + # always have a valid pipeline file — dotenv vars are not available in rules: + # at pipeline-creation time (gitlab-org/gitlab#235812). + for chunk_idx, chunk_libs in chunk_libraries.items(): + chunk_file = output_dir / f"generated-pipeline-chunk-{chunk_idx}.yml" + + if not chunk_libs: + chunk_file.write_text(noop_stub(stage)) + continue + + parts = [] + for lib_idx, library in enumerate(chunk_libs): + params_file = params_dir / f"params_{library}.json" + if not params_file.exists(): + print(f"ERROR: params file not found for library '{library}': {params_file}", file=sys.stderr) # noqa: T201 + sys.exit(1) + with open(params_file) as f: + params = json.load(f) + parts.append(render_library(library, params, skip_header=(lib_idx > 0), stage=stage, ci_image=ci_image, ref=ref, push_to_test_optimization=push_to_test_optimization)) # noqa: E501 + + chunk_file.write_text("\n".join(parts)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") + parser.add_argument("--libraries", required=True, help="Space-separated list of library names") + parser.add_argument("--params-dir", required=True, help="Directory containing params_.json files") + parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") + parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") + parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") + parser.add_argument("--output-dir", required=True, help="Directory where generated-pipeline-chunk-.yml files are written") + parser.add_argument("--chunks", type=int, default=3, help="Number of pipeline chunks (default: 3)") + + args = parser.parse_args(argv) + + libraries = args.libraries.split() + if not libraries: + print("No libraries specified, nothing to generate.", file=sys.stderr) # noqa: T201 + return 0 + + build( + libraries=libraries, + params_dir=Path(args.params_dir), + output_dir=Path(args.output_dir), + stage=args.stage, + ci_image=args.ci_image, + ref=args.ref, + push_to_test_optimization=args.push_to_test_optimization == "true", + chunks=args.chunks, + ) + return 0 - parts = [] - for lib_idx, library in enumerate(chunk_libs): - params_file = params_dir / f"params_{library}.json" - if not params_file.exists(): - print(f"ERROR: params file not found for library '{library}': {params_file}", file=sys.stderr) # noqa: T201 - sys.exit(1) - with open(params_file) as f: - params = json.load(f) - parts.append(render_library(library, params, skip_header=(lib_idx > 0))) - chunk_file.write_text("\n".join(parts)) +if __name__ == "__main__": + sys.exit(main()) From 18566f3461b81b500d448ec07e212cd0a5b2ad4d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 13:54:11 +0200 Subject: [PATCH 177/229] test(test_the_test): cover GitLab mode in compute_libraries_and_scenarios --- .../test_compute_libraries_and_scenarios.py | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index 625461ea806..ad0ef4be3c8 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -488,3 +488,40 @@ def test_missing_original_manifest(self): new_manifests=Path("./tests/test_the_test/manifests/manifests_ref/"), old_manifests=Path("./wrong/path"), ) + + +@scenarios.test_the_test +class Test_GitLabMode: + @pytest.mark.parametrize( + "source,expected_event", + [("merge_request_event", "pull_request"), ("push", "push"), ("schedule", "schedule")], + ) + def test_event_and_ref_normalization(self, monkeypatch, source, expected_event): + monkeypatch.setenv("GITLAB_CI", "true") + monkeypatch.setenv("CI_PIPELINE_SOURCE", source) + monkeypatch.setenv("CI_COMMIT_REF_NAME", "feat-x") + inputs = build_inputs() + assert inputs.event_name == expected_event + assert inputs.ref == "refs/heads/feat-x" + assert inputs.is_gitlab + + def test_empty_ref(self, monkeypatch): + monkeypatch.setenv("GITLAB_CI", "true") + monkeypatch.setenv("CI_PIPELINE_SOURCE", "push") + monkeypatch.setenv("CI_COMMIT_REF_NAME", "") + inputs = build_inputs() + assert inputs.ref == "" + + def test_libraries_output_sorted_no_rust(self, monkeypatch): + monkeypatch.setenv("GITLAB_CI", "true") + monkeypatch.setenv("CI_PIPELINE_SOURCE", "push") + monkeypatch.setenv("CI_COMMIT_REF_NAME", "feat-x") + # all-libs trigger file + inputs = build_inputs(modified_files=[".github/workflows/run-docker-ssi.yml"]) + output = process(inputs) + libs_line = next((l for l in output if l.startswith("libraries=")), None) + assert libs_line is not None, "GitLab mode must emit 'libraries=' output" + libs = json.loads(libs_line.split("=", 1)[1]) + parts = libs.split() + assert "rust" not in parts + assert parts == sorted(parts) From 1a1dd31103136329485d1f2bb26e0ab66428104c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 13:54:17 +0200 Subject: [PATCH 178/229] test(test_the_test): cover compute-workflow-parameters safe access regressions --- tests/test_the_test/test_ci_orchestrator.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_the_test/test_ci_orchestrator.py b/tests/test_the_test/test_ci_orchestrator.py index 8cb598c5ca9..27d98a85ba4 100644 --- a/tests/test_the_test/test_ci_orchestrator.py +++ b/tests/test_the_test/test_ci_orchestrator.py @@ -29,3 +29,16 @@ def test_ipv6_is_not_supported_for_uds_weblogs(): assert not _is_supported("dotnet", "uds", "IPV6", "dev") assert not _is_supported("python", "uds-flask", "IPV6", "dev") assert _is_supported("python", "flask-poc", "IPV6", "dev") + + +@scenarios.test_the_test +def test_get_endtoend_definitions_empty_scenario_map(): + # Regression: previously raised KeyError when "endtoend" or "parametric" keys were absent + defs = get_endtoend_definitions("ruby", {}, [], "dev", 200000, 256, "123", "") + assert isinstance(defs["endtoend_defs"]["parallel_jobs"], list) + + +@scenarios.test_the_test +def test_get_endtoend_definitions_missing_endtoend_key(): + defs = get_endtoend_definitions("ruby", {"other": ["X"]}, [], "dev", 200000, 256, "123", "") + assert defs["endtoend_defs"]["parallel_jobs"] == [] From 6b541e000caac3ae1e323c8d164529f2bc3de93d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 13:54:26 +0200 Subject: [PATCH 179/229] test(test_the_test): cover build_pipeline chunking and rendering --- tests/test_the_test/test_build_pipeline.py | 72 ++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 tests/test_the_test/test_build_pipeline.py diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py new file mode 100644 index 00000000000..b089cb4d403 --- /dev/null +++ b/tests/test_the_test/test_build_pipeline.py @@ -0,0 +1,72 @@ +"""Tests for utils/ci/gitlab/build_pipeline.py — chunking, rendering, and CLI.""" +import json +import re + +import pytest +import yaml + +from utils import scenarios +from utils.ci.gitlab.build_pipeline import build, main, noop_stub + + +MINIMAL_PARAMS = { + "endtoend_defs": { + "parallel_weblogs": [{"name": "flask"}], + "parallel_jobs": [{"weblog": "flask", "scenarios": ["DEFAULT"], "weblog_build_required": True}], + }, + "miscs": {"binaries_artifact": "flask-binaries"}, + "parametric": {"enable": False, "parallel_jobs": []}, +} + +BUILD_KWARGS = {"stage": "e2e", "ci_image": "myimage", "chunks": 3} + + +def write_params(tmp_path, *libs): + for lib in libs: + (tmp_path / f"params_{lib}.json").write_text(json.dumps(MINIMAL_PARAMS)) + + +@scenarios.test_the_test +class Test_BuildPipeline: + @pytest.mark.parametrize( + "libs", + [ + [], + ["a"], + ["a", "b"], + ["a", "b", "c"], + ["a", "b", "c", "d", "e", "f"], + ], + ) + def test_chunking_distribution(self, tmp_path, libs): + write_params(tmp_path, *libs) + out = tmp_path / "out" + build(libs, tmp_path, out, **BUILD_KWARGS) + + # Expected round-robin assignment + expected: dict[int, list[str]] = {i: [] for i in range(3)} + for idx, lib in enumerate(libs): + expected[idx % 3].append(lib) + + for i in range(3): + chunk = yaml.safe_load((out / f"generated-pipeline-chunk-{i}.yml").read_text()) + if not expected[i]: + assert "noop" in chunk, f"chunk {i} should be noop" + else: + jobs = {k: v for k, v in chunk.items() if isinstance(v, dict) and ("extends" in v or "script" in v)} + assert jobs, f"chunk {i} should have jobs for {expected[i]}" + + def test_missing_params_exits_nonzero(self, tmp_path): + with pytest.raises(SystemExit) as exc: + main(["--stage", "e2e", "--libraries", "python", "--params-dir", str(tmp_path), "--ci-image", "x", "--output-dir", str(tmp_path / "out"), "--chunks", "3"]) + assert exc.value.code != 0 + + def test_concatenation_no_duplicate_top_level_keys(self, tmp_path): + """Two libs in same chunk (chunks=1) must not produce duplicate top-level YAML keys.""" + write_params(tmp_path, "python", "java") + out = tmp_path / "out" + build(["python", "java"], tmp_path, out, stage="e2e", ci_image="img", chunks=1) + text = (out / "generated-pipeline-chunk-0.yml").read_text() + for key in ("workflow:", "stages:", "include:"): + count = len(re.findall(rf"^{re.escape(key)}", text, re.MULTILINE)) + assert count <= 1, f"duplicate top-level key '{key}' found {count} times" From cd815f4b8a4f5b1afe0fc30680665d0fe469f686 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 13:54:34 +0200 Subject: [PATCH 180/229] test(test_the_test): cover external_gitlab_pipeline local-include stripping --- .../test_external_gitlab_pipeline.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tests/test_the_test/test_external_gitlab_pipeline.py diff --git a/tests/test_the_test/test_external_gitlab_pipeline.py b/tests/test_the_test/test_external_gitlab_pipeline.py new file mode 100644 index 00000000000..5dad3f8b741 --- /dev/null +++ b/tests/test_the_test/test_external_gitlab_pipeline.py @@ -0,0 +1,56 @@ +"""Tests for utils/scripts/ci_orchestrators/external_gitlab_pipeline.py.""" +import yaml +import pytest + +from utils import scenarios +from utils.scripts.ci_orchestrators.external_gitlab_pipeline import ( + _is_local_include, + _strip_local_includes, + filter_yaml, + main, +) + + +@scenarios.test_the_test +class Test_ExternalGitlabPipeline: + @pytest.mark.parametrize( + "entry,expected", + [ + ({"local": "/foo"}, True), + ("bare-string", True), + ({"project": "p", "file": "f"}, False), + ({"remote": "https://example.com/x.yml"}, False), + ], + ) + def test_is_local_include(self, entry, expected): + assert _is_local_include(entry) is expected + + def test_strip_local_includes_removes_only_local(self): + remote = {"remote": "https://example.com/x.yml"} + data = {"include": [{"local": "/utils/ci/gitlab/main.yml"}, "bare", remote]} + _strip_local_includes(data) + assert data["include"] == [remote] + + def test_main_strips_locals_from_real_repo_yaml(self, capsys): + main(language=None) + out = capsys.readouterr().out + data = yaml.safe_load(out) + for entry in data.get("include", []): + if isinstance(entry, dict): + assert "local" not in entry + + def test_filter_yaml_keeps_only_requested_language(self): + data = { + "stages": ["configure", "python", "java", "pipeline-status"], + "variables": {}, + "include": [], + "configure_job": {"stage": "configure", "script": ["echo"]}, + "python_job": {"stage": "python", "script": ["echo"]}, + "java_job": {"stage": "java", "script": ["echo"]}, + "status_job": {"stage": "pipeline-status", "script": ["echo"]}, + } + result = filter_yaml(data, "python") + assert "java_job" not in result + assert "python_job" in result + assert "configure_job" in result + assert result["stages"] == ["configure", "python", "pipeline-status"] From 37cf6d99209a424bccbf61d5128be9d8b1c8698f Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 13:57:06 +0200 Subject: [PATCH 181/229] test(test_the_test): cover generated GitLab pipeline structural integrity --- .../test_gitlab_pipeline_structure.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/test_the_test/test_gitlab_pipeline_structure.py diff --git a/tests/test_the_test/test_gitlab_pipeline_structure.py b/tests/test_the_test/test_gitlab_pipeline_structure.py new file mode 100644 index 00000000000..86f5775efd5 --- /dev/null +++ b/tests/test_the_test/test_gitlab_pipeline_structure.py @@ -0,0 +1,32 @@ +"""Structural assertions on generated GitLab pipeline chunk YAML.""" +import json + +import yaml + +from utils import scenarios +from utils.ci.gitlab.build_pipeline import build + + +MINIMAL_PARAMS = { + "endtoend_defs": { + "parallel_weblogs": [{"name": "flask"}], + "parallel_jobs": [{"weblog": "flask", "scenarios": ["DEFAULT"], "weblog_build_required": True}], + }, + "miscs": {"binaries_artifact": "flask-binaries"}, + "parametric": {"enable": False, "parallel_jobs": []}, +} + + +@scenarios.test_the_test +def test_generated_chunk_jobs_have_required_keys(tmp_path): + (tmp_path / "params_python.json").write_text(json.dumps(MINIMAL_PARAMS)) + out = tmp_path / "out" + build(["python"], tmp_path, out, stage="e2e", ci_image="myimage", chunks=3) + + for i in range(3): + chunk = yaml.safe_load((out / f"generated-pipeline-chunk-{i}.yml").read_text()) + for name, job in chunk.items(): + if not isinstance(job, dict) or name.startswith(".") or name in ("workflow", "stages", "include", "variables"): + continue + has_stage = "stage" in job or "extends" in job + assert has_stage, f"job '{name}' in chunk {i} has no stage or extends" From 6bbd7e680536a01ec65a4882bc7643b984092761 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 14:02:56 +0200 Subject: [PATCH 182/229] fix: ruff and mypy issues in new test files and build_pipeline --- tests/test_the_test/test_build_pipeline.py | 16 +++++++--------- .../test_compute_libraries_and_scenarios.py | 10 +++++----- .../test_external_gitlab_pipeline.py | 6 +++--- .../test_gitlab_pipeline_structure.py | 3 ++- utils/ci/gitlab/build_pipeline.py | 2 +- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py index b089cb4d403..4d0c23a628a 100644 --- a/tests/test_the_test/test_build_pipeline.py +++ b/tests/test_the_test/test_build_pipeline.py @@ -1,12 +1,13 @@ """Tests for utils/ci/gitlab/build_pipeline.py — chunking, rendering, and CLI.""" import json import re +from pathlib import Path import pytest import yaml from utils import scenarios -from utils.ci.gitlab.build_pipeline import build, main, noop_stub +from utils.ci.gitlab.build_pipeline import build, main MINIMAL_PARAMS = { @@ -18,10 +19,7 @@ "parametric": {"enable": False, "parallel_jobs": []}, } -BUILD_KWARGS = {"stage": "e2e", "ci_image": "myimage", "chunks": 3} - - -def write_params(tmp_path, *libs): +def write_params(tmp_path: Path, *libs: str) -> None: for lib in libs: (tmp_path / f"params_{lib}.json").write_text(json.dumps(MINIMAL_PARAMS)) @@ -38,10 +36,10 @@ class Test_BuildPipeline: ["a", "b", "c", "d", "e", "f"], ], ) - def test_chunking_distribution(self, tmp_path, libs): + def test_chunking_distribution(self, tmp_path: Path, libs: list[str]): write_params(tmp_path, *libs) out = tmp_path / "out" - build(libs, tmp_path, out, **BUILD_KWARGS) + build(libs, tmp_path, out, stage="e2e", ci_image="myimage", chunks=3) # Expected round-robin assignment expected: dict[int, list[str]] = {i: [] for i in range(3)} @@ -56,12 +54,12 @@ def test_chunking_distribution(self, tmp_path, libs): jobs = {k: v for k, v in chunk.items() if isinstance(v, dict) and ("extends" in v or "script" in v)} assert jobs, f"chunk {i} should have jobs for {expected[i]}" - def test_missing_params_exits_nonzero(self, tmp_path): + def test_missing_params_exits_nonzero(self, tmp_path: Path): with pytest.raises(SystemExit) as exc: main(["--stage", "e2e", "--libraries", "python", "--params-dir", str(tmp_path), "--ci-image", "x", "--output-dir", str(tmp_path / "out"), "--chunks", "3"]) assert exc.value.code != 0 - def test_concatenation_no_duplicate_top_level_keys(self, tmp_path): + def test_concatenation_no_duplicate_top_level_keys(self, tmp_path: Path): """Two libs in same chunk (chunks=1) must not produce duplicate top-level YAML keys.""" write_params(tmp_path, "python", "java") out = tmp_path / "out" diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index ad0ef4be3c8..b85b2d4df86 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -493,10 +493,10 @@ def test_missing_original_manifest(self): @scenarios.test_the_test class Test_GitLabMode: @pytest.mark.parametrize( - "source,expected_event", + ("source", "expected_event"), [("merge_request_event", "pull_request"), ("push", "push"), ("schedule", "schedule")], ) - def test_event_and_ref_normalization(self, monkeypatch, source, expected_event): + def test_event_and_ref_normalization(self, monkeypatch: pytest.MonkeyPatch, source: str, expected_event: str): monkeypatch.setenv("GITLAB_CI", "true") monkeypatch.setenv("CI_PIPELINE_SOURCE", source) monkeypatch.setenv("CI_COMMIT_REF_NAME", "feat-x") @@ -505,21 +505,21 @@ def test_event_and_ref_normalization(self, monkeypatch, source, expected_event): assert inputs.ref == "refs/heads/feat-x" assert inputs.is_gitlab - def test_empty_ref(self, monkeypatch): + def test_empty_ref(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("GITLAB_CI", "true") monkeypatch.setenv("CI_PIPELINE_SOURCE", "push") monkeypatch.setenv("CI_COMMIT_REF_NAME", "") inputs = build_inputs() assert inputs.ref == "" - def test_libraries_output_sorted_no_rust(self, monkeypatch): + def test_libraries_output_sorted_no_rust(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("GITLAB_CI", "true") monkeypatch.setenv("CI_PIPELINE_SOURCE", "push") monkeypatch.setenv("CI_COMMIT_REF_NAME", "feat-x") # all-libs trigger file inputs = build_inputs(modified_files=[".github/workflows/run-docker-ssi.yml"]) output = process(inputs) - libs_line = next((l for l in output if l.startswith("libraries=")), None) + libs_line = next((line for line in output if line.startswith("libraries=")), None) assert libs_line is not None, "GitLab mode must emit 'libraries=' output" libs = json.loads(libs_line.split("=", 1)[1]) parts = libs.split() diff --git a/tests/test_the_test/test_external_gitlab_pipeline.py b/tests/test_the_test/test_external_gitlab_pipeline.py index 5dad3f8b741..18536bf505f 100644 --- a/tests/test_the_test/test_external_gitlab_pipeline.py +++ b/tests/test_the_test/test_external_gitlab_pipeline.py @@ -14,7 +14,7 @@ @scenarios.test_the_test class Test_ExternalGitlabPipeline: @pytest.mark.parametrize( - "entry,expected", + ("entry", "expected"), [ ({"local": "/foo"}, True), ("bare-string", True), @@ -22,7 +22,7 @@ class Test_ExternalGitlabPipeline: ({"remote": "https://example.com/x.yml"}, False), ], ) - def test_is_local_include(self, entry, expected): + def test_is_local_include(self, entry: object, expected: object): assert _is_local_include(entry) is expected def test_strip_local_includes_removes_only_local(self): @@ -31,7 +31,7 @@ def test_strip_local_includes_removes_only_local(self): _strip_local_includes(data) assert data["include"] == [remote] - def test_main_strips_locals_from_real_repo_yaml(self, capsys): + def test_main_strips_locals_from_real_repo_yaml(self, capsys: pytest.CaptureFixture): main(language=None) out = capsys.readouterr().out data = yaml.safe_load(out) diff --git a/tests/test_the_test/test_gitlab_pipeline_structure.py b/tests/test_the_test/test_gitlab_pipeline_structure.py index 86f5775efd5..5378b4ddedb 100644 --- a/tests/test_the_test/test_gitlab_pipeline_structure.py +++ b/tests/test_the_test/test_gitlab_pipeline_structure.py @@ -1,5 +1,6 @@ """Structural assertions on generated GitLab pipeline chunk YAML.""" import json +from pathlib import Path import yaml @@ -18,7 +19,7 @@ @scenarios.test_the_test -def test_generated_chunk_jobs_have_required_keys(tmp_path): +def test_generated_chunk_jobs_have_required_keys(tmp_path: Path): (tmp_path / "params_python.json").write_text(json.dumps(MINIMAL_PARAMS)) out = tmp_path / "out" build(["python"], tmp_path, out, stage="e2e", ci_image="myimage", chunks=3) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index cb2bd99a81f..363a45b3457 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -102,7 +102,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") - parser.add_argument("--output-dir", required=True, help="Directory where generated-pipeline-chunk-.yml files are written") + parser.add_argument("--output-dir", required=True, help="Output directory for generated-pipeline-chunk-N.yml files") parser.add_argument("--chunks", type=int, default=3, help="Number of pipeline chunks (default: 3)") args = parser.parse_args(argv) From da3626045f7a4f186e09279a66ff32c8ff97cc03 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 14:56:40 +0200 Subject: [PATCH 183/229] format: preserve blank lines in gitlab YAML files via yamlfmt config - Add retain_line_breaks_single: true to .yamlfmt so yamlfmt preserves single blank lines between top-level keys. - Drop utils/ci/gitlab/ from yamlfmt scope: GitLab's $[[ ]] interpolation syntax is not valid YAML and causes yamlfmt to abort on those files. yamllint (not yamlfmt) is the lint tool covering that directory. - Restore blank lines stripped by yamlfmt in main.yml, ssi.yml, and templates.yml (commit 612d10ef1 ran yamlfmt on them before the scope was narrowed). - Apply ruff format to test_build_pipeline.py, test_external_gitlab_pipeline.py, test_gitlab_pipeline_structure.py, and build_pipeline.py; remove two stale noqa: E501 directives that became redundant after reformatting. --- .yamlfmt | 1 + format.sh | 6 +++-- tests/test_the_test/test_build_pipeline.py | 19 ++++++++++++++- .../test_external_gitlab_pipeline.py | 1 + .../test_gitlab_pipeline_structure.py | 7 +++++- utils/ci/gitlab/build_pipeline.py | 23 +++++++++++++++++-- utils/ci/gitlab/main.yml | 10 ++++++++ utils/ci/gitlab/ssi.yml | 15 ++++++++++++ utils/ci/gitlab/templates.yml | 2 ++ 9 files changed, 78 insertions(+), 6 deletions(-) diff --git a/.yamlfmt b/.yamlfmt index 24441a65f0a..3ce4697bfce 100644 --- a/.yamlfmt +++ b/.yamlfmt @@ -1,6 +1,7 @@ formatter: type: basic include_document_start: true + retain_line_breaks_single: true indent: 2 pad_line_comments: 2 eof_newline: true diff --git a/format.sh b/format.sh index 8aa6f4b935b..f5ecdf91fac 100755 --- a/format.sh +++ b/format.sh @@ -145,10 +145,12 @@ if ! which yamlfmt > /dev/null; then fi echo "Running yamlfmt formatter..." +# utils/ci/gitlab/ is excluded from yamlfmt: GitLab's $[[ ]] interpolation syntax +# is not valid YAML and causes yamlfmt to fail on those files. if [ "$COMMAND" == "fix" ]; then - yamlfmt manifests/ utils/ci/gitlab/ + yamlfmt manifests/ else - yamlfmt -lint manifests/ utils/ci/gitlab/ + yamlfmt -lint manifests/ fi echo "Running yamllint checks..." diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py index 4d0c23a628a..9f18726649e 100644 --- a/tests/test_the_test/test_build_pipeline.py +++ b/tests/test_the_test/test_build_pipeline.py @@ -1,4 +1,5 @@ """Tests for utils/ci/gitlab/build_pipeline.py — chunking, rendering, and CLI.""" + import json import re from pathlib import Path @@ -19,6 +20,7 @@ "parametric": {"enable": False, "parallel_jobs": []}, } + def write_params(tmp_path: Path, *libs: str) -> None: for lib in libs: (tmp_path / f"params_{lib}.json").write_text(json.dumps(MINIMAL_PARAMS)) @@ -56,7 +58,22 @@ def test_chunking_distribution(self, tmp_path: Path, libs: list[str]): def test_missing_params_exits_nonzero(self, tmp_path: Path): with pytest.raises(SystemExit) as exc: - main(["--stage", "e2e", "--libraries", "python", "--params-dir", str(tmp_path), "--ci-image", "x", "--output-dir", str(tmp_path / "out"), "--chunks", "3"]) + main( + [ + "--stage", + "e2e", + "--libraries", + "python", + "--params-dir", + str(tmp_path), + "--ci-image", + "x", + "--output-dir", + str(tmp_path / "out"), + "--chunks", + "3", + ] + ) assert exc.value.code != 0 def test_concatenation_no_duplicate_top_level_keys(self, tmp_path: Path): diff --git a/tests/test_the_test/test_external_gitlab_pipeline.py b/tests/test_the_test/test_external_gitlab_pipeline.py index 18536bf505f..32252bff9dd 100644 --- a/tests/test_the_test/test_external_gitlab_pipeline.py +++ b/tests/test_the_test/test_external_gitlab_pipeline.py @@ -1,4 +1,5 @@ """Tests for utils/scripts/ci_orchestrators/external_gitlab_pipeline.py.""" + import yaml import pytest diff --git a/tests/test_the_test/test_gitlab_pipeline_structure.py b/tests/test_the_test/test_gitlab_pipeline_structure.py index 5378b4ddedb..01927ec73d0 100644 --- a/tests/test_the_test/test_gitlab_pipeline_structure.py +++ b/tests/test_the_test/test_gitlab_pipeline_structure.py @@ -1,4 +1,5 @@ """Structural assertions on generated GitLab pipeline chunk YAML.""" + import json from pathlib import Path @@ -27,7 +28,11 @@ def test_generated_chunk_jobs_have_required_keys(tmp_path: Path): for i in range(3): chunk = yaml.safe_load((out / f"generated-pipeline-chunk-{i}.yml").read_text()) for name, job in chunk.items(): - if not isinstance(job, dict) or name.startswith(".") or name in ("workflow", "stages", "include", "variables"): + if ( + not isinstance(job, dict) + or name.startswith(".") + or name in ("workflow", "stages", "include", "variables") + ): continue has_stage = "stage" in job or "extends" in job assert has_stage, f"job '{name}' in chunk {i} has no stage or extends" diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 363a45b3457..d978e096f12 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -27,7 +27,16 @@ def noop_stub(stage: str) -> str: """ -def render_library(library: str, params: dict, *, skip_header: bool, stage: str, ci_image: str, ref: str, push_to_test_optimization: bool) -> str: # noqa: E501 +def render_library( + library: str, + params: dict, + *, + skip_header: bool, + stage: str, + ci_image: str, + ref: str, + push_to_test_optimization: bool, +) -> str: parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) weblog_variants = [w["name"] for w in parallel_weblogs] @@ -89,7 +98,17 @@ def build( sys.exit(1) with open(params_file) as f: params = json.load(f) - parts.append(render_library(library, params, skip_header=(lib_idx > 0), stage=stage, ci_image=ci_image, ref=ref, push_to_test_optimization=push_to_test_optimization)) # noqa: E501 + parts.append( + render_library( + library, + params, + skip_header=(lib_idx > 0), + stage=stage, + ci_image=ci_image, + ref=ref, + push_to_test_optimization=push_to_test_optimization, + ) + ) chunk_file.write_text("\n".join(parts)) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 3509c2ae69f..d03266ef940 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -53,7 +53,9 @@ spec: ssi_injector_version: description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" default: "" + --- + include: - local: /utils/ci/gitlab/templates.yml inputs: @@ -69,11 +71,13 @@ include: ssi_library_version: $[[ inputs.ssi_library_version ]] k8s_lib_init_img: $[[ inputs.k8s_lib_init_img ]] ssi_injector_version: $[[ inputs.ssi_injector_version ]] + workflow: rules: - when: always auto_cancel: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] + resolve_ci_image: rules: - if: '$SKIP_RESOLVE_CI_IMAGE == "true"' @@ -99,6 +103,7 @@ resolve_ci_image: artifacts: reports: dotenv: build.env + build_test_pipeline: extends: .system_tests_base tags: @@ -130,6 +135,7 @@ build_test_pipeline: artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml + .run_test_pipeline_base: interruptible: true stage: $[[ inputs.stage ]] @@ -147,6 +153,7 @@ build_test_pipeline: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" LIBRARIES: $LIBRARIES + run_test_pipeline_0: extends: .run_test_pipeline_base trigger: @@ -154,6 +161,7 @@ run_test_pipeline_0: - artifact: system-tests/generated-pipeline-chunk-0.yml job: build_test_pipeline strategy: depend + run_test_pipeline_1: extends: .run_test_pipeline_base trigger: @@ -161,6 +169,7 @@ run_test_pipeline_1: - artifact: system-tests/generated-pipeline-chunk-1.yml job: build_test_pipeline strategy: depend + run_test_pipeline_2: extends: .run_test_pipeline_base trigger: @@ -168,3 +177,4 @@ run_test_pipeline_2: - artifact: system-tests/generated-pipeline-chunk-2.yml job: build_test_pipeline strategy: depend + diff --git a/utils/ci/gitlab/ssi.yml b/utils/ci/gitlab/ssi.yml index 00b56772487..baf91f271ed 100644 --- a/utils/ci/gitlab/ssi.yml +++ b/utils/ci/gitlab/ssi.yml @@ -21,12 +21,16 @@ spec: ssi_injector_version: description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" default: "" + --- + # SSI pipeline — mirrors the end-to-end pattern: # compute_pipeline → generated-ssi-pipeline.yml → run_ssi_pipeline # Covers all SSI workflows: aws_ssi, dockerssi, libinjection. + include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml + compute_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 tags: ["arch:amd64"] @@ -84,6 +88,7 @@ compute_pipeline: artifacts: paths: - "*_ssi_gitlab_pipeline.yml" + # SSI child pipelines run sequentially per language to respect AWS EC2 / subnet # quotas. Order mirrors the legacy pipeline on main: # nodejs → java → dotnet → python → php → ruby. @@ -105,6 +110,7 @@ nodejs_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/' when: always + java_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -122,6 +128,7 @@ java_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/' when: always + dotnet_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -141,6 +148,7 @@ dotnet_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/' when: always + python_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -162,6 +170,7 @@ python_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/' when: always + php_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -185,6 +194,7 @@ php_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/' when: always + ruby_ssi_pipeline: stage: $[[ inputs.stage ]] needs: @@ -210,6 +220,7 @@ ruby_ssi_pipeline: - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/' - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/' when: always + delete_amis: extends: .base_job_onboarding stage: $[[ inputs.stage ]] @@ -225,6 +236,7 @@ delete_amis: - if: '$SCHEDULED_JOB == "delete_amis"' after_script: echo "Finish" timeout: 3h + delete_amis_by_name_or_lang: extends: .base_job_onboarding stage: $[[ inputs.stage ]] @@ -247,6 +259,7 @@ delete_amis_by_name_or_lang: rules: - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' after_script: echo "Finish" + delete_ec2_instances: extends: .base_job_onboarding stage: $[[ inputs.stage ]] @@ -260,6 +273,7 @@ delete_ec2_instances: rules: - if: '$SCHEDULED_JOB == "delete_ec2_instances"' after_script: echo "Finish" + count_amis: extends: .base_job_onboarding stage: $[[ inputs.stage ]] @@ -271,6 +285,7 @@ count_amis: rules: - if: '$SCHEDULED_JOB == "count_amis"' after_script: echo "Finish" + .delayed_base_job: image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 tags: ["arch:amd64"] diff --git a/utils/ci/gitlab/templates.yml b/utils/ci/gitlab/templates.yml index a6f79d372d1..2df0483be2e 100644 --- a/utils/ci/gitlab/templates.yml +++ b/utils/ci/gitlab/templates.yml @@ -4,7 +4,9 @@ spec: stage: ref: default: main + --- + .system_tests_base: image: $CI_IMAGE tags: From 367fd252a708966a9c9d6619c881ea6ef7be03b5 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 14:56:52 +0200 Subject: [PATCH 184/229] format: scope yamllint indentation disable to gitlab files only Replace the global 'indentation: disable' added in 612d10ef1 with: - A per-rule 'indentation.ignore' covering only .gitlab-ci.yml, which uses intentional mixed indentation per GitLab CI conventions. - A top-level 'ignore' block for files that cannot be linted by yamllint at all: templates.yml (contains ANSI escape codes for CI section folding), main.yml (embedded newlines in block scalars break YAML parsing), and system-tests.yml.j2 (Jinja2 template, not plain YAML). - ssi.yml passes yamllint clean with the indentation rule enabled. - manifests/ indentation coverage is fully restored. --- .yamllint | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.yamllint b/.yamllint index c9b27c90bf3..d2a26d47b2b 100644 --- a/.yamllint +++ b/.yamllint @@ -1,6 +1,17 @@ extends: relaxed +# utils/ci/gitlab/templates.yml and main.yml contain non-YAML syntax +# (ANSI escape codes and embedded newlines in block scalars) that yamllint +# cannot parse. system-tests.yml.j2 is a Jinja2 template, not plain YAML. +ignore: | + utils/ci/gitlab/templates.yml + utils/ci/gitlab/main.yml + utils/ci/gitlab/system-tests.yml.j2 + rules: line-length: disable key-ordering: disable - indentation: disable + indentation: + # .gitlab-ci.yml uses intentional mixed indentation (GitLab CI style). + ignore: | + .gitlab-ci.yml From 1f8eb811c242e8139950704eae60aa400e0f7202 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 15:02:37 +0200 Subject: [PATCH 185/229] format: lint utils/ci/gitlab YAML files in full Remove unjustified yamllint ignore for templates.yml/main.yml/system-tests.yml.j2: those files lint cleanly. Drop trailing blank line in main.yml flagged by the empty-lines rule. --- .yamllint | 8 -------- utils/ci/gitlab/main.yml | 1 - 2 files changed, 9 deletions(-) diff --git a/.yamllint b/.yamllint index d2a26d47b2b..0230c802427 100644 --- a/.yamllint +++ b/.yamllint @@ -1,13 +1,5 @@ extends: relaxed -# utils/ci/gitlab/templates.yml and main.yml contain non-YAML syntax -# (ANSI escape codes and embedded newlines in block scalars) that yamllint -# cannot parse. system-tests.yml.j2 is a Jinja2 template, not plain YAML. -ignore: | - utils/ci/gitlab/templates.yml - utils/ci/gitlab/main.yml - utils/ci/gitlab/system-tests.yml.j2 - rules: line-length: disable key-ordering: disable diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index d03266ef940..194526e5d9a 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -177,4 +177,3 @@ run_test_pipeline_2: - artifact: system-tests/generated-pipeline-chunk-2.yml job: build_test_pipeline strategy: depend - From 3497375796a709459ef499bb36887760c389548b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 11 Jun 2026 16:44:06 +0200 Subject: [PATCH 186/229] Cleanup --- format.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/format.sh b/format.sh index f5ecdf91fac..a548303ee1e 100755 --- a/format.sh +++ b/format.sh @@ -145,8 +145,6 @@ if ! which yamlfmt > /dev/null; then fi echo "Running yamlfmt formatter..." -# utils/ci/gitlab/ is excluded from yamlfmt: GitLab's $[[ ]] interpolation syntax -# is not valid YAML and causes yamlfmt to fail on those files. if [ "$COMMAND" == "fix" ]; then yamlfmt manifests/ else From b8ca1273342fe1d3abb8a2ea38395a70f195baad Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 12 Jun 2026 14:43:12 +0200 Subject: [PATCH 187/229] Replaced local import with project imports --- utils/ci/gitlab/main.yml | 8 ++++++-- utils/ci/gitlab/system-tests.yml.j2 | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 194526e5d9a..fca0b8f3177 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -57,11 +57,15 @@ spec: --- include: - - local: /utils/ci/gitlab/templates.yml + - project: 'DataDog/system-tests' + ref: $[[ inputs.ref ]] + file: 'utils/ci/gitlab/templates.yml' inputs: stage: $[[ inputs.stage ]] ref: $[[ inputs.ref ]] - - local: /utils/ci/gitlab/ssi.yml + - project: 'DataDog/system-tests' + ref: $[[ inputs.ref ]] + file: 'utils/ci/gitlab/ssi.yml' if: $[[ inputs.ssi_enabled ]] inputs: stage: $[[ inputs.stage ]] diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index f423adf9093..210d10ccb41 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -16,7 +16,9 @@ workflow: name: "System-tests end to end" include: - - local: /utils/ci/gitlab/templates.yml + - project: 'DataDog/system-tests' + ref: {{ref}} + file: 'utils/ci/gitlab/templates.yml' inputs: stage: {{stage}} ref: {{ref}} From 5c2b902ff756483a689d1166a8dbf296a858ed5e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 12 Jun 2026 15:11:11 +0200 Subject: [PATCH 188/229] Optional docker auth --- .gitlab-ci.yml | 1 + utils/ci/gitlab/build_pipeline.py | 6 ++++++ utils/ci/gitlab/main.yml | 5 ++++- utils/ci/gitlab/system-tests.yml.j2 | 4 ++++ 4 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f39684eaf3..66342d77732 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ include: excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true + docker_auth: true stages: - configure - e2e diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index d978e096f12..f2441943bfb 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -36,6 +36,7 @@ def render_library( ci_image: str, ref: str, push_to_test_optimization: bool, + docker_auth: bool, ) -> str: parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) @@ -58,6 +59,7 @@ def render_library( ref=ref, push_to_test_optimization=push_to_test_optimization, skip_header=skip_header, + docker_auth=docker_auth, ) @@ -71,6 +73,7 @@ def build( ref: str = "", push_to_test_optimization: bool = False, chunks: int = 3, + docker_auth: bool = False, ) -> None: """Render pipeline chunk files into *output_dir*, one per chunk.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -107,6 +110,7 @@ def build( ci_image=ci_image, ref=ref, push_to_test_optimization=push_to_test_optimization, + docker_auth=docker_auth, ) ) @@ -123,6 +127,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") parser.add_argument("--output-dir", required=True, help="Output directory for generated-pipeline-chunk-N.yml files") parser.add_argument("--chunks", type=int, default=3, help="Number of pipeline chunks (default: 3)") + parser.add_argument("--docker-auth", default="false", help="Wether to authenticate calls to docker hub") args = parser.parse_args(argv) @@ -140,6 +145,7 @@ def main(argv: list[str] | None = None) -> int: ref=args.ref, push_to_test_optimization=args.push_to_test_optimization == "true", chunks=args.chunks, + docker_auth=args.docker_auth == "true", ) return 0 diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index fca0b8f3177..35a56d03894 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -53,6 +53,9 @@ spec: ssi_injector_version: description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" default: "" + docker_auth: + description: "Whether to authenticate calls to docker hub" + default: "false" --- @@ -135,7 +138,7 @@ build_test_pipeline: for library in $libraries; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json done - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$[[ inputs.docker_auth ]]" artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 210d10ccb41..3e19fb0dd01 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -78,7 +78,9 @@ build_{{library}}_{{variant}}: artifacts: true {% endif %} script: + {% if docker_auth_enabled %} {{ docker_auth() }} + {% endif %} - section_start "build" "Building weblog" - ./build.sh {{library}} -i weblog --save-to-binaries --weblog-variant {{variant}} - mv binaries .. @@ -104,7 +106,9 @@ run_{{library}}_{{scenario}}_{{variant}}: artifacts: true {% endif %} script: + {% if docker_auth_enabled %} {{ docker_auth() }} + {% endif %} - section_start "weblog_setup" "Setting up the weblog" {% if build_required %} - mv ../binaries/* binaries/ From d137e09cfba65e09a7605f8aba1f30c94eb95cca Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 12 Jun 2026 15:25:25 +0200 Subject: [PATCH 189/229] ci: enable docker hub auth --- utils/ci/gitlab/build_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index f2441943bfb..7afe8ccea0b 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -59,7 +59,7 @@ def render_library( ref=ref, push_to_test_optimization=push_to_test_optimization, skip_header=skip_header, - docker_auth=docker_auth, + docker_auth_enabled=docker_auth, ) From 2f008ec8ef872384c7e762e8890ae4c8d61d052b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 17 Jun 2026 15:21:54 +0200 Subject: [PATCH 190/229] Docker hub image mirroring --- .gitlab-ci.yml | 64 ++++++++++++++++++++++ mirror_images.yaml | 11 ++++ utils/scripts/mirror_images_list.py | 84 +++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 mirror_images.yaml create mode 100644 utils/scripts/mirror_images_list.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66342d77732..f0dff06e6c5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,6 +24,10 @@ variables: SKIP_RESOLVE_CI_IMAGE: "true" # TEMPORARY: rollout-control allowlist for system-tests-param LIBRARIES filter. E2E_SUPPORTED_LANGUAGES: "python" + # dd-repo-tools mirror_images.py (pinned). Used by the mirror_images_* jobs. + MIRROR_IMAGES_URL: "https://binaries.ddbuild.io/dd-repo-tools/default/ca/fb4f39a542e4dd42b646c300b539c7a9f4201531/mirror_images.py" + # Destination registry for mirrored CI images. + MIRROR_DEST_REGISTRY: "registry.ddbuild.io/system-tests/mirror" compute_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 @@ -393,3 +397,63 @@ build_ci_image: artifacts: reports: dotenv: build.env + +# ────────────────────────────────────────────── +# Mirror CI scenario images into registry.ddbuild.io/system-tests/mirror +# +# Two-step flow: +# * mirror_images_check (every push) regenerates the image list from the +# scenarios and fails if the committed mirror_images.yaml is out of date. +# Regenerate + commit it whenever scenarios or weblog Dockerfiles change +# (see utils/scripts/mirror_images_list.py). +# * mirror_images_mirror (main only) resolves digests and pushes every listed +# image to MIRROR_DEST_REGISTRY. +# ────────────────────────────────────────────── + +mirror_images_check: + image: $CI_IMAGE + stage: system-tests-utils + tags: + - arch:amd64 + needs: + - job: build_ci_image + artifacts: true + script: + - ln -sf /system-tests/venv venv + - source venv/bin/activate + - export PYTHONPATH=$(pwd) + - python -m pip install --quiet uv + - python utils/scripts/mirror_images_list.py > ci_images.txt + - cat ci_images.txt + - uv run --no-config --script "$MIRROR_IMAGES_URL" add $(cat ci_images.txt) + - rm -f ci_images.txt + - | + if ! git diff --exit-code -- mirror_images.yaml; then + echo "❌ mirror_images.yaml is out of date. Regenerate and commit it:" + echo " python utils/scripts/mirror_images_list.py | xargs uv run --no-config --script \"\$MIRROR_IMAGES_URL\" add" + exit 1 + fi + echo "✅ mirror_images.yaml is up to date." + rules: + - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' + +mirror_images_mirror: + image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 + stage: system-tests-utils + tags: + - docker-in-docker:amd64 + needs: + - job: mirror_images_check + artifacts: false + before_script: + # crane handles multi-arch manifest lists cleanly; mirror_images.py auto-detects + # it on PATH (falling back to skopeo/docker/podman otherwise). + - CRANE_VERSION=v0.20.2 + - curl -sSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" | tar -xz -C /usr/local/bin crane || true + - curl -LsSf https://astral.sh/uv/install.sh | sh + - export PATH="$HOME/.local/bin:$PATH" + script: + - uv run --no-config --script "$MIRROR_IMAGES_URL" lock + - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror + rules: + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" diff --git a/mirror_images.yaml b/mirror_images.yaml new file mode 100644 index 00000000000..5244f2dc9a0 --- /dev/null +++ b/mirror_images.yaml @@ -0,0 +1,11 @@ +# Docker images mirrored into registry.ddbuild.io/system-tests/mirror. +# +# This file is generated: it lists every image required by the CI scenarios. +# Regenerate after changing scenarios or weblog Dockerfiles with: +# +# python utils/scripts/mirror_images_list.py | xargs \ +# uv run --no-config --script \ +# https://binaries.ddbuild.io/dd-repo-tools/default/ca/fb4f39a542e4dd42b646c300b539c7a9f4201531/mirror_images.py \ +# add +# +# The `mirror_images_check` CI job fails if this file is out of date. diff --git a/utils/scripts/mirror_images_list.py b/utils/scripts/mirror_images_list.py new file mode 100644 index 00000000000..136179f5d23 --- /dev/null +++ b/utils/scripts/mirror_images_list.py @@ -0,0 +1,84 @@ +"""Enumerate every Docker image required by the CI scenarios, for mirroring. + +Companion to ``utils/scripts/get-image-list.py``: where that script emits a +docker-compose file of images to pull for a *single* scenario/library/weblog +(and drops images that already exist in the local Docker daemon), this script +unions the images across *all* CI-run ``DockerScenario`` instances and the full +library x weblog matrix, with no dependency on a running Docker daemon. + +The output (one image reference per line) feeds the dd-repo-tools +``mirror_images.py add`` command, which records them in ``mirror_images.yaml``. + +Run from the repository root (paths are resolved relative to the cwd, like +``get-image-list.py``). +""" + +import argparse +from pathlib import Path + +from utils._context._scenarios import get_all_scenarios, DockerScenario + +# Scenarios excluded from the GitLab end-to-end pipeline. Mirrors the +# `excluded_scenarios` input in .gitlab-ci.yml so the mirrored image set matches +# what actually runs in CI. Override with --exclude when the CI list changes. +DEFAULT_EXCLUDED = ( + "DEBUGGER_EXPRESSION_LANGUAGE", + "APM_TRACING_E2E_SINGLE_SPAN", + "APM_TRACING_E2E_OTEL", + "OTEL_COLLECTOR_E2E", +) + + +def _library_weblog_pairs() -> list[tuple[str, str]]: + """Every (library, weblog) pair, derived from the weblog Dockerfiles. + + WeblogContainer.get_image_list reads utils/build/docker//.Dockerfile, + so the directory name is the library and the file stem is the weblog. The + empty pair captures library-independent images (agent, integrations, + buddies) that every get_image_list call returns regardless of arguments. + """ + pairs = {("", "")} + for path in Path().glob("utils/build/docker/*/*.Dockerfile"): + pairs.add((path.parent.name, path.stem)) + return sorted(pairs) + + +def _is_mirrorable(image: str) -> bool: + """Keep only images that can be pulled from a public registry and mirrored.""" + if image.startswith("system_tests/"): + return False # built locally, never pulled from a registry + if "$" in image: + return False # unresolved Dockerfile/template placeholder (e.g. ${NORMALIZED_BRANCH}) + # already on the destination registry => nothing to mirror + return not image.startswith("registry.ddbuild.io/") + + +def collect_images(excluded: set[str]) -> list[str]: + pairs = _library_weblog_pairs() + images: set[str] = set() + for scenario in get_all_scenarios(): + if not isinstance(scenario, DockerScenario) or scenario.name in excluded: + continue + for library, weblog in pairs: + images.update(scenario.get_image_list(library, weblog)) + return sorted(image for image in images if _is_mirrorable(image)) + + +def main(excluded: set[str]) -> None: + for image in collect_images(excluded): + print(image) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="mirror_images_list", + description="List every Docker image required by the CI scenarios (for mirror_images add)", + ) + parser.add_argument( + "--exclude", + type=str, + default=",".join(DEFAULT_EXCLUDED), + help="Comma-separated scenario names to exclude (default: the CI excluded_scenarios set)", + ) + args = parser.parse_args() + main({name.strip() for name in args.exclude.split(",") if name.strip()}) From 0437e20dab86584d2628d7578691f3b4eedb38da Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 17 Jun 2026 16:24:07 +0200 Subject: [PATCH 191/229] ci: single-script mirror image regeneration (yaml + lock, no push) Replace the piped `mirror_images_list.py | xargs ... add` workflow with a self-contained `python utils/scripts/update_mirror_images.py` that: * enumerates every image required by the CI scenarios, * records them in mirror_images.yaml (mirror_images.py `add`), * resolves digests into mirror_images.lock.yaml (mirror_images.py `lock`), * never pushes/mirrors anything (commit both files yourself). The mirror_images_check CI job runs it with --skip-lock (no registry creds needed) and fails on mirror_images.yaml drift. Restores the 93-image mirror_images.yaml that was dropped during the branch squash. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitlab-ci.yml | 12 +- mirror_images.yaml | 101 ++++++++++++++++- utils/scripts/mirror_images_list.py | 84 -------------- utils/scripts/update_mirror_images.py | 151 ++++++++++++++++++++++++++ 4 files changed, 252 insertions(+), 96 deletions(-) delete mode 100644 utils/scripts/mirror_images_list.py create mode 100644 utils/scripts/update_mirror_images.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0dff06e6c5..c92a00f4c6e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -405,7 +405,7 @@ build_ci_image: # * mirror_images_check (every push) regenerates the image list from the # scenarios and fails if the committed mirror_images.yaml is out of date. # Regenerate + commit it whenever scenarios or weblog Dockerfiles change -# (see utils/scripts/mirror_images_list.py). +# (see utils/scripts/update_mirror_images.py). # * mirror_images_mirror (main only) resolves digests and pushes every listed # image to MIRROR_DEST_REGISTRY. # ────────────────────────────────────────────── @@ -421,16 +421,14 @@ mirror_images_check: script: - ln -sf /system-tests/venv venv - source venv/bin/activate - - export PYTHONPATH=$(pwd) - python -m pip install --quiet uv - - python utils/scripts/mirror_images_list.py > ci_images.txt - - cat ci_images.txt - - uv run --no-config --script "$MIRROR_IMAGES_URL" add $(cat ci_images.txt) - - rm -f ci_images.txt + # --skip-lock: the drift check only validates the image list; the lock file + # is regenerated/committed by developers (and refreshed by mirror_images_mirror). + - python utils/scripts/update_mirror_images.py --skip-lock - | if ! git diff --exit-code -- mirror_images.yaml; then echo "❌ mirror_images.yaml is out of date. Regenerate and commit it:" - echo " python utils/scripts/mirror_images_list.py | xargs uv run --no-config --script \"\$MIRROR_IMAGES_URL\" add" + echo " python utils/scripts/update_mirror_images.py" exit 1 fi echo "✅ mirror_images.yaml is up to date." diff --git a/mirror_images.yaml b/mirror_images.yaml index 5244f2dc9a0..b9f0a4dc483 100644 --- a/mirror_images.yaml +++ b/mirror_images.yaml @@ -3,9 +3,100 @@ # This file is generated: it lists every image required by the CI scenarios. # Regenerate after changing scenarios or weblog Dockerfiles with: # -# python utils/scripts/mirror_images_list.py | xargs \ -# uv run --no-config --script \ -# https://binaries.ddbuild.io/dd-repo-tools/default/ca/fb4f39a542e4dd42b646c300b539c7a9f4201531/mirror_images.py \ -# add +# python utils/scripts/update_mirror_images.py # -# The `mirror_images_check` CI job fails if this file is out of date. +# (also refreshes mirror_images.lock.yaml; commit both). The +# `mirror_images_check` CI job fails if this file is out of date. +- "apache/kafka:3.7.1" +- "cassandra:latest" +- "datadog/agent:latest" +- "datadog/dd-appsec-php-ci:php-8.0-release" +- "datadog/system-tests:apache-mod-7.0-zts.base-v1" +- "datadog/system-tests:apache-mod-7.0.base-v1" +- "datadog/system-tests:apache-mod-7.1-zts.base-v1" +- "datadog/system-tests:apache-mod-7.1.base-v1" +- "datadog/system-tests:apache-mod-7.2-zts.base-v1" +- "datadog/system-tests:apache-mod-7.2.base-v1" +- "datadog/system-tests:apache-mod-7.3-zts.base-v1" +- "datadog/system-tests:apache-mod-7.3.base-v1" +- "datadog/system-tests:apache-mod-7.4-zts.base-v1" +- "datadog/system-tests:apache-mod-7.4.base-v1" +- "datadog/system-tests:apache-mod-8.0-zts.base-v1" +- "datadog/system-tests:apache-mod-8.0.base-v1" +- "datadog/system-tests:apache-mod-8.1-zts.base-v1" +- "datadog/system-tests:apache-mod-8.1.base-v1" +- "datadog/system-tests:apache-mod-8.2-zts.base-v1" +- "datadog/system-tests:apache-mod-8.2.base-v1" +- "datadog/system-tests:django-poc.base-v11" +- "datadog/system-tests:django-py3.13.base-v10" +- "datadog/system-tests:express4-typescript.base-v1" +- "datadog/system-tests:express4.base-v1" +- "datadog/system-tests:express5.base-v1" +- "datadog/system-tests:fastapi.base-v9" +- "datadog/system-tests:fastify.base-v1" +- "datadog/system-tests:flask-poc.base-v1" +- "datadog/system-tests:flask-poc.base-v13" +- "datadog/system-tests:golang_buddy-v2" +- "datadog/system-tests:java_buddy-v1" +- "datadog/system-tests:lambda-proxy-v1" +- "datadog/system-tests:nextjs.base-v1" +- "datadog/system-tests:nodejs_buddy-v1" +- "datadog/system-tests:php-fpm-7.0.base-v1" +- "datadog/system-tests:php-fpm-7.1.base-v1" +- "datadog/system-tests:php-fpm-7.2.base-v1" +- "datadog/system-tests:php-fpm-7.3.base-v1" +- "datadog/system-tests:php-fpm-7.4.base-v1" +- "datadog/system-tests:php-fpm-8.0.base-v1" +- "datadog/system-tests:php-fpm-8.1.base-v1" +- "datadog/system-tests:php-fpm-8.2.base-v1" +- "datadog/system-tests:php-fpm-8.5.base-v1" +- "datadog/system-tests:proxy-v1" +- "datadog/system-tests:python3.12.base-v13" +- "datadog/system-tests:python_buddy-v2" +- "datadog/system-tests:ruby_buddy-v2" +- "datadog/system-tests:tornado.base-v2" +- "datadog/system-tests:uwsgi-poc.base-v9" +- "debian:bookworm-slim" +- "debian:stable-slim" +- "demisto/fastapi:0.116.1.4266494" +- "eclipse-temurin:11-jre" +- "eclipse-temurin:17-jre" +- "ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.59.0" +- "ghcr.io/datadog/images-rb/engines/ruby:2.5" +- "ghcr.io/datadog/images-rb/engines/ruby:2.7" +- "ghcr.io/datadog/images-rb/engines/ruby:3.0" +- "ghcr.io/datadog/images-rb/engines/ruby:3.1" +- "ghcr.io/datadog/images-rb/engines/ruby:3.4" +- "ghcr.io/graalvm/native-image-community:22.0.0" +- "golang:1.25" +- "golang:1.25-alpine" +- "kong/kong-gateway:3.4" +- "localstack/localstack:4.1" +- "maven:3.6-jdk-11" +- "maven:3.8-jdk-11" +- "maven:3.9-eclipse-temurin-11" +- "maven:3.9-eclipse-temurin-17" +- "maven:3.9.14-eclipse-temurin-17" +- "maven:3.9.9-eclipse-temurin-17-focal" +- "mcr.microsoft.com/dotnet/aspnet:8.0" +- "mcr.microsoft.com/dotnet/sdk:8.0" +- "mcr.microsoft.com/mssql/server:2022-latest" +- "mongo:latest" +- "mysql/mysql-server:8.0.32" +- "node:18-alpine" +- "node:20-alpine" +- "node:22-alpine" +- "otel/opentelemetry-collector-contrib:0.137.0" +- "postgres:alpine" +- "public.ecr.aws/lambda/java:17" +- "public.ecr.aws/lambda/nodejs:18" +- "public.ecr.aws/lambda/python:3.13" +- "public.ecr.aws/lambda/ruby:3.4" +- "python:3.11-slim" +- "python:3.12-slim" +- "python:3.13-slim" +- "python:3.14-slim" +- "rabbitmq:3.12-management-alpine" +- "rockylinux:9" +- "softwaremill/elasticmq-native:1.6.11" +- "ubuntu:24.04" diff --git a/utils/scripts/mirror_images_list.py b/utils/scripts/mirror_images_list.py deleted file mode 100644 index 136179f5d23..00000000000 --- a/utils/scripts/mirror_images_list.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Enumerate every Docker image required by the CI scenarios, for mirroring. - -Companion to ``utils/scripts/get-image-list.py``: where that script emits a -docker-compose file of images to pull for a *single* scenario/library/weblog -(and drops images that already exist in the local Docker daemon), this script -unions the images across *all* CI-run ``DockerScenario`` instances and the full -library x weblog matrix, with no dependency on a running Docker daemon. - -The output (one image reference per line) feeds the dd-repo-tools -``mirror_images.py add`` command, which records them in ``mirror_images.yaml``. - -Run from the repository root (paths are resolved relative to the cwd, like -``get-image-list.py``). -""" - -import argparse -from pathlib import Path - -from utils._context._scenarios import get_all_scenarios, DockerScenario - -# Scenarios excluded from the GitLab end-to-end pipeline. Mirrors the -# `excluded_scenarios` input in .gitlab-ci.yml so the mirrored image set matches -# what actually runs in CI. Override with --exclude when the CI list changes. -DEFAULT_EXCLUDED = ( - "DEBUGGER_EXPRESSION_LANGUAGE", - "APM_TRACING_E2E_SINGLE_SPAN", - "APM_TRACING_E2E_OTEL", - "OTEL_COLLECTOR_E2E", -) - - -def _library_weblog_pairs() -> list[tuple[str, str]]: - """Every (library, weblog) pair, derived from the weblog Dockerfiles. - - WeblogContainer.get_image_list reads utils/build/docker//.Dockerfile, - so the directory name is the library and the file stem is the weblog. The - empty pair captures library-independent images (agent, integrations, - buddies) that every get_image_list call returns regardless of arguments. - """ - pairs = {("", "")} - for path in Path().glob("utils/build/docker/*/*.Dockerfile"): - pairs.add((path.parent.name, path.stem)) - return sorted(pairs) - - -def _is_mirrorable(image: str) -> bool: - """Keep only images that can be pulled from a public registry and mirrored.""" - if image.startswith("system_tests/"): - return False # built locally, never pulled from a registry - if "$" in image: - return False # unresolved Dockerfile/template placeholder (e.g. ${NORMALIZED_BRANCH}) - # already on the destination registry => nothing to mirror - return not image.startswith("registry.ddbuild.io/") - - -def collect_images(excluded: set[str]) -> list[str]: - pairs = _library_weblog_pairs() - images: set[str] = set() - for scenario in get_all_scenarios(): - if not isinstance(scenario, DockerScenario) or scenario.name in excluded: - continue - for library, weblog in pairs: - images.update(scenario.get_image_list(library, weblog)) - return sorted(image for image in images if _is_mirrorable(image)) - - -def main(excluded: set[str]) -> None: - for image in collect_images(excluded): - print(image) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="mirror_images_list", - description="List every Docker image required by the CI scenarios (for mirror_images add)", - ) - parser.add_argument( - "--exclude", - type=str, - default=",".join(DEFAULT_EXCLUDED), - help="Comma-separated scenario names to exclude (default: the CI excluded_scenarios set)", - ) - args = parser.parse_args() - main({name.strip() for name in args.exclude.split(",") if name.strip()}) diff --git a/utils/scripts/update_mirror_images.py b/utils/scripts/update_mirror_images.py new file mode 100644 index 00000000000..65f45bcfc30 --- /dev/null +++ b/utils/scripts/update_mirror_images.py @@ -0,0 +1,151 @@ +"""Regenerate the mirror-images manifest and lock file from the CI scenarios. + +Run from a system-tests checkout, with the runner venv active: + + python utils/scripts/update_mirror_images.py + +It unions every Docker image required by the CI-run ``DockerScenario`` instances +(across the full library x weblog matrix), records them in ``mirror_images.yaml`` +via the dd-repo-tools ``mirror_images.py add`` command, then resolves digests +into ``mirror_images.lock.yaml`` via ``lock``. + +It never pushes or mirrors anything: commit the updated ``mirror_images.yaml`` +and ``mirror_images.lock.yaml`` yourself. + +Companion to ``utils/scripts/get-image-list.py`` (which emits a docker-compose +file of images to pull for a *single* scenario/library/weblog); this script is +docker-free for the enumeration step and covers *all* CI scenarios at once. +``lock`` does require a registry tool (crane/skopeo/docker) and network access. +""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# Make `utils` importable and resolve paths regardless of the caller's cwd, so a +# plain `python utils/scripts/update_mirror_images.py` works. +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) + +from utils._context._scenarios import get_all_scenarios, DockerScenario # noqa: E402 + +# Pinned dd-repo-tools mirror_images.py (override with $MIRROR_IMAGES_URL). +MIRROR_IMAGES_URL = os.environ.get( + "MIRROR_IMAGES_URL", + "https://binaries.ddbuild.io/dd-repo-tools/default/ca/fb4f39a542e4dd42b646c300b539c7a9f4201531/mirror_images.py", +) +# Destination registry for the mirrored images (override with $MIRROR_DEST_REGISTRY). +DEFAULT_DEST_REGISTRY = "registry.ddbuild.io/system-tests/mirror" + +MIRROR_YAML = REPO_ROOT / "mirror_images.yaml" + +# Header written when mirror_images.yaml does not exist yet. The mirror_images.py +# `add` command preserves existing comments, so this is only used on first run. +MIRROR_YAML_HEADER = """\ +# Docker images mirrored into registry.ddbuild.io/system-tests/mirror. +# +# Generated: this file lists every image required by the CI scenarios. +# Regenerate after changing scenarios or weblog Dockerfiles with: +# +# python utils/scripts/update_mirror_images.py +# +# The `mirror_images_check` CI job fails if this file is out of date. +""" + +# Scenarios excluded from the GitLab end-to-end pipeline. Mirrors the +# `excluded_scenarios` input in .gitlab-ci.yml so the mirrored image set matches +# what actually runs in CI. Override with --exclude when the CI list changes. +DEFAULT_EXCLUDED = ( + "DEBUGGER_EXPRESSION_LANGUAGE", + "APM_TRACING_E2E_SINGLE_SPAN", + "APM_TRACING_E2E_OTEL", + "OTEL_COLLECTOR_E2E", +) + + +def _library_weblog_pairs() -> list[tuple[str, str]]: + """Every (library, weblog) pair, derived from the weblog Dockerfiles. + + WeblogContainer.get_image_list reads utils/build/docker//.Dockerfile, + so the directory name is the library and the file stem is the weblog. The + empty pair captures library-independent images (agent, integrations, + buddies) that every get_image_list call returns regardless of arguments. + """ + pairs = {("", "")} + for path in Path("utils/build/docker").glob("*/*.Dockerfile"): + pairs.add((path.parent.name, path.stem)) + return sorted(pairs) + + +def _is_mirrorable(image: str) -> bool: + """Keep only images that can be pulled from a public registry and mirrored.""" + if image.startswith("system_tests/"): + return False # built locally, never pulled from a registry + if "$" in image: + return False # unresolved Dockerfile/template placeholder (e.g. ${NORMALIZED_BRANCH}) + # already on the destination registry => nothing to mirror + return not image.startswith("registry.ddbuild.io/") + + +def collect_images(excluded: set[str]) -> list[str]: + pairs = _library_weblog_pairs() + images: set[str] = set() + for scenario in get_all_scenarios(): + if not isinstance(scenario, DockerScenario) or scenario.name in excluded: + continue + for library, weblog in pairs: + images.update(scenario.get_image_list(library, weblog)) + return sorted(image for image in images if _is_mirrorable(image)) + + +def _run_mirror_images(*args: str) -> None: + """Invoke the pinned dd-repo-tools mirror_images.py via uv.""" + if shutil.which("uv") is None: + sys.exit("error: 'uv' is required to run mirror_images.py — install it from https://docs.astral.sh/uv/") + env = dict(os.environ) + env.setdefault("MIRROR_DEST_REGISTRY", DEFAULT_DEST_REGISTRY) + cmd = [ + "uv", "run", "--no-config", "--script", MIRROR_IMAGES_URL, + "--mirror-yaml", str(MIRROR_YAML), *args, + ] + print(f"+ {' '.join(cmd)}", flush=True) + subprocess.run(cmd, check=True, env=env) + + +def main(excluded: set[str], *, skip_lock: bool) -> None: + # get_image_list reads Dockerfiles relative to the repo root. + os.chdir(REPO_ROOT) + + images = collect_images(excluded) + print(f"Collected {len(images)} mirrorable image(s) from the CI scenarios.", flush=True) + + if not MIRROR_YAML.exists(): + MIRROR_YAML.write_text(MIRROR_YAML_HEADER) + + _run_mirror_images("add", *images) + if not skip_lock: + _run_mirror_images("lock") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="update_mirror_images", + description="Regenerate mirror_images.yaml and mirror_images.lock.yaml from the CI scenarios", + ) + parser.add_argument( + "--exclude", + type=str, + default=",".join(DEFAULT_EXCLUDED), + help="Comma-separated scenario names to exclude (default: the CI excluded_scenarios set)", + ) + parser.add_argument( + "--skip-lock", + action="store_true", + help="Only update mirror_images.yaml; do not resolve digests into the lock file " + "(used by the CI drift check, which has no registry credentials)", + ) + args = parser.parse_args() + main({name.strip() for name in args.exclude.split(",") if name.strip()}, skip_lock=args.skip_lock) From d2f2e3a1d9473c0108644be13b982fd71e8e9948 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 17 Jun 2026 16:32:20 +0200 Subject: [PATCH 192/229] adding image lock file --- mirror_images.lock.yaml | 376 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 376 insertions(+) create mode 100644 mirror_images.lock.yaml diff --git a/mirror_images.lock.yaml b/mirror_images.lock.yaml new file mode 100644 index 00000000000..1c53d5e9ce3 --- /dev/null +++ b/mirror_images.lock.yaml @@ -0,0 +1,376 @@ +# Auto-generated by: uv run bin/mirror_images.py lock +# Do not edit manually. +version: 1 +images: + apache/kafka:3.7.1: + digest: sha256:ed74d7d115968d5e8b00ba6822ac6a384cbaaf54ca38991828647000d7089b68 + target: registry.ddbuild.io/system-tests/mirror/apache/kafka:3.7.1 + tag: 3.7.1 + cassandra:latest: + digest: sha256:755db0086160b6898e890959343dec1544ff27e7fa8ecc955c2f4b28c40a2f1a + target: registry.ddbuild.io/system-tests/mirror/cassandra:latest + tag: 5.0.8 + datadog/agent:latest: + digest: sha256:2a798352bb00af4f64908809af46bbd211e1f087e6509e8ca0722fba45385c01 + target: registry.ddbuild.io/system-tests/mirror/datadog/agent:latest + tag: 7.80.1 + datadog/dd-appsec-php-ci:php-8.0-release: + digest: sha256:f6c7557c63d9da34d081bc681768798bc4f70d8be36c565c526e57cecd6c2249 + target: registry.ddbuild.io/system-tests/mirror/datadog/dd-appsec-php-ci:php-8.0-release + tag: php-8.0-release + datadog/system-tests:apache-mod-7.0-zts.base-v1: + digest: sha256:bafc072734341cafb86f45673df2de4851a04b951d0717c88f7958cac1815b20 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.0-zts.base-v1 + tag: apache-mod-7.0-zts.base-v1 + datadog/system-tests:apache-mod-7.0.base-v1: + digest: sha256:af5081af161f4e12645ccdc4717ffc79dbb010f46c8978a3c12331d504fd74cd + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.0.base-v1 + tag: apache-mod-7.0.base-v1 + datadog/system-tests:apache-mod-7.1-zts.base-v1: + digest: sha256:5aa5c49cd2a90fdd1b5e9291d7825de083d7f258bab9dbd0ebc2770cd311729b + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.1-zts.base-v1 + tag: apache-mod-7.1-zts.base-v1 + datadog/system-tests:apache-mod-7.1.base-v1: + digest: sha256:e3c1334ac870ceef365f6f6d870c91306aeaab27b7b164e98e9f685f96cdcaed + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.1.base-v1 + tag: apache-mod-7.1.base-v1 + datadog/system-tests:apache-mod-7.2-zts.base-v1: + digest: sha256:965f5684c0fbb633660d672603e2dfa7c1d9d11e44cd3a0a135240be4733d8d4 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.2-zts.base-v1 + tag: apache-mod-7.2-zts.base-v1 + datadog/system-tests:apache-mod-7.2.base-v1: + digest: sha256:f237f0269089c99af3bc0f125439f08a555925d78e1d03e96baa9d82e8308e54 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.2.base-v1 + tag: apache-mod-7.2.base-v1 + datadog/system-tests:apache-mod-7.3-zts.base-v1: + digest: sha256:cbb5d94151a26d44f9b0b7de3790161c0d39e7c0324fec6c2217495214166d2f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.3-zts.base-v1 + tag: apache-mod-7.3-zts.base-v1 + datadog/system-tests:apache-mod-7.3.base-v1: + digest: sha256:6d6cd038caa41a456a5918f258658a4d940f2473891fc17f478bc250a43b613b + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.3.base-v1 + tag: apache-mod-7.3.base-v1 + datadog/system-tests:apache-mod-7.4-zts.base-v1: + digest: sha256:0568dcc42a70b6f2aa142d32d8a3e5c0362c7f76149dfcdeaac1514c77fc4ce6 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.4-zts.base-v1 + tag: apache-mod-7.4-zts.base-v1 + datadog/system-tests:apache-mod-7.4.base-v1: + digest: sha256:007b8604e30cc4568e704b51d54e85a9e873bb17eac105b1abf690cab5888db5 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.4.base-v1 + tag: apache-mod-7.4.base-v1 + datadog/system-tests:apache-mod-8.0-zts.base-v1: + digest: sha256:08b761c6c942ab12731383fe84865efd2d15422dd8a23fe4d96d7454ea79e580 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.0-zts.base-v1 + tag: apache-mod-8.0-zts.base-v1 + datadog/system-tests:apache-mod-8.0.base-v1: + digest: sha256:8faf421f570e77665eaeccaac581f03e3144fbd4689e59aeca3bbeaad31760e3 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.0.base-v1 + tag: apache-mod-8.0.base-v1 + datadog/system-tests:apache-mod-8.1-zts.base-v1: + digest: sha256:13fed1524e18c20e0dac2546b44fd024bf9d20e1d8521a62d21fe9dc27877107 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.1-zts.base-v1 + tag: apache-mod-8.1-zts.base-v1 + datadog/system-tests:apache-mod-8.1.base-v1: + digest: sha256:79de2dcdc70282f2ec0641f8633a183985ab40bf9b6f0a8d6a51084527747e39 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.1.base-v1 + tag: apache-mod-8.1.base-v1 + datadog/system-tests:apache-mod-8.2-zts.base-v1: + digest: sha256:1d2d543af606021af9e80e6627de41928739a4a27346b60e0b806f31a9f694f5 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.2-zts.base-v1 + tag: apache-mod-8.2-zts.base-v1 + datadog/system-tests:apache-mod-8.2.base-v1: + digest: sha256:875f87c0c31bd65316c79e2904da03fb8d0089403fc8631f9244217ad02c8bc2 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.2.base-v1 + tag: apache-mod-8.2.base-v1 + datadog/system-tests:django-poc.base-v11: + digest: sha256:bc6089dc235abdb972965d004dc409c78636ba0b1e0f30b485057e10f2628e3b + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:django-poc.base-v11 + tag: django-poc.base-v11 + datadog/system-tests:django-py3.13.base-v10: + digest: sha256:3249e0d5df7b8c88f95a4dec4aee196ade0ac01b3f263a7162993c9a00ec892c + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:django-py3.13.base-v10 + tag: django-py3.13.base-v10 + datadog/system-tests:express4-typescript.base-v1: + digest: sha256:be33d685d24e57f478ece1258c722822c8fd496aef29abe409e1f9bbd63bb8a3 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4-typescript.base-v1 + tag: express4-typescript.base-v1 + datadog/system-tests:express4.base-v1: + digest: sha256:a4cf4545f6cf3e47dfdda1b9f33a98bd74ebdd7eb527ee53d6e513f93348290b + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4.base-v1 + tag: express4.base-v1 + datadog/system-tests:express5.base-v1: + digest: sha256:826625f5efc737fdee8d52e68fae6f5d3c8c70f18ed22e232a1762b5e74fa09f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express5.base-v1 + tag: express5.base-v1 + datadog/system-tests:fastapi.base-v9: + digest: sha256:c9cede8431eb9dd23e785c5740acf0f0a28c44cf0684e712b6ec0b87ed9e8e0f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastapi.base-v9 + tag: fastapi.base-v9 + datadog/system-tests:fastify.base-v1: + digest: sha256:6eeb9581247f14c6f033052779a8e3f7d56f82393d8a288434f4537789510e6f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastify.base-v1 + tag: fastify.base-v1 + datadog/system-tests:flask-poc.base-v1: + digest: sha256:c2985235013e3c83d498f55f4d8b39fc2121a1acc09c2b740ee07e6bdae6852e + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:flask-poc.base-v1 + tag: flask-poc.base-v1 + datadog/system-tests:flask-poc.base-v13: + digest: sha256:80a66d234e1545219b0418bd44d0ab30b7ed7ca8c81bbd6e488c8fbcd5902d44 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:flask-poc.base-v13 + tag: flask-poc.base-v13 + datadog/system-tests:golang_buddy-v2: + digest: sha256:b567fa2e328f24c74aab600e8bf07362b4265ac508413a83e06afddb399bee80 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:golang_buddy-v2 + tag: golang_buddy-v2 + datadog/system-tests:java_buddy-v1: + digest: sha256:67aac2b7c0c88b40da498fbc74b48f0a62702148e5c9256c5752a9e624d972ce + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:java_buddy-v1 + tag: java_buddy-v1 + datadog/system-tests:lambda-proxy-v1: + digest: sha256:2b09f33c61b2b19f4a239e4d8f10164ece66d925b72337720998ce43d7090d2f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:lambda-proxy-v1 + tag: lambda-proxy-v1 + datadog/system-tests:nextjs.base-v1: + digest: sha256:8500fa38a95c7dc7d97fd270f56331048d98c4e4181df079e32eadd03e9554e0 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:nextjs.base-v1 + tag: nextjs.base-v1 + datadog/system-tests:nodejs_buddy-v1: + digest: sha256:71d124f74d4d7ac706f1afa8d5950864e1e931ec3b511f540b49eb6328c0338e + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:nodejs_buddy-v1 + tag: nodejs_buddy-v1 + datadog/system-tests:php-fpm-7.0.base-v1: + digest: sha256:ca3e2cd0e0e1c39c5008213bac2f4130b43499eb5f6392173c02791e1cbbba95 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.0.base-v1 + tag: php-fpm-7.0.base-v1 + datadog/system-tests:php-fpm-7.1.base-v1: + digest: sha256:8efdc58bed6abc510b5b86c92f01fb1c361e9c2a0ceaa11de518ac4cbcbc9b49 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.1.base-v1 + tag: php-fpm-7.1.base-v1 + datadog/system-tests:php-fpm-7.2.base-v1: + digest: sha256:b0d8cdd1d6fcddcb6e67b5006f08b9c176196ad02ecebab97163b26b313d75ec + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.2.base-v1 + tag: php-fpm-7.2.base-v1 + datadog/system-tests:php-fpm-7.3.base-v1: + digest: sha256:a2f396b7f1a4a74212c899c5782a5cbef48e3d581cccaede210b4088b1db2e29 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.3.base-v1 + tag: php-fpm-7.3.base-v1 + datadog/system-tests:php-fpm-7.4.base-v1: + digest: sha256:65b4f41685a53e04ea090b5a5d6d36ed439f62806340dd2fd402d932556c0a00 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.4.base-v1 + tag: php-fpm-7.4.base-v1 + datadog/system-tests:php-fpm-8.0.base-v1: + digest: sha256:71cc5b9746c6271641ee7f56408476bb5457277e5d24d4c646f6b41807f5eca6 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-8.0.base-v1 + tag: php-fpm-8.0.base-v1 + datadog/system-tests:php-fpm-8.1.base-v1: + digest: sha256:abc6845df939364b5179c7c513366462426a75e30e7a69fe4d91ce860e7e2411 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-8.1.base-v1 + tag: php-fpm-8.1.base-v1 + datadog/system-tests:php-fpm-8.2.base-v1: + digest: sha256:c716acd35cf91cb8b1c646047372c85868657d29576aaa1fdfb43c754a4bea84 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-8.2.base-v1 + tag: php-fpm-8.2.base-v1 + datadog/system-tests:php-fpm-8.5.base-v1: + digest: sha256:d5ffab83a326dcd2f2d106949a3ff63a290fe396e3753cad221a5cb5d3ea4fc7 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-8.5.base-v1 + tag: php-fpm-8.5.base-v1 + datadog/system-tests:proxy-v1: + digest: sha256:02ce356eafacfe04d3022ff505e2dbb9e47cd3c55244b7918ff6fc824db85f02 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:proxy-v1 + tag: proxy-v1 + datadog/system-tests:python3.12.base-v13: + digest: sha256:89bfdf492c920e5653ca8099301d6710d0bbdf7dc54129d788cbfad5aafa46da + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:python3.12.base-v13 + tag: python3.12.base-v13 + datadog/system-tests:python_buddy-v2: + digest: sha256:abbd02d4495d83c66af12b5562ccca0077800cb920142375cd48dcfc8aeb1596 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:python_buddy-v2 + tag: python_buddy-v2 + datadog/system-tests:ruby_buddy-v2: + digest: sha256:939d3828d85666a598c0349a6dd933fe1d92c5daa5d01438512d950ea3ccc258 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:ruby_buddy-v2 + tag: ruby_buddy-v2 + datadog/system-tests:tornado.base-v2: + digest: sha256:cdea08fb0841b1c0b8ce632490123fafff8a18e62be4c60fb4882e41d0c9a2f0 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:tornado.base-v2 + tag: tornado.base-v2 + datadog/system-tests:uwsgi-poc.base-v9: + digest: sha256:7c8fe35e132b686b584c1f06bdc296502e0ae3c4cdac1d77e1f6fbc4226f8d03 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:uwsgi-poc.base-v9 + tag: uwsgi-poc.base-v9 + debian:bookworm-slim: + digest: sha256:96e378d7e6531ac9a15ad505478fcc2e69f371b10f5cdf87857c4b8188404716 + target: registry.ddbuild.io/system-tests/mirror/debian:bookworm-slim + tag: bookworm-slim + debian:stable-slim: + digest: sha256:34363c20bd149e41365fc77b086da067ed13ab2dff4cd0612788e12e6d52c44c + target: registry.ddbuild.io/system-tests/mirror/debian:stable-slim + tag: stable-slim + demisto/fastapi:0.116.1.4266494: + digest: sha256:057fb83c60e7685a35d99229ece21c4cdcc962adfdc50251bf829b5694a95722 + target: registry.ddbuild.io/system-tests/mirror/demisto/fastapi:0.116.1.4266494 + tag: 0.116.1.4266494 + eclipse-temurin:11-jre: + digest: sha256:c356cb1702bbc9f614d52f953a1853fc7f0b49e3e62ab65927108e5bb17d9292 + target: registry.ddbuild.io/system-tests/mirror/eclipse-temurin:11-jre + tag: 11-jre + eclipse-temurin:17-jre: + digest: sha256:8016378253033ebd971c237f6580f108b4afac11a188d3fc3b982497bbe55c65 + target: registry.ddbuild.io/system-tests/mirror/eclipse-temurin:17-jre + tag: 17-jre + ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.59.0: + digest: sha256:cf8c7e7c0d6a53d69e23fdf0d7a6d900d7b6cfb104fae4756805016b2c8bc1c5 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.59.0 + tag: v1.59.0 + ghcr.io/datadog/images-rb/engines/ruby:2.5: + digest: sha256:80a31383585fff1a3c0d33c149171460bca182deddd50f2ed8f54217252f5178 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:2.5 + tag: '2.5' + ghcr.io/datadog/images-rb/engines/ruby:2.7: + digest: sha256:7c0f84d46c8fd5982abd50243c6b6fa53ce86a8978d4b1ae63b537698acf3755 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:2.7 + tag: '2.7' + ghcr.io/datadog/images-rb/engines/ruby:3.0: + digest: sha256:69d8af29e1105110d553e0597a8dee886ce2b1a7a43dedee622434365eb711db + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:3.0 + tag: '3.0' + ghcr.io/datadog/images-rb/engines/ruby:3.1: + digest: sha256:87f74691edd471099442996bfeab1d70804a3b74ea9a6e2ccea9f7500aff92ec + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:3.1 + tag: '3.1' + ghcr.io/datadog/images-rb/engines/ruby:3.4: + digest: sha256:2d22ecb3e636e12c397c872ff147cbd7ce67a8ab70a39caa1a2b520812ccccd2 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:3.4 + tag: '3.4' + ghcr.io/graalvm/native-image-community:22.0.0: + digest: sha256:0e7b3ee194aadc80845a29cd4e6501b9ed4422e6b998267926df31d63caf4770 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/graalvm/native-image-community:22.0.0 + tag: 22.0.0 + golang:1.25: + digest: sha256:00feed335fe561979f2cdcc30a5191231977c5631fe79c40f7d3ab63b4fa222f + target: registry.ddbuild.io/system-tests/mirror/golang:1.25 + tag: '1.25' + golang:1.25-alpine: + digest: sha256:89f71d90dff0d7f30316963b3c3b8bfe5fb96b94641b3258963ce0c7a21dedda + target: registry.ddbuild.io/system-tests/mirror/golang:1.25-alpine + tag: 1.25-alpine + kong/kong-gateway:3.4: + digest: sha256:195b1dedb78d9fa4976d089af9cc2573f6bb667b2b4be40e50d9e8cef85b4b58 + target: registry.ddbuild.io/system-tests/mirror/kong/kong-gateway:3.4 + tag: '3.4' + localstack/localstack:4.1: + digest: sha256:97ccc65daec3542bd2cb3160d7355f11e89ad8027fd3c834b9d1197d197d866f + target: registry.ddbuild.io/system-tests/mirror/localstack/localstack:4.1 + tag: '4.1' + maven:3.6-jdk-11: + digest: sha256:1d29ccf46ef2a5e64f7de3d79a63f9bcffb4dc56be0ae3daed5ca5542b38aa2d + target: registry.ddbuild.io/system-tests/mirror/maven:3.6-jdk-11 + tag: 3.6-jdk-11 + maven:3.8-jdk-11: + digest: sha256:805f366910aea2a91ed263654d23df58bd239f218b2f9562ff51305be81fa215 + target: registry.ddbuild.io/system-tests/mirror/maven:3.8-jdk-11 + tag: 3.8-jdk-11 + maven:3.9-eclipse-temurin-11: + digest: sha256:f580cae20128b75c984853630c054baf486a8c2f345a7d47692ca8b3b5b775d9 + target: registry.ddbuild.io/system-tests/mirror/maven:3.9-eclipse-temurin-11 + tag: 3.9-eclipse-temurin-11 + maven:3.9-eclipse-temurin-17: + digest: sha256:e8ef73dbd33b69fe497fd96b3bbbd85aff84ac4c564a5784ab02ad941b32c12b + target: registry.ddbuild.io/system-tests/mirror/maven:3.9-eclipse-temurin-17 + tag: 3.9-eclipse-temurin-17 + maven:3.9.14-eclipse-temurin-17: + digest: sha256:674de0df5a38cfe793a587829a181835f8af3ea1f556be7172e0aa547274bcc9 + target: registry.ddbuild.io/system-tests/mirror/maven:3.9.14-eclipse-temurin-17 + tag: 3.9.14-eclipse-temurin-17 + maven:3.9.9-eclipse-temurin-17-focal: + digest: sha256:45ddc84a2e7f75ab7faf290085a0292dd8adc5e7674a89231f6a879b60f4358d + target: registry.ddbuild.io/system-tests/mirror/maven:3.9.9-eclipse-temurin-17-focal + tag: 3.9.9-eclipse-temurin-17-focal + mcr.microsoft.com/dotnet/aspnet:8.0: + digest: sha256:93b366e510c6cd01cee608447014f7d349cb7ff8809fd0f554aa3772e8587b7e + target: registry.ddbuild.io/system-tests/mirror/mcr.microsoft.com/dotnet/aspnet:8.0 + tag: '8.0' + mcr.microsoft.com/dotnet/sdk:8.0: + digest: sha256:d80fdd84f7e18eea12f8e45c52914f1353395009c95c41197178ea19944e6d48 + target: registry.ddbuild.io/system-tests/mirror/mcr.microsoft.com/dotnet/sdk:8.0 + tag: '8.0' + mcr.microsoft.com/mssql/server:2022-latest: + digest: sha256:e07b9699a2b749969f19d86563ceeea22bd3a69f7f1db85a8d1ac4bdaf0c6f56 + target: registry.ddbuild.io/system-tests/mirror/mcr.microsoft.com/mssql/server:2022-latest + tag: 2022-latest + mongo:latest: + digest: sha256:49f1d7b87c2ddf918372be5defe7edff8c46703d0b2a56023a3f825e32e1250c + target: registry.ddbuild.io/system-tests/mirror/mongo:latest + tag: 8.2.11 + mysql/mysql-server:8.0.32: + digest: sha256:d6c8301b7834c5b9c2b733b10b7e630f441af7bc917c74dba379f24eeeb6a313 + target: registry.ddbuild.io/system-tests/mirror/mysql/mysql-server:8.0.32 + tag: 8.0.32 + node:18-alpine: + digest: sha256:8d6421d663b4c28fd3ebc498332f249011d118945588d0a35cb9bc4b8ca09d9e + target: registry.ddbuild.io/system-tests/mirror/node:18-alpine + tag: 18-alpine + node:20-alpine: + digest: sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 + target: registry.ddbuild.io/system-tests/mirror/node:20-alpine + tag: 20-alpine + node:22-alpine: + digest: sha256:e58326d0d441090181ac150dc2078d3e2cf6a0d42e809aebba3ef5880935ffdd + target: registry.ddbuild.io/system-tests/mirror/node:22-alpine + tag: 22-alpine + otel/opentelemetry-collector-contrib:0.137.0: + digest: sha256:886722fe0f37af9d1fe24d29529253ec59fbf263b3b1df4facaf221373e19d23 + target: registry.ddbuild.io/system-tests/mirror/otel/opentelemetry-collector-contrib:0.137.0 + tag: 0.137.0 + postgres:alpine: + digest: sha256:d3e64d36360a9f40c30fbbc5dd2dde799fe35f8537500c8b067551a6497f50f4 + target: registry.ddbuild.io/system-tests/mirror/postgres:alpine + tag: alpine + public.ecr.aws/lambda/java:17: + digest: sha256:ba8e52b4a5eb669dfda85bc4b55fe2957961ef67a962b0dfffa2e7abbcb7fcbb + target: registry.ddbuild.io/system-tests/mirror/public.ecr.aws/lambda/java:17 + tag: '17' + public.ecr.aws/lambda/nodejs:18: + digest: sha256:daf6a5c0a2b36153b94c91f3563e8ef89b3b19e4129963c6ccb07b8c5251f7ef + target: registry.ddbuild.io/system-tests/mirror/public.ecr.aws/lambda/nodejs:18 + tag: '18' + public.ecr.aws/lambda/python:3.13: + digest: sha256:fb6a19c34ca64079d0042a49f05688b3f9fab3b7ed51b7baacd48d593c898335 + target: registry.ddbuild.io/system-tests/mirror/public.ecr.aws/lambda/python:3.13 + tag: '3.13' + public.ecr.aws/lambda/ruby:3.4: + digest: sha256:39eb6cac136bc7d86e63d56fc5fc9d78ca87f29051bc309ac62f4aceb6779886 + target: registry.ddbuild.io/system-tests/mirror/public.ecr.aws/lambda/ruby:3.4 + tag: '3.4' + python:3.11-slim: + digest: sha256:ae52c5bef62a6bdd42cd1e8dffef86b9cd284bde9427da79839de7a4b983e7ca + target: registry.ddbuild.io/system-tests/mirror/python:3.11-slim + tag: 3.11-slim + python:3.12-slim: + digest: sha256:d764629ce0ddd8c71fd371e9901efb324a95789d2315a47db7e4d27e78f1b0e9 + target: registry.ddbuild.io/system-tests/mirror/python:3.12-slim + tag: 3.12-slim + python:3.13-slim: + digest: sha256:c33f0bc4364a6881bed1ec0cc2665e6c53c87a43e774aaeab88e6f17af105e4f + target: registry.ddbuild.io/system-tests/mirror/python:3.13-slim + tag: 3.13-slim + python:3.14-slim: + digest: sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061 + target: registry.ddbuild.io/system-tests/mirror/python:3.14-slim + tag: 3.14-slim + rabbitmq:3.12-management-alpine: + digest: sha256:0b44fbcc3a4bf22d00090f1353127577dbe1fcb109c41669733a9d7ecf6c3a78 + target: registry.ddbuild.io/system-tests/mirror/rabbitmq:3.12-management-alpine + tag: 3.12-management-alpine + rockylinux:9: + digest: sha256:d7be1c094cc5845ee815d4632fe377514ee6ebcf8efaed6892889657e5ddaaa6 + target: registry.ddbuild.io/system-tests/mirror/rockylinux:9 + tag: '9' + softwaremill/elasticmq-native:1.6.11: + digest: sha256:fe3af4a8dd59f310b20e07ec3499cfdeb16a14f518c95ad7e70f200845dce3f5 + target: registry.ddbuild.io/system-tests/mirror/softwaremill/elasticmq-native:1.6.11 + tag: 1.6.11 + ubuntu:24.04: + digest: sha256:786a8b558f7be160c6c8c4a54f9a57274f3b4fb1491cf65146521ae77ff1dc54 + target: registry.ddbuild.io/system-tests/mirror/ubuntu:24.04 + tag: '24.04' From ae13347ac42b97199fa34251c6f6d716453901f7 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 17 Jun 2026 17:00:35 +0200 Subject: [PATCH 193/229] ci: add mirror_images_verify to check the mirror is fully populated Add a main-only (and manual) job that runs `mirror_images.py mirror --dry-run` against the committed lock. The dry-run reads only the destination registry and exits 0 even when images are missing, so the job greps its output for "needs copy" and fails if any locked image is absent from MIRROR_DEST_REGISTRY (a non-zero exit from registry auth/network errors also fails it). Factor the crane+uv setup into a shared .mirror_images_registry_job template, and drop the in-CI re-lock from mirror_images_mirror so mirror and verify both operate off the same committed digests. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitlab-ci.yml | 53 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c92a00f4c6e..2231b270091 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -401,13 +401,15 @@ build_ci_image: # ────────────────────────────────────────────── # Mirror CI scenario images into registry.ddbuild.io/system-tests/mirror # -# Two-step flow: +# Three jobs: # * mirror_images_check (every push) regenerates the image list from the # scenarios and fails if the committed mirror_images.yaml is out of date. # Regenerate + commit it whenever scenarios or weblog Dockerfiles change # (see utils/scripts/update_mirror_images.py). -# * mirror_images_mirror (main only) resolves digests and pushes every listed -# image to MIRROR_DEST_REGISTRY. +# * mirror_images_mirror (main only) pushes every image in the committed +# mirror_images.lock.yaml to MIRROR_DEST_REGISTRY. +# * mirror_images_verify (main only) confirms every locked image is actually +# present in MIRROR_DEST_REGISTRY (read-only dry-run). # ────────────────────────────────────────────── mirror_images_check: @@ -435,23 +437,54 @@ mirror_images_check: rules: - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' -mirror_images_mirror: +# Shared setup for jobs that talk to the destination registry: crane handles +# multi-arch manifest lists cleanly (mirror_images.py auto-detects it on PATH, +# falling back to skopeo/docker/podman), plus uv to run the pinned script. Both +# jobs operate off the committed mirror_images.lock.yaml so they agree on the +# exact digests that should be in the mirror. +.mirror_images_registry_job: image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 stage: system-tests-utils tags: - docker-in-docker:amd64 - needs: - - job: mirror_images_check - artifacts: false before_script: - # crane handles multi-arch manifest lists cleanly; mirror_images.py auto-detects - # it on PATH (falling back to skopeo/docker/podman otherwise). - CRANE_VERSION=v0.20.2 - curl -sSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" | tar -xz -C /usr/local/bin crane || true - curl -LsSf https://astral.sh/uv/install.sh | sh - export PATH="$HOME/.local/bin:$PATH" + +mirror_images_mirror: + extends: .mirror_images_registry_job + needs: + - job: mirror_images_check + artifacts: false script: - - uv run --no-config --script "$MIRROR_IMAGES_URL" lock - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror rules: - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" + +# Verify every image recorded in mirror_images.lock.yaml is actually present in +# the destination registry. `mirror --dry-run` only reads the destination (no +# source pulls), but it exits 0 even when images are missing, so we inspect its +# output for "needs copy". A non-zero exit (e.g. registry auth/network failure) +# also fails the job. +mirror_images_verify: + extends: .mirror_images_registry_job + needs: + - job: mirror_images_mirror + artifacts: false + script: + - set -o pipefail + - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror --dry-run | tee mirror_dryrun.log + - | + if grep -q "needs copy" mirror_dryrun.log; then + echo "❌ Images recorded in mirror_images.lock.yaml are missing from $MIRROR_DEST_REGISTRY:" + grep "needs copy" mirror_dryrun.log + echo "Run the mirror_images_mirror job (push to main) to populate them." + exit 1 + fi + echo "✅ All mirrored images are present in $MIRROR_DEST_REGISTRY." + rules: + - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" + - when: manual + allow_failure: true From 0363a06faa729d688213457a25a99320e13aed75 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 17 Jun 2026 17:07:13 +0200 Subject: [PATCH 194/229] ci: collapse mirror image jobs into a single mirror_images job Replace mirror_images_check / mirror_images_mirror / mirror_images_verify with one job that, on every push: * regenerates the image list and fails on mirror_images.yaml drift, * pushes any locked image missing from the mirror (mirror checks the destination first, so already-present images are skipped). It never updates the lock file. crane copies registry-to-registry over HTTPS, so no docker-in-docker is needed and it runs on the framework runner image. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitlab-ci.yml | 87 +++++++++++++------------------------------------- 1 file changed, 22 insertions(+), 65 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2231b270091..77767124c79 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -401,18 +401,23 @@ build_ci_image: # ────────────────────────────────────────────── # Mirror CI scenario images into registry.ddbuild.io/system-tests/mirror # -# Three jobs: -# * mirror_images_check (every push) regenerates the image list from the -# scenarios and fails if the committed mirror_images.yaml is out of date. -# Regenerate + commit it whenever scenarios or weblog Dockerfiles change -# (see utils/scripts/update_mirror_images.py). -# * mirror_images_mirror (main only) pushes every image in the committed -# mirror_images.lock.yaml to MIRROR_DEST_REGISTRY. -# * mirror_images_verify (main only) confirms every locked image is actually -# present in MIRROR_DEST_REGISTRY (read-only dry-run). +# A single job that, on every push: +# * regenerates the image list from the scenarios and fails if the committed +# mirror_images.yaml is out of date (regenerate + commit it with +# utils/scripts/update_mirror_images.py whenever scenarios or weblog +# Dockerfiles change); +# * pushes any image from the committed mirror_images.lock.yaml that is not yet +# present in MIRROR_DEST_REGISTRY. `mirror` checks the destination first and +# only copies what is missing, so re-running it is cheap and idempotent. +# +# It never updates the lock file: mirror_images.lock.yaml is the committed +# contract, regenerated by developers via update_mirror_images.py. +# +# crane copies registry-to-registry over HTTPS (no Docker daemon needed), so this +# runs on the framework runner image used for the scenario enumeration. # ────────────────────────────────────────────── -mirror_images_check: +mirror_images: image: $CI_IMAGE stage: system-tests-utils tags: @@ -420,12 +425,15 @@ mirror_images_check: needs: - job: build_ci_image artifacts: true + before_script: + # crane handles multi-arch manifest lists cleanly; mirror_images.py picks it up on PATH. + - CRANE_VERSION=v0.20.2 + - curl -sSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" | tar -xz -C /usr/local/bin crane script: - ln -sf /system-tests/venv venv - source venv/bin/activate - python -m pip install --quiet uv - # --skip-lock: the drift check only validates the image list; the lock file - # is regenerated/committed by developers (and refreshed by mirror_images_mirror). + # Fail if the committed image list is stale vs the scenarios (does not touch the lock). - python utils/scripts/update_mirror_images.py --skip-lock - | if ! git diff --exit-code -- mirror_images.yaml; then @@ -433,58 +441,7 @@ mirror_images_check: echo " python utils/scripts/update_mirror_images.py" exit 1 fi - echo "✅ mirror_images.yaml is up to date." - rules: - - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' - -# Shared setup for jobs that talk to the destination registry: crane handles -# multi-arch manifest lists cleanly (mirror_images.py auto-detects it on PATH, -# falling back to skopeo/docker/podman), plus uv to run the pinned script. Both -# jobs operate off the committed mirror_images.lock.yaml so they agree on the -# exact digests that should be in the mirror. -.mirror_images_registry_job: - image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 - stage: system-tests-utils - tags: - - docker-in-docker:amd64 - before_script: - - CRANE_VERSION=v0.20.2 - - curl -sSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" | tar -xz -C /usr/local/bin crane || true - - curl -LsSf https://astral.sh/uv/install.sh | sh - - export PATH="$HOME/.local/bin:$PATH" - -mirror_images_mirror: - extends: .mirror_images_registry_job - needs: - - job: mirror_images_check - artifacts: false - script: + # Push any locked image missing from the mirror (already-present images are skipped). - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror rules: - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" - -# Verify every image recorded in mirror_images.lock.yaml is actually present in -# the destination registry. `mirror --dry-run` only reads the destination (no -# source pulls), but it exits 0 even when images are missing, so we inspect its -# output for "needs copy". A non-zero exit (e.g. registry auth/network failure) -# also fails the job. -mirror_images_verify: - extends: .mirror_images_registry_job - needs: - - job: mirror_images_mirror - artifacts: false - script: - - set -o pipefail - - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror --dry-run | tee mirror_dryrun.log - - | - if grep -q "needs copy" mirror_dryrun.log; then - echo "❌ Images recorded in mirror_images.lock.yaml are missing from $MIRROR_DEST_REGISTRY:" - grep "needs copy" mirror_dryrun.log - echo "Run the mirror_images_mirror job (push to main) to populate them." - exit 1 - fi - echo "✅ All mirrored images are present in $MIRROR_DEST_REGISTRY." - rules: - - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == "main" - - when: manual - allow_failure: true + - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' From 9be8d85aa2441b04e6c755cbf7781952f3b3f205 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 17 Jun 2026 17:58:07 +0200 Subject: [PATCH 195/229] feat: pull scenario images from the mirror when USE_IMAGE_MIRROR is set Add an opt-in image-mirror layer so CI pulls images from registry.ddbuild.io/system-tests/mirror instead of docker.io/ghcr.io/etc. - utils/_context/_image_mirror.py: mirror_image(ref) maps a source ref to its mirror target from mirror_images.lock.yaml when USE_IMAGE_MIRROR is truthy; passthrough otherwise (off by default, safe where the mirror is unreachable). - containers.py: ImageInfo rewrites its ref to the mirror and, on pull failure, falls back to the original registry. - _test_agent.py: TestAgentFactory image routed through the mirror. - build.sh + utils/scripts/mirror_rewrite_dockerfile.py: when enabled, weblog FROM base images are rewritten to the mirror at build time (stage names and unmirrored refs untouched). - templates.yml: enable USE_IMAGE_MIRROR=1 for e2e jobs. Co-Authored-By: Claude Opus 4.8 (1M context) --- utils/_context/_image_mirror.py | 59 ++++++++++++++++++++++ utils/_context/containers.py | 17 ++++++- utils/build/build.sh | 10 ++++ utils/ci/gitlab/templates.yml | 4 ++ utils/docker_fixtures/_test_agent.py | 4 +- utils/scripts/mirror_rewrite_dockerfile.py | 47 +++++++++++++++++ 6 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 utils/_context/_image_mirror.py create mode 100644 utils/scripts/mirror_rewrite_dockerfile.py diff --git a/utils/_context/_image_mirror.py b/utils/_context/_image_mirror.py new file mode 100644 index 00000000000..db3a6882400 --- /dev/null +++ b/utils/_context/_image_mirror.py @@ -0,0 +1,59 @@ +"""Rewrite Docker image references to the system-tests mirror. + +When the `USE_IMAGE_MIRROR` env var is truthy, image references that have been +mirrored into `registry.ddbuild.io/system-tests/mirror` (see mirror_images.yaml / +mirror_images.lock.yaml and the `mirror_images` CI job) are rewritten to their +mirror target, so CI pulls from the mirror instead of docker.io / ghcr.io / etc. + +It is off by default: with the flag unset (local runs, or any environment where +the mirror is not reachable) every reference is returned unchanged. + +The source -> target mapping is read from mirror_images.lock.yaml, which holds +the exact target for each mirrored image. Anything not listed there (locally +built `system_tests/*` images, unmirrored refs, multi-stage build stage names) +is left untouched. +""" + +import functools +import os +from pathlib import Path + +import yaml + +# mirror_images.lock.yaml lives at the repo root (this file is utils/_context/). +_LOCK_PATH = Path(__file__).resolve().parents[2] / "mirror_images.lock.yaml" + + +def mirror_enabled() -> bool: + return os.environ.get("USE_IMAGE_MIRROR", "").strip().lower() in ("1", "true", "yes") + + +@functools.lru_cache(maxsize=1) +def _mapping() -> dict[str, str]: + """Return {source_ref: mirror_target_ref} from the lock file ({} if absent).""" + try: + with open(_LOCK_PATH, encoding="utf-8") as f: + data = yaml.safe_load(f) + except (FileNotFoundError, yaml.YAMLError): + return {} + + images = data.get("images") if isinstance(data, dict) else None + if not isinstance(images, dict): + return {} + + mapping: dict[str, str] = {} + for source, info in images.items(): + if isinstance(info, dict) and isinstance(info.get("target"), str): + mapping[source] = info["target"] + return mapping + + +def mirror_image(name: str) -> str: + """Return the mirror target for `name` when mirroring is enabled, else `name`. + + Only exact matches in the lock file are rewritten, so unmirrored and + locally-built references pass through unchanged. + """ + if not mirror_enabled(): + return name + return _mapping().get(name, name) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 891da7f9b2d..9d25984670b 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -19,6 +19,7 @@ from utils._context.component_version import ComponentVersion, Version from utils._context.docker import get_docker_client +from utils._context._image_mirror import mirror_image from utils._context.ports import ContainerPorts from utils.proxy.tuf import get_tuf_root_json from utils.proxy.ports import ProxyPorts @@ -525,7 +526,10 @@ def __init__(self, image_name: str, *, local_image_only: bool): self.env: dict[str, str] | None = None self.labels: dict[str, str] = {} - self.name = image_name + # When the image mirror is enabled (USE_IMAGE_MIRROR), pull from the + # mirror; keep the original ref to fall back to if it isn't mirrored. + self.original_name = image_name + self.name = mirror_image(image_name) self.local_image_only = local_image_only def _pull_with_retries(self, max_retries: int = 4, delay: int = 4): @@ -554,7 +558,16 @@ def load(self): pytest.exit(f"Image {self.name} not found locally, please build it", 1) logger.stdout(f"Pulling {self.name}") - self._image = self._pull_with_retries() + try: + self._image = self._pull_with_retries() + except (docker.errors.APIError, requests.exceptions.ConnectionError): + # If the mirrored ref can't be pulled (e.g. not mirrored yet), + # fall back to the original registry. + if self.name == self.original_name: + raise + logger.stdout(f"Could not pull mirrored image {self.name}, falling back to {self.original_name}") + self.name = self.original_name + self._image = self._pull_with_retries() self._init_from_attrs(self._image.attrs) diff --git a/utils/build/build.sh b/utils/build/build.sh index 159dc3c1e9a..186565ed2bd 100755 --- a/utils/build/build.sh +++ b/utils/build/build.sh @@ -303,6 +303,16 @@ build() { DOCKERFILE=utils/build/docker/${TEST_LIBRARY}/${WEBLOG_VARIANT}.Dockerfile + # When the image mirror is enabled, rewrite the weblog base images + # (FROM ...) to pull from registry.ddbuild.io/system-tests/mirror. + if [[ "${USE_IMAGE_MIRROR:-}" =~ ^(1|true|yes)$ ]]; then + MIRRORED_DOCKERFILE=$(mktemp /tmp/system-tests-weblog-XXXXXX.Dockerfile) + python utils/scripts/mirror_rewrite_dockerfile.py "${DOCKERFILE}" > "${MIRRORED_DOCKERFILE}" + echo "Using mirrored Dockerfile (${DOCKERFILE} -> mirror):" + grep '^FROM' "${MIRRORED_DOCKERFILE}" || true + DOCKERFILE="${MIRRORED_DOCKERFILE}" + fi + GITHUB_TOKEN_SECRET_ARG="" if [ -n "${GITHUB_TOKEN_FILE:-}" ]; then diff --git a/utils/ci/gitlab/templates.yml b/utils/ci/gitlab/templates.yml index 2df0483be2e..b00709c000d 100644 --- a/utils/ci/gitlab/templates.yml +++ b/utils/ci/gitlab/templates.yml @@ -17,6 +17,10 @@ spec: # resolve the weblog as docker:7777 instead of localhost:7777, breaking span # snapshot assertions. Force it back to localhost. SYSTEM_TESTS_WEBLOG_HOST: "localhost" + # Pull scenario images (and weblog FROM bases) from the system-tests mirror + # instead of docker.io/ghcr.io/etc. Runtime pulls fall back to the original + # registry if an image isn't mirrored yet (see utils/_context/_image_mirror.py). + USE_IMAGE_MIRROR: "1" stage: $[[ inputs.stage ]] interruptible: true before_script: diff --git a/utils/docker_fixtures/_test_agent.py b/utils/docker_fixtures/_test_agent.py index 28fb0003da1..1bb257a787e 100644 --- a/utils/docker_fixtures/_test_agent.py +++ b/utils/docker_fixtures/_test_agent.py @@ -18,6 +18,7 @@ from retry import retry from utils._logger import logger +from utils._context._image_mirror import mirror_image from utils.dd_constants import RemoteConfigApplyState, Capabilities from .spec import remoteconfig from .spec.trace import V06StatsPayload @@ -54,7 +55,8 @@ class TestAgentFactory: """ def __init__(self, image: str): - self.image = image + # Pull/run from the mirror when USE_IMAGE_MIRROR is enabled (no-op otherwise). + self.image = mirror_image(image) self.host_log_folder = "" def configure(self, host_log_folder: str): diff --git a/utils/scripts/mirror_rewrite_dockerfile.py b/utils/scripts/mirror_rewrite_dockerfile.py new file mode 100644 index 00000000000..0643ccf170c --- /dev/null +++ b/utils/scripts/mirror_rewrite_dockerfile.py @@ -0,0 +1,47 @@ +"""Rewrite a Dockerfile's `FROM` base images to the system-tests mirror. + +Used by utils/build/build.sh when USE_IMAGE_MIRROR is enabled: it prints the +given Dockerfile to stdout with every `FROM ` whose image is in the +mirror (mirror_images.lock.yaml) replaced by its mirror target, so weblog builds +pull their base images from registry.ddbuild.io/system-tests/mirror instead of +docker.io / ghcr.io / etc. + +Image references not present in the mirror mapping are left unchanged. This +matches multi-stage stage names (`FROM build`) too: they are never in the +mapping, so they pass through untouched. +""" + +import argparse +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from utils._context._image_mirror import mirror_image + +# FROM [--platform=

] [AS ] +_FROM_RE = re.compile(r"^(\s*FROM\s+)((?:--platform=\S+\s+)?)(\S+)(.*)$", re.IGNORECASE) + + +def rewrite(dockerfile: str) -> str: + out: list[str] = [] + with open(dockerfile, encoding="utf-8") as f: + for line in f: + match = _FROM_RE.match(line.rstrip("\n")) + if match: + prefix, platform, image, rest = match.groups() + out.append(f"{prefix}{platform}{mirror_image(image)}{rest}\n") + else: + out.append(line if line.endswith("\n") else line + "\n") + return "".join(out) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="mirror_rewrite_dockerfile", + description="Print a Dockerfile with FROM base images rewritten to the mirror", + ) + parser.add_argument("dockerfile", help="Path to the Dockerfile to rewrite") + args = parser.parse_args() + sys.stdout.write(rewrite(args.dockerfile)) From e2353a59bfc7a0599f0cf7f21177622859dabb0a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 17 Jun 2026 19:00:27 +0200 Subject: [PATCH 196/229] ci: gate build jobs on mirror_images; move Docker Hub auth to the mirror job - main.yml: .run_test_pipeline_base gains an optional `needs: mirror_images` so the e2e build/run jobs only start once the mirror is populated. Optional so repos that include this template without a mirror_images job are unaffected. - .gitlab-ci.yml: move mirror_images to the e2e stage (so earlier-DAG jobs can depend on it) and give it Docker Hub auth (crane auth login) since it is now the only job pulling the mirror sources from Docker Hub. - Drop `docker_auth: true` so the generated test jobs no longer authenticate to Docker Hub (they pull from the mirror); the shared docker_auth plumbing stays dormant (default false) for other repos. - Remove dead DOCKER_LOGIN exports from check_merge_labels (it never logs in). Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitlab-ci.yml | 12 ++++++++---- utils/ci/gitlab/main.yml | 6 ++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77767124c79..4c3e5a6fc17 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,6 @@ include: excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true - docker_auth: true stages: - configure - e2e @@ -266,8 +265,6 @@ check_merge_labels: - dd-octo-sts version - dd-octo-sts debug --scope DataDog/system-tests --policy self.gitlab-read - dd-octo-sts token --scope DataDog/system-tests --policy self.gitlab-read > token.txt - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) script: - export GITHUB_TOKEN=$(cat token.txt) - ./utils/scripts/get_pr_merged_labels.sh @@ -419,7 +416,9 @@ build_ci_image: mirror_images: image: $CI_IMAGE - stage: system-tests-utils + # Runs in e2e (not the last stage) so the build/run jobs can depend on it and + # only start once the mirror is populated. + stage: e2e tags: - arch:amd64 needs: @@ -429,6 +428,11 @@ mirror_images: # crane handles multi-arch manifest lists cleanly; mirror_images.py picks it up on PATH. - CRANE_VERSION=v0.20.2 - curl -sSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" | tar -xz -C /usr/local/bin crane + # This is the only job that pulls from Docker Hub (the mirror sources), so it + # is the only one that needs Docker Hub auth (avoids unauthenticated rate limits). + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | crane auth login index.docker.io --username "$DOCKER_LOGIN" --password-stdin script: - ln -sf /system-tests/venv venv - source venv/bin/activate diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 35a56d03894..e26994e1336 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -156,6 +156,12 @@ build_test_pipeline: - job: system-tests-param optional: true artifacts: true + # When the image mirror is in use (system-tests' own pipeline), wait for it to + # populate the mirror before building/running. Optional so repos that include + # this template without a mirror_images job are unaffected. + - job: mirror_images + optional: true + artifacts: false variables: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" From f33dc3395560bb2ae740a7cc60427464e516cf59 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 11:47:34 +0200 Subject: [PATCH 197/229] ci: add binaries_artifact_path so consumers can place pre-built binaries The generated build/run/parametric jobs read binaries from binaries/, but a consumer's build job may publish its artifact elsewhere (e.g. dd-trace-py's "build linux" puts wheels in pywheels/). Add a binaries_artifact_path input (threaded main.yml -> build_pipeline.py -> system-tests.yml.j2) that, when set alongside binaries_artifact, copies $CI_PROJECT_DIR//. into binaries/ before building/running. Default empty = unchanged behaviour. Co-Authored-By: Claude Opus 4.8 (1M context) --- utils/ci/gitlab/build_pipeline.py | 10 ++++++++++ utils/ci/gitlab/main.yml | 5 ++++- utils/ci/gitlab/system-tests.yml.j2 | 15 +++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 7afe8ccea0b..39356d53639 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -37,6 +37,7 @@ def render_library( ref: str, push_to_test_optimization: bool, docker_auth: bool, + binaries_artifact_path: str, ) -> str: parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) @@ -54,6 +55,7 @@ def render_library( library=library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact, + binaries_artifact_path=binaries_artifact_path, parametric=parametric, ci_image=ci_image, ref=ref, @@ -74,6 +76,7 @@ def build( push_to_test_optimization: bool = False, chunks: int = 3, docker_auth: bool = False, + binaries_artifact_path: str = "", ) -> None: """Render pipeline chunk files into *output_dir*, one per chunk.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -111,6 +114,7 @@ def build( ref=ref, push_to_test_optimization=push_to_test_optimization, docker_auth=docker_auth, + binaries_artifact_path=binaries_artifact_path, ) ) @@ -128,6 +132,11 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--output-dir", required=True, help="Output directory for generated-pipeline-chunk-N.yml files") parser.add_argument("--chunks", type=int, default=3, help="Number of pipeline chunks (default: 3)") parser.add_argument("--docker-auth", default="false", help="Wether to authenticate calls to docker hub") + parser.add_argument( + "--binaries-artifact-path", + default="", + help="Path (relative to CI_PROJECT_DIR) of the binaries_artifact contents, copied into binaries/", + ) args = parser.parse_args(argv) @@ -146,6 +155,7 @@ def main(argv: list[str] | None = None) -> int: push_to_test_optimization=args.push_to_test_optimization == "true", chunks=args.chunks, docker_auth=args.docker_auth == "true", + binaries_artifact_path=args.binaries_artifact_path, ) return 0 diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index e26994e1336..3b3785e162c 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -20,6 +20,9 @@ spec: binaries_artifact: description: "Name of the upstream job whose artifacts contain the pre-built binaries to test" default: "" + binaries_artifact_path: + description: "Path (relative to CI_PROJECT_DIR) of the binaries_artifact contents, copied into binaries/ before building/running" + default: "" force_execute: description: "Comma-separated test node IDs to force-execute" default: "" @@ -138,7 +141,7 @@ build_test_pipeline: for library in $libraries; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json done - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$[[ inputs.docker_auth ]]" + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$[[ inputs.docker_auth ]]" --binaries-artifact-path "$[[ inputs.binaries_artifact_path ]]" artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 3e19fb0dd01..971b4c5f6b2 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -11,6 +11,13 @@ done - section_end "docker_auth" {% endmacro %} +{% macro copy_binaries(path) %} + - section_start "copy_binaries" "Copying pre-built binaries into binaries/" + - mkdir -p binaries + - cp -r "$CI_PROJECT_DIR/{{path}}/." binaries/ + - ls -la binaries/ + - section_end "copy_binaries" +{% endmacro %} {% if not skip_header %} workflow: name: "System-tests end to end" @@ -81,6 +88,9 @@ build_{{library}}_{{variant}}: {% if docker_auth_enabled %} {{ docker_auth() }} {% endif %} + {% if binaries_artifact and binaries_artifact_path %} + {{ copy_binaries(binaries_artifact_path) }} + {% endif %} - section_start "build" "Building weblog" - ./build.sh {{library}} -i weblog --save-to-binaries --weblog-variant {{variant}} - mv binaries .. @@ -113,6 +123,8 @@ run_{{library}}_{{scenario}}_{{variant}}: {% if build_required %} - mv ../binaries/* binaries/ - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} + {% elif binaries_artifact and binaries_artifact_path %} + {{ copy_binaries(binaries_artifact_path) }} {% endif %} - section_end "weblog_setup" - section_start "scenario_run" "Running scenario" @@ -145,6 +157,9 @@ run_{{library}}_PARAMETRIC_{{job_index}}: {% endif %} script: - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" + {% if binaries_artifact and binaries_artifact_path %} + {{ copy_binaries(binaries_artifact_path) }} + {% endif %} - ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" artifacts: From a0299f0b182914af7e64c914dc77a298ad70e0df Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 11:56:35 +0200 Subject: [PATCH 198/229] ci: fix cross-pipeline artifact reference for binaries_artifact Generated child-pipeline jobs cannot needs: a parent pipeline job by name alone. When binaries_artifact_path is set (meaning the artifact comes from the parent), emit `pipeline: \$PARENT_PIPELINE_ID` on the needs entry so GitLab resolves it correctly. Pass PARENT_PIPELINE_ID: \$CI_PIPELINE_ID from the trigger variables in .run_test_pipeline_base so the child pipeline has the value available. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/main.yml | 3 +++ utils/ci/gitlab/system-tests.yml.j2 | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 3b3785e162c..7e7245bcd1d 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -169,6 +169,9 @@ build_test_pipeline: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" LIBRARIES: $LIBRARIES + # Passed to the child pipeline so generated jobs can reference artifacts from + # the parent pipeline (e.g. `needs: { pipeline: $PARENT_PIPELINE_ID, job: ... }`). + PARENT_PIPELINE_ID: $CI_PIPELINE_ID run_test_pipeline_0: extends: .run_test_pipeline_base diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 971b4c5f6b2..5e466426e57 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -83,6 +83,9 @@ build_{{library}}_{{variant}}: needs: - job: {{binaries_artifact}} artifacts: true + {% if binaries_artifact_path %} + pipeline: $PARENT_PIPELINE_ID + {% endif %} {% endif %} script: {% if docker_auth_enabled %} @@ -114,6 +117,9 @@ run_{{library}}_{{scenario}}_{{variant}}: {% elif binaries_artifact %} - job: {{binaries_artifact}} artifacts: true + {% if binaries_artifact_path %} + pipeline: $PARENT_PIPELINE_ID + {% endif %} {% endif %} script: {% if docker_auth_enabled %} @@ -154,6 +160,9 @@ run_{{library}}_PARAMETRIC_{{job_index}}: needs: - job: {{binaries_artifact}} artifacts: true + {% if binaries_artifact_path %} + pipeline: $PARENT_PIPELINE_ID + {% endif %} {% endif %} script: - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" From d533b182f290fbc4d04d498c73dd00a49f16efe6 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 11:57:33 +0200 Subject: [PATCH 199/229] ci: always use parent pipeline ref for binaries_artifact needs binaries_artifact names a job in the parent pipeline by definition, so pipeline: \$PARENT_PIPELINE_ID belongs on every binaries_artifact needs entry, not only when binaries_artifact_path is also set. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/system-tests.yml.j2 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 5e466426e57..f62cac315b1 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -83,9 +83,7 @@ build_{{library}}_{{variant}}: needs: - job: {{binaries_artifact}} artifacts: true - {% if binaries_artifact_path %} pipeline: $PARENT_PIPELINE_ID - {% endif %} {% endif %} script: {% if docker_auth_enabled %} @@ -117,9 +115,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% elif binaries_artifact %} - job: {{binaries_artifact}} artifacts: true - {% if binaries_artifact_path %} pipeline: $PARENT_PIPELINE_ID - {% endif %} {% endif %} script: {% if docker_auth_enabled %} @@ -160,9 +156,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: needs: - job: {{binaries_artifact}} artifacts: true - {% if binaries_artifact_path %} pipeline: $PARENT_PIPELINE_ID - {% endif %} {% endif %} script: - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" From 023763d8d34d83daaed29a001262750bc2c5127e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 12:00:07 +0200 Subject: [PATCH 200/229] Cleanup --- utils/ci/gitlab/main.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 7e7245bcd1d..8df20512cc0 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -159,9 +159,6 @@ build_test_pipeline: - job: system-tests-param optional: true artifacts: true - # When the image mirror is in use (system-tests' own pipeline), wait for it to - # populate the mirror before building/running. Optional so repos that include - # this template without a mirror_images job are unaffected. - job: mirror_images optional: true artifacts: false @@ -169,8 +166,6 @@ build_test_pipeline: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" LIBRARIES: $LIBRARIES - # Passed to the child pipeline so generated jobs can reference artifacts from - # the parent pipeline (e.g. `needs: { pipeline: $PARENT_PIPELINE_ID, job: ... }`). PARENT_PIPELINE_ID: $CI_PIPELINE_ID run_test_pipeline_0: From c289a4addb95089b059e8d6734ec322e653f4d23 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 12:20:24 +0200 Subject: [PATCH 201/229] ci: wait for binaries_artifact job before launching child pipeline The trigger jobs (.run_test_pipeline_base) now need the binaries_artifact job (optional) so the child pipeline doesn't start before the artifact is ready. Without this the child pipeline fired immediately and the cross-pipeline needs failed to retrieve artifacts. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 8df20512cc0..16cd931dc4c 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -162,6 +162,11 @@ build_test_pipeline: - job: mirror_images optional: true artifacts: false + # When binaries_artifact is set, wait for it before launching the child + # pipeline so the artifact is ready when generated jobs try to download it. + - job: $[[ inputs.binaries_artifact ]] + optional: true + artifacts: false variables: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" From 7b69ae142095d459721bb5e9b93a6cdc86f2683e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 12:56:24 +0200 Subject: [PATCH 202/229] ci: quote binaries_artifact job name in j2 template Job names containing ':' and '[' (e.g. parallel matrix instance names like 'build linux: [amd64, cp311-cp311, ...]') are special YAML characters and must be quoted, otherwise the generated pipeline YAML is invalid. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/main.yml | 5 ----- utils/ci/gitlab/system-tests.yml.j2 | 6 +++--- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 16cd931dc4c..8df20512cc0 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -162,11 +162,6 @@ build_test_pipeline: - job: mirror_images optional: true artifacts: false - # When binaries_artifact is set, wait for it before launching the child - # pipeline so the artifact is ready when generated jobs try to download it. - - job: $[[ inputs.binaries_artifact ]] - optional: true - artifacts: false variables: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index f62cac315b1..2a852d8582b 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -81,7 +81,7 @@ build_{{library}}_{{variant}}: extends: .system_tests_base {% if binaries_artifact %} needs: - - job: {{binaries_artifact}} + - job: "{{binaries_artifact}}" artifacts: true pipeline: $PARENT_PIPELINE_ID {% endif %} @@ -113,7 +113,7 @@ run_{{library}}_{{scenario}}_{{variant}}: - job: build_{{library}}_{{variant}} artifacts: true {% elif binaries_artifact %} - - job: {{binaries_artifact}} + - job: "{{binaries_artifact}}" artifacts: true pipeline: $PARENT_PIPELINE_ID {% endif %} @@ -154,7 +154,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: {% endif %} {% if binaries_artifact %} needs: - - job: {{binaries_artifact}} + - job: "{{binaries_artifact}}" artifacts: true pipeline: $PARENT_PIPELINE_ID {% endif %} From 44ac1d63ac7ccf2e8c4f209893631f8a7f933cd8 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 13:32:56 +0200 Subject: [PATCH 203/229] ci: wait for binaries_artifact before generating the child pipeline build_test_pipeline needs the binaries_artifact job (optional, so it's a no-op when the input is empty). Since run_test_pipeline_* always waits for build_test_pipeline before triggering the child pipeline, this guarantees the artifact is ready before any child job tries to download it via pipeline: \$PARENT_PIPELINE_ID. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 8df20512cc0..31241e76a39 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -128,6 +128,9 @@ build_test_pipeline: - job: system-tests-param optional: true artifacts: true + - job: $[[ inputs.binaries_artifact ]] + optional: true + artifacts: true script: - | libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) From 4463f6efa56e5c8f85e86c757f385499270d6970 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 13:42:25 +0200 Subject: [PATCH 204/229] =?UTF-8?q?ci:=20artifacts:=20false=20for=20binari?= =?UTF-8?q?es=5Fartifact=20wait=20=E2=80=94=20timing=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build_test_pipeline only needs to wait for the job to finish before generating the child pipeline; it never uses the artifacts itself. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 31241e76a39..a54eba77ec8 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -130,7 +130,7 @@ build_test_pipeline: artifacts: true - job: $[[ inputs.binaries_artifact ]] optional: true - artifacts: true + artifacts: false script: - | libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) From 38757e506b6d22f8db95fc726358a37c640c2884 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 13:47:00 +0200 Subject: [PATCH 205/229] ci: use \$CI_PARENT_PIPELINE_ID instead of a manually-passed variable \$CI_PARENT_PIPELINE_ID is provided automatically by GitLab in any triggered child pipeline. The manually-passed PARENT_PIPELINE_ID was redundant and potentially empty if not resolved correctly, causing the artifact download to fail. Since needs:pipeline: waits for the referenced job itself, build_test_pipeline also doesn't need to wait for binaries_artifact. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/main.yml | 4 ---- utils/ci/gitlab/system-tests.yml.j2 | 6 +++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index a54eba77ec8..1998ceddc5f 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -128,9 +128,6 @@ build_test_pipeline: - job: system-tests-param optional: true artifacts: true - - job: $[[ inputs.binaries_artifact ]] - optional: true - artifacts: false script: - | libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) @@ -169,7 +166,6 @@ build_test_pipeline: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" LIBRARIES: $LIBRARIES - PARENT_PIPELINE_ID: $CI_PIPELINE_ID run_test_pipeline_0: extends: .run_test_pipeline_base diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 2a852d8582b..61780b3d86e 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -83,7 +83,7 @@ build_{{library}}_{{variant}}: needs: - job: "{{binaries_artifact}}" artifacts: true - pipeline: $PARENT_PIPELINE_ID + pipeline: $CI_PARENT_PIPELINE_ID {% endif %} script: {% if docker_auth_enabled %} @@ -115,7 +115,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% elif binaries_artifact %} - job: "{{binaries_artifact}}" artifacts: true - pipeline: $PARENT_PIPELINE_ID + pipeline: $CI_PARENT_PIPELINE_ID {% endif %} script: {% if docker_auth_enabled %} @@ -156,7 +156,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: needs: - job: "{{binaries_artifact}}" artifacts: true - pipeline: $PARENT_PIPELINE_ID + pipeline: $CI_PARENT_PIPELINE_ID {% endif %} script: - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" From 62fbaa6a06b1c54578be52038d4e73a0271bd66a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 14:12:49 +0200 Subject: [PATCH 206/229] ci: wait for binaries_artifact before generating the child pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit needs:pipeline: in child jobs does NOT wait — it just tries to download the artifact and fails if the upstream job is not done yet. build_test_pipeline must therefore wait for the binaries_artifact job (optional, no-op when the input is empty) so the artifact is guaranteed to exist before any child pipeline job runs. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 1998ceddc5f..9191b9e8aa2 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -162,6 +162,9 @@ build_test_pipeline: - job: mirror_images optional: true artifacts: false + - job: $[[ inputs.binaries_artifact ]] + optional: true + artifacts: false variables: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" From 35dbaf2f23cf04cfef35fadd1942fa27583e5d8b Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 14:26:54 +0200 Subject: [PATCH 207/229] Fix upstream pipeline requirement --- utils/ci/gitlab/system-tests.yml.j2 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 61780b3d86e..54d12115bd6 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -83,7 +83,7 @@ build_{{library}}_{{variant}}: needs: - job: "{{binaries_artifact}}" artifacts: true - pipeline: $CI_PARENT_PIPELINE_ID + pipeline: $UPSTREAM_PIPELINE_ID {% endif %} script: {% if docker_auth_enabled %} @@ -115,7 +115,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% elif binaries_artifact %} - job: "{{binaries_artifact}}" artifacts: true - pipeline: $CI_PARENT_PIPELINE_ID + pipeline: $UPSTREAM_PIPELINE_ID {% endif %} script: {% if docker_auth_enabled %} @@ -156,7 +156,7 @@ run_{{library}}_PARAMETRIC_{{job_index}}: needs: - job: "{{binaries_artifact}}" artifacts: true - pipeline: $CI_PARENT_PIPELINE_ID + pipeline: $UPSTREAM_PIPELINE_ID {% endif %} script: - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" From 9d658db22262900214c2ff8c1744c3ca60ff861d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 14:36:46 +0200 Subject: [PATCH 208/229] ci: pass UPSTREAM_PIPELINE_ID to child pipeline The generated child pipeline jobs use pipeline: \$UPSTREAM_PIPELINE_ID to reference the parent pipeline's binaries artifact. This variable was never being set, so the artifact download had an empty pipeline reference and failed. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 9191b9e8aa2..6993d918145 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -169,6 +169,7 @@ build_test_pipeline: SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" LIBRARIES: $LIBRARIES + UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID run_test_pipeline_0: extends: .run_test_pipeline_base From 0c72538674e7b39aa5d602aed35f1a61ec47777d Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 15:00:19 +0200 Subject: [PATCH 209/229] ci: support a list of artifact jobs via binaries_artifacts input Different weblogs need different python version wheels. Add binaries_artifacts (semicolon-separated, since job names contain commas) as the full list of upstream artifact jobs to download from in the generated child pipeline. The existing single-job binaries_artifact remains the timing gate. Falls back to [binaries_artifact] when binaries_artifacts is empty, so existing callers that only set binaries_artifact are unaffected. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- utils/ci/gitlab/build_pipeline.py | 19 +++++++++++++++++++ utils/ci/gitlab/main.yml | 7 +++++-- utils/ci/gitlab/system-tests.yml.j2 | 24 +++++++++++++++--------- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 39356d53639..ca02b5fa0d7 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -38,6 +38,7 @@ def render_library( push_to_test_optimization: bool, docker_auth: bool, binaries_artifact_path: str, + binaries_artifacts: str, ) -> str: parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) @@ -49,12 +50,21 @@ def render_library( ] binaries_artifact = params["miscs"]["binaries_artifact"] parametric = params["parametric"] + # Build the full list of artifact jobs for cross-pipeline downloads. + # If binaries_artifacts is provided, use it; otherwise fall back to the single job. + if binaries_artifacts: + binaries_artifacts_list = [j.strip() for j in binaries_artifacts.split(";") if j.strip()] + elif binaries_artifact: + binaries_artifacts_list = [binaries_artifact] + else: + binaries_artifacts_list = [] return _template.render( scenario_pairs=scenario_pairs, stage=stage, library=library, weblog_variants=weblog_variants, binaries_artifact=binaries_artifact, + binaries_artifacts_list=binaries_artifacts_list, binaries_artifact_path=binaries_artifact_path, parametric=parametric, ci_image=ci_image, @@ -77,6 +87,7 @@ def build( chunks: int = 3, docker_auth: bool = False, binaries_artifact_path: str = "", + binaries_artifacts: str = "", ) -> None: """Render pipeline chunk files into *output_dir*, one per chunk.""" output_dir.mkdir(parents=True, exist_ok=True) @@ -115,6 +126,7 @@ def build( push_to_test_optimization=push_to_test_optimization, docker_auth=docker_auth, binaries_artifact_path=binaries_artifact_path, + binaries_artifacts=binaries_artifacts, ) ) @@ -137,6 +149,12 @@ def main(argv: list[str] | None = None) -> int: default="", help="Path (relative to CI_PROJECT_DIR) of the binaries_artifact contents, copied into binaries/", ) + parser.add_argument( + "--binaries-artifacts", + default="", + help="Comma-separated list of upstream jobs to download artifacts from in the child pipeline " + "(falls back to the single binaries_artifact from params if empty)", + ) args = parser.parse_args(argv) @@ -156,6 +174,7 @@ def main(argv: list[str] | None = None) -> int: chunks=args.chunks, docker_auth=args.docker_auth == "true", binaries_artifact_path=args.binaries_artifact_path, + binaries_artifacts=args.binaries_artifacts, ) return 0 diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 6993d918145..40953a5aa09 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -18,7 +18,10 @@ spec: description: "Comma-separated list of weblogs to run (all weblogs if empty)" default: "" binaries_artifact: - description: "Name of the upstream job whose artifacts contain the pre-built binaries to test" + description: "Name of the single upstream job to wait for before triggering the child pipeline (used as the timing gate)" + default: "" + binaries_artifacts: + description: "Semicolon-separated list of upstream jobs whose artifacts contain the pre-built binaries (downloaded by the generated build/run jobs via cross-pipeline needs). Use ';' not ',' because job names may contain commas (e.g. parallel matrix names)." default: "" binaries_artifact_path: description: "Path (relative to CI_PROJECT_DIR) of the binaries_artifact contents, copied into binaries/ before building/running" @@ -141,7 +144,7 @@ build_test_pipeline: for library in $libraries; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json done - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$[[ inputs.docker_auth ]]" --binaries-artifact-path "$[[ inputs.binaries_artifact_path ]]" + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$[[ inputs.docker_auth ]]" --binaries-artifact-path "$[[ inputs.binaries_artifact_path ]]" --binaries-artifacts "$[[ inputs.binaries_artifacts ]]" artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 54d12115bd6..c1f2d5f55a9 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -79,17 +79,19 @@ setup: {% for variant in weblog_variants %} build_{{library}}_{{variant}}: extends: .system_tests_base - {% if binaries_artifact %} + {% if binaries_artifacts_list %} needs: - - job: "{{binaries_artifact}}" + {% for artifact_job in binaries_artifacts_list %} + - job: "{{artifact_job}}" artifacts: true pipeline: $UPSTREAM_PIPELINE_ID + {% endfor %} {% endif %} script: {% if docker_auth_enabled %} {{ docker_auth() }} {% endif %} - {% if binaries_artifact and binaries_artifact_path %} + {% if binaries_artifacts_list and binaries_artifact_path %} {{ copy_binaries(binaries_artifact_path) }} {% endif %} - section_start "build" "Building weblog" @@ -112,10 +114,12 @@ run_{{library}}_{{scenario}}_{{variant}}: {% if build_required %} - job: build_{{library}}_{{variant}} artifacts: true - {% elif binaries_artifact %} - - job: "{{binaries_artifact}}" + {% elif binaries_artifacts_list %} + {% for artifact_job in binaries_artifacts_list %} + - job: "{{artifact_job}}" artifacts: true pipeline: $UPSTREAM_PIPELINE_ID + {% endfor %} {% endif %} script: {% if docker_auth_enabled %} @@ -125,7 +129,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% if build_required %} - mv ../binaries/* binaries/ - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} - {% elif binaries_artifact and binaries_artifact_path %} + {% elif binaries_artifacts_list and binaries_artifact_path %} {{ copy_binaries(binaries_artifact_path) }} {% endif %} - section_end "weblog_setup" @@ -152,15 +156,17 @@ run_{{library}}_PARAMETRIC_{{job_index}}: {% if push_to_test_optimization %} - .push_to_test_optimization {% endif %} - {% if binaries_artifact %} + {% if binaries_artifacts_list %} needs: - - job: "{{binaries_artifact}}" + {% for artifact_job in binaries_artifacts_list %} + - job: "{{artifact_job}}" artifacts: true pipeline: $UPSTREAM_PIPELINE_ID + {% endfor %} {% endif %} script: - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" - {% if binaries_artifact and binaries_artifact_path %} + {% if binaries_artifacts_list and binaries_artifact_path %} {{ copy_binaries(binaries_artifact_path) }} {% endif %} - ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} From 33cbc55ea9a4540a8f761d1c17340f1a4704cd3c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 18 Jun 2026 17:31:22 +0200 Subject: [PATCH 210/229] Static CI image tags --- .gitlab-ci.yml | 23 +++++++++++++---------- utils/ci/gitlab/main.yml | 35 ++++++----------------------------- 2 files changed, 19 insertions(+), 39 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4c3e5a6fc17..32dbcbf7091 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,6 @@ stages: variables: TEST: 1 - SKIP_RESOLVE_CI_IMAGE: "true" # TEMPORARY: rollout-control allowlist for system-tests-param LIBRARIES filter. E2E_SUPPORTED_LANGUAGES: "python" # dd-repo-tools mirror_images.py (pinned). Used by the mirror_images_* jobs. @@ -327,7 +326,7 @@ system-tests-param: - arch:amd64 needs: - job: build_ci_image - artifacts: true + artifacts: false script: - ln -sf /system-tests/venv venv - source venv/bin/activate @@ -383,17 +382,21 @@ build_ci_image: stage: e2e script: - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) - - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env + - EXPECTED_IMAGE="registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" + - | + if [ "$EXPECTED_IMAGE" != "$CI_IMAGE" ]; then + echo "❌ CI_IMAGE mismatch: hardcoded '$CI_IMAGE' does not match computed '$EXPECTED_IMAGE'" + echo " Update CI_IMAGE in utils/ci/gitlab/main.yml:" + echo " cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12" + exit 1 + fi - > - if docker manifest inspect registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG > /dev/null 2>&1; then + if docker manifest inspect $EXPECTED_IMAGE > /dev/null 2>&1; then echo "Image already exists, skipping build"; else - docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile -t registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG . && - docker push registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG; + docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile -t $EXPECTED_IMAGE . && + docker push $EXPECTED_IMAGE; fi - artifacts: - reports: - dotenv: build.env # ────────────────────────────────────────────── # Mirror CI scenario images into registry.ddbuild.io/system-tests/mirror @@ -423,7 +426,7 @@ mirror_images: - arch:amd64 needs: - job: build_ci_image - artifacts: true + artifacts: false before_script: # crane handles multi-arch manifest lists cleanly; mirror_images.py picks it up on PATH. - CRANE_VERSION=v0.20.2 diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 40953a5aa09..5436352cf1c 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -91,43 +91,20 @@ workflow: auto_cancel: on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] -resolve_ci_image: - rules: - - if: '$SKIP_RESOLVE_CI_IMAGE == "true"' - when: never - - when: always - image: registry.ddbuild.io/system-tests/git - tags: - - arch:amd64 - stage: $[[ inputs.stage ]] - interruptible: true - variables: - GIT_STRATEGY: none - needs: - - job: build_ci_image - optional: true - artifacts: false - script: - - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git - - git -C system-tests checkout $[[ inputs.ref ]] - - IMAGE_TAG=$(cat system-tests/utils/ci/gitlab/docker/system-tests.Dockerfile system-tests/requirements.txt | sha256sum | cut -c1-12) - - echo "CI_IMAGE=registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" >> build.env - - cat build.env - artifacts: - reports: - dotenv: build.env +variables: + # Tag = first 12 chars of sha256(utils/ci/gitlab/docker/system-tests.Dockerfile + requirements.txt) + # Update this when either file changes: + # cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12 + CI_IMAGE: "registry.ddbuild.io/system-tests/ci-runner:6bf5fa49e86c" build_test_pipeline: extends: .system_tests_base tags: - arch:amd64 needs: - - job: resolve_ci_image - optional: true - artifacts: true - job: build_ci_image optional: true - artifacts: true + artifacts: false - job: system-tests-param optional: true artifacts: true From b8840fd551351cc1bbfbbdaef2418c83ab9be197 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 19 Jun 2026 11:13:05 +0200 Subject: [PATCH 211/229] fix --- utils/ci/gitlab/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 5436352cf1c..8a6eb912bb2 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -19,7 +19,7 @@ spec: default: "" binaries_artifact: description: "Name of the single upstream job to wait for before triggering the child pipeline (used as the timing gate)" - default: "" + default: "__no_binaries_artifact__" binaries_artifacts: description: "Semicolon-separated list of upstream jobs whose artifacts contain the pre-built binaries (downloaded by the generated build/run jobs via cross-pipeline needs). Use ';' not ',' because job names may contain commas (e.g. parallel matrix names)." default: "" From 7895dd44b8c60aaa089a6551546700ed45d29ae3 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 19 Jun 2026 11:41:04 +0200 Subject: [PATCH 212/229] fix --- utils/ci/gitlab/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 8a6eb912bb2..42393c837e9 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -118,8 +118,10 @@ build_test_pipeline: echo "scenarios: $scenarios" echo "scenario_groups: $scenario_groups" echo "weblogs: $weblogs" + binaries_artifact="$[[ inputs.binaries_artifact ]]" + [ "$binaries_artifact" = "__no_binaries_artifact__" ] && binaries_artifact="" for library in $libraries; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$[[ inputs.binaries_artifact ]]" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$binaries_artifact" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json done python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$[[ inputs.docker_auth ]]" --binaries-artifact-path "$[[ inputs.binaries_artifact_path ]]" --binaries-artifacts "$[[ inputs.binaries_artifacts ]]" artifacts: From 6161ea61f192ee7699f72bd911f142fd9c8a6281 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 19 Jun 2026 13:27:12 +0200 Subject: [PATCH 213/229] Removing SSI --- utils/ci/gitlab/main.yml | 12 -- utils/ci/gitlab/ssi.yml | 295 --------------------------------------- 2 files changed, 307 deletions(-) delete mode 100644 utils/ci/gitlab/ssi.yml diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 42393c837e9..9f1fd1d95ab 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -72,18 +72,6 @@ include: inputs: stage: $[[ inputs.stage ]] ref: $[[ inputs.ref ]] - - project: 'DataDog/system-tests' - ref: $[[ inputs.ref ]] - file: 'utils/ci/gitlab/ssi.yml' - if: $[[ inputs.ssi_enabled ]] - inputs: - stage: $[[ inputs.stage ]] - libraries: $[[ inputs.libraries ]] - scenarios: $[[ inputs.scenarios ]] - scenarios_groups: $[[ inputs.scenarios_groups ]] - ssi_library_version: $[[ inputs.ssi_library_version ]] - k8s_lib_init_img: $[[ inputs.k8s_lib_init_img ]] - ssi_injector_version: $[[ inputs.ssi_injector_version ]] workflow: rules: diff --git a/utils/ci/gitlab/ssi.yml b/utils/ci/gitlab/ssi.yml deleted file mode 100644 index baf91f271ed..00000000000 --- a/utils/ci/gitlab/ssi.yml +++ /dev/null @@ -1,295 +0,0 @@ ---- -spec: - inputs: - stage: - description: "CI stage for all jobs in this component" - libraries: - description: "Space- or comma-separated list of libraries to test (e.g. 'java python nodejs' or 'java,python,nodejs')" - default: "java python nodejs" - scenarios: - description: "Comma-separated list of scenarios to run (all if empty)" - default: "" - scenarios_groups: - description: "Comma-separated list of scenario groups to run (all if empty)" - default: "" - ssi_library_version: - description: "DD_INSTALLER_LIBRARY_VERSION to inject into generated jobs (Docker/K8s SSI)" - default: "" - k8s_lib_init_img: - description: "K8S_LIB_INIT_IMG to inject into generated libinjection jobs" - default: "" - ssi_injector_version: - description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" - default: "" - ---- - -# SSI pipeline — mirrors the end-to-end pattern: -# compute_pipeline → generated-ssi-pipeline.yml → run_ssi_pipeline -# Covers all SSI workflows: aws_ssi, dockerssi, libinjection. - -include: - - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/9cf7d7609ff62e4723c9cbc061ca2a25345ce5d6055b9acad9a13dbf736261f0/single-step-instrumentation-tests.yml - -compute_pipeline: - image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 - tags: ["arch:amd64"] - stage: $[[ inputs.stage ]] - variables: - CI_ENVIRONMENT: "prod" - script: - - | - if [ -z "$SYSTEM_TESTS_SCENARIOS" ] && [ -z "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then - echo "❌ Both SYSTEM_TESTS_SCENARIOS and SYSTEM_TESTS_SCENARIOS_GROUPS are not defined. Computing changes..." - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - source venv/bin/activate - ./run.sh MOCK_THE_TEST --collect-only --scenario-report - git clone https://github.com/DataDog/system-tests.git original - git fetch --all - git branch --track $CI_COMMIT_REF_NAME origin/$CI_COMMIT_REF_NAME || true - git checkout $CI_COMMIT_REF_NAME - BASE_COMMIT=$(git merge-base origin/main $CI_COMMIT_REF_NAME) - echo "Branch was created from commit--> $BASE_COMMIT" - git diff --name-only $BASE_COMMIT $CI_COMMIT_SHA >> modified_files.txt - cat modified_files.txt - python utils/scripts/compute_libraries_and_scenarios.py >> impacted_scenarios.txt - cat impacted_scenarios.txt - source impacted_scenarios.txt - else - echo "✅ SYSTEM_TESTS_SCENARIOS OR SYSTEM_TESTS_SCENARIOS_GROUPS are set." - export scenarios=$SYSTEM_TESTS_SCENARIOS - export scenarios_groups=$SYSTEM_TESTS_SCENARIOS_GROUPS - cd /system-tests - git pull - if [ -n "$SYSTEM_TESTS_REF" ]; then - git checkout $SYSTEM_TESTS_REF - else - echo "⚠️ SYSTEM_TESTS_REF variable is not set, skipping git checkout" - fi - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - source venv/bin/activate - fi - - | - [ -n "$[[ inputs.scenarios ]]" ] && scenarios="$[[ inputs.scenarios ]]" - [ -n "$[[ inputs.scenarios_groups ]]" ] && scenarios_groups="$[[ inputs.scenarios_groups ]]" - [ -n "$[[ inputs.ssi_library_version ]]" ] && export DD_INSTALLER_LIBRARY_VERSION="$[[ inputs.ssi_library_version ]]" - [ -n "$[[ inputs.k8s_lib_init_img ]]" ] && export K8S_LIB_INIT_IMG="$[[ inputs.k8s_lib_init_img ]]" - [ -n "$[[ inputs.ssi_injector_version ]]" ] && export DD_INSTALLER_INJECTOR_VERSION="$[[ inputs.ssi_injector_version ]]" - for library in $(echo "$[[ inputs.libraries ]]" | tr ',' ' '); do - python utils/scripts/compute-workflow-parameters.py $library -s "$scenarios" -g "$scenarios_groups" --parametric-job-count 1 --ci-environment "${CI_ENVIRONMENT}" --format gitlab - done - - | - if [ -n "$SYSTEM_TESTS_SCENARIOS" ] || [ -n "$SYSTEM_TESTS_SCENARIOS_GROUPS" ]; then - cp *.yml "$CI_PROJECT_DIR" - fi - rules: - - if: '$SCHEDULED_JOB == ""' - - if: '$SCHEDULED_JOB == null' - artifacts: - paths: - - "*_ssi_gitlab_pipeline.yml" - -# SSI child pipelines run sequentially per language to respect AWS EC2 / subnet -# quotas. Order mirrors the legacy pipeline on main: -# nodejs → java → dotnet → python → php → ruby. -# Each trigger is gated on whether the language is in `inputs.libraries`; -# `needs.optional: true` on every earlier library makes the chain skip -# unrequested languages while keeping order between the ones that are selected. -nodejs_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: nodejs_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /nodejs/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /nodejs/' - when: always - -java_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - - job: nodejs_ssi_pipeline - optional: true - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: java_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /java/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /java/' - when: always - -dotnet_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - - job: nodejs_ssi_pipeline - optional: true - - job: java_ssi_pipeline - optional: true - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: dotnet_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /dotnet/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /dotnet/' - when: always - -python_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - - job: nodejs_ssi_pipeline - optional: true - - job: java_ssi_pipeline - optional: true - - job: dotnet_ssi_pipeline - optional: true - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: python_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /python/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /python/' - when: always - -php_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - - job: nodejs_ssi_pipeline - optional: true - - job: java_ssi_pipeline - optional: true - - job: dotnet_ssi_pipeline - optional: true - - job: python_ssi_pipeline - optional: true - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: php_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /php/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /php/' - when: always - -ruby_ssi_pipeline: - stage: $[[ inputs.stage ]] - needs: - - job: compute_pipeline - - job: nodejs_ssi_pipeline - optional: true - - job: java_ssi_pipeline - optional: true - - job: dotnet_ssi_pipeline - optional: true - - job: python_ssi_pipeline - optional: true - - job: php_ssi_pipeline - optional: true - variables: - PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE - trigger: - include: - - artifact: ruby_ssi_gitlab_pipeline.yml - job: compute_pipeline - strategy: depend - rules: - - if: '$SCHEDULED_JOB == "" && "$[[ inputs.libraries ]]" =~ /ruby/' - - if: '$SCHEDULED_JOB == null && "$[[ inputs.libraries ]]" =~ /ruby/' - when: always - -delete_amis: - extends: .base_job_onboarding - stage: $[[ inputs.stage ]] - allow_failure: false - variables: - AMI_RETENTION_DAYS: 10 - AMI_LAST_LAUNCHED_DAYS: 10 - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - python utils/scripts/pulumi_clean_up.py --component amis --ami-retention-days $AMI_RETENTION_DAYS --ami-last-launched-days $AMI_LAST_LAUNCHED_DAYS - rules: - - if: '$SCHEDULED_JOB == "delete_amis"' - after_script: echo "Finish" - timeout: 3h - -delete_amis_by_name_or_lang: - extends: .base_job_onboarding - stage: $[[ inputs.stage ]] - allow_failure: false - script: - - | - if [ -z "$AMI_NAME" ] && [ -z "$AMI_LANG" ]; then - echo "❌ ERROR: Either AMI_NAME or AMI_LANG must be set." - exit 1 - fi - echo "✅ Proceeding with AMI_NAME=$AMI_NAME, AMI_LANG=$AMI_LANG" - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - | - CMD="python utils/scripts/pulumi_clean_up.py --component amis_by_name" - if [ -n "$AMI_NAME" ]; then CMD="$CMD --ami-name $AMI_NAME"; fi - if [ -n "$AMI_LANG" ]; then CMD="$CMD --ami-lang $AMI_LANG"; fi - echo "Running: $CMD" - eval $CMD - rules: - - if: '$SCHEDULED_JOB == "delete_amis_by_name_or_lang"' - after_script: echo "Finish" - -delete_ec2_instances: - extends: .base_job_onboarding - stage: $[[ inputs.stage ]] - allow_failure: false - variables: - EC2_AGE_MINUTES: 45 - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - python utils/scripts/pulumi_clean_up.py --component ec2 --ec2-age-minutes $EC2_AGE_MINUTES - rules: - - if: '$SCHEDULED_JOB == "delete_ec2_instances"' - after_script: echo "Finish" - -count_amis: - extends: .base_job_onboarding - stage: $[[ inputs.stage ]] - allow_failure: false - script: - - SYSTEM_TEST_BUILD_ATTEMPTS=3 SYSTEM_TEST_BUILD_TIMEOUT=240 ./build.sh -i runner - - source venv/bin/activate - - python utils/scripts/pulumi_clean_up.py --component amis_count - rules: - - if: '$SCHEDULED_JOB == "count_amis"' - after_script: echo "Finish" - -.delayed_base_job: - image: registry.ddbuild.io/ci/libdatadog-build/ci_docker_base:100425777 - tags: ["arch:amd64"] - script: - - echo "⏳ Waiting before triggering the child pipeline..." - when: delayed - start_in: 5 minutes From 16e65b1a6e775d6ae28d6b50e467be0e00fb6218 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 19 Jun 2026 14:49:23 +0200 Subject: [PATCH 214/229] Variable input cleanup --- .gitlab-ci.yml | 6 +++++- utils/ci/gitlab/main.yml | 45 +++++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 32dbcbf7091..4a1fdc6504e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,7 +3,6 @@ include: - local: /utils/ci/gitlab/main.yml inputs: stage: "e2e" - excluded_scenarios: "DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true stages: @@ -26,6 +25,9 @@ variables: MIRROR_IMAGES_URL: "https://binaries.ddbuild.io/dd-repo-tools/default/ca/fb4f39a542e4dd42b646c300b539c7a9f4201531/mirror_images.py" # Destination registry for mirrored CI images. MIRROR_DEST_REGISTRY: "registry.ddbuild.io/system-tests/mirror" + # Timing-gate job from an upstream pipeline that provides pre-built binaries. + # Set to a real job name via a trigger variable to activate cross-pipeline artifact download. + BINARIES_ARTIFACT: "something" compute_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 @@ -370,6 +372,8 @@ system-tests-param: mv param.env.tmp param.env cat param.env fi + - echo 'EXCLUDED_SCENARIOS=DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E' >> param.env + - echo 'SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION=true' >> param.env artifacts: reports: dotenv: param.env diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 9f1fd1d95ab..bf32eac53fe 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -47,18 +47,6 @@ spec: ref: description: "system-tests ref to use when called from another repository (branch, tag or SHA)" default: "main" - ssi_enabled: - description: "Whether to run SSI tests" - default: false - ssi_library_version: - description: "DD_INSTALLER_LIBRARY_VERSION to inject into generated jobs (Docker/K8s SSI)" - default: "" - k8s_lib_init_img: - description: "K8S_LIB_INIT_IMG to inject into generated libinjection jobs" - default: "" - ssi_injector_version: - description: "DD_INSTALLER_INJECTOR_VERSION to inject into generated dockerssi jobs" - default: "" docker_auth: description: "Whether to authenticate calls to docker hub" default: "false" @@ -101,20 +89,35 @@ build_test_pipeline: libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') - weblogs=$(echo "$[[ inputs.weblogs ]]" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') - echo "libraries: $libraries" - echo "scenarios: $scenarios" - echo "scenario_groups: $scenario_groups" - echo "weblogs: $weblogs" + weblogs=$(echo "$[[ inputs.weblogs ]],$WEBLOGS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') + excluded_scenarios=$(echo "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" | tr ',' '\n' | sed '/^$/d' | sort -u | tr '\n' ',' | sed 's/,$//') + force_execute=$(echo "$[[ inputs.force_execute ]],$FORCE_EXECUTE" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') + parametric_job_count="${SYSTEM_TESTS_PARAMETRIC_JOB_COUNT:-$[[ inputs.parametric_job_count ]]}" + push_to_test_optimization="${SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION:-$[[ inputs.push_to_test_optimization ]]}" + docker_auth="${SYSTEM_TESTS_DOCKER_AUTH:-$[[ inputs.docker_auth ]]}" + skip_empty_scenario="${SYSTEM_TESTS_SKIP_EMPTY_SCENARIO:-$[[ inputs.skip_empty_scenarios ]]}" + echo "libraries: $libraries" + echo "scenarios: $scenarios" + echo "scenario_groups: $scenario_groups" + echo "weblogs: $weblogs" + echo "excluded_scenarios: $excluded_scenarios" + echo "force_execute: $force_execute" + echo "parametric_job_count: $parametric_job_count" + echo "push_to_test_opt: $push_to_test_optimization" + echo "skip_empty_scenario: $skip_empty_scenario" + printf 'SYSTEM_TESTS_FORCE_EXECUTE=%s\nSYSTEM_TESTS_SKIP_EMPTY_SCENARIO=%s\n' "$force_execute" "$skip_empty_scenario" > build_params.env binaries_artifact="$[[ inputs.binaries_artifact ]]" [ "$binaries_artifact" = "__no_binaries_artifact__" ] && binaries_artifact="" for library in $libraries; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$[[ inputs.excluded_scenarios ]]" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$binaries_artifact" --parametric-job-count $[[ inputs.parametric_job_count ]] --output params_${library}.json + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$excluded_scenarios" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$binaries_artifact" --parametric-job-count $parametric_job_count --output params_${library}.json done - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$[[ inputs.push_to_test_optimization ]]" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$[[ inputs.docker_auth ]]" --binaries-artifact-path "$[[ inputs.binaries_artifact_path ]]" --binaries-artifacts "$[[ inputs.binaries_artifacts ]]" + binaries_artifact_path="${BINARIES_ARTIFACT_PATH:-$[[ inputs.binaries_artifact_path ]]}" + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$docker_auth" --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$[[ inputs.binaries_artifacts ]]" artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml + reports: + dotenv: system-tests/build_params.env .run_test_pipeline_base: interruptible: true @@ -136,8 +139,8 @@ build_test_pipeline: optional: true artifacts: false variables: - SYSTEM_TESTS_FORCE_EXECUTE: "$[[ inputs.force_execute ]]" - SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$[[ inputs.skip_empty_scenarios ]]" + SYSTEM_TESTS_FORCE_EXECUTE: "$SYSTEM_TESTS_FORCE_EXECUTE" + SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$SYSTEM_TESTS_SKIP_EMPTY_SCENARIO" LIBRARIES: $LIBRARIES UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID From 8faeaf5d6ed3fc10753041fe4964d7d2cb23f064 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 19 Jun 2026 14:55:00 +0200 Subject: [PATCH 215/229] Cleanup --- .gitlab-ci.yml | 3 --- utils/ci/gitlab/system-tests.yml.j2 | 17 ----------------- 2 files changed, 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a1fdc6504e..58af3ded9de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,9 +25,6 @@ variables: MIRROR_IMAGES_URL: "https://binaries.ddbuild.io/dd-repo-tools/default/ca/fb4f39a542e4dd42b646c300b539c7a9f4201531/mirror_images.py" # Destination registry for mirrored CI images. MIRROR_DEST_REGISTRY: "registry.ddbuild.io/system-tests/mirror" - # Timing-gate job from an upstream pipeline that provides pre-built binaries. - # Set to a real job name via a trigger variable to activate cross-pipeline artifact download. - BINARIES_ARTIFACT: "something" compute_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index c1f2d5f55a9..3c724694410 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -41,23 +41,6 @@ variables: DD_STS_OIDC_TOKEN: aud: dd-sts after_script: - - | - ( - set +e - cd "$CI_PROJECT_DIR/system-tests" 2>/dev/null || cd system-tests 2>/dev/null || true - mkdir -p logs - { - echo "===== docker version ====="; docker version - echo "===== docker info ====="; docker info - echo "===== docker network ls ====="; docker network ls - echo "===== docker network inspect system-tests-ipv4 ====="; docker network inspect system-tests-ipv4 - echo "===== docker ps -a ====="; docker ps -a - echo "===== docker inspect system-tests-postgres ====="; docker inspect system-tests-postgres - echo "===== docker inspect system-tests-weblog ====="; docker inspect system-tests-weblog - echo "===== weblog /etc/resolv.conf and DNS lookups =====" - docker exec system-tests-weblog sh -c 'cat /etc/resolv.conf; echo ---; getent hosts postgres; getent hosts agent; getent hosts proxy; getent hosts weblog' - } > logs/network-diagnostics.txt 2>&1 - ) || true - echo -e "section_start:$(date +%s):test_optim[collapsed=true]\r\e[0KPush to tests optimization" - dd-sts debug - > From 63dabb38e7ac002cfc8a1ef48cab77b17dff36f6 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 19 Jun 2026 15:56:28 +0200 Subject: [PATCH 216/229] Binary artifact cleanup --- utils/ci/gitlab/main.yml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index bf32eac53fe..b0098d01b16 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -17,9 +17,6 @@ spec: weblogs: description: "Comma-separated list of weblogs to run (all weblogs if empty)" default: "" - binaries_artifact: - description: "Name of the single upstream job to wait for before triggering the child pipeline (used as the timing gate)" - default: "__no_binaries_artifact__" binaries_artifacts: description: "Semicolon-separated list of upstream jobs whose artifacts contain the pre-built binaries (downloaded by the generated build/run jobs via cross-pipeline needs). Use ';' not ',' because job names may contain commas (e.g. parallel matrix names)." default: "" @@ -106,13 +103,12 @@ build_test_pipeline: echo "push_to_test_opt: $push_to_test_optimization" echo "skip_empty_scenario: $skip_empty_scenario" printf 'SYSTEM_TESTS_FORCE_EXECUTE=%s\nSYSTEM_TESTS_SKIP_EMPTY_SCENARIO=%s\n' "$force_execute" "$skip_empty_scenario" > build_params.env - binaries_artifact="$[[ inputs.binaries_artifact ]]" - [ "$binaries_artifact" = "__no_binaries_artifact__" ] && binaries_artifact="" + binaries_artifacts="${BINARIES_ARTIFACTS:-$[[ inputs.binaries_artifacts ]]}" + binaries_artifact_path="${BINARIES_ARTIFACT_PATH:-$[[ inputs.binaries_artifact_path ]]}" for library in $libraries; do - python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$excluded_scenarios" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$binaries_artifact" --parametric-job-count $parametric_job_count --output params_${library}.json + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$excluded_scenarios" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$binaries_artifacts" --parametric-job-count $parametric_job_count --output params_${library}.json done - binaries_artifact_path="${BINARIES_ARTIFACT_PATH:-$[[ inputs.binaries_artifact_path ]]}" - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$docker_auth" --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$[[ inputs.binaries_artifacts ]]" + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$docker_auth" --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$binaries_artifacts" artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml @@ -135,9 +131,6 @@ build_test_pipeline: - job: mirror_images optional: true artifacts: false - - job: $[[ inputs.binaries_artifact ]] - optional: true - artifacts: false variables: SYSTEM_TESTS_FORCE_EXECUTE: "$SYSTEM_TESTS_FORCE_EXECUTE" SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$SYSTEM_TESTS_SKIP_EMPTY_SCENARIO" From c44196a2f4a3bdece2af55b7f9bfeeff0e9b9190 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 22 Jun 2026 13:53:16 +0200 Subject: [PATCH 217/229] Improved param job --- .gitlab-ci.yml | 33 ++--- .../ci/gitlab/docker/system-tests.Dockerfile | 1 + utils/ci/gitlab/main.yml | 12 +- utils/ci/gitlab/validate_param_env.py | 134 ++++++++++++++++++ 4 files changed, 158 insertions(+), 22 deletions(-) create mode 100644 utils/ci/gitlab/validate_param_env.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58af3ded9de..aaa9efc4e22 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -319,14 +319,12 @@ generate_system_tests_lib_injection_images: # ────────────────────────────────────────────── system-tests-param: - image: $CI_IMAGE + extends: .system_tests_param_base stage: e2e - tags: - - arch:amd64 needs: - job: build_ci_image artifacts: false - script: + before_script: - ln -sf /system-tests/venv venv - source venv/bin/activate - export PYTHONPATH=$(pwd) @@ -337,26 +335,25 @@ system-tests-param: - python3 utils/scripts/compute_libraries_and_scenarios.py --output raw_params.txt - | python3 -c " - import json + import json, shlex with open('raw_params.txt') as f: for line in f: line = line.strip() if '=' not in line: continue key, val = line.split('=', 1) val = json.loads(val) - if key == 'libraries': print(f'LIBRARIES={val}') - elif key == 'scenarios': print(f'SCENARIOS={val}') - elif key == 'scenarios_groups': print(f'SCENARIO_GROUPS={val}') - " > param.env - - cat param.env + if key == 'libraries': print('export LIBRARIES=' + shlex.quote(str(val))) + elif key == 'scenarios': print('export SCENARIOS=' + shlex.quote(str(val))) + elif key == 'scenarios_groups': print('export SCENARIO_GROUPS=' + shlex.quote(str(val))) + " > set_vars.sh + source set_vars.sh # TEMPORARY: filter LIBRARIES via E2E_SUPPORTED_LANGUAGES (space-separated allowlist) # used to control rollout, to be removed once rollout is complete. - | if [ -n "${E2E_SUPPORTED_LANGUAGES:-}" ]; then echo "Filtering LIBRARIES with E2E_SUPPORTED_LANGUAGES='$E2E_SUPPORTED_LANGUAGES'" - current=$(grep '^LIBRARIES=' param.env | sed 's/^LIBRARIES=//') filtered="" - for lib in $current; do + for lib in $LIBRARIES; do for allowed in $E2E_SUPPORTED_LANGUAGES; do if [ "$lib" = "$allowed" ]; then filtered="${filtered:+$filtered }$lib" @@ -364,16 +361,10 @@ system-tests-param: fi done done - grep -v '^LIBRARIES=' param.env > param.env.tmp || true - echo "LIBRARIES=$filtered" >> param.env.tmp - mv param.env.tmp param.env - cat param.env + export LIBRARIES="$filtered" fi - - echo 'EXCLUDED_SCENARIOS=DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E' >> param.env - - echo 'SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION=true' >> param.env - artifacts: - reports: - dotenv: param.env + - export EXCLUDED_SCENARIOS='DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E' + - export SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION=true build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy diff --git a/utils/ci/gitlab/docker/system-tests.Dockerfile b/utils/ci/gitlab/docker/system-tests.Dockerfile index 149e9c268ae..f8f017f9a59 100644 --- a/utils/ci/gitlab/docker/system-tests.Dockerfile +++ b/utils/ci/gitlab/docker/system-tests.Dockerfile @@ -53,5 +53,6 @@ COPY --from=registry.ddbuild.io/dd-sts:v0.1.4@sha256:1f4bc8861cca86b0c977ae70843 /usr/local/bin/dd-sts /usr/local/bin/dd-sts COPY --from=builder /system-tests/venv /system-tests/venv +COPY --from=builder /system-tests/utils/ci/gitlab/validate_param_env.py /system-tests/utils/ci/gitlab/validate_param_env.py WORKDIR / diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index b0098d01b16..27b22c546fe 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -68,7 +68,17 @@ variables: # Tag = first 12 chars of sha256(utils/ci/gitlab/docker/system-tests.Dockerfile + requirements.txt) # Update this when either file changes: # cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12 - CI_IMAGE: "registry.ddbuild.io/system-tests/ci-runner:6bf5fa49e86c" + CI_IMAGE: "registry.ddbuild.io/system-tests/ci-runner:f4f0d91aa8dc" + +.system_tests_param_base: + image: $CI_IMAGE + tags: + - arch:amd64 + script: + - python3 /system-tests/utils/ci/gitlab/validate_param_env.py + artifacts: + reports: + dotenv: param.env build_test_pipeline: extends: .system_tests_base diff --git a/utils/ci/gitlab/validate_param_env.py b/utils/ci/gitlab/validate_param_env.py new file mode 100644 index 00000000000..51a9f64b8ae --- /dev/null +++ b/utils/ci/gitlab/validate_param_env.py @@ -0,0 +1,134 @@ +"""Validate environment variables and write them to param.env. + +Called by .system_tests_param_base in main.yml. The caller (before_script) is +responsible for exporting the variables into the environment before this script runs. +""" + +from __future__ import annotations + +import os +import re +import sys + + +def _error(name: str, value: str, msg: str) -> str: + return f"❌ {name}: '{value}' — {msg}" + + +def validate_space_lower(name: str, value: str) -> list[str]: + """Space-separated lowercase identifiers, e.g. 'python java_otel'.""" + errors = [] + for item in value.split(): + if not re.match(r"^[a-z][a-z0-9_]*$", item): + errors.append( + _error(name, item, "expected space-separated lowercase identifiers (e.g. 'python java_otel')") + ) + return errors + + +def validate_comma_upper(name: str, value: str) -> list[str]: + """Comma-separated uppercase identifiers, e.g. 'DEFAULT,PARAMETRIC'.""" + errors = [] + for item in (i.strip() for i in value.split(",")): + if item and not re.match(r"^[A-Z][A-Z0-9_]*$", item): + errors.append( + _error(name, item, "expected comma-separated uppercase identifiers (e.g. 'DEFAULT,PARAMETRIC')") + ) + return errors + + +def validate_comma_lower(name: str, value: str) -> list[str]: + """Comma-separated lowercase identifiers, e.g. 'all,appsec'.""" + errors = [] + for item in (i.strip() for i in value.split(",")): + if item and not re.match(r"^[a-z][a-z0-9_]*$", item): + errors.append( + _error(name, item, "expected comma-separated lowercase identifiers (e.g. 'all,appsec')") + ) + return errors + + +def validate_comma_lower_hyphen(name: str, value: str) -> list[str]: + """Comma-separated lowercase identifiers allowing hyphens, e.g. 'flask,django-realworld'.""" + errors = [] + for item in (i.strip() for i in value.split(",")): + if item and not re.match(r"^[a-z][a-z0-9_-]*$", item): + errors.append( + _error(name, item, "expected comma-separated lowercase identifiers (e.g. 'flask,django-realworld')") + ) + return errors + + +def validate_bool(name: str, value: str) -> list[str]: + """Must be 'true' or 'false'.""" + if value not in ("true", "false"): + return [_error(name, value, "expected 'true' or 'false'")] + return [] + + +def validate_positive_int(name: str, value: str) -> list[str]: + """Must be a positive integer.""" + if not re.match(r"^[1-9][0-9]*$", value): + return [_error(name, value, "expected a positive integer")] + return [] + + +def validate_path(name: str, value: str) -> list[str]: + """Must be a valid relative path.""" + if not re.match(r"^[a-zA-Z0-9_][a-zA-Z0-9_./-]*$", value): + return [_error(name, value, "expected a valid relative path")] + return [] + + +def validate_semicolon_nonempty(name: str, value: str) -> list[str]: + """Semicolon-separated non-empty entries, e.g. job names.""" + errors = [] + for item in (i.strip() for i in value.split(";")): + if not item: + errors.append(_error(name, value, "empty entry found — expected semicolon-separated non-empty job names")) + return errors + + +CHECKS: list[tuple[str, object]] = [ + ("LIBRARIES", validate_space_lower), + ("SCENARIOS", validate_comma_upper), + ("SCENARIO_GROUPS", validate_comma_lower), + ("WEBLOGS", validate_comma_lower_hyphen), + ("EXCLUDED_SCENARIOS", validate_comma_upper), + ("FORCE_EXECUTE", None), # pytest node IDs are too complex to validate by format + ("SYSTEM_TESTS_PARAMETRIC_JOB_COUNT", validate_positive_int), + ("SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION", validate_bool), + ("SYSTEM_TESTS_DOCKER_AUTH", validate_bool), + ("SYSTEM_TESTS_SKIP_EMPTY_SCENARIO", validate_bool), + ("BINARIES_ARTIFACTS", validate_semicolon_nonempty), + ("BINARIES_ARTIFACT_PATH", validate_path), +] + +PARAM_ENV = "param.env" + + +def main() -> None: + errors: list[str] = [] + + for name, validator in CHECKS: + value = os.environ.get(name, "") + if value and validator is not None: + errors.extend(validator(name, value)) # type: ignore[operator] + + if errors: + for e in errors: + print(e, file=sys.stderr) + sys.exit(1) + + with open(PARAM_ENV, "w") as f: + for name, _ in CHECKS: + value = os.environ.get(name, "") + if value: + f.write(f"{name}={value}\n") + + with open(PARAM_ENV) as f: + print(f.read(), end="") + + +if __name__ == "__main__": + main() From e2bd59f9f8c7cb9f735e22d583d3b5ecb593663a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 22 Jun 2026 14:38:10 +0200 Subject: [PATCH 218/229] cleanup --- .../test_compute_libraries_and_scenarios.py | 3 +-- .../test_external_gitlab_pipeline.py | 10 ---------- utils/_context/containers.py | 20 +++++++++++-------- utils/ci/gitlab/build_pipeline.py | 7 ++++--- utils/ci/gitlab/system-tests.yml.j2 | 2 +- .../compute_libraries_and_scenarios.py | 10 +++++----- 6 files changed, 23 insertions(+), 29 deletions(-) diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index b85b2d4df86..447c7b65fb2 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -512,7 +512,7 @@ def test_empty_ref(self, monkeypatch: pytest.MonkeyPatch): inputs = build_inputs() assert inputs.ref == "" - def test_libraries_output_sorted_no_rust(self, monkeypatch: pytest.MonkeyPatch): + def test_libraries_output_sorted(self, monkeypatch: pytest.MonkeyPatch): monkeypatch.setenv("GITLAB_CI", "true") monkeypatch.setenv("CI_PIPELINE_SOURCE", "push") monkeypatch.setenv("CI_COMMIT_REF_NAME", "feat-x") @@ -523,5 +523,4 @@ def test_libraries_output_sorted_no_rust(self, monkeypatch: pytest.MonkeyPatch): assert libs_line is not None, "GitLab mode must emit 'libraries=' output" libs = json.loads(libs_line.split("=", 1)[1]) parts = libs.split() - assert "rust" not in parts assert parts == sorted(parts) diff --git a/tests/test_the_test/test_external_gitlab_pipeline.py b/tests/test_the_test/test_external_gitlab_pipeline.py index 32252bff9dd..6dd7fee9fed 100644 --- a/tests/test_the_test/test_external_gitlab_pipeline.py +++ b/tests/test_the_test/test_external_gitlab_pipeline.py @@ -1,6 +1,5 @@ """Tests for utils/scripts/ci_orchestrators/external_gitlab_pipeline.py.""" -import yaml import pytest from utils import scenarios @@ -8,7 +7,6 @@ _is_local_include, _strip_local_includes, filter_yaml, - main, ) @@ -32,14 +30,6 @@ def test_strip_local_includes_removes_only_local(self): _strip_local_includes(data) assert data["include"] == [remote] - def test_main_strips_locals_from_real_repo_yaml(self, capsys: pytest.CaptureFixture): - main(language=None) - out = capsys.readouterr().out - data = yaml.safe_load(out) - for entry in data.get("include", []): - if isinstance(entry, dict): - assert "local" not in entry - def test_filter_yaml_keeps_only_requested_language(self): data = { "stages": ["configure", "python", "java", "pipeline-status"], diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 9d25984670b..513331d88b6 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -527,7 +527,7 @@ def __init__(self, image_name: str, *, local_image_only: bool): self.env: dict[str, str] | None = None self.labels: dict[str, str] = {} # When the image mirror is enabled (USE_IMAGE_MIRROR), pull from the - # mirror; keep the original ref to fall back to if it isn't mirrored. + # mirror; keep the original ref for diagnostics if the mirror pull fails. self.original_name = image_name self.name = mirror_image(image_name) self.local_image_only = local_image_only @@ -561,13 +561,17 @@ def load(self): try: self._image = self._pull_with_retries() except (docker.errors.APIError, requests.exceptions.ConnectionError): - # If the mirrored ref can't be pulled (e.g. not mirrored yet), - # fall back to the original registry. - if self.name == self.original_name: - raise - logger.stdout(f"Could not pull mirrored image {self.name}, falling back to {self.original_name}") - self.name = self.original_name - self._image = self._pull_with_retries() + if self.name != self.original_name: + # The mirror rewrote this ref but the mirror copy could not be + # pulled. Don't silently fall back to the original registry: + # that would mask an incomplete mirror and could pull a + # different digest than the one that was mirrored. + logger.stdout( + f"Could not pull mirrored image {self.name} (mirror of {self.original_name}). " + "The mirror may be incomplete — regenerate it with " + "utils/scripts/update_mirror_images.py and check the mirror_images CI job." + ) + raise self._init_from_attrs(self._image.attrs) diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index ca02b5fa0d7..04bac8a42f5 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -143,7 +143,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") parser.add_argument("--output-dir", required=True, help="Output directory for generated-pipeline-chunk-N.yml files") parser.add_argument("--chunks", type=int, default=3, help="Number of pipeline chunks (default: 3)") - parser.add_argument("--docker-auth", default="false", help="Wether to authenticate calls to docker hub") + parser.add_argument("--docker-auth", default="false", help="Whether to authenticate calls to docker hub") parser.add_argument( "--binaries-artifact-path", default="", @@ -152,8 +152,9 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument( "--binaries-artifacts", default="", - help="Comma-separated list of upstream jobs to download artifacts from in the child pipeline " - "(falls back to the single binaries_artifact from params if empty)", + help="Semicolon-separated list of upstream jobs to download artifacts from in the child pipeline " + "(';' not ',' because job names may contain commas, e.g. parallel matrix names; " + "falls back to the single binaries_artifact from params if empty)", ) args = parser.parse_args(argv) diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index 3c724694410..cefac1d8be5 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -117,7 +117,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% endif %} - section_end "weblog_setup" - section_start "scenario_run" "Running scenario" - {% if scenario in ("INTEGRATION_FRAMEWORKS") %} + {% if scenario == "INTEGRATION_FRAMEWORKS" %} - ./run.sh {{scenario}} -L {{library}} --weblog {{variant}} {% elif scenario in ("GO_PROXIES_DEFAULT", "GO_PROXIES_APPSEC_BLOCKING") %} - ./run.sh {{scenario}} --weblog {{variant}} diff --git a/utils/scripts/compute_libraries_and_scenarios.py b/utils/scripts/compute_libraries_and_scenarios.py index cf286d032b7..dff74aca5fb 100644 --- a/utils/scripts/compute_libraries_and_scenarios.py +++ b/utils/scripts/compute_libraries_and_scenarios.py @@ -347,6 +347,10 @@ def load_scenario_mappings(self) -> None: self.scenario_map = json.load(f) +def extra_gitlab_output(inputs: Inputs) -> dict[str, str]: + return {"CI_PIPELINE_SOURCE": inputs.event_name, "CI_COMMIT_REF_NAME": inputs.ref} + + def stringify_outputs(outputs: dict[str, Any]) -> list[str]: ret = [] for name, value in outputs.items(): @@ -366,10 +370,6 @@ def print_ci_outputs(strings_out: list[str], f: Any) -> None: # noqa: ANN401 print_ci_outputs(strings_out, sys.stdout) -def extra_gitlab_output(inputs: Inputs) -> dict[str, str]: - return {"CI_PIPELINE_SOURCE": inputs.event_name, "CI_COMMIT_REF_NAME": inputs.ref} - - def process(inputs: Inputs) -> list[str]: outputs: dict[str, Any] = {} if inputs.is_gitlab: @@ -407,7 +407,7 @@ def process(inputs: Inputs) -> list[str]: library_processor.selected |= scenario_processor.impacted_libraries if inputs.is_gitlab: - libraries = " ".join(sorted(lib for lib in library_processor.selected if lib not in ("rust",))) + libraries = " ".join(sorted(library_processor.selected)) if libraries: outputs["libraries"] = libraries outputs |= scenario_processor.get_outputs() From e52ea4b5a10266528a67322af12ab4c5f806532e Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Mon, 22 Jun 2026 17:01:42 +0200 Subject: [PATCH 219/229] Skip condition --- utils/ci/gitlab/main.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index 27b22c546fe..a048019dd66 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -44,6 +44,9 @@ spec: ref: description: "system-tests ref to use when called from another repository (branch, tag or SHA)" default: "main" + condition: + description: "GitLab CI rule expression controlling whether system-tests jobs run (e.g. '$NIGHTLY_BUILD == \"true\"'). Defaults to always-run." + default: "null == null" docker_auth: description: "Whether to authenticate calls to docker hub" default: "false" @@ -84,6 +87,9 @@ build_test_pipeline: extends: .system_tests_base tags: - arch:amd64 + rules: + - if: $[[ inputs.condition ]] + - when: never needs: - job: build_ci_image optional: true @@ -94,6 +100,14 @@ build_test_pipeline: script: - | libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) + if [ -z "$libraries" ]; then + echo "No libraries configured, producing noop pipeline chunks" + for i in 0 1 2; do + printf 'noop:\n image: registry.ddbuild.io/images/mirror/alpine:latest\n tags:\n - arch:amd64\n script:\n - echo "no system tests configured"\n' > generated-pipeline-chunk-${i}.yml + done + touch build_params.env + exit 0 + fi scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') weblogs=$(echo "$[[ inputs.weblogs ]],$WEBLOGS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') @@ -129,7 +143,7 @@ build_test_pipeline: interruptible: true stage: $[[ inputs.stage ]] rules: - - if: '"$[[ inputs.libraries ]],$LIBRARIES" =~ /\w/' + - if: $[[ inputs.condition ]] - when: never needs: - job: build_test_pipeline From 2f500dda0a733a643dc41d017261c8af28037c2a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 24 Jun 2026 16:06:07 +0200 Subject: [PATCH 220/229] Use buildx for forwarding pulls to the mirror --- .gitlab-ci.yml | 4 +- utils/_context/_image_mirror.py | 14 +- utils/build/build.sh | 19 +- utils/build/docker/buildkitd.toml | 15 + utils/scripts/mirror_images.py | 1658 ++++++++++++++++++++ utils/scripts/mirror_rewrite_dockerfile.py | 47 - utils/scripts/update_mirror_images.py | 15 +- 7 files changed, 1705 insertions(+), 67 deletions(-) create mode 100644 utils/build/docker/buildkitd.toml create mode 100755 utils/scripts/mirror_images.py delete mode 100644 utils/scripts/mirror_rewrite_dockerfile.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aaa9efc4e22..343e1dc7ef6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,8 +21,6 @@ variables: TEST: 1 # TEMPORARY: rollout-control allowlist for system-tests-param LIBRARIES filter. E2E_SUPPORTED_LANGUAGES: "python" - # dd-repo-tools mirror_images.py (pinned). Used by the mirror_images_* jobs. - MIRROR_IMAGES_URL: "https://binaries.ddbuild.io/dd-repo-tools/default/ca/fb4f39a542e4dd42b646c300b539c7a9f4201531/mirror_images.py" # Destination registry for mirrored CI images. MIRROR_DEST_REGISTRY: "registry.ddbuild.io/system-tests/mirror" @@ -441,6 +439,6 @@ mirror_images: exit 1 fi # Push any locked image missing from the mirror (already-present images are skipped). - - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror + - uv run --no-config --script utils/scripts/mirror_images.py mirror rules: - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' diff --git a/utils/_context/_image_mirror.py b/utils/_context/_image_mirror.py index db3a6882400..56a13423824 100644 --- a/utils/_context/_image_mirror.py +++ b/utils/_context/_image_mirror.py @@ -9,9 +9,17 @@ the mirror is not reachable) every reference is returned unchanged. The source -> target mapping is read from mirror_images.lock.yaml, which holds -the exact target for each mirrored image. Anything not listed there (locally -built `system_tests/*` images, unmirrored refs, multi-stage build stage names) -is left untouched. +the exact target for each mirrored image. Mirror paths are laid out so that +BuildKit's per-registry mirror protocol works without Dockerfile rewriting: + + docker.io official images -> mirror/library/ (e.g. library/node:22-alpine) + docker.io namespaced -> mirror// + ghcr.io -> mirror/ghcr.io/ + mcr.microsoft.com -> mirror/mcr.microsoft.com/ + public.ecr.aws -> mirror/public.ecr.aws/ + +Anything not listed in the lock file (locally built `system_tests/*` images, +unmirrored refs) is left untouched. """ import functools diff --git a/utils/build/build.sh b/utils/build/build.sh index 186565ed2bd..fc902452eaa 100755 --- a/utils/build/build.sh +++ b/utils/build/build.sh @@ -303,14 +303,17 @@ build() { DOCKERFILE=utils/build/docker/${TEST_LIBRARY}/${WEBLOG_VARIANT}.Dockerfile - # When the image mirror is enabled, rewrite the weblog base images - # (FROM ...) to pull from registry.ddbuild.io/system-tests/mirror. + # When the image mirror is enabled, create (or reuse) a buildx + # builder whose buildkitd daemon is configured to redirect all + # FROM pulls through registry.ddbuild.io/system-tests/mirror. + MIRROR_BUILDER_ARG="" if [[ "${USE_IMAGE_MIRROR:-}" =~ ^(1|true|yes)$ ]]; then - MIRRORED_DOCKERFILE=$(mktemp /tmp/system-tests-weblog-XXXXXX.Dockerfile) - python utils/scripts/mirror_rewrite_dockerfile.py "${DOCKERFILE}" > "${MIRRORED_DOCKERFILE}" - echo "Using mirrored Dockerfile (${DOCKERFILE} -> mirror):" - grep '^FROM' "${MIRRORED_DOCKERFILE}" || true - DOCKERFILE="${MIRRORED_DOCKERFILE}" + if ! docker buildx inspect system-tests-mirror &>/dev/null 2>&1; then + docker buildx create \ + --name system-tests-mirror \ + --config "${SCRIPT_DIR}/docker/buildkitd.toml" + fi + MIRROR_BUILDER_ARG="--builder system-tests-mirror" fi GITHUB_TOKEN_SECRET_ARG="" @@ -348,6 +351,7 @@ build() { -t system_tests/weblog \ $CACHE_TO \ $CACHE_FROM \ + $MIRROR_BUILDER_ARG \ $EXTRA_DOCKER_ARGS \ . @@ -360,6 +364,7 @@ build() { ${DOCKER_PLATFORM_ARGS} \ -f utils/build/docker/overwrite_waf_rules.Dockerfile \ -t system_tests/weblog \ + $MIRROR_BUILDER_ARG \ $EXTRA_DOCKER_ARGS \ . fi diff --git a/utils/build/docker/buildkitd.toml b/utils/build/docker/buildkitd.toml new file mode 100644 index 00000000000..ebd07e1ab6e --- /dev/null +++ b/utils/build/docker/buildkitd.toml @@ -0,0 +1,15 @@ +# Auto-generated by: mirror_images buildkitd +# Do not edit manually — regenerate with: +# mirror_images buildkitd + +[registry."docker.io"] + mirrors = ["registry.ddbuild.io/system-tests/mirror"] + +[registry."ghcr.io"] + mirrors = ["registry.ddbuild.io/system-tests/mirror/ghcr.io"] + +[registry."mcr.microsoft.com"] + mirrors = ["registry.ddbuild.io/system-tests/mirror/mcr.microsoft.com"] + +[registry."public.ecr.aws"] + mirrors = ["registry.ddbuild.io/system-tests/mirror/public.ecr.aws"] diff --git a/utils/scripts/mirror_images.py b/utils/scripts/mirror_images.py new file mode 100755 index 00000000000..3e0288d1524 --- /dev/null +++ b/utils/scripts/mirror_images.py @@ -0,0 +1,1658 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.10" +# dependencies = ["pyyaml==6.0.3", "dulwich==1.2.4"] +# /// +"""Manage mirrored Docker images: generate lock files, mirror, and lint. + +Uses CLI tools (crane, skopeo, docker, podman, nerdctl) for registry +operations instead of Python registry libraries. Whichever tool is +available on PATH will be used. +""" + +import argparse +import collections +import fnmatch +import functools +import glob +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import threading +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field + +import yaml +from dulwich.errors import NotGitRepository +from dulwich.repo import Repo + +# Force line-buffered output so parallel progress lines appear immediately. +print = functools.partial(print, flush=True) # type: ignore[assignment] + +# Log-line prefix convention: human-facing diagnostics use " [level] message" +# with level in {warning, error, fatal}. Stick to this so log filters and +# eyeballed output stay consistent across modules. + +# Mirror destination registry is resolved lazily via `_dest_registry()` +# from `MIRROR_DEST_REGISTRY` or the origin remote — see the helper below. + + +class Digest(str): + """A `sha256:<64 hex>` container image digest. + + Subclassing `str` so it round-trips through YAML and string formatting + transparently while still rejecting malformed values at construction. + """ + _RE = re.compile(r"^sha256:[0-9a-f]{64}$") + + def __new__(cls, value: str) -> "Digest": + if not cls._RE.match(value): + raise ValueError( + f"invalid digest (expected sha256:<64 hex>): {value!r}") + return super().__new__(cls, value) + + +# Substrings that strongly indicate an authentication/authorization failure +# from a container registry CLI's stderr. Used both to fail-fast in mirror +# Phase 2 and to distinguish "not present" from "auth broken" in checks. +_AUTH_ERROR_SIGNALS = ("unauthorized", "401", "403", "denied", "forbidden", + "authentication required") +# Substrings registries / CLI tools use for "manifest not found" / 404. +_NOT_FOUND_SIGNALS = ("manifest_unknown", "manifest unknown", "not found", + "name unknown", "no such manifest", "404") + + +def _is_auth_error(stderr: str) -> bool: + s = stderr.lower() + return any(sig in s for sig in _AUTH_ERROR_SIGNALS) + + +def _is_not_found(stderr: str) -> bool: + s = stderr.lower() + return any(sig in s for sig in _NOT_FOUND_SIGNALS) + + +def _find_project_dir() -> str: + """Locate the project root: walk up from cwd until a `.git` entry is found. + + `MIRROR_IMAGES_PROJECT_DIR` overrides the auto-detection and is used as-is. + Falls back to cwd if no `.git` is found anywhere up the tree, in which + case path lookups will fail later with a clear FileNotFoundError. + """ + override = os.environ.get("MIRROR_IMAGES_PROJECT_DIR") + if override: + return override + + current = os.getcwd() + while True: + if os.path.exists(os.path.join(current, ".git")): + return current + parent = os.path.dirname(current) + if parent == current: + return os.getcwd() + current = parent + + +PROJECT_DIR = _find_project_dir() +MIRROR_YAML = os.path.join(PROJECT_DIR, "mirror_images.yaml") +LOCK_YAML = os.path.join(PROJECT_DIR, "mirror_images.lock.yaml") + + +def _detect_repo_name(project_dir: str) -> str | None: + """Best-effort repo name from the `origin` remote URL. + + Returns the last path segment with a trailing `.git` stripped, for + both SSH (`git@host:org/repo.git`) and HTTPS URLs. None when the + directory isn't a git repo or has no `origin` remote. + """ + try: + repo = Repo(project_dir) + except NotGitRepository: + return None + try: + url = repo.get_config().get((b"remote", b"origin"), b"url").decode() + except KeyError: + return None + finally: + repo.close() + url = url.strip().removesuffix(".git") + name = url.rsplit("/", 1)[-1].rsplit(":", 1)[-1] + return name or None + + +@functools.cache +def _dest_registry() -> str: + """Mirror destination registry, resolved lazily so non-mirroring + subcommands (`add`, `--help`) work outside a git checkout. + + `MIRROR_DEST_REGISTRY` wins; otherwise build it from the origin + remote name; otherwise exit with instructions. + """ + env = os.environ.get("MIRROR_DEST_REGISTRY") + if env: + return env + repo = _detect_repo_name(PROJECT_DIR) + if repo: + return f"registry.ddbuild.io/ci/{repo}/mirror" + print( + " [error] cannot determine mirror destination registry — no\n" + " 'origin' remote found in this checkout. Set\n" + " MIRROR_DEST_REGISTRY=registry.ddbuild.io/ci//mirror.", + file=sys.stderr, + ) + sys.exit(2) + +# --------------------------------------------------------------------------- +# Registry CLI abstraction +# --------------------------------------------------------------------------- + + +def _find_tool(names: list[str]) -> str | None: + """Return the first tool name found on PATH.""" + for name in names: + if shutil.which(name): + return name + return None + + +def _find_digest_tool() -> str: + """Find a tool that can report registry-canonical image digests. + + Only crane, skopeo, and `docker buildx imagetools` surface the canonical + manifest digest. `docker manifest inspect`, `podman manifest inspect` + and `nerdctl manifest inspect` return reformatted JSON whose hash does + not match the registry's bytes, so they're rejected — the resulting + lock file would pin digests that the source registry does not store. + """ + tool = _find_tool(["crane", "skopeo", "docker"]) + if not tool: + print( + "No digest tool found. Install one of: crane, skopeo, docker (with buildx).", + file=sys.stderr) + sys.exit(1) + return tool + + +def _find_copy_tool() -> str: + """Find a tool that can copy images between registries.""" + tool = _find_tool(["crane", "skopeo", "docker"]) + if not tool: + print( + "No image copy tool found. Install one of: crane, skopeo, docker", + file=sys.stderr) + sys.exit(1) + return tool + + +def resolve_digest(image_ref: str, tool: str) -> Digest: + """Resolve image:tag -> sha256:... using the given CLI tool. + + All three supported tools return the registry-canonical manifest digest: + `crane digest`, `skopeo inspect --format '{{.Digest}}'`, and + `docker buildx imagetools inspect --format '{{.Manifest.Digest}}'`. + Plain `docker manifest inspect` does *not* — it returns reformatted + JSON whose sha256 doesn't match the registry's bytes — so we use the + buildx imagetools path instead. + """ + if tool == "crane": + cmd = ["crane", "digest", image_ref] + elif tool == "skopeo": + cmd = [ + "skopeo", "inspect", "--format", "{{.Digest}}", + f"docker://{image_ref}", + ] + elif tool == "docker": + cmd = [ + "docker", "buildx", "imagetools", "inspect", + "--format", "{{.Manifest.Digest}}", image_ref, + ] + else: + raise ValueError(f"Unknown tool: {tool}") + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError( + f"{tool} failed for {image_ref}: {result.stderr.strip()}") + + return Digest(result.stdout.strip()) + + +def check_digest_exists(image_ref_with_digest: str, tool: str) -> bool: + """Return True if digest is present at the target, False if confirmed absent. + + Raises RuntimeError on auth/network/registry errors so the caller can + distinguish "needs copy" (404) from "operator must investigate" + (everything else). Treating any non-zero exit as "needs copy" hides + expired credentials behind a flood of redundant copy attempts. + """ + if tool == "crane": + cmd = ["crane", "manifest", image_ref_with_digest] + elif tool == "skopeo": + cmd = [ + "skopeo", "inspect", "--raw", + f"docker://{image_ref_with_digest}", + ] + elif tool == "docker": + cmd = [ + "docker", "buildx", "imagetools", "inspect", + image_ref_with_digest, + ] + else: + raise ValueError(f"Unknown tool: {tool}") + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode == 0: + return True + + stderr = result.stderr.strip() + if _is_not_found(stderr): + return False + + raise RuntimeError( + f"{tool} manifest check failed for {image_ref_with_digest} " + f"(exit {result.returncode}): {stderr or '(no stderr)'}") + + +def copy_image(source_ref: str, target: str, tool: str) -> tuple[bool, str]: + """Copy an image from source to target. Returns (success, error_msg).""" + if tool == "crane": + cmd = [tool, "copy", "--platform", "all", source_ref, target] + elif tool == "skopeo": + cmd = [ + tool, "copy", "--all", f"docker://{source_ref}", + f"docker://{target}" + ] + elif tool == "docker": + cmd = [ + tool, "buildx", "imagetools", "create", "-t", target, source_ref + ] + else: + raise ValueError(f"Unknown copy tool: {tool}") + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + return False, result.stderr.strip() + return True, "" + + +def list_tags(image_repo: str, tool: str) -> list[str]: + """List all tags for a repository. + + Only crane and skopeo expose tag listing; docker has no equivalent in + the buildx/imagetools surface. When the active tool is docker we skip + vague-tag pinning instead of failing the run. + """ + if tool == "docker": + print(" [warning] docker has no tag-listing API; " + "version pinning for vague tags will be skipped. Install " + "crane or skopeo for full tag resolution.", file=sys.stderr) + return [] + + if tool == "crane": + result = subprocess.run(["crane", "ls", image_repo], + capture_output=True, + text=True) + elif tool == "skopeo": + result = subprocess.run( + ["skopeo", "list-tags", f"docker://{image_repo}"], + capture_output=True, + text=True, + ) + else: + raise ValueError(f"Unknown tool for tag listing: {tool}") + + if result.returncode != 0: + print( + f" [warning] failed to list tags for {image_repo}: " + f"{result.stderr.strip()}", + file=sys.stderr) + return [] + + if tool == "crane": + return result.stdout.strip().splitlines() + + try: + data = json.loads(result.stdout) + return data.get("Tags", []) + except json.JSONDecodeError as exc: + print(f" [warning] failed to parse tags for {image_repo}: {exc}", + file=sys.stderr) + return [] + + +# --------------------------------------------------------------------------- +# Image reference parsing & YAML config +# --------------------------------------------------------------------------- + + +def parse_image_ref(raw: str) -> str: + """Normalize a Docker image reference to registry/repo:tag form.""" + ref = raw.strip() + parts = ref.split("/", 1) + if len(parts) == 1: + registry = "index.docker.io" + repo = f"library/{ref}" + elif "." in parts[0] or ":" in parts[0]: + registry = parts[0] + repo = parts[1] + else: + registry = "index.docker.io" + repo = ref + + if ":" not in repo: + repo = f"{repo}:latest" + + return f"{registry}/{repo}" + + +@dataclass(frozen=True) +class MirrorEntry: + """A single entry in mirror_images.yaml. + + Two on-disk shapes exist so the common case stays terse: + + - alpine:3.20 + + when the destination follows the default `DEST_REGISTRY/` + convention, or a single-key mapping with an explicit override: + + - nginx:1.27.5: + target: registry.ddbuild.io/ci/libdatadog-build/custom/nginx:1.27.5 + + `to_yaml` round-trips back to whichever form the entry's `target` + implies; an unset `target` always emits the bare-string form. + + Two computed attributes are available but not persisted to YAML: + + ``source_registry`` + The normalised source registry hostname (e.g. ``docker.io``, + ``ghcr.io``). Stored eagerly at construction time. + + ``target_registry`` + The mirror host+path prefix for this source registry, e.g. + ``registry.ddbuild.io/ci/myrepo/mirror`` for ``docker.io`` or + ``registry.ddbuild.io/ci/myrepo/mirror/ghcr.io`` for ``ghcr.io``. + Computed lazily. + """ + source: str + target: str | None = None + source_registry: str = field(init=False) + + def __post_init__(self) -> None: + raw_registry = parse_image_ref(self.source).split("/", 1)[0] + registry = "docker.io" if raw_registry == "index.docker.io" else raw_registry + object.__setattr__(self, "source_registry", registry) + + @property + def target_registry(self) -> "str | None": + """Mirror prefix for this source registry. + """ + image_path = ( + self.source if self.source_registry == "docker.io" + else self.source[len(self.source_registry) + 1:] + ) + t = self.resolved_target() + suffix = f"/{image_path}" + return t[: -len(suffix)] if t.endswith(suffix) else None + + def resolved_target(self) -> str: + return self.target if self.target is not None else f"{_dest_registry()}/{self.source}" + + @classmethod + def from_yaml(cls, raw: object) -> "MirrorEntry": + if isinstance(raw, str): + return cls(source=raw) + if isinstance(raw, dict) and len(raw) == 1: + (source, opts), = raw.items() + if not isinstance(source, str): + raise ValueError( + f"expected entry key to be str, got {type(source).__name__}: {source!r}") + if opts is None: + opts = {} + elif not isinstance(opts, dict): + raise ValueError( + f"expected options for {source!r} to be a mapping or null, " + f"got {type(opts).__name__}: {opts!r}") + target = opts.get("target") + if target is not None and not isinstance(target, str): + raise ValueError( + f"expected target for {source!r} to be a str, " + f"got {type(target).__name__}: {target!r}") + return cls(source=source, target=target) + raise ValueError( + f"expected str or single-key dict, got {type(raw).__name__}: {raw!r}") + + def to_yaml(self) -> "str | dict[str, dict[str, str]]": + if self.target is None: + return self.source + return {self.source: {"target": self.target}} + + +@dataclass(frozen=True) +class LockEntry: + """A single resolved image in mirror_images.lock.yaml.""" + digest: Digest + target: str + tag: str + + @classmethod + def from_yaml(cls, raw: dict) -> "LockEntry": + try: + return cls( + digest=Digest(raw["digest"]), + target=raw["target"], + tag=raw["tag"], + ) + except KeyError as exc: + raise ValueError(f"missing field in lock entry: {exc}") from exc + + def to_yaml(self) -> "dict[str, str]": + # Field order matters for stable yaml output. + return {"digest": str(self.digest), "target": self.target, "tag": self.tag} + + +@dataclass(frozen=True) +class MirrorManifest: + """Parsed mirror_images.yaml: images plus lint ignore lists. + + `ignore_patterns` suppresses `lint` flags for image references that + are intentionally not mirrored (test fixtures, refs that resolve at + build time via a variable, alternate registries the repo trusts). + + `ignore_paths` skips entire directories from lint scanning (e.g. + vendored submodules, example/ trees). + """ + entries: list[MirrorEntry] + ignore_patterns: list[re.Pattern[str]] + # Raw `ignore.images` strings kept alongside the compiled patterns so + # `add` can round-trip them verbatim (preserving the user's quoting) + # without re-reading the file. + ignore_images_raw: list[str] + ignore_paths: list[str] + was_mapping_form: bool + + def is_ignored(self, image: str) -> bool: + return any(p.fullmatch(image) for p in self.ignore_patterns) + + +def _parse_str_list(value: object, what: str, path: str) -> list[str]: + """Validate a YAML value as a list[str], filtering+warning on bad entries.""" + if value is None: + return [] + if not isinstance(value, list): + raise ValueError(f"{what} in {path} must be a list") + out: list[str] = [] + for i, raw in enumerate(value): + if not isinstance(raw, str): + print( + f" [warning] {what}[{i}] in {path} must be a string, " + f"got {type(raw).__name__}", file=sys.stderr) + continue + out.append(raw) + return out + + +def parse_mirror_yaml(path: str) -> MirrorManifest: + """Parse mirror_images.yaml. + + Two on-disk shapes are accepted: + + # legacy flat list + - alpine:3.20 + + # mapping form, enables `ignore:` + images: + - alpine:3.20 + ignore: + images: + - public\\.ecr\\.aws/.* + paths: + - vendor + + Empty / comments-only / explicit `null` files yield an empty manifest + so a freshly `init`'d project works immediately with `add`, `lock`, + etc. + """ + with open(path) as f: + data = yaml.safe_load(f) + + if data is None: + return MirrorManifest( + entries=[], ignore_patterns=[], ignore_images_raw=[], + ignore_paths=[], was_mapping_form=False, + ) + + ignore_images_raw: list[str] = [] + ignore_paths: list[str] = [] + + if isinstance(data, list): + raw_entries: list = data + was_mapping = False + elif isinstance(data, dict): + raw_entries = data.get("images") or [] + if not isinstance(raw_entries, list): + raise ValueError(f"'images' in {path} must be a list") + ignore_section = data.get("ignore") + if ignore_section is not None: + if not isinstance(ignore_section, dict): + raise ValueError( + f"'ignore' in {path} must be a mapping with 'images' " + f"and/or 'paths' keys") + ignore_images_raw = _parse_str_list( + ignore_section.get("images"), "ignore.images", path) + ignore_paths = _parse_str_list( + ignore_section.get("paths"), "ignore.paths", path) + was_mapping = True + else: + raise ValueError( + f"Expected YAML list or mapping in {path}, got {type(data).__name__}") + + entries: list[MirrorEntry] = [] + for i, raw in enumerate(raw_entries): + try: + entries.append(MirrorEntry.from_yaml(raw)) + except ValueError as exc: + print(f" [warning] ignoring entry #{i} in {path}: {exc}", + file=sys.stderr) + + ignore_patterns: list[re.Pattern[str]] = [] + for i, raw in enumerate(ignore_images_raw): + try: + ignore_patterns.append(re.compile(raw)) + except re.error as exc: + print(f" [warning] ignore.images[{i}] {raw!r} in {path} is " + f"not a valid regex: {exc}", file=sys.stderr) + + return MirrorManifest( + entries=entries, + ignore_patterns=ignore_patterns, + ignore_images_raw=ignore_images_raw, + ignore_paths=ignore_paths, + was_mapping_form=was_mapping, + ) + + +def parse_lock_yaml(path: str) -> dict[str, LockEntry]: + """Parse mirror_images.lock.yaml into {source: LockEntry}.""" + with open(path) as f: + data = yaml.safe_load(f) + if not isinstance(data, dict) or "images" not in data: + raise ValueError( + f"Malformed lock file {path}: expected a YAML mapping with an 'images' key" + ) + images = data["images"] + if not isinstance(images, dict): + raise ValueError( + f"Malformed lock file {path}: 'images' must be a mapping, " + f"got {type(images).__name__}") + return {src: LockEntry.from_yaml(info) for src, info in images.items()} + + +# --------------------------------------------------------------------------- +# Tag resolution for vague tags (e.g. "latest") +# --------------------------------------------------------------------------- + + +def _version_sort_key(tag: str) -> list[tuple[int, int | str]]: + """Sort key that orders version-like tags so the highest version comes last. + + Numeric parts sort before string parts so that "1.2.3" < "1.2.10". + """ + parts = re.split(r"[.\-+]", tag.lstrip("v")) + return [(0, int(p)) if p.isdigit() else (1, p) for p in parts] + + +def find_specific_tag(image_ref: str, digest: str, tool: str) -> str: + """Given a vague tag (e.g. 'latest'), find the most specific version tag + that shares the same digest. + + Uses a dedicated inner ThreadPoolExecutor so this function is safe to + call from a worker of an outer pool. (Submitting back into the *same* + pool you are running in deadlocks once every worker is a parent.) + """ + if ":" in image_ref: + repo = image_ref.rsplit(":", 1)[0] + else: + repo = image_ref + + all_tags = list_tags(repo, tool) + if not all_tags: + return "" + + # "clean" = pure version tags like "1.2.3" or "v1.2.3" + clean_re = re.compile(r"^v?\d+(\.\d+)+$") + # "suffixed" = version + single suffix like "1.2.3-alpine" + suffixed_re = re.compile(r"^v?\d+(\.\d+)+-[a-zA-Z][a-zA-Z0-9]*$") + + clean_tags = sorted( + [t for t in all_tags if clean_re.match(t)], + key=_version_sort_key, reverse=True, + ) + clean_set = set(clean_tags) + suffixed_tags = sorted( + [t for t in all_tags if suffixed_re.match(t) and t not in clean_set], + key=_version_sort_key, reverse=True, + ) + + candidates = clean_tags[:30] + suffixed_tags[:20] + if not candidates: + return "" + + def _resolve_tag(tag: str) -> tuple[str, Digest | None]: + try: + d = resolve_digest(f"{repo}:{tag}", tool) + return tag, d + except (RuntimeError, subprocess.SubprocessError, ValueError) as exc: + print(f" [warning] could not resolve {repo}:{tag}: {exc}", + file=sys.stderr) + return tag, None + + matches = [] + with ThreadPoolExecutor(max_workers=min(8, len(candidates))) as inner_pool: + futures = [inner_pool.submit(_resolve_tag, tag) for tag in candidates] + for future in as_completed(futures): + tag, tag_digest = future.result() + if tag_digest == digest: + matches.append(tag) + + if not matches: + return "" + + # Prefer clean version tags (e.g. "1.2.3") over suffixed ones (e.g. "1.2.3-alpine") + clean_matches = sorted( + [t for t in matches if clean_re.match(t)], + key=_version_sort_key, reverse=True, + ) + if clean_matches: + return clean_matches[0] + + matches.sort(key=_version_sort_key, reverse=True) + return matches[0] + + +# --------------------------------------------------------------------------- +# Lock file +# --------------------------------------------------------------------------- + + +def _write_lock_file(path: str, results: dict[str, LockEntry]) -> None: + """Write the lock file with current results, sorted for stable output.""" + lock_data = { + "version": 1, + "images": {k: results[k].to_yaml() for k in sorted(results)}, + } + dir_name = os.path.dirname(path) or "." + fd, tmp = tempfile.mkstemp(dir=dir_name, suffix=".tmp", prefix=".lock_") + try: + with os.fdopen(fd, "w") as f: + f.write("# Auto-generated by: uv run bin/mirror_images.py lock\n") + f.write("# Do not edit manually.\n") + yaml.dump(lock_data, f, default_flow_style=False, sort_keys=False) + os.replace(tmp, path) + except BaseException: + os.unlink(tmp) + raise + + +def cmd_lock(args: argparse.Namespace) -> int: + """Generate mirror_images.lock.yaml with resolved digests.""" + tool = _find_digest_tool() + entries = parse_mirror_yaml(args.mirror_yaml).entries + declared_sources = {e.source for e in entries} + output_path = args.output or LOCK_YAML + + # Load existing lock file to preserve previously resolved entries + results: dict[str, LockEntry] = {} + if os.path.exists(output_path): + try: + existing = parse_lock_yaml(output_path) + results.update(existing) + print(f"Loaded {len(results)} existing entries from {output_path}") + except (OSError, yaml.YAMLError, ValueError) as exc: + print( + f" [warning] could not parse existing lock file {output_path}: {exc}", + file=sys.stderr) + print(" [warning] resolving all images from scratch.", + file=sys.stderr) + + def _current_lock_entries() -> dict[str, LockEntry]: + """Return only the results that are still declared in mirror_images.yaml.""" + return {k: results[k] for k in results if k in declared_sources} + + # Only resolve entries not already in the lock file + to_resolve = [e for e in entries if e.source not in results] + if not to_resolve: + print(f"All {len(entries)} images already resolved in {output_path}") + _write_lock_file(output_path, _current_lock_entries()) + return 0 + + print( + f"Resolving {len(to_resolve)} of {len(entries)} images using {tool} (parallelism={args.jobs})..." + ) + + errors: list[tuple[str, str]] = [] + pinning_failures: list[str] = [] + fatal_write_error: Exception | None = None + + def _resolve(entry: MirrorEntry): + digest = resolve_digest(entry.source, tool) + normalized = parse_image_ref(entry.source) + tag = normalized.rsplit(":", 1)[1] + + pinning_failed = False + if tag == "latest": + pinned = find_specific_tag(entry.source, digest, tool) or tag + pinning_failed = (pinned == tag) + tag = pinned + + return entry, digest, tag, pinning_failed + + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + futures = { + pool.submit(_resolve, entry): entry.source + for entry in to_resolve + } + for future in as_completed(futures): + src = futures[future] + try: + entry, digest, tag, pinning_failed = future.result() + except (OSError, RuntimeError, subprocess.SubprocessError, + ValueError) as exc: + errors.append((src, str(exc))) + print(f" [error] resolving {src}: {exc}", file=sys.stderr) + continue + + original_tag = entry.source.rsplit(":", 1)[-1] + tag_info = f" (tag: {tag})" if tag != original_tag else "" + print(f" {entry.source}: {digest}{tag_info}") + if pinning_failed: + pinning_failures.append(entry.source) + + results[entry.source] = LockEntry( + digest=digest, + target=entry.resolved_target(), + tag=tag, + ) + try: + _write_lock_file(output_path, _current_lock_entries()) + except (OSError, yaml.YAMLError) as exc: + fatal_write_error = exc + print(f"\n [fatal] failed to write {output_path}: {exc}", + file=sys.stderr) + for f in futures: + f.cancel() + break + + if fatal_write_error is not None: + return 1 + + if pinning_failures: + print( + f"\n [warning] {len(pinning_failures)} entr{'y' if len(pinning_failures) == 1 else 'ies'} " + "kept a vague tag — tag pinning failed (transient registry error?). " + "The digest is pinned, but the human-readable tag is not specific:", + file=sys.stderr) + for src in pinning_failures: + print(f" - {src}: tag={results[src].tag}", file=sys.stderr) + + if errors: + print(f"\nFailed to resolve {len(errors)} image(s):", file=sys.stderr) + for src, err in errors: + print(f" - {src}: {err}", file=sys.stderr) + return 1 + + print(f"\nWrote {output_path} ({len(results)} images)") + return 0 + + +# --------------------------------------------------------------------------- +# Mirror +# --------------------------------------------------------------------------- + + +_AUTH_FAILURE_THRESHOLD = 3 + + +def cmd_mirror(args: argparse.Namespace) -> int: + """Sync images from lock file to target registry using crane/skopeo.""" + lock_path = args.lock_yaml + if not os.path.exists(lock_path): + print(f"Lock file not found: {lock_path}", file=sys.stderr) + print("Run 'mirror_images.py lock' first.", file=sys.stderr) + return 1 + + digest_tool = _find_digest_tool() + copy_tool = _find_copy_tool() + images = parse_lock_yaml(lock_path) + + # Phase 1: check which images need copying + print( + f"Phase 1: checking {len(images)} images using {digest_tool} ({args.jobs} workers)..." + ) + + def _check_if_mirrored(source: str, info: LockEntry): + # Resolve the target *tag* (not just the digest) so we re-copy when + # the bytes already live in the repo under a different tag — e.g. + # a previous run mirrored the same digest as `mirror/foo:1` and + # this run wants `mirror/foo:1.2.3`. Skipping by digest alone would + # leave the new tag alias unset and consumers pulling by tag get + # `manifest unknown`. + print(f" ? {source} -> {info.target}") + try: + actual = resolve_digest(info.target, digest_tool) + except RuntimeError as exc: + if _is_not_found(str(exc)): + return source, info, False + raise + return source, info, actual == info.digest + + to_copy: list[tuple[str, LockEntry]] = [] + already_present = 0 + checked = 0 + check_errors: list[tuple[str, str]] = [] + + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + futures = { + pool.submit(_check_if_mirrored, src, info): src + for src, info in images.items() + } + for future in as_completed(futures): + src = futures[future] + checked += 1 + try: + source, info, present = future.result() + except (OSError, RuntimeError, subprocess.SubprocessError, + ValueError) as exc: + check_errors.append((src, str(exc))) + print(f" [error] checking {src}: {exc}", file=sys.stderr) + continue + if present: + already_present += 1 + print( + f" = [{checked}/{len(images)}] {source} (already mirrored)" + ) + else: + to_copy.append((source, info)) + print(f" + [{checked}/{len(images)}] {source} (needs copy)") + + if check_errors: + print( + f"\n [fatal] {len(check_errors)} image(s) failed pre-flight check " + "(not a 404). This usually means broken auth, network, or registry " + "config. Refusing to attempt copies until checks pass:", + file=sys.stderr) + for src, err in check_errors: + print(f" - {src}: {err}", file=sys.stderr) + return 2 + + if not to_copy: + print(f"\nAll {len(images)} images are up to date.") + return 0 + + print( + f"\n{already_present} already mirrored, {len(to_copy)} to copy using {copy_tool}.\n" + ) + + # Phase 2: copy missing images in parallel, fail-fast on persistent auth errors. + print( + f"\nPhase 2: copying {len(to_copy)} images using {copy_tool} ({args.jobs} workers)..." + ) + + errors: list[tuple[str, str]] = [] + auth_failures = 0 + aborted_for_auth = False + abort_lock = threading.Lock() + + def _do_copy(idx: int, source: str, info: LockEntry): + source_ref = f"{parse_image_ref(source).rsplit(':', 1)[0]}@{info.digest}" + if args.dry_run: + return idx, source, info, source_ref, True, "[dry-run]" + ok, err = copy_image(source_ref, info.target, copy_tool) + return idx, source, info, source_ref, ok, err + + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + indexed = list(enumerate(sorted(to_copy), 1)) + future_map = { + pool.submit(_do_copy, i, src, info): (src, info) + for i, (src, info) in indexed + } + for future in as_completed(future_map): + if future.cancelled(): + continue + idx, source, info, source_ref, ok, err = future.result() + print( + f" -> [{idx}/{len(to_copy)}] {source} ({info.digest[:19]}...)\n" + f" src: {source_ref}\n" + f" dest: {info.target}") + if args.dry_run: + print( + f" [dry-run] {copy_tool} copy {source_ref} {info.target}" + ) + continue + if ok: + print(f" OK") + continue + errors.append((source, err)) + print( + f" [error] ({copy_tool}) {err or '(no stderr)'}", + file=sys.stderr) + if _is_auth_error(err): + with abort_lock: + auth_failures += 1 + if (auth_failures >= _AUTH_FAILURE_THRESHOLD + and not aborted_for_auth): + aborted_for_auth = True + print( + f"\n [fatal] {auth_failures} auth-shaped copy failures — " + "cancelling remaining copies. Check destination registry " + "credentials.", + file=sys.stderr) + for f in future_map: + f.cancel() + + if errors: + print(f"\nFailed to copy {len(errors)} image(s):", file=sys.stderr) + for src, err in errors: + print(f" - {src}: {err}", file=sys.stderr) + return 1 + + action = "would copy" if args.dry_run else "copied" + print( + f"\nDone: {already_present} already present, {len(to_copy)} {action}.") + return 0 + + +# --------------------------------------------------------------------------- +# BuildKit daemon config generation +# --------------------------------------------------------------------------- + + +def cmd_buildkitd(args: argparse.Namespace) -> int: + entries = parse_mirror_yaml(args.mirror_yaml).entries + if not entries: + print(" [error] no entries in mirror_images.yaml.", file=sys.stderr) + return 1 + + registry_to_mirror: dict[str, str] = {} + + for entry in entries: + mirror = entry.target_registry + if mirror is None: + print( + f" [warning] skipping {entry.source!r}: explicit target does not " + "follow the standard {dest}/{image_path} convention.", + file=sys.stderr, + ) + continue + + reg = entry.source_registry + existing = registry_to_mirror.get(reg) + if existing is not None and existing != mirror: + print( + f" [error] conflicting mirror addresses for {reg!r}:\n" + f" {existing!r} (seen earlier)\n" + f" {mirror!r} (from {entry.source!r})", + file=sys.stderr, + ) + return 1 + + registry_to_mirror[reg] = mirror + + lines = [ + "# Auto-generated by: mirror_images buildkitd", + "# Do not edit manually — regenerate with:", + "# mirror_images buildkitd", + "", + ] + for reg in sorted(registry_to_mirror): + lines.append(f'[registry."{reg}"]') + lines.append(f' mirrors = ["{registry_to_mirror[reg]}"]') + lines.append("") + + content = "\n".join(lines) + + if args.output: + with open(args.output, "w") as f: + f.write(content) + print(f"Wrote {args.output} ({len(registry_to_mirror)} registry mirror(s))") + else: + print(content, end="") + + return 0 + + +# --------------------------------------------------------------------------- +# Lint +# --------------------------------------------------------------------------- + + +def find_public_image_refs( + skip_dirs: tuple[str, ...] = (), +) -> list[tuple[str, int, str, str]]: + """Scan the repo for image references not using registry.ddbuild.io. + + `skip_dirs` are repo-relative directory paths to skip (typically + sourced from `mirror_images.yaml#ignore.paths`). + """ + internal_prefixes = ("registry.ddbuild.io/", "486234852809.dkr.ecr") + hits = [] + + _ignore_prefixes = internal_prefixes + ( + "$", "{", "nginx-datadog-test-", + ) + + def _is_external(img: str) -> bool: + if any(img.startswith(p) for p in _ignore_prefixes): + return False + return bool(re.match(r"^[a-zA-Z0-9]", img)) + + def _read_lines(filepath): + try: + with open(filepath) as f: + return list(enumerate(f, 1)) + except OSError as exc: + print(f" [warning] Could not read {filepath}: {exc}", + file=sys.stderr) + return [] + + # Trailing `/` keeps "foo" from matching "foo-bar". + _skip_dir_prefixes = tuple( + os.path.join(PROJECT_DIR, d) + os.sep for d in skip_dirs + ) + + def _in_skip_dir(filepath): + return filepath.startswith(_skip_dir_prefixes) + + def _scan_dockerfiles(): + for filepath in glob.glob(os.path.join(PROJECT_DIR, "**/Dockerfile*"), + recursive=True): + if "/.git/" in filepath or _in_skip_dir(filepath): + continue + for lineno, line in _read_lines(filepath): + stripped = line.strip() + m = re.match(r"^FROM\s+(\S+)", stripped, re.IGNORECASE) + if m and _is_external(m.group(1)): + hits.append((filepath, lineno, stripped, m.group(1))) + for m in re.finditer(r"--from=(\S+)", stripped): + img = m.group(1) + if ("/" in img or ":" in img) and _is_external(img): + hits.append((filepath, lineno, stripped, img)) + + def _scan_yaml_image_fields(): + skip_path_fragments = ("/.git/", "/.gitlab/") + for pattern in ("**/*.yml", "**/*.yaml"): + for filepath in glob.glob(os.path.join(PROJECT_DIR, pattern), + recursive=True): + if any(frag in filepath for frag in skip_path_fragments): + continue + if _in_skip_dir(filepath): + continue + for lineno, line in _read_lines(filepath): + m = re.match(r"^\s*image:\s+['\"]?(\S+?)['\"]?\s*$", line) + if m and _is_external(m.group(1)): + hits.append( + (filepath, lineno, line.rstrip(), m.group(1))) + + _scan_dockerfiles() + _scan_yaml_image_fields() + return hits + + +def _collect_gitlab_ci_files(entry: str) -> list[str]: + """Starting from a GitLab CI YAML file, follow local includes and return all file paths.""" + collected = [] + visited: set[str] = set() + queue: collections.deque[str] = collections.deque([entry]) + while queue: + path = queue.popleft() + abspath = os.path.join(PROJECT_DIR, path) if not os.path.isabs(path) else path + if abspath in visited or not os.path.isfile(abspath): + continue + visited.add(abspath) + collected.append(abspath) + try: + with open(abspath) as f: + data = yaml.safe_load(f) + except (OSError, yaml.YAMLError) as exc: + print(f" [warning] Could not parse {abspath}: {exc}", + file=sys.stderr) + continue + if not isinstance(data, dict): + continue + includes = data.get("include", []) + if isinstance(includes, dict): + includes = [includes] + if not isinstance(includes, list): + continue + for inc in includes: + if isinstance(inc, str): + queue.append(os.path.join(PROJECT_DIR, inc)) + elif isinstance(inc, dict) and "local" in inc: + queue.append(os.path.join(PROJECT_DIR, inc["local"])) + return collected + + +# Templates that map matrix variable names to image name patterns. +# {value} is replaced with the matrix variable value. +_GITLAB_MATRIX_IMAGE_TEMPLATES: dict[str, list[str]] = { + "BASE_IMAGE": ["{value}"], + "INGRESS_NGINX_VERSION": [ + "registry.k8s.io/ingress-nginx/controller:v{value}" + ], + "RESTY_VERSION": ["openresty/openresty:{value}-alpine"], +} + + +def _extract_matrix_combos(job: dict) -> list[dict]: + """Extract the parallel:matrix combo list from a GitLab CI job definition.""" + parallel = job.get("parallel") + if not isinstance(parallel, dict): + return [] + matrix = parallel.get("matrix") + if not isinstance(matrix, list): + return [] + return [c for c in matrix if isinstance(c, dict)] + + +def _expand_matrix_images(combo: dict) -> list[str]: + """Expand a single matrix combo into image references using known templates.""" + images = [] + for var_name, templates in _GITLAB_MATRIX_IMAGE_TEMPLATES.items(): + values = combo.get(var_name, []) + if isinstance(values, str): + values = [values] + if not isinstance(values, list): + continue + for val in values: + for tmpl in templates: + images.append(tmpl.format(value=val)) + return images + + +def find_gitlab_ci_images() -> list[tuple[str, str]]: + """Extract public image references from GitLab CI matrix variables. + + Returns a list of (source_file_relpath, image_ref) tuples. + """ + ci_entry = os.path.join(PROJECT_DIR, ".gitlab-ci.yml") + if not os.path.isfile(ci_entry): + return [] + + results = [] + for filepath in _collect_gitlab_ci_files(ci_entry): + try: + with open(filepath) as f: + data = yaml.safe_load(f) + except (OSError, yaml.YAMLError) as exc: + print(f" [warning] Could not parse {filepath}: {exc}", + file=sys.stderr) + continue + if not isinstance(data, dict): + continue + + relpath = os.path.relpath(filepath, PROJECT_DIR) + for _job_name, job in data.items(): + if not isinstance(job, dict): + continue + for combo in _extract_matrix_combos(job): + for img in _expand_matrix_images(combo): + results.append((relpath, img)) + + return results + + +def cmd_lint(args: argparse.Namespace) -> int: + """Check that all images are referenced from registry.ddbuild.io.""" + manifest = parse_mirror_yaml(args.mirror_yaml) + declared = {e.source: e.resolved_target() for e in manifest.entries} + rc = 0 + + # --- Check 1: public image references in Dockerfiles and YAML --- + public_refs = [r for r in find_public_image_refs(tuple(manifest.ignore_paths)) + if not manifest.is_ignored(r[3])] + + if public_refs: + by_image: dict[str, list[tuple[str, int, str]]] = {} + for filepath, lineno, line, img in public_refs: + by_image.setdefault(img, []).append((filepath, lineno, line)) + + undeclared = [] + print( + "Public image references found (should use registry.ddbuild.io mirror):\n" + ) + for img in sorted(by_image): + replacement = declared.get(img) + if not replacement: + replacement = f"{_dest_registry()}/{img}" + undeclared.append(img) + + print(f" {img}") + print(f" -> {replacement}") + for filepath, lineno, line in by_image[img]: + relpath = os.path.relpath(filepath, PROJECT_DIR) + print(f" {relpath}:{lineno}") + print() + + if undeclared: + print( + "Images not declared in mirror_images.yaml (add them first):") + for img in sorted(undeclared): + print(f" - {img}") + print() + + print( + f"{len(public_refs)} public image reference(s) across {len(by_image)} image(s)." + ) + rc = 1 + + # --- Check 2: GitLab CI matrix images must be in mirror_images.yaml --- + ci_images = [(src, img) for src, img in find_gitlab_ci_images() + if not manifest.is_ignored(img)] + if ci_images: + missing: dict[str, list[str]] = {} + for source_file, img in ci_images: + if img not in declared: + missing.setdefault(img, []).append(source_file) + + if missing: + print("GitLab CI matrix images not declared in mirror_images.yaml:\n") + for img in sorted(missing): + sources = sorted(set(missing[img])) + print(f" - {img}") + for src in sources: + print(f" {src}") + print( + f"\n{len(missing)} image(s) from GitLab CI not in mirror_images.yaml." + ) + rc = 1 + else: + print("All GitLab CI matrix images are declared in mirror_images.yaml.") + + if rc == 0: + print("All image references use registry.ddbuild.io.") + return rc + + +# --------------------------------------------------------------------------- +# List ignored image references +# --------------------------------------------------------------------------- + + +def cmd_ignored(args: argparse.Namespace) -> int: + """Emit JSON listing image refs suppressed by mirror_images.yaml ignores. + + Default: only refs outside `ignore.paths` whose source matches + `ignore.images`. With `--scan-paths`, also walks into `ignore.paths` + directories and reports refs found there (rule="path"). + """ + manifest = parse_mirror_yaml(args.mirror_yaml) + + skip_dirs: tuple[str, ...] = () if args.scan_paths else tuple(manifest.ignore_paths) + refs = find_public_image_refs(skip_dirs) + + def _match_path(relpath: str) -> str | None: + for p in manifest.ignore_paths: + if relpath == p or relpath.startswith(p + os.sep): + return p + return None + + def _match_pattern(img: str) -> str | None: + for p in manifest.ignore_patterns: + if p.fullmatch(img): + return p.pattern + return None + + def _rec(img, rule, matched, file, line): + return {"image": img, "rule": rule, "matched": matched, + "file": file, "line": line} + + suppressed: list[dict] = [] + for filepath, lineno, _line, img in refs: + relpath = os.path.relpath(filepath, PROJECT_DIR) + # Path rule wins over image-pattern when both apply, since lint + # would never reach the image check for files inside an ignored + # directory. + path_match = _match_path(relpath) + if path_match is not None: + suppressed.append(_rec(img, "path", path_match, relpath, lineno)) + continue + pattern_match = _match_pattern(img) + if pattern_match is not None: + suppressed.append(_rec(img, "pattern", pattern_match, relpath, lineno)) + + for src_file, img in find_gitlab_ci_images(): + pattern_match = _match_pattern(img) + if pattern_match is not None: + suppressed.append(_rec(img, "pattern", pattern_match, src_file, None)) + + suppressed.sort(key=lambda r: (r["image"], r["file"], r["line"] or 0)) + + output = { + "patterns": [p.pattern for p in manifest.ignore_patterns], + "paths": manifest.ignore_paths, + "scanned_ignored_paths": args.scan_paths, + "suppressed": suppressed, + } + print(json.dumps(output, indent=2)) + return 0 + + +# --------------------------------------------------------------------------- +# Add images +# --------------------------------------------------------------------------- + + +def _write_mirror_yaml( + path: str, + entries: list[MirrorEntry], + ignore_section: dict[str, list[str]], + use_mapping_form: bool, +) -> None: + """Rewrite mirror_images.yaml preserving leading comments. + + Uses mapping form (`images:` / `ignore:`) when either the file already + used that form or the ignore section is non-empty; otherwise emits + the legacy flat list. Bare-string entries keep their quoted shape; + custom-target entries round-trip through yaml.dump. + """ + with open(path) as f: + header_lines = [] + for line in f: + if line.startswith("#") or line.strip() == "": + header_lines.append(line) + else: + break + + def _entry_lines(entry: MirrorEntry, indent: str = "") -> str: + shape = entry.to_yaml() + if isinstance(shape, str): + return f'{indent}- "{shape}"\n' + return "".join( + f"{indent}{line}\n" + for line in yaml.dump([shape], default_flow_style=False, sort_keys=False).splitlines() + ) + + with open(path, "w") as f: + f.writelines(header_lines) + if use_mapping_form: + f.write("images:\n") + for entry in entries: + f.write(_entry_lines(entry, indent=" ")) + if any(ignore_section.values()): + f.write("\nignore:\n") + for key in ("images", "paths"): + items = ignore_section.get(key) or [] + if not items: + continue + f.write(f" {key}:\n") + for item in items: + for line in yaml.dump( + [item], default_flow_style=False, + sort_keys=False).splitlines(): + f.write(f" {line}\n") + else: + for entry in entries: + f.write(_entry_lines(entry)) + + +def cmd_add(args: argparse.Namespace) -> int: + """Add one or more images to mirror_images.yaml (if not already present).""" + try: + manifest = parse_mirror_yaml(args.mirror_yaml) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + entries = list(manifest.entries) + existing = {e.source for e in entries} + + added = [] + for img in args.images: + if img in existing: + print(f" already listed: {img}") + else: + entries.append(MirrorEntry(source=img)) + existing.add(img) + added.append(img) + print(f" added: {img}") + + if not added: + print("\nNo new images to add.") + return 0 + + # Sort bare-string entries (default target) by source; keep entries + # with a custom target after them in their original order. + bare = sorted([e for e in entries if e.target is None], key=lambda e: e.source) + custom = [e for e in entries if e.target is not None] + entries = bare + custom + + ignore_section = {"images": manifest.ignore_images_raw, "paths": manifest.ignore_paths} + use_mapping = manifest.was_mapping_form or any(ignore_section.values()) + _write_mirror_yaml(args.mirror_yaml, entries, ignore_section, use_mapping) + + print(f"\nAdded {len(added)} image(s) to {args.mirror_yaml}") + return 0 + + +# --------------------------------------------------------------------------- +# Init +# --------------------------------------------------------------------------- + + +_INIT_TEMPLATE = """\ +# Images to mirror into registry.ddbuild.io. +# See: https://github.com/DataDog/dd-repo-tools/blob/main/shared/mirror-images/mirror_images.md +# +# Flat-list form: +# - "alpine:3.20" +# - "nginx:1.27.5": +# target: registry.ddbuild.io/ci/example/custom/nginx:1.27.5 +# +# Mapping form (enables `ignore`): +# images: +# - "alpine:3.20" +# ignore: +# images: +# - "public\\.ecr\\.aws/.*" +# paths: +# - "vendor" +""" + + +def cmd_init(args: argparse.Namespace) -> int: + """Write a starter mirror_images.yaml at the project root.""" + path = args.mirror_yaml + if os.path.exists(path) and not args.force: + print(f"{path} already exists. Use --force to overwrite.", + file=sys.stderr) + return 1 + with open(path, "w") as f: + f.write(_INIT_TEMPLATE) + print(f"Wrote {path}") + print("Next: `mirror_images lock` to resolve digests, " + "then `mirror_images mirror` to copy.") + return 0 + + +# --------------------------------------------------------------------------- +# Relock images +# --------------------------------------------------------------------------- + + +def cmd_relock(args: argparse.Namespace) -> int: + """Re-resolve digests for images matching the given patterns. + + Removes matching entries from the lock file, then runs the lock + command to re-resolve them. Patterns use fnmatch-style wildcards + (e.g. 'nginx:1.29.*', 'openresty/*'). + """ + lock_path = args.output or LOCK_YAML + if not os.path.exists(lock_path): + print(f"Lock file not found: {lock_path}", file=sys.stderr) + print("Run 'lock' first to create it.", file=sys.stderr) + return 1 + + existing = parse_lock_yaml(lock_path) + patterns = args.patterns + + matched = [] + for src in list(existing): + if any(fnmatch.fnmatch(src, pat) for pat in patterns): + matched.append(src) + del existing[src] + + if not matched: + print(f"No images in lock file matched: {', '.join(patterns)}") + return 1 + + print(f"Removing {len(matched)} image(s) from lock file to re-resolve:") + for src in sorted(matched): + print(f" - {src}") + + # Write out the lock file without the matched entries + _write_lock_file(lock_path, existing) + + # Now run the normal lock flow which will resolve the missing entries + print() + return cmd_lock(args) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> int: + parser = argparse.ArgumentParser( + prog="mirror_images", + description="Manage mirrored Docker images") + parser.add_argument( + "--mirror-yaml", + default=MIRROR_YAML, + help=f"Path to mirror_images.yaml (default: {MIRROR_YAML})", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + init_parser = subparsers.add_parser( + "init", + help="Write a starter mirror_images.yaml", + description="Create mirror_images.yaml at the project root with a " + "small example. Refuses to overwrite an existing file unless --force.", + ) + init_parser.set_defaults(func=cmd_init) + init_parser.add_argument( + "-f", "--force", action="store_true", + help="Overwrite mirror_images.yaml if it already exists") + + lock_parser = subparsers.add_parser( + "lock", help="Generate mirror_images.lock.yaml with resolved digests") + lock_parser.set_defaults(func=cmd_lock) + lock_parser.add_argument("-j", + "--jobs", + type=int, + default=16, + help="Parallel workers (default: 16)") + lock_parser.add_argument( + "-o", + "--output", + help="Output path (default: mirror_images.lock.yaml)") + + mirror_parser = subparsers.add_parser( + "mirror", help="Sync images from lock file to target registry") + mirror_parser.set_defaults(func=cmd_mirror) + mirror_parser.add_argument( + "-j", + "--jobs", + type=int, + default=16, + help="Parallel workers for checking (default: 16)") + mirror_parser.add_argument("--lock-yaml", + default=LOCK_YAML, + help="Path to lock file") + mirror_parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be copied without copying") + + buildkitd_parser = subparsers.add_parser( + "buildkitd", + help="Generate a buildkitd.toml mirror config from mirror_images.yaml", + description=( + "Read mirror_images.yaml and emit a buildkitd.toml with one\n" + "[registry.\"X\"] section per source registry." + ), + ) + buildkitd_parser.set_defaults(func=cmd_buildkitd) + buildkitd_parser.add_argument( + "-o", + "--output", + help="Output path for buildkitd.toml (default: print to stdout)", + ) + + lint_parser = subparsers.add_parser( + "lint", help="Check that all image references use registry.ddbuild.io") + lint_parser.set_defaults(func=cmd_lint) + + ignored_parser = subparsers.add_parser( + "ignored", + help="List image references suppressed by mirror_images.yaml `ignore:` rules (JSON)", + description="Print JSON listing image refs that lint is silently " + "skipping. By default only `ignore.images` regex hits are scanned. " + "Pass --scan-paths to also walk into `ignore.paths` directories.", + ) + ignored_parser.set_defaults(func=cmd_ignored) + ignored_parser.add_argument( + "--scan-paths", action="store_true", + help="Also descend into `ignore.paths` directories and report refs found there", + ) + + add_parser = subparsers.add_parser( + "add", + help="Add images to mirror_images.yaml", + description= + "Add one or more Docker image references to mirror_images.yaml. " + "Images already listed are skipped. The file is re-sorted after adding." + ) + add_parser.set_defaults(func=cmd_add) + add_parser.add_argument( + "images", + nargs="+", + metavar="IMAGE", + help="Image references to add (e.g. 'nginx:1.30.0' 'alpine:3.21')") + + relock_parser = subparsers.add_parser( + "relock", + help="Re-resolve digests for specific images in the lock file", + description="Remove matching entries from mirror_images.lock.yaml and " + "re-resolve their digests. Uses fnmatch-style wildcards. " + "Example: 'nginx:1.29.*' matches nginx:1.29.0, nginx:1.29.5-alpine, etc." + ) + relock_parser.set_defaults(func=cmd_relock) + relock_parser.add_argument( + "patterns", + nargs="+", + metavar="PATTERN", + help= + "Glob patterns to match image names (e.g. 'nginx:1.29.*' 'openresty/*')" + ) + relock_parser.add_argument("-j", + "--jobs", + type=int, + default=16, + help="Parallel workers (default: 16)") + relock_parser.add_argument( + "-o", + "--output", + help="Path to lock file (default: mirror_images.lock.yaml)") + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/scripts/mirror_rewrite_dockerfile.py b/utils/scripts/mirror_rewrite_dockerfile.py deleted file mode 100644 index 0643ccf170c..00000000000 --- a/utils/scripts/mirror_rewrite_dockerfile.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Rewrite a Dockerfile's `FROM` base images to the system-tests mirror. - -Used by utils/build/build.sh when USE_IMAGE_MIRROR is enabled: it prints the -given Dockerfile to stdout with every `FROM ` whose image is in the -mirror (mirror_images.lock.yaml) replaced by its mirror target, so weblog builds -pull their base images from registry.ddbuild.io/system-tests/mirror instead of -docker.io / ghcr.io / etc. - -Image references not present in the mirror mapping are left unchanged. This -matches multi-stage stage names (`FROM build`) too: they are never in the -mapping, so they pass through untouched. -""" - -import argparse -import re -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from utils._context._image_mirror import mirror_image - -# FROM [--platform=

] [AS ] -_FROM_RE = re.compile(r"^(\s*FROM\s+)((?:--platform=\S+\s+)?)(\S+)(.*)$", re.IGNORECASE) - - -def rewrite(dockerfile: str) -> str: - out: list[str] = [] - with open(dockerfile, encoding="utf-8") as f: - for line in f: - match = _FROM_RE.match(line.rstrip("\n")) - if match: - prefix, platform, image, rest = match.groups() - out.append(f"{prefix}{platform}{mirror_image(image)}{rest}\n") - else: - out.append(line if line.endswith("\n") else line + "\n") - return "".join(out) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - prog="mirror_rewrite_dockerfile", - description="Print a Dockerfile with FROM base images rewritten to the mirror", - ) - parser.add_argument("dockerfile", help="Path to the Dockerfile to rewrite") - args = parser.parse_args() - sys.stdout.write(rewrite(args.dockerfile)) diff --git a/utils/scripts/update_mirror_images.py b/utils/scripts/update_mirror_images.py index 65f45bcfc30..c58a7cdb0b6 100644 --- a/utils/scripts/update_mirror_images.py +++ b/utils/scripts/update_mirror_images.py @@ -32,15 +32,15 @@ from utils._context._scenarios import get_all_scenarios, DockerScenario # noqa: E402 -# Pinned dd-repo-tools mirror_images.py (override with $MIRROR_IMAGES_URL). -MIRROR_IMAGES_URL = os.environ.get( - "MIRROR_IMAGES_URL", - "https://binaries.ddbuild.io/dd-repo-tools/default/ca/fb4f39a542e4dd42b646c300b539c7a9f4201531/mirror_images.py", -) +# Local copy of mirror_images.py (from dd-repo-tools). Replace with the +# published URL once the buildkitd subcommand is released upstream. +MIRROR_IMAGES_SCRIPT = REPO_ROOT / "utils" / "scripts" / "mirror_images.py" + # Destination registry for the mirrored images (override with $MIRROR_DEST_REGISTRY). DEFAULT_DEST_REGISTRY = "registry.ddbuild.io/system-tests/mirror" MIRROR_YAML = REPO_ROOT / "mirror_images.yaml" +BUILDKITD_TOML = REPO_ROOT / "utils" / "build" / "docker" / "buildkitd.toml" # Header written when mirror_images.yaml does not exist yet. The mirror_images.py # `add` command preserves existing comments, so this is only used on first run. @@ -102,13 +102,13 @@ def collect_images(excluded: set[str]) -> list[str]: def _run_mirror_images(*args: str) -> None: - """Invoke the pinned dd-repo-tools mirror_images.py via uv.""" + """Invoke the local mirror_images.py via uv.""" if shutil.which("uv") is None: sys.exit("error: 'uv' is required to run mirror_images.py — install it from https://docs.astral.sh/uv/") env = dict(os.environ) env.setdefault("MIRROR_DEST_REGISTRY", DEFAULT_DEST_REGISTRY) cmd = [ - "uv", "run", "--no-config", "--script", MIRROR_IMAGES_URL, + "uv", "run", "--no-config", "--script", str(MIRROR_IMAGES_SCRIPT), "--mirror-yaml", str(MIRROR_YAML), *args, ] print(f"+ {' '.join(cmd)}", flush=True) @@ -128,6 +128,7 @@ def main(excluded: set[str], *, skip_lock: bool) -> None: _run_mirror_images("add", *images) if not skip_lock: _run_mirror_images("lock") + _run_mirror_images("buildkitd", "--output", str(BUILDKITD_TOML)) if __name__ == "__main__": From 65ddb0969519e42391e9d2bef851786e86aa167a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 24 Jun 2026 19:21:09 +0200 Subject: [PATCH 221/229] Use remote mirror_images.py instead of local copy --- .gitlab-ci.yml | 4 +- utils/scripts/mirror_images.py | 1658 ------------------------- utils/scripts/update_mirror_images.py | 22 +- 3 files changed, 17 insertions(+), 1667 deletions(-) delete mode 100755 utils/scripts/mirror_images.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 343e1dc7ef6..a333402ecb2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,8 @@ variables: TEST: 1 # TEMPORARY: rollout-control allowlist for system-tests-param LIBRARIES filter. E2E_SUPPORTED_LANGUAGES: "python" + # dd-repo-tools mirror_images.py (pinned). Used by the mirror_images_* jobs. + MIRROR_IMAGES_URL: "https://binaries.ddbuild.io/dd-repo-tools/default/ca/d6c8f8ec4b1fb69b36e96dc204e5f79bed6bf067/mirror_images.py" # Destination registry for mirrored CI images. MIRROR_DEST_REGISTRY: "registry.ddbuild.io/system-tests/mirror" @@ -439,6 +441,6 @@ mirror_images: exit 1 fi # Push any locked image missing from the mirror (already-present images are skipped). - - uv run --no-config --script utils/scripts/mirror_images.py mirror + - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror rules: - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' diff --git a/utils/scripts/mirror_images.py b/utils/scripts/mirror_images.py deleted file mode 100755 index 3e0288d1524..00000000000 --- a/utils/scripts/mirror_images.py +++ /dev/null @@ -1,1658 +0,0 @@ -#!/usr/bin/env -S uv run --script -# /// script -# requires-python = ">=3.10" -# dependencies = ["pyyaml==6.0.3", "dulwich==1.2.4"] -# /// -"""Manage mirrored Docker images: generate lock files, mirror, and lint. - -Uses CLI tools (crane, skopeo, docker, podman, nerdctl) for registry -operations instead of Python registry libraries. Whichever tool is -available on PATH will be used. -""" - -import argparse -import collections -import fnmatch -import functools -import glob -import hashlib -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile -import threading -from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass, field - -import yaml -from dulwich.errors import NotGitRepository -from dulwich.repo import Repo - -# Force line-buffered output so parallel progress lines appear immediately. -print = functools.partial(print, flush=True) # type: ignore[assignment] - -# Log-line prefix convention: human-facing diagnostics use " [level] message" -# with level in {warning, error, fatal}. Stick to this so log filters and -# eyeballed output stay consistent across modules. - -# Mirror destination registry is resolved lazily via `_dest_registry()` -# from `MIRROR_DEST_REGISTRY` or the origin remote — see the helper below. - - -class Digest(str): - """A `sha256:<64 hex>` container image digest. - - Subclassing `str` so it round-trips through YAML and string formatting - transparently while still rejecting malformed values at construction. - """ - _RE = re.compile(r"^sha256:[0-9a-f]{64}$") - - def __new__(cls, value: str) -> "Digest": - if not cls._RE.match(value): - raise ValueError( - f"invalid digest (expected sha256:<64 hex>): {value!r}") - return super().__new__(cls, value) - - -# Substrings that strongly indicate an authentication/authorization failure -# from a container registry CLI's stderr. Used both to fail-fast in mirror -# Phase 2 and to distinguish "not present" from "auth broken" in checks. -_AUTH_ERROR_SIGNALS = ("unauthorized", "401", "403", "denied", "forbidden", - "authentication required") -# Substrings registries / CLI tools use for "manifest not found" / 404. -_NOT_FOUND_SIGNALS = ("manifest_unknown", "manifest unknown", "not found", - "name unknown", "no such manifest", "404") - - -def _is_auth_error(stderr: str) -> bool: - s = stderr.lower() - return any(sig in s for sig in _AUTH_ERROR_SIGNALS) - - -def _is_not_found(stderr: str) -> bool: - s = stderr.lower() - return any(sig in s for sig in _NOT_FOUND_SIGNALS) - - -def _find_project_dir() -> str: - """Locate the project root: walk up from cwd until a `.git` entry is found. - - `MIRROR_IMAGES_PROJECT_DIR` overrides the auto-detection and is used as-is. - Falls back to cwd if no `.git` is found anywhere up the tree, in which - case path lookups will fail later with a clear FileNotFoundError. - """ - override = os.environ.get("MIRROR_IMAGES_PROJECT_DIR") - if override: - return override - - current = os.getcwd() - while True: - if os.path.exists(os.path.join(current, ".git")): - return current - parent = os.path.dirname(current) - if parent == current: - return os.getcwd() - current = parent - - -PROJECT_DIR = _find_project_dir() -MIRROR_YAML = os.path.join(PROJECT_DIR, "mirror_images.yaml") -LOCK_YAML = os.path.join(PROJECT_DIR, "mirror_images.lock.yaml") - - -def _detect_repo_name(project_dir: str) -> str | None: - """Best-effort repo name from the `origin` remote URL. - - Returns the last path segment with a trailing `.git` stripped, for - both SSH (`git@host:org/repo.git`) and HTTPS URLs. None when the - directory isn't a git repo or has no `origin` remote. - """ - try: - repo = Repo(project_dir) - except NotGitRepository: - return None - try: - url = repo.get_config().get((b"remote", b"origin"), b"url").decode() - except KeyError: - return None - finally: - repo.close() - url = url.strip().removesuffix(".git") - name = url.rsplit("/", 1)[-1].rsplit(":", 1)[-1] - return name or None - - -@functools.cache -def _dest_registry() -> str: - """Mirror destination registry, resolved lazily so non-mirroring - subcommands (`add`, `--help`) work outside a git checkout. - - `MIRROR_DEST_REGISTRY` wins; otherwise build it from the origin - remote name; otherwise exit with instructions. - """ - env = os.environ.get("MIRROR_DEST_REGISTRY") - if env: - return env - repo = _detect_repo_name(PROJECT_DIR) - if repo: - return f"registry.ddbuild.io/ci/{repo}/mirror" - print( - " [error] cannot determine mirror destination registry — no\n" - " 'origin' remote found in this checkout. Set\n" - " MIRROR_DEST_REGISTRY=registry.ddbuild.io/ci//mirror.", - file=sys.stderr, - ) - sys.exit(2) - -# --------------------------------------------------------------------------- -# Registry CLI abstraction -# --------------------------------------------------------------------------- - - -def _find_tool(names: list[str]) -> str | None: - """Return the first tool name found on PATH.""" - for name in names: - if shutil.which(name): - return name - return None - - -def _find_digest_tool() -> str: - """Find a tool that can report registry-canonical image digests. - - Only crane, skopeo, and `docker buildx imagetools` surface the canonical - manifest digest. `docker manifest inspect`, `podman manifest inspect` - and `nerdctl manifest inspect` return reformatted JSON whose hash does - not match the registry's bytes, so they're rejected — the resulting - lock file would pin digests that the source registry does not store. - """ - tool = _find_tool(["crane", "skopeo", "docker"]) - if not tool: - print( - "No digest tool found. Install one of: crane, skopeo, docker (with buildx).", - file=sys.stderr) - sys.exit(1) - return tool - - -def _find_copy_tool() -> str: - """Find a tool that can copy images between registries.""" - tool = _find_tool(["crane", "skopeo", "docker"]) - if not tool: - print( - "No image copy tool found. Install one of: crane, skopeo, docker", - file=sys.stderr) - sys.exit(1) - return tool - - -def resolve_digest(image_ref: str, tool: str) -> Digest: - """Resolve image:tag -> sha256:... using the given CLI tool. - - All three supported tools return the registry-canonical manifest digest: - `crane digest`, `skopeo inspect --format '{{.Digest}}'`, and - `docker buildx imagetools inspect --format '{{.Manifest.Digest}}'`. - Plain `docker manifest inspect` does *not* — it returns reformatted - JSON whose sha256 doesn't match the registry's bytes — so we use the - buildx imagetools path instead. - """ - if tool == "crane": - cmd = ["crane", "digest", image_ref] - elif tool == "skopeo": - cmd = [ - "skopeo", "inspect", "--format", "{{.Digest}}", - f"docker://{image_ref}", - ] - elif tool == "docker": - cmd = [ - "docker", "buildx", "imagetools", "inspect", - "--format", "{{.Manifest.Digest}}", image_ref, - ] - else: - raise ValueError(f"Unknown tool: {tool}") - - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - raise RuntimeError( - f"{tool} failed for {image_ref}: {result.stderr.strip()}") - - return Digest(result.stdout.strip()) - - -def check_digest_exists(image_ref_with_digest: str, tool: str) -> bool: - """Return True if digest is present at the target, False if confirmed absent. - - Raises RuntimeError on auth/network/registry errors so the caller can - distinguish "needs copy" (404) from "operator must investigate" - (everything else). Treating any non-zero exit as "needs copy" hides - expired credentials behind a flood of redundant copy attempts. - """ - if tool == "crane": - cmd = ["crane", "manifest", image_ref_with_digest] - elif tool == "skopeo": - cmd = [ - "skopeo", "inspect", "--raw", - f"docker://{image_ref_with_digest}", - ] - elif tool == "docker": - cmd = [ - "docker", "buildx", "imagetools", "inspect", - image_ref_with_digest, - ] - else: - raise ValueError(f"Unknown tool: {tool}") - - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode == 0: - return True - - stderr = result.stderr.strip() - if _is_not_found(stderr): - return False - - raise RuntimeError( - f"{tool} manifest check failed for {image_ref_with_digest} " - f"(exit {result.returncode}): {stderr or '(no stderr)'}") - - -def copy_image(source_ref: str, target: str, tool: str) -> tuple[bool, str]: - """Copy an image from source to target. Returns (success, error_msg).""" - if tool == "crane": - cmd = [tool, "copy", "--platform", "all", source_ref, target] - elif tool == "skopeo": - cmd = [ - tool, "copy", "--all", f"docker://{source_ref}", - f"docker://{target}" - ] - elif tool == "docker": - cmd = [ - tool, "buildx", "imagetools", "create", "-t", target, source_ref - ] - else: - raise ValueError(f"Unknown copy tool: {tool}") - - result = subprocess.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - return False, result.stderr.strip() - return True, "" - - -def list_tags(image_repo: str, tool: str) -> list[str]: - """List all tags for a repository. - - Only crane and skopeo expose tag listing; docker has no equivalent in - the buildx/imagetools surface. When the active tool is docker we skip - vague-tag pinning instead of failing the run. - """ - if tool == "docker": - print(" [warning] docker has no tag-listing API; " - "version pinning for vague tags will be skipped. Install " - "crane or skopeo for full tag resolution.", file=sys.stderr) - return [] - - if tool == "crane": - result = subprocess.run(["crane", "ls", image_repo], - capture_output=True, - text=True) - elif tool == "skopeo": - result = subprocess.run( - ["skopeo", "list-tags", f"docker://{image_repo}"], - capture_output=True, - text=True, - ) - else: - raise ValueError(f"Unknown tool for tag listing: {tool}") - - if result.returncode != 0: - print( - f" [warning] failed to list tags for {image_repo}: " - f"{result.stderr.strip()}", - file=sys.stderr) - return [] - - if tool == "crane": - return result.stdout.strip().splitlines() - - try: - data = json.loads(result.stdout) - return data.get("Tags", []) - except json.JSONDecodeError as exc: - print(f" [warning] failed to parse tags for {image_repo}: {exc}", - file=sys.stderr) - return [] - - -# --------------------------------------------------------------------------- -# Image reference parsing & YAML config -# --------------------------------------------------------------------------- - - -def parse_image_ref(raw: str) -> str: - """Normalize a Docker image reference to registry/repo:tag form.""" - ref = raw.strip() - parts = ref.split("/", 1) - if len(parts) == 1: - registry = "index.docker.io" - repo = f"library/{ref}" - elif "." in parts[0] or ":" in parts[0]: - registry = parts[0] - repo = parts[1] - else: - registry = "index.docker.io" - repo = ref - - if ":" not in repo: - repo = f"{repo}:latest" - - return f"{registry}/{repo}" - - -@dataclass(frozen=True) -class MirrorEntry: - """A single entry in mirror_images.yaml. - - Two on-disk shapes exist so the common case stays terse: - - - alpine:3.20 - - when the destination follows the default `DEST_REGISTRY/` - convention, or a single-key mapping with an explicit override: - - - nginx:1.27.5: - target: registry.ddbuild.io/ci/libdatadog-build/custom/nginx:1.27.5 - - `to_yaml` round-trips back to whichever form the entry's `target` - implies; an unset `target` always emits the bare-string form. - - Two computed attributes are available but not persisted to YAML: - - ``source_registry`` - The normalised source registry hostname (e.g. ``docker.io``, - ``ghcr.io``). Stored eagerly at construction time. - - ``target_registry`` - The mirror host+path prefix for this source registry, e.g. - ``registry.ddbuild.io/ci/myrepo/mirror`` for ``docker.io`` or - ``registry.ddbuild.io/ci/myrepo/mirror/ghcr.io`` for ``ghcr.io``. - Computed lazily. - """ - source: str - target: str | None = None - source_registry: str = field(init=False) - - def __post_init__(self) -> None: - raw_registry = parse_image_ref(self.source).split("/", 1)[0] - registry = "docker.io" if raw_registry == "index.docker.io" else raw_registry - object.__setattr__(self, "source_registry", registry) - - @property - def target_registry(self) -> "str | None": - """Mirror prefix for this source registry. - """ - image_path = ( - self.source if self.source_registry == "docker.io" - else self.source[len(self.source_registry) + 1:] - ) - t = self.resolved_target() - suffix = f"/{image_path}" - return t[: -len(suffix)] if t.endswith(suffix) else None - - def resolved_target(self) -> str: - return self.target if self.target is not None else f"{_dest_registry()}/{self.source}" - - @classmethod - def from_yaml(cls, raw: object) -> "MirrorEntry": - if isinstance(raw, str): - return cls(source=raw) - if isinstance(raw, dict) and len(raw) == 1: - (source, opts), = raw.items() - if not isinstance(source, str): - raise ValueError( - f"expected entry key to be str, got {type(source).__name__}: {source!r}") - if opts is None: - opts = {} - elif not isinstance(opts, dict): - raise ValueError( - f"expected options for {source!r} to be a mapping or null, " - f"got {type(opts).__name__}: {opts!r}") - target = opts.get("target") - if target is not None and not isinstance(target, str): - raise ValueError( - f"expected target for {source!r} to be a str, " - f"got {type(target).__name__}: {target!r}") - return cls(source=source, target=target) - raise ValueError( - f"expected str or single-key dict, got {type(raw).__name__}: {raw!r}") - - def to_yaml(self) -> "str | dict[str, dict[str, str]]": - if self.target is None: - return self.source - return {self.source: {"target": self.target}} - - -@dataclass(frozen=True) -class LockEntry: - """A single resolved image in mirror_images.lock.yaml.""" - digest: Digest - target: str - tag: str - - @classmethod - def from_yaml(cls, raw: dict) -> "LockEntry": - try: - return cls( - digest=Digest(raw["digest"]), - target=raw["target"], - tag=raw["tag"], - ) - except KeyError as exc: - raise ValueError(f"missing field in lock entry: {exc}") from exc - - def to_yaml(self) -> "dict[str, str]": - # Field order matters for stable yaml output. - return {"digest": str(self.digest), "target": self.target, "tag": self.tag} - - -@dataclass(frozen=True) -class MirrorManifest: - """Parsed mirror_images.yaml: images plus lint ignore lists. - - `ignore_patterns` suppresses `lint` flags for image references that - are intentionally not mirrored (test fixtures, refs that resolve at - build time via a variable, alternate registries the repo trusts). - - `ignore_paths` skips entire directories from lint scanning (e.g. - vendored submodules, example/ trees). - """ - entries: list[MirrorEntry] - ignore_patterns: list[re.Pattern[str]] - # Raw `ignore.images` strings kept alongside the compiled patterns so - # `add` can round-trip them verbatim (preserving the user's quoting) - # without re-reading the file. - ignore_images_raw: list[str] - ignore_paths: list[str] - was_mapping_form: bool - - def is_ignored(self, image: str) -> bool: - return any(p.fullmatch(image) for p in self.ignore_patterns) - - -def _parse_str_list(value: object, what: str, path: str) -> list[str]: - """Validate a YAML value as a list[str], filtering+warning on bad entries.""" - if value is None: - return [] - if not isinstance(value, list): - raise ValueError(f"{what} in {path} must be a list") - out: list[str] = [] - for i, raw in enumerate(value): - if not isinstance(raw, str): - print( - f" [warning] {what}[{i}] in {path} must be a string, " - f"got {type(raw).__name__}", file=sys.stderr) - continue - out.append(raw) - return out - - -def parse_mirror_yaml(path: str) -> MirrorManifest: - """Parse mirror_images.yaml. - - Two on-disk shapes are accepted: - - # legacy flat list - - alpine:3.20 - - # mapping form, enables `ignore:` - images: - - alpine:3.20 - ignore: - images: - - public\\.ecr\\.aws/.* - paths: - - vendor - - Empty / comments-only / explicit `null` files yield an empty manifest - so a freshly `init`'d project works immediately with `add`, `lock`, - etc. - """ - with open(path) as f: - data = yaml.safe_load(f) - - if data is None: - return MirrorManifest( - entries=[], ignore_patterns=[], ignore_images_raw=[], - ignore_paths=[], was_mapping_form=False, - ) - - ignore_images_raw: list[str] = [] - ignore_paths: list[str] = [] - - if isinstance(data, list): - raw_entries: list = data - was_mapping = False - elif isinstance(data, dict): - raw_entries = data.get("images") or [] - if not isinstance(raw_entries, list): - raise ValueError(f"'images' in {path} must be a list") - ignore_section = data.get("ignore") - if ignore_section is not None: - if not isinstance(ignore_section, dict): - raise ValueError( - f"'ignore' in {path} must be a mapping with 'images' " - f"and/or 'paths' keys") - ignore_images_raw = _parse_str_list( - ignore_section.get("images"), "ignore.images", path) - ignore_paths = _parse_str_list( - ignore_section.get("paths"), "ignore.paths", path) - was_mapping = True - else: - raise ValueError( - f"Expected YAML list or mapping in {path}, got {type(data).__name__}") - - entries: list[MirrorEntry] = [] - for i, raw in enumerate(raw_entries): - try: - entries.append(MirrorEntry.from_yaml(raw)) - except ValueError as exc: - print(f" [warning] ignoring entry #{i} in {path}: {exc}", - file=sys.stderr) - - ignore_patterns: list[re.Pattern[str]] = [] - for i, raw in enumerate(ignore_images_raw): - try: - ignore_patterns.append(re.compile(raw)) - except re.error as exc: - print(f" [warning] ignore.images[{i}] {raw!r} in {path} is " - f"not a valid regex: {exc}", file=sys.stderr) - - return MirrorManifest( - entries=entries, - ignore_patterns=ignore_patterns, - ignore_images_raw=ignore_images_raw, - ignore_paths=ignore_paths, - was_mapping_form=was_mapping, - ) - - -def parse_lock_yaml(path: str) -> dict[str, LockEntry]: - """Parse mirror_images.lock.yaml into {source: LockEntry}.""" - with open(path) as f: - data = yaml.safe_load(f) - if not isinstance(data, dict) or "images" not in data: - raise ValueError( - f"Malformed lock file {path}: expected a YAML mapping with an 'images' key" - ) - images = data["images"] - if not isinstance(images, dict): - raise ValueError( - f"Malformed lock file {path}: 'images' must be a mapping, " - f"got {type(images).__name__}") - return {src: LockEntry.from_yaml(info) for src, info in images.items()} - - -# --------------------------------------------------------------------------- -# Tag resolution for vague tags (e.g. "latest") -# --------------------------------------------------------------------------- - - -def _version_sort_key(tag: str) -> list[tuple[int, int | str]]: - """Sort key that orders version-like tags so the highest version comes last. - - Numeric parts sort before string parts so that "1.2.3" < "1.2.10". - """ - parts = re.split(r"[.\-+]", tag.lstrip("v")) - return [(0, int(p)) if p.isdigit() else (1, p) for p in parts] - - -def find_specific_tag(image_ref: str, digest: str, tool: str) -> str: - """Given a vague tag (e.g. 'latest'), find the most specific version tag - that shares the same digest. - - Uses a dedicated inner ThreadPoolExecutor so this function is safe to - call from a worker of an outer pool. (Submitting back into the *same* - pool you are running in deadlocks once every worker is a parent.) - """ - if ":" in image_ref: - repo = image_ref.rsplit(":", 1)[0] - else: - repo = image_ref - - all_tags = list_tags(repo, tool) - if not all_tags: - return "" - - # "clean" = pure version tags like "1.2.3" or "v1.2.3" - clean_re = re.compile(r"^v?\d+(\.\d+)+$") - # "suffixed" = version + single suffix like "1.2.3-alpine" - suffixed_re = re.compile(r"^v?\d+(\.\d+)+-[a-zA-Z][a-zA-Z0-9]*$") - - clean_tags = sorted( - [t for t in all_tags if clean_re.match(t)], - key=_version_sort_key, reverse=True, - ) - clean_set = set(clean_tags) - suffixed_tags = sorted( - [t for t in all_tags if suffixed_re.match(t) and t not in clean_set], - key=_version_sort_key, reverse=True, - ) - - candidates = clean_tags[:30] + suffixed_tags[:20] - if not candidates: - return "" - - def _resolve_tag(tag: str) -> tuple[str, Digest | None]: - try: - d = resolve_digest(f"{repo}:{tag}", tool) - return tag, d - except (RuntimeError, subprocess.SubprocessError, ValueError) as exc: - print(f" [warning] could not resolve {repo}:{tag}: {exc}", - file=sys.stderr) - return tag, None - - matches = [] - with ThreadPoolExecutor(max_workers=min(8, len(candidates))) as inner_pool: - futures = [inner_pool.submit(_resolve_tag, tag) for tag in candidates] - for future in as_completed(futures): - tag, tag_digest = future.result() - if tag_digest == digest: - matches.append(tag) - - if not matches: - return "" - - # Prefer clean version tags (e.g. "1.2.3") over suffixed ones (e.g. "1.2.3-alpine") - clean_matches = sorted( - [t for t in matches if clean_re.match(t)], - key=_version_sort_key, reverse=True, - ) - if clean_matches: - return clean_matches[0] - - matches.sort(key=_version_sort_key, reverse=True) - return matches[0] - - -# --------------------------------------------------------------------------- -# Lock file -# --------------------------------------------------------------------------- - - -def _write_lock_file(path: str, results: dict[str, LockEntry]) -> None: - """Write the lock file with current results, sorted for stable output.""" - lock_data = { - "version": 1, - "images": {k: results[k].to_yaml() for k in sorted(results)}, - } - dir_name = os.path.dirname(path) or "." - fd, tmp = tempfile.mkstemp(dir=dir_name, suffix=".tmp", prefix=".lock_") - try: - with os.fdopen(fd, "w") as f: - f.write("# Auto-generated by: uv run bin/mirror_images.py lock\n") - f.write("# Do not edit manually.\n") - yaml.dump(lock_data, f, default_flow_style=False, sort_keys=False) - os.replace(tmp, path) - except BaseException: - os.unlink(tmp) - raise - - -def cmd_lock(args: argparse.Namespace) -> int: - """Generate mirror_images.lock.yaml with resolved digests.""" - tool = _find_digest_tool() - entries = parse_mirror_yaml(args.mirror_yaml).entries - declared_sources = {e.source for e in entries} - output_path = args.output or LOCK_YAML - - # Load existing lock file to preserve previously resolved entries - results: dict[str, LockEntry] = {} - if os.path.exists(output_path): - try: - existing = parse_lock_yaml(output_path) - results.update(existing) - print(f"Loaded {len(results)} existing entries from {output_path}") - except (OSError, yaml.YAMLError, ValueError) as exc: - print( - f" [warning] could not parse existing lock file {output_path}: {exc}", - file=sys.stderr) - print(" [warning] resolving all images from scratch.", - file=sys.stderr) - - def _current_lock_entries() -> dict[str, LockEntry]: - """Return only the results that are still declared in mirror_images.yaml.""" - return {k: results[k] for k in results if k in declared_sources} - - # Only resolve entries not already in the lock file - to_resolve = [e for e in entries if e.source not in results] - if not to_resolve: - print(f"All {len(entries)} images already resolved in {output_path}") - _write_lock_file(output_path, _current_lock_entries()) - return 0 - - print( - f"Resolving {len(to_resolve)} of {len(entries)} images using {tool} (parallelism={args.jobs})..." - ) - - errors: list[tuple[str, str]] = [] - pinning_failures: list[str] = [] - fatal_write_error: Exception | None = None - - def _resolve(entry: MirrorEntry): - digest = resolve_digest(entry.source, tool) - normalized = parse_image_ref(entry.source) - tag = normalized.rsplit(":", 1)[1] - - pinning_failed = False - if tag == "latest": - pinned = find_specific_tag(entry.source, digest, tool) or tag - pinning_failed = (pinned == tag) - tag = pinned - - return entry, digest, tag, pinning_failed - - with ThreadPoolExecutor(max_workers=args.jobs) as pool: - futures = { - pool.submit(_resolve, entry): entry.source - for entry in to_resolve - } - for future in as_completed(futures): - src = futures[future] - try: - entry, digest, tag, pinning_failed = future.result() - except (OSError, RuntimeError, subprocess.SubprocessError, - ValueError) as exc: - errors.append((src, str(exc))) - print(f" [error] resolving {src}: {exc}", file=sys.stderr) - continue - - original_tag = entry.source.rsplit(":", 1)[-1] - tag_info = f" (tag: {tag})" if tag != original_tag else "" - print(f" {entry.source}: {digest}{tag_info}") - if pinning_failed: - pinning_failures.append(entry.source) - - results[entry.source] = LockEntry( - digest=digest, - target=entry.resolved_target(), - tag=tag, - ) - try: - _write_lock_file(output_path, _current_lock_entries()) - except (OSError, yaml.YAMLError) as exc: - fatal_write_error = exc - print(f"\n [fatal] failed to write {output_path}: {exc}", - file=sys.stderr) - for f in futures: - f.cancel() - break - - if fatal_write_error is not None: - return 1 - - if pinning_failures: - print( - f"\n [warning] {len(pinning_failures)} entr{'y' if len(pinning_failures) == 1 else 'ies'} " - "kept a vague tag — tag pinning failed (transient registry error?). " - "The digest is pinned, but the human-readable tag is not specific:", - file=sys.stderr) - for src in pinning_failures: - print(f" - {src}: tag={results[src].tag}", file=sys.stderr) - - if errors: - print(f"\nFailed to resolve {len(errors)} image(s):", file=sys.stderr) - for src, err in errors: - print(f" - {src}: {err}", file=sys.stderr) - return 1 - - print(f"\nWrote {output_path} ({len(results)} images)") - return 0 - - -# --------------------------------------------------------------------------- -# Mirror -# --------------------------------------------------------------------------- - - -_AUTH_FAILURE_THRESHOLD = 3 - - -def cmd_mirror(args: argparse.Namespace) -> int: - """Sync images from lock file to target registry using crane/skopeo.""" - lock_path = args.lock_yaml - if not os.path.exists(lock_path): - print(f"Lock file not found: {lock_path}", file=sys.stderr) - print("Run 'mirror_images.py lock' first.", file=sys.stderr) - return 1 - - digest_tool = _find_digest_tool() - copy_tool = _find_copy_tool() - images = parse_lock_yaml(lock_path) - - # Phase 1: check which images need copying - print( - f"Phase 1: checking {len(images)} images using {digest_tool} ({args.jobs} workers)..." - ) - - def _check_if_mirrored(source: str, info: LockEntry): - # Resolve the target *tag* (not just the digest) so we re-copy when - # the bytes already live in the repo under a different tag — e.g. - # a previous run mirrored the same digest as `mirror/foo:1` and - # this run wants `mirror/foo:1.2.3`. Skipping by digest alone would - # leave the new tag alias unset and consumers pulling by tag get - # `manifest unknown`. - print(f" ? {source} -> {info.target}") - try: - actual = resolve_digest(info.target, digest_tool) - except RuntimeError as exc: - if _is_not_found(str(exc)): - return source, info, False - raise - return source, info, actual == info.digest - - to_copy: list[tuple[str, LockEntry]] = [] - already_present = 0 - checked = 0 - check_errors: list[tuple[str, str]] = [] - - with ThreadPoolExecutor(max_workers=args.jobs) as pool: - futures = { - pool.submit(_check_if_mirrored, src, info): src - for src, info in images.items() - } - for future in as_completed(futures): - src = futures[future] - checked += 1 - try: - source, info, present = future.result() - except (OSError, RuntimeError, subprocess.SubprocessError, - ValueError) as exc: - check_errors.append((src, str(exc))) - print(f" [error] checking {src}: {exc}", file=sys.stderr) - continue - if present: - already_present += 1 - print( - f" = [{checked}/{len(images)}] {source} (already mirrored)" - ) - else: - to_copy.append((source, info)) - print(f" + [{checked}/{len(images)}] {source} (needs copy)") - - if check_errors: - print( - f"\n [fatal] {len(check_errors)} image(s) failed pre-flight check " - "(not a 404). This usually means broken auth, network, or registry " - "config. Refusing to attempt copies until checks pass:", - file=sys.stderr) - for src, err in check_errors: - print(f" - {src}: {err}", file=sys.stderr) - return 2 - - if not to_copy: - print(f"\nAll {len(images)} images are up to date.") - return 0 - - print( - f"\n{already_present} already mirrored, {len(to_copy)} to copy using {copy_tool}.\n" - ) - - # Phase 2: copy missing images in parallel, fail-fast on persistent auth errors. - print( - f"\nPhase 2: copying {len(to_copy)} images using {copy_tool} ({args.jobs} workers)..." - ) - - errors: list[tuple[str, str]] = [] - auth_failures = 0 - aborted_for_auth = False - abort_lock = threading.Lock() - - def _do_copy(idx: int, source: str, info: LockEntry): - source_ref = f"{parse_image_ref(source).rsplit(':', 1)[0]}@{info.digest}" - if args.dry_run: - return idx, source, info, source_ref, True, "[dry-run]" - ok, err = copy_image(source_ref, info.target, copy_tool) - return idx, source, info, source_ref, ok, err - - with ThreadPoolExecutor(max_workers=args.jobs) as pool: - indexed = list(enumerate(sorted(to_copy), 1)) - future_map = { - pool.submit(_do_copy, i, src, info): (src, info) - for i, (src, info) in indexed - } - for future in as_completed(future_map): - if future.cancelled(): - continue - idx, source, info, source_ref, ok, err = future.result() - print( - f" -> [{idx}/{len(to_copy)}] {source} ({info.digest[:19]}...)\n" - f" src: {source_ref}\n" - f" dest: {info.target}") - if args.dry_run: - print( - f" [dry-run] {copy_tool} copy {source_ref} {info.target}" - ) - continue - if ok: - print(f" OK") - continue - errors.append((source, err)) - print( - f" [error] ({copy_tool}) {err or '(no stderr)'}", - file=sys.stderr) - if _is_auth_error(err): - with abort_lock: - auth_failures += 1 - if (auth_failures >= _AUTH_FAILURE_THRESHOLD - and not aborted_for_auth): - aborted_for_auth = True - print( - f"\n [fatal] {auth_failures} auth-shaped copy failures — " - "cancelling remaining copies. Check destination registry " - "credentials.", - file=sys.stderr) - for f in future_map: - f.cancel() - - if errors: - print(f"\nFailed to copy {len(errors)} image(s):", file=sys.stderr) - for src, err in errors: - print(f" - {src}: {err}", file=sys.stderr) - return 1 - - action = "would copy" if args.dry_run else "copied" - print( - f"\nDone: {already_present} already present, {len(to_copy)} {action}.") - return 0 - - -# --------------------------------------------------------------------------- -# BuildKit daemon config generation -# --------------------------------------------------------------------------- - - -def cmd_buildkitd(args: argparse.Namespace) -> int: - entries = parse_mirror_yaml(args.mirror_yaml).entries - if not entries: - print(" [error] no entries in mirror_images.yaml.", file=sys.stderr) - return 1 - - registry_to_mirror: dict[str, str] = {} - - for entry in entries: - mirror = entry.target_registry - if mirror is None: - print( - f" [warning] skipping {entry.source!r}: explicit target does not " - "follow the standard {dest}/{image_path} convention.", - file=sys.stderr, - ) - continue - - reg = entry.source_registry - existing = registry_to_mirror.get(reg) - if existing is not None and existing != mirror: - print( - f" [error] conflicting mirror addresses for {reg!r}:\n" - f" {existing!r} (seen earlier)\n" - f" {mirror!r} (from {entry.source!r})", - file=sys.stderr, - ) - return 1 - - registry_to_mirror[reg] = mirror - - lines = [ - "# Auto-generated by: mirror_images buildkitd", - "# Do not edit manually — regenerate with:", - "# mirror_images buildkitd", - "", - ] - for reg in sorted(registry_to_mirror): - lines.append(f'[registry."{reg}"]') - lines.append(f' mirrors = ["{registry_to_mirror[reg]}"]') - lines.append("") - - content = "\n".join(lines) - - if args.output: - with open(args.output, "w") as f: - f.write(content) - print(f"Wrote {args.output} ({len(registry_to_mirror)} registry mirror(s))") - else: - print(content, end="") - - return 0 - - -# --------------------------------------------------------------------------- -# Lint -# --------------------------------------------------------------------------- - - -def find_public_image_refs( - skip_dirs: tuple[str, ...] = (), -) -> list[tuple[str, int, str, str]]: - """Scan the repo for image references not using registry.ddbuild.io. - - `skip_dirs` are repo-relative directory paths to skip (typically - sourced from `mirror_images.yaml#ignore.paths`). - """ - internal_prefixes = ("registry.ddbuild.io/", "486234852809.dkr.ecr") - hits = [] - - _ignore_prefixes = internal_prefixes + ( - "$", "{", "nginx-datadog-test-", - ) - - def _is_external(img: str) -> bool: - if any(img.startswith(p) for p in _ignore_prefixes): - return False - return bool(re.match(r"^[a-zA-Z0-9]", img)) - - def _read_lines(filepath): - try: - with open(filepath) as f: - return list(enumerate(f, 1)) - except OSError as exc: - print(f" [warning] Could not read {filepath}: {exc}", - file=sys.stderr) - return [] - - # Trailing `/` keeps "foo" from matching "foo-bar". - _skip_dir_prefixes = tuple( - os.path.join(PROJECT_DIR, d) + os.sep for d in skip_dirs - ) - - def _in_skip_dir(filepath): - return filepath.startswith(_skip_dir_prefixes) - - def _scan_dockerfiles(): - for filepath in glob.glob(os.path.join(PROJECT_DIR, "**/Dockerfile*"), - recursive=True): - if "/.git/" in filepath or _in_skip_dir(filepath): - continue - for lineno, line in _read_lines(filepath): - stripped = line.strip() - m = re.match(r"^FROM\s+(\S+)", stripped, re.IGNORECASE) - if m and _is_external(m.group(1)): - hits.append((filepath, lineno, stripped, m.group(1))) - for m in re.finditer(r"--from=(\S+)", stripped): - img = m.group(1) - if ("/" in img or ":" in img) and _is_external(img): - hits.append((filepath, lineno, stripped, img)) - - def _scan_yaml_image_fields(): - skip_path_fragments = ("/.git/", "/.gitlab/") - for pattern in ("**/*.yml", "**/*.yaml"): - for filepath in glob.glob(os.path.join(PROJECT_DIR, pattern), - recursive=True): - if any(frag in filepath for frag in skip_path_fragments): - continue - if _in_skip_dir(filepath): - continue - for lineno, line in _read_lines(filepath): - m = re.match(r"^\s*image:\s+['\"]?(\S+?)['\"]?\s*$", line) - if m and _is_external(m.group(1)): - hits.append( - (filepath, lineno, line.rstrip(), m.group(1))) - - _scan_dockerfiles() - _scan_yaml_image_fields() - return hits - - -def _collect_gitlab_ci_files(entry: str) -> list[str]: - """Starting from a GitLab CI YAML file, follow local includes and return all file paths.""" - collected = [] - visited: set[str] = set() - queue: collections.deque[str] = collections.deque([entry]) - while queue: - path = queue.popleft() - abspath = os.path.join(PROJECT_DIR, path) if not os.path.isabs(path) else path - if abspath in visited or not os.path.isfile(abspath): - continue - visited.add(abspath) - collected.append(abspath) - try: - with open(abspath) as f: - data = yaml.safe_load(f) - except (OSError, yaml.YAMLError) as exc: - print(f" [warning] Could not parse {abspath}: {exc}", - file=sys.stderr) - continue - if not isinstance(data, dict): - continue - includes = data.get("include", []) - if isinstance(includes, dict): - includes = [includes] - if not isinstance(includes, list): - continue - for inc in includes: - if isinstance(inc, str): - queue.append(os.path.join(PROJECT_DIR, inc)) - elif isinstance(inc, dict) and "local" in inc: - queue.append(os.path.join(PROJECT_DIR, inc["local"])) - return collected - - -# Templates that map matrix variable names to image name patterns. -# {value} is replaced with the matrix variable value. -_GITLAB_MATRIX_IMAGE_TEMPLATES: dict[str, list[str]] = { - "BASE_IMAGE": ["{value}"], - "INGRESS_NGINX_VERSION": [ - "registry.k8s.io/ingress-nginx/controller:v{value}" - ], - "RESTY_VERSION": ["openresty/openresty:{value}-alpine"], -} - - -def _extract_matrix_combos(job: dict) -> list[dict]: - """Extract the parallel:matrix combo list from a GitLab CI job definition.""" - parallel = job.get("parallel") - if not isinstance(parallel, dict): - return [] - matrix = parallel.get("matrix") - if not isinstance(matrix, list): - return [] - return [c for c in matrix if isinstance(c, dict)] - - -def _expand_matrix_images(combo: dict) -> list[str]: - """Expand a single matrix combo into image references using known templates.""" - images = [] - for var_name, templates in _GITLAB_MATRIX_IMAGE_TEMPLATES.items(): - values = combo.get(var_name, []) - if isinstance(values, str): - values = [values] - if not isinstance(values, list): - continue - for val in values: - for tmpl in templates: - images.append(tmpl.format(value=val)) - return images - - -def find_gitlab_ci_images() -> list[tuple[str, str]]: - """Extract public image references from GitLab CI matrix variables. - - Returns a list of (source_file_relpath, image_ref) tuples. - """ - ci_entry = os.path.join(PROJECT_DIR, ".gitlab-ci.yml") - if not os.path.isfile(ci_entry): - return [] - - results = [] - for filepath in _collect_gitlab_ci_files(ci_entry): - try: - with open(filepath) as f: - data = yaml.safe_load(f) - except (OSError, yaml.YAMLError) as exc: - print(f" [warning] Could not parse {filepath}: {exc}", - file=sys.stderr) - continue - if not isinstance(data, dict): - continue - - relpath = os.path.relpath(filepath, PROJECT_DIR) - for _job_name, job in data.items(): - if not isinstance(job, dict): - continue - for combo in _extract_matrix_combos(job): - for img in _expand_matrix_images(combo): - results.append((relpath, img)) - - return results - - -def cmd_lint(args: argparse.Namespace) -> int: - """Check that all images are referenced from registry.ddbuild.io.""" - manifest = parse_mirror_yaml(args.mirror_yaml) - declared = {e.source: e.resolved_target() for e in manifest.entries} - rc = 0 - - # --- Check 1: public image references in Dockerfiles and YAML --- - public_refs = [r for r in find_public_image_refs(tuple(manifest.ignore_paths)) - if not manifest.is_ignored(r[3])] - - if public_refs: - by_image: dict[str, list[tuple[str, int, str]]] = {} - for filepath, lineno, line, img in public_refs: - by_image.setdefault(img, []).append((filepath, lineno, line)) - - undeclared = [] - print( - "Public image references found (should use registry.ddbuild.io mirror):\n" - ) - for img in sorted(by_image): - replacement = declared.get(img) - if not replacement: - replacement = f"{_dest_registry()}/{img}" - undeclared.append(img) - - print(f" {img}") - print(f" -> {replacement}") - for filepath, lineno, line in by_image[img]: - relpath = os.path.relpath(filepath, PROJECT_DIR) - print(f" {relpath}:{lineno}") - print() - - if undeclared: - print( - "Images not declared in mirror_images.yaml (add them first):") - for img in sorted(undeclared): - print(f" - {img}") - print() - - print( - f"{len(public_refs)} public image reference(s) across {len(by_image)} image(s)." - ) - rc = 1 - - # --- Check 2: GitLab CI matrix images must be in mirror_images.yaml --- - ci_images = [(src, img) for src, img in find_gitlab_ci_images() - if not manifest.is_ignored(img)] - if ci_images: - missing: dict[str, list[str]] = {} - for source_file, img in ci_images: - if img not in declared: - missing.setdefault(img, []).append(source_file) - - if missing: - print("GitLab CI matrix images not declared in mirror_images.yaml:\n") - for img in sorted(missing): - sources = sorted(set(missing[img])) - print(f" - {img}") - for src in sources: - print(f" {src}") - print( - f"\n{len(missing)} image(s) from GitLab CI not in mirror_images.yaml." - ) - rc = 1 - else: - print("All GitLab CI matrix images are declared in mirror_images.yaml.") - - if rc == 0: - print("All image references use registry.ddbuild.io.") - return rc - - -# --------------------------------------------------------------------------- -# List ignored image references -# --------------------------------------------------------------------------- - - -def cmd_ignored(args: argparse.Namespace) -> int: - """Emit JSON listing image refs suppressed by mirror_images.yaml ignores. - - Default: only refs outside `ignore.paths` whose source matches - `ignore.images`. With `--scan-paths`, also walks into `ignore.paths` - directories and reports refs found there (rule="path"). - """ - manifest = parse_mirror_yaml(args.mirror_yaml) - - skip_dirs: tuple[str, ...] = () if args.scan_paths else tuple(manifest.ignore_paths) - refs = find_public_image_refs(skip_dirs) - - def _match_path(relpath: str) -> str | None: - for p in manifest.ignore_paths: - if relpath == p or relpath.startswith(p + os.sep): - return p - return None - - def _match_pattern(img: str) -> str | None: - for p in manifest.ignore_patterns: - if p.fullmatch(img): - return p.pattern - return None - - def _rec(img, rule, matched, file, line): - return {"image": img, "rule": rule, "matched": matched, - "file": file, "line": line} - - suppressed: list[dict] = [] - for filepath, lineno, _line, img in refs: - relpath = os.path.relpath(filepath, PROJECT_DIR) - # Path rule wins over image-pattern when both apply, since lint - # would never reach the image check for files inside an ignored - # directory. - path_match = _match_path(relpath) - if path_match is not None: - suppressed.append(_rec(img, "path", path_match, relpath, lineno)) - continue - pattern_match = _match_pattern(img) - if pattern_match is not None: - suppressed.append(_rec(img, "pattern", pattern_match, relpath, lineno)) - - for src_file, img in find_gitlab_ci_images(): - pattern_match = _match_pattern(img) - if pattern_match is not None: - suppressed.append(_rec(img, "pattern", pattern_match, src_file, None)) - - suppressed.sort(key=lambda r: (r["image"], r["file"], r["line"] or 0)) - - output = { - "patterns": [p.pattern for p in manifest.ignore_patterns], - "paths": manifest.ignore_paths, - "scanned_ignored_paths": args.scan_paths, - "suppressed": suppressed, - } - print(json.dumps(output, indent=2)) - return 0 - - -# --------------------------------------------------------------------------- -# Add images -# --------------------------------------------------------------------------- - - -def _write_mirror_yaml( - path: str, - entries: list[MirrorEntry], - ignore_section: dict[str, list[str]], - use_mapping_form: bool, -) -> None: - """Rewrite mirror_images.yaml preserving leading comments. - - Uses mapping form (`images:` / `ignore:`) when either the file already - used that form or the ignore section is non-empty; otherwise emits - the legacy flat list. Bare-string entries keep their quoted shape; - custom-target entries round-trip through yaml.dump. - """ - with open(path) as f: - header_lines = [] - for line in f: - if line.startswith("#") or line.strip() == "": - header_lines.append(line) - else: - break - - def _entry_lines(entry: MirrorEntry, indent: str = "") -> str: - shape = entry.to_yaml() - if isinstance(shape, str): - return f'{indent}- "{shape}"\n' - return "".join( - f"{indent}{line}\n" - for line in yaml.dump([shape], default_flow_style=False, sort_keys=False).splitlines() - ) - - with open(path, "w") as f: - f.writelines(header_lines) - if use_mapping_form: - f.write("images:\n") - for entry in entries: - f.write(_entry_lines(entry, indent=" ")) - if any(ignore_section.values()): - f.write("\nignore:\n") - for key in ("images", "paths"): - items = ignore_section.get(key) or [] - if not items: - continue - f.write(f" {key}:\n") - for item in items: - for line in yaml.dump( - [item], default_flow_style=False, - sort_keys=False).splitlines(): - f.write(f" {line}\n") - else: - for entry in entries: - f.write(_entry_lines(entry)) - - -def cmd_add(args: argparse.Namespace) -> int: - """Add one or more images to mirror_images.yaml (if not already present).""" - try: - manifest = parse_mirror_yaml(args.mirror_yaml) - except ValueError as exc: - print(f"ERROR: {exc}", file=sys.stderr) - return 1 - - entries = list(manifest.entries) - existing = {e.source for e in entries} - - added = [] - for img in args.images: - if img in existing: - print(f" already listed: {img}") - else: - entries.append(MirrorEntry(source=img)) - existing.add(img) - added.append(img) - print(f" added: {img}") - - if not added: - print("\nNo new images to add.") - return 0 - - # Sort bare-string entries (default target) by source; keep entries - # with a custom target after them in their original order. - bare = sorted([e for e in entries if e.target is None], key=lambda e: e.source) - custom = [e for e in entries if e.target is not None] - entries = bare + custom - - ignore_section = {"images": manifest.ignore_images_raw, "paths": manifest.ignore_paths} - use_mapping = manifest.was_mapping_form or any(ignore_section.values()) - _write_mirror_yaml(args.mirror_yaml, entries, ignore_section, use_mapping) - - print(f"\nAdded {len(added)} image(s) to {args.mirror_yaml}") - return 0 - - -# --------------------------------------------------------------------------- -# Init -# --------------------------------------------------------------------------- - - -_INIT_TEMPLATE = """\ -# Images to mirror into registry.ddbuild.io. -# See: https://github.com/DataDog/dd-repo-tools/blob/main/shared/mirror-images/mirror_images.md -# -# Flat-list form: -# - "alpine:3.20" -# - "nginx:1.27.5": -# target: registry.ddbuild.io/ci/example/custom/nginx:1.27.5 -# -# Mapping form (enables `ignore`): -# images: -# - "alpine:3.20" -# ignore: -# images: -# - "public\\.ecr\\.aws/.*" -# paths: -# - "vendor" -""" - - -def cmd_init(args: argparse.Namespace) -> int: - """Write a starter mirror_images.yaml at the project root.""" - path = args.mirror_yaml - if os.path.exists(path) and not args.force: - print(f"{path} already exists. Use --force to overwrite.", - file=sys.stderr) - return 1 - with open(path, "w") as f: - f.write(_INIT_TEMPLATE) - print(f"Wrote {path}") - print("Next: `mirror_images lock` to resolve digests, " - "then `mirror_images mirror` to copy.") - return 0 - - -# --------------------------------------------------------------------------- -# Relock images -# --------------------------------------------------------------------------- - - -def cmd_relock(args: argparse.Namespace) -> int: - """Re-resolve digests for images matching the given patterns. - - Removes matching entries from the lock file, then runs the lock - command to re-resolve them. Patterns use fnmatch-style wildcards - (e.g. 'nginx:1.29.*', 'openresty/*'). - """ - lock_path = args.output or LOCK_YAML - if not os.path.exists(lock_path): - print(f"Lock file not found: {lock_path}", file=sys.stderr) - print("Run 'lock' first to create it.", file=sys.stderr) - return 1 - - existing = parse_lock_yaml(lock_path) - patterns = args.patterns - - matched = [] - for src in list(existing): - if any(fnmatch.fnmatch(src, pat) for pat in patterns): - matched.append(src) - del existing[src] - - if not matched: - print(f"No images in lock file matched: {', '.join(patterns)}") - return 1 - - print(f"Removing {len(matched)} image(s) from lock file to re-resolve:") - for src in sorted(matched): - print(f" - {src}") - - # Write out the lock file without the matched entries - _write_lock_file(lock_path, existing) - - # Now run the normal lock flow which will resolve the missing entries - print() - return cmd_lock(args) - - -# --------------------------------------------------------------------------- -# CLI -# --------------------------------------------------------------------------- - - -def main() -> int: - parser = argparse.ArgumentParser( - prog="mirror_images", - description="Manage mirrored Docker images") - parser.add_argument( - "--mirror-yaml", - default=MIRROR_YAML, - help=f"Path to mirror_images.yaml (default: {MIRROR_YAML})", - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - init_parser = subparsers.add_parser( - "init", - help="Write a starter mirror_images.yaml", - description="Create mirror_images.yaml at the project root with a " - "small example. Refuses to overwrite an existing file unless --force.", - ) - init_parser.set_defaults(func=cmd_init) - init_parser.add_argument( - "-f", "--force", action="store_true", - help="Overwrite mirror_images.yaml if it already exists") - - lock_parser = subparsers.add_parser( - "lock", help="Generate mirror_images.lock.yaml with resolved digests") - lock_parser.set_defaults(func=cmd_lock) - lock_parser.add_argument("-j", - "--jobs", - type=int, - default=16, - help="Parallel workers (default: 16)") - lock_parser.add_argument( - "-o", - "--output", - help="Output path (default: mirror_images.lock.yaml)") - - mirror_parser = subparsers.add_parser( - "mirror", help="Sync images from lock file to target registry") - mirror_parser.set_defaults(func=cmd_mirror) - mirror_parser.add_argument( - "-j", - "--jobs", - type=int, - default=16, - help="Parallel workers for checking (default: 16)") - mirror_parser.add_argument("--lock-yaml", - default=LOCK_YAML, - help="Path to lock file") - mirror_parser.add_argument( - "--dry-run", - action="store_true", - help="Show what would be copied without copying") - - buildkitd_parser = subparsers.add_parser( - "buildkitd", - help="Generate a buildkitd.toml mirror config from mirror_images.yaml", - description=( - "Read mirror_images.yaml and emit a buildkitd.toml with one\n" - "[registry.\"X\"] section per source registry." - ), - ) - buildkitd_parser.set_defaults(func=cmd_buildkitd) - buildkitd_parser.add_argument( - "-o", - "--output", - help="Output path for buildkitd.toml (default: print to stdout)", - ) - - lint_parser = subparsers.add_parser( - "lint", help="Check that all image references use registry.ddbuild.io") - lint_parser.set_defaults(func=cmd_lint) - - ignored_parser = subparsers.add_parser( - "ignored", - help="List image references suppressed by mirror_images.yaml `ignore:` rules (JSON)", - description="Print JSON listing image refs that lint is silently " - "skipping. By default only `ignore.images` regex hits are scanned. " - "Pass --scan-paths to also walk into `ignore.paths` directories.", - ) - ignored_parser.set_defaults(func=cmd_ignored) - ignored_parser.add_argument( - "--scan-paths", action="store_true", - help="Also descend into `ignore.paths` directories and report refs found there", - ) - - add_parser = subparsers.add_parser( - "add", - help="Add images to mirror_images.yaml", - description= - "Add one or more Docker image references to mirror_images.yaml. " - "Images already listed are skipped. The file is re-sorted after adding." - ) - add_parser.set_defaults(func=cmd_add) - add_parser.add_argument( - "images", - nargs="+", - metavar="IMAGE", - help="Image references to add (e.g. 'nginx:1.30.0' 'alpine:3.21')") - - relock_parser = subparsers.add_parser( - "relock", - help="Re-resolve digests for specific images in the lock file", - description="Remove matching entries from mirror_images.lock.yaml and " - "re-resolve their digests. Uses fnmatch-style wildcards. " - "Example: 'nginx:1.29.*' matches nginx:1.29.0, nginx:1.29.5-alpine, etc." - ) - relock_parser.set_defaults(func=cmd_relock) - relock_parser.add_argument( - "patterns", - nargs="+", - metavar="PATTERN", - help= - "Glob patterns to match image names (e.g. 'nginx:1.29.*' 'openresty/*')" - ) - relock_parser.add_argument("-j", - "--jobs", - type=int, - default=16, - help="Parallel workers (default: 16)") - relock_parser.add_argument( - "-o", - "--output", - help="Path to lock file (default: mirror_images.lock.yaml)") - - args = parser.parse_args() - return args.func(args) - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/utils/scripts/update_mirror_images.py b/utils/scripts/update_mirror_images.py index c58a7cdb0b6..4562f06f13c 100644 --- a/utils/scripts/update_mirror_images.py +++ b/utils/scripts/update_mirror_images.py @@ -32,15 +32,16 @@ from utils._context._scenarios import get_all_scenarios, DockerScenario # noqa: E402 -# Local copy of mirror_images.py (from dd-repo-tools). Replace with the -# published URL once the buildkitd subcommand is released upstream. -MIRROR_IMAGES_SCRIPT = REPO_ROOT / "utils" / "scripts" / "mirror_images.py" +# Pinned dd-repo-tools mirror_images.py (override with $MIRROR_IMAGES_URL). +MIRROR_IMAGES_URL = os.environ.get( + "MIRROR_IMAGES_URL", + "https://binaries.ddbuild.io/dd-repo-tools/default/ca/d6c8f8ec4b1fb69b36e96dc204e5f79bed6bf067/mirror_images.py", +) # Destination registry for the mirrored images (override with $MIRROR_DEST_REGISTRY). DEFAULT_DEST_REGISTRY = "registry.ddbuild.io/system-tests/mirror" MIRROR_YAML = REPO_ROOT / "mirror_images.yaml" -BUILDKITD_TOML = REPO_ROOT / "utils" / "build" / "docker" / "buildkitd.toml" # Header written when mirror_images.yaml does not exist yet. The mirror_images.py # `add` command preserves existing comments, so this is only used on first run. @@ -102,14 +103,20 @@ def collect_images(excluded: set[str]) -> list[str]: def _run_mirror_images(*args: str) -> None: - """Invoke the local mirror_images.py via uv.""" + """Invoke the pinned dd-repo-tools mirror_images.py via uv.""" if shutil.which("uv") is None: sys.exit("error: 'uv' is required to run mirror_images.py — install it from https://docs.astral.sh/uv/") env = dict(os.environ) env.setdefault("MIRROR_DEST_REGISTRY", DEFAULT_DEST_REGISTRY) cmd = [ - "uv", "run", "--no-config", "--script", str(MIRROR_IMAGES_SCRIPT), - "--mirror-yaml", str(MIRROR_YAML), *args, + "uv", + "run", + "--no-config", + "--script", + MIRROR_IMAGES_URL, + "--mirror-yaml", + str(MIRROR_YAML), + *args, ] print(f"+ {' '.join(cmd)}", flush=True) subprocess.run(cmd, check=True, env=env) @@ -128,7 +135,6 @@ def main(excluded: set[str], *, skip_lock: bool) -> None: _run_mirror_images("add", *images) if not skip_lock: _run_mirror_images("lock") - _run_mirror_images("buildkitd", "--output", str(BUILDKITD_TOML)) if __name__ == "__main__": From 43c3ec5bca2ccba5279e5df7325873303693537c Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Wed, 24 Jun 2026 19:42:31 +0200 Subject: [PATCH 222/229] Fix ruff T201 and formatting in validate_param_env --- utils/ci/gitlab/validate_param_env.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/utils/ci/gitlab/validate_param_env.py b/utils/ci/gitlab/validate_param_env.py index 51a9f64b8ae..d950314c210 100644 --- a/utils/ci/gitlab/validate_param_env.py +++ b/utils/ci/gitlab/validate_param_env.py @@ -42,9 +42,7 @@ def validate_comma_lower(name: str, value: str) -> list[str]: errors = [] for item in (i.strip() for i in value.split(",")): if item and not re.match(r"^[a-z][a-z0-9_]*$", item): - errors.append( - _error(name, item, "expected comma-separated lowercase identifiers (e.g. 'all,appsec')") - ) + errors.append(_error(name, item, "expected comma-separated lowercase identifiers (e.g. 'all,appsec')")) return errors @@ -117,7 +115,7 @@ def main() -> None: if errors: for e in errors: - print(e, file=sys.stderr) + print(e, file=sys.stderr) # noqa: T201 sys.exit(1) with open(PARAM_ENV, "w") as f: @@ -127,7 +125,7 @@ def main() -> None: f.write(f"{name}={value}\n") with open(PARAM_ENV) as f: - print(f.read(), end="") + print(f.read(), end="") # noqa: T201 if __name__ == "__main__": From f017c70fed37de663700207feffb5e9de449b5f4 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 25 Jun 2026 14:31:20 +0200 Subject: [PATCH 223/229] Optional chunking --- .gitlab-ci.yml | 1 + utils/ci/gitlab/build_pipeline.py | 3 --- utils/ci/gitlab/main.yml | 28 +++++++++++++++++++--------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e180e03fbb7..3cb0cf54fcb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ include: stage: "e2e" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true + split_pipeline: true stages: - configure - e2e diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py index 04bac8a42f5..db7f4c27a3a 100644 --- a/utils/ci/gitlab/build_pipeline.py +++ b/utils/ci/gitlab/build_pipeline.py @@ -160,9 +160,6 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) libraries = args.libraries.split() - if not libraries: - print("No libraries specified, nothing to generate.", file=sys.stderr) # noqa: T201 - return 0 build( libraries=libraries, diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index a048019dd66..d6fea444154 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -50,6 +50,10 @@ spec: docker_auth: description: "Whether to authenticate calls to docker hub" default: "false" + split_pipeline: + description: "Split jobs across multiple child pipelines" + type: boolean + default: false --- @@ -72,6 +76,7 @@ variables: # Update this when either file changes: # cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12 CI_IMAGE: "registry.ddbuild.io/system-tests/ci-runner:f4f0d91aa8dc" + SYSTEM_TESTS_SPLIT_PIPELINE: "$[[ inputs.split_pipeline ]]" .system_tests_param_base: image: $CI_IMAGE @@ -100,14 +105,7 @@ build_test_pipeline: script: - | libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) - if [ -z "$libraries" ]; then - echo "No libraries configured, producing noop pipeline chunks" - for i in 0 1 2; do - printf 'noop:\n image: registry.ddbuild.io/images/mirror/alpine:latest\n tags:\n - arch:amd64\n script:\n - echo "no system tests configured"\n' > generated-pipeline-chunk-${i}.yml - done - touch build_params.env - exit 0 - fi + touch build_params.env scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') weblogs=$(echo "$[[ inputs.weblogs ]],$WEBLOGS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') @@ -132,7 +130,9 @@ build_test_pipeline: for library in $libraries; do python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$excluded_scenarios" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$binaries_artifacts" --parametric-job-count $parametric_job_count --output params_${library}.json done - python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks 3 --docker-auth "$docker_auth" --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$binaries_artifacts" + chunks=1 + if [ "$SYSTEM_TESTS_SPLIT_PIPELINE" = "true" ]; then chunks=3; fi + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks $chunks --docker-auth "$docker_auth" --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$binaries_artifacts" artifacts: paths: - system-tests/generated-pipeline-chunk-*.yml @@ -171,6 +171,11 @@ run_test_pipeline_0: run_test_pipeline_1: extends: .run_test_pipeline_base + rules: + - if: $SYSTEM_TESTS_SPLIT_PIPELINE != "true" + when: never + - if: $[[ inputs.condition ]] + - when: never trigger: include: - artifact: system-tests/generated-pipeline-chunk-1.yml @@ -179,6 +184,11 @@ run_test_pipeline_1: run_test_pipeline_2: extends: .run_test_pipeline_base + rules: + - if: $SYSTEM_TESTS_SPLIT_PIPELINE != "true" + when: never + - if: $[[ inputs.condition ]] + - when: never trigger: include: - artifact: system-tests/generated-pipeline-chunk-2.yml From f387d56e7a481112e72472a382a4d91e30fb552a Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 25 Jun 2026 14:33:01 +0200 Subject: [PATCH 224/229] Move all e2e pipeline to the earliest stage --- .gitlab-ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3cb0cf54fcb..3a5620f0d7b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,13 +2,12 @@ include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/0b4d88122bb64a87131eeb9446a3671598790917ec30fc31851ccb6ee8375b8a/single-step-instrumentation-tests.yml - local: /utils/ci/gitlab/main.yml inputs: - stage: "e2e" + stage: "configure" ref: "$CI_COMMIT_SHA" push_to_test_optimization: true split_pipeline: true stages: - configure - - e2e - nodejs - java - dotnet @@ -322,7 +321,7 @@ generate_system_tests_lib_injection_images: system-tests-param: extends: .system_tests_param_base - stage: e2e + stage: configure needs: - job: build_ci_image artifacts: false @@ -373,7 +372,7 @@ build_ci_image: interruptible: true tags: - docker-in-docker:amd64 - stage: e2e + stage: configure script: - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) - EXPECTED_IMAGE="registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" @@ -415,7 +414,7 @@ mirror_images: image: $CI_IMAGE # Runs in e2e (not the last stage) so the build/run jobs can depend on it and # only start once the mirror is populated. - stage: e2e + stage: configure tags: - arch:amd64 needs: From b30bf98cea74c7eaf33a007a9e60918c2b6418bf Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 25 Jun 2026 16:02:37 +0200 Subject: [PATCH 225/229] buildkitd auto gen --- .gitlab-ci.yml | 6 ++++++ utils/scripts/update_mirror_images.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a5620f0d7b..21b091583f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -441,6 +441,12 @@ mirror_images: echo " python utils/scripts/update_mirror_images.py" exit 1 fi + - | + if ! git diff --exit-code -- utils/build/docker/buildkitd.toml; then + echo "❌ utils/build/docker/buildkitd.toml is out of date. Regenerate and commit it:" + echo " python utils/scripts/update_mirror_images.py" + exit 1 + fi # Push any locked image missing from the mirror (already-present images are skipped). - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror rules: diff --git a/utils/scripts/update_mirror_images.py b/utils/scripts/update_mirror_images.py index 4562f06f13c..75ddd5968e5 100644 --- a/utils/scripts/update_mirror_images.py +++ b/utils/scripts/update_mirror_images.py @@ -42,6 +42,7 @@ DEFAULT_DEST_REGISTRY = "registry.ddbuild.io/system-tests/mirror" MIRROR_YAML = REPO_ROOT / "mirror_images.yaml" +BUILDKITD_TOML = REPO_ROOT / "utils" / "build" / "docker" / "buildkitd.toml" # Header written when mirror_images.yaml does not exist yet. The mirror_images.py # `add` command preserves existing comments, so this is only used on first run. @@ -135,6 +136,7 @@ def main(excluded: set[str], *, skip_lock: bool) -> None: _run_mirror_images("add", *images) if not skip_lock: _run_mirror_images("lock") + _run_mirror_images("buildkitd", "--output", str(BUILDKITD_TOML)) if __name__ == "__main__": From d100a9450a733bb4645488435782a45708fe1309 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 25 Jun 2026 16:02:48 +0200 Subject: [PATCH 226/229] Mirror update --- mirror_images.lock.yaml | 32 ++++++++++++++++++++++++++++++++ mirror_images.yaml | 8 ++++++++ 2 files changed, 40 insertions(+) diff --git a/mirror_images.lock.yaml b/mirror_images.lock.yaml index 1c53d5e9ce3..d27b2367b6d 100644 --- a/mirror_images.lock.yaml +++ b/mirror_images.lock.yaml @@ -94,14 +94,26 @@ images: digest: sha256:be33d685d24e57f478ece1258c722822c8fd496aef29abe409e1f9bbd63bb8a3 target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4-typescript.base-v1 tag: express4-typescript.base-v1 + datadog/system-tests:express4-typescript.base-v2: + digest: sha256:0b1ce417ef02e50c4e215deb21504ef476bca1e597034c56c91e0341744991cb + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4-typescript.base-v2 + tag: express4-typescript.base-v2 datadog/system-tests:express4.base-v1: digest: sha256:a4cf4545f6cf3e47dfdda1b9f33a98bd74ebdd7eb527ee53d6e513f93348290b target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4.base-v1 tag: express4.base-v1 + datadog/system-tests:express4.base-v2: + digest: sha256:8c64a9c7ff8b394d4be359ce0b08113f1cc246fbb1d8213f1ce9e0ceb3cf45c3 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4.base-v2 + tag: express4.base-v2 datadog/system-tests:express5.base-v1: digest: sha256:826625f5efc737fdee8d52e68fae6f5d3c8c70f18ed22e232a1762b5e74fa09f target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express5.base-v1 tag: express5.base-v1 + datadog/system-tests:express5.base-v2: + digest: sha256:25f04a53e2ceac5f9bb2363e6b28303b7a5c24823cb115c80c91e16706a18ba0 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express5.base-v2 + tag: express5.base-v2 datadog/system-tests:fastapi.base-v9: digest: sha256:c9cede8431eb9dd23e785c5740acf0f0a28c44cf0684e712b6ec0b87ed9e8e0f target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastapi.base-v9 @@ -110,6 +122,10 @@ images: digest: sha256:6eeb9581247f14c6f033052779a8e3f7d56f82393d8a288434f4537789510e6f target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastify.base-v1 tag: fastify.base-v1 + datadog/system-tests:fastify.base-v2: + digest: sha256:37820a425c93afba518339e9de411c246d785b6fde80d7c683c0ba347ee51813 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastify.base-v2 + tag: fastify.base-v2 datadog/system-tests:flask-poc.base-v1: digest: sha256:c2985235013e3c83d498f55f4d8b39fc2121a1acc09c2b740ee07e6bdae6852e target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:flask-poc.base-v1 @@ -118,6 +134,10 @@ images: digest: sha256:80a66d234e1545219b0418bd44d0ab30b7ed7ca8c81bbd6e488c8fbcd5902d44 target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:flask-poc.base-v13 tag: flask-poc.base-v13 + datadog/system-tests:flask-poc.base-v14: + digest: sha256:d46c08ea596e8cd5576fc6cacf3c3dbdc77a5f8d9b3ecb92673155d15728554d + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:flask-poc.base-v14 + tag: flask-poc.base-v14 datadog/system-tests:golang_buddy-v2: digest: sha256:b567fa2e328f24c74aab600e8bf07362b4265ac508413a83e06afddb399bee80 target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:golang_buddy-v2 @@ -134,6 +154,10 @@ images: digest: sha256:8500fa38a95c7dc7d97fd270f56331048d98c4e4181df079e32eadd03e9554e0 target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:nextjs.base-v1 tag: nextjs.base-v1 + datadog/system-tests:nextjs.base-v2: + digest: sha256:507c7a63f3ec29a5ff7e6a65f23f4e29ccdc94806affc81a94d2a59ff9385e16 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:nextjs.base-v2 + tag: nextjs.base-v2 datadog/system-tests:nodejs_buddy-v1: digest: sha256:71d124f74d4d7ac706f1afa8d5950864e1e931ec3b511f540b49eb6328c0338e target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:nodejs_buddy-v1 @@ -194,6 +218,10 @@ images: digest: sha256:cdea08fb0841b1c0b8ce632490123fafff8a18e62be4c60fb4882e41d0c9a2f0 target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:tornado.base-v2 tag: tornado.base-v2 + datadog/system-tests:uwsgi-poc.base-v10: + digest: sha256:939106a94fa4a4d8a2a8038b7e25933bc00811dad2847441c07df0c228e81f39 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:uwsgi-poc.base-v10 + tag: uwsgi-poc.base-v10 datadog/system-tests:uwsgi-poc.base-v9: digest: sha256:7c8fe35e132b686b584c1f06bdc296502e0ae3c4cdac1d77e1f6fbc4226f8d03 target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:uwsgi-poc.base-v9 @@ -222,6 +250,10 @@ images: digest: sha256:cf8c7e7c0d6a53d69e23fdf0d7a6d900d7b6cfb104fae4756805016b2c8bc1c5 target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.59.0 tag: v1.59.0 + ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.62.0: + digest: sha256:3b2d2b1287d7e5443e89dbcad1946e7cdac804494e3ad55a87ed02b916750006 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.62.0 + tag: v1.62.0 ghcr.io/datadog/images-rb/engines/ruby:2.5: digest: sha256:80a31383585fff1a3c0d33c149171460bca182deddd50f2ed8f54217252f5178 target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:2.5 diff --git a/mirror_images.yaml b/mirror_images.yaml index b9f0a4dc483..e0831216b23 100644 --- a/mirror_images.yaml +++ b/mirror_images.yaml @@ -30,16 +30,22 @@ - "datadog/system-tests:django-poc.base-v11" - "datadog/system-tests:django-py3.13.base-v10" - "datadog/system-tests:express4-typescript.base-v1" +- "datadog/system-tests:express4-typescript.base-v2" - "datadog/system-tests:express4.base-v1" +- "datadog/system-tests:express4.base-v2" - "datadog/system-tests:express5.base-v1" +- "datadog/system-tests:express5.base-v2" - "datadog/system-tests:fastapi.base-v9" - "datadog/system-tests:fastify.base-v1" +- "datadog/system-tests:fastify.base-v2" - "datadog/system-tests:flask-poc.base-v1" - "datadog/system-tests:flask-poc.base-v13" +- "datadog/system-tests:flask-poc.base-v14" - "datadog/system-tests:golang_buddy-v2" - "datadog/system-tests:java_buddy-v1" - "datadog/system-tests:lambda-proxy-v1" - "datadog/system-tests:nextjs.base-v1" +- "datadog/system-tests:nextjs.base-v2" - "datadog/system-tests:nodejs_buddy-v1" - "datadog/system-tests:php-fpm-7.0.base-v1" - "datadog/system-tests:php-fpm-7.1.base-v1" @@ -55,6 +61,7 @@ - "datadog/system-tests:python_buddy-v2" - "datadog/system-tests:ruby_buddy-v2" - "datadog/system-tests:tornado.base-v2" +- "datadog/system-tests:uwsgi-poc.base-v10" - "datadog/system-tests:uwsgi-poc.base-v9" - "debian:bookworm-slim" - "debian:stable-slim" @@ -62,6 +69,7 @@ - "eclipse-temurin:11-jre" - "eclipse-temurin:17-jre" - "ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.59.0" +- "ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.62.0" - "ghcr.io/datadog/images-rb/engines/ruby:2.5" - "ghcr.io/datadog/images-rb/engines/ruby:2.7" - "ghcr.io/datadog/images-rb/engines/ruby:3.0" From 1d7463be346709c72e8e4523dc4ac4c3521c9327 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Thu, 25 Jun 2026 17:22:21 +0200 Subject: [PATCH 227/229] test --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21b091583f2..49ba4f78751 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -362,7 +362,7 @@ system-tests-param: fi done done - export LIBRARIES="$filtered" + export LIBRARIES="" fi - export EXCLUDED_SCENARIOS='DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E' - export SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION=true From d1548070ef6d7bab83cc9d7c18865adc4e5ec557 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 26 Jun 2026 18:18:22 +0200 Subject: [PATCH 228/229] ci: build_ci_image only on relevant file changes, add CI_IMAGE check to mirror_images --- .gitlab-ci.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 49ba4f78751..eec1d6ac9a8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -325,6 +325,7 @@ system-tests-param: needs: - job: build_ci_image artifacts: false + optional: true before_script: - ln -sf /system-tests/venv venv - source venv/bin/activate @@ -369,7 +370,7 @@ system-tests-param: build_ci_image: image: registry.ddbuild.io/images/docker:20.10.13-jammy - interruptible: true + interruptible: false tags: - docker-in-docker:amd64 stage: configure @@ -390,6 +391,10 @@ build_ci_image: docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile -t $EXPECTED_IMAGE . && docker push $EXPECTED_IMAGE; fi + rules: + - changes: + - utils/ci/gitlab/docker/system-tests.Dockerfile + - requirements.txt # ────────────────────────────────────────────── # Mirror CI scenario images into registry.ddbuild.io/system-tests/mirror @@ -420,6 +425,7 @@ mirror_images: needs: - job: build_ci_image artifacts: false + optional: true before_script: # crane handles multi-arch manifest lists cleanly; mirror_images.py picks it up on PATH. - CRANE_VERSION=v0.20.2 @@ -430,6 +436,15 @@ mirror_images: - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) - echo "$DOCKER_LOGIN_PASS" | crane auth login index.docker.io --username "$DOCKER_LOGIN" --password-stdin script: + - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) + - EXPECTED_IMAGE="registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" + - | + if [ "$EXPECTED_IMAGE" != "$CI_IMAGE" ]; then + echo "❌ CI_IMAGE mismatch: hardcoded '$CI_IMAGE' does not match computed '$EXPECTED_IMAGE'" + echo " Update CI_IMAGE in utils/ci/gitlab/main.yml:" + echo " cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12" + exit 1 + fi - ln -sf /system-tests/venv venv - source venv/bin/activate - python -m pip install --quiet uv From da4683f13c3f2396e12935ec5dd17f9b7b0e1064 Mon Sep 17 00:00:00 2001 From: Nicolas Catoni Date: Fri, 26 Jun 2026 19:45:21 +0200 Subject: [PATCH 229/229] Job name prefix --- .gitlab-ci.yml | 4 ++-- utils/ci/gitlab/main.yml | 20 ++++++++++---------- utils/ci/gitlab/system-tests.yml.j2 | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eec1d6ac9a8..3d9a3c57396 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -319,7 +319,7 @@ generate_system_tests_lib_injection_images: # End-to-end pipeline — only for the system-tests repo # ────────────────────────────────────────────── -system-tests-param: +system_tests_param: extends: .system_tests_param_base stage: configure needs: @@ -363,7 +363,7 @@ system-tests-param: fi done done - export LIBRARIES="" + export LIBRARIES="$filtered" fi - export EXCLUDED_SCENARIOS='DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E' - export SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION=true diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml index d6fea444154..cd37696a8e3 100644 --- a/utils/ci/gitlab/main.yml +++ b/utils/ci/gitlab/main.yml @@ -88,7 +88,7 @@ variables: reports: dotenv: param.env -build_test_pipeline: +system_tests_build_pipeline: extends: .system_tests_base tags: - arch:amd64 @@ -99,7 +99,7 @@ build_test_pipeline: - job: build_ci_image optional: true artifacts: false - - job: system-tests-param + - job: system_tests_param optional: true artifacts: true script: @@ -146,10 +146,10 @@ build_test_pipeline: - if: $[[ inputs.condition ]] - when: never needs: - - job: build_test_pipeline + - job: system_tests_build_pipeline optional: true artifacts: true - - job: system-tests-param + - job: system_tests_param optional: true artifacts: true - job: mirror_images @@ -161,15 +161,15 @@ build_test_pipeline: LIBRARIES: $LIBRARIES UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID -run_test_pipeline_0: +system_tests_run_pipeline_0: extends: .run_test_pipeline_base trigger: include: - artifact: system-tests/generated-pipeline-chunk-0.yml - job: build_test_pipeline + job: system_tests_build_pipeline strategy: depend -run_test_pipeline_1: +system_tests_run_pipeline_1: extends: .run_test_pipeline_base rules: - if: $SYSTEM_TESTS_SPLIT_PIPELINE != "true" @@ -179,10 +179,10 @@ run_test_pipeline_1: trigger: include: - artifact: system-tests/generated-pipeline-chunk-1.yml - job: build_test_pipeline + job: system_tests_build_pipeline strategy: depend -run_test_pipeline_2: +system_tests_run_pipeline_2: extends: .run_test_pipeline_base rules: - if: $SYSTEM_TESTS_SPLIT_PIPELINE != "true" @@ -192,5 +192,5 @@ run_test_pipeline_2: trigger: include: - artifact: system-tests/generated-pipeline-chunk-2.yml - job: build_test_pipeline + job: system_tests_build_pipeline strategy: depend diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 index cefac1d8be5..dc7e9b04920 100644 --- a/utils/ci/gitlab/system-tests.yml.j2 +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -60,7 +60,7 @@ setup: {% endif %} {% for variant in weblog_variants %} -build_{{library}}_{{variant}}: +system_tests_build_{{library}}_{{variant}}: extends: .system_tests_base {% if binaries_artifacts_list %} needs: @@ -87,7 +87,7 @@ build_{{library}}_{{variant}}: {% endfor %} {% for variant, scenario, build_required in scenario_pairs %} -run_{{library}}_{{scenario}}_{{variant}}: +system_tests_run_{{library}}_{{scenario}}_{{variant}}: extends: - .system_tests_base {% if push_to_test_optimization %} @@ -95,7 +95,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% endif %} needs: {% if build_required %} - - job: build_{{library}}_{{variant}} + - job: system_tests_build_{{library}}_{{variant}} artifacts: true {% elif binaries_artifacts_list %} {% for artifact_job in binaries_artifacts_list %} @@ -133,7 +133,7 @@ run_{{library}}_{{scenario}}_{{variant}}: {% endfor %} {% if parametric.enable %} {% for job_index in parametric.job_matrix %} -run_{{library}}_PARAMETRIC_{{job_index}}: +system_tests_run_{{library}}_PARAMETRIC_{{job_index}}: extends: - .system_tests_base {% if push_to_test_optimization %}