diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1d90e3a6d48..3d9a3c57396 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,11 @@ include: - remote: https://gitlab-templates.ddbuild.io/libdatadog/one-pipeline/ca/0b4d88122bb64a87131eeb9446a3671598790917ec30fc31851ccb6ee8375b8a/single-step-instrumentation-tests.yml + - local: /utils/ci/gitlab/main.yml + inputs: + stage: "configure" + ref: "$CI_COMMIT_SHA" + push_to_test_optimization: true + split_pipeline: true stages: - configure - nodejs @@ -13,6 +19,12 @@ stages: variables: TEST: 1 + # TEMPORARY: rollout-control allowlist for system-tests-param LIBRARIES filter. + E2E_SUPPORTED_LANGUAGES: "python" + # dd-repo-tools mirror_images.py (pinned). Used by the mirror_images_* jobs. + MIRROR_IMAGES_URL: "https://binaries.ddbuild.io/dd-repo-tools/default/ca/d6c8f8ec4b1fb69b36e96dc204e5f79bed6bf067/mirror_images.py" + # Destination registry for mirrored CI images. + MIRROR_DEST_REGISTRY: "registry.ddbuild.io/system-tests/mirror" compute_pipeline: image: registry.ddbuild.io/ci/libdatadog-build/system-tests:100425777 @@ -251,8 +263,6 @@ check_merge_labels: - dd-octo-sts version - dd-octo-sts debug --scope DataDog/system-tests --policy self.gitlab-read - dd-octo-sts token --scope DataDog/system-tests --policy self.gitlab-read > token.txt - - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) - - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) script: - docker buildx version - export GITHUB_TOKEN=$(cat token.txt) @@ -304,3 +314,155 @@ generate_system_tests_lib_injection_images: - echo "⏳ Waiting before triggering the child pipeline..." when: delayed start_in: 5 minutes + +# ────────────────────────────────────────────── +# End-to-end pipeline — only for the system-tests repo +# ────────────────────────────────────────────── + +system_tests_param: + extends: .system_tests_param_base + stage: configure + needs: + - job: build_ci_image + artifacts: false + optional: true + before_script: + - ln -sf /system-tests/venv venv + - source venv/bin/activate + - export PYTHONPATH=$(pwd) + - git fetch origin main + - mkdir -p original && git archive origin/main -- manifests/ | tar -xC original/ + - git diff origin/main...HEAD --name-only > modified_files.txt + - ./run.sh MOCK_THE_TEST --collect-only --scenario-report + - python3 utils/scripts/compute_libraries_and_scenarios.py --output raw_params.txt + - | + python3 -c " + import json, shlex + with open('raw_params.txt') as f: + for line in f: + line = line.strip() + if '=' not in line: continue + key, val = line.split('=', 1) + val = json.loads(val) + if key == 'libraries': print('export LIBRARIES=' + shlex.quote(str(val))) + elif key == 'scenarios': print('export SCENARIOS=' + shlex.quote(str(val))) + elif key == 'scenarios_groups': print('export SCENARIO_GROUPS=' + shlex.quote(str(val))) + " > set_vars.sh + source set_vars.sh + # TEMPORARY: filter LIBRARIES via E2E_SUPPORTED_LANGUAGES (space-separated allowlist) + # used to control rollout, to be removed once rollout is complete. + - | + if [ -n "${E2E_SUPPORTED_LANGUAGES:-}" ]; then + echo "Filtering LIBRARIES with E2E_SUPPORTED_LANGUAGES='$E2E_SUPPORTED_LANGUAGES'" + filtered="" + for lib in $LIBRARIES; do + for allowed in $E2E_SUPPORTED_LANGUAGES; do + if [ "$lib" = "$allowed" ]; then + filtered="${filtered:+$filtered }$lib" + break + fi + done + done + export LIBRARIES="$filtered" + fi + - export EXCLUDED_SCENARIOS='DEBUGGER_EXPRESSION_LANGUAGE,APM_TRACING_E2E_SINGLE_SPAN,APM_TRACING_E2E_OTEL,OTEL_COLLECTOR_E2E' + - export SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION=true + +build_ci_image: + image: registry.ddbuild.io/images/docker:20.10.13-jammy + interruptible: false + tags: + - docker-in-docker:amd64 + stage: configure + script: + - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) + - EXPECTED_IMAGE="registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" + - | + if [ "$EXPECTED_IMAGE" != "$CI_IMAGE" ]; then + echo "❌ CI_IMAGE mismatch: hardcoded '$CI_IMAGE' does not match computed '$EXPECTED_IMAGE'" + echo " Update CI_IMAGE in utils/ci/gitlab/main.yml:" + echo " cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12" + exit 1 + fi + - > + if docker manifest inspect $EXPECTED_IMAGE > /dev/null 2>&1; then + echo "Image already exists, skipping build"; + else + docker build -f utils/ci/gitlab/docker/system-tests.Dockerfile -t $EXPECTED_IMAGE . && + docker push $EXPECTED_IMAGE; + fi + rules: + - changes: + - utils/ci/gitlab/docker/system-tests.Dockerfile + - requirements.txt + +# ────────────────────────────────────────────── +# Mirror CI scenario images into registry.ddbuild.io/system-tests/mirror +# +# A single job that, on every push: +# * regenerates the image list from the scenarios and fails if the committed +# mirror_images.yaml is out of date (regenerate + commit it with +# utils/scripts/update_mirror_images.py whenever scenarios or weblog +# Dockerfiles change); +# * pushes any image from the committed mirror_images.lock.yaml that is not yet +# present in MIRROR_DEST_REGISTRY. `mirror` checks the destination first and +# only copies what is missing, so re-running it is cheap and idempotent. +# +# It never updates the lock file: mirror_images.lock.yaml is the committed +# contract, regenerated by developers via update_mirror_images.py. +# +# crane copies registry-to-registry over HTTPS (no Docker daemon needed), so this +# runs on the framework runner image used for the scenario enumeration. +# ────────────────────────────────────────────── + +mirror_images: + image: $CI_IMAGE + # Runs in e2e (not the last stage) so the build/run jobs can depend on it and + # only start once the mirror is populated. + stage: configure + tags: + - arch:amd64 + needs: + - job: build_ci_image + artifacts: false + optional: true + before_script: + # crane handles multi-arch manifest lists cleanly; mirror_images.py picks it up on PATH. + - CRANE_VERSION=v0.20.2 + - curl -sSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" | tar -xz -C /usr/local/bin crane + # This is the only job that pulls from Docker Hub (the mirror sources), so it + # is the only one that needs Docker Hub auth (avoids unauthenticated rate limits). + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - echo "$DOCKER_LOGIN_PASS" | crane auth login index.docker.io --username "$DOCKER_LOGIN" --password-stdin + script: + - IMAGE_TAG=$(cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12) + - EXPECTED_IMAGE="registry.ddbuild.io/system-tests/ci-runner:$IMAGE_TAG" + - | + if [ "$EXPECTED_IMAGE" != "$CI_IMAGE" ]; then + echo "❌ CI_IMAGE mismatch: hardcoded '$CI_IMAGE' does not match computed '$EXPECTED_IMAGE'" + echo " Update CI_IMAGE in utils/ci/gitlab/main.yml:" + echo " cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12" + exit 1 + fi + - ln -sf /system-tests/venv venv + - source venv/bin/activate + - python -m pip install --quiet uv + # Fail if the committed image list is stale vs the scenarios (does not touch the lock). + - python utils/scripts/update_mirror_images.py --skip-lock + - | + if ! git diff --exit-code -- mirror_images.yaml; then + echo "❌ mirror_images.yaml is out of date. Regenerate and commit it:" + echo " python utils/scripts/update_mirror_images.py" + exit 1 + fi + - | + if ! git diff --exit-code -- utils/build/docker/buildkitd.toml; then + echo "❌ utils/build/docker/buildkitd.toml is out of date. Regenerate and commit it:" + echo " python utils/scripts/update_mirror_images.py" + exit 1 + fi + # Push any locked image missing from the mirror (already-present images are skipped). + - uv run --no-config --script "$MIRROR_IMAGES_URL" mirror + rules: + - if: '$SCHEDULED_JOB == "" || $SCHEDULED_JOB == null' diff --git a/.yamlfmt b/.yamlfmt index 24441a65f0a..3ce4697bfce 100644 --- a/.yamlfmt +++ b/.yamlfmt @@ -1,6 +1,7 @@ formatter: type: basic include_document_start: true + retain_line_breaks_single: true indent: 2 pad_line_comments: 2 eof_newline: true diff --git a/.yamllint b/.yamllint index 8b27b985f77..0230c802427 100644 --- a/.yamllint +++ b/.yamllint @@ -3,3 +3,7 @@ extends: relaxed rules: line-length: disable key-ordering: disable + indentation: + # .gitlab-ci.yml uses intentional mixed indentation (GitLab CI style). + ignore: | + .gitlab-ci.yml diff --git a/format.sh b/format.sh index af0d1ba25ee..a548303ee1e 100755 --- a/format.sh +++ b/format.sh @@ -152,7 +152,7 @@ else fi echo "Running yamllint checks..." -if ! ./venv/bin/yamllint -s manifests/; then +if ! ./venv/bin/yamllint -s manifests/ utils/ci/gitlab/ .gitlab-ci.yml; then echo "yamllint checks failed. Please fix the errors above. 💥 💔 💥" exit 1 fi diff --git a/mirror_images.lock.yaml b/mirror_images.lock.yaml new file mode 100644 index 00000000000..d27b2367b6d --- /dev/null +++ b/mirror_images.lock.yaml @@ -0,0 +1,408 @@ +# Auto-generated by: uv run bin/mirror_images.py lock +# Do not edit manually. +version: 1 +images: + apache/kafka:3.7.1: + digest: sha256:ed74d7d115968d5e8b00ba6822ac6a384cbaaf54ca38991828647000d7089b68 + target: registry.ddbuild.io/system-tests/mirror/apache/kafka:3.7.1 + tag: 3.7.1 + cassandra:latest: + digest: sha256:755db0086160b6898e890959343dec1544ff27e7fa8ecc955c2f4b28c40a2f1a + target: registry.ddbuild.io/system-tests/mirror/cassandra:latest + tag: 5.0.8 + datadog/agent:latest: + digest: sha256:2a798352bb00af4f64908809af46bbd211e1f087e6509e8ca0722fba45385c01 + target: registry.ddbuild.io/system-tests/mirror/datadog/agent:latest + tag: 7.80.1 + datadog/dd-appsec-php-ci:php-8.0-release: + digest: sha256:f6c7557c63d9da34d081bc681768798bc4f70d8be36c565c526e57cecd6c2249 + target: registry.ddbuild.io/system-tests/mirror/datadog/dd-appsec-php-ci:php-8.0-release + tag: php-8.0-release + datadog/system-tests:apache-mod-7.0-zts.base-v1: + digest: sha256:bafc072734341cafb86f45673df2de4851a04b951d0717c88f7958cac1815b20 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.0-zts.base-v1 + tag: apache-mod-7.0-zts.base-v1 + datadog/system-tests:apache-mod-7.0.base-v1: + digest: sha256:af5081af161f4e12645ccdc4717ffc79dbb010f46c8978a3c12331d504fd74cd + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.0.base-v1 + tag: apache-mod-7.0.base-v1 + datadog/system-tests:apache-mod-7.1-zts.base-v1: + digest: sha256:5aa5c49cd2a90fdd1b5e9291d7825de083d7f258bab9dbd0ebc2770cd311729b + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.1-zts.base-v1 + tag: apache-mod-7.1-zts.base-v1 + datadog/system-tests:apache-mod-7.1.base-v1: + digest: sha256:e3c1334ac870ceef365f6f6d870c91306aeaab27b7b164e98e9f685f96cdcaed + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.1.base-v1 + tag: apache-mod-7.1.base-v1 + datadog/system-tests:apache-mod-7.2-zts.base-v1: + digest: sha256:965f5684c0fbb633660d672603e2dfa7c1d9d11e44cd3a0a135240be4733d8d4 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.2-zts.base-v1 + tag: apache-mod-7.2-zts.base-v1 + datadog/system-tests:apache-mod-7.2.base-v1: + digest: sha256:f237f0269089c99af3bc0f125439f08a555925d78e1d03e96baa9d82e8308e54 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.2.base-v1 + tag: apache-mod-7.2.base-v1 + datadog/system-tests:apache-mod-7.3-zts.base-v1: + digest: sha256:cbb5d94151a26d44f9b0b7de3790161c0d39e7c0324fec6c2217495214166d2f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.3-zts.base-v1 + tag: apache-mod-7.3-zts.base-v1 + datadog/system-tests:apache-mod-7.3.base-v1: + digest: sha256:6d6cd038caa41a456a5918f258658a4d940f2473891fc17f478bc250a43b613b + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.3.base-v1 + tag: apache-mod-7.3.base-v1 + datadog/system-tests:apache-mod-7.4-zts.base-v1: + digest: sha256:0568dcc42a70b6f2aa142d32d8a3e5c0362c7f76149dfcdeaac1514c77fc4ce6 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.4-zts.base-v1 + tag: apache-mod-7.4-zts.base-v1 + datadog/system-tests:apache-mod-7.4.base-v1: + digest: sha256:007b8604e30cc4568e704b51d54e85a9e873bb17eac105b1abf690cab5888db5 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-7.4.base-v1 + tag: apache-mod-7.4.base-v1 + datadog/system-tests:apache-mod-8.0-zts.base-v1: + digest: sha256:08b761c6c942ab12731383fe84865efd2d15422dd8a23fe4d96d7454ea79e580 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.0-zts.base-v1 + tag: apache-mod-8.0-zts.base-v1 + datadog/system-tests:apache-mod-8.0.base-v1: + digest: sha256:8faf421f570e77665eaeccaac581f03e3144fbd4689e59aeca3bbeaad31760e3 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.0.base-v1 + tag: apache-mod-8.0.base-v1 + datadog/system-tests:apache-mod-8.1-zts.base-v1: + digest: sha256:13fed1524e18c20e0dac2546b44fd024bf9d20e1d8521a62d21fe9dc27877107 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.1-zts.base-v1 + tag: apache-mod-8.1-zts.base-v1 + datadog/system-tests:apache-mod-8.1.base-v1: + digest: sha256:79de2dcdc70282f2ec0641f8633a183985ab40bf9b6f0a8d6a51084527747e39 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.1.base-v1 + tag: apache-mod-8.1.base-v1 + datadog/system-tests:apache-mod-8.2-zts.base-v1: + digest: sha256:1d2d543af606021af9e80e6627de41928739a4a27346b60e0b806f31a9f694f5 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.2-zts.base-v1 + tag: apache-mod-8.2-zts.base-v1 + datadog/system-tests:apache-mod-8.2.base-v1: + digest: sha256:875f87c0c31bd65316c79e2904da03fb8d0089403fc8631f9244217ad02c8bc2 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:apache-mod-8.2.base-v1 + tag: apache-mod-8.2.base-v1 + datadog/system-tests:django-poc.base-v11: + digest: sha256:bc6089dc235abdb972965d004dc409c78636ba0b1e0f30b485057e10f2628e3b + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:django-poc.base-v11 + tag: django-poc.base-v11 + datadog/system-tests:django-py3.13.base-v10: + digest: sha256:3249e0d5df7b8c88f95a4dec4aee196ade0ac01b3f263a7162993c9a00ec892c + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:django-py3.13.base-v10 + tag: django-py3.13.base-v10 + datadog/system-tests:express4-typescript.base-v1: + digest: sha256:be33d685d24e57f478ece1258c722822c8fd496aef29abe409e1f9bbd63bb8a3 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4-typescript.base-v1 + tag: express4-typescript.base-v1 + datadog/system-tests:express4-typescript.base-v2: + digest: sha256:0b1ce417ef02e50c4e215deb21504ef476bca1e597034c56c91e0341744991cb + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4-typescript.base-v2 + tag: express4-typescript.base-v2 + datadog/system-tests:express4.base-v1: + digest: sha256:a4cf4545f6cf3e47dfdda1b9f33a98bd74ebdd7eb527ee53d6e513f93348290b + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4.base-v1 + tag: express4.base-v1 + datadog/system-tests:express4.base-v2: + digest: sha256:8c64a9c7ff8b394d4be359ce0b08113f1cc246fbb1d8213f1ce9e0ceb3cf45c3 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express4.base-v2 + tag: express4.base-v2 + datadog/system-tests:express5.base-v1: + digest: sha256:826625f5efc737fdee8d52e68fae6f5d3c8c70f18ed22e232a1762b5e74fa09f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express5.base-v1 + tag: express5.base-v1 + datadog/system-tests:express5.base-v2: + digest: sha256:25f04a53e2ceac5f9bb2363e6b28303b7a5c24823cb115c80c91e16706a18ba0 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:express5.base-v2 + tag: express5.base-v2 + datadog/system-tests:fastapi.base-v9: + digest: sha256:c9cede8431eb9dd23e785c5740acf0f0a28c44cf0684e712b6ec0b87ed9e8e0f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastapi.base-v9 + tag: fastapi.base-v9 + datadog/system-tests:fastify.base-v1: + digest: sha256:6eeb9581247f14c6f033052779a8e3f7d56f82393d8a288434f4537789510e6f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastify.base-v1 + tag: fastify.base-v1 + datadog/system-tests:fastify.base-v2: + digest: sha256:37820a425c93afba518339e9de411c246d785b6fde80d7c683c0ba347ee51813 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:fastify.base-v2 + tag: fastify.base-v2 + datadog/system-tests:flask-poc.base-v1: + digest: sha256:c2985235013e3c83d498f55f4d8b39fc2121a1acc09c2b740ee07e6bdae6852e + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:flask-poc.base-v1 + tag: flask-poc.base-v1 + datadog/system-tests:flask-poc.base-v13: + digest: sha256:80a66d234e1545219b0418bd44d0ab30b7ed7ca8c81bbd6e488c8fbcd5902d44 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:flask-poc.base-v13 + tag: flask-poc.base-v13 + datadog/system-tests:flask-poc.base-v14: + digest: sha256:d46c08ea596e8cd5576fc6cacf3c3dbdc77a5f8d9b3ecb92673155d15728554d + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:flask-poc.base-v14 + tag: flask-poc.base-v14 + datadog/system-tests:golang_buddy-v2: + digest: sha256:b567fa2e328f24c74aab600e8bf07362b4265ac508413a83e06afddb399bee80 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:golang_buddy-v2 + tag: golang_buddy-v2 + datadog/system-tests:java_buddy-v1: + digest: sha256:67aac2b7c0c88b40da498fbc74b48f0a62702148e5c9256c5752a9e624d972ce + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:java_buddy-v1 + tag: java_buddy-v1 + datadog/system-tests:lambda-proxy-v1: + digest: sha256:2b09f33c61b2b19f4a239e4d8f10164ece66d925b72337720998ce43d7090d2f + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:lambda-proxy-v1 + tag: lambda-proxy-v1 + datadog/system-tests:nextjs.base-v1: + digest: sha256:8500fa38a95c7dc7d97fd270f56331048d98c4e4181df079e32eadd03e9554e0 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:nextjs.base-v1 + tag: nextjs.base-v1 + datadog/system-tests:nextjs.base-v2: + digest: sha256:507c7a63f3ec29a5ff7e6a65f23f4e29ccdc94806affc81a94d2a59ff9385e16 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:nextjs.base-v2 + tag: nextjs.base-v2 + datadog/system-tests:nodejs_buddy-v1: + digest: sha256:71d124f74d4d7ac706f1afa8d5950864e1e931ec3b511f540b49eb6328c0338e + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:nodejs_buddy-v1 + tag: nodejs_buddy-v1 + datadog/system-tests:php-fpm-7.0.base-v1: + digest: sha256:ca3e2cd0e0e1c39c5008213bac2f4130b43499eb5f6392173c02791e1cbbba95 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.0.base-v1 + tag: php-fpm-7.0.base-v1 + datadog/system-tests:php-fpm-7.1.base-v1: + digest: sha256:8efdc58bed6abc510b5b86c92f01fb1c361e9c2a0ceaa11de518ac4cbcbc9b49 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.1.base-v1 + tag: php-fpm-7.1.base-v1 + datadog/system-tests:php-fpm-7.2.base-v1: + digest: sha256:b0d8cdd1d6fcddcb6e67b5006f08b9c176196ad02ecebab97163b26b313d75ec + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.2.base-v1 + tag: php-fpm-7.2.base-v1 + datadog/system-tests:php-fpm-7.3.base-v1: + digest: sha256:a2f396b7f1a4a74212c899c5782a5cbef48e3d581cccaede210b4088b1db2e29 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.3.base-v1 + tag: php-fpm-7.3.base-v1 + datadog/system-tests:php-fpm-7.4.base-v1: + digest: sha256:65b4f41685a53e04ea090b5a5d6d36ed439f62806340dd2fd402d932556c0a00 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-7.4.base-v1 + tag: php-fpm-7.4.base-v1 + datadog/system-tests:php-fpm-8.0.base-v1: + digest: sha256:71cc5b9746c6271641ee7f56408476bb5457277e5d24d4c646f6b41807f5eca6 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-8.0.base-v1 + tag: php-fpm-8.0.base-v1 + datadog/system-tests:php-fpm-8.1.base-v1: + digest: sha256:abc6845df939364b5179c7c513366462426a75e30e7a69fe4d91ce860e7e2411 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-8.1.base-v1 + tag: php-fpm-8.1.base-v1 + datadog/system-tests:php-fpm-8.2.base-v1: + digest: sha256:c716acd35cf91cb8b1c646047372c85868657d29576aaa1fdfb43c754a4bea84 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-8.2.base-v1 + tag: php-fpm-8.2.base-v1 + datadog/system-tests:php-fpm-8.5.base-v1: + digest: sha256:d5ffab83a326dcd2f2d106949a3ff63a290fe396e3753cad221a5cb5d3ea4fc7 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:php-fpm-8.5.base-v1 + tag: php-fpm-8.5.base-v1 + datadog/system-tests:proxy-v1: + digest: sha256:02ce356eafacfe04d3022ff505e2dbb9e47cd3c55244b7918ff6fc824db85f02 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:proxy-v1 + tag: proxy-v1 + datadog/system-tests:python3.12.base-v13: + digest: sha256:89bfdf492c920e5653ca8099301d6710d0bbdf7dc54129d788cbfad5aafa46da + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:python3.12.base-v13 + tag: python3.12.base-v13 + datadog/system-tests:python_buddy-v2: + digest: sha256:abbd02d4495d83c66af12b5562ccca0077800cb920142375cd48dcfc8aeb1596 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:python_buddy-v2 + tag: python_buddy-v2 + datadog/system-tests:ruby_buddy-v2: + digest: sha256:939d3828d85666a598c0349a6dd933fe1d92c5daa5d01438512d950ea3ccc258 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:ruby_buddy-v2 + tag: ruby_buddy-v2 + datadog/system-tests:tornado.base-v2: + digest: sha256:cdea08fb0841b1c0b8ce632490123fafff8a18e62be4c60fb4882e41d0c9a2f0 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:tornado.base-v2 + tag: tornado.base-v2 + datadog/system-tests:uwsgi-poc.base-v10: + digest: sha256:939106a94fa4a4d8a2a8038b7e25933bc00811dad2847441c07df0c228e81f39 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:uwsgi-poc.base-v10 + tag: uwsgi-poc.base-v10 + datadog/system-tests:uwsgi-poc.base-v9: + digest: sha256:7c8fe35e132b686b584c1f06bdc296502e0ae3c4cdac1d77e1f6fbc4226f8d03 + target: registry.ddbuild.io/system-tests/mirror/datadog/system-tests:uwsgi-poc.base-v9 + tag: uwsgi-poc.base-v9 + debian:bookworm-slim: + digest: sha256:96e378d7e6531ac9a15ad505478fcc2e69f371b10f5cdf87857c4b8188404716 + target: registry.ddbuild.io/system-tests/mirror/debian:bookworm-slim + tag: bookworm-slim + debian:stable-slim: + digest: sha256:34363c20bd149e41365fc77b086da067ed13ab2dff4cd0612788e12e6d52c44c + target: registry.ddbuild.io/system-tests/mirror/debian:stable-slim + tag: stable-slim + demisto/fastapi:0.116.1.4266494: + digest: sha256:057fb83c60e7685a35d99229ece21c4cdcc962adfdc50251bf829b5694a95722 + target: registry.ddbuild.io/system-tests/mirror/demisto/fastapi:0.116.1.4266494 + tag: 0.116.1.4266494 + eclipse-temurin:11-jre: + digest: sha256:c356cb1702bbc9f614d52f953a1853fc7f0b49e3e62ab65927108e5bb17d9292 + target: registry.ddbuild.io/system-tests/mirror/eclipse-temurin:11-jre + tag: 11-jre + eclipse-temurin:17-jre: + digest: sha256:8016378253033ebd971c237f6580f108b4afac11a188d3fc3b982497bbe55c65 + target: registry.ddbuild.io/system-tests/mirror/eclipse-temurin:17-jre + tag: 17-jre + ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.59.0: + digest: sha256:cf8c7e7c0d6a53d69e23fdf0d7a6d900d7b6cfb104fae4756805016b2c8bc1c5 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.59.0 + tag: v1.59.0 + ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.62.0: + digest: sha256:3b2d2b1287d7e5443e89dbcad1946e7cdac804494e3ad55a87ed02b916750006 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.62.0 + tag: v1.62.0 + ghcr.io/datadog/images-rb/engines/ruby:2.5: + digest: sha256:80a31383585fff1a3c0d33c149171460bca182deddd50f2ed8f54217252f5178 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:2.5 + tag: '2.5' + ghcr.io/datadog/images-rb/engines/ruby:2.7: + digest: sha256:7c0f84d46c8fd5982abd50243c6b6fa53ce86a8978d4b1ae63b537698acf3755 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:2.7 + tag: '2.7' + ghcr.io/datadog/images-rb/engines/ruby:3.0: + digest: sha256:69d8af29e1105110d553e0597a8dee886ce2b1a7a43dedee622434365eb711db + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:3.0 + tag: '3.0' + ghcr.io/datadog/images-rb/engines/ruby:3.1: + digest: sha256:87f74691edd471099442996bfeab1d70804a3b74ea9a6e2ccea9f7500aff92ec + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:3.1 + tag: '3.1' + ghcr.io/datadog/images-rb/engines/ruby:3.4: + digest: sha256:2d22ecb3e636e12c397c872ff147cbd7ce67a8ab70a39caa1a2b520812ccccd2 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/datadog/images-rb/engines/ruby:3.4 + tag: '3.4' + ghcr.io/graalvm/native-image-community:22.0.0: + digest: sha256:0e7b3ee194aadc80845a29cd4e6501b9ed4422e6b998267926df31d63caf4770 + target: registry.ddbuild.io/system-tests/mirror/ghcr.io/graalvm/native-image-community:22.0.0 + tag: 22.0.0 + golang:1.25: + digest: sha256:00feed335fe561979f2cdcc30a5191231977c5631fe79c40f7d3ab63b4fa222f + target: registry.ddbuild.io/system-tests/mirror/golang:1.25 + tag: '1.25' + golang:1.25-alpine: + digest: sha256:89f71d90dff0d7f30316963b3c3b8bfe5fb96b94641b3258963ce0c7a21dedda + target: registry.ddbuild.io/system-tests/mirror/golang:1.25-alpine + tag: 1.25-alpine + kong/kong-gateway:3.4: + digest: sha256:195b1dedb78d9fa4976d089af9cc2573f6bb667b2b4be40e50d9e8cef85b4b58 + target: registry.ddbuild.io/system-tests/mirror/kong/kong-gateway:3.4 + tag: '3.4' + localstack/localstack:4.1: + digest: sha256:97ccc65daec3542bd2cb3160d7355f11e89ad8027fd3c834b9d1197d197d866f + target: registry.ddbuild.io/system-tests/mirror/localstack/localstack:4.1 + tag: '4.1' + maven:3.6-jdk-11: + digest: sha256:1d29ccf46ef2a5e64f7de3d79a63f9bcffb4dc56be0ae3daed5ca5542b38aa2d + target: registry.ddbuild.io/system-tests/mirror/maven:3.6-jdk-11 + tag: 3.6-jdk-11 + maven:3.8-jdk-11: + digest: sha256:805f366910aea2a91ed263654d23df58bd239f218b2f9562ff51305be81fa215 + target: registry.ddbuild.io/system-tests/mirror/maven:3.8-jdk-11 + tag: 3.8-jdk-11 + maven:3.9-eclipse-temurin-11: + digest: sha256:f580cae20128b75c984853630c054baf486a8c2f345a7d47692ca8b3b5b775d9 + target: registry.ddbuild.io/system-tests/mirror/maven:3.9-eclipse-temurin-11 + tag: 3.9-eclipse-temurin-11 + maven:3.9-eclipse-temurin-17: + digest: sha256:e8ef73dbd33b69fe497fd96b3bbbd85aff84ac4c564a5784ab02ad941b32c12b + target: registry.ddbuild.io/system-tests/mirror/maven:3.9-eclipse-temurin-17 + tag: 3.9-eclipse-temurin-17 + maven:3.9.14-eclipse-temurin-17: + digest: sha256:674de0df5a38cfe793a587829a181835f8af3ea1f556be7172e0aa547274bcc9 + target: registry.ddbuild.io/system-tests/mirror/maven:3.9.14-eclipse-temurin-17 + tag: 3.9.14-eclipse-temurin-17 + maven:3.9.9-eclipse-temurin-17-focal: + digest: sha256:45ddc84a2e7f75ab7faf290085a0292dd8adc5e7674a89231f6a879b60f4358d + target: registry.ddbuild.io/system-tests/mirror/maven:3.9.9-eclipse-temurin-17-focal + tag: 3.9.9-eclipse-temurin-17-focal + mcr.microsoft.com/dotnet/aspnet:8.0: + digest: sha256:93b366e510c6cd01cee608447014f7d349cb7ff8809fd0f554aa3772e8587b7e + target: registry.ddbuild.io/system-tests/mirror/mcr.microsoft.com/dotnet/aspnet:8.0 + tag: '8.0' + mcr.microsoft.com/dotnet/sdk:8.0: + digest: sha256:d80fdd84f7e18eea12f8e45c52914f1353395009c95c41197178ea19944e6d48 + target: registry.ddbuild.io/system-tests/mirror/mcr.microsoft.com/dotnet/sdk:8.0 + tag: '8.0' + mcr.microsoft.com/mssql/server:2022-latest: + digest: sha256:e07b9699a2b749969f19d86563ceeea22bd3a69f7f1db85a8d1ac4bdaf0c6f56 + target: registry.ddbuild.io/system-tests/mirror/mcr.microsoft.com/mssql/server:2022-latest + tag: 2022-latest + mongo:latest: + digest: sha256:49f1d7b87c2ddf918372be5defe7edff8c46703d0b2a56023a3f825e32e1250c + target: registry.ddbuild.io/system-tests/mirror/mongo:latest + tag: 8.2.11 + mysql/mysql-server:8.0.32: + digest: sha256:d6c8301b7834c5b9c2b733b10b7e630f441af7bc917c74dba379f24eeeb6a313 + target: registry.ddbuild.io/system-tests/mirror/mysql/mysql-server:8.0.32 + tag: 8.0.32 + node:18-alpine: + digest: sha256:8d6421d663b4c28fd3ebc498332f249011d118945588d0a35cb9bc4b8ca09d9e + target: registry.ddbuild.io/system-tests/mirror/node:18-alpine + tag: 18-alpine + node:20-alpine: + digest: sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 + target: registry.ddbuild.io/system-tests/mirror/node:20-alpine + tag: 20-alpine + node:22-alpine: + digest: sha256:e58326d0d441090181ac150dc2078d3e2cf6a0d42e809aebba3ef5880935ffdd + target: registry.ddbuild.io/system-tests/mirror/node:22-alpine + tag: 22-alpine + otel/opentelemetry-collector-contrib:0.137.0: + digest: sha256:886722fe0f37af9d1fe24d29529253ec59fbf263b3b1df4facaf221373e19d23 + target: registry.ddbuild.io/system-tests/mirror/otel/opentelemetry-collector-contrib:0.137.0 + tag: 0.137.0 + postgres:alpine: + digest: sha256:d3e64d36360a9f40c30fbbc5dd2dde799fe35f8537500c8b067551a6497f50f4 + target: registry.ddbuild.io/system-tests/mirror/postgres:alpine + tag: alpine + public.ecr.aws/lambda/java:17: + digest: sha256:ba8e52b4a5eb669dfda85bc4b55fe2957961ef67a962b0dfffa2e7abbcb7fcbb + target: registry.ddbuild.io/system-tests/mirror/public.ecr.aws/lambda/java:17 + tag: '17' + public.ecr.aws/lambda/nodejs:18: + digest: sha256:daf6a5c0a2b36153b94c91f3563e8ef89b3b19e4129963c6ccb07b8c5251f7ef + target: registry.ddbuild.io/system-tests/mirror/public.ecr.aws/lambda/nodejs:18 + tag: '18' + public.ecr.aws/lambda/python:3.13: + digest: sha256:fb6a19c34ca64079d0042a49f05688b3f9fab3b7ed51b7baacd48d593c898335 + target: registry.ddbuild.io/system-tests/mirror/public.ecr.aws/lambda/python:3.13 + tag: '3.13' + public.ecr.aws/lambda/ruby:3.4: + digest: sha256:39eb6cac136bc7d86e63d56fc5fc9d78ca87f29051bc309ac62f4aceb6779886 + target: registry.ddbuild.io/system-tests/mirror/public.ecr.aws/lambda/ruby:3.4 + tag: '3.4' + python:3.11-slim: + digest: sha256:ae52c5bef62a6bdd42cd1e8dffef86b9cd284bde9427da79839de7a4b983e7ca + target: registry.ddbuild.io/system-tests/mirror/python:3.11-slim + tag: 3.11-slim + python:3.12-slim: + digest: sha256:d764629ce0ddd8c71fd371e9901efb324a95789d2315a47db7e4d27e78f1b0e9 + target: registry.ddbuild.io/system-tests/mirror/python:3.12-slim + tag: 3.12-slim + python:3.13-slim: + digest: sha256:c33f0bc4364a6881bed1ec0cc2665e6c53c87a43e774aaeab88e6f17af105e4f + target: registry.ddbuild.io/system-tests/mirror/python:3.13-slim + tag: 3.13-slim + python:3.14-slim: + digest: sha256:44dd04494ee8f3b538294360e7c4b3acb87c8268e4d0a4828a6500b1eff50061 + target: registry.ddbuild.io/system-tests/mirror/python:3.14-slim + tag: 3.14-slim + rabbitmq:3.12-management-alpine: + digest: sha256:0b44fbcc3a4bf22d00090f1353127577dbe1fcb109c41669733a9d7ecf6c3a78 + target: registry.ddbuild.io/system-tests/mirror/rabbitmq:3.12-management-alpine + tag: 3.12-management-alpine + rockylinux:9: + digest: sha256:d7be1c094cc5845ee815d4632fe377514ee6ebcf8efaed6892889657e5ddaaa6 + target: registry.ddbuild.io/system-tests/mirror/rockylinux:9 + tag: '9' + softwaremill/elasticmq-native:1.6.11: + digest: sha256:fe3af4a8dd59f310b20e07ec3499cfdeb16a14f518c95ad7e70f200845dce3f5 + target: registry.ddbuild.io/system-tests/mirror/softwaremill/elasticmq-native:1.6.11 + tag: 1.6.11 + ubuntu:24.04: + digest: sha256:786a8b558f7be160c6c8c4a54f9a57274f3b4fb1491cf65146521ae77ff1dc54 + target: registry.ddbuild.io/system-tests/mirror/ubuntu:24.04 + tag: '24.04' diff --git a/mirror_images.yaml b/mirror_images.yaml new file mode 100644 index 00000000000..e0831216b23 --- /dev/null +++ b/mirror_images.yaml @@ -0,0 +1,110 @@ +# Docker images mirrored into registry.ddbuild.io/system-tests/mirror. +# +# This file is generated: it lists every image required by the CI scenarios. +# Regenerate after changing scenarios or weblog Dockerfiles with: +# +# python utils/scripts/update_mirror_images.py +# +# (also refreshes mirror_images.lock.yaml; commit both). The +# `mirror_images_check` CI job fails if this file is out of date. +- "apache/kafka:3.7.1" +- "cassandra:latest" +- "datadog/agent:latest" +- "datadog/dd-appsec-php-ci:php-8.0-release" +- "datadog/system-tests:apache-mod-7.0-zts.base-v1" +- "datadog/system-tests:apache-mod-7.0.base-v1" +- "datadog/system-tests:apache-mod-7.1-zts.base-v1" +- "datadog/system-tests:apache-mod-7.1.base-v1" +- "datadog/system-tests:apache-mod-7.2-zts.base-v1" +- "datadog/system-tests:apache-mod-7.2.base-v1" +- "datadog/system-tests:apache-mod-7.3-zts.base-v1" +- "datadog/system-tests:apache-mod-7.3.base-v1" +- "datadog/system-tests:apache-mod-7.4-zts.base-v1" +- "datadog/system-tests:apache-mod-7.4.base-v1" +- "datadog/system-tests:apache-mod-8.0-zts.base-v1" +- "datadog/system-tests:apache-mod-8.0.base-v1" +- "datadog/system-tests:apache-mod-8.1-zts.base-v1" +- "datadog/system-tests:apache-mod-8.1.base-v1" +- "datadog/system-tests:apache-mod-8.2-zts.base-v1" +- "datadog/system-tests:apache-mod-8.2.base-v1" +- "datadog/system-tests:django-poc.base-v11" +- "datadog/system-tests:django-py3.13.base-v10" +- "datadog/system-tests:express4-typescript.base-v1" +- "datadog/system-tests:express4-typescript.base-v2" +- "datadog/system-tests:express4.base-v1" +- "datadog/system-tests:express4.base-v2" +- "datadog/system-tests:express5.base-v1" +- "datadog/system-tests:express5.base-v2" +- "datadog/system-tests:fastapi.base-v9" +- "datadog/system-tests:fastify.base-v1" +- "datadog/system-tests:fastify.base-v2" +- "datadog/system-tests:flask-poc.base-v1" +- "datadog/system-tests:flask-poc.base-v13" +- "datadog/system-tests:flask-poc.base-v14" +- "datadog/system-tests:golang_buddy-v2" +- "datadog/system-tests:java_buddy-v1" +- "datadog/system-tests:lambda-proxy-v1" +- "datadog/system-tests:nextjs.base-v1" +- "datadog/system-tests:nextjs.base-v2" +- "datadog/system-tests:nodejs_buddy-v1" +- "datadog/system-tests:php-fpm-7.0.base-v1" +- "datadog/system-tests:php-fpm-7.1.base-v1" +- "datadog/system-tests:php-fpm-7.2.base-v1" +- "datadog/system-tests:php-fpm-7.3.base-v1" +- "datadog/system-tests:php-fpm-7.4.base-v1" +- "datadog/system-tests:php-fpm-8.0.base-v1" +- "datadog/system-tests:php-fpm-8.1.base-v1" +- "datadog/system-tests:php-fpm-8.2.base-v1" +- "datadog/system-tests:php-fpm-8.5.base-v1" +- "datadog/system-tests:proxy-v1" +- "datadog/system-tests:python3.12.base-v13" +- "datadog/system-tests:python_buddy-v2" +- "datadog/system-tests:ruby_buddy-v2" +- "datadog/system-tests:tornado.base-v2" +- "datadog/system-tests:uwsgi-poc.base-v10" +- "datadog/system-tests:uwsgi-poc.base-v9" +- "debian:bookworm-slim" +- "debian:stable-slim" +- "demisto/fastapi:0.116.1.4266494" +- "eclipse-temurin:11-jre" +- "eclipse-temurin:17-jre" +- "ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.59.0" +- "ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.62.0" +- "ghcr.io/datadog/images-rb/engines/ruby:2.5" +- "ghcr.io/datadog/images-rb/engines/ruby:2.7" +- "ghcr.io/datadog/images-rb/engines/ruby:3.0" +- "ghcr.io/datadog/images-rb/engines/ruby:3.1" +- "ghcr.io/datadog/images-rb/engines/ruby:3.4" +- "ghcr.io/graalvm/native-image-community:22.0.0" +- "golang:1.25" +- "golang:1.25-alpine" +- "kong/kong-gateway:3.4" +- "localstack/localstack:4.1" +- "maven:3.6-jdk-11" +- "maven:3.8-jdk-11" +- "maven:3.9-eclipse-temurin-11" +- "maven:3.9-eclipse-temurin-17" +- "maven:3.9.14-eclipse-temurin-17" +- "maven:3.9.9-eclipse-temurin-17-focal" +- "mcr.microsoft.com/dotnet/aspnet:8.0" +- "mcr.microsoft.com/dotnet/sdk:8.0" +- "mcr.microsoft.com/mssql/server:2022-latest" +- "mongo:latest" +- "mysql/mysql-server:8.0.32" +- "node:18-alpine" +- "node:20-alpine" +- "node:22-alpine" +- "otel/opentelemetry-collector-contrib:0.137.0" +- "postgres:alpine" +- "public.ecr.aws/lambda/java:17" +- "public.ecr.aws/lambda/nodejs:18" +- "public.ecr.aws/lambda/python:3.13" +- "public.ecr.aws/lambda/ruby:3.4" +- "python:3.11-slim" +- "python:3.12-slim" +- "python:3.13-slim" +- "python:3.14-slim" +- "rabbitmq:3.12-management-alpine" +- "rockylinux:9" +- "softwaremill/elasticmq-native:1.6.11" +- "ubuntu:24.04" diff --git a/requirements.txt b/requirements.txt index 7a8fd0a0734..5fd1783bdcf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -47,6 +47,7 @@ types-protobuf==5.29.1.20241207 types-python-dateutil==2.9.0.20241206 types-PyYAML==6.0.12.20241230 types-requests==2.31.0 +Jinja2==3.1.6 types-retry==0.9.9.20250322 watchdog==3.0.0 yamllint==1.35.1 diff --git a/tests/test_the_test/test_build_pipeline.py b/tests/test_the_test/test_build_pipeline.py new file mode 100644 index 00000000000..9f18726649e --- /dev/null +++ b/tests/test_the_test/test_build_pipeline.py @@ -0,0 +1,87 @@ +"""Tests for utils/ci/gitlab/build_pipeline.py — chunking, rendering, and CLI.""" + +import json +import re +from pathlib import Path + +import pytest +import yaml + +from utils import scenarios +from utils.ci.gitlab.build_pipeline import build, main + + +MINIMAL_PARAMS = { + "endtoend_defs": { + "parallel_weblogs": [{"name": "flask"}], + "parallel_jobs": [{"weblog": "flask", "scenarios": ["DEFAULT"], "weblog_build_required": True}], + }, + "miscs": {"binaries_artifact": "flask-binaries"}, + "parametric": {"enable": False, "parallel_jobs": []}, +} + + +def write_params(tmp_path: Path, *libs: str) -> None: + for lib in libs: + (tmp_path / f"params_{lib}.json").write_text(json.dumps(MINIMAL_PARAMS)) + + +@scenarios.test_the_test +class Test_BuildPipeline: + @pytest.mark.parametrize( + "libs", + [ + [], + ["a"], + ["a", "b"], + ["a", "b", "c"], + ["a", "b", "c", "d", "e", "f"], + ], + ) + def test_chunking_distribution(self, tmp_path: Path, libs: list[str]): + write_params(tmp_path, *libs) + out = tmp_path / "out" + build(libs, tmp_path, out, stage="e2e", ci_image="myimage", chunks=3) + + # Expected round-robin assignment + expected: dict[int, list[str]] = {i: [] for i in range(3)} + for idx, lib in enumerate(libs): + expected[idx % 3].append(lib) + + for i in range(3): + chunk = yaml.safe_load((out / f"generated-pipeline-chunk-{i}.yml").read_text()) + if not expected[i]: + assert "noop" in chunk, f"chunk {i} should be noop" + else: + jobs = {k: v for k, v in chunk.items() if isinstance(v, dict) and ("extends" in v or "script" in v)} + assert jobs, f"chunk {i} should have jobs for {expected[i]}" + + def test_missing_params_exits_nonzero(self, tmp_path: Path): + with pytest.raises(SystemExit) as exc: + main( + [ + "--stage", + "e2e", + "--libraries", + "python", + "--params-dir", + str(tmp_path), + "--ci-image", + "x", + "--output-dir", + str(tmp_path / "out"), + "--chunks", + "3", + ] + ) + assert exc.value.code != 0 + + def test_concatenation_no_duplicate_top_level_keys(self, tmp_path: Path): + """Two libs in same chunk (chunks=1) must not produce duplicate top-level YAML keys.""" + write_params(tmp_path, "python", "java") + out = tmp_path / "out" + build(["python", "java"], tmp_path, out, stage="e2e", ci_image="img", chunks=1) + text = (out / "generated-pipeline-chunk-0.yml").read_text() + for key in ("workflow:", "stages:", "include:"): + count = len(re.findall(rf"^{re.escape(key)}", text, re.MULTILINE)) + assert count <= 1, f"duplicate top-level key '{key}' found {count} times" diff --git a/tests/test_the_test/test_ci_orchestrator.py b/tests/test_the_test/test_ci_orchestrator.py index 8cb598c5ca9..27d98a85ba4 100644 --- a/tests/test_the_test/test_ci_orchestrator.py +++ b/tests/test_the_test/test_ci_orchestrator.py @@ -29,3 +29,16 @@ def test_ipv6_is_not_supported_for_uds_weblogs(): assert not _is_supported("dotnet", "uds", "IPV6", "dev") assert not _is_supported("python", "uds-flask", "IPV6", "dev") assert _is_supported("python", "flask-poc", "IPV6", "dev") + + +@scenarios.test_the_test +def test_get_endtoend_definitions_empty_scenario_map(): + # Regression: previously raised KeyError when "endtoend" or "parametric" keys were absent + defs = get_endtoend_definitions("ruby", {}, [], "dev", 200000, 256, "123", "") + assert isinstance(defs["endtoend_defs"]["parallel_jobs"], list) + + +@scenarios.test_the_test +def test_get_endtoend_definitions_missing_endtoend_key(): + defs = get_endtoend_definitions("ruby", {"other": ["X"]}, [], "dev", 200000, 256, "123", "") + assert defs["endtoend_defs"]["parallel_jobs"] == [] diff --git a/tests/test_the_test/test_compute_libraries_and_scenarios.py b/tests/test_the_test/test_compute_libraries_and_scenarios.py index 625461ea806..447c7b65fb2 100644 --- a/tests/test_the_test/test_compute_libraries_and_scenarios.py +++ b/tests/test_the_test/test_compute_libraries_and_scenarios.py @@ -488,3 +488,39 @@ def test_missing_original_manifest(self): new_manifests=Path("./tests/test_the_test/manifests/manifests_ref/"), old_manifests=Path("./wrong/path"), ) + + +@scenarios.test_the_test +class Test_GitLabMode: + @pytest.mark.parametrize( + ("source", "expected_event"), + [("merge_request_event", "pull_request"), ("push", "push"), ("schedule", "schedule")], + ) + def test_event_and_ref_normalization(self, monkeypatch: pytest.MonkeyPatch, source: str, expected_event: str): + monkeypatch.setenv("GITLAB_CI", "true") + monkeypatch.setenv("CI_PIPELINE_SOURCE", source) + monkeypatch.setenv("CI_COMMIT_REF_NAME", "feat-x") + inputs = build_inputs() + assert inputs.event_name == expected_event + assert inputs.ref == "refs/heads/feat-x" + assert inputs.is_gitlab + + def test_empty_ref(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("GITLAB_CI", "true") + monkeypatch.setenv("CI_PIPELINE_SOURCE", "push") + monkeypatch.setenv("CI_COMMIT_REF_NAME", "") + inputs = build_inputs() + assert inputs.ref == "" + + def test_libraries_output_sorted(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("GITLAB_CI", "true") + monkeypatch.setenv("CI_PIPELINE_SOURCE", "push") + monkeypatch.setenv("CI_COMMIT_REF_NAME", "feat-x") + # all-libs trigger file + inputs = build_inputs(modified_files=[".github/workflows/run-docker-ssi.yml"]) + output = process(inputs) + libs_line = next((line for line in output if line.startswith("libraries=")), None) + assert libs_line is not None, "GitLab mode must emit 'libraries=' output" + libs = json.loads(libs_line.split("=", 1)[1]) + parts = libs.split() + assert parts == sorted(parts) diff --git a/tests/test_the_test/test_external_gitlab_pipeline.py b/tests/test_the_test/test_external_gitlab_pipeline.py new file mode 100644 index 00000000000..6dd7fee9fed --- /dev/null +++ b/tests/test_the_test/test_external_gitlab_pipeline.py @@ -0,0 +1,47 @@ +"""Tests for utils/scripts/ci_orchestrators/external_gitlab_pipeline.py.""" + +import pytest + +from utils import scenarios +from utils.scripts.ci_orchestrators.external_gitlab_pipeline import ( + _is_local_include, + _strip_local_includes, + filter_yaml, +) + + +@scenarios.test_the_test +class Test_ExternalGitlabPipeline: + @pytest.mark.parametrize( + ("entry", "expected"), + [ + ({"local": "/foo"}, True), + ("bare-string", True), + ({"project": "p", "file": "f"}, False), + ({"remote": "https://example.com/x.yml"}, False), + ], + ) + def test_is_local_include(self, entry: object, expected: object): + assert _is_local_include(entry) is expected + + def test_strip_local_includes_removes_only_local(self): + remote = {"remote": "https://example.com/x.yml"} + data = {"include": [{"local": "/utils/ci/gitlab/main.yml"}, "bare", remote]} + _strip_local_includes(data) + assert data["include"] == [remote] + + def test_filter_yaml_keeps_only_requested_language(self): + data = { + "stages": ["configure", "python", "java", "pipeline-status"], + "variables": {}, + "include": [], + "configure_job": {"stage": "configure", "script": ["echo"]}, + "python_job": {"stage": "python", "script": ["echo"]}, + "java_job": {"stage": "java", "script": ["echo"]}, + "status_job": {"stage": "pipeline-status", "script": ["echo"]}, + } + result = filter_yaml(data, "python") + assert "java_job" not in result + assert "python_job" in result + assert "configure_job" in result + assert result["stages"] == ["configure", "python", "pipeline-status"] diff --git a/tests/test_the_test/test_gitlab_pipeline_structure.py b/tests/test_the_test/test_gitlab_pipeline_structure.py new file mode 100644 index 00000000000..01927ec73d0 --- /dev/null +++ b/tests/test_the_test/test_gitlab_pipeline_structure.py @@ -0,0 +1,38 @@ +"""Structural assertions on generated GitLab pipeline chunk YAML.""" + +import json +from pathlib import Path + +import yaml + +from utils import scenarios +from utils.ci.gitlab.build_pipeline import build + + +MINIMAL_PARAMS = { + "endtoend_defs": { + "parallel_weblogs": [{"name": "flask"}], + "parallel_jobs": [{"weblog": "flask", "scenarios": ["DEFAULT"], "weblog_build_required": True}], + }, + "miscs": {"binaries_artifact": "flask-binaries"}, + "parametric": {"enable": False, "parallel_jobs": []}, +} + + +@scenarios.test_the_test +def test_generated_chunk_jobs_have_required_keys(tmp_path: Path): + (tmp_path / "params_python.json").write_text(json.dumps(MINIMAL_PARAMS)) + out = tmp_path / "out" + build(["python"], tmp_path, out, stage="e2e", ci_image="myimage", chunks=3) + + for i in range(3): + chunk = yaml.safe_load((out / f"generated-pipeline-chunk-{i}.yml").read_text()) + for name, job in chunk.items(): + if ( + not isinstance(job, dict) + or name.startswith(".") + or name in ("workflow", "stages", "include", "variables") + ): + continue + has_stage = "stage" in job or "extends" in job + assert has_stage, f"job '{name}' in chunk {i} has no stage or extends" diff --git a/utils/_context/_image_mirror.py b/utils/_context/_image_mirror.py new file mode 100644 index 00000000000..56a13423824 --- /dev/null +++ b/utils/_context/_image_mirror.py @@ -0,0 +1,67 @@ +"""Rewrite Docker image references to the system-tests mirror. + +When the `USE_IMAGE_MIRROR` env var is truthy, image references that have been +mirrored into `registry.ddbuild.io/system-tests/mirror` (see mirror_images.yaml / +mirror_images.lock.yaml and the `mirror_images` CI job) are rewritten to their +mirror target, so CI pulls from the mirror instead of docker.io / ghcr.io / etc. + +It is off by default: with the flag unset (local runs, or any environment where +the mirror is not reachable) every reference is returned unchanged. + +The source -> target mapping is read from mirror_images.lock.yaml, which holds +the exact target for each mirrored image. Mirror paths are laid out so that +BuildKit's per-registry mirror protocol works without Dockerfile rewriting: + + docker.io official images -> mirror/library/ (e.g. library/node:22-alpine) + docker.io namespaced -> mirror// + ghcr.io -> mirror/ghcr.io/ + mcr.microsoft.com -> mirror/mcr.microsoft.com/ + public.ecr.aws -> mirror/public.ecr.aws/ + +Anything not listed in the lock file (locally built `system_tests/*` images, +unmirrored refs) is left untouched. +""" + +import functools +import os +from pathlib import Path + +import yaml + +# mirror_images.lock.yaml lives at the repo root (this file is utils/_context/). +_LOCK_PATH = Path(__file__).resolve().parents[2] / "mirror_images.lock.yaml" + + +def mirror_enabled() -> bool: + return os.environ.get("USE_IMAGE_MIRROR", "").strip().lower() in ("1", "true", "yes") + + +@functools.lru_cache(maxsize=1) +def _mapping() -> dict[str, str]: + """Return {source_ref: mirror_target_ref} from the lock file ({} if absent).""" + try: + with open(_LOCK_PATH, encoding="utf-8") as f: + data = yaml.safe_load(f) + except (FileNotFoundError, yaml.YAMLError): + return {} + + images = data.get("images") if isinstance(data, dict) else None + if not isinstance(images, dict): + return {} + + mapping: dict[str, str] = {} + for source, info in images.items(): + if isinstance(info, dict) and isinstance(info.get("target"), str): + mapping[source] = info["target"] + return mapping + + +def mirror_image(name: str) -> str: + """Return the mirror target for `name` when mirroring is enabled, else `name`. + + Only exact matches in the lock file are rewritten, so unmirrored and + locally-built references pass through unchanged. + """ + if not mirror_enabled(): + return name + return _mapping().get(name, name) diff --git a/utils/_context/containers.py b/utils/_context/containers.py index e21bf2ca686..89c18b42835 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -19,6 +19,7 @@ from utils._context.component_version import ComponentVersion, Version from utils._context.docker import get_docker_client +from utils._context._image_mirror import mirror_image from utils._context.ports import ContainerPorts from utils.proxy.tuf import get_tuf_root_json from utils.proxy.ports import ProxyPorts @@ -526,7 +527,10 @@ def __init__(self, image_name: str, *, local_image_only: bool): self.env: dict[str, str] | None = None self.labels: dict[str, str] = {} - self.name = image_name + # When the image mirror is enabled (USE_IMAGE_MIRROR), pull from the + # mirror; keep the original ref for diagnostics if the mirror pull fails. + self.original_name = image_name + self.name = mirror_image(image_name) self.local_image_only = local_image_only def _pull_with_retries(self, max_retries: int = 4, delay: int = 4): @@ -555,7 +559,20 @@ def load(self): pytest.exit(f"Image {self.name} not found locally, please build it", 1) logger.stdout(f"Pulling {self.name}") - self._image = self._pull_with_retries() + try: + self._image = self._pull_with_retries() + except (docker.errors.APIError, requests.exceptions.ConnectionError): + if self.name != self.original_name: + # The mirror rewrote this ref but the mirror copy could not be + # pulled. Don't silently fall back to the original registry: + # that would mask an incomplete mirror and could pull a + # different digest than the one that was mirrored. + logger.stdout( + f"Could not pull mirrored image {self.name} (mirror of {self.original_name}). " + "The mirror may be incomplete — regenerate it with " + "utils/scripts/update_mirror_images.py and check the mirror_images CI job." + ) + raise self._init_from_attrs(self._image.attrs) diff --git a/utils/build/build.sh b/utils/build/build.sh index 159dc3c1e9a..fc902452eaa 100755 --- a/utils/build/build.sh +++ b/utils/build/build.sh @@ -303,6 +303,19 @@ build() { DOCKERFILE=utils/build/docker/${TEST_LIBRARY}/${WEBLOG_VARIANT}.Dockerfile + # When the image mirror is enabled, create (or reuse) a buildx + # builder whose buildkitd daemon is configured to redirect all + # FROM pulls through registry.ddbuild.io/system-tests/mirror. + MIRROR_BUILDER_ARG="" + if [[ "${USE_IMAGE_MIRROR:-}" =~ ^(1|true|yes)$ ]]; then + if ! docker buildx inspect system-tests-mirror &>/dev/null 2>&1; then + docker buildx create \ + --name system-tests-mirror \ + --config "${SCRIPT_DIR}/docker/buildkitd.toml" + fi + MIRROR_BUILDER_ARG="--builder system-tests-mirror" + fi + GITHUB_TOKEN_SECRET_ARG="" if [ -n "${GITHUB_TOKEN_FILE:-}" ]; then @@ -338,6 +351,7 @@ build() { -t system_tests/weblog \ $CACHE_TO \ $CACHE_FROM \ + $MIRROR_BUILDER_ARG \ $EXTRA_DOCKER_ARGS \ . @@ -350,6 +364,7 @@ build() { ${DOCKER_PLATFORM_ARGS} \ -f utils/build/docker/overwrite_waf_rules.Dockerfile \ -t system_tests/weblog \ + $MIRROR_BUILDER_ARG \ $EXTRA_DOCKER_ARGS \ . fi diff --git a/utils/build/docker/buildkitd.toml b/utils/build/docker/buildkitd.toml new file mode 100644 index 00000000000..ebd07e1ab6e --- /dev/null +++ b/utils/build/docker/buildkitd.toml @@ -0,0 +1,15 @@ +# Auto-generated by: mirror_images buildkitd +# Do not edit manually — regenerate with: +# mirror_images buildkitd + +[registry."docker.io"] + mirrors = ["registry.ddbuild.io/system-tests/mirror"] + +[registry."ghcr.io"] + mirrors = ["registry.ddbuild.io/system-tests/mirror/ghcr.io"] + +[registry."mcr.microsoft.com"] + mirrors = ["registry.ddbuild.io/system-tests/mirror/mcr.microsoft.com"] + +[registry."public.ecr.aws"] + mirrors = ["registry.ddbuild.io/system-tests/mirror/public.ecr.aws"] diff --git a/utils/ci/__init__.py b/utils/ci/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/utils/ci/gitlab/__init__.py b/utils/ci/gitlab/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/utils/ci/gitlab/build_pipeline.py b/utils/ci/gitlab/build_pipeline.py new file mode 100644 index 00000000000..db7f4c27a3a --- /dev/null +++ b/utils/ci/gitlab/build_pipeline.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +_env = Environment(loader=FileSystemLoader(Path(__file__).resolve().parent), autoescape=select_autoescape()) +_template = _env.get_template("system-tests.yml.j2") + + +def noop_stub(stage: str) -> str: + """Return a minimal valid GitLab pipeline YAML for an empty chunk.""" + return f"""workflow: + name: "System-tests end to end (empty chunk)" +stages: + - {stage} +noop: + stage: {stage} + image: registry.ddbuild.io/images/mirror/alpine:latest + tags: + - arch:amd64 + script: + - echo "no libraries assigned to this chunk" +""" + + +def render_library( + library: str, + params: dict, + *, + skip_header: bool, + stage: str, + ci_image: str, + ref: str, + push_to_test_optimization: bool, + docker_auth: bool, + binaries_artifact_path: str, + binaries_artifacts: str, +) -> str: + parallel_weblogs = params.get("endtoend_defs", {}).get("parallel_weblogs", []) + parallel_jobs = params.get("endtoend_defs", {}).get("parallel_jobs", []) + weblog_variants = [w["name"] for w in parallel_weblogs] + scenario_pairs = [ + (job["weblog"], scenario, job.get("weblog_build_required", True)) + for job in parallel_jobs + for scenario in job.get("scenarios", []) + ] + binaries_artifact = params["miscs"]["binaries_artifact"] + parametric = params["parametric"] + # Build the full list of artifact jobs for cross-pipeline downloads. + # If binaries_artifacts is provided, use it; otherwise fall back to the single job. + if binaries_artifacts: + binaries_artifacts_list = [j.strip() for j in binaries_artifacts.split(";") if j.strip()] + elif binaries_artifact: + binaries_artifacts_list = [binaries_artifact] + else: + binaries_artifacts_list = [] + return _template.render( + scenario_pairs=scenario_pairs, + stage=stage, + library=library, + weblog_variants=weblog_variants, + binaries_artifact=binaries_artifact, + binaries_artifacts_list=binaries_artifacts_list, + binaries_artifact_path=binaries_artifact_path, + parametric=parametric, + ci_image=ci_image, + ref=ref, + push_to_test_optimization=push_to_test_optimization, + skip_header=skip_header, + docker_auth_enabled=docker_auth, + ) + + +def build( + libraries: list[str], + params_dir: Path, + output_dir: Path, + *, + stage: str, + ci_image: str, + ref: str = "", + push_to_test_optimization: bool = False, + chunks: int = 3, + docker_auth: bool = False, + binaries_artifact_path: str = "", + binaries_artifacts: str = "", +) -> None: + """Render pipeline chunk files into *output_dir*, one per chunk.""" + output_dir.mkdir(parents=True, exist_ok=True) + + # Assign libraries to chunks via simple round-robin + chunk_libraries: dict[int, list[str]] = {i: [] for i in range(chunks)} + for idx, library in enumerate(libraries): + chunk_libraries[idx % chunks].append(library) + + # Render and write all chunks; empty chunks get a noop stub so trigger jobs + # always have a valid pipeline file — dotenv vars are not available in rules: + # at pipeline-creation time (gitlab-org/gitlab#235812). + for chunk_idx, chunk_libs in chunk_libraries.items(): + chunk_file = output_dir / f"generated-pipeline-chunk-{chunk_idx}.yml" + + if not chunk_libs: + chunk_file.write_text(noop_stub(stage)) + continue + + parts = [] + for lib_idx, library in enumerate(chunk_libs): + params_file = params_dir / f"params_{library}.json" + if not params_file.exists(): + print(f"ERROR: params file not found for library '{library}': {params_file}", file=sys.stderr) # noqa: T201 + sys.exit(1) + with open(params_file) as f: + params = json.load(f) + parts.append( + render_library( + library, + params, + skip_header=(lib_idx > 0), + stage=stage, + ci_image=ci_image, + ref=ref, + push_to_test_optimization=push_to_test_optimization, + docker_auth=docker_auth, + binaries_artifact_path=binaries_artifact_path, + binaries_artifacts=binaries_artifacts, + ) + ) + + chunk_file.write_text("\n".join(parts)) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--stage", required=True, help="GitLab CI stage for the generated jobs") + parser.add_argument("--libraries", required=True, help="Space-separated list of library names") + parser.add_argument("--params-dir", required=True, help="Directory containing params_.json files") + parser.add_argument("--ci-image", required=True, help="Full CI image reference for generated jobs") + parser.add_argument("--ref", default="", help="system-tests ref to clone when called from another repository") + parser.add_argument("--push-to-test-optimization", default="false", help="Generate the push_test_optimization job") + parser.add_argument("--output-dir", required=True, help="Output directory for generated-pipeline-chunk-N.yml files") + parser.add_argument("--chunks", type=int, default=3, help="Number of pipeline chunks (default: 3)") + parser.add_argument("--docker-auth", default="false", help="Whether to authenticate calls to docker hub") + parser.add_argument( + "--binaries-artifact-path", + default="", + help="Path (relative to CI_PROJECT_DIR) of the binaries_artifact contents, copied into binaries/", + ) + parser.add_argument( + "--binaries-artifacts", + default="", + help="Semicolon-separated list of upstream jobs to download artifacts from in the child pipeline " + "(';' not ',' because job names may contain commas, e.g. parallel matrix names; " + "falls back to the single binaries_artifact from params if empty)", + ) + + args = parser.parse_args(argv) + + libraries = args.libraries.split() + + build( + libraries=libraries, + params_dir=Path(args.params_dir), + output_dir=Path(args.output_dir), + stage=args.stage, + ci_image=args.ci_image, + ref=args.ref, + push_to_test_optimization=args.push_to_test_optimization == "true", + chunks=args.chunks, + docker_auth=args.docker_auth == "true", + binaries_artifact_path=args.binaries_artifact_path, + binaries_artifacts=args.binaries_artifacts, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/ci/gitlab/docker/git.Dockerfile b/utils/ci/gitlab/docker/git.Dockerfile new file mode 100644 index 00000000000..b0b33abb390 --- /dev/null +++ b/utils/ci/gitlab/docker/git.Dockerfile @@ -0,0 +1,3 @@ +FROM alpine/git + +# docker build --push --platform linux/amd64 -t registry.ddbuild.io/system-tests/git -f utils/ci/gitlab/docker/git.Dockerfile . diff --git a/utils/ci/gitlab/docker/system-tests.Dockerfile b/utils/ci/gitlab/docker/system-tests.Dockerfile new file mode 100644 index 00000000000..f8f017f9a59 --- /dev/null +++ b/utils/ci/gitlab/docker/system-tests.Dockerfile @@ -0,0 +1,58 @@ +FROM registry.ddbuild.io/images/docker:29.4.0-noble AS builder +USER root + +RUN apt update && apt upgrade -y +RUN apt install software-properties-common -y +RUN add-apt-repository ppa:deadsnakes/ppa -y + +RUN clean-apt install \ + build-essential \ + ca-certificates \ + git \ + python3.12 \ + python3-pip \ + python3.12-venv \ + python3.12-dev + +COPY . /system-tests +WORKDIR /system-tests +RUN ./build.sh -i runner + +FROM registry.ddbuild.io/images/docker:29.4.0-noble +USER root + +RUN apt update && apt upgrade -y +RUN apt install software-properties-common -y +RUN add-apt-repository ppa:deadsnakes/ppa -y + +RUN clean-apt install \ + jq \ + zstd \ + ca-certificates \ + curl \ + git \ + python3.12 \ + python3.12-venv + +RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \ + clean-apt install nodejs + +RUN npm install -g @datadog/datadog-ci + +ARG TARGETARCH +RUN if [ "${TARGETARCH}" = "arm64" ]; then \ + curl --retry 10 -fsSLo awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip; \ + else \ + curl --retry 10 -fsSLo awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip; \ + fi && \ + unzip -q awscliv2.zip && \ + ./aws/install && \ + apt-get clean + +COPY --from=registry.ddbuild.io/dd-sts:v0.1.4@sha256:1f4bc8861cca86b0c977ae70843990f9368f9b69dbfc4979cf5c515a97a3ea15 \ + /usr/local/bin/dd-sts /usr/local/bin/dd-sts + +COPY --from=builder /system-tests/venv /system-tests/venv +COPY --from=builder /system-tests/utils/ci/gitlab/validate_param_env.py /system-tests/utils/ci/gitlab/validate_param_env.py + +WORKDIR / diff --git a/utils/ci/gitlab/main.yml b/utils/ci/gitlab/main.yml new file mode 100644 index 00000000000..cd37696a8e3 --- /dev/null +++ b/utils/ci/gitlab/main.yml @@ -0,0 +1,196 @@ +--- +spec: + inputs: + stage: + description: "CI stage for all jobs in this component and the generated child pipeline" + libraries: + default: "" + scenarios: + description: "Comma-separated list of scenarios to run" + default: "" + scenarios_groups: + description: "Comma-separated list of scenario groups to run" + default: "" + excluded_scenarios: + description: "Comma-separated list of scenarios not to run" + default: "APM_TRACING_E2E_OTEL,APM_TRACING_E2E_SINGLE_SPAN,OTEL_TRACING_E2E,OTEL_METRIC_E2E,OTEL_LOG_E2E,INTEGRATION_FRAMEWORKS" + weblogs: + description: "Comma-separated list of weblogs to run (all weblogs if empty)" + default: "" + binaries_artifacts: + description: "Semicolon-separated list of upstream jobs whose artifacts contain the pre-built binaries (downloaded by the generated build/run jobs via cross-pipeline needs). Use ';' not ',' because job names may contain commas (e.g. parallel matrix names)." + default: "" + binaries_artifact_path: + description: "Path (relative to CI_PROJECT_DIR) of the binaries_artifact contents, copied into binaries/ before building/running" + default: "" + force_execute: + description: "Comma-separated test node IDs to force-execute" + default: "" + skip_empty_scenarios: + description: "Skip scenarios that contain only xfail or irrelevant tests" + type: boolean + default: false + parametric_job_count: + description: "Number of parallel jobs for the PARAMETRIC scenario" + type: number + default: 1 + push_to_test_optimization: + description: "Push test results to Datadog Test Optimization" + type: boolean + default: false + auto_cancel_on_new_commit: + description: "Auto-cancel strategy when a new commit is pushed (interruptible, conservative, disabled)" + default: "interruptible" + ref: + description: "system-tests ref to use when called from another repository (branch, tag or SHA)" + default: "main" + condition: + description: "GitLab CI rule expression controlling whether system-tests jobs run (e.g. '$NIGHTLY_BUILD == \"true\"'). Defaults to always-run." + default: "null == null" + docker_auth: + description: "Whether to authenticate calls to docker hub" + default: "false" + split_pipeline: + description: "Split jobs across multiple child pipelines" + type: boolean + default: false + +--- + +include: + - project: 'DataDog/system-tests' + ref: $[[ inputs.ref ]] + file: 'utils/ci/gitlab/templates.yml' + inputs: + stage: $[[ inputs.stage ]] + ref: $[[ inputs.ref ]] + +workflow: + rules: + - when: always + auto_cancel: + on_new_commit: $[[ inputs.auto_cancel_on_new_commit ]] + +variables: + # Tag = first 12 chars of sha256(utils/ci/gitlab/docker/system-tests.Dockerfile + requirements.txt) + # Update this when either file changes: + # cat utils/ci/gitlab/docker/system-tests.Dockerfile requirements.txt | sha256sum | cut -c1-12 + CI_IMAGE: "registry.ddbuild.io/system-tests/ci-runner:f4f0d91aa8dc" + SYSTEM_TESTS_SPLIT_PIPELINE: "$[[ inputs.split_pipeline ]]" + +.system_tests_param_base: + image: $CI_IMAGE + tags: + - arch:amd64 + script: + - python3 /system-tests/utils/ci/gitlab/validate_param_env.py + artifacts: + reports: + dotenv: param.env + +system_tests_build_pipeline: + extends: .system_tests_base + tags: + - arch:amd64 + rules: + - if: $[[ inputs.condition ]] + - when: never + needs: + - job: build_ci_image + optional: true + artifacts: false + - job: system_tests_param + optional: true + artifacts: true + script: + - | + libraries=$(echo "$[[ inputs.libraries ]],$LIBRARIES" | tr ',' ' ' | xargs) + touch build_params.env + scenarios=$(echo "$[[ inputs.scenarios ]],$SCENARIOS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') + scenario_groups=$(echo "$[[ inputs.scenarios_groups ]],$SCENARIO_GROUPS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') + weblogs=$(echo "$[[ inputs.weblogs ]],$WEBLOGS" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') + excluded_scenarios=$(echo "$[[ inputs.excluded_scenarios ]],$EXCLUDED_SCENARIOS" | tr ',' '\n' | sed '/^$/d' | sort -u | tr '\n' ',' | sed 's/,$//') + force_execute=$(echo "$[[ inputs.force_execute ]],$FORCE_EXECUTE" | tr ',' '\n' | sed '/^$/d' | tr '\n' ',' | sed 's/,$//') + parametric_job_count="${SYSTEM_TESTS_PARAMETRIC_JOB_COUNT:-$[[ inputs.parametric_job_count ]]}" + push_to_test_optimization="${SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION:-$[[ inputs.push_to_test_optimization ]]}" + docker_auth="${SYSTEM_TESTS_DOCKER_AUTH:-$[[ inputs.docker_auth ]]}" + skip_empty_scenario="${SYSTEM_TESTS_SKIP_EMPTY_SCENARIO:-$[[ inputs.skip_empty_scenarios ]]}" + echo "libraries: $libraries" + echo "scenarios: $scenarios" + echo "scenario_groups: $scenario_groups" + echo "weblogs: $weblogs" + echo "excluded_scenarios: $excluded_scenarios" + echo "force_execute: $force_execute" + echo "parametric_job_count: $parametric_job_count" + echo "push_to_test_opt: $push_to_test_optimization" + echo "skip_empty_scenario: $skip_empty_scenario" + printf 'SYSTEM_TESTS_FORCE_EXECUTE=%s\nSYSTEM_TESTS_SKIP_EMPTY_SCENARIO=%s\n' "$force_execute" "$skip_empty_scenario" > build_params.env + binaries_artifacts="${BINARIES_ARTIFACTS:-$[[ inputs.binaries_artifacts ]]}" + binaries_artifact_path="${BINARIES_ARTIFACT_PATH:-$[[ inputs.binaries_artifact_path ]]}" + for library in $libraries; do + python3 utils/scripts/compute-workflow-parameters.py $library --format json --groups "$scenario_groups" --excluded-scenarios "$excluded_scenarios" --scenarios "$scenarios" --weblogs "$weblogs" --explicit-binaries-artifact "$binaries_artifacts" --parametric-job-count $parametric_job_count --output params_${library}.json + done + chunks=1 + if [ "$SYSTEM_TESTS_SPLIT_PIPELINE" = "true" ]; then chunks=3; fi + python3 utils/ci/gitlab/build_pipeline.py --stage $[[ inputs.stage ]] --ci-image "$CI_IMAGE" --ref "$[[ inputs.ref ]]" --push-to-test-optimization "$push_to_test_optimization" --libraries "$libraries" --params-dir . --output-dir . --chunks $chunks --docker-auth "$docker_auth" --binaries-artifact-path "$binaries_artifact_path" --binaries-artifacts "$binaries_artifacts" + artifacts: + paths: + - system-tests/generated-pipeline-chunk-*.yml + reports: + dotenv: system-tests/build_params.env + +.run_test_pipeline_base: + interruptible: true + stage: $[[ inputs.stage ]] + rules: + - if: $[[ inputs.condition ]] + - when: never + needs: + - job: system_tests_build_pipeline + optional: true + artifacts: true + - job: system_tests_param + optional: true + artifacts: true + - job: mirror_images + optional: true + artifacts: false + variables: + SYSTEM_TESTS_FORCE_EXECUTE: "$SYSTEM_TESTS_FORCE_EXECUTE" + SYSTEM_TESTS_SKIP_EMPTY_SCENARIO: "$SYSTEM_TESTS_SKIP_EMPTY_SCENARIO" + LIBRARIES: $LIBRARIES + UPSTREAM_PIPELINE_ID: $CI_PIPELINE_ID + +system_tests_run_pipeline_0: + extends: .run_test_pipeline_base + trigger: + include: + - artifact: system-tests/generated-pipeline-chunk-0.yml + job: system_tests_build_pipeline + strategy: depend + +system_tests_run_pipeline_1: + extends: .run_test_pipeline_base + rules: + - if: $SYSTEM_TESTS_SPLIT_PIPELINE != "true" + when: never + - if: $[[ inputs.condition ]] + - when: never + trigger: + include: + - artifact: system-tests/generated-pipeline-chunk-1.yml + job: system_tests_build_pipeline + strategy: depend + +system_tests_run_pipeline_2: + extends: .run_test_pipeline_base + rules: + - if: $SYSTEM_TESTS_SPLIT_PIPELINE != "true" + when: never + - if: $[[ inputs.condition ]] + - when: never + trigger: + include: + - artifact: system-tests/generated-pipeline-chunk-2.yml + job: system_tests_build_pipeline + strategy: depend diff --git a/utils/ci/gitlab/section.sh b/utils/ci/gitlab/section.sh new file mode 100644 index 00000000000..4bbcd09834f --- /dev/null +++ b/utils/ci/gitlab/section.sh @@ -0,0 +1,15 @@ +# shellcheck shell=bash +# function for starting the section +function section_start () { + local section_title="${1}" + local section_description="${2:-$section_title}" + + echo -e "section_start:$(date +%s):${section_title}[collapsed=true]\r\e[0K${section_description}" +} + +# Function for ending the section +function section_end () { + local section_title="${1}" + + echo -e "section_end:$(date +%s):${section_title}\r\e[0K" +} diff --git a/utils/ci/gitlab/system-tests.yml.j2 b/utils/ci/gitlab/system-tests.yml.j2 new file mode 100644 index 00000000000..dc7e9b04920 --- /dev/null +++ b/utils/ci/gitlab/system-tests.yml.j2 @@ -0,0 +1,163 @@ +{% macro docker_auth() %} + - section_start "docker_auth" "Docker hub auth" + - export DOCKER_LOGIN=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-write --with-decryption --query "Parameter.Value" --out text) + - export DOCKER_LOGIN_PASS=$(aws ssm get-parameter --region us-east-1 --name ci.system-tests.docker-login-pass-write --with-decryption --query "Parameter.Value" --out text) + - | + for i in 1 2 3; do + echo "$DOCKER_LOGIN_PASS" | docker login --username "$DOCKER_LOGIN" --password-stdin && break + if [ "$i" -eq 3 ]; then echo "docker login failed after 3 attempts"; exit 1; fi + echo "docker login failed (attempt $i), retrying in $((i*5))s..." + sleep $((i*5)) + done + - section_end "docker_auth" +{% endmacro %} +{% macro copy_binaries(path) %} + - section_start "copy_binaries" "Copying pre-built binaries into binaries/" + - mkdir -p binaries + - cp -r "$CI_PROJECT_DIR/{{path}}/." binaries/ + - ls -la binaries/ + - section_end "copy_binaries" +{% endmacro %} +{% if not skip_header %} +workflow: + name: "System-tests end to end" + +include: + - project: 'DataDog/system-tests' + ref: {{ref}} + file: 'utils/ci/gitlab/templates.yml' + inputs: + stage: {{stage}} + ref: {{ref}} + +stages: + - {{stage}} + +variables: + CI_IMAGE: {{ci_image}} + +.push_to_test_optimization: + id_tokens: + DD_STS_OIDC_TOKEN: + aud: dd-sts + after_script: + - echo -e "section_start:$(date +%s):test_optim[collapsed=true]\r\e[0KPush to tests optimization" + - dd-sts debug + - > + dd-sts exchange --policy system-tests-gitlab -- datadog-ci junit upload system-tests/logs*/reportJunit.xml + --service system-tests + --env ci + --xpath-tag "test.codeowners=/testcase/properties/property[@name='test.codeowners']" + - echo -e "section_end:$(date +%s):test_optim\r\e[0K" + +setup: + extends: .system_tests_base + id_tokens: + DD_STS_OIDC_TOKEN: + aud: dd-sts + script: + - dd-sts exchange --policy system-tests-gitlab -- datadog-ci tag --level pipeline --tags "system-tests:true" +{% endif %} + +{% for variant in weblog_variants %} +system_tests_build_{{library}}_{{variant}}: + extends: .system_tests_base + {% if binaries_artifacts_list %} + needs: + {% for artifact_job in binaries_artifacts_list %} + - job: "{{artifact_job}}" + artifacts: true + pipeline: $UPSTREAM_PIPELINE_ID + {% endfor %} + {% endif %} + script: + {% if docker_auth_enabled %} + {{ docker_auth() }} + {% endif %} + {% if binaries_artifacts_list and binaries_artifact_path %} + {{ copy_binaries(binaries_artifact_path) }} + {% endif %} + - section_start "build" "Building weblog" + - ./build.sh {{library}} -i weblog --save-to-binaries --weblog-variant {{variant}} + - mv binaries .. + - section_end "build" + artifacts: + paths: + - binaries/ + +{% endfor %} +{% for variant, scenario, build_required in scenario_pairs %} +system_tests_run_{{library}}_{{scenario}}_{{variant}}: + extends: + - .system_tests_base +{% if push_to_test_optimization %} + - .push_to_test_optimization +{% endif %} + needs: + {% if build_required %} + - job: system_tests_build_{{library}}_{{variant}} + artifacts: true + {% elif binaries_artifacts_list %} + {% for artifact_job in binaries_artifacts_list %} + - job: "{{artifact_job}}" + artifacts: true + pipeline: $UPSTREAM_PIPELINE_ID + {% endfor %} + {% endif %} + script: + {% if docker_auth_enabled %} + {{ docker_auth() }} + {% endif %} + - section_start "weblog_setup" "Setting up the weblog" + {% if build_required %} + - mv ../binaries/* binaries/ + - ./build.sh {{library}} -i weblog --weblog-variant {{variant}} + {% elif binaries_artifacts_list and binaries_artifact_path %} + {{ copy_binaries(binaries_artifact_path) }} + {% endif %} + - section_end "weblog_setup" + - section_start "scenario_run" "Running scenario" + {% if scenario == "INTEGRATION_FRAMEWORKS" %} + - ./run.sh {{scenario}} -L {{library}} --weblog {{variant}} + {% elif scenario in ("GO_PROXIES_DEFAULT", "GO_PROXIES_APPSEC_BLOCKING") %} + - ./run.sh {{scenario}} --weblog {{variant}} + {% else %} + - ./run.sh {{scenario}} + {% endif %} + - section_end "scenario_run" + artifacts: + when: always + paths: + - system-tests/logs* + +{% endfor %} +{% if parametric.enable %} +{% for job_index in parametric.job_matrix %} +system_tests_run_{{library}}_PARAMETRIC_{{job_index}}: + extends: + - .system_tests_base +{% if push_to_test_optimization %} + - .push_to_test_optimization +{% endif %} + {% if binaries_artifacts_list %} + needs: + {% for artifact_job in binaries_artifacts_list %} + - job: "{{artifact_job}}" + artifacts: true + pipeline: $UPSTREAM_PIPELINE_ID + {% endfor %} + {% endif %} + script: + - echo -e "\e[0Ksection_start:$(date +%s):run\r\e[0KRunning PARAMETRIC (group {{job_index}}/{{parametric.job_count}})" + {% if binaries_artifacts_list and binaries_artifact_path %} + {{ copy_binaries(binaries_artifact_path) }} + {% endif %} + - ./run.sh PARAMETRIC -L {{library}} --splits={{parametric.job_count}} --group={{job_index}} + - echo -e "\e[0Ksection_end:$(date +%s):run\r\e[0K" + artifacts: + when: always + paths: + - system-tests/logs* + +{% endfor %} +{% endif %} diff --git a/utils/ci/gitlab/templates.yml b/utils/ci/gitlab/templates.yml new file mode 100644 index 00000000000..b00709c000d --- /dev/null +++ b/utils/ci/gitlab/templates.yml @@ -0,0 +1,35 @@ +--- +spec: + inputs: + stage: + ref: + default: main + +--- + +.system_tests_base: + image: $CI_IMAGE + tags: + - docker-in-docker:amd64 + variables: + GIT_STRATEGY: none + # In DinD, DOCKER_HOST is set to tcp://docker:2376 which causes _Weblog to + # resolve the weblog as docker:7777 instead of localhost:7777, breaking span + # snapshot assertions. Force it back to localhost. + SYSTEM_TESTS_WEBLOG_HOST: "localhost" + # Pull scenario images (and weblog FROM bases) from the system-tests mirror + # instead of docker.io/ghcr.io/etc. Runtime pulls fall back to the original + # registry if an image isn't mirrored yet (see utils/_context/_image_mirror.py). + USE_IMAGE_MIRROR: "1" + stage: $[[ inputs.stage ]] + interruptible: true + before_script: + - echo -e "section_start:`date +%s`:setup[collapsed=true]\r\e[0KSystem-tests runner setup" + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/system-tests.git + - cd system-tests + - git checkout $[[ inputs.ref ]] + - ln -sf /system-tests/venv venv + - source venv/bin/activate + - export PYTHONPATH="$CI_PROJECT_DIR/system-tests" + - source utils/ci/gitlab/section.sh + - echo -e "section_end:`date +%s`:setup\r\e[0K" diff --git a/utils/ci/gitlab/validate_param_env.py b/utils/ci/gitlab/validate_param_env.py new file mode 100644 index 00000000000..d950314c210 --- /dev/null +++ b/utils/ci/gitlab/validate_param_env.py @@ -0,0 +1,132 @@ +"""Validate environment variables and write them to param.env. + +Called by .system_tests_param_base in main.yml. The caller (before_script) is +responsible for exporting the variables into the environment before this script runs. +""" + +from __future__ import annotations + +import os +import re +import sys + + +def _error(name: str, value: str, msg: str) -> str: + return f"❌ {name}: '{value}' — {msg}" + + +def validate_space_lower(name: str, value: str) -> list[str]: + """Space-separated lowercase identifiers, e.g. 'python java_otel'.""" + errors = [] + for item in value.split(): + if not re.match(r"^[a-z][a-z0-9_]*$", item): + errors.append( + _error(name, item, "expected space-separated lowercase identifiers (e.g. 'python java_otel')") + ) + return errors + + +def validate_comma_upper(name: str, value: str) -> list[str]: + """Comma-separated uppercase identifiers, e.g. 'DEFAULT,PARAMETRIC'.""" + errors = [] + for item in (i.strip() for i in value.split(",")): + if item and not re.match(r"^[A-Z][A-Z0-9_]*$", item): + errors.append( + _error(name, item, "expected comma-separated uppercase identifiers (e.g. 'DEFAULT,PARAMETRIC')") + ) + return errors + + +def validate_comma_lower(name: str, value: str) -> list[str]: + """Comma-separated lowercase identifiers, e.g. 'all,appsec'.""" + errors = [] + for item in (i.strip() for i in value.split(",")): + if item and not re.match(r"^[a-z][a-z0-9_]*$", item): + errors.append(_error(name, item, "expected comma-separated lowercase identifiers (e.g. 'all,appsec')")) + return errors + + +def validate_comma_lower_hyphen(name: str, value: str) -> list[str]: + """Comma-separated lowercase identifiers allowing hyphens, e.g. 'flask,django-realworld'.""" + errors = [] + for item in (i.strip() for i in value.split(",")): + if item and not re.match(r"^[a-z][a-z0-9_-]*$", item): + errors.append( + _error(name, item, "expected comma-separated lowercase identifiers (e.g. 'flask,django-realworld')") + ) + return errors + + +def validate_bool(name: str, value: str) -> list[str]: + """Must be 'true' or 'false'.""" + if value not in ("true", "false"): + return [_error(name, value, "expected 'true' or 'false'")] + return [] + + +def validate_positive_int(name: str, value: str) -> list[str]: + """Must be a positive integer.""" + if not re.match(r"^[1-9][0-9]*$", value): + return [_error(name, value, "expected a positive integer")] + return [] + + +def validate_path(name: str, value: str) -> list[str]: + """Must be a valid relative path.""" + if not re.match(r"^[a-zA-Z0-9_][a-zA-Z0-9_./-]*$", value): + return [_error(name, value, "expected a valid relative path")] + return [] + + +def validate_semicolon_nonempty(name: str, value: str) -> list[str]: + """Semicolon-separated non-empty entries, e.g. job names.""" + errors = [] + for item in (i.strip() for i in value.split(";")): + if not item: + errors.append(_error(name, value, "empty entry found — expected semicolon-separated non-empty job names")) + return errors + + +CHECKS: list[tuple[str, object]] = [ + ("LIBRARIES", validate_space_lower), + ("SCENARIOS", validate_comma_upper), + ("SCENARIO_GROUPS", validate_comma_lower), + ("WEBLOGS", validate_comma_lower_hyphen), + ("EXCLUDED_SCENARIOS", validate_comma_upper), + ("FORCE_EXECUTE", None), # pytest node IDs are too complex to validate by format + ("SYSTEM_TESTS_PARAMETRIC_JOB_COUNT", validate_positive_int), + ("SYSTEM_TESTS_PUSH_TO_TEST_OPTIMIZATION", validate_bool), + ("SYSTEM_TESTS_DOCKER_AUTH", validate_bool), + ("SYSTEM_TESTS_SKIP_EMPTY_SCENARIO", validate_bool), + ("BINARIES_ARTIFACTS", validate_semicolon_nonempty), + ("BINARIES_ARTIFACT_PATH", validate_path), +] + +PARAM_ENV = "param.env" + + +def main() -> None: + errors: list[str] = [] + + for name, validator in CHECKS: + value = os.environ.get(name, "") + if value and validator is not None: + errors.extend(validator(name, value)) # type: ignore[operator] + + if errors: + for e in errors: + print(e, file=sys.stderr) # noqa: T201 + sys.exit(1) + + with open(PARAM_ENV, "w") as f: + for name, _ in CHECKS: + value = os.environ.get(name, "") + if value: + f.write(f"{name}={value}\n") + + with open(PARAM_ENV) as f: + print(f.read(), end="") # noqa: T201 + + +if __name__ == "__main__": + main() diff --git a/utils/docker_fixtures/_test_agent.py b/utils/docker_fixtures/_test_agent.py index 1d4756bf229..aa1f441bef2 100644 --- a/utils/docker_fixtures/_test_agent.py +++ b/utils/docker_fixtures/_test_agent.py @@ -18,6 +18,7 @@ from retry import retry from utils._logger import logger +from utils._context._image_mirror import mirror_image from utils.dd_constants import RemoteConfigApplyState, Capabilities from .spec import remoteconfig from .spec.trace import V06StatsPayload @@ -54,7 +55,8 @@ class TestAgentFactory: """ def __init__(self, image: str): - self.image = image + # Pull/run from the mirror when USE_IMAGE_MIRROR is enabled (no-op otherwise). + self.image = mirror_image(image) self.host_log_folder = "" def configure(self, host_log_folder: str): diff --git a/utils/scripts/ci_orchestrators/external_gitlab_pipeline.py b/utils/scripts/ci_orchestrators/external_gitlab_pipeline.py index dd7c0e04777..bb10b53adfb 100644 --- a/utils/scripts/ci_orchestrators/external_gitlab_pipeline.py +++ b/utils/scripts/ci_orchestrators/external_gitlab_pipeline.py @@ -18,6 +18,37 @@ LANG_STAGES = sorted(COMPONENT_GROUPS.ssi) +def _is_local_include(entry: object) -> bool: + """Return True if a GitLab include entry refers to a local file path. + + local: includes are resolved against the root project, not the project + that wrote the YAML. When this file's output is used as a child pipeline + in a tracer repo, any local: path from system-tests' .gitlab-ci.yml would + be looked up inside the tracer repo and cause a pipeline compilation error. + """ + if isinstance(entry, str): + return True # bare strings are local file paths + if isinstance(entry, dict): + return "local" in entry + return False + + +def _strip_local_includes(data: dict) -> None: + """Remove local: include entries from *data* in-place. + + Only remote: (and other non-local) entries are kept so that the generated + pipeline is safe to run from any project. + """ + raw = data.get("include") + if raw is None: + return + if isinstance(raw, list): + data["include"] = [e for e in raw if not _is_local_include(e)] + else: + # Single mapping or bare string + data["include"] = [] if _is_local_include(raw) else [raw] + + def main(language: str | None = None) -> None: """Main function to generate the gitlab system-tests pipeline Args: @@ -30,6 +61,11 @@ def main(language: str | None = None) -> None: with open(".gitlab-ci.yml", "r") as file: data = yaml.safe_load(file) + # Drop local: includes — they resolve against the root project (the tracer + # repo) when this output is used as a child pipeline, not against + # system-tests, so any local: path would cause a compilation error there. + _strip_local_includes(data) + # Ensure 'variables' section exists and update with new values data.setdefault("variables", {}).update(new_variables) diff --git a/utils/scripts/ci_orchestrators/gitlab_exporter.py b/utils/scripts/ci_orchestrators/gitlab_exporter.py index f2dbd5729cd..fe9851daa8f 100644 --- a/utils/scripts/ci_orchestrators/gitlab_exporter.py +++ b/utils/scripts/ci_orchestrators/gitlab_exporter.py @@ -66,6 +66,7 @@ def print_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_enviro def print_ssi_gitlab_pipeline(language: str, matrix_data: dict[str, dict], ci_environment: str) -> None: result_pipeline = {} # type: dict + result_pipeline["workflow"] = {"name": f"{language} SSI"} result_pipeline["include"] = [] result_pipeline["stages"] = [] pipeline_file = ".gitlab/ssi_gitlab-ci.yml" diff --git a/utils/scripts/ci_orchestrators/workflow_data.py b/utils/scripts/ci_orchestrators/workflow_data.py index 4c686a28aac..39a6d072e93 100644 --- a/utils/scripts/ci_orchestrators/workflow_data.py +++ b/utils/scripts/ci_orchestrators/workflow_data.py @@ -344,7 +344,7 @@ def get_endtoend_definitions( unique_id: str, binaries_artifact: str, ) -> dict: - scenarios = scenario_map["endtoend"] + scenarios = scenario_map.get("endtoend", []) # get time stats with open("utils/scripts/ci_orchestrators/time-stats.json", "r") as file: diff --git a/utils/scripts/compute-workflow-parameters.py b/utils/scripts/compute-workflow-parameters.py index 2f730e72c4c..07836492c02 100644 --- a/utils/scripts/compute-workflow-parameters.py +++ b/utils/scripts/compute-workflow-parameters.py @@ -92,7 +92,7 @@ def __init__( self.data["parametric"] = { "job_count": parametric_job_count, "job_matrix": list(range(1, parametric_job_count + 1)), - "enable": len(scenario_map["parametric"]) > 0 + "enable": len(scenario_map.get("parametric", [])) > 0 and "otel" not in library and library not in ( diff --git a/utils/scripts/compute_libraries_and_scenarios.py b/utils/scripts/compute_libraries_and_scenarios.py index 4e6d15526bc..dff74aca5fb 100644 --- a/utils/scripts/compute_libraries_and_scenarios.py +++ b/utils/scripts/compute_libraries_and_scenarios.py @@ -306,8 +306,10 @@ def __init__( def load_git_info(self) -> None: # Get all relevant environment variables. if "GITLAB_CI" in os.environ: - self.event_name = os.environ.get("CI_PIPELINE_SOURCE", "push") - self.ref = os.environ.get("CI_COMMIT_REF_NAME", "") + source = os.environ.get("CI_PIPELINE_SOURCE", "push") + self.event_name = "pull_request" if source == "merge_request_event" else source + branch = os.environ.get("CI_COMMIT_REF_NAME", "") + self.ref = f"refs/heads/{branch}" if branch else "" self.pr_title = "" self.is_gitlab = True else: @@ -405,6 +407,9 @@ def process(inputs: Inputs) -> list[str]: library_processor.selected |= scenario_processor.impacted_libraries if inputs.is_gitlab: + libraries = " ".join(sorted(library_processor.selected)) + if libraries: + outputs["libraries"] = libraries outputs |= scenario_processor.get_outputs() else: outputs |= ( diff --git a/utils/scripts/update_mirror_images.py b/utils/scripts/update_mirror_images.py new file mode 100644 index 00000000000..75ddd5968e5 --- /dev/null +++ b/utils/scripts/update_mirror_images.py @@ -0,0 +1,160 @@ +"""Regenerate the mirror-images manifest and lock file from the CI scenarios. + +Run from a system-tests checkout, with the runner venv active: + + python utils/scripts/update_mirror_images.py + +It unions every Docker image required by the CI-run ``DockerScenario`` instances +(across the full library x weblog matrix), records them in ``mirror_images.yaml`` +via the dd-repo-tools ``mirror_images.py add`` command, then resolves digests +into ``mirror_images.lock.yaml`` via ``lock``. + +It never pushes or mirrors anything: commit the updated ``mirror_images.yaml`` +and ``mirror_images.lock.yaml`` yourself. + +Companion to ``utils/scripts/get-image-list.py`` (which emits a docker-compose +file of images to pull for a *single* scenario/library/weblog); this script is +docker-free for the enumeration step and covers *all* CI scenarios at once. +``lock`` does require a registry tool (crane/skopeo/docker) and network access. +""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# Make `utils` importable and resolve paths regardless of the caller's cwd, so a +# plain `python utils/scripts/update_mirror_images.py` works. +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT)) + +from utils._context._scenarios import get_all_scenarios, DockerScenario # noqa: E402 + +# Pinned dd-repo-tools mirror_images.py (override with $MIRROR_IMAGES_URL). +MIRROR_IMAGES_URL = os.environ.get( + "MIRROR_IMAGES_URL", + "https://binaries.ddbuild.io/dd-repo-tools/default/ca/d6c8f8ec4b1fb69b36e96dc204e5f79bed6bf067/mirror_images.py", +) + +# Destination registry for the mirrored images (override with $MIRROR_DEST_REGISTRY). +DEFAULT_DEST_REGISTRY = "registry.ddbuild.io/system-tests/mirror" + +MIRROR_YAML = REPO_ROOT / "mirror_images.yaml" +BUILDKITD_TOML = REPO_ROOT / "utils" / "build" / "docker" / "buildkitd.toml" + +# Header written when mirror_images.yaml does not exist yet. The mirror_images.py +# `add` command preserves existing comments, so this is only used on first run. +MIRROR_YAML_HEADER = """\ +# Docker images mirrored into registry.ddbuild.io/system-tests/mirror. +# +# Generated: this file lists every image required by the CI scenarios. +# Regenerate after changing scenarios or weblog Dockerfiles with: +# +# python utils/scripts/update_mirror_images.py +# +# The `mirror_images_check` CI job fails if this file is out of date. +""" + +# Scenarios excluded from the GitLab end-to-end pipeline. Mirrors the +# `excluded_scenarios` input in .gitlab-ci.yml so the mirrored image set matches +# what actually runs in CI. Override with --exclude when the CI list changes. +DEFAULT_EXCLUDED = ( + "DEBUGGER_EXPRESSION_LANGUAGE", + "APM_TRACING_E2E_SINGLE_SPAN", + "APM_TRACING_E2E_OTEL", + "OTEL_COLLECTOR_E2E", +) + + +def _library_weblog_pairs() -> list[tuple[str, str]]: + """Every (library, weblog) pair, derived from the weblog Dockerfiles. + + WeblogContainer.get_image_list reads utils/build/docker//.Dockerfile, + so the directory name is the library and the file stem is the weblog. The + empty pair captures library-independent images (agent, integrations, + buddies) that every get_image_list call returns regardless of arguments. + """ + pairs = {("", "")} + for path in Path("utils/build/docker").glob("*/*.Dockerfile"): + pairs.add((path.parent.name, path.stem)) + return sorted(pairs) + + +def _is_mirrorable(image: str) -> bool: + """Keep only images that can be pulled from a public registry and mirrored.""" + if image.startswith("system_tests/"): + return False # built locally, never pulled from a registry + if "$" in image: + return False # unresolved Dockerfile/template placeholder (e.g. ${NORMALIZED_BRANCH}) + # already on the destination registry => nothing to mirror + return not image.startswith("registry.ddbuild.io/") + + +def collect_images(excluded: set[str]) -> list[str]: + pairs = _library_weblog_pairs() + images: set[str] = set() + for scenario in get_all_scenarios(): + if not isinstance(scenario, DockerScenario) or scenario.name in excluded: + continue + for library, weblog in pairs: + images.update(scenario.get_image_list(library, weblog)) + return sorted(image for image in images if _is_mirrorable(image)) + + +def _run_mirror_images(*args: str) -> None: + """Invoke the pinned dd-repo-tools mirror_images.py via uv.""" + if shutil.which("uv") is None: + sys.exit("error: 'uv' is required to run mirror_images.py — install it from https://docs.astral.sh/uv/") + env = dict(os.environ) + env.setdefault("MIRROR_DEST_REGISTRY", DEFAULT_DEST_REGISTRY) + cmd = [ + "uv", + "run", + "--no-config", + "--script", + MIRROR_IMAGES_URL, + "--mirror-yaml", + str(MIRROR_YAML), + *args, + ] + print(f"+ {' '.join(cmd)}", flush=True) + subprocess.run(cmd, check=True, env=env) + + +def main(excluded: set[str], *, skip_lock: bool) -> None: + # get_image_list reads Dockerfiles relative to the repo root. + os.chdir(REPO_ROOT) + + images = collect_images(excluded) + print(f"Collected {len(images)} mirrorable image(s) from the CI scenarios.", flush=True) + + if not MIRROR_YAML.exists(): + MIRROR_YAML.write_text(MIRROR_YAML_HEADER) + + _run_mirror_images("add", *images) + if not skip_lock: + _run_mirror_images("lock") + _run_mirror_images("buildkitd", "--output", str(BUILDKITD_TOML)) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="update_mirror_images", + description="Regenerate mirror_images.yaml and mirror_images.lock.yaml from the CI scenarios", + ) + parser.add_argument( + "--exclude", + type=str, + default=",".join(DEFAULT_EXCLUDED), + help="Comma-separated scenario names to exclude (default: the CI excluded_scenarios set)", + ) + parser.add_argument( + "--skip-lock", + action="store_true", + help="Only update mirror_images.yaml; do not resolve digests into the lock file " + "(used by the CI drift check, which has no registry credentials)", + ) + args = parser.parse_args() + main({name.strip() for name in args.exclude.split(",") if name.strip()}, skip_lock=args.skip_lock)