diff --git a/.github/workflows/get-test-runner.yml b/.github/workflows/get-test-runner.yml new file mode 100644 index 0000000000..b185141bf0 --- /dev/null +++ b/.github/workflows/get-test-runner.yml @@ -0,0 +1,41 @@ +on: + workflow_call: + inputs: + runner_type: + description: 'The type of test runners' + required: true + type: string + outputs: + matrix: + description: "The list of runners" + value: ${{ jobs.get_runners.outputs.matrix }} +jobs: + get_runners: + name: "get_platforms type=${{inputs.runner_type}}" + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: SCM Checkout + uses: actions/checkout@v6 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v5 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + + - id: set-matrix + run: | + RUNNERS=`poetry run -- nox --non-interactive -s 'get-test-runners' -- --type=$RUNNER_TYPE` + echo "$RUNNERS" + echo "matrix=$RUNNERS" >> "$GITHUB_OUTPUT" + env: + RUNNER_TYPE: ${{ inputs.runner_type }} + + - name: Print matrix + run: echo ${{ steps.set-matrix.outputs.matrix }} + + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/.github/workflows/slow-tests.yml b/.github/workflows/slow-tests.yml index 6e005ba98c..b739063a0f 100644 --- a/.github/workflows/slow-tests.yml +++ b/.github/workflows/slow-tests.yml @@ -22,3 +22,10 @@ jobs: permissions: contents: read uses: ./.github/workflows/test-db-versions-gpu.yml + +# TODO: https://github.com/exasol/integration-tasks/issues/815 +# additional-platforms-tests: +# name: Run on additional platforms +# permissions: +# contents: read +# uses: ./.github/workflows/test-db-versions-additional-platforms.yml diff --git a/.github/workflows/test-db-versions-additional-platforms.yml b/.github/workflows/test-db-versions-additional-platforms.yml new file mode 100644 index 0000000000..798ce2d95d --- /dev/null +++ b/.github/workflows/test-db-versions-additional-platforms.yml @@ -0,0 +1,48 @@ +name: Test DB Versions with all tests + +# python-version: default as specified +# database-version: Default +# tests: all +# platform: all except default platform +# +# Use the specified default version of python and call nox task run_all_tests for +# all versions of Exasol database currently supported by the ITDE. + +on: + workflow_call: + +jobs: + get_test_runners: + name: Read all additional test runners + permissions: + contents: read + uses: ./.github/workflows/get-test-runner.yml + with: + runner_type: all_except_default + + test-add-platforms-all-tests: + needs: get_test_runners + strategy: + fail-fast: false + matrix: + runner: ${{fromJson(needs.get_test_runners.outputs.matrix)}} + runs-on: ${{matrix.runner}} + permissions: + contents: read + name: Run all tests on ${{matrix.runner}} + + steps: + - name: SCM Checkout + uses: actions/checkout@v6 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v5 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Allow unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Run all tests + run: poetry run -- nox -s run-all-tests -- --test-set=default diff --git a/.github/workflows/test-db-versions-all-tests.yml b/.github/workflows/test-db-versions-all-tests.yml index 2f3cba9067..88dbbbbe98 100644 --- a/.github/workflows/test-db-versions-all-tests.yml +++ b/.github/workflows/test-db-versions-all-tests.yml @@ -3,6 +3,7 @@ name: Test DB Versions with all tests # python-version: default as specified # database-version: all supported by ITDE # tests: all +# platform: default # # Use the specified default version of python and call nox task run_all_tests for # all versions of Exasol database currently supported by the ITDE. @@ -11,6 +12,15 @@ on: workflow_call: jobs: + + get_test_runners: + name: Read all additional test runners + permissions: + contents: read + uses: ./.github/workflows/get-test-runner.yml + with: + runner_type: only_default + get_exasol_versions: name: Read all supported Exasol DB versions permissions: @@ -18,15 +28,18 @@ jobs: uses: ./.github/workflows/get-exasol-versions.yml test-db-versions-all-tests: - needs: get_exasol_versions + needs: + - get_exasol_versions + - get_test_runners strategy: fail-fast: false matrix: exasol_version: ${{fromJson(needs.get_exasol_versions.outputs.matrix)}} - runs-on: ubuntu-24.04 + runner: ${{fromJson(needs.get_test_runners.outputs.matrix)}} + runs-on: ${{matrix.runner}} permissions: contents: read - name: Run all tests for Exasol ${{ matrix.exasol_version }} + name: Run all tests for Exasol ${{ matrix.exasol_version }} on ${{matrix.runner}} steps: - name: SCM Checkout diff --git a/.github/workflows/test-db-versions-minimal.yml b/.github/workflows/test-db-versions-minimal.yml index 95745fd721..9da135c300 100644 --- a/.github/workflows/test-db-versions-minimal.yml +++ b/.github/workflows/test-db-versions-minimal.yml @@ -3,6 +3,7 @@ name: Test DB Versions with minimal tests # python-version: default as specified # database-version: all supported by ITDE # tests: minimal (subset of all tests) +# platform: all # # Use the specified default version of python and call nox task # run_minimal_tests to execute a few selected tests for all versions of Exasol @@ -12,19 +13,30 @@ on: workflow_call: jobs: + get_test_runners: + name: Read all additional test runners + permissions: + contents: read + uses: ./.github/workflows/get-test-runner.yml + with: + runner_type: all + get_exasol_versions: permissions: contents: read uses: ./.github/workflows/get-exasol-versions.yml test-db-versions-minimal: - name: Run Minimal Tests for Exasol ${{ matrix.exasol_version }} - needs: get_exasol_versions + name: Run Minimal Tests for Exasol ${{ matrix.exasol_version }} on ${{matrix.runner}} + needs: + - get_exasol_versions + - get_test_runners strategy: fail-fast: false matrix: exasol_version: ${{fromJson(needs.get_exasol_versions.outputs.matrix)}} - runs-on: ubuntu-24.04 + runner: ${{fromJson(needs.get_test_runners.outputs.matrix)}} + runs-on: ${{matrix.runner}} permissions: contents: read steps: diff --git a/.github/workflows/test-python-versions.yml b/.github/workflows/test-python-versions.yml index 567a642a61..3821bb7ef4 100644 --- a/.github/workflows/test-python-versions.yml +++ b/.github/workflows/test-python-versions.yml @@ -3,6 +3,7 @@ name: Test Python Versions with all tests # python-version: All supported Python version # database-version: Default # tests: all +# platform: default # # Call nox task run_tests for all version of Python as defined in the # python-toolbox (PTB) using the default version of the Exasol database. @@ -11,6 +12,13 @@ on: workflow_call: jobs: + get_test_runners: + name: Read all additional test runners + permissions: + contents: read + uses: ./.github/workflows/get-test-runner.yml + with: + runner_type: only_default build-matrix: name: Generate Build Matrix @@ -19,14 +27,16 @@ jobs: uses: ./.github/workflows/matrix-python.yml test-python-version-all-tests: - name: "Run Tests with Python ${{ matrix.python-version }} and Default Exasol Version" + name: "Run Tests with Python ${{ matrix.python-version }} and Default Exasol Version on ${{matrix.runner}}" permissions: contents: read - needs: [ build-matrix ] + needs: + - get_test_runners + - build-matrix strategy: fail-fast: false matrix: ${{ fromJson(needs.build-matrix.outputs.matrix) }} - runs-on: ubuntu-24.04 + runs-on: ${{ fromJson(needs.get_test_runners.outputs.matrix) }} steps: - name: SCM Checkout uses: actions/checkout@v6 diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 0a5b3f30a6..e5d11c5f9d 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -3,6 +3,7 @@ ## Features - #558: Added docker-db 2025.1.8 + - #583: Added platform support for managed docker images ## Bugs diff --git a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py index 97813487b2..85bdff77ed 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/create/docker_image_analyze_task.py @@ -23,6 +23,7 @@ ImageDescription, ImageInfo, ImageState, + current_platform, ) from exasol_integration_test_docker_environment.lib.models.config.build_config import ( build_config, @@ -185,6 +186,7 @@ def run_task(self) -> None: depends_on_images=image_info_of_dependencies, image_state=None, image_description=self.image_description, + platform=current_platform(), ) target_image_target = DockerImageTarget( self._target_repository_name, image_info.get_target_complete_tag() diff --git a/exasol_integration_test_docker_environment/lib/docker/images/image_info.py b/exasol_integration_test_docker_environment/lib/docker/images/image_info.py index e38f59b59b..c279582ea4 100644 --- a/exasol_integration_test_docker_environment/lib/docker/images/image_info.py +++ b/exasol_integration_test_docker_environment/lib/docker/images/image_info.py @@ -10,6 +10,23 @@ from exasol_integration_test_docker_environment.lib.base.info import Info +class Platform(Enum): + X64 = "x64" + ARM_64 = "arm64" + + +def current_platform() -> Platform: + import platform + + supported_platforms = { + "x86_64": Platform.X64, + "amd64": Platform.X64, + "aarch64": Platform.ARM_64, + "arm64": Platform.ARM_64, + } + return supported_platforms[platform.machine()] + + class ImageState(Enum): NOT_EXISTING = auto() # After analyze phase or if build phase did touch the image @@ -62,6 +79,7 @@ def __init__( build_date_time: datetime = datetime.utcnow(), image_state: ImageState | None = ImageState.NOT_EXISTING, depends_on_images: dict[str, "ImageInfo"] | None = None, + platform: Platform | None = None, ) -> None: self.build_name = build_name self.date_time = str(build_date_time) @@ -80,6 +98,7 @@ def __init__( self.source_tag = source_tag self.target_tag = target_tag self.hash = hash_value + self.platform = platform self.check_complete_tag_length(self.source_tag) self.check_complete_tag_length(self.target_tag) @@ -111,7 +130,9 @@ def _create_truncated_complete_tag(self, tag: str) -> str: return truncated_tag def _create_complete_tag(self, tag): - if self.hash == "": - return f"{tag}" - else: - return f"{tag}_{self.hash}" + tag_elements = [tag] + if self.platform: + tag_elements.append(self.platform.value) + if self.hash: + tag_elements.append(self.hash) + return "_".join(tag_elements) diff --git a/noxconfig.py b/noxconfig.py index 9faf74c567..a025cb4dc8 100644 --- a/noxconfig.py +++ b/noxconfig.py @@ -3,7 +3,15 @@ from pathlib import Path from exasol.toolbox.config import BaseConfig -from pydantic import computed_field +from pydantic import ( + BaseModel, + computed_field, +) + + +class TestRunnersConfig(BaseModel): + default_runner: str + test_runners: list[str] class Config(BaseConfig): @@ -19,10 +27,18 @@ def source_code_path(self) -> Path: """ return self.root_path / self.project_name + test_runners_config: TestRunnersConfig + PROJECT_CONFIG = Config( root_path=Path(__file__).parent, project_name="exasol_integration_test_docker_environment", python_versions=("3.10", "3.11", "3.12", "3.13"), add_to_excluded_python_paths=("resources",), + test_runners_config=TestRunnersConfig( + default_runner="ubuntu-24.04", + test_runners=[ + "ubuntu-24.04" + ], # TODO https://github.com/exasol/integration-tasks/issues/815 + ), ) diff --git a/noxfile.py b/noxfile.py index 7da00d22b0..787f5ccd63 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,7 +19,7 @@ from exasol.toolbox.nox.tasks import * # type: ignore # default actions to be run if nothing is explicitly specified with the -s option -nox.options.sessions = ["project:fix"] +nox.options.sessions = ["format:fix"] class TestSet(Enum): @@ -305,3 +305,37 @@ def execute_itde(session: nox.Session): session.run( f"{exe_name}", "spawn-test-environment", "--environment-name", "test_01" ) + + +class TestRunnerType(Enum): + ONLY_DEFAULT = "only_default" + ALL = "all" + ALL_EXCEPT_DEFAULT = "all_except_default" + + +@nox.session(name="get-test-runners", python=False) +def get_test_runners(session: nox.Session): + + p = ArgumentParser( + usage="nox -s get-test-runners -- --type=only_default|all|all_except_default", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument("--type", choices=[str(t.value) for t in TestRunnerType]) + args = p.parse_args(session.posargs) + + test_runner_config = PROJECT_CONFIG.test_runners_config + + if args.type == "only_default": + print(json.dumps(test_runner_config.default_runner)) + elif args.type == "all_except_default": + result = [ + runner + for runner in test_runner_config.test_runners + if runner != test_runner_config.default_runner + ] + print(json.dumps(result)) + + elif args.type == "all": + print(json.dumps(test_runner_config.test_runners)) + else: + raise ValueError(f"Invalid test runner type '{args.type}'")