GPU Tests #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: GPU Tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| instance_type: | |
| description: 'EC2 instance type' | |
| required: false | |
| type: choice | |
| default: 'g6.2xlarge' | |
| options: | |
| - g5.xlarge # 4 vCPUs, 16GB RAM, A10G GPU, ≈$1.11/hr | |
| - g5.2xlarge # 8 vCPUs, 32GB RAM, A10G GPU, ≈$1.33/hr | |
| - g5.4xlarge # 16 vCPUs, 64GB RAM, A10G GPU, ≈$1.79/hr | |
| - g6.xlarge # 4 vCPUs, 16GB RAM, L4 GPU, ≈$0.89/hr | |
| - g6.2xlarge # 8 vCPUs, 32GB RAM, L4 GPU, ≈$1.08/hr | |
| - g6.4xlarge # 16 vCPUs, 64GB RAM, L4 GPU, ≈$1.46/hr | |
| pull_request: | |
| paths: | |
| - 'cubed/backend_array_api.py' | |
| - 'cubed/storage/virtual.py' | |
| - 'cubed/tests/**' | |
| - '.github/workflows/gpu-tests.yml' | |
| - '.github/scripts/**' | |
| - 'pyproject.toml' | |
| schedule: | |
| # Run weekly on Mondays at 03:00 UTC | |
| - cron: "0 3 * * 1" | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| ec2: | |
| name: Start EC2 runner | |
| uses: Open-Athena/ec2-gha/.github/workflows/runner.yml@v2 | |
| with: | |
| ec2_instance_type: ${{ inputs.instance_type || 'g6.2xlarge' }} | |
| ec2_image_id: ami-0aee7b90d684e107d # Deep Learning OSS Nvidia Driver AMI GPU PyTorch 2.4.1 (Ubuntu 22.04) | |
| secrets: | |
| GH_SA_TOKEN: ${{ secrets.GH_SA_TOKEN }} | |
| gpu-test-cupy: | |
| name: GPU Tests (CuPy) | |
| needs: ec2 | |
| runs-on: ${{ needs.ec2.outputs.id }} | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check GPU | |
| run: nvidia-smi | |
| - name: Setup Python environment | |
| run: | | |
| # Use the DLAMI's pre-installed PyTorch conda environment | |
| echo "/opt/conda/envs/pytorch/bin" >> $GITHUB_PATH | |
| echo "CONDA_DEFAULT_ENV=pytorch" >> $GITHUB_ENV | |
| - name: Verify Python | |
| run: python --version | |
| - name: Install dependencies | |
| run: | | |
| pip install -e .[test] | |
| pip install cupy-cuda12x array-api-compat | |
| - name: Verify CuPy installation | |
| run: | | |
| python -c "import cupy; print(f'CuPy version: {cupy.__version__}')" | |
| python -c "import cupy; cupy.cuda.Device(0).compute_capability" | |
| - name: Test CuPy with cubed (basic functionality) | |
| env: | |
| CUBED_BACKEND_ARRAY_API_MODULE: "array_api_compat.cupy" | |
| run: | | |
| echo "Running basic GPU functionality tests..." | |
| python cubed/tests/test_gpu_basic.py 2>&1 | tee basic_test_results.txt | |
| echo "Exit code: $?" | |
| - name: Run array API tests with CuPy | |
| env: | |
| CUBED_BACKEND_ARRAY_API_MODULE: "array_api_compat.cupy" | |
| continue-on-error: true | |
| run: | | |
| # Run tests that are known to work with CuPy | |
| # Skip processes executor which doesn't work with CuPy | |
| # Use -k to select specific tests by pattern | |
| pytest -v cubed/tests/test_array_api.py \ | |
| -k "test_arange or test_ones or test_asarray" \ | |
| --ignore-glob="*processes*" \ | |
| --maxfail=5 2>&1 | tee pytest_results.txt || echo "Note: Some tests failed (investigating GPU/zarr compatibility)" | |
| - name: Document known issues | |
| if: always() | |
| run: | | |
| chmod +x .github/scripts/document_gpu_issues.sh | |
| .github/scripts/document_gpu_issues.sh | |
| gpu-performance-test: | |
| name: GPU Performance Comparison | |
| needs: ec2 | |
| runs-on: ${{ needs.ec2.outputs.id }} | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v4 | |
| - name: Setup Python environment | |
| run: | | |
| echo "/opt/conda/envs/pytorch/bin" >> $GITHUB_PATH | |
| echo "CONDA_DEFAULT_ENV=pytorch" >> $GITHUB_ENV | |
| - name: Install dependencies | |
| run: | | |
| pip install -e .[test] | |
| pip install cupy-cuda12x array-api-compat | |
| - name: Run performance comparison | |
| run: | | |
| echo "Running GPU performance benchmarks..." | |
| python cubed/tests/benchmark_gpu.py \ | |
| --instance-type "${{ inputs.instance_type || 'g6.2xlarge' }}" \ | |
| 2>&1 | tee performance_results.txt | |
| echo "Benchmark exit code: $?" | |
| - name: Create test summary | |
| if: always() | |
| run: | | |
| chmod +x .github/scripts/create_test_summary.sh | |
| .github/scripts/create_test_summary.sh "${{ inputs.instance_type || 'g6.2xlarge' }}" | |
| - name: Upload test artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gpu-test-results | |
| path: | | |
| test_summary.md | |
| performance_results.txt | |
| cupy_test_results.md | |
| basic_test_results.txt | |
| pytest_results.txt |