From 98e0b5994aecd9a5f7bbe65c671b0abb731acc10 Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 3 Dec 2025 11:06:11 -0500 Subject: [PATCH] Introduce build-and-test-python-wheels workflow --- .../build-and-test-python-wheels.yml | 110 ++++++++++++ .github/workflows/pr.yml | 163 ++++++++---------- 2 files changed, 183 insertions(+), 90 deletions(-) create mode 100644 .github/workflows/build-and-test-python-wheels.yml diff --git a/.github/workflows/build-and-test-python-wheels.yml b/.github/workflows/build-and-test-python-wheels.yml new file mode 100644 index 0000000..a1ad77a --- /dev/null +++ b/.github/workflows/build-and-test-python-wheels.yml @@ -0,0 +1,110 @@ +name: Build and Test Python Wheels + +on: + workflow_call: + workflow_dispatch: + +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} + +jobs: + # Build wheels for all CUDA/Python combinations + build-wheels: + name: Build wheel (CUDA ${{ matrix.cuda }}, Python ${{ matrix.python }}) + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + cuda: ['12', '13'] + python: ['3.10', '3.11', '3.12', '3.13'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build wheel + run: | + bash ci/build_pynvbench_wheel.sh -py-version ${{ matrix.python }} -cuda-version ${{ matrix.cuda }} + + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 + with: + name: wheel-pynvbench-cu${{ matrix.cuda }}-py${{ matrix.python }} + path: wheelhouse/*.whl + retention-days: 7 + if-no-files-found: error + + # Test wheels for all CUDA/Python combinations + test-wheels: + name: Test wheel (CUDA ${{ matrix.cuda }}, Python ${{ matrix.python }}) + needs: build-wheels + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + cuda: ['12', '13'] + python: ['3.10', '3.11', '3.12', '3.13'] + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Download wheel artifact + uses: actions/download-artifact@v4 + with: + name: wheel-pynvbench-cu${{ matrix.cuda }}-py${{ matrix.python }} + path: /home/coder/nvbench/wheel-pynvbench-cu${{ matrix.cuda }}-py${{ matrix.python }} + + - name: Test wheel + run: | + # Use the same rapidsai/ci-wheel Docker image as build + if [[ "${{ matrix.cuda }}" == "12" ]]; then + cuda_full_version="12.9.1" + else + cuda_full_version="13.0.1" + fi + + docker run --rm \ + --workdir /home/coder/nvbench \ + --mount type=bind,source=$(pwd),target=/home/coder/nvbench/ \ + --env py_version=${{ matrix.python }} \ + --env cuda_version=${{ matrix.cuda }} \ + rapidsai/ci-wheel:25.12-cuda${cuda_full_version}-rockylinux8-py${{ matrix.python }} \ + /home/coder/nvbench/ci/test_pynvbench.sh -py-version ${{ matrix.python }} -cuda-version ${{ matrix.cuda }} + + verify-workflow: + name: Verify all builds and tests succeeded + if: ${{ always() }} + needs: + - build-wheels + - test-wheels + runs-on: ubuntu-latest + steps: + - name: Check build results + run: | + if [[ "${{ needs.build-wheels.result }}" != "success" ]]; then + echo "Wheel builds failed!" + exit 1 + fi + if [[ "${{ needs.test-wheels.result }}" != "success" ]]; then + echo "Wheel tests failed!" + exit 1 + fi + echo "All wheels built and tested successfully!" + diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7c37075..c42cda0 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -14,119 +14,102 @@ # limitations under the License. # This is the main workflow that runs on every PR and push to main -name: Pull Request +name: pr defaults: run: shell: bash -euo pipefail {0} on: - pull_request: - types: [opened, synchronize, reopened] push: branches: - - main + - "pull-request/[0-9]+" -defaults: - run: - shell: bash --noprofile --norc -euo pipefail {0} +# Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts. +concurrency: + group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: read jobs: - # Build wheels for all CUDA/Python combinations - build-wheels: - name: Build wheel (CUDA ${{ matrix.cuda }}, Python ${{ matrix.python }}) + compute-matrix: + name: Compute matrix runs-on: ubuntu-latest + outputs: + DEVCONTAINER_VERSION: ${{steps.set-outputs.outputs.DEVCONTAINER_VERSION}} + PER_CUDA_COMPILER_MATRIX: ${{steps.set-outputs.outputs.PER_CUDA_COMPILER_MATRIX}} + PER_CUDA_COMPILER_KEYS: ${{steps.set-outputs.outputs.PER_CUDA_COMPILER_KEYS}} + base_sha: ${{ steps.export-pr-info.outputs.base_sha }} + pr_number: ${{ steps.export-pr-info.outputs.pr_number }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Lookup PR info + id: get-pr-info + uses: nv-gha-runners/get-pr-info@main + - name: Export PR info + id: export-pr-info + run: | + echo "base_sha=${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }}" | tee -a "${GITHUB_OUTPUT}" + echo "pr_number=${{ fromJSON(steps.get-pr-info.outputs.pr-info).number }}" | tee -a "${GITHUB_OUTPUT}" + - name: Compute matrix outputs + id: set-outputs + run: | + .github/actions/compute-matrix/compute-matrix.sh ci/matrix.yaml pull_request + + nvbench: + name: NVBench CUDA${{ matrix.cuda_host_combination }} permissions: id-token: write contents: read + needs: compute-matrix + uses: ./.github/workflows/dispatch-build-and-test.yml strategy: fail-fast: false matrix: - cuda: ['12', '13'] - python: ['3.10', '3.11', '3.12', '3.13'] + cuda_host_combination: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_KEYS) }} + with: + project_name: "nvbench" + per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ matrix.cuda_host_combination ]) }} + devcontainer_version: ${{ needs.compute-matrix.outputs.DEVCONTAINER_VERSION }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build wheel - run: | - bash ci/build_pynvbench_wheel.sh -py-version ${{ matrix.python }} -cuda-version ${{ matrix.cuda }} - - - name: Upload wheel artifact - uses: actions/upload-artifact@v4 - with: - name: wheel-pynvbench-cu${{ matrix.cuda }}-py${{ matrix.python }} - path: wheelhouse/*.whl - retention-days: 7 - if-no-files-found: error - - # Test wheels for all CUDA/Python combinations - test-wheels: - name: Test wheel (CUDA ${{ matrix.cuda }}, Python ${{ matrix.python }}) - needs: build-wheels - runs-on: ubuntu-latest + python-wheels: + name: Python Wheels permissions: id-token: write contents: read - strategy: - fail-fast: false - matrix: - cuda: ['12', '13'] - python: ['3.10', '3.11', '3.12', '3.13'] + uses: ./.github/workflows/build-and-test-python-wheels.yml - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false + verify-devcontainers: + name: Verify Dev Containers + if: ${{ !contains(github.event.head_commit.message, '[skip-vdc]') }} + needs: compute-matrix + permissions: + id-token: write + contents: read + uses: ./.github/workflows/verify-devcontainers.yml + with: + base_sha: ${{ needs.compute-matrix.outputs.base_sha }} - - name: Download wheel artifact - uses: actions/download-artifact@v4 - with: - name: wheel-pynvbench-cu${{ matrix.cuda }}-py${{ matrix.python }} - path: /home/coder/nvbench/wheel-pynvbench-cu${{ matrix.cuda }}-py${{ matrix.python }} - - - name: Test wheel - run: | - # Use the same rapidsai/ci-wheel Docker image as build - if [[ "${{ matrix.cuda }}" == "12" ]]; then - cuda_full_version="12.9.1" - else - cuda_full_version="13.0.1" - fi - - docker run --rm \ - --workdir /home/coder/nvbench \ - --mount type=bind,source=$(pwd),target=/home/coder/nvbench/ \ - --env py_version=${{ matrix.python }} \ - --env cuda_version=${{ matrix.cuda }} \ - rapidsai/ci-wheel:25.12-cuda${cuda_full_version}-rockylinux8-py${{ matrix.python }} \ - /home/coder/nvbench/ci/test_pynvbench.sh -py-version ${{ matrix.python }} -cuda-version ${{ matrix.cuda }} - - verify-workflow: - name: Verify all builds and tests succeeded - if: ${{ always() }} + # This job is the final job that runs after all other jobs and is used for branch protection status checks. + # See: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks + # https://github.com/orgs/community/discussions/26822#discussioncomment-5122101 + ci: + runs-on: ubuntu-latest + name: CI + if: ${{ always() }} # need to use always() instead of !cancelled() because skipped jobs count as success needs: - - build-wheels - - test-wheels - runs-on: ubuntu-latest + - nvbench + - python-wheels + - verify-devcontainers steps: - - name: Check build results - run: | - if [[ "${{ needs.build-wheels.result }}" != "success" ]]; then - echo "Wheel builds failed!" - exit 1 - fi - if [[ "${{ needs.test-wheels.result }}" != "success" ]]; then - echo "Wheel tests failed!" - exit 1 - fi - echo "All wheels built and tested successfully!" + - name: Check status of all precursor jobs + if: >- + ${{ + contains(needs.*.result, 'failure') + || contains(needs.*.result, 'cancelled') + }} + run: exit 1