From e9cf53a1a4ccf1bc2e8bd76dfd515d34c026550a Mon Sep 17 00:00:00 2001 From: Ashwin Srinath Date: Wed, 3 Dec 2025 10:30:27 -0500 Subject: [PATCH] Add PR workflow for building and testing wheels --- .github/workflows/build-wheels.yml | 3 +- .github/workflows/pr.yml | 159 +++++++++++++++++------------ ci/test_pynvbench.sh | 72 +++++++++++++ 3 files changed, 166 insertions(+), 68 deletions(-) create mode 100755 ci/test_pynvbench.sh diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index bd52be1..37dfb3d 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -1,7 +1,8 @@ -name: Build Python Wheels +name: Build Python Wheels (Manual) on: workflow_dispatch: + workflow_call: defaults: run: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9ff31e0..7c37075 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -14,94 +14,119 @@ # limitations under the License. # This is the main workflow that runs on every PR and push to main -name: pr +name: Pull Request defaults: run: shell: bash -euo pipefail {0} on: + pull_request: + types: [opened, synchronize, reopened] push: branches: - - "pull-request/[0-9]+" + - main -# Only runs one instance of this workflow at a time for a given PR and cancels any in-progress runs when a new one starts. -concurrency: - group: ${{ github.workflow }}-on-${{ github.event_name }}-from-${{ github.ref_name }} - cancel-in-progress: true - -permissions: - contents: read - pull-requests: read +defaults: + run: + shell: bash --noprofile --norc -euo pipefail {0} jobs: - compute-matrix: - name: Compute matrix + # Build wheels for all CUDA/Python combinations + build-wheels: + name: Build wheel (CUDA ${{ matrix.cuda }}, Python ${{ matrix.python }}) runs-on: ubuntu-latest - outputs: - DEVCONTAINER_VERSION: ${{steps.set-outputs.outputs.DEVCONTAINER_VERSION}} - PER_CUDA_COMPILER_MATRIX: ${{steps.set-outputs.outputs.PER_CUDA_COMPILER_MATRIX}} - PER_CUDA_COMPILER_KEYS: ${{steps.set-outputs.outputs.PER_CUDA_COMPILER_KEYS}} - base_sha: ${{ steps.export-pr-info.outputs.base_sha }} - pr_number: ${{ steps.export-pr-info.outputs.pr_number }} - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - name: Lookup PR info - id: get-pr-info - uses: nv-gha-runners/get-pr-info@main - - name: Export PR info - id: export-pr-info - run: | - echo "base_sha=${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }}" | tee -a "${GITHUB_OUTPUT}" - echo "pr_number=${{ fromJSON(steps.get-pr-info.outputs.pr-info).number }}" | tee -a "${GITHUB_OUTPUT}" - - name: Compute matrix outputs - id: set-outputs - run: | - .github/actions/compute-matrix/compute-matrix.sh ci/matrix.yaml pull_request - - nvbench: - name: NVBench CUDA${{ matrix.cuda_host_combination }} permissions: id-token: write contents: read - needs: compute-matrix - uses: ./.github/workflows/dispatch-build-and-test.yml strategy: fail-fast: false matrix: - cuda_host_combination: ${{ fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_KEYS) }} - with: - project_name: "nvbench" - per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ matrix.cuda_host_combination ]) }} - devcontainer_version: ${{ needs.compute-matrix.outputs.DEVCONTAINER_VERSION }} + cuda: ['12', '13'] + python: ['3.10', '3.11', '3.12', '3.13'] - verify-devcontainers: - name: Verify Dev Containers - if: ${{ !contains(github.event.head_commit.message, '[skip-vdc]') }} - needs: compute-matrix + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build wheel + run: | + bash ci/build_pynvbench_wheel.sh -py-version ${{ matrix.python }} -cuda-version ${{ matrix.cuda }} + + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 + with: + name: wheel-pynvbench-cu${{ matrix.cuda }}-py${{ matrix.python }} + path: wheelhouse/*.whl + retention-days: 7 + if-no-files-found: error + + # Test wheels for all CUDA/Python combinations + test-wheels: + name: Test wheel (CUDA ${{ matrix.cuda }}, Python ${{ matrix.python }}) + needs: build-wheels + runs-on: ubuntu-latest permissions: id-token: write contents: read - uses: ./.github/workflows/verify-devcontainers.yml - with: - base_sha: ${{ needs.compute-matrix.outputs.base_sha }} + strategy: + fail-fast: false + matrix: + cuda: ['12', '13'] + python: ['3.10', '3.11', '3.12', '3.13'] - # This job is the final job that runs after all other jobs and is used for branch protection status checks. - # See: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks - # https://github.com/orgs/community/discussions/26822#discussioncomment-5122101 - ci: - runs-on: ubuntu-latest - name: CI - if: ${{ always() }} # need to use always() instead of !cancelled() because skipped jobs count as success - needs: - - nvbench - - verify-devcontainers steps: - - name: Check status of all precursor jobs - if: >- - ${{ - contains(needs.*.result, 'failure') - || contains(needs.*.result, 'cancelled') - }} - run: exit 1 + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Download wheel artifact + uses: actions/download-artifact@v4 + with: + name: wheel-pynvbench-cu${{ matrix.cuda }}-py${{ matrix.python }} + path: /home/coder/nvbench/wheel-pynvbench-cu${{ matrix.cuda }}-py${{ matrix.python }} + + - name: Test wheel + run: | + # Use the same rapidsai/ci-wheel Docker image as build + if [[ "${{ matrix.cuda }}" == "12" ]]; then + cuda_full_version="12.9.1" + else + cuda_full_version="13.0.1" + fi + + docker run --rm \ + --workdir /home/coder/nvbench \ + --mount type=bind,source=$(pwd),target=/home/coder/nvbench/ \ + --env py_version=${{ matrix.python }} \ + --env cuda_version=${{ matrix.cuda }} \ + rapidsai/ci-wheel:25.12-cuda${cuda_full_version}-rockylinux8-py${{ matrix.python }} \ + /home/coder/nvbench/ci/test_pynvbench.sh -py-version ${{ matrix.python }} -cuda-version ${{ matrix.cuda }} + + verify-workflow: + name: Verify all builds and tests succeeded + if: ${{ always() }} + needs: + - build-wheels + - test-wheels + runs-on: ubuntu-latest + steps: + - name: Check build results + run: | + if [[ "${{ needs.build-wheels.result }}" != "success" ]]; then + echo "Wheel builds failed!" + exit 1 + fi + if [[ "${{ needs.test-wheels.result }}" != "success" ]]; then + echo "Wheel tests failed!" + exit 1 + fi + echo "All wheels built and tested successfully!" diff --git a/ci/test_pynvbench.sh b/ci/test_pynvbench.sh new file mode 100755 index 0000000..bc148ed --- /dev/null +++ b/ci/test_pynvbench.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -euo pipefail +ci_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "$ci_dir/pyenv_helper.sh" + +# Parse common arguments +source "$ci_dir/util/python/common_arg_parser.sh" +parse_python_args "$@" + +# Parse CUDA version +cuda_version="" +while [[ $# -gt 0 ]]; do + case $1 in + -cuda-version=*) + cuda_version="${1#*=}" + shift + ;; + -cuda-version) + if [[ $# -lt 2 ]]; then + echo "Error: -cuda-version requires a value" >&2 + exit 1 + fi + cuda_version="$2" + shift 2 + ;; + *) + shift + ;; + esac +done + +if [[ -z "$cuda_version" ]]; then + echo "Error: -cuda-version is required" + exit 1 +fi + +# Determine CUDA major version from environment +cuda_major_version=$(nvcc --version | grep release | awk '{print $6}' | tr -d ',' | cut -d '.' -f 1 | cut -d 'V' -f 2) + +# Setup Python environment +setup_python_env "${py_version}" + +# Fetch the pynvbench wheel from artifacts +if [[ -n "${GITHUB_ACTIONS:-}" ]]; then + # In GitHub Actions, download from artifacts + wheel_name="wheel-pynvbench-cu${cuda_version}-py${py_version}" + mkdir -p /home/coder/nvbench/wheelhouse + + # Download artifact (assumes it's already been downloaded by the workflow) + if [[ -d "/home/coder/nvbench/${wheel_name}" ]]; then + cp /home/coder/nvbench/${wheel_name}/*.whl /home/coder/nvbench/wheelhouse/ + fi +else + # For local testing, build the wheel + "$ci_dir/build_pynvbench_wheel.sh" -py-version "${py_version}" -cuda-version "${cuda_version}" +fi + +# Install pynvbench +PYNVBENCH_WHEEL_PATH="$(ls /home/coder/nvbench/wheelhouse/pynvbench-*.whl | head -1)" +if [[ -z "$PYNVBENCH_WHEEL_PATH" ]]; then + echo "Error: No pynvbench wheel found" + exit 1 +fi + +echo "Installing wheel: $PYNVBENCH_WHEEL_PATH" +python -m pip install "${PYNVBENCH_WHEEL_PATH}[test]" + +# Run tests +cd "/home/coder/nvbench/python/test/" +python -m pytest -v test_nvbench.py