From b3bb31ca51c4af3fb1278f917cb9bbbe2336b812 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 14 Apr 2025 17:25:32 -0400 Subject: [PATCH] ci: work on speeding up further (#5613) * ci: work on speeding up further Signed-off-by: Henry Schreiner * chore: move uv's index-strategy to pyproject.toml Signed-off-by: Henry Schreiner * Update .github/workflows/ci.yml --------- Signed-off-by: Henry Schreiner --- .github/workflows/ci.yml | 44 ++++++++++-------------- .github/workflows/configure.yml | 24 ++++--------- .github/workflows/pip.yml | 24 ++++++------- pyproject.toml | 3 ++ tests/CMakeLists.txt | 7 ++-- tests/extra_python_package/test_files.py | 16 ++++++--- tests/requirements.txt | 1 + 7 files changed, 55 insertions(+), 64 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96e5b9b93..d9308decb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,33 +122,28 @@ jobs: if: runner.os == 'macOS' run: brew install boost - - name: Update CMake - uses: jwlawson/actions-setup-cmake@v2.0 - - - name: Cache wheels - if: runner.os == 'macOS' - uses: actions/cache@v4 + - name: Install uv + uses: astral-sh/setup-uv@v5 with: - # This path is specific to macOS - we really only need it for PyPy NumPy wheels - # See https://github.com/actions/cache/blob/master/examples.md#python---pip - # for ways to do this more generally - path: ~/Library/Caches/pip - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ matrix.python }}-x64-${{ hashFiles('tests/requirements.txt') }} + enable-cache: true - name: Prepare env - run: | - python -m pip install -r tests/requirements.txt + run: uv pip install --python=python --system -r tests/requirements.txt - name: Setup annotations on Linux if: runner.os == 'Linux' - run: python -m pip install pytest-github-actions-annotate-failures + run: uv pip install --python=python --system pytest-github-actions-annotate-failures + + # TODO Resolve Windows Ninja shared object issue on Python 3.8+ + - name: Use Ninja except on Windows + if: runner.os != 'Windows' + run: echo "CMAKE_GENERATOR=Ninja" >> "$GITHUB_ENV" # First build - C++11 mode and inplace # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON here - name: Configure C++11 ${{ matrix.args }} run: > - cmake -S . -B . + cmake -S. -B. -DPYBIND11_WERROR=ON -DPYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=ON @@ -159,13 +154,13 @@ jobs: ${{ matrix.args }} - name: Build C++11 - run: cmake --build . -j 2 + run: cmake --build . - name: Python tests C++11 - run: cmake --build . --target pytest -j 2 + run: cmake --build . --target pytest - name: C++11 tests - run: cmake --build . --target cpptest -j 2 + run: cmake --build . --target cpptest - name: Interface test C++11 run: cmake --build . --target test_cmake_build @@ -177,7 +172,7 @@ jobs: # More-or-less randomly adding -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF here. - name: Configure C++17 run: > - cmake -S . -B build2 -Werror=dev + cmake -S. -Bbuild2 -Werror=dev -DPYBIND11_WERROR=ON -DPYBIND11_SIMPLE_GIL_MANAGEMENT=OFF -DPYBIND11_PYTEST_ARGS=-v @@ -187,7 +182,7 @@ jobs: ${{ matrix.args }} - name: Build - run: cmake --build build2 -j 2 + run: cmake --build build2 - name: Python tests run: cmake --build build2 --target pytest @@ -202,7 +197,7 @@ jobs: # setuptools - name: Setuptools helpers test run: | - pip install setuptools + uv pip install --python=python --system setuptools pytest tests/extra_setuptools if: "!(matrix.runs-on == 'windows-2022')" @@ -913,10 +908,7 @@ jobs: python-version: ${{ matrix.python }} - name: Prepare env - # Ensure use of NumPy 2 (via NumPy nightlies but can be changed soon) - run: | - python3 -m pip install -r tests/requirements.txt - python3 -m pip install 'numpy>=2.0.0b1' 'scipy>=1.13.0rc1' + run: python3 -m pip install -r tests/requirements.txt - name: Update CMake uses: jwlawson/actions-setup-cmake@v2.0 diff --git a/.github/workflows/configure.yml b/.github/workflows/configure.yml index 8a5cf34d5..6408b5382 100644 --- a/.github/workflows/configure.yml +++ b/.github/workflows/configure.yml @@ -12,11 +12,6 @@ on: permissions: contents: read -env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - # For cmake: - VERBOSE: 1 - jobs: # This tests various versions of CMake in various combinations, to make sure # the configure step passes. @@ -28,10 +23,10 @@ jobs: - runs-on: ubuntu-22.04 cmake: "3.15" - - runs-on: ubuntu-22.04 + - runs-on: ubuntu-24.04 cmake: "3.26" - - runs-on: ubuntu-22.04 + - runs-on: ubuntu-24.04 cmake: "3.29" - runs-on: macos-13 @@ -57,8 +52,11 @@ jobs: with: python-version: 3.11 + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Prepare env - run: python -m pip install -r tests/requirements.txt + run: uv pip install --python=python --system -r tests/requirements.txt # An action for adding a specific version of CMake: # https://github.com/jwlawson/actions-setup-cmake @@ -68,17 +66,9 @@ jobs: cmake-version: ${{ matrix.cmake }} # These steps use a directory with a space in it intentionally - - name: Make build directories - run: mkdir "build dir" - - name: Configure - working-directory: build dir shell: bash - run: > - cmake .. - -DPYBIND11_WERROR=ON - -DDOWNLOAD_CATCH=ON - -DPYTHON_EXECUTABLE=$(python -c "import sys; print(sys.executable)") + run: cmake -S. -B"build dir" -DPYBIND11_WERROR=ON -DDOWNLOAD_CATCH=ON # Only build and test if this was manually triggered in the GitHub UI - name: Build diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml index 79c2963b9..694b55079 100644 --- a/.github/workflows/pip.yml +++ b/.github/workflows/pip.yml @@ -15,10 +15,6 @@ on: permissions: contents: read -env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - PIP_ONLY_BINARY: numpy - jobs: # This builds the sdists and wheels and makes sure the files are exactly as # expected. @@ -34,9 +30,11 @@ jobs: with: python-version: 3.8 + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Prepare env - run: | - python -m pip install -r tests/requirements.txt + run: uv pip install --system -r tests/requirements.txt - name: Python Packaging tests run: pytest tests/extra_python_package/ @@ -56,17 +54,19 @@ jobs: with: python-version: 3.8 + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Prepare env - run: | - python -m pip install -r tests/requirements.txt build twine + run: uv pip install --system -r tests/requirements.txt twine - name: Python Packaging tests run: pytest tests/extra_python_package/ - name: Build SDist and wheels run: | - python -m build - PYBIND11_GLOBAL_SDIST=1 python -m build + uv build + PYBIND11_GLOBAL_SDIST=1 uv build - name: Check metadata run: twine check dist/* @@ -103,7 +103,7 @@ jobs: - uses: actions/download-artifact@v4 - name: Generate artifact attestation for sdist and wheel - uses: actions/attest-build-provenance@c074443f1aee8d4aeeae555aebba3282517141b2 # v2.2.3 + uses: actions/attest-build-provenance@v2 with: subject-path: "*/pybind11*" @@ -111,10 +111,8 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: standard/ - attestations: true - name: Publish global package uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: global/ - attestations: true diff --git a/pyproject.toml b/pyproject.toml index 07e8edabb..38deff474 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,9 @@ ignore = [ "noxfile.py", ] +# Can't use tool.uv.sources with requirements.txt +[tool.uv] +index-strategy = "unsafe-best-match" [tool.mypy] files = ["pybind11"] diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9ee8131b1..e5e613df4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -541,10 +541,9 @@ if(NOT CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR) endif() -# cmake 3.12 added list(transform prepend -# but we can't use it yet -string(REPLACE "test_" "${CMAKE_CURRENT_SOURCE_DIR}/test_" PYBIND11_ABS_PYTEST_FILES - "${PYBIND11_PYTEST_FILES}") +# Convert relative to full file names +list(TRANSFORM PYBIND11_PYTEST_FILES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/" + OUTPUT_VARIABLE PYBIND11_ABS_PYTEST_FILES) set(PYBIND11_TEST_PREFIX_COMMAND "" diff --git a/tests/extra_python_package/test_files.py b/tests/extra_python_package/test_files.py index 5c6015050..017b52f6a 100644 --- a/tests/extra_python_package/test_files.py +++ b/tests/extra_python_package/test_files.py @@ -2,6 +2,7 @@ from __future__ import annotations import contextlib import os +import shutil import string import subprocess import sys @@ -13,6 +14,9 @@ import zipfile DIR = os.path.abspath(os.path.dirname(__file__)) MAIN_DIR = os.path.dirname(os.path.dirname(DIR)) +HAS_UV = shutil.which("uv") is not None +UV_ARGS = ["--installer=uv"] if HAS_UV else [] + PKGCONFIG = """\ prefix=${{pcfiledir}}/../../ includedir=${{prefix}}/include @@ -168,7 +172,8 @@ def test_build_sdist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) subprocess.run( - [sys.executable, "-m", "build", "--sdist", f"--outdir={tmpdir}"], check=True + [sys.executable, "-m", "build", "--sdist", f"--outdir={tmpdir}", *UV_ARGS], + check=True, ) (sdist,) = tmpdir.visit("*.tar.gz") @@ -218,7 +223,8 @@ def test_build_global_dist(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") subprocess.run( - [sys.executable, "-m", "build", "--sdist", "--outdir", str(tmpdir)], check=True + [sys.executable, "-m", "build", "--sdist", "--outdir", str(tmpdir), *UV_ARGS], + check=True, ) (sdist,) = tmpdir.visit("*.tar.gz") @@ -266,7 +272,8 @@ def tests_build_wheel(monkeypatch, tmpdir): monkeypatch.chdir(MAIN_DIR) subprocess.run( - [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True + [sys.executable, "-m", "build", "--wheel", "--outdir", str(tmpdir), *UV_ARGS], + check=True, ) (wheel,) = tmpdir.visit("*.whl") @@ -294,7 +301,8 @@ def tests_build_global_wheel(monkeypatch, tmpdir): monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1") subprocess.run( - [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)], check=True + [sys.executable, "-m", "build", "--wheel", "--outdir", str(tmpdir), *UV_ARGS], + check=True, ) (wheel,) = tmpdir.visit("*.whl") diff --git a/tests/requirements.txt b/tests/requirements.txt index 58c732c91..96c0cbcee 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,4 @@ +--extra-index-url=https://www.graalvm.org/python/wheels --only-binary=:all: build>=1 numpy~=1.23.0; python_version=="3.8" and platform_python_implementation=="PyPy"