diff --git a/_build_version_metadata.py b/_build_version_metadata.py new file mode 100644 index 00000000..951f7b8e --- /dev/null +++ b/_build_version_metadata.py @@ -0,0 +1,135 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Custom build backend wrapper to ensure version generation.""" + +import os +import sys +import re +import logging +from pathlib import Path + +# Import the original backend +from scikit_build_core.build import * # noqa: F401, F403 +from scikit_build_core.build import build_wheel as _orig_build_wheel +from scikit_build_core.build import build_sdist as _orig_build_sdist +from scikit_build_core.build import build_editable as _orig_build_editable +from scikit_build_core.build import get_requires_for_build_wheel as _orig_get_requires_for_build_wheel +from scikit_build_core.build import prepare_metadata_for_build_wheel as _orig_prepare_metadata_for_build_wheel + +logging.basicConfig(level=logging.INFO) + + +def _get_version(): + """Get version using setuptools-scm, VERSION and clean it up""" + try: + with open("VERSION", "r") as vf: + base_version = vf.read().strip() + except FileNotFoundError: + base_version = "0.0.0" + + try: + import setuptools_scm + + version = setuptools_scm.get_version(root=".") + + # Remove the .dYYYYMMDD timestamp if present + # Convert "0.7.1.dev36+g6e2360d69.d20250926" to "0.7.1.dev36+g6e2360d69" + version = re.sub(r"\.d\d{8}", "", version) + + # Use the value in VERSION as the base version + # Change to "0.7.0.dev36+g6e2360d69" + version = re.sub(r"^[0-9]+\.[0-9]+\.[0-9]+", base_version, version) + + logging.info(f"Generated version with setuptools-scm: {version}") + return version + except Exception as e: + logging.warning(f"setuptools-scm failed: {e}, using fallback") + return base_version + "+unknown" + + +def _generate_version_file(): + """Generate _version.py file using setuptools-scm""" + version = _get_version() + + # Write version file + version_file = Path("python/mscclpp/_version.py") + version_file.parent.mkdir(parents=True, exist_ok=True) + + with open(version_file, "w") as f: + f.write(f"# Generated by build backend\n") + f.write(f'__version__ = "{version}"\n') + f.write(f'version = "{version}"\n') + + logging.info(f"Wrote version {version} to {version_file}") + + # Also write a metadata file that scikit-build-core can read + metadata_file = Path("python/mscclpp/PKG-INFO") + metadata_file.parent.mkdir(parents=True, exist_ok=True) + with open(metadata_file, "w") as f: + f.write(f"Metadata-Version: 2.1\n") + f.write(f"Name: mscclpp\n") + f.write(f"Version: {version}\n") + + # Set environment variable for scikit-build-core + os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version + + return version + + +def get_requires_for_build_wheel(config_settings=None): + """Get requirements with version generation""" + _generate_version_file() + return _orig_get_requires_for_build_wheel(config_settings) + + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None): + """Prepare metadata with version generation""" + version = _generate_version_file() + + # Call original function + result = _orig_prepare_metadata_for_build_wheel(metadata_directory, config_settings) + + # Patch the metadata with correct version + import configparser + + metadata_file = Path(metadata_directory) / f"{result}/METADATA" + if metadata_file.exists(): + with open(metadata_file, "r") as f: + content = f.read() + + # Replace version line + lines = content.split("\n") + for i, line in enumerate(lines): + if line.startswith("Version:"): + lines[i] = f"Version: {version}" + break + + with open(metadata_file, "w") as f: + f.write("\n".join(lines)) + + return result + + +def build_wheel(wheel_directory, config_settings=None, metadata_directory=None): + """Build wheel with version generation""" + version = _generate_version_file() + + # Set version in environment for scikit-build-core to pick up + os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version + + return _orig_build_wheel(wheel_directory, config_settings, metadata_directory) + + +def build_sdist(sdist_directory, config_settings=None): + """Build sdist with version generation""" + version = _generate_version_file() + os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version + return _orig_build_sdist(sdist_directory, config_settings) + + +def build_editable(wheel_directory, config_settings=None, metadata_directory=None): + """Build editable with version generation""" + version = _generate_version_file() + os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version + return _orig_build_editable(wheel_directory, config_settings, metadata_directory) diff --git a/docs/quickstart.md b/docs/quickstart.md index ceda0e32..0960ec82 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -205,3 +205,38 @@ export LD_AUDIT=$MSCCLPP_INSTALL_DIR/libmscclpp_audit_nccl.so export LD_LIBRARY_PATH=$MSCCLPP_INSTALL_DIR:$LD_LIBRARY_PATH torchrun --nnodes=1 --nproc_per_node=8 your_script.py ``` + +### Version Tracking + +The MSCCL++ Python package includes comprehensive version tracking that captures git repository information at build time. This feature allows users to identify the exact source code version of their installed package. + +#### Version Format + +The package version includes the git commit hash directly in the version string for development builds: +- **Release version**: `0.7.0` +- **Development version**: `0.7.0.dev36+g6e2360d69` (includes short commit hash) +- **Development with uncommitted changes**: `0.7.0.dev36+g6e2360d69.dirty` + +#### Checking Version Information + +After installation, you can check the version information in several ways: + +**From Python:** +```python +import mscclpp + +# Access individual attributes +print(f"Version: {mscclpp.__version__}") # Full version with commit +Version: 0.7.0.dev36+g6e2360d69 + +# Get as dictionary +mscclpp.version +{'version': '0.7.0.dev36+g6e2360d69', 'base_version': '0.7.0', 'git_commit': '6e2360d69'} +``` + +#### Version Information Details + +The version tracking captures: +- **Package Version** (`mscclpp.__version__`): Full version string including git commit (e.g., `0.7.1.dev36+g6e2360d69`) + +This information is embedded during the package build process and remains accessible even after distribution, making it easier to debug issues and ensure reproducibility. diff --git a/pyproject.toml b/pyproject.toml index cd109e20..0af8ded5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,17 +2,28 @@ # Licensed under the MIT license. [build-system] -requires = ["scikit-build-core"] -build-backend = "scikit_build_core.build" +requires = [ + "scikit-build-core>=0.4.3", + "setuptools-scm[toml]>=6.2" +] +build-backend = "_build_version_metadata" +backend-path = ["."] [project] name = "mscclpp" -version = "0.7.0" +dynamic = ["version"] +description = "MSCCL++ Python API" +requires-python = ">=3.8" + +[tool.setuptools_scm] +write_to = "python/mscclpp/_version.py" [tool.scikit-build] cmake.version = ">=3.25.0" cmake.build-type = "Release" build-dir = "build/{wheel_tag}" +# Tell scikit-build-core to get version from setuptools-scm +metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" [tool.scikit-build.wheel] packages = ["python/mscclpp", "python/mscclpp_benchmark"] diff --git a/python/mscclpp/__init__.py b/python/mscclpp/__init__.py index 229a2902..12b836d8 100644 --- a/python/mscclpp/__init__.py +++ b/python/mscclpp/__init__.py @@ -5,8 +5,59 @@ import os import warnings +import re from functools import wraps + +# Get version +def _get_version(): + """Get version from the best available source""" + + # Try setuptools-scm generated _version.py (most reliable) + try: + from ._version import __version__ + + return __version__ + except ImportError: + raise RuntimeError("Could not determine MSCCL++ version from setuptools-scm generated _version.py.") + + +# Parse version components +def _parse_version(version_string): + """Parse version components from setuptools-scm generated version""" + # Pattern for versions like "0.7.0.dev36+g6e2360d69" (without .dYYYYMMDD) + pattern = r"^v?(?P[\d\.]+)(?:\.dev(?P\d+))?(?:\+g(?P[a-f0-9]+))?(?P\.dirty)?$" + match = re.match(pattern, version_string) + + if match: + return {"base_version": match.group("base"), "git_commit": match.group("commit") or "unknown"} + else: + # Fallback parsing - try to extract what we can + base = version_string.split("+")[0].lstrip("v").split(".dev")[0] + commit = "unknown" + + return {"base_version": base, "git_commit": commit} + + +__version__ = _get_version() + +# Parse the version +_version_info = _parse_version(__version__) +__base_version__ = _version_info["base_version"] +__git_commit__ = _version_info["git_commit"] + + +def _version(): + """Get complete version information as a dictionary""" + return { + "version": __version__, + "base_version": __base_version__, + "git_commit": __git_commit__, + } + + +version: dict = _version() + from ._mscclpp import ( Env, ErrorCode, @@ -40,12 +91,10 @@ from ._mscclpp import ( PacketType, RawGpuBuffer, env, - version, is_nvls_supported, npkit, ) - __all__ = [ "Device", "DeviceType", @@ -69,11 +118,12 @@ __all__ = [ "Executor", "ExecutionPlan", "PacketType", - "version", "is_nvls_supported", "alloc_shared_physical_cuda", "npkit", + # Version information "__version__", + "version", "get_include", "get_lib", ### Deprecated ### @@ -82,8 +132,6 @@ __all__ = [ "SmDevice2DeviceSemaphore", ] -__version__: str = str(version()) - if os.environ.get("MSCCLPP_HOME", None) is None: os.environ["MSCCLPP_HOME"] = os.path.abspath(os.path.dirname(__file__))