mirror of
https://github.com/NVIDIA/nvbench.git
synced 2026-05-14 02:02:16 +00:00
338936b6fe
Implements `cuda.bench.results.BenchmarkResult` class to represent data from JSON output of benchmark execution.
The contains implements two class methods `BenchmarkResult.from_json(filename : str | os.PathLike, *, metadata : Any = None)` which expects well-formed JSON filename and `BenchmarkResult.empty(*, metadata : Any = None)` intended to represent failed result with reasons that can be recorded in metadata at user's discretion.
The `BenchmarkResult` implements mapping interface, supporting `.keys()`, `.values()`, `.items()` methods, `__len__`, `__contains__`, `__getitem__` and `__iter__` special methods.
Values in `BenchmarkResult` has type `cuda.bench.results.SubBenchmarkResult` which implements a list-like interface, i.e. implements `__len__`, `__getitem__`, and `__iter__` special methods. Values in this list-like structure correspond to measurements of individual states of a particular benchmark (the key in `BenchmarkResult`).
Elements of `SubBenchmarkResult` structure have type `SubBenchmarkState` that supports mapping protocol with axis_values as a key and represent data corresponding to measurements for a particular state (combination of settings for each axis).
The state provides `.samples` and `.frequencies` attributes storing raw execution duration values and estimates for average GPU frequencies.
Example usage:
```
import array, numpy as np, cuda.bench.results
r = cuda.bench.results.BenchmarkResult("perf_data/axes_run1.json")
r["copy_sweep_grid_shape"].centers_with_frequencies(
lambda t, f: np.median(np.asarray(t)*np.asarray(f)))
```
```
In [1]: import array, numpy as np, cuda.bench.results
In [2]: r = cuda.bench.results.BenchmarkResult("temp_data/axes_run1.json")
In [3]: list(r)
Out[3]:
['simple',
'single_float64_axis',
'copy_sweep_grid_shape',
'copy_type_sweep',
'copy_type_conversion_sweep',
'copy_type_and_block_size_sweep']
In [4]: r["simple"].centers(lambda t: np.percentile(t, [25,75]))
Out[4]: {'Device=0': array([0.00100966, 0.00101299])}
In [5]: r.centers(lambda t: np.percentile(t, [25,75]))["simple"]
Out[5]: {'Device=0': array([0.00100966, 0.00101299])}
In [6]: len(r)
Out[6]: 6
In [7]: "fake" in r
Out[7]: False
```
Each `SubBenchmarkState` implements
`.summaries` attribute - rich object that retains tag/name/hint/hide/description metadata.
* Add nvbench-json-summary to render NVBench JSON output as an NVBench-style
markdown summary table, including axis formatting, device sections, hidden
summary filtering, and summary hint formatting.
Update packaging, type stubs, and tests for the new namespace, renamed
classes, Python 3.10-compatible annotations, and summary-table generation.
* Split tests in test_benchmark_result into smaller tests
* Fix break due to file name change
* Add python/examples/benchmark_result_autotune.py
This example demonstrates using cuda.bench and cuda.bench.results
to implement simple auto-tuning, demonstrated on selecting of
tile shape hyperparameter for naive stencil kernel implemented
in numba-cuda.
* Resolve ruff PLE0604
* Fix for format_axis_value in json format script to handle None value
Add tests to cover such input.
* Address code rabbit review feedback
* Fix license header, add validation
* Addressed both issues raised in review
Malformed values are now represented in result as None.
Skipped benchmarks are no longer dropped, i.e., they are present
in BenchmarkResult data, but they are not reflected in summary
table in line with what NVBench-instrumented benchmarks do.
94 lines
2.6 KiB
TOML
94 lines
2.6 KiB
TOML
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
[build-system]
|
|
requires = ["scikit-build-core>=0.10", "setuptools_scm"]
|
|
build-backend = "scikit_build_core.build"
|
|
|
|
[project]
|
|
name = "cuda-bench"
|
|
description = "CUDA Kernel Benchmarking Package"
|
|
authors = [{ name = "NVIDIA Corporation" }]
|
|
license = { text = "Apache-2.0 WITH LLVM-exception" }
|
|
classifiers = [
|
|
"Development Status :: 4 - Beta",
|
|
"Intended Audience :: Developers",
|
|
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
"Programming Language :: Python :: 3 :: Only",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
"Environment :: GPU :: NVIDIA CUDA",
|
|
"License :: OSI Approved :: Apache Software License",
|
|
"Operating System :: POSIX :: Linux",
|
|
]
|
|
requires-python = ">=3.10"
|
|
dynamic = ["version"]
|
|
readme = { file = "README.md", content-type = "text/markdown" }
|
|
|
|
[project.optional-dependencies]
|
|
# CUDA 12.x dependencies
|
|
cu12 = ["cuda-bindings>=12.0.0,<13.0.0"]
|
|
|
|
# CUDA 13.x dependencies
|
|
cu13 = ["cuda-bindings>=13.0.0,<14.0.0"]
|
|
|
|
# Test dependencies for CUDA 12
|
|
test-cu12 = ["cuda-bench[cu12]", "pytest", "cupy-cuda12x", "numba"]
|
|
|
|
# Test dependencies for CUDA 13
|
|
test-cu13 = ["cuda-bench[cu13]", "pytest", "cupy-cuda13x", "numba"]
|
|
|
|
# Generic test dependencies (defaults to CUDA 12)
|
|
test = ["pytest", "cupy-cuda12x", "numba"]
|
|
tools = [
|
|
"colorama",
|
|
"jsondiff",
|
|
"matplotlib",
|
|
"numpy",
|
|
"pandas",
|
|
"seaborn",
|
|
"tabulate",
|
|
]
|
|
|
|
[project.scripts]
|
|
nvbench-compare = "scripts.nvbench_compare:main"
|
|
nvbench-histogram = "scripts.nvbench_histogram:main"
|
|
nvbench-json-summary = "scripts.nvbench_json_summary:main"
|
|
nvbench-walltime = "scripts.nvbench_walltime:main"
|
|
|
|
[project.urls]
|
|
Homepage = "https://github.com/NVIDIA/nvbench"
|
|
Repository = "https://github.com/NVIDIA/nvbench"
|
|
Issues = "https://github.com/NVIDIA/nvbench/issues"
|
|
|
|
[tool.scikit-build]
|
|
minimum-version = "build-system.requires"
|
|
build-dir = "build/{wheel_tag}"
|
|
|
|
[tool.scikit-build.cmake]
|
|
version = ">=3.30.4"
|
|
args = []
|
|
build-type = "Release"
|
|
source-dir = "."
|
|
|
|
[tool.scikit-build.ninja]
|
|
version = ">=1.11"
|
|
make-fallback = true
|
|
|
|
[tool.scikit-build.metadata.version]
|
|
provider = "scikit_build_core.metadata.setuptools_scm"
|
|
|
|
[tool.setuptools_scm]
|
|
root = ".."
|
|
git_describe_command = ["git", "describe", "--tags", "--match", "python-*"]
|
|
tag_regex = "^python-(?P<version>.*)$"
|
|
fallback_version = "0.0.0"
|
|
|
|
[tool.scikit-build.wheel.packages]
|
|
"cuda" = "cuda"
|
|
"cuda/bench" = "cuda/bench"
|
|
"cuda/bench/results" = "cuda/bench/results"
|
|
"scripts" = "scripts"
|