mirror of
https://github.com/NVIDIA/nvbench.git
synced 2026-06-29 02:37:36 +00:00
338936b6fe
Implements `cuda.bench.results.BenchmarkResult` class to represent data from JSON output of benchmark execution.
The contains implements two class methods `BenchmarkResult.from_json(filename : str | os.PathLike, *, metadata : Any = None)` which expects well-formed JSON filename and `BenchmarkResult.empty(*, metadata : Any = None)` intended to represent failed result with reasons that can be recorded in metadata at user's discretion.
The `BenchmarkResult` implements mapping interface, supporting `.keys()`, `.values()`, `.items()` methods, `__len__`, `__contains__`, `__getitem__` and `__iter__` special methods.
Values in `BenchmarkResult` has type `cuda.bench.results.SubBenchmarkResult` which implements a list-like interface, i.e. implements `__len__`, `__getitem__`, and `__iter__` special methods. Values in this list-like structure correspond to measurements of individual states of a particular benchmark (the key in `BenchmarkResult`).
Elements of `SubBenchmarkResult` structure have type `SubBenchmarkState` that supports mapping protocol with axis_values as a key and represent data corresponding to measurements for a particular state (combination of settings for each axis).
The state provides `.samples` and `.frequencies` attributes storing raw execution duration values and estimates for average GPU frequencies.
Example usage:
```
import array, numpy as np, cuda.bench.results
r = cuda.bench.results.BenchmarkResult("perf_data/axes_run1.json")
r["copy_sweep_grid_shape"].centers_with_frequencies(
lambda t, f: np.median(np.asarray(t)*np.asarray(f)))
```
```
In [1]: import array, numpy as np, cuda.bench.results
In [2]: r = cuda.bench.results.BenchmarkResult("temp_data/axes_run1.json")
In [3]: list(r)
Out[3]:
['simple',
'single_float64_axis',
'copy_sweep_grid_shape',
'copy_type_sweep',
'copy_type_conversion_sweep',
'copy_type_and_block_size_sweep']
In [4]: r["simple"].centers(lambda t: np.percentile(t, [25,75]))
Out[4]: {'Device=0': array([0.00100966, 0.00101299])}
In [5]: r.centers(lambda t: np.percentile(t, [25,75]))["simple"]
Out[5]: {'Device=0': array([0.00100966, 0.00101299])}
In [6]: len(r)
Out[6]: 6
In [7]: "fake" in r
Out[7]: False
```
Each `SubBenchmarkState` implements
`.summaries` attribute - rich object that retains tag/name/hint/hide/description metadata.
* Add nvbench-json-summary to render NVBench JSON output as an NVBench-style
markdown summary table, including axis formatting, device sections, hidden
summary filtering, and summary hint formatting.
Update packaging, type stubs, and tests for the new namespace, renamed
classes, Python 3.10-compatible annotations, and summary-table generation.
* Split tests in test_benchmark_result into smaller tests
* Fix break due to file name change
* Add python/examples/benchmark_result_autotune.py
This example demonstrates using cuda.bench and cuda.bench.results
to implement simple auto-tuning, demonstrated on selecting of
tile shape hyperparameter for naive stencil kernel implemented
in numba-cuda.
* Resolve ruff PLE0604
* Fix for format_axis_value in json format script to handle None value
Add tests to cover such input.
* Address code rabbit review feedback
* Fix license header, add validation
* Addressed both issues raised in review
Malformed values are now represented in result as None.
Skipped benchmarks are no longer dropped, i.e., they are present
in BenchmarkResult data, but they are not reflected in summary
table in line with what NVBench-instrumented benchmarks do.
97 lines
3.2 KiB
Python
97 lines
3.2 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
from array import array
|
|
from collections.abc import Callable, ItemsView, Iterator, KeysView, ValuesView
|
|
from os import PathLike
|
|
from typing import Any, TypeVar, overload
|
|
|
|
ResultT = TypeVar("ResultT")
|
|
BenchmarkResultT = TypeVar("BenchmarkResultT", bound="BenchmarkResult")
|
|
_SummaryValue = int | float | str | None
|
|
|
|
class BenchmarkResultDevice:
|
|
id: int
|
|
name: str
|
|
data: dict[str, Any]
|
|
|
|
class BenchmarkResultSummary:
|
|
tag: str
|
|
name: str | None
|
|
hint: str | None
|
|
hide: str | None
|
|
description: str | None
|
|
data: dict[str, _SummaryValue]
|
|
@property
|
|
def value(self) -> _SummaryValue | None: ...
|
|
def __getitem__(self, key: str) -> _SummaryValue: ...
|
|
def get(
|
|
self, key: str, default: _SummaryValue | None = None
|
|
) -> _SummaryValue | None: ...
|
|
|
|
class SubBenchmarkState:
|
|
state_name: str
|
|
device: int | None
|
|
type_config_index: int | None
|
|
axis_values: list[dict[str, Any]]
|
|
is_skipped: bool
|
|
skip_reason: str | None
|
|
summaries: dict[str, BenchmarkResultSummary]
|
|
samples: array | None
|
|
frequencies: array | None
|
|
bw: float | None
|
|
point: dict[str, str]
|
|
def name(self) -> str: ...
|
|
def center(self, estimator: Callable[[array], ResultT]) -> ResultT | None: ...
|
|
def center_with_frequencies(
|
|
self, estimator: Callable[[array, array], ResultT]
|
|
) -> ResultT | None: ...
|
|
|
|
class SubBenchmarkResult:
|
|
name: str
|
|
devices: list[int]
|
|
axes: list[dict[str, Any]]
|
|
states: list[SubBenchmarkState]
|
|
def __len__(self) -> int: ...
|
|
@overload
|
|
def __getitem__(self, state_index: int) -> SubBenchmarkState: ...
|
|
@overload
|
|
def __getitem__(self, state_index: slice) -> list[SubBenchmarkState]: ...
|
|
def __iter__(self) -> Iterator[SubBenchmarkState]: ...
|
|
def centers(
|
|
self, estimator: Callable[[array], ResultT]
|
|
) -> dict[str, ResultT | None]: ...
|
|
def centers_with_frequencies(
|
|
self, estimator: Callable[[array, array], ResultT]
|
|
) -> dict[str, ResultT | None]: ...
|
|
|
|
class BenchmarkResult:
|
|
metadata: Any
|
|
devices: dict[int, BenchmarkResultDevice]
|
|
subbenches: dict[str, SubBenchmarkResult]
|
|
def __init__(self, token: object | None = None) -> None: ...
|
|
@classmethod
|
|
def empty(
|
|
cls: type[BenchmarkResultT], *, metadata: Any = None
|
|
) -> BenchmarkResultT: ...
|
|
@classmethod
|
|
def from_json(
|
|
cls: type[BenchmarkResultT],
|
|
json_path: str | PathLike[str],
|
|
*,
|
|
metadata: Any = None,
|
|
) -> BenchmarkResultT: ...
|
|
def __len__(self) -> int: ...
|
|
def __iter__(self) -> Iterator[str]: ...
|
|
def __contains__(self, subbench_name: object) -> bool: ...
|
|
def __getitem__(self, subbench_name: str) -> SubBenchmarkResult: ...
|
|
def keys(self) -> KeysView[str]: ...
|
|
def values(self) -> ValuesView[SubBenchmarkResult]: ...
|
|
def items(self) -> ItemsView[str, SubBenchmarkResult]: ...
|
|
def centers(
|
|
self, estimator: Callable[[array], ResultT]
|
|
) -> dict[str, dict[str, ResultT | None]]: ...
|
|
def centers_with_frequencies(
|
|
self, estimator: Callable[[array, array], ResultT]
|
|
) -> dict[str, dict[str, ResultT | None]]: ...
|