NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-05-24 23:04:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add BenchmarkResult JSON results namespace and summary CLI

Browse Source 
Move NVBench JSON result parsing into cuda.bench.results with explicit
BenchmarkResult, BenchmarkResultDevice, BenchmarkResultSummary,
SubBenchmarkResult, and SubBenchmarkState types. Remove the result reader
from the top-level cuda.bench namespace and require construction through
BenchmarkResult.from_json() or BenchmarkResult.empty().

Preserve bulk sample/frequency parsing and estimator helpers while making
summaries rich objects that retain tag/name/hint/hide/description metadata.

Add nvbench-json-summary to render NVBench JSON output as an NVBench-style
markdown summary table, including axis formatting, device sections, hidden
summary filtering, and summary hint formatting.

Update packaging, type stubs, and tests for the new namespace, renamed
classes, Python 3.10-compatible annotations, and summary-table generation.
This commit is contained in:
Oleksandr Pavlyk
2026-05-12 09:58:30 -05:00
parent 231e9d7b1d
commit 4a22923721
9 changed files with 826 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
python/test/test_bench_result.py → python/test/test_benchmark_result.py
View File

@@ -1,8 +1,25 @@
# Copyright 2026 NVIDIA Corporation
#
# Licensed under the Apache License, Version 2.0 with the LLVM exception
# (the "License"); you may not use this file except in compliance with
# the License.
#
# You may obtain a copy of the License at
#
# http://llvm.org/foundation/relicensing/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import struct
from dataclasses import dataclass
import cuda.bench as bench
import cuda.bench
import cuda.bench.results as results
import pytest
@@ -63,6 +80,9 @@ def test_benchmark_result_reads_jsonbin_relative_to_json_path(tmp_path):
},
{
"tag": "nv/cold/bw/global/utilization",
"name": "BWUtil",
"hint": "percentage",
"description": "Global memory utilization",
"data": [
{
"name": "value",
@@ -98,10 +118,12 @@ def test_benchmark_result_reads_jsonbin_relative_to_json_path(tmp_path):
)
metadata = {"returncode": 0, "elapsed_seconds": 0.25}
default_result = bench.BenchmarkResult.from_json(json_fn)
result = bench.BenchmarkResult(json_path=json_fn, metadata=metadata)
default_result = results.BenchmarkResult.from_json(json_fn)
result = results.BenchmarkResult.from_json(json_fn, metadata=metadata)
assert bench.BenchmarkResult.__module__ == bench.__name__
assert results.BenchmarkResult.__module__ == results.__name__
assert results.BenchmarkResultSummary.__module__ == results.__name__
assert not hasattr(cuda.bench, "BenchmarkResult")
assert default_result.metadata is None
assert result.metadata is metadata
subbench = result["copy"]
@@ -121,12 +143,19 @@ def test_benchmark_result_reads_jsonbin_relative_to_json_path(tmp_path):
subbench[1]
assert state.name() == "BlockSize[pow2]=8"
assert state.bw == 0.75
assert state.summaries["nv/cold/bw/global/utilization"] == pytest.approx(0.75)
assert state.summaries["nv/json/bin:nv/cold/sample_times"] == {
bw_summary = state.summaries["nv/cold/bw/global/utilization"]
assert bw_summary.tag == "nv/cold/bw/global/utilization"
assert bw_summary.name == "BWUtil"
assert bw_summary.hint == "percentage"
assert bw_summary.hide is None
assert bw_summary.description == "Global memory utilization"
assert bw_summary.value == pytest.approx(0.75)
assert bw_summary["value"] == pytest.approx(0.75)
assert state.summaries["nv/json/bin:nv/cold/sample_times"].data == {
"filename": "result.json-bin/0.bin",
"size": 3,
}
assert state.summaries["nv/json/freqs-bin:nv/cold/sample_freqs"] == {
assert state.summaries["nv/json/freqs-bin:nv/cold/sample_freqs"].data == {
"filename": "result.json-freqs-bin/0.bin",
"size": 3,
}
@@ -154,13 +183,15 @@ def test_benchmark_result_reads_jsonbin_relative_to_json_path(tmp_path):
result["missing"]
def test_benchmark_result_json_path_is_required_keyword():
def test_benchmark_result_constructor_is_private():
with pytest.raises(TypeError, match="from_json\\(\\).*empty\\(\\)"):
results.BenchmarkResult()
with pytest.raises(TypeError, match="from_json\\(\\).*empty\\(\\)"):
results.BenchmarkResult("result.json")
with pytest.raises(TypeError):
bench.BenchmarkResult("result.json")
results.BenchmarkResult(metadata=None)
with pytest.raises(TypeError):
bench.BenchmarkResult(metadata=None)
with pytest.raises(TypeError):
bench.BenchmarkResult(json_path="result.json", parse=False)
results.BenchmarkResult(json_path="result.json", parse=False)
def test_benchmark_result_empty_does_not_read_json(tmp_path):
@@ -172,15 +203,15 @@ def test_benchmark_result_empty_does_not_read_json(tmp_path):
metadata = RunMetadata(returncode=1, elapsed_seconds=0.25)
missing_json = tmp_path / "missing.json"
result = bench.BenchmarkResult.empty(metadata=metadata)
result = results.BenchmarkResult.empty(metadata=metadata)
assert result.metadata is metadata
assert result.subbenches == {}
with pytest.raises(FileNotFoundError):
bench.BenchmarkResult(json_path=missing_json, metadata=metadata)
results.BenchmarkResult.from_json(missing_json, metadata=metadata)
with pytest.raises(FileNotFoundError):
bench.BenchmarkResult.from_json(json_path=missing_json, metadata=metadata)
results.BenchmarkResult.from_json(json_path=missing_json, metadata=metadata)
def test_benchmark_result_accepts_no_axis_benchmark_with_recorded_binary_path(
@@ -251,7 +282,7 @@ def test_benchmark_result_accepts_no_axis_benchmark_with_recorded_binary_path(
monkeypatch.chdir(tmp_path)
result = bench.BenchmarkResult(json_path="temp_data/axes_run1.json")
result = results.BenchmarkResult.from_json("temp_data/axes_run1.json")
state = result.subbenches["simple"].states[0]
assert state.name() == "Device=0"
@@ -263,7 +294,7 @@ def test_benchmark_result_accepts_no_axis_benchmark_with_recorded_binary_path(
def test_benchmark_result_accepts_axis_value_input_string():
result = bench.SubBenchResult(
result = results.SubBenchmarkResult(
{
"name": "single_float64_axis",
"axes": [
@@ -304,7 +335,7 @@ def test_benchmark_result_accepts_axis_value_input_string():
def test_benchmark_result_ignores_skipped_state_with_no_summaries():
result = bench.SubBenchResult(
result = results.SubBenchmarkResult(
{
"name": "copy_sweep_grid_shape",
"axes": [
@@ -451,7 +482,7 @@ def test_benchmark_result_uses_none_for_unavailable_samples(tmp_path):
encoding="utf-8",
)
result = bench.BenchmarkResult(json_path=json_fn)
result = results.BenchmarkResult.from_json(json_fn)
states = result.subbenches["copy"].states
assert states[0].samples is None
@@ -556,4 +587,4 @@ def test_benchmark_result_rejects_mismatched_sample_and_frequency_counts(tmp_pat
)
with pytest.raises(ValueError, match="sample count .* frequency count"):
bench.BenchmarkResult(json_path=json_fn)
results.BenchmarkResult.from_json(json_fn)

221
python/test/test_nvbench_json_summary.py Normal file
View File

@@ -0,0 +1,221 @@
# Copyright 2026 NVIDIA Corporation
#
# Licensed under the Apache License, Version 2.0 with the LLVM exception
# (the "License"); you may not use this file except in compliance with
# the License.
#
# You may obtain a copy of the License at
#
# http://llvm.org/foundation/relicensing/LICENSE.txt
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import importlib.util
import json
from pathlib import Path
def load_nvbench_json_summary():
module_path = (
Path(__file__).resolve().parents[1] / "scripts" / "nvbench_json_summary.py"
)
spec = importlib.util.spec_from_file_location("nvbench_json_summary", module_path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
nvbench_json_summary = load_nvbench_json_summary()
def write_result_json(path):
path.write_text(
json.dumps(
{
"devices": [
{
"id": 0,
"name": "Test GPU",
}
],
"benchmarks": [
{
"name": "copy",
"devices": [0],
"axes": [
{
"name": "BlockSize",
"type": "int64",
"flags": "pow2",
"values": [
{
"input_string": "8",
"description": "2^8 = 256",
"value": 256,
}
],
}
],
"states": [
{
"name": "Device=0 BlockSize=2^8",
"device": 0,
"type_config_index": 0,
"axis_values": [
{
"name": "BlockSize",
"type": "int64",
"value": "256",
}
],
"summaries": [
{
"tag": "nv/cold/time/gpu/sample_size",
"name": "Samples",
"hint": "sample_size",
"data": [
{
"name": "value",
"type": "int64",
"value": "12",
}
],
},
{
"tag": "nv/cold/time/gpu/mean",
"name": "GPU Time",
"hint": "duration",
"data": [
{
"name": "value",
"type": "float64",
"value": "1.25e-6",
}
],
},
{
"tag": "nv/cold/time/gpu/stdev/relative",
"name": "Noise",
"hint": "percentage",
"data": [
{
"name": "value",
"type": "float64",
"value": "0.015",
}
],
},
{
"tag": "nv/cold/bw/global/bytes_per_second",
"name": "GlobalMem BW",
"hint": "byte_rate",
"data": [
{
"name": "value",
"type": "float64",
"value": "2.5e9",
}
],
},
{
"tag": "nv/cold/bw/global/utilization",
"name": "BWUtil",
"hint": "percentage",
"data": [
{
"name": "value",
"type": "float64",
"value": "0.625",
}
],
},
{
"tag": "nv/cold/time/gpu/min",
"name": "Min GPU Time",
"hint": "duration",
"hide": "Hidden by default.",
"data": [
{
"name": "value",
"type": "float64",
"value": "1.0e-6",
}
],
},
],
"is_skipped": False,
}
],
}
],
}
),
encoding="utf-8",
)
def test_json_summary_formats_nvbench_style_markdown(tmp_path):
json_path = tmp_path / "result.json"
write_result_json(json_path)
result = nvbench_json_summary.BenchmarkResult.from_json(json_path)
report = nvbench_json_summary.format_result(result)
assert "# Benchmark Results" in report
assert "## copy" in report
assert "### [0] Test GPU" in report
assert (
"| BlockSize | Samples | GPU Time | Noise | GlobalMem BW | BWUtil |" in report
)
assert (
"| 2^8 = 256 | 12x | 1.250 us | 1.50% | 2.500 GB/s | 62.50% |" in report
)
assert "Min GPU Time" not in report
def test_json_summary_formats_axis_values_like_markdown_printer():
axes_by_name = {
"BlockSize": {
"name": "BlockSize",
"type": "int64",
"flags": "pow2",
},
"NumBlocks": {
"name": "NumBlocks",
"type": "int64",
"flags": "",
},
"Duration": {
"name": "Duration",
"type": "float64",
"flags": "",
},
}
assert nvbench_json_summary.format_axis_value(
{"name": "BlockSize", "type": "int64", "value": "256"}, axes_by_name
) == ("BlockSize", "2^8 = 256")
assert nvbench_json_summary.format_axis_value(
{"name": "NumBlocks", "type": "int64", "value": "64"}, axes_by_name
) == ("NumBlocks", "64")
assert nvbench_json_summary.format_axis_value(
{"name": "Duration", "type": "float64", "value": "0.123456789"},
axes_by_name,
) == ("Duration", "0.12346")
def test_json_summary_cli_writes_output_file(tmp_path):
json_path = tmp_path / "result.json"
output_path = tmp_path / "summary.md"
write_result_json(json_path)
rc = nvbench_json_summary.main([str(json_path), "--output", str(output_path)])
assert rc == 0
assert "GlobalMem BW" in output_path.read_text(encoding="utf-8")