mirror of
https://github.com/NVIDIA/nvbench.git
synced 2026-05-19 12:28:52 +00:00
Addressed both issues raised in review
Malformed values are now represented in result as None. Skipped benchmarks are no longer dropped, i.e., they are present in BenchmarkResult data, but they are not reflected in summary table in line with what NVBench-instrumented benchmarks do.
This commit is contained in:
Oleksandr Pavlyk
@@ -207,6 +207,94 @@ def test_state_stores_rich_summary_metadata(sample_state):
|
||||
}
|
||||
|
||||
|
||||
def test_state_preserves_null_summary_values(tmp_path):
|
||||
json_fn = tmp_path / "result.json"
|
||||
write_json(
|
||||
json_fn,
|
||||
{
|
||||
"benchmarks": [
|
||||
{
|
||||
"name": "copy",
|
||||
"axes": [],
|
||||
"states": [
|
||||
{
|
||||
"name": "Device=0",
|
||||
"axis_values": [],
|
||||
"summaries": [
|
||||
{
|
||||
"tag": "nv/cold/time/gpu/stdev/relative",
|
||||
"name": "Noise",
|
||||
"hint": "percentage",
|
||||
"data": [
|
||||
{
|
||||
"name": "value",
|
||||
"type": "float64",
|
||||
"value": None,
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"is_skipped": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
summary = results.BenchmarkResult.from_json(json_fn)["copy"][0].summaries[
|
||||
"nv/cold/time/gpu/stdev/relative"
|
||||
]
|
||||
|
||||
assert summary.value is None
|
||||
assert summary["value"] is None
|
||||
|
||||
|
||||
def test_state_reports_malformed_numeric_summary_values(tmp_path):
|
||||
json_fn = tmp_path / "result.json"
|
||||
write_json(
|
||||
json_fn,
|
||||
{
|
||||
"benchmarks": [
|
||||
{
|
||||
"name": "copy",
|
||||
"axes": [],
|
||||
"states": [
|
||||
{
|
||||
"name": "Device=0",
|
||||
"axis_values": [],
|
||||
"summaries": [
|
||||
{
|
||||
"tag": "nv/cold/time/gpu/mean",
|
||||
"name": "GPU Time",
|
||||
"hint": "duration",
|
||||
"data": [
|
||||
{
|
||||
"name": "value",
|
||||
"type": "float64",
|
||||
"value": "not-a-number",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"is_skipped": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match=(
|
||||
"summary 'nv/cold/time/gpu/mean' field 'value' "
|
||||
"value 'not-a-number' is not a float64"
|
||||
),
|
||||
):
|
||||
results.BenchmarkResult.from_json(json_fn)
|
||||
|
||||
|
||||
def test_state_loads_samples_and_frequencies(sample_state):
|
||||
assert sample_state.samples is not None
|
||||
assert list(sample_state.samples) == pytest.approx([1.0, 2.0, 4.0])
|
||||
@@ -432,7 +520,7 @@ def test_benchmark_result_normalizes_axis_value_lookup_key():
|
||||
assert result.states[2].point == {"NumBlocks": "64"}
|
||||
|
||||
|
||||
def test_benchmark_result_ignores_skipped_state_with_no_summaries():
|
||||
def test_benchmark_result_preserves_skipped_state_with_no_summaries():
|
||||
result = results.SubBenchmarkResult(
|
||||
{
|
||||
"name": "copy_sweep_grid_shape",
|
||||
@@ -467,8 +555,14 @@ def test_benchmark_result_ignores_skipped_state_with_no_summaries():
|
||||
"",
|
||||
)
|
||||
|
||||
assert len(result.states) == 1
|
||||
assert result.states[0].name() == "BlockSize[pow2]=6"
|
||||
assert len(result.states) == 2
|
||||
assert result.states[0].name() == "BlockSize[pow2]=8"
|
||||
assert result.states[0].is_skipped is True
|
||||
assert result.states[0].summaries == {}
|
||||
assert result.states[0].samples is None
|
||||
assert result.states[0].frequencies is None
|
||||
assert result.states[1].name() == "BlockSize[pow2]=6"
|
||||
assert result.states[1].is_skipped is False
|
||||
|
||||
|
||||
def test_benchmark_result_uses_empty_summaries_when_field_is_missing():
|
||||
|
||||
@@ -167,6 +167,19 @@ def test_json_summary_formats_nvbench_style_markdown(tmp_path):
|
||||
assert "Min GPU Time" not in report
|
||||
|
||||
|
||||
def test_json_summary_formats_null_summary_value_as_blank():
|
||||
summary = nvbench_json_summary.BenchmarkResultSummary(
|
||||
tag="nv/cold/time/gpu/stdev/relative",
|
||||
name="Noise",
|
||||
hint="percentage",
|
||||
hide=None,
|
||||
description=None,
|
||||
data={"value": None},
|
||||
)
|
||||
|
||||
assert nvbench_json_summary.format_summary(summary) == ""
|
||||
|
||||
|
||||
def test_json_summary_formats_axis_values_like_markdown_printer():
|
||||
axes_by_name = {
|
||||
"BlockSize": {
|
||||
@@ -259,6 +272,99 @@ def test_json_summary_formats_state_with_null_axis_values(tmp_path):
|
||||
assert "| 7x |" in report
|
||||
|
||||
|
||||
def test_json_summary_omits_skipped_states(tmp_path):
|
||||
json_path = tmp_path / "result.json"
|
||||
json_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"devices": [
|
||||
{
|
||||
"id": 0,
|
||||
"name": "Test GPU",
|
||||
}
|
||||
],
|
||||
"benchmarks": [
|
||||
{
|
||||
"name": "copy",
|
||||
"devices": [0],
|
||||
"axes": [
|
||||
{
|
||||
"name": "BlockSize",
|
||||
"type": "int64",
|
||||
"flags": "pow2",
|
||||
"values": [
|
||||
{
|
||||
"input_string": "8",
|
||||
"description": "2^8 = 256",
|
||||
"value": 256,
|
||||
},
|
||||
{
|
||||
"input_string": "9",
|
||||
"description": "2^9 = 512",
|
||||
"value": 512,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
"states": [
|
||||
{
|
||||
"name": "Device=0 BlockSize=2^8",
|
||||
"device": 0,
|
||||
"axis_values": [
|
||||
{
|
||||
"name": "BlockSize",
|
||||
"type": "int64",
|
||||
"value": "256",
|
||||
}
|
||||
],
|
||||
"summaries": None,
|
||||
"is_skipped": True,
|
||||
"skip_reason": "Deadlock detected",
|
||||
},
|
||||
{
|
||||
"name": "Device=0 BlockSize=2^9",
|
||||
"device": 0,
|
||||
"axis_values": [
|
||||
{
|
||||
"name": "BlockSize",
|
||||
"type": "int64",
|
||||
"value": "512",
|
||||
}
|
||||
],
|
||||
"summaries": [
|
||||
{
|
||||
"tag": "nv/cold/time/gpu/sample_size",
|
||||
"name": "Samples",
|
||||
"hint": "sample_size",
|
||||
"data": [
|
||||
{
|
||||
"name": "value",
|
||||
"type": "int64",
|
||||
"value": "3",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"is_skipped": False,
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = nvbench_json_summary.BenchmarkResult.from_json(json_path)
|
||||
report = nvbench_json_summary.format_result(result)
|
||||
|
||||
assert "Skip Reason" not in report
|
||||
assert "Deadlock detected" not in report
|
||||
assert "2^8 = 256" not in report
|
||||
assert "2^9 = 512" in report
|
||||
assert "3x" in report
|
||||
|
||||
|
||||
def test_json_summary_cli_writes_output_file(tmp_path):
|
||||
json_path = tmp_path / "result.json"
|
||||
output_path = tmp_path / "summary.md"
|
||||
|
||||
Reference in New Issue
Block a user