NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-07-12 18:17:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Match benchmark-axis scoping to NVBench CLI

Browse Source 
When any --benchmark filter is present, keep comparison limited to the
explicitly selected benchmarks. Leading --axis filters are still replayed onto
each selected benchmark, matching native NVBench option parsing, but they no
longer cause unrelated benchmarks to be compared.

E.g., `-a A=2 -b bench1` now compares only bench1,
`-a A=2 -b bench1 -b bench2` applies A=2 to both selected benchmarks

Update tests for global axis filters with benchmark scopes and document the
selection behavior.
This commit is contained in:
Oleksandr Pavlyk
2026-06-28 07:10:38 -05:00
parent f4aff067c7
commit 2e0f635b6c
3 changed files with 65 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
python/test/test_nvbench_compare.py
View File

@@ -2077,7 +2077,7 @@ def test_axis_filter_applies_to_most_recent_benchmark(monkeypatch, nvbench_compa
assert run_data.stats.unknown_count == 0
def test_global_axis_filter_still_applies_after_benchmark_scope(
def test_global_axis_filter_does_not_select_unmatched_benchmark(
monkeypatch, nvbench_compare
):
run_data = make_comparison_run_data(nvbench_compare)
@@ -2130,6 +2130,67 @@ def test_global_axis_filter_still_applies_after_benchmark_scope(
no_color=True,
)
assert run_data.stats.config_count == 1
assert run_data.stats.pass_count == 0
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.unknown_count == 0
def test_global_axis_filter_applies_to_each_selected_benchmark(
monkeypatch, nvbench_compare
):
run_data = make_comparison_run_data(nvbench_compare)
ref_benches = [
make_benchmark(
[
make_state(nvbench_compare, "state", mean="1.0", axis_value=1),
make_state(nvbench_compare, "state", mean="2.0", axis_value=2),
],
name="bench1",
),
make_benchmark(
[
make_state(nvbench_compare, "state", mean="3.0", axis_value=1),
make_state(nvbench_compare, "state", mean="4.0", axis_value=2),
],
name="bench2",
),
]
cmp_benches = [
make_benchmark(
[
make_state(nvbench_compare, "state", mean="1.0", axis_value=1),
make_state(nvbench_compare, "state", mean="2.0", axis_value=2),
],
name="bench1",
),
make_benchmark(
[
make_state(nvbench_compare, "state", mean="3.0", axis_value=1),
make_state(nvbench_compare, "state", mean="4.0", axis_value=2),
],
name="bench2",
),
]
nvbench_compare.compare_benches(
run_data,
ref_benches,
cmp_benches,
threshold=0.0,
plot_along=None,
plot=False,
dark=False,
filter_plan=make_filter_plan(
nvbench_compare,
[("axis", "A=2"), ("benchmark", "bench1"), ("benchmark", "bench2")],
),
no_color=True,
)
assert run_data.stats.config_count == 2
assert run_data.stats.pass_count == 0
assert run_data.stats.improvement_count == 0