NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-07-12 10:07:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Match benchmark-axis scoping to NVBench CLI

Browse Source 
When any --benchmark filter is present, keep comparison limited to the
explicitly selected benchmarks. Leading --axis filters are still replayed onto
each selected benchmark, matching native NVBench option parsing, but they no
longer cause unrelated benchmarks to be compared.

E.g., `-a A=2 -b bench1` now compares only bench1,
`-a A=2 -b bench1 -b bench2` applies A=2 to both selected benchmarks

Update tests for global axis filters with benchmark scopes and document the
selection behavior.
This commit is contained in:
Oleksandr Pavlyk
2026-06-28 07:10:38 -05:00
parent f4aff067c7
commit 2e0f635b6c
3 changed files with 65 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
docs/nvbench_compare.md
View File

@@ -150,7 +150,9 @@ nvbench-compare -b base -a "Elements{io}[pow2]=20" reference.json compare.json
```
`-a` / `--axis` applies to the most recent `-b` / `--benchmark`, or to all
benchmarks if it appears before any benchmark filter.
benchmarks if it appears before any benchmark filter. If any benchmark filters
are provided, only those benchmarks are compared; leading axis filters are
replayed onto each selected benchmark, matching NVBench CLI behavior.
## Timing Data Used

2
python/scripts/nvbench_compare.py
View File

@@ -2285,8 +2285,6 @@ def axis_filter_groups_for_benchmark(benchmark_name, filter_plan):
filter_plan.global_axis_filters + scope.axis_filters
for scope in matching_scopes
]
if filter_plan.global_axis_filters:
return [filter_plan.global_axis_filters]
return []

63
python/test/test_nvbench_compare.py
View File

@@ -2077,7 +2077,7 @@ def test_axis_filter_applies_to_most_recent_benchmark(monkeypatch, nvbench_compa
assert run_data.stats.unknown_count == 0
def test_global_axis_filter_still_applies_after_benchmark_scope(
def test_global_axis_filter_does_not_select_unmatched_benchmark(
monkeypatch, nvbench_compare
):
run_data = make_comparison_run_data(nvbench_compare)
@@ -2130,6 +2130,67 @@ def test_global_axis_filter_still_applies_after_benchmark_scope(
no_color=True,
)
assert run_data.stats.config_count == 1
assert run_data.stats.pass_count == 0
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.unknown_count == 0
def test_global_axis_filter_applies_to_each_selected_benchmark(
monkeypatch, nvbench_compare
):
run_data = make_comparison_run_data(nvbench_compare)
ref_benches = [
make_benchmark(
[
make_state(nvbench_compare, "state", mean="1.0", axis_value=1),
make_state(nvbench_compare, "state", mean="2.0", axis_value=2),
],
name="bench1",
),
make_benchmark(
[
make_state(nvbench_compare, "state", mean="3.0", axis_value=1),
make_state(nvbench_compare, "state", mean="4.0", axis_value=2),
],
name="bench2",
),
]
cmp_benches = [
make_benchmark(
[
make_state(nvbench_compare, "state", mean="1.0", axis_value=1),
make_state(nvbench_compare, "state", mean="2.0", axis_value=2),
],
name="bench1",
),
make_benchmark(
[
make_state(nvbench_compare, "state", mean="3.0", axis_value=1),
make_state(nvbench_compare, "state", mean="4.0", axis_value=2),
],
name="bench2",
),
]
nvbench_compare.compare_benches(
run_data,
ref_benches,
cmp_benches,
threshold=0.0,
plot_along=None,
plot=False,
dark=False,
filter_plan=make_filter_plan(
nvbench_compare,
[("axis", "A=2"), ("benchmark", "bench1"), ("benchmark", "bench2")],
),
no_color=True,
)
assert run_data.stats.config_count == 2
assert run_data.stats.pass_count == 0
assert run_data.stats.improvement_count == 0