diff --git a/docs/nvbench_compare.md b/docs/nvbench_compare.md index b83a198..6e7500c 100644 --- a/docs/nvbench_compare.md +++ b/docs/nvbench_compare.md @@ -150,7 +150,9 @@ nvbench-compare -b base -a "Elements{io}[pow2]=20" reference.json compare.json ``` `-a` / `--axis` applies to the most recent `-b` / `--benchmark`, or to all -benchmarks if it appears before any benchmark filter. +benchmarks if it appears before any benchmark filter. If any benchmark filters +are provided, only those benchmarks are compared; leading axis filters are +replayed onto each selected benchmark, matching NVBench CLI behavior. ## Timing Data Used diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index cff504b..0828766 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -2285,8 +2285,6 @@ def axis_filter_groups_for_benchmark(benchmark_name, filter_plan): filter_plan.global_axis_filters + scope.axis_filters for scope in matching_scopes ] - if filter_plan.global_axis_filters: - return [filter_plan.global_axis_filters] return [] diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index fc54cb2..46f3626 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -2077,7 +2077,7 @@ def test_axis_filter_applies_to_most_recent_benchmark(monkeypatch, nvbench_compa assert run_data.stats.unknown_count == 0 -def test_global_axis_filter_still_applies_after_benchmark_scope( +def test_global_axis_filter_does_not_select_unmatched_benchmark( monkeypatch, nvbench_compare ): run_data = make_comparison_run_data(nvbench_compare) @@ -2130,6 +2130,67 @@ def test_global_axis_filter_still_applies_after_benchmark_scope( no_color=True, ) + assert run_data.stats.config_count == 1 + assert run_data.stats.pass_count == 0 + assert run_data.stats.improvement_count == 0 + assert run_data.stats.regression_count == 0 + assert run_data.stats.undecided_count == 1 + assert run_data.stats.unknown_count == 0 + + +def test_global_axis_filter_applies_to_each_selected_benchmark( + monkeypatch, nvbench_compare +): + run_data = make_comparison_run_data(nvbench_compare) + + ref_benches = [ + make_benchmark( + [ + make_state(nvbench_compare, "state", mean="1.0", axis_value=1), + make_state(nvbench_compare, "state", mean="2.0", axis_value=2), + ], + name="bench1", + ), + make_benchmark( + [ + make_state(nvbench_compare, "state", mean="3.0", axis_value=1), + make_state(nvbench_compare, "state", mean="4.0", axis_value=2), + ], + name="bench2", + ), + ] + cmp_benches = [ + make_benchmark( + [ + make_state(nvbench_compare, "state", mean="1.0", axis_value=1), + make_state(nvbench_compare, "state", mean="2.0", axis_value=2), + ], + name="bench1", + ), + make_benchmark( + [ + make_state(nvbench_compare, "state", mean="3.0", axis_value=1), + make_state(nvbench_compare, "state", mean="4.0", axis_value=2), + ], + name="bench2", + ), + ] + + nvbench_compare.compare_benches( + run_data, + ref_benches, + cmp_benches, + threshold=0.0, + plot_along=None, + plot=False, + dark=False, + filter_plan=make_filter_plan( + nvbench_compare, + [("axis", "A=2"), ("benchmark", "bench1"), ("benchmark", "bench2")], + ), + no_color=True, + ) + assert run_data.stats.config_count == 2 assert run_data.stats.pass_count == 0 assert run_data.stats.improvement_count == 0