diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index 0d29a04..5afa28a 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -3179,7 +3179,7 @@ def main() -> int: type=float, dest="threshold", default=0.0, - help="only show benchmarks where percentage diff is >= THRESHOLD", + help="only show rows where abs(%%Diff) is >= THRESHOLD percent", ) parser.add_argument( "--preset", @@ -3373,7 +3373,7 @@ def main() -> int: run_data, ref_root["benchmarks"], cmp_root["benchmarks"], - threshold=args.threshold, + threshold=args.threshold / 100.0, plot_along=args.plot_along, plot=args.plot, dark=args.dark, diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index cd40398..92221fa 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -2961,3 +2961,34 @@ def test_main_passes_selected_preset_to_compare_benches(monkeypatch, nvbench_com "comparison_thresholds" ] == nvbench_compare.get_comparison_thresholds("strict") assert captured["display"] == "explain" + + +def test_main_converts_threshold_diff_percent_to_fraction(monkeypatch, nvbench_compare): + devices = [{"id": 0, "name": "Test GPU"}] + root = { + "devices": devices, + "benchmarks": [], + } + captured = {} + + monkeypatch.setattr(nvbench_compare.reader, "read_file", lambda _: root) + + def fake_compare_benches(*args, **kwargs): + del args + captured["threshold"] = kwargs["threshold"] + + monkeypatch.setattr(nvbench_compare, "compare_benches", fake_compare_benches) + monkeypatch.setattr( + sys, + "argv", + [ + "nvbench_compare", + "--threshold-diff", + "5", + "ref.json", + "cmp.json", + ], + ) + + assert nvbench_compare.main() == 0 + assert captured["threshold"] == pytest.approx(0.05)