From 149a2d257ab3580bed9575ce442e7ea6108fb81e Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:28:16 -0500 Subject: [PATCH] Allow timing-only bulk data to confirm SAME Permit nvbench-compare to use bulk timing samples to confirm SAME even when sample frequency sidecars are unavailable. This lets legacy --jsonbin outputs, which recorded sample times but not frequencies, avoid AMBG results when summary noise is high but timing coverage clearly supports equality. Cycle coverage is still required when frequency samples are present, and FAST/SLOW clear-gap confirmation remains cycle-based. Add regression coverage for noisy bulk samples that exceed the SAME noise ceiling but are classified SAME from bulk timing/cycle coverage, including the timing-only legacy-data path. --- python/scripts/nvbench_compare.py | 124 ++++++++++++++++++++++------ python/test/test_nvbench_compare.py | 102 ++++++++++++++++++++++- 2 files changed, 197 insertions(+), 29 deletions(-) diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index c16b8eb..b64ffde 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -585,6 +585,8 @@ REASON_DISPLAY_CODES = { "bulk_same": "bulk-same", "bulk_time_data_unusable": "bt-bad", "bulk_time_same": "bt-same", + "bulk_time_same_confirmed_by_summary_cycles": "bt-same-sc", + "bulk_time_same_without_cycles": "bt-same-no-cyc", "bulk_time_support_mismatch": "bt-sup-miss", "centers_not_close": "centers-far", "clear_gap_confirmed_by_bulk_cycles": "bc-gap", @@ -1895,24 +1897,53 @@ def compare_values_for_bulk_same(ref_values, cmp_values, *, label, thresholds): def compare_timings_for_bulk_same(ref_timing, cmp_timing, thresholds): - ref_bulk = get_bulk_time_and_cycles(ref_timing) - cmp_bulk = get_bulk_time_and_cycles(cmp_timing) - if ref_bulk is None or cmp_bulk is None: + ref_times = positive_finite_array(ref_timing.samples) + cmp_times = positive_finite_array(cmp_timing.samples) + if ref_times is None or cmp_times is None: + if ( + ref_timing.sample_source is not None + and cmp_timing.sample_source is not None + ): + return make_decision( + ComparisonStatus.UNDECIDED, + "bulk_time_data_unusable", + "bulk time data is empty or unusable", + ) return make_decision( ComparisonStatus.UNDECIDED, "bulk_data_unavailable", - "bulk sample time and frequency data are unavailable", + "bulk sample time data are unavailable", ) - ref_times, ref_cycles = ref_bulk - cmp_times, cmp_cycles = cmp_bulk - time_decision = compare_values_for_bulk_same( ref_times, cmp_times, label="time", thresholds=thresholds ) if time_decision.status != ComparisonStatus.SAME: return time_decision + if ref_timing.frequency_source is None or cmp_timing.frequency_source is None: + return make_decision( + ComparisonStatus.SAME, + "bulk_time_same_without_cycles", + "bulk time nearest-neighbor coverage supports same; sample frequencies are unavailable", + ) + + ref_frequencies = positive_finite_array(ref_timing.frequencies) + cmp_frequencies = positive_finite_array(cmp_timing.frequencies) + if ( + ref_frequencies is None + or cmp_frequencies is None + or len(ref_times) != len(ref_frequencies) + or len(cmp_times) != len(cmp_frequencies) + ): + return make_decision( + ComparisonStatus.UNDECIDED, + "bulk_cycle_data_unusable", + "bulk cycle data is empty or unusable", + ) + + ref_cycles = ref_times * ref_frequencies + cmp_cycles = cmp_times * cmp_frequencies cycle_decision = compare_values_for_bulk_same( ref_cycles, cmp_cycles, label="cycle", thresholds=thresholds ) @@ -1926,6 +1957,34 @@ def compare_timings_for_bulk_same(ref_timing, cmp_timing, thresholds): ) +def compare_timings_for_bulk_same_if_available(ref_timing, cmp_timing, thresholds): + decision = compare_timings_for_bulk_same(ref_timing, cmp_timing, thresholds) + if decision.reason.code == "bulk_data_unavailable": + return None + return decision + + +def confirm_bulk_time_same_without_cycles( + ref_timing, cmp_timing, ref_interval, cmp_interval, thresholds +): + decision = confirm_same_with_clock_rate( + ref_timing, cmp_timing, ref_interval, cmp_interval, thresholds + ) + if decision.reason.code == "same_without_clock_rate": + return make_decision( + ComparisonStatus.SAME, + "bulk_time_same_without_cycles", + "bulk time nearest-neighbor coverage supports same; cycle data are unavailable", + ) + if decision.status == ComparisonStatus.SAME: + return make_decision( + ComparisonStatus.SAME, + "bulk_time_same_confirmed_by_summary_cycles", + "bulk time nearest-neighbor coverage supports same and SM-clock-adjusted summary intervals confirm same", + ) + return decision + + def compare_timings_for_same( ref_timing, cmp_timing, ref_noise, cmp_noise, ref_interval, cmp_interval, thresholds ): @@ -1935,12 +1994,6 @@ def compare_timings_for_same( "noise_unavailable", "relative dispersion is unavailable, negative, or non-finite", ) - if max(ref_noise, cmp_noise) > thresholds.same_relative_dispersion_ceiling: - return make_decision( - ComparisonStatus.UNDECIDED, - "noise_too_high", - "relative dispersion is too high to declare same", - ) if ref_interval is None or cmp_interval is None: return make_decision( @@ -1953,6 +2006,29 @@ def compare_timings_for_same( if decision.status != ComparisonStatus.SAME: return decision + bulk_decision = compare_timings_for_bulk_same_if_available( + ref_timing, cmp_timing, thresholds + ) + if ( + bulk_decision is not None + and bulk_decision.reason.code == "bulk_time_same_without_cycles" + ): + bulk_decision = confirm_bulk_time_same_without_cycles( + ref_timing, cmp_timing, ref_interval, cmp_interval, thresholds + ) + + if max(ref_noise, cmp_noise) > thresholds.same_relative_dispersion_ceiling: + if bulk_decision is not None: + return bulk_decision + return make_decision( + ComparisonStatus.UNDECIDED, + "noise_too_high", + "relative dispersion is too high to declare same", + ) + + if bulk_decision is not None: + return bulk_decision + return confirm_same_with_clock_rate( ref_timing, cmp_timing, ref_interval, cmp_interval, thresholds ) @@ -2175,21 +2251,15 @@ def compare_gpu_timings(ref_timing, cmp_timing, comparison_thresholds=None): "no_clear_gap", "missing_interval", }: - bulk_decision = compare_timings_for_bulk_same( - ref_timing, cmp_timing, comparison_thresholds + decision = compare_timings_for_same( + ref_timing, + cmp_timing, + ref_noise, + cmp_noise, + ref_interval, + cmp_interval, + comparison_thresholds, ) - if bulk_decision.reason.code == "bulk_data_unavailable": - decision = compare_timings_for_same( - ref_timing, - cmp_timing, - ref_noise, - cmp_noise, - ref_interval, - cmp_interval, - comparison_thresholds, - ) - else: - decision = bulk_decision return SummaryComparison( ref_interval=ref_interval, diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index c3e8c58..e2d5b56 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -974,7 +974,7 @@ def test_compare_gpu_timings_classifies_common_cases(tmp_path, nvbench_compare): assert undecided.diff == pytest.approx(0.03) assert undecided.frac_diff == pytest.approx(0.03) assert undecided.max_noise == pytest.approx(0.05) - assert undecided.reason.code == "noise_too_high" + assert undecided.reason.code == "centers_not_close" legacy_same = nvbench_compare.compare_gpu_timings( make_gpu_timing_data(nvbench_compare, mean=1.0, stdev_relative=0.005), @@ -1241,6 +1241,104 @@ def test_compare_gpu_timings_classifies_common_cases(tmp_path, nvbench_compare): assert noisy_same.status == nvbench_compare.ComparisonStatus.UNDECIDED assert noisy_same.reason.code == "noise_too_high" + noisy_bulk_same = nvbench_compare.compare_gpu_timings( + make_gpu_timing_data( + nvbench_compare, + sample_values=[1.0, 1.0, 1.1, 1.1], + frequency_values=[100.0] * 4, + ), + make_gpu_timing_data( + nvbench_compare, + sample_values=[1.0, 1.0, 1.1, 1.1], + frequency_values=[100.0] * 4, + ), + ) + assert noisy_bulk_same is not None + assert noisy_bulk_same.status == nvbench_compare.ComparisonStatus.SAME + assert ( + noisy_bulk_same.max_noise + > nvbench_compare.get_default_thresholds().same_relative_dispersion_ceiling + ) + assert noisy_bulk_same.reason.code == "bulk_same" + + noisy_bulk_time_same = nvbench_compare.compare_gpu_timings( + make_gpu_timing_data( + nvbench_compare, + sample_values=[1.0, 1.0, 1.1, 1.1], + ), + make_gpu_timing_data( + nvbench_compare, + sample_values=[1.0, 1.0, 1.1, 1.1], + ), + ) + assert noisy_bulk_time_same is not None + assert noisy_bulk_time_same.status == nvbench_compare.ComparisonStatus.SAME + assert ( + noisy_bulk_time_same.max_noise + > nvbench_compare.get_default_thresholds().same_relative_dispersion_ceiling + ) + assert noisy_bulk_time_same.reason.code == "bulk_time_same_without_cycles" + + noisy_bulk_time_same_with_summary_cycles = nvbench_compare.compare_gpu_timings( + make_gpu_timing_data( + nvbench_compare, + sm_clock_rate_mean=100.0, + sample_values=[1.0, 1.0, 1.1, 1.1], + ), + make_gpu_timing_data( + nvbench_compare, + sm_clock_rate_mean=100.0, + sample_values=[1.0, 1.0, 1.1, 1.1], + ), + ) + assert noisy_bulk_time_same_with_summary_cycles is not None + assert ( + noisy_bulk_time_same_with_summary_cycles.status + == nvbench_compare.ComparisonStatus.SAME + ) + assert ( + noisy_bulk_time_same_with_summary_cycles.reason.code + == "bulk_time_same_confirmed_by_summary_cycles" + ) + + noisy_bulk_time_summary_cycle_disagreement = nvbench_compare.compare_gpu_timings( + make_gpu_timing_data( + nvbench_compare, + sm_clock_rate_mean=100.0, + sample_values=[1.0, 1.0, 1.1, 1.1], + ), + make_gpu_timing_data( + nvbench_compare, + sm_clock_rate_mean=200.0, + sample_values=[1.0, 1.0, 1.1, 1.1], + ), + ) + assert noisy_bulk_time_summary_cycle_disagreement is not None + assert ( + noisy_bulk_time_summary_cycle_disagreement.status + == nvbench_compare.ComparisonStatus.UNDECIDED + ) + assert ( + noisy_bulk_time_summary_cycle_disagreement.reason.code + == "cycle_same_not_confirmed" + ) + + noisy_bulk_bad_cycles = nvbench_compare.compare_gpu_timings( + make_gpu_timing_data( + nvbench_compare, + sample_values=[1.0, 1.0, 1.1, 1.1], + frequency_values=[0.0, 100.0, 100.0, 100.0], + ), + make_gpu_timing_data( + nvbench_compare, + sample_values=[1.0, 1.0, 1.1, 1.1], + frequency_values=[100.0] * 4, + ), + ) + assert noisy_bulk_bad_cycles is not None + assert noisy_bulk_bad_cycles.status == nvbench_compare.ComparisonStatus.UNDECIDED + assert noisy_bulk_bad_cycles.reason.code == "bulk_cycle_data_unusable" + clock_disagreement = nvbench_compare.compare_gpu_timings( ref_interval_timing, make_gpu_timing_data( @@ -2528,7 +2626,7 @@ def test_main_prints_undecided_reason_summary(monkeypatch, capsys, nvbench_compa "devices": devices, "benchmarks": [ make_benchmark( - [make_state(nvbench_compare, "state", mean="1.01", noise="0.05")] + [make_state(nvbench_compare, "state", mean="1.001", noise="0.05")] ) ], }