diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index d02f5a3..432f445 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -990,14 +990,20 @@ def extract_gpu_timing_data(summaries, json_dir=None, float32_reader=read_float3 sample_source = None frequency_source = None + mean = extract_summary_float(summaries, GPU_TIME_MEAN_TAG) + stdev = extract_summary_float(summaries, GPU_TIME_STDEV_TAG, null_value=math.inf) + stdev_relative = extract_summary_float( + summaries, GPU_TIME_STDEV_RELATIVE_TAG, null_value=math.inf + ) + if stdev is None: + stdev = derive_absolute_dispersion(stdev_relative, mean) + return GpuTimingData( minimum=extract_summary_float(summaries, GPU_TIME_MIN_TAG), maximum=extract_summary_float(summaries, GPU_TIME_MAX_TAG), - mean=extract_summary_float(summaries, GPU_TIME_MEAN_TAG), - stdev=extract_summary_float(summaries, GPU_TIME_STDEV_TAG, null_value=math.inf), - stdev_relative=extract_summary_float( - summaries, GPU_TIME_STDEV_RELATIVE_TAG, null_value=math.inf - ), + mean=mean, + stdev=stdev, + stdev_relative=stdev_relative, first_quartile=extract_summary_float(summaries, GPU_TIME_Q1_TAG), median=extract_summary_float(summaries, GPU_TIME_MEDIAN_TAG), third_quartile=extract_summary_float(summaries, GPU_TIME_Q3_TAG), @@ -1192,6 +1198,12 @@ def is_nonnegative_finite(value): return value is not None and math.isfinite(value) and value >= 0.0 +def derive_absolute_dispersion(relative_dispersion, center): + if is_nonnegative_finite(relative_dispersion) and is_positive_finite(center): + return relative_dispersion * center + return None + + def parse_plot_axis_value(axis_name, axis_value): try: value = float(axis_value) @@ -1240,21 +1252,17 @@ def compute_robust_summary_interval(timing): def compute_mean_summary_interval(timing): - if ( - is_positive_finite(timing.minimum) - and is_positive_finite(timing.maximum) - and is_positive_finite(timing.mean) - and is_nonnegative_finite(timing.stdev) - and timing.minimum <= timing.mean - and timing.mean <= timing.maximum - ): - return make_timing_interval( - lower=max(timing.minimum, timing.mean - timing.stdev), - upper=min(timing.maximum, timing.mean + timing.stdev), - center=timing.mean, - ) + if not is_positive_finite(timing.mean) or not is_nonnegative_finite(timing.stdev): + return None - return None + lower = max(timing.mean - timing.stdev, timing.mean * 0.001) + upper = timing.mean + timing.stdev + if is_positive_finite(timing.minimum): + lower = max(lower, timing.minimum) + if is_positive_finite(timing.maximum): + upper = min(upper, timing.maximum) + + return make_timing_interval(lower=lower, upper=upper, center=timing.mean) def compute_timing_interval(timing): diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index b2a6eac..12c0f5f 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -198,6 +198,9 @@ def make_gpu_timing_data( sample_values=None, frequency_values=None, ): + if stdev is None: + stdev = nvbench_compare.derive_absolute_dispersion(stdev_relative, mean) + return nvbench_compare.GpuTimingData( minimum=minimum, maximum=maximum, @@ -287,10 +290,10 @@ def test_compare_benches_accepts_matching_duplicate_state_counts( ) assert run_data.stats.config_count == 3 - assert run_data.stats.pass_count == 0 + assert run_data.stats.pass_count == 3 assert run_data.stats.improvement_count == 0 assert run_data.stats.regression_count == 0 - assert run_data.stats.undecided_count == 3 + assert run_data.stats.undecided_count == 0 assert run_data.stats.unknown_count == 0 @@ -371,10 +374,10 @@ def test_compare_benches_matches_duplicate_states_after_axis_filter( ) assert run_data.stats.config_count == 1 - assert run_data.stats.pass_count == 0 + assert run_data.stats.pass_count == 1 assert run_data.stats.improvement_count == 0 assert run_data.stats.regression_count == 0 - assert run_data.stats.undecided_count == 1 + assert run_data.stats.undecided_count == 0 assert run_data.stats.unknown_count == 0 @@ -485,10 +488,10 @@ def test_compare_benches_counts_non_finite_centers_as_unknown( ) assert run_data.stats.config_count == 3 - assert run_data.stats.pass_count == 0 + assert run_data.stats.pass_count == 1 assert run_data.stats.improvement_count == 0 assert run_data.stats.regression_count == 0 - assert run_data.stats.undecided_count == 1 + assert run_data.stats.undecided_count == 0 assert run_data.stats.unknown_count == 2 @@ -737,6 +740,18 @@ def test_gpu_timing_data_parses_quartiles_and_sm_clock_rate_mean(nvbench_compare assert timing.frequencies is None +def test_gpu_timing_data_derives_stdev_from_relative_stdev(nvbench_compare): + timing = nvbench_compare.extract_gpu_timing_data( + [ + make_summary(nvbench_compare, "GPU_TIME_MEAN_TAG", "2.0"), + make_summary(nvbench_compare, "GPU_TIME_STDEV_RELATIVE_TAG", "0.25"), + ], + ) + + assert timing.stdev == pytest.approx(0.5) + assert timing.stdev_relative == pytest.approx(0.25) + + def test_gpu_timing_data_accepts_legacy_ir_tags(nvbench_compare): timing = nvbench_compare.extract_gpu_timing_data( [ @@ -833,6 +848,17 @@ def test_compare_gpu_timings_classifies_common_cases(tmp_path, nvbench_compare): assert undecided.max_noise == pytest.approx(0.05) assert undecided.reason.code == "noise_too_high" + legacy_same = nvbench_compare.compare_gpu_timings( + make_gpu_timing_data(nvbench_compare, mean=1.0, stdev_relative=0.005), + make_gpu_timing_data(nvbench_compare, mean=1.001, stdev_relative=0.005), + ) + assert legacy_same is not None + assert legacy_same.status == nvbench_compare.ComparisonStatus.SAME + assert legacy_same.reason.code == "same_without_clock_rate" + assert legacy_same.ref_interval is not None + assert legacy_same.ref_interval.lower == pytest.approx(0.995) + assert legacy_same.ref_interval.upper == pytest.approx(1.005) + partial_robust = nvbench_compare.compare_gpu_timings( make_gpu_timing_data( nvbench_compare, @@ -1906,10 +1932,10 @@ def test_plot_along_skips_states_without_selected_axis(monkeypatch, nvbench_comp ) assert run_data.stats.config_count == 3 - assert run_data.stats.pass_count == 0 + assert run_data.stats.pass_count == 3 assert run_data.stats.improvement_count == 0 assert run_data.stats.regression_count == 0 - assert run_data.stats.undecided_count == 3 + assert run_data.stats.undecided_count == 0 assert run_data.stats.unknown_count == 0 assert xscale_calls == ["log"] assert yscale_calls == ["log"] @@ -2105,10 +2131,10 @@ def test_compare_benches_pairs_filtered_devices_by_position( ) assert run_data.stats.config_count == 1 - assert run_data.stats.pass_count == 0 + assert run_data.stats.pass_count == 1 assert run_data.stats.improvement_count == 0 assert run_data.stats.regression_count == 0 - assert run_data.stats.undecided_count == 1 + assert run_data.stats.undecided_count == 0 assert run_data.stats.unknown_count == 0 @@ -2164,10 +2190,10 @@ def test_axis_filter_applies_to_most_recent_benchmark(monkeypatch, nvbench_compa ) assert run_data.stats.config_count == 3 - assert run_data.stats.pass_count == 0 + assert run_data.stats.pass_count == 3 assert run_data.stats.improvement_count == 0 assert run_data.stats.regression_count == 0 - assert run_data.stats.undecided_count == 3 + assert run_data.stats.undecided_count == 0 assert run_data.stats.unknown_count == 0 @@ -2225,10 +2251,10 @@ def test_global_axis_filter_does_not_select_unmatched_benchmark( ) assert run_data.stats.config_count == 1 - assert run_data.stats.pass_count == 0 + assert run_data.stats.pass_count == 1 assert run_data.stats.improvement_count == 0 assert run_data.stats.regression_count == 0 - assert run_data.stats.undecided_count == 1 + assert run_data.stats.undecided_count == 0 assert run_data.stats.unknown_count == 0 @@ -2286,10 +2312,10 @@ def test_global_axis_filter_applies_to_each_selected_benchmark( ) assert run_data.stats.config_count == 2 - assert run_data.stats.pass_count == 0 + assert run_data.stats.pass_count == 2 assert run_data.stats.improvement_count == 0 assert run_data.stats.regression_count == 0 - assert run_data.stats.undecided_count == 2 + assert run_data.stats.undecided_count == 0 assert run_data.stats.unknown_count == 0