Handle legacy nvbench-compare timing summaries

Derive absolute standard deviation from relative stdev and mean when
nv/cold/time/gpu/stdev/absolute is absent. This lets older JSON files
that only contain mean and relative stdev still construct timing
intervals.

Also allow mean/stdev intervals to be built without min/max summaries,
using min/max only as optional clipping bounds when present. This
restores SAME classification for legacy fixture data instead of treating
those rows as missing-interval AMBG cases.

Update nvbench_compare tests to cover derived stdev handling and the
legacy mean/stdev comparison path.
This commit is contained in:
Oleksandr Pavlyk
2026-06-29 05:29:02 -05:00
parent 46e61aa12b
commit 45f28e2855
2 changed files with 69 additions and 35 deletions

View File

@@ -198,6 +198,9 @@ def make_gpu_timing_data(
sample_values=None,
frequency_values=None,
):
if stdev is None:
stdev = nvbench_compare.derive_absolute_dispersion(stdev_relative, mean)
return nvbench_compare.GpuTimingData(
minimum=minimum,
maximum=maximum,
@@ -287,10 +290,10 @@ def test_compare_benches_accepts_matching_duplicate_state_counts(
)
assert run_data.stats.config_count == 3
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 3
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 3
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -371,10 +374,10 @@ def test_compare_benches_matches_duplicate_states_after_axis_filter(
)
assert run_data.stats.config_count == 1
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 1
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -485,10 +488,10 @@ def test_compare_benches_counts_non_finite_centers_as_unknown(
)
assert run_data.stats.config_count == 3
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 1
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 2
@@ -737,6 +740,18 @@ def test_gpu_timing_data_parses_quartiles_and_sm_clock_rate_mean(nvbench_compare
assert timing.frequencies is None
def test_gpu_timing_data_derives_stdev_from_relative_stdev(nvbench_compare):
timing = nvbench_compare.extract_gpu_timing_data(
[
make_summary(nvbench_compare, "GPU_TIME_MEAN_TAG", "2.0"),
make_summary(nvbench_compare, "GPU_TIME_STDEV_RELATIVE_TAG", "0.25"),
],
)
assert timing.stdev == pytest.approx(0.5)
assert timing.stdev_relative == pytest.approx(0.25)
def test_gpu_timing_data_accepts_legacy_ir_tags(nvbench_compare):
timing = nvbench_compare.extract_gpu_timing_data(
[
@@ -833,6 +848,17 @@ def test_compare_gpu_timings_classifies_common_cases(tmp_path, nvbench_compare):
assert undecided.max_noise == pytest.approx(0.05)
assert undecided.reason.code == "noise_too_high"
legacy_same = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(nvbench_compare, mean=1.0, stdev_relative=0.005),
make_gpu_timing_data(nvbench_compare, mean=1.001, stdev_relative=0.005),
)
assert legacy_same is not None
assert legacy_same.status == nvbench_compare.ComparisonStatus.SAME
assert legacy_same.reason.code == "same_without_clock_rate"
assert legacy_same.ref_interval is not None
assert legacy_same.ref_interval.lower == pytest.approx(0.995)
assert legacy_same.ref_interval.upper == pytest.approx(1.005)
partial_robust = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(
nvbench_compare,
@@ -1906,10 +1932,10 @@ def test_plot_along_skips_states_without_selected_axis(monkeypatch, nvbench_comp
)
assert run_data.stats.config_count == 3
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 3
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 3
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
assert xscale_calls == ["log"]
assert yscale_calls == ["log"]
@@ -2105,10 +2131,10 @@ def test_compare_benches_pairs_filtered_devices_by_position(
)
assert run_data.stats.config_count == 1
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 1
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -2164,10 +2190,10 @@ def test_axis_filter_applies_to_most_recent_benchmark(monkeypatch, nvbench_compa
)
assert run_data.stats.config_count == 3
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 3
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 3
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -2225,10 +2251,10 @@ def test_global_axis_filter_does_not_select_unmatched_benchmark(
)
assert run_data.stats.config_count == 1
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 1
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -2286,10 +2312,10 @@ def test_global_axis_filter_applies_to_each_selected_benchmark(
)
assert run_data.stats.config_count == 2
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 2
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 2
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0