NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-07-12 10:07:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Handle legacy nvbench-compare timing summaries

Browse Source 
Derive absolute standard deviation from relative stdev and mean when
nv/cold/time/gpu/stdev/absolute is absent. This lets older JSON files
that only contain mean and relative stdev still construct timing
intervals.

Also allow mean/stdev intervals to be built without min/max summaries,
using min/max only as optional clipping bounds when present. This
restores SAME classification for legacy fixture data instead of treating
those rows as missing-interval AMBG cases.

Update nvbench_compare tests to cover derived stdev handling and the
legacy mean/stdev comparison path.
This commit is contained in:
Oleksandr Pavlyk
2026-06-29 05:29:02 -05:00
parent 46e61aa12b
commit 45f28e2855
2 changed files with 69 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
python/scripts/nvbench_compare.py
View File

@@ -990,14 +990,20 @@ def extract_gpu_timing_data(summaries, json_dir=None, float32_reader=read_float3
sample_source = None
frequency_source = None
mean = extract_summary_float(summaries, GPU_TIME_MEAN_TAG)
stdev = extract_summary_float(summaries, GPU_TIME_STDEV_TAG, null_value=math.inf)
stdev_relative = extract_summary_float(
summaries, GPU_TIME_STDEV_RELATIVE_TAG, null_value=math.inf
)
if stdev is None:
stdev = derive_absolute_dispersion(stdev_relative, mean)
return GpuTimingData(
minimum=extract_summary_float(summaries, GPU_TIME_MIN_TAG),
maximum=extract_summary_float(summaries, GPU_TIME_MAX_TAG),
mean=extract_summary_float(summaries, GPU_TIME_MEAN_TAG),
stdev=extract_summary_float(summaries, GPU_TIME_STDEV_TAG, null_value=math.inf),
stdev_relative=extract_summary_float(
summaries, GPU_TIME_STDEV_RELATIVE_TAG, null_value=math.inf
),
mean=mean,
stdev=stdev,
stdev_relative=stdev_relative,
first_quartile=extract_summary_float(summaries, GPU_TIME_Q1_TAG),
median=extract_summary_float(summaries, GPU_TIME_MEDIAN_TAG),
third_quartile=extract_summary_float(summaries, GPU_TIME_Q3_TAG),
@@ -1192,6 +1198,12 @@ def is_nonnegative_finite(value):
return value is not None and math.isfinite(value) and value >= 0.0
def derive_absolute_dispersion(relative_dispersion, center):
if is_nonnegative_finite(relative_dispersion) and is_positive_finite(center):
return relative_dispersion * center
return None
def parse_plot_axis_value(axis_name, axis_value):
try:
value = float(axis_value)
@@ -1240,21 +1252,17 @@ def compute_robust_summary_interval(timing):
def compute_mean_summary_interval(timing):
if (
is_positive_finite(timing.minimum)
and is_positive_finite(timing.maximum)
and is_positive_finite(timing.mean)
and is_nonnegative_finite(timing.stdev)
and timing.minimum <= timing.mean
and timing.mean <= timing.maximum
):
return make_timing_interval(
lower=max(timing.minimum, timing.mean - timing.stdev),
upper=min(timing.maximum, timing.mean + timing.stdev),
center=timing.mean,
)
if not is_positive_finite(timing.mean) or not is_nonnegative_finite(timing.stdev):
return None
return None
lower = max(timing.mean - timing.stdev, timing.mean * 0.001)
upper = timing.mean + timing.stdev
if is_positive_finite(timing.minimum):
lower = max(lower, timing.minimum)
if is_positive_finite(timing.maximum):
upper = min(upper, timing.maximum)
return make_timing_interval(lower=lower, upper=upper, center=timing.mean)
def compute_timing_interval(timing):

58
python/test/test_nvbench_compare.py
View File

@@ -198,6 +198,9 @@ def make_gpu_timing_data(
sample_values=None,
frequency_values=None,
):
if stdev is None:
stdev = nvbench_compare.derive_absolute_dispersion(stdev_relative, mean)
return nvbench_compare.GpuTimingData(
minimum=minimum,
maximum=maximum,
@@ -287,10 +290,10 @@ def test_compare_benches_accepts_matching_duplicate_state_counts(
)
assert run_data.stats.config_count == 3
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 3
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 3
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -371,10 +374,10 @@ def test_compare_benches_matches_duplicate_states_after_axis_filter(
)
assert run_data.stats.config_count == 1
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 1
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -485,10 +488,10 @@ def test_compare_benches_counts_non_finite_centers_as_unknown(
)
assert run_data.stats.config_count == 3
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 1
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 2
@@ -737,6 +740,18 @@ def test_gpu_timing_data_parses_quartiles_and_sm_clock_rate_mean(nvbench_compare
assert timing.frequencies is None
def test_gpu_timing_data_derives_stdev_from_relative_stdev(nvbench_compare):
timing = nvbench_compare.extract_gpu_timing_data(
[
make_summary(nvbench_compare, "GPU_TIME_MEAN_TAG", "2.0"),
make_summary(nvbench_compare, "GPU_TIME_STDEV_RELATIVE_TAG", "0.25"),
],
)
assert timing.stdev == pytest.approx(0.5)
assert timing.stdev_relative == pytest.approx(0.25)
def test_gpu_timing_data_accepts_legacy_ir_tags(nvbench_compare):
timing = nvbench_compare.extract_gpu_timing_data(
[
@@ -833,6 +848,17 @@ def test_compare_gpu_timings_classifies_common_cases(tmp_path, nvbench_compare):
assert undecided.max_noise == pytest.approx(0.05)
assert undecided.reason.code == "noise_too_high"
legacy_same = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(nvbench_compare, mean=1.0, stdev_relative=0.005),
make_gpu_timing_data(nvbench_compare, mean=1.001, stdev_relative=0.005),
)
assert legacy_same is not None
assert legacy_same.status == nvbench_compare.ComparisonStatus.SAME
assert legacy_same.reason.code == "same_without_clock_rate"
assert legacy_same.ref_interval is not None
assert legacy_same.ref_interval.lower == pytest.approx(0.995)
assert legacy_same.ref_interval.upper == pytest.approx(1.005)
partial_robust = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(
nvbench_compare,
@@ -1906,10 +1932,10 @@ def test_plot_along_skips_states_without_selected_axis(monkeypatch, nvbench_comp
)
assert run_data.stats.config_count == 3
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 3
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 3
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
assert xscale_calls == ["log"]
assert yscale_calls == ["log"]
@@ -2105,10 +2131,10 @@ def test_compare_benches_pairs_filtered_devices_by_position(
)
assert run_data.stats.config_count == 1
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 1
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -2164,10 +2190,10 @@ def test_axis_filter_applies_to_most_recent_benchmark(monkeypatch, nvbench_compa
)
assert run_data.stats.config_count == 3
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 3
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 3
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -2225,10 +2251,10 @@ def test_global_axis_filter_does_not_select_unmatched_benchmark(
)
assert run_data.stats.config_count == 1
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 1
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 1
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0
@@ -2286,10 +2312,10 @@ def test_global_axis_filter_applies_to_each_selected_benchmark(
)
assert run_data.stats.config_count == 2
assert run_data.stats.pass_count == 0
assert run_data.stats.pass_count == 2
assert run_data.stats.improvement_count == 0
assert run_data.stats.regression_count == 0
assert run_data.stats.undecided_count == 2
assert run_data.stats.undecided_count == 0
assert run_data.stats.unknown_count == 0