NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-07-17 00:58:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow timing-only bulk data to confirm SAME

Browse Source 
Permit nvbench-compare to use bulk timing samples to confirm SAME even when
sample frequency sidecars are unavailable. This lets legacy --jsonbin outputs,
which recorded sample times but not frequencies, avoid AMBG results when
summary noise is high but timing coverage clearly supports equality.

Cycle coverage is still required when frequency samples are present, and
FAST/SLOW clear-gap confirmation remains cycle-based.

Add regression coverage for noisy bulk samples that exceed the SAME noise
ceiling but are classified SAME from bulk timing/cycle coverage, including the
timing-only legacy-data path.
This commit is contained in:
Oleksandr Pavlyk
2026-07-14 14:28:16 -05:00
parent 25005fc9c4
commit 149a2d257a
2 changed files with 197 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

124
python/scripts/nvbench_compare.py
View File

@@ -585,6 +585,8 @@ REASON_DISPLAY_CODES = {
"bulk_same": "bulk-same",
"bulk_time_data_unusable": "bt-bad",
"bulk_time_same": "bt-same",
"bulk_time_same_confirmed_by_summary_cycles": "bt-same-sc",
"bulk_time_same_without_cycles": "bt-same-no-cyc",
"bulk_time_support_mismatch": "bt-sup-miss",
"centers_not_close": "centers-far",
"clear_gap_confirmed_by_bulk_cycles": "bc-gap",
@@ -1895,24 +1897,53 @@ def compare_values_for_bulk_same(ref_values, cmp_values, *, label, thresholds):
def compare_timings_for_bulk_same(ref_timing, cmp_timing, thresholds):
ref_bulk = get_bulk_time_and_cycles(ref_timing)
cmp_bulk = get_bulk_time_and_cycles(cmp_timing)
if ref_bulk is None or cmp_bulk is None:
ref_times = positive_finite_array(ref_timing.samples)
cmp_times = positive_finite_array(cmp_timing.samples)
if ref_times is None or cmp_times is None:
if (
ref_timing.sample_source is not None
and cmp_timing.sample_source is not None
):
return make_decision(
ComparisonStatus.UNDECIDED,
"bulk_time_data_unusable",
"bulk time data is empty or unusable",
)
return make_decision(
ComparisonStatus.UNDECIDED,
"bulk_data_unavailable",
"bulk sample time and frequency data are unavailable",
"bulk sample time data are unavailable",
)
ref_times, ref_cycles = ref_bulk
cmp_times, cmp_cycles = cmp_bulk
time_decision = compare_values_for_bulk_same(
ref_times, cmp_times, label="time", thresholds=thresholds
)
if time_decision.status != ComparisonStatus.SAME:
return time_decision
if ref_timing.frequency_source is None or cmp_timing.frequency_source is None:
return make_decision(
ComparisonStatus.SAME,
"bulk_time_same_without_cycles",
"bulk time nearest-neighbor coverage supports same; sample frequencies are unavailable",
)
ref_frequencies = positive_finite_array(ref_timing.frequencies)
cmp_frequencies = positive_finite_array(cmp_timing.frequencies)
if (
ref_frequencies is None
or cmp_frequencies is None
or len(ref_times) != len(ref_frequencies)
or len(cmp_times) != len(cmp_frequencies)
):
return make_decision(
ComparisonStatus.UNDECIDED,
"bulk_cycle_data_unusable",
"bulk cycle data is empty or unusable",
)
ref_cycles = ref_times * ref_frequencies
cmp_cycles = cmp_times * cmp_frequencies
cycle_decision = compare_values_for_bulk_same(
ref_cycles, cmp_cycles, label="cycle", thresholds=thresholds
)
@@ -1926,6 +1957,34 @@ def compare_timings_for_bulk_same(ref_timing, cmp_timing, thresholds):
)
def compare_timings_for_bulk_same_if_available(ref_timing, cmp_timing, thresholds):
decision = compare_timings_for_bulk_same(ref_timing, cmp_timing, thresholds)
if decision.reason.code == "bulk_data_unavailable":
return None
return decision
def confirm_bulk_time_same_without_cycles(
ref_timing, cmp_timing, ref_interval, cmp_interval, thresholds
):
decision = confirm_same_with_clock_rate(
ref_timing, cmp_timing, ref_interval, cmp_interval, thresholds
)
if decision.reason.code == "same_without_clock_rate":
return make_decision(
ComparisonStatus.SAME,
"bulk_time_same_without_cycles",
"bulk time nearest-neighbor coverage supports same; cycle data are unavailable",
)
if decision.status == ComparisonStatus.SAME:
return make_decision(
ComparisonStatus.SAME,
"bulk_time_same_confirmed_by_summary_cycles",
"bulk time nearest-neighbor coverage supports same and SM-clock-adjusted summary intervals confirm same",
)
return decision
def compare_timings_for_same(
ref_timing, cmp_timing, ref_noise, cmp_noise, ref_interval, cmp_interval, thresholds
):
@@ -1935,12 +1994,6 @@ def compare_timings_for_same(
"noise_unavailable",
"relative dispersion is unavailable, negative, or non-finite",
)
if max(ref_noise, cmp_noise) > thresholds.same_relative_dispersion_ceiling:
return make_decision(
ComparisonStatus.UNDECIDED,
"noise_too_high",
"relative dispersion is too high to declare same",
)
if ref_interval is None or cmp_interval is None:
return make_decision(
@@ -1953,6 +2006,29 @@ def compare_timings_for_same(
if decision.status != ComparisonStatus.SAME:
return decision
bulk_decision = compare_timings_for_bulk_same_if_available(
ref_timing, cmp_timing, thresholds
)
if (
bulk_decision is not None
and bulk_decision.reason.code == "bulk_time_same_without_cycles"
):
bulk_decision = confirm_bulk_time_same_without_cycles(
ref_timing, cmp_timing, ref_interval, cmp_interval, thresholds
)
if max(ref_noise, cmp_noise) > thresholds.same_relative_dispersion_ceiling:
if bulk_decision is not None:
return bulk_decision
return make_decision(
ComparisonStatus.UNDECIDED,
"noise_too_high",
"relative dispersion is too high to declare same",
)
if bulk_decision is not None:
return bulk_decision
return confirm_same_with_clock_rate(
ref_timing, cmp_timing, ref_interval, cmp_interval, thresholds
)
@@ -2175,21 +2251,15 @@ def compare_gpu_timings(ref_timing, cmp_timing, comparison_thresholds=None):
"no_clear_gap",
"missing_interval",
}:
bulk_decision = compare_timings_for_bulk_same(
ref_timing, cmp_timing, comparison_thresholds
decision = compare_timings_for_same(
ref_timing,
cmp_timing,
ref_noise,
cmp_noise,
ref_interval,
cmp_interval,
comparison_thresholds,
)
if bulk_decision.reason.code == "bulk_data_unavailable":
decision = compare_timings_for_same(
ref_timing,
cmp_timing,
ref_noise,
cmp_noise,
ref_interval,
cmp_interval,
comparison_thresholds,
)
else:
decision = bulk_decision
return SummaryComparison(
ref_interval=ref_interval,

102
python/test/test_nvbench_compare.py
View File

@@ -974,7 +974,7 @@ def test_compare_gpu_timings_classifies_common_cases(tmp_path, nvbench_compare):
assert undecided.diff == pytest.approx(0.03)
assert undecided.frac_diff == pytest.approx(0.03)
assert undecided.max_noise == pytest.approx(0.05)
assert undecided.reason.code == "noise_too_high"
assert undecided.reason.code == "centers_not_close"
legacy_same = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(nvbench_compare, mean=1.0, stdev_relative=0.005),
@@ -1241,6 +1241,104 @@ def test_compare_gpu_timings_classifies_common_cases(tmp_path, nvbench_compare):
assert noisy_same.status == nvbench_compare.ComparisonStatus.UNDECIDED
assert noisy_same.reason.code == "noise_too_high"
noisy_bulk_same = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(
nvbench_compare,
sample_values=[1.0, 1.0, 1.1, 1.1],
frequency_values=[100.0] * 4,
),
make_gpu_timing_data(
nvbench_compare,
sample_values=[1.0, 1.0, 1.1, 1.1],
frequency_values=[100.0] * 4,
),
)
assert noisy_bulk_same is not None
assert noisy_bulk_same.status == nvbench_compare.ComparisonStatus.SAME
assert (
noisy_bulk_same.max_noise
> nvbench_compare.get_default_thresholds().same_relative_dispersion_ceiling
)
assert noisy_bulk_same.reason.code == "bulk_same"
noisy_bulk_time_same = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(
nvbench_compare,
sample_values=[1.0, 1.0, 1.1, 1.1],
),
make_gpu_timing_data(
nvbench_compare,
sample_values=[1.0, 1.0, 1.1, 1.1],
),
)
assert noisy_bulk_time_same is not None
assert noisy_bulk_time_same.status == nvbench_compare.ComparisonStatus.SAME
assert (
noisy_bulk_time_same.max_noise
> nvbench_compare.get_default_thresholds().same_relative_dispersion_ceiling
)
assert noisy_bulk_time_same.reason.code == "bulk_time_same_without_cycles"
noisy_bulk_time_same_with_summary_cycles = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(
nvbench_compare,
sm_clock_rate_mean=100.0,
sample_values=[1.0, 1.0, 1.1, 1.1],
),
make_gpu_timing_data(
nvbench_compare,
sm_clock_rate_mean=100.0,
sample_values=[1.0, 1.0, 1.1, 1.1],
),
)
assert noisy_bulk_time_same_with_summary_cycles is not None
assert (
noisy_bulk_time_same_with_summary_cycles.status
== nvbench_compare.ComparisonStatus.SAME
)
assert (
noisy_bulk_time_same_with_summary_cycles.reason.code
== "bulk_time_same_confirmed_by_summary_cycles"
)
noisy_bulk_time_summary_cycle_disagreement = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(
nvbench_compare,
sm_clock_rate_mean=100.0,
sample_values=[1.0, 1.0, 1.1, 1.1],
),
make_gpu_timing_data(
nvbench_compare,
sm_clock_rate_mean=200.0,
sample_values=[1.0, 1.0, 1.1, 1.1],
),
)
assert noisy_bulk_time_summary_cycle_disagreement is not None
assert (
noisy_bulk_time_summary_cycle_disagreement.status
== nvbench_compare.ComparisonStatus.UNDECIDED
)
assert (
noisy_bulk_time_summary_cycle_disagreement.reason.code
== "cycle_same_not_confirmed"
)
noisy_bulk_bad_cycles = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(
nvbench_compare,
sample_values=[1.0, 1.0, 1.1, 1.1],
frequency_values=[0.0, 100.0, 100.0, 100.0],
),
make_gpu_timing_data(
nvbench_compare,
sample_values=[1.0, 1.0, 1.1, 1.1],
frequency_values=[100.0] * 4,
),
)
assert noisy_bulk_bad_cycles is not None
assert noisy_bulk_bad_cycles.status == nvbench_compare.ComparisonStatus.UNDECIDED
assert noisy_bulk_bad_cycles.reason.code == "bulk_cycle_data_unusable"
clock_disagreement = nvbench_compare.compare_gpu_timings(
ref_interval_timing,
make_gpu_timing_data(
@@ -2528,7 +2626,7 @@ def test_main_prints_undecided_reason_summary(monkeypatch, capsys, nvbench_compa
"devices": devices,
"benchmarks": [
make_benchmark(
[make_state(nvbench_compare, "state", mean="1.01", noise="0.05")]
[make_state(nvbench_compare, "state", mean="1.001", noise="0.05")]
)
],
}