NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-07-12 18:17:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add nvbench_compare display modes and interval-based table views

Browse Source 
Extend nvbench_compare with multiple table display modes and richer interval
formatting for timing comparisons.

Highlights:
  - add `--display` with `intervals`, `legacy`, and `explain` modes
  - keep `legacy` output using scalar Diff/%Diff
  - make `intervals` the default, showing compact center-plus-delta timing
    intervals
  - add `explain` mode with explicit `[L | C | H]` interval rendering and
    self-describing headers
  - compute and store diff and relative-diff intervals in SummaryComparison
  - add formatting helpers for absolute and relative interval displays
  - make default preset slightly more permissive by lowering
    `bulk_same_sample_coverage` to 0.97

Add focused tests covering:
  - diff/%diff interval computation
  - compact and explicit interval formatting
  - default, legacy, and explain table layouts
  - CLI propagation of `--display` and preset selection
This commit is contained in:
Oleksandr Pavlyk
2026-06-04 08:49:06 -05:00
parent 541356e60a
commit 5ceba06ce2
2 changed files with 400 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
python/scripts/nvbench_compare.py
View File

@@ -77,7 +77,7 @@ COMPARISON_THRESHOLD_PRESET_VALUES = {
"same_center_relative": 0.005,
"same_overlap_fraction": 0.5,
"same_relative_dispersion_ceiling": 0.02,
"bulk_same_sample_coverage": 0.99,
"bulk_same_sample_coverage": 0.97,
"bulk_same_support_coverage": 0.80,
"bulk_support_rare_sample_fraction": 0.001,
"bulk_support_max_removed_sample_fraction": 0.01,
@@ -204,6 +204,8 @@ class TimingDecision:
@dataclass(frozen=True)
class SummaryComparison:
ref_interval: TimingInterval | None
cmp_interval: TimingInterval | None
ref_estimate: TimeEstimate
cmp_estimate: TimeEstimate
ref_time: float
@@ -212,6 +214,8 @@ class SummaryComparison:
cmp_noise: float | None
diff: float
frac_diff: float
diff_interval: tuple[float, float] | None
frac_diff_interval: tuple[float, float] | None
max_noise: float | None
status: ComparisonStatus
reason: DecisionReason
@@ -680,6 +684,20 @@ def compare_intervals_for_clear_gap(ref_interval, cmp_interval, thresholds):
return None
def compute_diff_interval(ref_interval, cmp_interval):
return (
cmp_interval.lower - ref_interval.upper,
cmp_interval.upper - ref_interval.lower,
)
def compute_frac_diff_interval(ref_interval, cmp_interval):
return (
cmp_interval.lower / ref_interval.upper - 1.0,
cmp_interval.upper / ref_interval.lower - 1.0,
)
def centers_are_close(ref_center, cmp_center, thresholds):
if not is_positive_finite(ref_center) or not is_positive_finite(cmp_center):
return False
@@ -1240,8 +1258,15 @@ def compare_gpu_timings(ref_timing, cmp_timing, comparison_thresholds=None):
cmp_noise = cmp_estimate.relative_dispersion
ref_noise = ref_estimate.relative_dispersion
ref_interval = compute_timing_interval(ref_timing)
cmp_interval = compute_timing_interval(cmp_timing)
diff = cmp_time - ref_time
frac_diff = diff / ref_time
diff_interval = None
frac_diff_interval = None
if ref_interval is not None and cmp_interval is not None:
diff_interval = compute_diff_interval(ref_interval, cmp_interval)
frac_diff_interval = compute_frac_diff_interval(ref_interval, cmp_interval)
if not has_finite_noise(ref_noise) or not has_finite_noise(cmp_noise):
max_noise = None
@@ -1266,6 +1291,8 @@ def compare_gpu_timings(ref_timing, cmp_timing, comparison_thresholds=None):
decision = bulk_decision
return SummaryComparison(
ref_interval=ref_interval,
cmp_interval=cmp_interval,
ref_estimate=ref_estimate,
cmp_estimate=cmp_estimate,
ref_time=ref_time,
@@ -1274,6 +1301,8 @@ def compare_gpu_timings(ref_timing, cmp_timing, comparison_thresholds=None):
cmp_noise=cmp_noise,
diff=diff,
frac_diff=frac_diff,
diff_interval=diff_interval,
frac_diff_interval=frac_diff_interval,
max_noise=max_noise,
status=decision.status,
reason=decision.reason,
@@ -1486,6 +1515,82 @@ def format_duration(seconds):
return f"{seconds * multiplier:0.3f} {units}"
def select_duration_units(*seconds_values):
max_abs_seconds = max(abs(value) for value in seconds_values)
if max_abs_seconds >= 1:
return 1.0, "s"
if max_abs_seconds >= 1e-3:
return 1e3, "ms"
return 1e6, "us"
def duration_precision_for_center(center, delta_multiplier):
center_multiplier, _ = select_duration_units(center)
center_quantum = 10.0**-3 * (delta_multiplier / center_multiplier)
if center_quantum >= 1.0:
return 0
return int(math.ceil(-math.log10(center_quantum)))
def format_duration_range(bounds):
if bounds is None:
return "n/a"
lower, upper = bounds
multiplier, units = select_duration_units(lower, upper)
return f"[{lower * multiplier:0.2f}, {upper * multiplier:0.2f}] {units}"
def format_timing_with_interval(center, interval):
if center is None:
return "n/a"
if interval is None:
return format_duration(center)
lower_delta = interval.lower - interval.center
upper_delta = interval.upper - interval.center
multiplier, units = select_duration_units(lower_delta, upper_delta)
precision = duration_precision_for_center(center, multiplier)
return (
f"{format_duration(center)} "
f"[{lower_delta * multiplier:+0.{precision}f}, "
f"{upper_delta * multiplier:+0.{precision}f}] {units}"
)
def longest_common_prefix(strings):
if not strings:
return ""
prefix = strings[0]
for text in strings[1:]:
while not text.startswith(prefix):
prefix = prefix[:-1]
if not prefix:
return ""
return prefix
def format_timing_with_explicit_interval(center, interval):
if center is None:
return "n/a"
if interval is None:
return format_duration(center)
multiplier, units = select_duration_units(
interval.lower, interval.center, interval.upper
)
values = [
f"{interval.lower * multiplier:0.3f}",
f"{interval.center * multiplier:0.3f}",
f"{interval.upper * multiplier:0.3f}",
]
prefix = longest_common_prefix(values)
if not prefix:
return f"[{values[0]} | {values[1]} | {values[2]}] {units}"
suffixes = [value[len(prefix) :] for value in values]
return f"{prefix}[{suffixes[0]} | {suffixes[1]} | {suffixes[2]}] {units}"
def format_percentage(percentage):
if percentage is None:
return "n/a"
@@ -1496,6 +1601,79 @@ def format_percentage(percentage):
return f"{percentage * 100.0:0.2f}%"
def format_percentage_bounds(bounds, status):
if bounds is None:
return "n/a"
lower, upper = bounds
if status == ComparisonStatus.FAST:
return f"<= {upper * 100.0:+0.1f}%"
if status == ComparisonStatus.SLOW:
return f">= {lower * 100.0:+0.1f}%"
return f"in [{lower * 100.0:+0.1f}%, {upper * 100.0:+0.1f}%]"
def get_display_headers(display):
if display == "legacy":
return (
[
"Ref Time",
"Ref Noise",
"Cmp Time",
"Cmp Noise",
"Diff",
"%Diff",
"Status",
],
["right", "right", "right", "right", "right", "right", "center"],
)
if display == "explain":
return (
[
"Ref [L | C | H]",
"Cmp [L | C | H]",
"Ref Noise",
"Cmp Noise",
"Reason",
"Status",
],
["right", "right", "right", "right", "left", "center"],
)
return (
["Ref", "Cmp", "Status"],
["right", "right", "center"],
)
def append_display_row(row, comparison, no_color, display):
if display == "legacy":
row.append(format_duration(comparison.ref_time))
row.append(format_percentage(comparison.ref_noise))
row.append(format_duration(comparison.cmp_time))
row.append(format_percentage(comparison.cmp_noise))
row.append(format_duration(comparison.diff))
row.append(format_percentage(comparison.frac_diff))
row.append(colorize_comparison_status(comparison.status, no_color))
return
row.append(
format_timing_with_interval(comparison.ref_time, comparison.ref_interval)
)
row.append(
format_timing_with_interval(comparison.cmp_time, comparison.cmp_interval)
)
if display == "explain":
row[-2] = format_timing_with_explicit_interval(
comparison.ref_time, comparison.ref_interval
)
row[-1] = format_timing_with_explicit_interval(
comparison.cmp_time, comparison.cmp_interval
)
row.append(format_percentage(comparison.ref_noise))
row.append(format_percentage(comparison.cmp_noise))
row.append(comparison.reason.code)
row.append(colorize_comparison_status(comparison.status, no_color))
def has_finite_noise(noise):
return noise is not None and math.isfinite(noise)
@@ -1618,6 +1796,7 @@ def compare_benches(
ref_json_dir=None,
cmp_json_dir=None,
comparison_thresholds=None,
display="intervals",
):
if comparison_thresholds is None:
comparison_thresholds = ComparisonThresholds()
@@ -1664,21 +1843,9 @@ def compare_benches(
headers = [x["name"] for x in axes]
colalign = ["center"] * len(headers)
headers.append("Ref Time")
colalign.append("right")
headers.append("Ref Noise")
colalign.append("right")
headers.append("Cmp Time")
colalign.append("right")
headers.append("Cmp Noise")
colalign.append("right")
headers.append("Diff")
colalign.append("right")
headers.append("%Diff")
colalign.append("right")
headers.append("Status")
colalign.append("center")
display_headers, display_colalign = get_display_headers(display)
headers.extend(display_headers)
colalign.extend(display_colalign)
for cmp_device_index, cmp_device_id in enumerate(cmp_device_ids):
ref_device_id = ref_device_ids[cmp_device_index]
@@ -1774,17 +1941,9 @@ def compare_benches(
)
run_data.stats.record(comparison.status, comparison.reason)
status = colorize_comparison_status(comparison.status, no_color)
if abs(comparison.frac_diff) >= threshold:
axis_filters = matching_axis_filters(cmp_state, axis_filter_groups)
row.append(format_duration(comparison.ref_time))
row.append(format_percentage(comparison.ref_noise))
row.append(format_duration(comparison.cmp_time))
row.append(format_percentage(comparison.cmp_noise))
row.append(format_duration(comparison.diff))
row.append(format_percentage(comparison.frac_diff))
row.append(status)
append_display_row(row, comparison, no_color, display)
rows.append(row)
if plot:
@@ -1953,6 +2112,12 @@ def main() -> int:
default="default",
help="comparison threshold preset",
)
parser.add_argument(
"--display",
choices=["intervals", "legacy", "explain"],
default="intervals",
help="comparison table display mode",
)
parser.add_argument(
"--plot-along", type=str, dest="plot_along", default=None, help="plot results"
)
@@ -2091,17 +2256,18 @@ def main() -> int:
run_data,
ref_root["benchmarks"],
cmp_root["benchmarks"],
args.threshold,
args.plot_along,
args.plot,
args.dark,
filter_plan,
args.no_color,
reference_device_filter,
compare_device_filter,
os.path.dirname(ref),
os.path.dirname(comp),
comparison_thresholds,
threshold=args.threshold,
plot_along=args.plot_along,
plot=args.plot,
dark=args.dark,
filter_plan=filter_plan,
no_color=args.no_color,
reference_device_filter=reference_device_filter,
compare_device_filter=compare_device_filter,
ref_json_dir=os.path.dirname(ref),
cmp_json_dir=os.path.dirname(comp),
comparison_thresholds=comparison_thresholds,
display=args.display,
)
except ValueError as exc:
print(str(exc))

201
python/test/test_nvbench_compare.py
View File

@@ -511,6 +511,8 @@ def test_compare_gpu_timings_classifies_common_cases(nvbench_compare):
assert fast is not None
assert fast.status == nvbench_compare.ComparisonStatus.FAST
assert fast.reason.code == "clear_gap_confirmed_by_summary_cycles"
assert fast.diff_interval == pytest.approx((-0.5, -0.05))
assert fast.frac_diff_interval == pytest.approx((-0.3846153846, -0.05))
slow = nvbench_compare.compare_gpu_timings(
ref_interval_timing,
@@ -528,6 +530,8 @@ def test_compare_gpu_timings_classifies_common_cases(nvbench_compare):
assert slow is not None
assert slow.status == nvbench_compare.ComparisonStatus.SLOW
assert slow.reason.code == "clear_gap_confirmed_by_summary_cycles"
assert slow.diff_interval == pytest.approx((0.1, 0.55))
assert slow.frac_diff_interval == pytest.approx((0.0769230769, 0.55))
same = nvbench_compare.compare_gpu_timings(
ref_interval_timing,
@@ -545,6 +549,8 @@ def test_compare_gpu_timings_classifies_common_cases(nvbench_compare):
assert same is not None
assert same.status == nvbench_compare.ComparisonStatus.SAME
assert same.reason.code == "same_confirmed_by_cycles"
assert same.diff_interval == pytest.approx((-0.28, 0.28))
assert same.frac_diff_interval == pytest.approx((-0.2153846154, 0.28))
weak_overlap = nvbench_compare.compare_gpu_timings(
make_gpu_timing_data(
@@ -741,6 +747,48 @@ def test_compare_gpu_timings_uses_bulk_data_to_confirm_same(nvbench_compare):
assert comparison.reason.code == "bulk_same"
def test_format_diff_and_percent_ranges(nvbench_compare):
assert nvbench_compare.format_duration_range((-12e-6, 8e-6)) == "[-12.00, 8.00] us"
assert (
nvbench_compare.format_percentage_bounds(
(-0.2153846154, 0.28), nvbench_compare.ComparisonStatus.UNDECIDED
)
== "in [-21.5%, +28.0%]"
)
assert (
nvbench_compare.format_percentage_bounds(
(-0.3076923077, -0.05), nvbench_compare.ComparisonStatus.FAST
)
== "<= -5.0%"
)
assert (
nvbench_compare.format_percentage_bounds(
(0.0769230769, 0.55), nvbench_compare.ComparisonStatus.SLOW
)
== ">= +7.7%"
)
def test_format_timing_with_interval(nvbench_compare):
interval = nvbench_compare.TimingInterval(
lower=0.002237, upper=0.002389, center=0.0023
)
assert (
nvbench_compare.format_timing_with_interval(0.0023, interval)
== "2.300 ms [-63, +89] us"
)
def test_format_timing_with_explicit_interval(nvbench_compare):
interval = nvbench_compare.TimingInterval(
lower=0.001434, upper=0.001458, center=0.001446
)
assert (
nvbench_compare.format_timing_with_explicit_interval(0.001446, interval)
== "1.4[34 | 46 | 58] ms"
)
def test_compare_gpu_timings_keeps_bulk_mismatch_undecided(nvbench_compare):
ref_timing = make_gpu_timing_data(
nvbench_compare,
@@ -1321,7 +1369,9 @@ def test_get_comparison_thresholds_returns_named_presets(nvbench_compare):
strict = nvbench_compare.get_comparison_thresholds("strict")
permissive = nvbench_compare.get_comparison_thresholds("permissive")
assert default == nvbench_compare.ComparisonThresholds()
assert default == nvbench_compare.ComparisonThresholds(
**nvbench_compare.COMPARISON_THRESHOLD_PRESET_VALUES["default"]
)
assert strict.clear_gap_relative > default.clear_gap_relative
assert strict.same_center_relative < default.same_center_relative
assert strict.bulk_same_sample_coverage > default.bulk_same_sample_coverage
@@ -1330,6 +1380,141 @@ def test_get_comparison_thresholds_returns_named_presets(nvbench_compare):
assert permissive.bulk_same_support_coverage < default.bulk_same_support_coverage
def test_compare_benches_defaults_to_interval_display(monkeypatch, nvbench_compare):
run_data = make_comparison_run_data(nvbench_compare)
captured = {}
def fake_tabulate(rows, headers, *args, **kwargs):
captured["rows"] = rows
captured["headers"] = headers
return ""
monkeypatch.setattr(nvbench_compare.tabulate, "tabulate", fake_tabulate)
ref_benches = [make_benchmark([make_state(nvbench_compare, "state", mean="1.0")])]
cmp_benches = [make_benchmark([make_state(nvbench_compare, "state", mean="1.01")])]
nvbench_compare.compare_benches(
run_data,
ref_benches,
cmp_benches,
threshold=0.0,
plot_along=None,
plot=False,
dark=False,
filter_plan=make_filter_plan(nvbench_compare),
no_color=True,
)
assert captured["headers"][-3:] == ["Ref", "Cmp", "Status"]
row = captured["rows"][0]
assert row[-3].startswith("1.000 s")
assert row[-2].startswith("1.010 s")
def test_compare_benches_legacy_display_uses_scalar_diff(monkeypatch, nvbench_compare):
run_data = make_comparison_run_data(nvbench_compare)
captured = {}
def fake_tabulate(rows, headers, *args, **kwargs):
captured["rows"] = rows
captured["headers"] = headers
return ""
monkeypatch.setattr(nvbench_compare.tabulate, "tabulate", fake_tabulate)
ref_benches = [make_benchmark([make_state(nvbench_compare, "state", mean="1.0")])]
cmp_benches = [make_benchmark([make_state(nvbench_compare, "state", mean="1.01")])]
nvbench_compare.compare_benches(
run_data,
ref_benches,
cmp_benches,
threshold=0.0,
plot_along=None,
plot=False,
dark=False,
filter_plan=make_filter_plan(nvbench_compare),
no_color=True,
display="legacy",
)
assert captured["headers"][-7:] == [
"Ref Time",
"Ref Noise",
"Cmp Time",
"Cmp Noise",
"Diff",
"%Diff",
"Status",
]
row = captured["rows"][0]
assert row[-7] == "1.000 s"
assert row[-5] == "1.010 s"
assert row[-3] == "10.000 ms"
assert row[-2] == "1.00%"
def test_compare_benches_explain_display_uses_explicit_intervals(
monkeypatch, nvbench_compare
):
run_data = make_comparison_run_data(nvbench_compare)
captured = {}
def fake_tabulate(rows, headers, *args, **kwargs):
captured["rows"] = rows
captured["headers"] = headers
return ""
monkeypatch.setattr(nvbench_compare.tabulate, "tabulate", fake_tabulate)
ref_state = make_state(nvbench_compare, "state", mean="1.0")
ref_state["summaries"].extend(
[
make_summary(nvbench_compare, "GPU_TIME_MIN_TAG", "1.0"),
make_summary(nvbench_compare, "GPU_TIME_Q1_TAG", "1.01"),
make_summary(nvbench_compare, "GPU_TIME_MEDIAN_TAG", "1.02"),
make_summary(nvbench_compare, "GPU_TIME_Q3_TAG", "1.03"),
make_summary(nvbench_compare, "GPU_SM_CLOCK_RATE_MEAN_TAG", "100.0"),
]
)
cmp_state = make_state(nvbench_compare, "state", mean="1.01")
cmp_state["summaries"].extend(
[
make_summary(nvbench_compare, "GPU_TIME_MIN_TAG", "1.01"),
make_summary(nvbench_compare, "GPU_TIME_Q1_TAG", "1.02"),
make_summary(nvbench_compare, "GPU_TIME_MEDIAN_TAG", "1.03"),
make_summary(nvbench_compare, "GPU_TIME_Q3_TAG", "1.04"),
make_summary(nvbench_compare, "GPU_SM_CLOCK_RATE_MEAN_TAG", "100.0"),
]
)
nvbench_compare.compare_benches(
run_data,
[make_benchmark([ref_state])],
[make_benchmark([cmp_state])],
threshold=0.0,
plot_along=None,
plot=False,
dark=False,
filter_plan=make_filter_plan(nvbench_compare),
no_color=True,
display="explain",
)
assert captured["headers"][-6:] == [
"Ref [L | C | H]",
"Cmp [L | C | H]",
"Ref Noise",
"Cmp Noise",
"Reason",
"Status",
]
row = captured["rows"][0]
assert row[-6] == "1.0[00 | 20 | 30] s"
assert row[-5] == "1.0[10 | 30 | 40] s"
def test_main_passes_selected_preset_to_compare_benches(monkeypatch, nvbench_compare):
devices = [{"id": 0, "name": "Test GPU"}]
root = {
@@ -1341,16 +1526,26 @@ def test_main_passes_selected_preset_to_compare_benches(monkeypatch, nvbench_com
monkeypatch.setattr(nvbench_compare.reader, "read_file", lambda _: root)
def fake_compare_benches(*args, **kwargs):
captured["comparison_thresholds"] = args[-1]
captured["comparison_thresholds"] = kwargs["comparison_thresholds"]
captured["display"] = kwargs["display"]
monkeypatch.setattr(nvbench_compare, "compare_benches", fake_compare_benches)
monkeypatch.setattr(
sys,
"argv",
["nvbench_compare", "--preset", "strict", "ref.json", "cmp.json"],
[
"nvbench_compare",
"--preset",
"strict",
"--display",
"explain",
"ref.json",
"cmp.json",
],
)
assert nvbench_compare.main() == 0
assert captured[
"comparison_thresholds"
] == nvbench_compare.get_comparison_thresholds("strict")
assert captured["display"] == "explain"