From 5ceba06ce2d592423a4f53a1cb9820882bf2890c Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Thu, 4 Jun 2026 08:49:06 -0500 Subject: [PATCH] Add nvbench_compare display modes and interval-based table views Extend nvbench_compare with multiple table display modes and richer interval formatting for timing comparisons. Highlights: - add `--display` with `intervals`, `legacy`, and `explain` modes - keep `legacy` output using scalar Diff/%Diff - make `intervals` the default, showing compact center-plus-delta timing intervals - add `explain` mode with explicit `[L | C | H]` interval rendering and self-describing headers - compute and store diff and relative-diff intervals in SummaryComparison - add formatting helpers for absolute and relative interval displays - make default preset slightly more permissive by lowering `bulk_same_sample_coverage` to 0.97 Add focused tests covering: - diff/%diff interval computation - compact and explicit interval formatting - default, legacy, and explain table layouts - CLI propagation of `--display` and preset selection --- python/scripts/nvbench_compare.py | 238 +++++++++++++++++++++++----- python/test/test_nvbench_compare.py | 201 ++++++++++++++++++++++- 2 files changed, 400 insertions(+), 39 deletions(-) diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index 7f77910..b86be01 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -77,7 +77,7 @@ COMPARISON_THRESHOLD_PRESET_VALUES = { "same_center_relative": 0.005, "same_overlap_fraction": 0.5, "same_relative_dispersion_ceiling": 0.02, - "bulk_same_sample_coverage": 0.99, + "bulk_same_sample_coverage": 0.97, "bulk_same_support_coverage": 0.80, "bulk_support_rare_sample_fraction": 0.001, "bulk_support_max_removed_sample_fraction": 0.01, @@ -204,6 +204,8 @@ class TimingDecision: @dataclass(frozen=True) class SummaryComparison: + ref_interval: TimingInterval | None + cmp_interval: TimingInterval | None ref_estimate: TimeEstimate cmp_estimate: TimeEstimate ref_time: float @@ -212,6 +214,8 @@ class SummaryComparison: cmp_noise: float | None diff: float frac_diff: float + diff_interval: tuple[float, float] | None + frac_diff_interval: tuple[float, float] | None max_noise: float | None status: ComparisonStatus reason: DecisionReason @@ -680,6 +684,20 @@ def compare_intervals_for_clear_gap(ref_interval, cmp_interval, thresholds): return None +def compute_diff_interval(ref_interval, cmp_interval): + return ( + cmp_interval.lower - ref_interval.upper, + cmp_interval.upper - ref_interval.lower, + ) + + +def compute_frac_diff_interval(ref_interval, cmp_interval): + return ( + cmp_interval.lower / ref_interval.upper - 1.0, + cmp_interval.upper / ref_interval.lower - 1.0, + ) + + def centers_are_close(ref_center, cmp_center, thresholds): if not is_positive_finite(ref_center) or not is_positive_finite(cmp_center): return False @@ -1240,8 +1258,15 @@ def compare_gpu_timings(ref_timing, cmp_timing, comparison_thresholds=None): cmp_noise = cmp_estimate.relative_dispersion ref_noise = ref_estimate.relative_dispersion + ref_interval = compute_timing_interval(ref_timing) + cmp_interval = compute_timing_interval(cmp_timing) diff = cmp_time - ref_time frac_diff = diff / ref_time + diff_interval = None + frac_diff_interval = None + if ref_interval is not None and cmp_interval is not None: + diff_interval = compute_diff_interval(ref_interval, cmp_interval) + frac_diff_interval = compute_frac_diff_interval(ref_interval, cmp_interval) if not has_finite_noise(ref_noise) or not has_finite_noise(cmp_noise): max_noise = None @@ -1266,6 +1291,8 @@ def compare_gpu_timings(ref_timing, cmp_timing, comparison_thresholds=None): decision = bulk_decision return SummaryComparison( + ref_interval=ref_interval, + cmp_interval=cmp_interval, ref_estimate=ref_estimate, cmp_estimate=cmp_estimate, ref_time=ref_time, @@ -1274,6 +1301,8 @@ def compare_gpu_timings(ref_timing, cmp_timing, comparison_thresholds=None): cmp_noise=cmp_noise, diff=diff, frac_diff=frac_diff, + diff_interval=diff_interval, + frac_diff_interval=frac_diff_interval, max_noise=max_noise, status=decision.status, reason=decision.reason, @@ -1486,6 +1515,82 @@ def format_duration(seconds): return f"{seconds * multiplier:0.3f} {units}" +def select_duration_units(*seconds_values): + max_abs_seconds = max(abs(value) for value in seconds_values) + if max_abs_seconds >= 1: + return 1.0, "s" + if max_abs_seconds >= 1e-3: + return 1e3, "ms" + return 1e6, "us" + + +def duration_precision_for_center(center, delta_multiplier): + center_multiplier, _ = select_duration_units(center) + center_quantum = 10.0**-3 * (delta_multiplier / center_multiplier) + if center_quantum >= 1.0: + return 0 + return int(math.ceil(-math.log10(center_quantum))) + + +def format_duration_range(bounds): + if bounds is None: + return "n/a" + lower, upper = bounds + multiplier, units = select_duration_units(lower, upper) + return f"[{lower * multiplier:0.2f}, {upper * multiplier:0.2f}] {units}" + + +def format_timing_with_interval(center, interval): + if center is None: + return "n/a" + if interval is None: + return format_duration(center) + + lower_delta = interval.lower - interval.center + upper_delta = interval.upper - interval.center + multiplier, units = select_duration_units(lower_delta, upper_delta) + precision = duration_precision_for_center(center, multiplier) + return ( + f"{format_duration(center)} " + f"[{lower_delta * multiplier:+0.{precision}f}, " + f"{upper_delta * multiplier:+0.{precision}f}] {units}" + ) + + +def longest_common_prefix(strings): + if not strings: + return "" + prefix = strings[0] + for text in strings[1:]: + while not text.startswith(prefix): + prefix = prefix[:-1] + if not prefix: + return "" + return prefix + + +def format_timing_with_explicit_interval(center, interval): + if center is None: + return "n/a" + if interval is None: + return format_duration(center) + + multiplier, units = select_duration_units( + interval.lower, interval.center, interval.upper + ) + values = [ + f"{interval.lower * multiplier:0.3f}", + f"{interval.center * multiplier:0.3f}", + f"{interval.upper * multiplier:0.3f}", + ] + prefix = longest_common_prefix(values) + if not prefix: + return f"[{values[0]} | {values[1]} | {values[2]}] {units}" + + suffixes = [value[len(prefix) :] for value in values] + return f"{prefix}[{suffixes[0]} | {suffixes[1]} | {suffixes[2]}] {units}" + + def format_percentage(percentage): if percentage is None: return "n/a" @@ -1496,6 +1601,79 @@ def format_percentage(percentage): return f"{percentage * 100.0:0.2f}%" +def format_percentage_bounds(bounds, status): + if bounds is None: + return "n/a" + lower, upper = bounds + if status == ComparisonStatus.FAST: + return f"<= {upper * 100.0:+0.1f}%" + if status == ComparisonStatus.SLOW: + return f">= {lower * 100.0:+0.1f}%" + return f"in [{lower * 100.0:+0.1f}%, {upper * 100.0:+0.1f}%]" + + +def get_display_headers(display): + if display == "legacy": + return ( + [ + "Ref Time", + "Ref Noise", + "Cmp Time", + "Cmp Noise", + "Diff", + "%Diff", + "Status", + ], + ["right", "right", "right", "right", "right", "right", "center"], + ) + if display == "explain": + return ( + [ + "Ref [L | C | H]", + "Cmp [L | C | H]", + "Ref Noise", + "Cmp Noise", + "Reason", + "Status", + ], + ["right", "right", "right", "right", "left", "center"], + ) + return ( + ["Ref", "Cmp", "Status"], + ["right", "right", "center"], + ) + + +def append_display_row(row, comparison, no_color, display): + if display == "legacy": + row.append(format_duration(comparison.ref_time)) + row.append(format_percentage(comparison.ref_noise)) + row.append(format_duration(comparison.cmp_time)) + row.append(format_percentage(comparison.cmp_noise)) + row.append(format_duration(comparison.diff)) + row.append(format_percentage(comparison.frac_diff)) + row.append(colorize_comparison_status(comparison.status, no_color)) + return + + row.append( + format_timing_with_interval(comparison.ref_time, comparison.ref_interval) + ) + row.append( + format_timing_with_interval(comparison.cmp_time, comparison.cmp_interval) + ) + if display == "explain": + row[-2] = format_timing_with_explicit_interval( + comparison.ref_time, comparison.ref_interval + ) + row[-1] = format_timing_with_explicit_interval( + comparison.cmp_time, comparison.cmp_interval + ) + row.append(format_percentage(comparison.ref_noise)) + row.append(format_percentage(comparison.cmp_noise)) + row.append(comparison.reason.code) + row.append(colorize_comparison_status(comparison.status, no_color)) + + def has_finite_noise(noise): return noise is not None and math.isfinite(noise) @@ -1618,6 +1796,7 @@ def compare_benches( ref_json_dir=None, cmp_json_dir=None, comparison_thresholds=None, + display="intervals", ): if comparison_thresholds is None: comparison_thresholds = ComparisonThresholds() @@ -1664,21 +1843,9 @@ def compare_benches( headers = [x["name"] for x in axes] colalign = ["center"] * len(headers) - - headers.append("Ref Time") - colalign.append("right") - headers.append("Ref Noise") - colalign.append("right") - headers.append("Cmp Time") - colalign.append("right") - headers.append("Cmp Noise") - colalign.append("right") - headers.append("Diff") - colalign.append("right") - headers.append("%Diff") - colalign.append("right") - headers.append("Status") - colalign.append("center") + display_headers, display_colalign = get_display_headers(display) + headers.extend(display_headers) + colalign.extend(display_colalign) for cmp_device_index, cmp_device_id in enumerate(cmp_device_ids): ref_device_id = ref_device_ids[cmp_device_index] @@ -1774,17 +1941,9 @@ def compare_benches( ) run_data.stats.record(comparison.status, comparison.reason) - status = colorize_comparison_status(comparison.status, no_color) - if abs(comparison.frac_diff) >= threshold: axis_filters = matching_axis_filters(cmp_state, axis_filter_groups) - row.append(format_duration(comparison.ref_time)) - row.append(format_percentage(comparison.ref_noise)) - row.append(format_duration(comparison.cmp_time)) - row.append(format_percentage(comparison.cmp_noise)) - row.append(format_duration(comparison.diff)) - row.append(format_percentage(comparison.frac_diff)) - row.append(status) + append_display_row(row, comparison, no_color, display) rows.append(row) if plot: @@ -1953,6 +2112,12 @@ def main() -> int: default="default", help="comparison threshold preset", ) + parser.add_argument( + "--display", + choices=["intervals", "legacy", "explain"], + default="intervals", + help="comparison table display mode", + ) parser.add_argument( "--plot-along", type=str, dest="plot_along", default=None, help="plot results" ) @@ -2091,17 +2256,18 @@ def main() -> int: run_data, ref_root["benchmarks"], cmp_root["benchmarks"], - args.threshold, - args.plot_along, - args.plot, - args.dark, - filter_plan, - args.no_color, - reference_device_filter, - compare_device_filter, - os.path.dirname(ref), - os.path.dirname(comp), - comparison_thresholds, + threshold=args.threshold, + plot_along=args.plot_along, + plot=args.plot, + dark=args.dark, + filter_plan=filter_plan, + no_color=args.no_color, + reference_device_filter=reference_device_filter, + compare_device_filter=compare_device_filter, + ref_json_dir=os.path.dirname(ref), + cmp_json_dir=os.path.dirname(comp), + comparison_thresholds=comparison_thresholds, + display=args.display, ) except ValueError as exc: print(str(exc)) diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index b75a0a8..e982b9d 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -511,6 +511,8 @@ def test_compare_gpu_timings_classifies_common_cases(nvbench_compare): assert fast is not None assert fast.status == nvbench_compare.ComparisonStatus.FAST assert fast.reason.code == "clear_gap_confirmed_by_summary_cycles" + assert fast.diff_interval == pytest.approx((-0.5, -0.05)) + assert fast.frac_diff_interval == pytest.approx((-0.3846153846, -0.05)) slow = nvbench_compare.compare_gpu_timings( ref_interval_timing, @@ -528,6 +530,8 @@ def test_compare_gpu_timings_classifies_common_cases(nvbench_compare): assert slow is not None assert slow.status == nvbench_compare.ComparisonStatus.SLOW assert slow.reason.code == "clear_gap_confirmed_by_summary_cycles" + assert slow.diff_interval == pytest.approx((0.1, 0.55)) + assert slow.frac_diff_interval == pytest.approx((0.0769230769, 0.55)) same = nvbench_compare.compare_gpu_timings( ref_interval_timing, @@ -545,6 +549,8 @@ def test_compare_gpu_timings_classifies_common_cases(nvbench_compare): assert same is not None assert same.status == nvbench_compare.ComparisonStatus.SAME assert same.reason.code == "same_confirmed_by_cycles" + assert same.diff_interval == pytest.approx((-0.28, 0.28)) + assert same.frac_diff_interval == pytest.approx((-0.2153846154, 0.28)) weak_overlap = nvbench_compare.compare_gpu_timings( make_gpu_timing_data( @@ -741,6 +747,48 @@ def test_compare_gpu_timings_uses_bulk_data_to_confirm_same(nvbench_compare): assert comparison.reason.code == "bulk_same" +def test_format_diff_and_percent_ranges(nvbench_compare): + assert nvbench_compare.format_duration_range((-12e-6, 8e-6)) == "[-12.00, 8.00] us" + assert ( + nvbench_compare.format_percentage_bounds( + (-0.2153846154, 0.28), nvbench_compare.ComparisonStatus.UNDECIDED + ) + == "in [-21.5%, +28.0%]" + ) + assert ( + nvbench_compare.format_percentage_bounds( + (-0.3076923077, -0.05), nvbench_compare.ComparisonStatus.FAST + ) + == "<= -5.0%" + ) + assert ( + nvbench_compare.format_percentage_bounds( + (0.0769230769, 0.55), nvbench_compare.ComparisonStatus.SLOW + ) + == ">= +7.7%" + ) + + +def test_format_timing_with_interval(nvbench_compare): + interval = nvbench_compare.TimingInterval( + lower=0.002237, upper=0.002389, center=0.0023 + ) + assert ( + nvbench_compare.format_timing_with_interval(0.0023, interval) + == "2.300 ms [-63, +89] us" + ) + + +def test_format_timing_with_explicit_interval(nvbench_compare): + interval = nvbench_compare.TimingInterval( + lower=0.001434, upper=0.001458, center=0.001446 + ) + assert ( + nvbench_compare.format_timing_with_explicit_interval(0.001446, interval) + == "1.4[34 | 46 | 58] ms" + ) + + def test_compare_gpu_timings_keeps_bulk_mismatch_undecided(nvbench_compare): ref_timing = make_gpu_timing_data( nvbench_compare, @@ -1321,7 +1369,9 @@ def test_get_comparison_thresholds_returns_named_presets(nvbench_compare): strict = nvbench_compare.get_comparison_thresholds("strict") permissive = nvbench_compare.get_comparison_thresholds("permissive") - assert default == nvbench_compare.ComparisonThresholds() + assert default == nvbench_compare.ComparisonThresholds( + **nvbench_compare.COMPARISON_THRESHOLD_PRESET_VALUES["default"] + ) assert strict.clear_gap_relative > default.clear_gap_relative assert strict.same_center_relative < default.same_center_relative assert strict.bulk_same_sample_coverage > default.bulk_same_sample_coverage @@ -1330,6 +1380,141 @@ def test_get_comparison_thresholds_returns_named_presets(nvbench_compare): assert permissive.bulk_same_support_coverage < default.bulk_same_support_coverage +def test_compare_benches_defaults_to_interval_display(monkeypatch, nvbench_compare): + run_data = make_comparison_run_data(nvbench_compare) + captured = {} + + def fake_tabulate(rows, headers, *args, **kwargs): + captured["rows"] = rows + captured["headers"] = headers + return "" + + monkeypatch.setattr(nvbench_compare.tabulate, "tabulate", fake_tabulate) + + ref_benches = [make_benchmark([make_state(nvbench_compare, "state", mean="1.0")])] + cmp_benches = [make_benchmark([make_state(nvbench_compare, "state", mean="1.01")])] + + nvbench_compare.compare_benches( + run_data, + ref_benches, + cmp_benches, + threshold=0.0, + plot_along=None, + plot=False, + dark=False, + filter_plan=make_filter_plan(nvbench_compare), + no_color=True, + ) + + assert captured["headers"][-3:] == ["Ref", "Cmp", "Status"] + row = captured["rows"][0] + assert row[-3].startswith("1.000 s") + assert row[-2].startswith("1.010 s") + + +def test_compare_benches_legacy_display_uses_scalar_diff(monkeypatch, nvbench_compare): + run_data = make_comparison_run_data(nvbench_compare) + captured = {} + + def fake_tabulate(rows, headers, *args, **kwargs): + captured["rows"] = rows + captured["headers"] = headers + return "" + + monkeypatch.setattr(nvbench_compare.tabulate, "tabulate", fake_tabulate) + + ref_benches = [make_benchmark([make_state(nvbench_compare, "state", mean="1.0")])] + cmp_benches = [make_benchmark([make_state(nvbench_compare, "state", mean="1.01")])] + + nvbench_compare.compare_benches( + run_data, + ref_benches, + cmp_benches, + threshold=0.0, + plot_along=None, + plot=False, + dark=False, + filter_plan=make_filter_plan(nvbench_compare), + no_color=True, + display="legacy", + ) + + assert captured["headers"][-7:] == [ + "Ref Time", + "Ref Noise", + "Cmp Time", + "Cmp Noise", + "Diff", + "%Diff", + "Status", + ] + row = captured["rows"][0] + assert row[-7] == "1.000 s" + assert row[-5] == "1.010 s" + assert row[-3] == "10.000 ms" + assert row[-2] == "1.00%" + + +def test_compare_benches_explain_display_uses_explicit_intervals( + monkeypatch, nvbench_compare +): + run_data = make_comparison_run_data(nvbench_compare) + captured = {} + + def fake_tabulate(rows, headers, *args, **kwargs): + captured["rows"] = rows + captured["headers"] = headers + return "" + + monkeypatch.setattr(nvbench_compare.tabulate, "tabulate", fake_tabulate) + + ref_state = make_state(nvbench_compare, "state", mean="1.0") + ref_state["summaries"].extend( + [ + make_summary(nvbench_compare, "GPU_TIME_MIN_TAG", "1.0"), + make_summary(nvbench_compare, "GPU_TIME_Q1_TAG", "1.01"), + make_summary(nvbench_compare, "GPU_TIME_MEDIAN_TAG", "1.02"), + make_summary(nvbench_compare, "GPU_TIME_Q3_TAG", "1.03"), + make_summary(nvbench_compare, "GPU_SM_CLOCK_RATE_MEAN_TAG", "100.0"), + ] + ) + cmp_state = make_state(nvbench_compare, "state", mean="1.01") + cmp_state["summaries"].extend( + [ + make_summary(nvbench_compare, "GPU_TIME_MIN_TAG", "1.01"), + make_summary(nvbench_compare, "GPU_TIME_Q1_TAG", "1.02"), + make_summary(nvbench_compare, "GPU_TIME_MEDIAN_TAG", "1.03"), + make_summary(nvbench_compare, "GPU_TIME_Q3_TAG", "1.04"), + make_summary(nvbench_compare, "GPU_SM_CLOCK_RATE_MEAN_TAG", "100.0"), + ] + ) + + nvbench_compare.compare_benches( + run_data, + [make_benchmark([ref_state])], + [make_benchmark([cmp_state])], + threshold=0.0, + plot_along=None, + plot=False, + dark=False, + filter_plan=make_filter_plan(nvbench_compare), + no_color=True, + display="explain", + ) + + assert captured["headers"][-6:] == [ + "Ref [L | C | H]", + "Cmp [L | C | H]", + "Ref Noise", + "Cmp Noise", + "Reason", + "Status", + ] + row = captured["rows"][0] + assert row[-6] == "1.0[00 | 20 | 30] s" + assert row[-5] == "1.0[10 | 30 | 40] s" + + def test_main_passes_selected_preset_to_compare_benches(monkeypatch, nvbench_compare): devices = [{"id": 0, "name": "Test GPU"}] root = { @@ -1341,16 +1526,26 @@ def test_main_passes_selected_preset_to_compare_benches(monkeypatch, nvbench_com monkeypatch.setattr(nvbench_compare.reader, "read_file", lambda _: root) def fake_compare_benches(*args, **kwargs): - captured["comparison_thresholds"] = args[-1] + captured["comparison_thresholds"] = kwargs["comparison_thresholds"] + captured["display"] = kwargs["display"] monkeypatch.setattr(nvbench_compare, "compare_benches", fake_compare_benches) monkeypatch.setattr( sys, "argv", - ["nvbench_compare", "--preset", "strict", "ref.json", "cmp.json"], + [ + "nvbench_compare", + "--preset", + "strict", + "--display", + "explain", + "ref.json", + "cmp.json", + ], ) assert nvbench_compare.main() == 0 assert captured[ "comparison_thresholds" ] == nvbench_compare.get_comparison_thresholds("strict") + assert captured["display"] == "explain"