NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-07-12 18:17:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Address review feedback for Python tooling scripts

Browse Source 
Tighten missing-dependency install hints so compare-only dependencies point
to cuda-bench[compare] and plotting/dataframe dependencies point to
cuda-bench[plot], instead of defaulting every script to the broader tools
extra.

Also harden nvbench_compare_legacy by reporting missing or skipped state
summaries as UNKNOWN rows instead of silently dropping them, and by converting
missing axis metadata into the existing JSON-structure error path rather than
leaking StopIteration.

Finally, consolidate duplicate finite-number predicates in both compare
scripts so duration formatting and numeric validation share the same helper.
This commit is contained in:
Oleksandr Pavlyk
2026-07-09 16:50:36 -05:00
parent cc17e96c3b
commit 25005fc9c4
5 changed files with 108 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
python/scripts/nvbench_compare.py
View File

@@ -59,12 +59,16 @@ def load_nvbench_compare_tooling(*, load_color: bool = True) -> None:
if np is None:
np = require_tooling_dependency(
ToolingDependency("numpy", "numpy", "bulk timing analysis"),
ToolingDependency(
"numpy", "numpy", "bulk timing analysis", extra="compare"
),
tool_name="nvbench-compare",
)
if load_color and Fore is None:
colorama = require_tooling_dependency(
ToolingDependency("colorama", "colorama", "colored status output"),
ToolingDependency(
"colorama", "colorama", "colored status output", extra="compare"
),
tool_name="nvbench-compare",
)
Fore = colorama.Fore
@@ -72,7 +76,7 @@ def load_nvbench_compare_tooling(*, load_color: bool = True) -> None:
def load_tabulate_for_table_output() -> tuple[Any, tuple[int, ...]]:
tabulate_module = require_tooling_dependency(
ToolingDependency("tabulate", "tabulate", "table output"),
ToolingDependency("tabulate", "tabulate", "table output", extra="compare"),
tool_name="nvbench-compare",
)
return tabulate_module, version_tuple(tabulate_module.__version__)
@@ -80,7 +84,9 @@ def load_tabulate_for_table_output() -> tuple[Any, tuple[int, ...]]:
def load_jsondiff_for_device_diff() -> Any:
return require_tooling_dependency(
ToolingDependency("jsondiff", "jsondiff", "device metadata diffs"),
ToolingDependency(
"jsondiff", "jsondiff", "device metadata diffs", extra="compare"
),
tool_name="nvbench-compare",
)
@@ -2442,13 +2448,9 @@ def matching_axis_filters(state, axis_filter_groups):
)
def is_finite_number(value):
return value is not None and math.isfinite(value)
def format_duration(seconds, *, allow_negative=False, allow_zero=False):
if (
not is_finite_number(seconds)
not is_finite(seconds)
or (seconds < 0.0 and not allow_negative)
or (seconds == 0.0 and not allow_zero)
):
@@ -2470,7 +2472,7 @@ def format_duration(seconds, *, allow_negative=False, allow_zero=False):
def select_duration_units(*seconds_values):
seconds_values = [value for value in seconds_values if is_finite_number(value)]
seconds_values = [value for value in seconds_values if is_finite(value)]
if not seconds_values:
return 1e6, "us"
@@ -2483,7 +2485,7 @@ def select_duration_units(*seconds_values):
def duration_precision_for_center(center, delta_multiplier):
if not is_finite_number(center):
if not is_finite(center):
return 3
center_multiplier, _ = select_duration_units(center)
@@ -2497,7 +2499,7 @@ def format_duration_range(bounds):
if bounds is None:
return "n/a"
lower, upper = bounds
if not is_finite_number(lower) or not is_finite_number(upper):
if not is_finite(lower) or not is_finite(upper):
return "n/a"
multiplier, units = select_duration_units(lower, upper)
@@ -2853,18 +2855,22 @@ def plot_comparison_entries(entries, title=None, dark=False):
return 1
matplotlib = require_tooling_dependency(
ToolingDependency("matplotlib", "matplotlib", "plot rendering"),
ToolingDependency("matplotlib", "matplotlib", "plot rendering", extra="plot"),
tool_name="nvbench-compare",
)
if not os.environ.get("DISPLAY"):
matplotlib.use("Agg")
plt = require_tooling_dependency(
ToolingDependency("matplotlib.pyplot", "matplotlib", "plot rendering"),
ToolingDependency(
"matplotlib.pyplot", "matplotlib", "plot rendering", extra="plot"
),
tool_name="nvbench-compare",
)
ticker = require_tooling_dependency(
ToolingDependency("matplotlib.ticker", "matplotlib", "plot axis formatting"),
ToolingDependency(
"matplotlib.ticker", "matplotlib", "plot axis formatting", extra="plot"
),
tool_name="nvbench-compare",
)
PercentFormatter = ticker.PercentFormatter
@@ -2953,12 +2959,17 @@ def compare_benches(
if plot_along:
plt = require_tooling_dependency(
ToolingDependency(
"matplotlib.pyplot", "matplotlib", "per-axis plot rendering"
"matplotlib.pyplot",
"matplotlib",
"per-axis plot rendering",
extra="plot",
),
tool_name="nvbench-compare",
)
sns = require_tooling_dependency(
ToolingDependency("seaborn", "seaborn", "per-axis plot styling"),
ToolingDependency(
"seaborn", "seaborn", "per-axis plot styling", extra="plot"
),
tool_name="nvbench-compare",
)

92
python/scripts/nvbench_compare_legacy.py
View File

@@ -44,7 +44,9 @@ def load_nvbench_compare_tooling(*, load_color: bool = True) -> None:
if load_color and Fore is None:
colorama = require_tooling_dependency(
ToolingDependency("colorama", "colorama", "colored status output"),
ToolingDependency(
"colorama", "colorama", "colored status output", extra="compare"
),
tool_name="nvbench-compare-legacy",
)
Fore = colorama.Fore
@@ -52,7 +54,7 @@ def load_nvbench_compare_tooling(*, load_color: bool = True) -> None:
def load_tabulate_for_table_output() -> tuple[Any, tuple[int, ...]]:
tabulate_module = require_tooling_dependency(
ToolingDependency("tabulate", "tabulate", "table output"),
ToolingDependency("tabulate", "tabulate", "table output", extra="compare"),
tool_name="nvbench-compare-legacy",
)
return tabulate_module, version_tuple(tabulate_module.__version__)
@@ -60,7 +62,9 @@ def load_tabulate_for_table_output() -> tuple[Any, tuple[int, ...]]:
def load_jsondiff_for_device_diff() -> Any:
return require_tooling_dependency(
ToolingDependency("jsondiff", "jsondiff", "device metadata diffs"),
ToolingDependency(
"jsondiff", "jsondiff", "device metadata diffs", extra="compare"
),
tool_name="nvbench-compare-legacy",
)
@@ -687,8 +691,14 @@ def find_device_by_id(device_id, all_devices):
return None
def format_int64_axis_value(axis_name, axis_value, axes):
axis = next(filter(lambda ax: ax["name"] == axis_name, axes))
def find_axis_by_name(axis_name, axes):
for axis in axes:
if axis["name"] == axis_name:
return axis
raise KeyError(f"axis metadata not found for {axis_name!r}")
def format_int64_axis_value(axis_name, axis_value, axis):
axis_flags = axis["flags"]
value = int(axis_value["value"])
if axis_flags == "pow2":
@@ -710,10 +720,10 @@ def format_string_axis_value(axis_name, axis_value, axes):
def format_axis_value(axis_name, axis_value, axes):
axis = next(filter(lambda ax: ax["name"] == axis_name, axes))
axis = find_axis_by_name(axis_name, axes)
axis_type = axis["type"]
if axis_type == "int64":
return format_int64_axis_value(axis_name, axis_value, axes)
return format_int64_axis_value(axis_name, axis_value, axis)
elif axis_type == "float64":
return format_float64_axis_value(axis_name, axis_value, axes)
elif axis_type == "type":
@@ -865,13 +875,9 @@ def matching_axis_filters(state, axis_filter_groups):
)
def is_finite_number(value):
return value is not None and math.isfinite(value)
def format_duration(seconds, *, allow_negative=False, allow_zero=False):
if (
not is_finite_number(seconds)
not is_finite(seconds)
or (seconds < 0.0 and not allow_negative)
or (seconds == 0.0 and not allow_zero)
):
@@ -893,7 +899,7 @@ def format_duration(seconds, *, allow_negative=False, allow_zero=False):
def select_duration_units(*seconds_values):
seconds_values = [value for value in seconds_values if is_finite_number(value)]
seconds_values = [value for value in seconds_values if is_finite(value)]
if not seconds_values:
return 1e6, "us"
@@ -906,7 +912,7 @@ def select_duration_units(*seconds_values):
def duration_precision_for_center(center, delta_multiplier):
if not is_finite_number(center):
if not is_finite(center):
return 3
center_multiplier, _ = select_duration_units(center)
@@ -920,7 +926,7 @@ def format_duration_range(bounds):
if bounds is None:
return "n/a"
lower, upper = bounds
if not is_finite_number(lower) or not is_finite_number(upper):
if not is_finite(lower) or not is_finite(upper):
return "n/a"
multiplier, units = select_duration_units(lower, upper)
@@ -1010,18 +1016,22 @@ def plot_comparison_entries(entries, title=None, dark=False):
return 1
matplotlib = require_tooling_dependency(
ToolingDependency("matplotlib", "matplotlib", "plot rendering"),
ToolingDependency("matplotlib", "matplotlib", "plot rendering", extra="plot"),
tool_name="nvbench-compare-legacy",
)
if not os.environ.get("DISPLAY"):
matplotlib.use("Agg")
plt = require_tooling_dependency(
ToolingDependency("matplotlib.pyplot", "matplotlib", "plot rendering"),
ToolingDependency(
"matplotlib.pyplot", "matplotlib", "plot rendering", extra="plot"
),
tool_name="nvbench-compare-legacy",
)
ticker = require_tooling_dependency(
ToolingDependency("matplotlib.ticker", "matplotlib", "plot axis formatting"),
ToolingDependency(
"matplotlib.ticker", "matplotlib", "plot axis formatting", extra="plot"
),
tool_name="nvbench-compare-legacy",
)
PercentFormatter = ticker.PercentFormatter
@@ -1104,12 +1114,17 @@ def compare_benches(
if plot_along:
plt = require_tooling_dependency(
ToolingDependency(
"matplotlib.pyplot", "matplotlib", "per-axis plot rendering"
"matplotlib.pyplot",
"matplotlib",
"per-axis plot rendering",
extra="plot",
),
tool_name="nvbench-compare-legacy",
)
sns = require_tooling_dependency(
ToolingDependency("seaborn", "seaborn", "per-axis plot styling"),
ToolingDependency(
"seaborn", "seaborn", "per-axis plot styling", extra="plot"
),
tool_name="nvbench-compare-legacy",
)
@@ -1215,15 +1230,34 @@ def compare_benches(
ref_state, cmp_state
)
if missing_summaries_decision is not None:
continue
cmp_gpu_time = extract_gpu_timing_data(
cmp_summaries, cmp_json_dir, json_path=cmp_json_path
)
ref_gpu_time = extract_gpu_timing_data(
ref_summaries, ref_json_dir, json_path=ref_json_path
)
comparison = compare_gpu_timings(ref_gpu_time, cmp_gpu_time)
ref_gpu_time = (
extract_gpu_timing_data(
ref_summaries, ref_json_dir, json_path=ref_json_path
)
if ref_summaries
else make_empty_gpu_timing_data()
)
cmp_gpu_time = (
extract_gpu_timing_data(
cmp_summaries, cmp_json_dir, json_path=cmp_json_path
)
if cmp_summaries
else make_empty_gpu_timing_data()
)
timing_inputs = compute_legacy_timing_comparison_inputs(
ref_gpu_time, cmp_gpu_time
)
comparison = make_unavailable_timing_comparison(
missing_summaries_decision, timing_inputs
)
else:
cmp_gpu_time = extract_gpu_timing_data(
cmp_summaries, cmp_json_dir, json_path=cmp_json_path
)
ref_gpu_time = extract_gpu_timing_data(
ref_summaries, ref_json_dir, json_path=ref_json_path
)
comparison = compare_gpu_timings(ref_gpu_time, cmp_gpu_time)
if comparison is None:
continue

12
python/scripts/nvbench_histogram.py
View File

@@ -30,22 +30,26 @@ def load_nvbench_histogram_tooling():
if plt is None:
plt = require_tooling_dependency(
ToolingDependency("matplotlib.pyplot", "matplotlib", "histogram plotting"),
ToolingDependency(
"matplotlib.pyplot", "matplotlib", "histogram plotting", extra="plot"
),
tool_name="nvbench-histogram",
)
if np is None:
np = require_tooling_dependency(
ToolingDependency("numpy", "numpy", "sample loading"),
ToolingDependency("numpy", "numpy", "sample loading", extra="plot"),
tool_name="nvbench-histogram",
)
if pd is None:
pd = require_tooling_dependency(
ToolingDependency("pandas", "pandas", "sample table construction"),
ToolingDependency(
"pandas", "pandas", "sample table construction", extra="plot"
),
tool_name="nvbench-histogram",
)
if sns is None:
sns = require_tooling_dependency(
ToolingDependency("seaborn", "seaborn", "histogram plotting"),
ToolingDependency("seaborn", "seaborn", "histogram plotting", extra="plot"),
tool_name="nvbench-histogram",
)

10
python/scripts/nvbench_plot_bwutil.py
View File

@@ -29,14 +29,20 @@ def load_nvbench_plot_bwutil_tooling():
if plt is None:
plt = require_tooling_dependency(
ToolingDependency(
"matplotlib.pyplot", "matplotlib", "bandwidth plot rendering"
"matplotlib.pyplot",
"matplotlib",
"bandwidth plot rendering",
extra="plot",
),
tool_name="nvbench-plot-bwutil",
)
if PercentFormatter is None:
ticker = require_tooling_dependency(
ToolingDependency(
"matplotlib.ticker", "matplotlib", "plot axis formatting"
"matplotlib.ticker",
"matplotlib",
"plot axis formatting",
extra="plot",
),
tool_name="nvbench-plot-bwutil",
)

2
python/scripts/nvbench_walltime.py
View File

@@ -36,7 +36,7 @@ def load_nvbench_walltime_tooling():
return
tabulate = require_tooling_dependency(
ToolingDependency("tabulate", "tabulate", "table output"),
ToolingDependency("tabulate", "tabulate", "table output", extra="compare"),
tool_name="nvbench-walltime",
)
tabulate_version = version_tuple(tabulate.__version__)