diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index 39639f2..c16b8eb 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -59,12 +59,16 @@ def load_nvbench_compare_tooling(*, load_color: bool = True) -> None: if np is None: np = require_tooling_dependency( - ToolingDependency("numpy", "numpy", "bulk timing analysis"), + ToolingDependency( + "numpy", "numpy", "bulk timing analysis", extra="compare" + ), tool_name="nvbench-compare", ) if load_color and Fore is None: colorama = require_tooling_dependency( - ToolingDependency("colorama", "colorama", "colored status output"), + ToolingDependency( + "colorama", "colorama", "colored status output", extra="compare" + ), tool_name="nvbench-compare", ) Fore = colorama.Fore @@ -72,7 +76,7 @@ def load_nvbench_compare_tooling(*, load_color: bool = True) -> None: def load_tabulate_for_table_output() -> tuple[Any, tuple[int, ...]]: tabulate_module = require_tooling_dependency( - ToolingDependency("tabulate", "tabulate", "table output"), + ToolingDependency("tabulate", "tabulate", "table output", extra="compare"), tool_name="nvbench-compare", ) return tabulate_module, version_tuple(tabulate_module.__version__) @@ -80,7 +84,9 @@ def load_tabulate_for_table_output() -> tuple[Any, tuple[int, ...]]: def load_jsondiff_for_device_diff() -> Any: return require_tooling_dependency( - ToolingDependency("jsondiff", "jsondiff", "device metadata diffs"), + ToolingDependency( + "jsondiff", "jsondiff", "device metadata diffs", extra="compare" + ), tool_name="nvbench-compare", ) @@ -2442,13 +2448,9 @@ def matching_axis_filters(state, axis_filter_groups): ) -def is_finite_number(value): - return value is not None and math.isfinite(value) - - def format_duration(seconds, *, allow_negative=False, allow_zero=False): if ( - not is_finite_number(seconds) + not is_finite(seconds) or (seconds < 0.0 and not allow_negative) or (seconds == 0.0 and not allow_zero) ): @@ -2470,7 +2472,7 @@ def format_duration(seconds, *, allow_negative=False, allow_zero=False): def select_duration_units(*seconds_values): - seconds_values = [value for value in seconds_values if is_finite_number(value)] + seconds_values = [value for value in seconds_values if is_finite(value)] if not seconds_values: return 1e6, "us" @@ -2483,7 +2485,7 @@ def select_duration_units(*seconds_values): def duration_precision_for_center(center, delta_multiplier): - if not is_finite_number(center): + if not is_finite(center): return 3 center_multiplier, _ = select_duration_units(center) @@ -2497,7 +2499,7 @@ def format_duration_range(bounds): if bounds is None: return "n/a" lower, upper = bounds - if not is_finite_number(lower) or not is_finite_number(upper): + if not is_finite(lower) or not is_finite(upper): return "n/a" multiplier, units = select_duration_units(lower, upper) @@ -2853,18 +2855,22 @@ def plot_comparison_entries(entries, title=None, dark=False): return 1 matplotlib = require_tooling_dependency( - ToolingDependency("matplotlib", "matplotlib", "plot rendering"), + ToolingDependency("matplotlib", "matplotlib", "plot rendering", extra="plot"), tool_name="nvbench-compare", ) if not os.environ.get("DISPLAY"): matplotlib.use("Agg") plt = require_tooling_dependency( - ToolingDependency("matplotlib.pyplot", "matplotlib", "plot rendering"), + ToolingDependency( + "matplotlib.pyplot", "matplotlib", "plot rendering", extra="plot" + ), tool_name="nvbench-compare", ) ticker = require_tooling_dependency( - ToolingDependency("matplotlib.ticker", "matplotlib", "plot axis formatting"), + ToolingDependency( + "matplotlib.ticker", "matplotlib", "plot axis formatting", extra="plot" + ), tool_name="nvbench-compare", ) PercentFormatter = ticker.PercentFormatter @@ -2953,12 +2959,17 @@ def compare_benches( if plot_along: plt = require_tooling_dependency( ToolingDependency( - "matplotlib.pyplot", "matplotlib", "per-axis plot rendering" + "matplotlib.pyplot", + "matplotlib", + "per-axis plot rendering", + extra="plot", ), tool_name="nvbench-compare", ) sns = require_tooling_dependency( - ToolingDependency("seaborn", "seaborn", "per-axis plot styling"), + ToolingDependency( + "seaborn", "seaborn", "per-axis plot styling", extra="plot" + ), tool_name="nvbench-compare", ) diff --git a/python/scripts/nvbench_compare_legacy.py b/python/scripts/nvbench_compare_legacy.py index 6cc02e5..c51acad 100644 --- a/python/scripts/nvbench_compare_legacy.py +++ b/python/scripts/nvbench_compare_legacy.py @@ -44,7 +44,9 @@ def load_nvbench_compare_tooling(*, load_color: bool = True) -> None: if load_color and Fore is None: colorama = require_tooling_dependency( - ToolingDependency("colorama", "colorama", "colored status output"), + ToolingDependency( + "colorama", "colorama", "colored status output", extra="compare" + ), tool_name="nvbench-compare-legacy", ) Fore = colorama.Fore @@ -52,7 +54,7 @@ def load_nvbench_compare_tooling(*, load_color: bool = True) -> None: def load_tabulate_for_table_output() -> tuple[Any, tuple[int, ...]]: tabulate_module = require_tooling_dependency( - ToolingDependency("tabulate", "tabulate", "table output"), + ToolingDependency("tabulate", "tabulate", "table output", extra="compare"), tool_name="nvbench-compare-legacy", ) return tabulate_module, version_tuple(tabulate_module.__version__) @@ -60,7 +62,9 @@ def load_tabulate_for_table_output() -> tuple[Any, tuple[int, ...]]: def load_jsondiff_for_device_diff() -> Any: return require_tooling_dependency( - ToolingDependency("jsondiff", "jsondiff", "device metadata diffs"), + ToolingDependency( + "jsondiff", "jsondiff", "device metadata diffs", extra="compare" + ), tool_name="nvbench-compare-legacy", ) @@ -687,8 +691,14 @@ def find_device_by_id(device_id, all_devices): return None -def format_int64_axis_value(axis_name, axis_value, axes): - axis = next(filter(lambda ax: ax["name"] == axis_name, axes)) +def find_axis_by_name(axis_name, axes): + for axis in axes: + if axis["name"] == axis_name: + return axis + raise KeyError(f"axis metadata not found for {axis_name!r}") + + +def format_int64_axis_value(axis_name, axis_value, axis): axis_flags = axis["flags"] value = int(axis_value["value"]) if axis_flags == "pow2": @@ -710,10 +720,10 @@ def format_string_axis_value(axis_name, axis_value, axes): def format_axis_value(axis_name, axis_value, axes): - axis = next(filter(lambda ax: ax["name"] == axis_name, axes)) + axis = find_axis_by_name(axis_name, axes) axis_type = axis["type"] if axis_type == "int64": - return format_int64_axis_value(axis_name, axis_value, axes) + return format_int64_axis_value(axis_name, axis_value, axis) elif axis_type == "float64": return format_float64_axis_value(axis_name, axis_value, axes) elif axis_type == "type": @@ -865,13 +875,9 @@ def matching_axis_filters(state, axis_filter_groups): ) -def is_finite_number(value): - return value is not None and math.isfinite(value) - - def format_duration(seconds, *, allow_negative=False, allow_zero=False): if ( - not is_finite_number(seconds) + not is_finite(seconds) or (seconds < 0.0 and not allow_negative) or (seconds == 0.0 and not allow_zero) ): @@ -893,7 +899,7 @@ def format_duration(seconds, *, allow_negative=False, allow_zero=False): def select_duration_units(*seconds_values): - seconds_values = [value for value in seconds_values if is_finite_number(value)] + seconds_values = [value for value in seconds_values if is_finite(value)] if not seconds_values: return 1e6, "us" @@ -906,7 +912,7 @@ def select_duration_units(*seconds_values): def duration_precision_for_center(center, delta_multiplier): - if not is_finite_number(center): + if not is_finite(center): return 3 center_multiplier, _ = select_duration_units(center) @@ -920,7 +926,7 @@ def format_duration_range(bounds): if bounds is None: return "n/a" lower, upper = bounds - if not is_finite_number(lower) or not is_finite_number(upper): + if not is_finite(lower) or not is_finite(upper): return "n/a" multiplier, units = select_duration_units(lower, upper) @@ -1010,18 +1016,22 @@ def plot_comparison_entries(entries, title=None, dark=False): return 1 matplotlib = require_tooling_dependency( - ToolingDependency("matplotlib", "matplotlib", "plot rendering"), + ToolingDependency("matplotlib", "matplotlib", "plot rendering", extra="plot"), tool_name="nvbench-compare-legacy", ) if not os.environ.get("DISPLAY"): matplotlib.use("Agg") plt = require_tooling_dependency( - ToolingDependency("matplotlib.pyplot", "matplotlib", "plot rendering"), + ToolingDependency( + "matplotlib.pyplot", "matplotlib", "plot rendering", extra="plot" + ), tool_name="nvbench-compare-legacy", ) ticker = require_tooling_dependency( - ToolingDependency("matplotlib.ticker", "matplotlib", "plot axis formatting"), + ToolingDependency( + "matplotlib.ticker", "matplotlib", "plot axis formatting", extra="plot" + ), tool_name="nvbench-compare-legacy", ) PercentFormatter = ticker.PercentFormatter @@ -1104,12 +1114,17 @@ def compare_benches( if plot_along: plt = require_tooling_dependency( ToolingDependency( - "matplotlib.pyplot", "matplotlib", "per-axis plot rendering" + "matplotlib.pyplot", + "matplotlib", + "per-axis plot rendering", + extra="plot", ), tool_name="nvbench-compare-legacy", ) sns = require_tooling_dependency( - ToolingDependency("seaborn", "seaborn", "per-axis plot styling"), + ToolingDependency( + "seaborn", "seaborn", "per-axis plot styling", extra="plot" + ), tool_name="nvbench-compare-legacy", ) @@ -1215,15 +1230,34 @@ def compare_benches( ref_state, cmp_state ) if missing_summaries_decision is not None: - continue - - cmp_gpu_time = extract_gpu_timing_data( - cmp_summaries, cmp_json_dir, json_path=cmp_json_path - ) - ref_gpu_time = extract_gpu_timing_data( - ref_summaries, ref_json_dir, json_path=ref_json_path - ) - comparison = compare_gpu_timings(ref_gpu_time, cmp_gpu_time) + ref_gpu_time = ( + extract_gpu_timing_data( + ref_summaries, ref_json_dir, json_path=ref_json_path + ) + if ref_summaries + else make_empty_gpu_timing_data() + ) + cmp_gpu_time = ( + extract_gpu_timing_data( + cmp_summaries, cmp_json_dir, json_path=cmp_json_path + ) + if cmp_summaries + else make_empty_gpu_timing_data() + ) + timing_inputs = compute_legacy_timing_comparison_inputs( + ref_gpu_time, cmp_gpu_time + ) + comparison = make_unavailable_timing_comparison( + missing_summaries_decision, timing_inputs + ) + else: + cmp_gpu_time = extract_gpu_timing_data( + cmp_summaries, cmp_json_dir, json_path=cmp_json_path + ) + ref_gpu_time = extract_gpu_timing_data( + ref_summaries, ref_json_dir, json_path=ref_json_path + ) + comparison = compare_gpu_timings(ref_gpu_time, cmp_gpu_time) if comparison is None: continue diff --git a/python/scripts/nvbench_histogram.py b/python/scripts/nvbench_histogram.py index fd93de1..92ac3d4 100644 --- a/python/scripts/nvbench_histogram.py +++ b/python/scripts/nvbench_histogram.py @@ -30,22 +30,26 @@ def load_nvbench_histogram_tooling(): if plt is None: plt = require_tooling_dependency( - ToolingDependency("matplotlib.pyplot", "matplotlib", "histogram plotting"), + ToolingDependency( + "matplotlib.pyplot", "matplotlib", "histogram plotting", extra="plot" + ), tool_name="nvbench-histogram", ) if np is None: np = require_tooling_dependency( - ToolingDependency("numpy", "numpy", "sample loading"), + ToolingDependency("numpy", "numpy", "sample loading", extra="plot"), tool_name="nvbench-histogram", ) if pd is None: pd = require_tooling_dependency( - ToolingDependency("pandas", "pandas", "sample table construction"), + ToolingDependency( + "pandas", "pandas", "sample table construction", extra="plot" + ), tool_name="nvbench-histogram", ) if sns is None: sns = require_tooling_dependency( - ToolingDependency("seaborn", "seaborn", "histogram plotting"), + ToolingDependency("seaborn", "seaborn", "histogram plotting", extra="plot"), tool_name="nvbench-histogram", ) diff --git a/python/scripts/nvbench_plot_bwutil.py b/python/scripts/nvbench_plot_bwutil.py index 740bd73..7dfce44 100644 --- a/python/scripts/nvbench_plot_bwutil.py +++ b/python/scripts/nvbench_plot_bwutil.py @@ -29,14 +29,20 @@ def load_nvbench_plot_bwutil_tooling(): if plt is None: plt = require_tooling_dependency( ToolingDependency( - "matplotlib.pyplot", "matplotlib", "bandwidth plot rendering" + "matplotlib.pyplot", + "matplotlib", + "bandwidth plot rendering", + extra="plot", ), tool_name="nvbench-plot-bwutil", ) if PercentFormatter is None: ticker = require_tooling_dependency( ToolingDependency( - "matplotlib.ticker", "matplotlib", "plot axis formatting" + "matplotlib.ticker", + "matplotlib", + "plot axis formatting", + extra="plot", ), tool_name="nvbench-plot-bwutil", ) diff --git a/python/scripts/nvbench_walltime.py b/python/scripts/nvbench_walltime.py index 9d2f7f7..31e0b89 100644 --- a/python/scripts/nvbench_walltime.py +++ b/python/scripts/nvbench_walltime.py @@ -36,7 +36,7 @@ def load_nvbench_walltime_tooling(): return tabulate = require_tooling_dependency( - ToolingDependency("tabulate", "tabulate", "table output"), + ToolingDependency("tabulate", "tabulate", "table output", extra="compare"), tool_name="nvbench-walltime", ) tabulate_version = version_tuple(tabulate.__version__)