From 0be190b4073ac103f2bfd0a0e891dd888a4b1b1d Mon Sep 17 00:00:00 2001 From: Bernhard Manfred Gruber Date: Thu, 5 Feb 2026 10:36:52 +0100 Subject: [PATCH 1/6] Add a script to plot benchmark results --- python/scripts/nvbench_plot.py | 239 +++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 python/scripts/nvbench_plot.py diff --git a/python/scripts/nvbench_plot.py b/python/scripts/nvbench_plot.py new file mode 100644 index 0000000..249abfd --- /dev/null +++ b/python/scripts/nvbench_plot.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python + +import argparse +import os +import sys + +import matplotlib.pyplot as plt +from matplotlib.ticker import PercentFormatter + +try: + from nvbench_json import reader +except ImportError: + from scripts.nvbench_json import reader + + +UTILIZATION_TAG = "nv/cold/bw/global/utilization" + + +def parse_files(): + help_text = "%(prog)s [nvbench.out.json | dir/] ..." + parser = argparse.ArgumentParser(prog="nvbench_plot", usage=help_text) + parser.add_argument( + "-o", + "--output", + default=None, + help="Save plot to this file instead of showing it", + ) + parser.add_argument( + "--title", + default=None, + help="Optional plot title", + ) + parser.add_argument( + "-a", + "--axis", + action="append", + default=[], + help="Filter on axis value, e.g. -a T{ct}=I8 (can repeat)", + ) + parser.add_argument( + "-b", + "--benchmark", + action="append", + default=[], + help="Filter by benchmark name (can repeat)", + ) + args, files_or_dirs = parser.parse_known_args() + + filenames = [] + for file_or_dir in files_or_dirs: + if os.path.isdir(file_or_dir): + for f in os.listdir(file_or_dir): + if os.path.splitext(f)[1] != ".json": + continue + filename = os.path.join(file_or_dir, f) + if os.path.isfile(filename) and os.path.getsize(filename) > 0: + filenames.append(filename) + else: + filenames.append(file_or_dir) + + filenames.sort() + + if not filenames: + parser.print_help() + sys.exit(0) + + return args, filenames + + +def extract_utilization(state): + summaries = state.get("summaries") or [] + summary = next( + filter(lambda s: s["tag"] == UTILIZATION_TAG, summaries), + None, + ) + if not summary: + return None + + value_data = next( + filter(lambda v: v["name"] == "value", summary["data"]), + None, + ) + if not value_data: + return None + + return float(value_data["value"]) + + +def parse_axis_filters(axis_args): + filters = [] + for axis_arg in axis_args: + if "=" not in axis_arg: + raise ValueError("Axis filter must be NAME=VALUE: {}".format(axis_arg)) + name, value = axis_arg.split("=", 1) + name = name.strip() + value = value.strip() + if not name or not value: + raise ValueError("Axis filter must be NAME=VALUE: {}".format(axis_arg)) + if name.endswith("[pow2]"): + name = name[: -len("[pow2]")].strip() + if not name: + raise ValueError( + "Axis filter missing name before [pow2]: {}".format(axis_arg) + ) + try: + exponent = int(value) + except ValueError as exc: + raise ValueError( + "Axis filter [pow2] value must be integer: {}".format(axis_arg) + ) from exc + value = str(2**exponent) + filters.append((name, value)) + return filters + + +def matches_axis_filters(state, axis_filters): + if not axis_filters: + return True + + axis_values = state.get("axis_values") or [] + for filter_name, filter_value in axis_filters: + matched = False + for axis_value in axis_values: + if axis_value.get("name") != filter_name: + continue + value = axis_value.get("value") + if value is None: + continue + if str(value) == filter_value: + matched = True + break + if not matched: + return False + return True + + +def collect_entries(filename, axis_filters, benchmark_filters): + json_root = reader.read_file(filename) + entries = [] + device_names = set() + devices = {device["id"]: device["name"] for device in json_root.get("devices", [])} + for bench in json_root["benchmarks"]: + bench_name = bench["name"] + if benchmark_filters and bench_name not in benchmark_filters: + continue + for state in bench["states"]: + if not matches_axis_filters(state, axis_filters): + continue + utilization = extract_utilization(state) + if utilization is None: + continue + + state_name = state["name"] + if state_name.startswith("Device="): + parts = state_name.split(" ", 1) + if len(parts) == 2: + state_name = parts[1] + label = "{} | {}".format(bench_name, state_name) + device_name = devices.get(state.get("device")) + if device_name: + device_names.add(device_name) + entries.append((label, utilization, bench_name)) + + return entries, device_names + + +def plot_entries(entries, title=None, output=None): + if not entries: + print("No utilization data found.") + return 1 + + labels = [entry[0] for entry in entries] + values = [entry[1] for entry in entries] + bench_names = [entry[2] for entry in entries] + + unique_benches = [] + for bench in bench_names: + if bench not in unique_benches: + unique_benches.append(bench) + + cmap = plt.get_cmap("tab20", max(len(unique_benches), 1)) + bench_colors = {bench: cmap(index) for index, bench in enumerate(unique_benches)} + colors = [bench_colors[bench] for bench in bench_names] + + fig_height = max(4.0, 0.3 * len(entries) + 1.5) + fig, ax = plt.subplots(figsize=(10, fig_height)) + + y_pos = range(len(labels)) + ax.barh(y_pos, values, color=colors) + ax.set_yticks(y_pos) + ax.set_yticklabels(labels) + ax.invert_yaxis() + ax.set_ylim(len(labels) - 0.5, -0.5) + + ax.set_xlabel("Global BW Utilization") + ax.xaxis.set_major_formatter(PercentFormatter(1.0)) + + if title: + ax.set_title(title) + + fig.tight_layout() + + if output: + fig.savefig(output, dpi=150) + else: + plt.show() + + return 0 + + +def main(): + args, filenames = parse_files() + try: + axis_filters = parse_axis_filters(args.axis) + except ValueError as exc: + print(str(exc)) + return 1 + entries = [] + device_names = set() + for filename in filenames: + file_entries, file_device_names = collect_entries( + filename, + axis_filters, + args.benchmark, + ) + entries.extend(file_entries) + device_names.update(file_device_names) + + title = args.title + if title is None: + title = "%SOL Bandwidth" + if len(device_names) == 1: + title = "{} - {}".format(title, next(iter(device_names))) + + return plot_entries(entries, title=title, output=args.output) + + +if __name__ == "__main__": + sys.exit(main()) From ccde9fc4d4eb1f675ce614a84cd6addd1a02a347 Mon Sep 17 00:00:00 2001 From: Bernhard Manfred Gruber Date: Thu, 5 Feb 2026 10:56:36 +0100 Subject: [PATCH 2/6] More --- python/scripts/nvbench_plot.py | 51 ++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/python/scripts/nvbench_plot.py b/python/scripts/nvbench_plot.py index 249abfd..63330a0 100644 --- a/python/scripts/nvbench_plot.py +++ b/python/scripts/nvbench_plot.py @@ -30,6 +30,11 @@ def parse_files(): default=None, help="Optional plot title", ) + parser.add_argument( + "--dark", + action="store_true", + help="Use dark theme (black background, white text)", + ) parser.add_argument( "-a", "--axis", @@ -96,6 +101,7 @@ def parse_axis_filters(axis_args): value = value.strip() if not name or not value: raise ValueError("Axis filter must be NAME=VALUE: {}".format(axis_arg)) + display_value = value if name.endswith("[pow2]"): name = name[: -len("[pow2]")].strip() if not name: @@ -109,7 +115,14 @@ def parse_axis_filters(axis_args): "Axis filter [pow2] value must be integer: {}".format(axis_arg) ) from exc value = str(2**exponent) - filters.append((name, value)) + display_value = "2^{}".format(exponent) + filters.append( + { + "name": name, + "value": value, + "display": "{}={}".format(name, display_value), + } + ) return filters @@ -118,7 +131,9 @@ def matches_axis_filters(state, axis_filters): return True axis_values = state.get("axis_values") or [] - for filter_name, filter_value in axis_filters: + for axis_filter in axis_filters: + filter_name = axis_filter["name"] + filter_value = axis_filter["value"] matched = False for axis_value in axis_values: if axis_value.get("name") != filter_name: @@ -134,6 +149,20 @@ def matches_axis_filters(state, axis_filters): return True +def strip_axis_filters_from_state_name(state_name, axis_filters): + if not axis_filters: + return state_name + + tokens = state_name.split() + tokens_to_remove = set( + axis_filter["display"] + for axis_filter in axis_filters + if " " not in axis_filter["display"] + ) + tokens = [token for token in tokens if token not in tokens_to_remove] + return " ".join(tokens) + + def collect_entries(filename, axis_filters, benchmark_filters): json_root = reader.read_file(filename) entries = [] @@ -155,6 +184,7 @@ def collect_entries(filename, axis_filters, benchmark_filters): parts = state_name.split(" ", 1) if len(parts) == 2: state_name = parts[1] + state_name = strip_axis_filters_from_state_name(state_name, axis_filters) label = "{} | {}".format(bench_name, state_name) device_name = devices.get(state.get("device")) if device_name: @@ -164,7 +194,7 @@ def collect_entries(filename, axis_filters, benchmark_filters): return entries, device_names -def plot_entries(entries, title=None, output=None): +def plot_entries(entries, title=None, output=None, dark=False): if not entries: print("No utilization data found.") return 1 @@ -184,6 +214,15 @@ def plot_entries(entries, title=None, output=None): fig_height = max(4.0, 0.3 * len(entries) + 1.5) fig, ax = plt.subplots(figsize=(10, fig_height)) + if dark: + fig.patch.set_facecolor("black") + ax.set_facecolor("black") + ax.tick_params(colors="white") + ax.xaxis.label.set_color("white") + ax.yaxis.label.set_color("white") + ax.title.set_color("white") + for spine in ax.spines.values(): + spine.set_color("white") y_pos = range(len(labels)) ax.barh(y_pos, values, color=colors) @@ -192,7 +231,6 @@ def plot_entries(entries, title=None, output=None): ax.invert_yaxis() ax.set_ylim(len(labels) - 0.5, -0.5) - ax.set_xlabel("Global BW Utilization") ax.xaxis.set_major_formatter(PercentFormatter(1.0)) if title: @@ -231,8 +269,11 @@ def main(): title = "%SOL Bandwidth" if len(device_names) == 1: title = "{} - {}".format(title, next(iter(device_names))) + if axis_filters: + axis_label = ", ".join(axis_filter["display"] for axis_filter in axis_filters) + title = "{} ({})".format(title, axis_label) - return plot_entries(entries, title=title, output=args.output) + return plot_entries(entries, title=title, output=args.output, dark=args.dark) if __name__ == "__main__": From ec9759037d06cf7586e2631b7d0ffe94fa6f6751 Mon Sep 17 00:00:00 2001 From: Bernhard Manfred Gruber Date: Thu, 5 Feb 2026 11:15:27 +0100 Subject: [PATCH 3/6] I have no idea what I am doing --- python/scripts/nvbench_plot.py | 50 ++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/python/scripts/nvbench_plot.py b/python/scripts/nvbench_plot.py index 63330a0..4cdf4bd 100644 --- a/python/scripts/nvbench_plot.py +++ b/python/scripts/nvbench_plot.py @@ -101,7 +101,19 @@ def parse_axis_filters(axis_args): value = value.strip() if not name or not value: raise ValueError("Axis filter must be NAME=VALUE: {}".format(axis_arg)) - display_value = value + + values = [] + display_values = [] + if value.startswith("[") and value.endswith("]"): + inner = value[1:-1].strip() + if inner: + values = [item.strip() for item in inner.split(",") if item.strip()] + else: + values = [] + else: + values = [value] + display_values = list(values) + if name.endswith("[pow2]"): name = name[: -len("[pow2]")].strip() if not name: @@ -109,18 +121,28 @@ def parse_axis_filters(axis_args): "Axis filter missing name before [pow2]: {}".format(axis_arg) ) try: - exponent = int(value) + exponents = [int(v) for v in values] except ValueError as exc: raise ValueError( "Axis filter [pow2] value must be integer: {}".format(axis_arg) ) from exc - value = str(2**exponent) - display_value = "2^{}".format(exponent) + values = [str(2**exponent) for exponent in exponents] + display_values = ["2^{}".format(exponent) for exponent in exponents] + + if not values: + raise ValueError( + "Axis filter must specify at least one value: {}".format(axis_arg) + ) + + if len(display_values) == 1: + display = "{}={}".format(name, display_values[0]) + else: + display = "{}=[{}]".format(name, ",".join(display_values)) filters.append( { "name": name, - "value": value, - "display": "{}={}".format(name, display_value), + "values": values, + "display": display, } ) return filters @@ -133,7 +155,7 @@ def matches_axis_filters(state, axis_filters): axis_values = state.get("axis_values") or [] for axis_filter in axis_filters: filter_name = axis_filter["name"] - filter_value = axis_filter["value"] + filter_values = axis_filter["values"] matched = False for axis_value in axis_values: if axis_value.get("name") != filter_name: @@ -141,7 +163,7 @@ def matches_axis_filters(state, axis_filters): value = axis_value.get("value") if value is None: continue - if str(value) == filter_value: + if str(value) in filter_values: matched = True break if not matched: @@ -154,12 +176,16 @@ def strip_axis_filters_from_state_name(state_name, axis_filters): return state_name tokens = state_name.split() - tokens_to_remove = set( - axis_filter["display"] + filter_prefixes = set( + "{}=".format(axis_filter["name"]) for axis_filter in axis_filters - if " " not in axis_filter["display"] + if len(axis_filter["values"]) == 1 ) - tokens = [token for token in tokens if token not in tokens_to_remove] + tokens = [ + token + for token in tokens + if not any(token.startswith(prefix) for prefix in filter_prefixes) + ] return " ".join(tokens) From 28ed32bb474cd05e20d56acb54e1c71f88b3b249 Mon Sep 17 00:00:00 2001 From: Bernhard Manfred Gruber Date: Thu, 5 Feb 2026 14:00:33 +0100 Subject: [PATCH 4/6] Implement dark mode using style sheets --- python/scripts/nvbench_plot.py | 41 ++++++++++++++-------------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/python/scripts/nvbench_plot.py b/python/scripts/nvbench_plot.py index 4cdf4bd..8ef6086 100644 --- a/python/scripts/nvbench_plot.py +++ b/python/scripts/nvbench_plot.py @@ -239,35 +239,28 @@ def plot_entries(entries, title=None, output=None, dark=False): colors = [bench_colors[bench] for bench in bench_names] fig_height = max(4.0, 0.3 * len(entries) + 1.5) - fig, ax = plt.subplots(figsize=(10, fig_height)) - if dark: - fig.patch.set_facecolor("black") - ax.set_facecolor("black") - ax.tick_params(colors="white") - ax.xaxis.label.set_color("white") - ax.yaxis.label.set_color("white") - ax.title.set_color("white") - for spine in ax.spines.values(): - spine.set_color("white") + style = "dark_background" if dark else None + with plt.style.context(style) if style else plt.style.context("default"): + fig, ax = plt.subplots(figsize=(10, fig_height)) - y_pos = range(len(labels)) - ax.barh(y_pos, values, color=colors) - ax.set_yticks(y_pos) - ax.set_yticklabels(labels) - ax.invert_yaxis() - ax.set_ylim(len(labels) - 0.5, -0.5) + y_pos = range(len(labels)) + ax.barh(y_pos, values, color=colors) + ax.set_yticks(y_pos) + ax.set_yticklabels(labels) + ax.invert_yaxis() + ax.set_ylim(len(labels) - 0.5, -0.5) - ax.xaxis.set_major_formatter(PercentFormatter(1.0)) + ax.xaxis.set_major_formatter(PercentFormatter(1.0)) - if title: - ax.set_title(title) + if title: + ax.set_title(title) - fig.tight_layout() + fig.tight_layout() - if output: - fig.savefig(output, dpi=150) - else: - plt.show() + if output: + fig.savefig(output, dpi=150) + else: + plt.show() return 0 From d3a0bec4a860ad8863d7c67baabbca3580829073 Mon Sep 17 00:00:00 2001 From: Bernhard Manfred Gruber Date: Thu, 5 Feb 2026 14:13:16 +0100 Subject: [PATCH 5/6] Feedback from review --- python/scripts/nvbench_plot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/scripts/nvbench_plot.py b/python/scripts/nvbench_plot.py index 8ef6086..f8a34df 100644 --- a/python/scripts/nvbench_plot.py +++ b/python/scripts/nvbench_plot.py @@ -239,8 +239,8 @@ def plot_entries(entries, title=None, output=None, dark=False): colors = [bench_colors[bench] for bench in bench_names] fig_height = max(4.0, 0.3 * len(entries) + 1.5) - style = "dark_background" if dark else None - with plt.style.context(style) if style else plt.style.context("default"): + style = "dark_background" if dark else "default" + with plt.style.context(style): fig, ax = plt.subplots(figsize=(10, fig_height)) y_pos = range(len(labels)) From 800f640c20de865d9175e2aa25fdbce7e651f3ef Mon Sep 17 00:00:00 2001 From: Bernhard Manfred Gruber Date: Thu, 26 Feb 2026 19:23:51 +0100 Subject: [PATCH 6/6] Apply reviewer feedback --- ...nvbench_plot.py => nvbench_plot_bwutil.py} | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) rename python/scripts/{nvbench_plot.py => nvbench_plot_bwutil.py} (94%) diff --git a/python/scripts/nvbench_plot.py b/python/scripts/nvbench_plot_bwutil.py similarity index 94% rename from python/scripts/nvbench_plot.py rename to python/scripts/nvbench_plot_bwutil.py index f8a34df..a7e7148 100644 --- a/python/scripts/nvbench_plot.py +++ b/python/scripts/nvbench_plot_bwutil.py @@ -12,7 +12,6 @@ try: except ImportError: from scripts.nvbench_json import reader - UTILIZATION_TAG = "nv/cold/bw/global/utilization" @@ -55,12 +54,11 @@ def parse_files(): for file_or_dir in files_or_dirs: if os.path.isdir(file_or_dir): for f in os.listdir(file_or_dir): - if os.path.splitext(f)[1] != ".json": - continue filename = os.path.join(file_or_dir, f) if os.path.isfile(filename) and os.path.getsize(filename) > 0: filenames.append(filename) else: + assert os.path.isfile(file_or_dir) filenames.append(file_or_dir) filenames.sort() @@ -107,7 +105,9 @@ def parse_axis_filters(axis_args): if value.startswith("[") and value.endswith("]"): inner = value[1:-1].strip() if inner: - values = [item.strip() for item in inner.split(",") if item.strip()] + values = [ + stripped for item in inner.split(",") if (stripped := item.strip()) + ] else: values = [] else: @@ -189,7 +189,9 @@ def strip_axis_filters_from_state_name(state_name, axis_filters): return " ".join(tokens) -def collect_entries(filename, axis_filters, benchmark_filters): +def collect_entries( + filename: str, axis_filters: list[dict], benchmark_filters: list[str] +) -> tuple[list[tuple[str, float, str]], set[str]]: json_root = reader.read_file(filename) entries = [] device_names = set() @@ -225,14 +227,8 @@ def plot_entries(entries, title=None, output=None, dark=False): print("No utilization data found.") return 1 - labels = [entry[0] for entry in entries] - values = [entry[1] for entry in entries] - bench_names = [entry[2] for entry in entries] - - unique_benches = [] - for bench in bench_names: - if bench not in unique_benches: - unique_benches.append(bench) + labels, values, bench_names = map(list, zip(*entries)) + unique_benches = list(set(bench_names)) cmap = plt.get_cmap("tab20", max(len(unique_benches), 1)) bench_colors = {bench: cmap(index) for index, bench in enumerate(unique_benches)}