NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-04-20 14:58:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Formatting updates.

Browse Source
This commit is contained in:
Allison Piper
2025-04-14 17:26:12 +00:00
parent de36f1a248
commit 3440855dbd
107 changed files with 808 additions and 967 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
scripts/nvbench_compare.py
View File

@@ -5,12 +5,11 @@ import math
import os
import sys
from colorama import Fore
import tabulate
from colorama import Fore
from nvbench_json import reader
# Parse version string into tuple, "x.y.z" -> (x, y, z)
def version_tuple(v):
return tuple(map(int, (v.split("."))))
@@ -139,15 +138,14 @@ def compare_benches(ref_benches, cmp_benches, threshold, plot):
colalign.append("center")
for device_id in device_ids:
rows = []
plot_data = {'cmp': {}, 'ref': {}, 'cmp_noise': {}, 'ref_noise': {}}
plot_data = {"cmp": {}, "ref": {}, "cmp_noise": {}, "ref_noise": {}}
for cmp_state in cmp_states:
cmp_state_name = cmp_state["name"]
ref_state = next(filter(lambda st: st["name"] == cmp_state_name,
ref_states),
None)
ref_state = next(
filter(lambda st: st["name"] == cmp_state_name, ref_states), None
)
if not ref_state:
continue
@@ -158,9 +156,7 @@ def compare_benches(ref_benches, cmp_benches, threshold, plot):
row = []
for axis_value in axis_values:
axis_value_name = axis_value["name"]
row.append(format_axis_value(axis_value_name,
axis_value,
axes))
row.append(format_axis_value(axis_value_name, axis_value, axes))
cmp_summaries = cmp_state["summaries"]
ref_summaries = ref_state["summaries"]
@@ -171,23 +167,37 @@ def compare_benches(ref_benches, cmp_benches, threshold, plot):
def lookup_summary(summaries, tag):
return next(filter(lambda s: s["tag"] == tag, summaries), None)
cmp_time_summary = lookup_summary(cmp_summaries, "nv/cold/time/gpu/mean")
ref_time_summary = lookup_summary(ref_summaries, "nv/cold/time/gpu/mean")
cmp_noise_summary = lookup_summary(cmp_summaries, "nv/cold/time/gpu/stdev/relative")
ref_noise_summary = lookup_summary(ref_summaries, "nv/cold/time/gpu/stdev/relative")
cmp_time_summary = lookup_summary(
cmp_summaries, "nv/cold/time/gpu/mean"
)
ref_time_summary = lookup_summary(
ref_summaries, "nv/cold/time/gpu/mean"
)
cmp_noise_summary = lookup_summary(
cmp_summaries, "nv/cold/time/gpu/stdev/relative"
)
ref_noise_summary = lookup_summary(
ref_summaries, "nv/cold/time/gpu/stdev/relative"
)
# TODO: Use other timings, too. Maybe multiple rows, with a
# "Timing" column + values "CPU/GPU/Batch"?
if not all([cmp_time_summary,
ref_time_summary,
cmp_noise_summary,
ref_noise_summary]):
if not all(
[
cmp_time_summary,
ref_time_summary,
cmp_noise_summary,
ref_noise_summary,
]
):
continue
def extract_value(summary):
summary_data = summary["data"]
value_data = next(filter(lambda v: v["name"] == "value", summary_data))
assert(value_data["type"] == "float64")
value_data = next(
filter(lambda v: v["name"] == "value", summary_data)
)
assert value_data["type"] == "float64"
return value_data["value"]
cmp_time = extract_value(cmp_time_summary)
@@ -218,23 +228,27 @@ def compare_benches(ref_benches, cmp_benches, threshold, plot):
if plot:
axis_name = []
axis_value = "--"
for aid in range(len(axis_values)):
for aid in range(len(axis_values)):
if axis_values[aid]["name"] != plot:
axis_name.append("{} = {}".format(axis_values[aid]["name"], axis_values[aid]["value"]))
axis_name.append(
"{} = {}".format(
axis_values[aid]["name"], axis_values[aid]["value"]
)
)
else:
axis_value = float(axis_values[aid]["value"])
axis_name = ', '.join(axis_name)
axis_value = float(axis_values[aid]["value"])
axis_name = ", ".join(axis_name)
if axis_name not in plot_data['cmp']:
plot_data['cmp'][axis_name] = {}
plot_data['ref'][axis_name] = {}
plot_data['cmp_noise'][axis_name] = {}
plot_data['ref_noise'][axis_name] = {}
if axis_name not in plot_data["cmp"]:
plot_data["cmp"][axis_name] = {}
plot_data["ref"][axis_name] = {}
plot_data["cmp_noise"][axis_name] = {}
plot_data["ref_noise"][axis_name] = {}
plot_data['cmp'][axis_name][axis_value] = cmp_time
plot_data['ref'][axis_name][axis_value] = ref_time
plot_data['cmp_noise'][axis_name][axis_value] = cmp_noise
plot_data['ref_noise'][axis_name][axis_value] = ref_noise
plot_data["cmp"][axis_name][axis_value] = cmp_time
plot_data["ref"][axis_name][axis_value] = ref_time
plot_data["cmp_noise"][axis_name][axis_value] = cmp_noise
plot_data["ref_noise"][axis_name][axis_value] = ref_noise
global config_count
global unknown_count
@@ -273,14 +287,13 @@ def compare_benches(ref_benches, cmp_benches, threshold, plot):
print("## [%d] %s\n" % (device["id"], device["name"]))
# colalign and github format require tabulate 0.8.3
if tabulate_version >= (0, 8, 3):
print(tabulate.tabulate(rows,
headers=headers,
colalign=colalign,
tablefmt="github"))
print(
tabulate.tabulate(
rows, headers=headers, colalign=colalign, tablefmt="github"
)
)
else:
print(tabulate.tabulate(rows,
headers=headers,
tablefmt="markdown"))
print(tabulate.tabulate(rows, headers=headers, tablefmt="markdown"))
print("")
@@ -295,18 +308,17 @@ def compare_benches(ref_benches, cmp_benches, threshold, plot):
x = [float(x) for x in plot_data[key][axis].keys()]
y = list(plot_data[key][axis].values())
noise = list(plot_data[key + '_noise'][axis].values())
noise = list(plot_data[key + "_noise"][axis].values())
top = [y[i] + y[i] * noise[i] for i in range(len(x))]
bottom = [y[i] - y[i] * noise[i] for i in range(len(x))]
p = plt.plot(x, y, shape, marker='o', label=label)
p = plt.plot(x, y, shape, marker="o", label=label)
plt.fill_between(x, bottom, top, color=p[0].get_color(), alpha=0.1)
for axis in plot_data['cmp'].keys():
plot_line('cmp', '-', axis)
plot_line('ref', '--', axis + ' ref')
for axis in plot_data["cmp"].keys():
plot_line("cmp", "-", axis)
plot_line("ref", "--", axis + " ref")
plt.legend()
plt.show()
@@ -314,11 +326,17 @@ def compare_benches(ref_benches, cmp_benches, threshold, plot):
def main():
help_text = "%(prog)s [reference.json compare.json | reference_dir/ compare_dir/]"
parser = argparse.ArgumentParser(prog='nvbench_compare', usage=help_text)
parser.add_argument('--threshold-diff', type=float, dest='threshold', default=0.0,
help='only show benchmarks where percentage diff is >= THRESHOLD')
parser.add_argument('--plot-along', type=str, dest='plot', default=None,
help='plot results')
parser = argparse.ArgumentParser(prog="nvbench_compare", usage=help_text)
parser.add_argument(
"--threshold-diff",
type=float,
dest="threshold",
default=0.0,
help="only show benchmarks where percentage diff is >= THRESHOLD",
)
parser.add_argument(
"--plot-along", type=str, dest="plot", default=None, help="plot results"
)
args, files_or_dirs = parser.parse_known_args()
print(files_or_dirs)
@@ -336,14 +354,17 @@ def main():
continue
r = os.path.join(files_or_dirs[0], f)
c = os.path.join(files_or_dirs[1], f)
if os.path.isfile(r) and os.path.isfile(c) and \
os.path.getsize(r) > 0 and os.path.getsize(c) > 0:
if (
os.path.isfile(r)
and os.path.isfile(c)
and os.path.getsize(r) > 0
and os.path.getsize(c) > 0
):
to_compare.append((r, c))
else:
to_compare = [(files_or_dirs[0], files_or_dirs[1])]
for ref, comp in to_compare:
ref_root = reader.read_file(ref)
cmp_root = reader.read_file(comp)
@@ -355,7 +376,9 @@ def main():
print("Device sections do not match.")
sys.exit(1)
compare_benches(ref_root["benchmarks"], cmp_root["benchmarks"], args.threshold, args.plot)
compare_benches(
ref_root["benchmarks"], cmp_root["benchmarks"], args.threshold, args.plot
)
print("# Summary\n")
print("- Total Matches: %d" % config_count)
@@ -365,5 +388,5 @@ def main():
return failure_count
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())

27
scripts/nvbench_histogram.py
View File

@@ -1,19 +1,19 @@
#!/usr/bin/env python
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import argparse
import os
import sys
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from nvbench_json import reader
def parse_files():
help_text = "%(prog)s [nvbench.out.json | dir/] ..."
parser = argparse.ArgumentParser(prog='nvbench_histogram', usage=help_text)
parser = argparse.ArgumentParser(prog="nvbench_histogram", usage=help_text)
args, files_or_dirs = parser.parse_known_args()
@@ -41,14 +41,14 @@ def parse_files():
def extract_filename(summary):
summary_data = summary["data"]
value_data = next(filter(lambda v: v["name"] == "filename", summary_data))
assert(value_data["type"] == "string")
assert value_data["type"] == "string"
return value_data["value"]
def extract_size(summary):
summary_data = summary["data"]
value_data = next(filter(lambda v: v["name"] == "size", summary_data))
assert(value_data["type"] == "int64")
assert value_data["type"] == "int64"
return int(value_data["value"])
@@ -57,9 +57,10 @@ def parse_samples_meta(filename, state):
if not summaries:
return None, None
summary = next(filter(lambda s: s["tag"] == "nv/json/bin:nv/cold/sample_times",
summaries),
None)
summary = next(
filter(lambda s: s["tag"] == "nv/json/bin:nv/cold/sample_times", summaries),
None,
)
if not summary:
return None, None
@@ -81,7 +82,7 @@ def parse_samples(filename, state):
with open(samples_filename, "rb") as f:
samples = np.fromfile(f, "<f4")
assert (sample_count == len(samples))
assert sample_count == len(samples)
return samples
@@ -118,5 +119,5 @@ def main():
plt.show()
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())

5
scripts/nvbench_json/__init__.py
View File

@@ -1,2 +1,3 @@
from . import reader
from . import version
from . import reader, version
__all__ = ["reader", "version"]

20
scripts/nvbench_json/version.py
View File

@@ -1,8 +1,8 @@
file_version = (1, 0, 0)
file_version_string = "{}.{}.{}".format(file_version[0],
file_version[1],
file_version[2])
file_version_string = "{}.{}.{}".format(
file_version[0], file_version[1], file_version[2]
)
def check_file_version(filename, root_node):
@@ -19,8 +19,14 @@ def check_file_version(filename, root_node):
# for now just warn on mismatch.
if version_node["string"] != file_version_string:
print("WARNING:")
print(" {} was written using a different NVBench JSON file version."
.format(filename))
print(
" {} was written using a different NVBench JSON file version.".format(
filename
)
)
print(" It may not read correctly.")
print(" (file version: {} reader version: {})"
.format(version_node["string"], file_version_string))
print(
" (file version: {} reader version: {})".format(
version_node["string"], file_version_string
)
)

54
scripts/nvbench_walltime.py
View File

@@ -5,9 +5,8 @@ import math
import os
import sys
from nvbench_json import reader
import tabulate
from nvbench_json import reader
# Parse version string into tuple, "x.y.z" -> (x, y, z)
@@ -39,7 +38,8 @@ def format_walltime(seconds_in):
"{:0>2d}:".format(h) if h > 1e-9 else "",
"{:0>2d}:".format(m) if (h > 1e-9 or m > 1e-9) else "",
"{:0>2d}.".format(s) if (h > 1e-9 or m > 1e-9) else "{:d}.".format(s),
"{:0>3d}".format(ms))
"{:0>3d}".format(ms),
)
def format_percentage(percentage):
@@ -58,7 +58,7 @@ measure_column_names = {"cold": "Isolated", "batch": "Batch", "cupti": "CUPTI"}
def init_measures():
out = {}
for name in measure_names:
out[name] = 0.
out[name] = 0.0
return out
@@ -67,17 +67,17 @@ def get_measures(state):
times = {}
for name in measure_names:
measure_walltime_tag = "nv/{}/walltime".format(name)
summary = next(filter(lambda s: s["tag"] == measure_walltime_tag,
summaries),
None)
summary = next(
filter(lambda s: s["tag"] == measure_walltime_tag, summaries), None
)
if not summary:
continue
walltime_data = next(filter(lambda d: d["name"] == "value", summary["data"]))
assert(walltime_data["type"] == "float64")
assert walltime_data["type"] == "float64"
walltime = walltime_data["value"]
walltime = float(walltime)
times[name] = walltime if walltime else 0.
times[name] = walltime if walltime else 0.0
return times
@@ -87,7 +87,7 @@ def merge_measures(target, src):
def sum_measures(measures):
total_time = 0.
total_time = 0.0
for time in measures.values():
total_time += time
return total_time
@@ -194,20 +194,21 @@ def print_overview_section(data):
# colalign and github format require tabulate 0.8.3
if tabulate_version >= (0, 8, 3):
print(tabulate.tabulate(rows,
headers=headers,
colalign=colalign,
tablefmt="github"))
print(
tabulate.tabulate(
rows, headers=headers, colalign=colalign, tablefmt="github"
)
)
else:
print(tabulate.tabulate(rows,
headers=headers,
tablefmt="markdown"))
print(tabulate.tabulate(rows, headers=headers, tablefmt="markdown"))
print()
# append_data_row_lambda args: (row_list, name, items[name])
def print_measures_table(headers, colalign, items, total_measures, append_item_row_lambda):
def print_measures_table(
headers, colalign, items, total_measures, append_item_row_lambda
):
total_time = sum_measures(total_measures)
active_measures = get_active_measure_names(total_measures)
num_user_columns = len(headers)
@@ -248,14 +249,13 @@ def print_measures_table(headers, colalign, items, total_measures, append_item_r
# colalign and github format require tabulate 0.8.3
if tabulate_version >= (0, 8, 3):
print(tabulate.tabulate(rows,
headers=headers,
colalign=colalign,
tablefmt="github"))
print(
tabulate.tabulate(
rows, headers=headers, colalign=colalign, tablefmt="github"
)
)
else:
print(tabulate.tabulate(rows,
headers=headers,
tablefmt="markdown"))
print(tabulate.tabulate(rows, headers=headers, tablefmt="markdown"))
def print_files_section(data):
@@ -319,7 +319,7 @@ def print_bench_section(bench_name, bench):
def main():
help_text = "%(prog)s [nvbench.out.json | dir/]..."
parser = argparse.ArgumentParser(prog='nvbench_walltime', usage=help_text)
parser = argparse.ArgumentParser(prog="nvbench_walltime", usage=help_text)
args, files_or_dirs = parser.parse_known_args()
@@ -353,5 +353,5 @@ def main():
print_files_section(data)
if __name__ == '__main__':
if __name__ == "__main__":
sys.exit(main())