diff --git a/scripts/nvbench_compare.py b/scripts/nvbench_compare.py index b995bf1..a9359fa 100755 --- a/scripts/nvbench_compare.py +++ b/scripts/nvbench_compare.py @@ -92,7 +92,7 @@ def format_percentage(percentage): # When there aren't enough samples for a meaningful noise measurement, # the noise is recorded as infinity. Unfortunately, JSON spec doesn't # allow for inf, so these get turned into null. - if not percentage: + if percentage is None: return "inf" return "%0.2f%%" % (percentage * 100.0) diff --git a/scripts/nvbench_walltime.py b/scripts/nvbench_walltime.py new file mode 100644 index 0000000..611ecfc --- /dev/null +++ b/scripts/nvbench_walltime.py @@ -0,0 +1,352 @@ +#!/usr/bin/env python + +import argparse +import math +import json +import os +import sys + +import tabulate + + +# Parse version string into tuple, "x.y.z" -> (x, y, z) +def version_tuple(v): + return tuple(map(int, (v.split(".")))) + + +tabulate_version = version_tuple(tabulate.__version__) + +all_devices = [] + + +def format_axis_value(axis_value, axis_type): + if axis_type == "int64": + return "%d" % int(axis_value) + elif axis_type == "float64": + return "%.5g" % float(axis_value) + else: + return axis_value + + +def format_walltime(seconds_in): + h = math.floor(seconds_in / (60 * 60)) + m = math.floor((seconds_in / 60) % 60) + s = math.floor(seconds_in % 60) + ms = math.floor((seconds_in * 1000) % 1000) + + return "{}{}{}{}".format( + "{:0>2d}:".format(h) if h > 1e-9 else "", + "{:0>2d}:".format(m) if (h > 1e-9 or m > 1e-9) else "", + "{:0>2d}.".format(s) if (h > 1e-9 or m > 1e-9) else "{:d}.".format(s), + "{:0>3d}".format(ms)) + + +def format_percentage(percentage): + # When there aren't enough samples for a meaningful noise measurement, + # the noise is recorded as infinity. Unfortunately, JSON spec doesn't + # allow for inf, so these get turned into null. + if percentage is None: + return "inf" + return "%0.2f%%" % (percentage * 100.0) + + +measure_names = ["cold", "batch", "cupti"] +measure_column_names = {"cold": "Isolated", "batch": "Batch", "cupti": "CUPTI"} + + +def init_measures(): + out = {} + for name in measure_names: + out[name] = 0. + return out + + +def get_measures(state): + times = {} + for name in measure_names: + try: + time = state["summaries"]["nv/%s/walltime" % name]["value"]["value"] + time = float(time) + except KeyError: + time = None + except TypeError: + time = None + times[name] = time if time else 0. + return times + + +def merge_measures(target, src): + for name, src_val in src.items(): + target[name] += src_val + + +def sum_measures(measures): + total_time = 0. + for time in measures.values(): + total_time += time + return total_time + + +def get_active_measure_names(measures): + names = [] + for name, time in measures.items(): + if time > 1e-9: + names.append(name) + return names + + +def append_measure_headers(headers, active=measure_names): + for name in active: + headers.append(measure_column_names[name]) + + +def append_measure_values(row, measures, active=measure_names): + for name in active: + row.append(format_walltime(measures[name])) + + +def consume_file(filename): + with open(filename, "r") as f: + file_root = json.load(f) + + file_out = {} + file_measures = init_measures() + + benches = {} + for bench in file_root["benchmarks"]: + bench_data = consume_benchmark(bench, file_root) + merge_measures(file_measures, bench_data["measures"]) + benches[bench["name"]] = bench_data + + file_out["benches"] = benches + file_out["measures"] = file_measures + return file_out + + +def consume_benchmark(bench, file_root): + bench_out = {} + + # Initialize axis map + axes_out = {} + axes = bench["axes"] + if axes: + for axis_name, axis in axes.items(): + values_out = {} + axis_type = axis["type"] + for value in axis["values"]: + if axis_type == "type": + value = value["input_string"] + else: + value = format_axis_value(value["value"], axis_type) + values_out[value] = {"measures": init_measures()} + axes_out[axis_name] = values_out + + states_out = {} + bench_measures = init_measures() + + for state_name, state in bench["states"].items(): + # Get walltimes for each measurement: + state_measures = get_measures(state) + state_out = {} + state_out["measures"] = state_measures + states_out[state_name] = state_out + + # Update the benchmark measures walltimes + merge_measures(bench_measures, state_measures) + + # Update the axis measurements: + axis_values = state["axis_values"] + if axis_values: + for axis_name, value_data in axis_values.items(): + value = format_axis_value(value_data["value"], value_data["type"]) + merge_measures(axes_out[axis_name][value]["measures"], state_measures) + + bench_out["axes"] = axes_out + bench_out["measures"] = bench_measures + bench_out["states"] = states_out + return bench_out + + +def print_overview_section(data): + print("# Walltime Overview\n") + + measures = data["measures"] + active_measures = get_active_measure_names(measures) + + headers = ["Walltime"] + append_measure_headers(headers, active_measures) + + colalign = ["right"] * len(headers) + + rows = [] + + row = [format_walltime(sum_measures(measures))] + append_measure_values(row, measures, active_measures) + rows.append(row) + + # colalign and github format require tabulate 0.8.3 + if tabulate_version >= (0, 8, 3): + print(tabulate.tabulate(rows, + headers=headers, + colalign=colalign, + tablefmt="github")) + else: + print(tabulate.tabulate(rows, + headers=headers, + tablefmt="markdown")) + + print() + + +# append_data_row_lambda args: (row_list, name, items[name]) +def print_measures_table(headers, colalign, items, total_measures, append_item_row_lambda): + total_time = sum_measures(total_measures) + active_measures = get_active_measure_names(total_measures) + num_user_columns = len(headers) + + headers.append("%") + headers.append("Walltime") + append_measure_headers(headers, active_measures) + + while len(colalign) < len(headers): + colalign.append("right") + + rows = [] + + for name, item in items.items(): + item_measures = item["measures"] + item_time = sum_measures(item_measures) + + row = [] + append_item_row_lambda(row, name, item) + if total_time > 1e-9: + row.append(format_percentage(item_time / total_time)) + else: + row.append(format_percentage(0)) + row.append(format_walltime(item_time)) + append_measure_values(row, item_measures, active_measures) + rows.append(row) + + # Totals: + row = [] + if num_user_columns != 0: + row.append("Total") + while len(row) < num_user_columns: + row.append("") + row.append(format_percentage(1)) + row.append(format_walltime(total_time)) + append_measure_values(row, total_measures, active_measures) + rows.append(row) + + # colalign and github format require tabulate 0.8.3 + if tabulate_version >= (0, 8, 3): + print(tabulate.tabulate(rows, + headers=headers, + colalign=colalign, + tablefmt="github")) + else: + print(tabulate.tabulate(rows, + headers=headers, + tablefmt="markdown")) + + +def print_files_section(data): + print("# Files\n") + + items = data["files"] + total_measures = data["measures"] + headers = ["Filename"] + colalign = ["left"] + + def append_row(row, name, item): + row.append(name) + + print_measures_table(headers, colalign, items, total_measures, append_row) + print() + + for filename, file in items.items(): + print_file_section(filename, file) + + +def print_file_section(filename, file): + print("## File: {}\n".format(filename)) + + items = file["benches"] + total_measures = file["measures"] + headers = ["Benchmark"] + colalign = ["left"] + + def append_row_name(row, name, item): + row.append(name) + + print_measures_table(headers, colalign, items, total_measures, append_row_name) + print() + + for bench_name, bench in items.items(): + print_bench_section(bench_name, bench) + + +def print_bench_section(bench_name, bench): + print("### Benchmark: {}\n".format(bench_name)) + + # TODO split this up so each axis is a column + items = bench["states"] + total_measures = bench["measures"] + headers = ["Configuration"] + colalign = ["left"] + + def append_row_name(row, name, item): + row.append(name) + + print_measures_table(headers, colalign, items, total_measures, append_row_name) + print() + + for axis_name, axis in bench["axes"].items(): + total_measures = bench["measures"] + headers = ["Axis: " + axis_name] + colalign = ["left"] + print_measures_table(headers, colalign, axis, total_measures, append_row_name) + print() + + +def main(): + help_text = "%(prog)s [nvbench.out.json | dir/]..." + parser = argparse.ArgumentParser(prog='nvbench_walltime', usage=help_text) + + args, files_or_dirs = parser.parse_known_args() + + filenames = [] + for file_or_dir in files_or_dirs: + if os.path.isdir(file_or_dir): + for f in os.listdir(file_or_dir): + if os.path.splitext(f)[1] != ".json": + continue + filename = os.path.join(file_or_dir, f) + if os.path.isfile(filename) and os.path.getsize(filename) > 0: + filenames.append(filename) + else: + filenames.append(file_or_dir) + + filenames.sort() + + data = {} + + files_out = {} + measures = init_measures() + for filename in filenames: + file_data = consume_file(filename) + merge_measures(measures, file_data["measures"]) + files_out[filename] = file_data + + data["files"] = files_out + data["measures"] = measures + + # Debug data structure: + # print(json.dumps(data, indent=2)) + + print_overview_section(data) + print_files_section(data) + + +if __name__ == '__main__': + sys.exit(main())