Add nvbench_walltime.py script.

scripts/nvbench_compare.py

@@ -92,7 +92,7 @@ def format_percentage(percentage):
     # When there aren't enough samples for a meaningful noise measurement,
     # the noise is recorded as infinity. Unfortunately, JSON spec doesn't
     # allow for inf, so these get turned into null.
-    if not percentage:
+    if percentage is None:
         return "inf"
     return "%0.2f%%" % (percentage * 100.0)
 

scripts/nvbench_walltime.py

@@ -0,0 +1,352 @@
+#!/usr/bin/env python
+
+import argparse
+import math
+import json
+import os
+import sys
+
+import tabulate
+
+
+# Parse version string into tuple, "x.y.z" -> (x, y, z)
+def version_tuple(v):
+    return tuple(map(int, (v.split("."))))
+
+
+tabulate_version = version_tuple(tabulate.__version__)
+
+all_devices = []
+
+
+def format_axis_value(axis_value, axis_type):
+    if axis_type == "int64":
+        return "%d" % int(axis_value)
+    elif axis_type == "float64":
+        return "%.5g" % float(axis_value)
+    else:
+        return axis_value
+
+
+def format_walltime(seconds_in):
+    h = math.floor(seconds_in / (60 * 60))
+    m = math.floor((seconds_in / 60) % 60)
+    s = math.floor(seconds_in % 60)
+    ms = math.floor((seconds_in * 1000) % 1000)
+
+    return "{}{}{}{}".format(
+        "{:0>2d}:".format(h) if h > 1e-9 else "",
+        "{:0>2d}:".format(m) if (h > 1e-9 or m > 1e-9) else "",
+        "{:0>2d}.".format(s) if (h > 1e-9 or m > 1e-9) else "{:d}.".format(s),
+        "{:0>3d}".format(ms))
+
+
+def format_percentage(percentage):
+    # When there aren't enough samples for a meaningful noise measurement,
+    # the noise is recorded as infinity. Unfortunately, JSON spec doesn't
+    # allow for inf, so these get turned into null.
+    if percentage is None:
+        return "inf"
+    return "%0.2f%%" % (percentage * 100.0)
+
+
+measure_names = ["cold", "batch", "cupti"]
+measure_column_names = {"cold": "Isolated", "batch": "Batch", "cupti": "CUPTI"}
+
+
+def init_measures():
+    out = {}
+    for name in measure_names:
+        out[name] = 0.
+    return out
+
+
+def get_measures(state):
+    times = {}
+    for name in measure_names:
+        try:
+            time = state["summaries"]["nv/%s/walltime" % name]["value"]["value"]
+            time = float(time)
+        except KeyError:
+            time = None
+        except TypeError:
+            time = None
+        times[name] = time if time else 0.
+    return times
+
+
+def merge_measures(target, src):
+    for name, src_val in src.items():
+        target[name] += src_val
+
+
+def sum_measures(measures):
+    total_time = 0.
+    for time in measures.values():
+        total_time += time
+    return total_time
+
+
+def get_active_measure_names(measures):
+    names = []
+    for name, time in measures.items():
+        if time > 1e-9:
+            names.append(name)
+    return names
+
+
+def append_measure_headers(headers, active=measure_names):
+    for name in active:
+        headers.append(measure_column_names[name])
+
+
+def append_measure_values(row, measures, active=measure_names):
+    for name in active:
+        row.append(format_walltime(measures[name]))
+
+
+def consume_file(filename):
+    with open(filename, "r") as f:
+        file_root = json.load(f)
+
+    file_out = {}
+    file_measures = init_measures()
+
+    benches = {}
+    for bench in file_root["benchmarks"]:
+        bench_data = consume_benchmark(bench, file_root)
+        merge_measures(file_measures, bench_data["measures"])
+        benches[bench["name"]] = bench_data
+
+    file_out["benches"] = benches
+    file_out["measures"] = file_measures
+    return file_out
+
+
+def consume_benchmark(bench, file_root):
+    bench_out = {}
+
+    # Initialize axis map
+    axes_out = {}
+    axes = bench["axes"]
+    if axes:
+        for axis_name, axis in axes.items():
+            values_out = {}
+            axis_type = axis["type"]
+            for value in axis["values"]:
+                if axis_type == "type":
+                    value = value["input_string"]
+                else:
+                    value = format_axis_value(value["value"], axis_type)
+                values_out[value] = {"measures": init_measures()}
+            axes_out[axis_name] = values_out
+
+    states_out = {}
+    bench_measures = init_measures()
+
+    for state_name, state in bench["states"].items():
+        # Get walltimes for each measurement:
+        state_measures = get_measures(state)
+        state_out = {}
+        state_out["measures"] = state_measures
+        states_out[state_name] = state_out
+
+        # Update the benchmark measures walltimes
+        merge_measures(bench_measures, state_measures)
+
+        # Update the axis measurements:
+        axis_values = state["axis_values"]
+        if axis_values:
+            for axis_name, value_data in axis_values.items():
+                value = format_axis_value(value_data["value"], value_data["type"])
+                merge_measures(axes_out[axis_name][value]["measures"], state_measures)
+
+    bench_out["axes"] = axes_out
+    bench_out["measures"] = bench_measures
+    bench_out["states"] = states_out
+    return bench_out
+
+
+def print_overview_section(data):
+    print("# Walltime Overview\n")
+
+    measures = data["measures"]
+    active_measures = get_active_measure_names(measures)
+
+    headers = ["Walltime"]
+    append_measure_headers(headers, active_measures)
+
+    colalign = ["right"] * len(headers)
+
+    rows = []
+
+    row = [format_walltime(sum_measures(measures))]
+    append_measure_values(row, measures, active_measures)
+    rows.append(row)
+
+    # colalign and github format require tabulate 0.8.3
+    if tabulate_version >= (0, 8, 3):
+        print(tabulate.tabulate(rows,
+                                headers=headers,
+                                colalign=colalign,
+                                tablefmt="github"))
+    else:
+        print(tabulate.tabulate(rows,
+                                headers=headers,
+                                tablefmt="markdown"))
+
+    print()
+
+
+# append_data_row_lambda args: (row_list, name, items[name])
+def print_measures_table(headers, colalign, items, total_measures, append_item_row_lambda):
+    total_time = sum_measures(total_measures)
+    active_measures = get_active_measure_names(total_measures)
+    num_user_columns = len(headers)
+
+    headers.append("%")
+    headers.append("Walltime")
+    append_measure_headers(headers, active_measures)
+
+    while len(colalign) < len(headers):
+        colalign.append("right")
+
+    rows = []
+
+    for name, item in items.items():
+        item_measures = item["measures"]
+        item_time = sum_measures(item_measures)
+
+        row = []
+        append_item_row_lambda(row, name, item)
+        if total_time > 1e-9:
+            row.append(format_percentage(item_time / total_time))
+        else:
+            row.append(format_percentage(0))
+        row.append(format_walltime(item_time))
+        append_measure_values(row, item_measures, active_measures)
+        rows.append(row)
+
+    # Totals:
+    row = []
+    if num_user_columns != 0:
+        row.append("Total")
+    while len(row) < num_user_columns:
+        row.append("")
+    row.append(format_percentage(1))
+    row.append(format_walltime(total_time))
+    append_measure_values(row, total_measures, active_measures)
+    rows.append(row)
+
+    # colalign and github format require tabulate 0.8.3
+    if tabulate_version >= (0, 8, 3):
+        print(tabulate.tabulate(rows,
+                                headers=headers,
+                                colalign=colalign,
+                                tablefmt="github"))
+    else:
+        print(tabulate.tabulate(rows,
+                                headers=headers,
+                                tablefmt="markdown"))
+
+
+def print_files_section(data):
+    print("# Files\n")
+
+    items = data["files"]
+    total_measures = data["measures"]
+    headers = ["Filename"]
+    colalign = ["left"]
+
+    def append_row(row, name, item):
+        row.append(name)
+
+    print_measures_table(headers, colalign, items, total_measures, append_row)
+    print()
+
+    for filename, file in items.items():
+        print_file_section(filename, file)
+
+
+def print_file_section(filename, file):
+    print("## File: {}\n".format(filename))
+
+    items = file["benches"]
+    total_measures = file["measures"]
+    headers = ["Benchmark"]
+    colalign = ["left"]
+
+    def append_row_name(row, name, item):
+        row.append(name)
+
+    print_measures_table(headers, colalign, items, total_measures, append_row_name)
+    print()
+
+    for bench_name, bench in items.items():
+        print_bench_section(bench_name, bench)
+
+
+def print_bench_section(bench_name, bench):
+    print("### Benchmark: {}\n".format(bench_name))
+
+    # TODO split this up so each axis is a column
+    items = bench["states"]
+    total_measures = bench["measures"]
+    headers = ["Configuration"]
+    colalign = ["left"]
+
+    def append_row_name(row, name, item):
+        row.append(name)
+
+    print_measures_table(headers, colalign, items, total_measures, append_row_name)
+    print()
+
+    for axis_name, axis in bench["axes"].items():
+        total_measures = bench["measures"]
+        headers = ["Axis: " + axis_name]
+        colalign = ["left"]
+        print_measures_table(headers, colalign, axis, total_measures, append_row_name)
+        print()
+
+
+def main():
+    help_text = "%(prog)s [nvbench.out.json | dir/]..."
+    parser = argparse.ArgumentParser(prog='nvbench_walltime', usage=help_text)
+
+    args, files_or_dirs = parser.parse_known_args()
+
+    filenames = []
+    for file_or_dir in files_or_dirs:
+        if os.path.isdir(file_or_dir):
+            for f in os.listdir(file_or_dir):
+                if os.path.splitext(f)[1] != ".json":
+                    continue
+                filename = os.path.join(file_or_dir, f)
+                if os.path.isfile(filename) and os.path.getsize(filename) > 0:
+                    filenames.append(filename)
+        else:
+            filenames.append(file_or_dir)
+
+    filenames.sort()
+
+    data = {}
+
+    files_out = {}
+    measures = init_measures()
+    for filename in filenames:
+        file_data = consume_file(filename)
+        merge_measures(measures, file_data["measures"])
+        files_out[filename] = file_data
+
+    data["files"] = files_out
+    data["measures"] = measures
+
+    # Debug data structure:
+    # print(json.dumps(data, indent=2))
+
+    print_overview_section(data)
+    print_files_section(data)
+
+
+if __name__ == '__main__':
+    sys.exit(main())