diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index 99d6485..13c4938 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -12,6 +12,7 @@ from dataclasses import dataclass from enum import Enum import jsondiff +import numpy as np import tabulate from colorama import Fore @@ -44,6 +45,8 @@ GPU_TIME_STDEV_RELATIVE_TAG = "nv/cold/time/gpu/stdev/relative" GPU_TIME_MEDIAN_TAG = "nv/cold/time/gpu/median" GPU_TIME_IR_TAG = "nv/cold/time/gpu/ir/absolute" GPU_TIME_IR_RELATIVE_TAG = "nv/cold/time/gpu/ir/relative" +SAMPLE_TIMES_TAG = "nv/json/bin:nv/cold/sample_times" +SAMPLE_FREQUENCIES_TAG = "nv/json/freqs-bin:nv/cold/sample_freqs" # These dataclasses are treated as parsed value objects. frozen=True prevents # accidental field reassignment but does not imply deep immutability. @@ -59,6 +62,8 @@ class GpuTimeSummary: median: float | None interquartile_range: float | None interquartile_range_relative: float | None + samples: np.ndarray | None = None + frequencies: np.ndarray | None = None @dataclass(frozen=True) @@ -205,23 +210,27 @@ def lookup_summary(summaries, tag): return next((summary for summary in summaries if summary["tag"] == tag), None) -def extract_summary_value(summary): +def extract_summary_data_value(summary, name, expected_type): summary_tag = summary.get("tag", "") for value_data in summary.get("data", []): - if value_data.get("name") != "value": + if value_data.get("name") != name: continue value_type = value_data.get("type") - if value_type != "float64": + if value_type != expected_type: raise ValueError( - f"summary {summary_tag!r} field 'value' has type " - f"{value_type!r}; expected 'float64'" + f"summary {summary_tag!r} field {name!r} has type " + f"{value_type!r}; expected {expected_type!r}" ) if "value" not in value_data: - raise ValueError(f"summary {summary_tag!r} field 'value' is missing value") + raise ValueError(f"summary {summary_tag!r} field {name!r} is missing value") return value_data["value"] - raise ValueError(f"summary {summary_tag!r} is missing field 'value'") + raise ValueError(f"summary {summary_tag!r} is missing field {name!r}") + + +def extract_summary_value(summary): + return extract_summary_data_value(summary, "value", "float64") def normalize_float_value(value, *, null_value=None): @@ -237,7 +246,92 @@ def extract_summary_float(summaries, tag, *, null_value=None): return normalize_float_value(extract_summary_value(summary), null_value=null_value) -def extract_gpu_time_summary(summaries): +def extract_binary_filename(summary): + value = extract_summary_data_value(summary, "filename", "string") + if not isinstance(value, str): + raise ValueError( + f"summary {summary.get('tag', '')!r} field 'filename' " + "value must be a string" + ) + return value + + +def extract_binary_size(summary): + value = extract_summary_data_value(summary, "size", "int64") + try: + return int(value) + except (TypeError, ValueError) as exc: + raise ValueError( + f"summary {summary.get('tag', '')!r} field 'size' " + f"value {value!r} is not an int64" + ) from exc + + +def extract_binary_meta(summaries, tag): + summary = lookup_summary(summaries, tag) + if summary is None: + return None, None + return extract_binary_size(summary), extract_binary_filename(summary) + + +def resolve_binary_filename(json_dir, binary_filename): + if os.path.isabs(binary_filename): + return binary_filename + + json_relative_filename = os.path.join(json_dir, binary_filename) + if os.path.exists(json_relative_filename): + return json_relative_filename + + parent_relative_filename = os.path.join(os.path.dirname(json_dir), binary_filename) + if os.path.exists(parent_relative_filename): + return parent_relative_filename + + if os.path.exists(binary_filename): + return binary_filename + + return json_relative_filename + + +def read_float32_binary(count, filename, json_dir): + if count is None or filename is None or json_dir is None: + return None + + filename = resolve_binary_filename(json_dir, filename) + try: + values = np.fromfile(filename, dtype=" int: args.no_color, reference_device_filter, compare_device_filter, + os.path.dirname(ref), + os.path.dirname(comp), ) except ValueError as exc: print(str(exc)) diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index c6d8c14..6bc754b 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -6,6 +6,7 @@ import sys import types from pathlib import Path +import numpy as np import pytest @@ -99,6 +100,16 @@ def make_summary(nvbench_compare, tag, value): } +def make_binary_summary(nvbench_compare, tag, filename, size): + return { + "tag": getattr(nvbench_compare, tag), + "data": [ + {"name": "filename", "type": "string", "value": filename}, + {"name": "size", "type": "int64", "value": str(size)}, + ], + } + + def make_benchmark(states, *, name="bench"): devices = [] for state in states: @@ -301,6 +312,64 @@ def test_compare_benches_skips_non_finite_centers(monkeypatch, nvbench_compare): assert nvbench_compare.unknown_count == 0 +def test_gpu_time_summary_loads_samples_and_frequencies(tmp_path, nvbench_compare): + samples_dir = tmp_path / "result.json-bin" + freqs_dir = tmp_path / "result.json-freqs-bin" + samples_dir.mkdir() + freqs_dir.mkdir() + samples_file = samples_dir / "0.bin" + freqs_file = freqs_dir / "0.bin" + + np.array([1.0, 2.0, 4.0], dtype="