From 2604547eebefea585fd9fef78895d2f220707d63 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Fri, 8 May 2026 15:51:09 -0500 Subject: [PATCH] Improve Python BenchResult parsing and container APIs Add arbitrary BenchResult metadata and explicit parse control, replacing the previous code/elapsed fields. Make BenchResult subscriptable by subbenchmark name and make SubBenchResult list-like over its states. Extend SubBenchState parsing to expose summaries by tag, read paired sample frequency data, return None for unavailable sample/frequency files, and validate matching sample/frequency lengths. Harden parsing for NVBench JSON output with no-axis benchmarks, null axis_values, skipped states with null summaries, float axis input_string lookups, and recorded sidecar binary paths. Expand BenchResult tests to cover metadata, parse=False, sequence-style access, frequency-aware centers, missing binary data, skipped states, and mismatched sample/frequency counts. Example usage: ``` import array, numpy as np, cuda.bench r = cuda.bench.BenchResult("perf_data/axes_run1.json") r["copy_sweep_grid_shape"].centers_with_frequencies( lambda t, f: np.median(np.asarray(t)*np.asarray(f))) ``` --- python/cuda/bench/__init__.pyi | 57 +++- python/cuda/bench/_bench_result.py | 218 +++++++++++--- python/test/test_bench_result.py | 463 ++++++++++++++++++++++++++++- 3 files changed, 671 insertions(+), 67 deletions(-) diff --git a/python/cuda/bench/__init__.pyi b/python/cuda/bench/__init__.pyi index c498f27..db723f0 100644 --- a/python/cuda/bench/__init__.pyi +++ b/python/cuda/bench/__init__.pyi @@ -26,8 +26,21 @@ # with definitions given here. from array import array -from collections.abc import Callable, Sequence -from typing import Optional, Self, SupportsFloat, SupportsInt, Union +from collections.abc import Callable, Iterator, Sequence +from typing import ( + Any, + Optional, + Self, + SupportsFloat, + SupportsInt, + TypeVar, + Union, + overload, +) + +ResultT = TypeVar("ResultT") +_SummaryValue = int | float | str +_SummaryData = _SummaryValue | dict[str, _SummaryValue] class CudaStream: def __cuda_stream__(self) -> tuple[int, int]: ... @@ -119,25 +132,47 @@ def run_all_benchmarks(argv: Sequence[str]) -> None: ... class NVBenchRuntimeError(RuntimeError): ... class SubBenchState: - samples: array + state_name: str + summaries: dict[str, _SummaryData] + samples: array | None + frequencies: array | None bw: float | None point: dict[str, str] def name(self) -> str: ... - def center(self, estimator: Callable[[array], SupportsFloat]) -> SupportsFloat: ... + def center(self, estimator: Callable[[array], ResultT]) -> ResultT | None: ... + def center_with_frequencies( + self, estimator: Callable[[array, array], ResultT] + ) -> ResultT | None: ... class SubBenchResult: states: list[SubBenchState] + def __len__(self) -> int: ... + @overload + def __getitem__(self, state_index: int) -> SubBenchState: ... + @overload + def __getitem__(self, state_index: slice) -> list[SubBenchState]: ... + def __iter__(self) -> Iterator[SubBenchState]: ... def centers( - self, estimator: Callable[[array], SupportsFloat] - ) -> dict[str, SupportsFloat]: ... + self, estimator: Callable[[array], ResultT] + ) -> dict[str, ResultT | None]: ... + def centers_with_frequencies( + self, estimator: Callable[[array, array], ResultT] + ) -> dict[str, ResultT | None]: ... class BenchResult: - code: int - elapsed: float + metadata: Any subbenches: dict[str, SubBenchResult] def __init__( - self, json_fn: str, *, code: int = 0, elapsed: float = 0.0 + self, + json_fn: str | None = None, + *, + metadata: Any = None, + parse: bool = True, ) -> None: ... + def __getitem__(self, subbench_name: str) -> SubBenchResult: ... def centers( - self, estimator: Callable[[array], SupportsFloat] - ) -> dict[str, dict[str, SupportsFloat]]: ... + self, estimator: Callable[[array], ResultT] + ) -> dict[str, dict[str, ResultT | None]]: ... + def centers_with_frequencies( + self, estimator: Callable[[array, array], ResultT] + ) -> dict[str, dict[str, ResultT | None]]: ... diff --git a/python/cuda/bench/_bench_result.py b/python/cuda/bench/_bench_result.py index 9114371..a5de2f1 100644 --- a/python/cuda/bench/_bench_result.py +++ b/python/cuda/bench/_bench_result.py @@ -18,10 +18,15 @@ import array import json import os import sys -from typing import Callable, SupportsFloat +from collections.abc import Iterator +from typing import Any, Callable, TypeVar __all__ = ["BenchResult", "SubBenchResult", "SubBenchState"] +ResultT = TypeVar("ResultT") +_SummaryValue = int | float | str +_SummaryData = _SummaryValue | dict[str, _SummaryValue] + def read_json(filename: str) -> dict: with open(filename, "r", encoding="utf-8") as f: @@ -50,13 +55,41 @@ def extract_bw(summary: dict) -> float: return float(value_data["value"]) -def parse_samples_meta(state: dict) -> tuple[int | None, str | None]: +def parse_summary_value(value_data: dict) -> _SummaryValue: + value_type = value_data["type"] + value = value_data["value"] + if value_type == "int64": + return int(value) + if value_type == "float64": + return float(value) + if value_type == "string": + return value + raise ValueError(f"unsupported summary value type: {value_type}") + + +def parse_summary_data(summary: dict) -> _SummaryData: + summary_values = { + value_data["name"]: parse_summary_value(value_data) + for value_data in summary["data"] + } + if len(summary_values) == 1 and "value" in summary_values: + return summary_values["value"] + return summary_values + + +def parse_summaries(state: dict) -> dict[str, _SummaryData]: + return { + summary["tag"]: parse_summary_data(summary) for summary in state["summaries"] + } + + +def parse_binary_meta(state: dict, tag: str) -> tuple[int | None, str | None]: summaries = state["summaries"] if not summaries: return None, None summary = next( - filter(lambda s: s["tag"] == "nv/json/bin:nv/cold/sample_times", summaries), + filter(lambda s: s["tag"] == tag, summaries), None, ) if not summary: @@ -67,40 +100,72 @@ def parse_samples_meta(state: dict) -> tuple[int | None, str | None]: return sample_count, sample_filename -def resolve_sample_filename(json_dir: str, samples_filename: str) -> str: - if os.path.isabs(samples_filename): - return samples_filename - return os.path.join(json_dir, samples_filename) +def parse_samples_meta(state: dict) -> tuple[int | None, str | None]: + return parse_binary_meta(state, "nv/json/bin:nv/cold/sample_times") -def parse_samples(state: dict, json_dir: str) -> array.array: - """Return the state's sample times as an array of float32 values.""" - sample_count, samples_filename = parse_samples_meta(state) - if sample_count is None or samples_filename is None: - return array.array("f", []) +def parse_frequencies_meta(state: dict) -> tuple[int | None, str | None]: + return parse_binary_meta(state, "nv/json/freqs-bin:nv/cold/sample_freqs") - samples = array.array("f") - if samples.itemsize != 4: + +def resolve_binary_filename(json_dir: str, binary_filename: str) -> str: + if os.path.isabs(binary_filename): + return binary_filename + + json_relative_filename = os.path.join(json_dir, binary_filename) + if os.path.exists(json_relative_filename): + return json_relative_filename + + parent_relative_filename = os.path.join(os.path.dirname(json_dir), binary_filename) + if os.path.exists(parent_relative_filename): + return parent_relative_filename + + if os.path.exists(binary_filename): + return binary_filename + + return json_relative_filename + + +def parse_float32_binary( + count: int | None, filename: str | None, json_dir: str +) -> array.array | None: + if count is None or filename is None: + return None + + values = array.array("f") + if values.itemsize != 4: raise RuntimeError("array('f') is not a 32-bit float on this platform") - samples_filename = resolve_sample_filename(json_dir, samples_filename) - with open(samples_filename, "rb") as f: - size = os.fstat(f.fileno()).st_size - if size % samples.itemsize: - raise ValueError("file size is not a multiple of float size") + filename = resolve_binary_filename(json_dir, filename) + try: + with open(filename, "rb") as f: + size = os.fstat(f.fileno()).st_size + if size % values.itemsize: + raise ValueError("file size is not a multiple of float size") - samples.fromfile(f, size // samples.itemsize) + values.fromfile(f, size // values.itemsize) + except FileNotFoundError: + return None # Match np.fromfile(fn, " array.array | None: + """Return the state's sample times, or None if sample data is unavailable.""" + sample_count, samples_filename = parse_samples_meta(state) + return parse_float32_binary(sample_count, samples_filename, json_dir) + + +def parse_frequencies(state: dict, json_dir: str) -> array.array | None: + """Return the state's sample frequencies, or None if data is unavailable.""" + frequency_count, frequencies_filename = parse_frequencies_meta(state) + return parse_float32_binary(frequency_count, frequencies_filename, json_dir) def parse_bw(state: dict) -> float | None: @@ -125,11 +190,23 @@ def get_axis_name(axis: dict) -> str: class SubBenchState: def __init__(self, state: dict, axes_names: dict, axes_values: dict, json_dir: str): + self.state_name = state["name"] + self.summaries = parse_summaries(state) self.samples = parse_samples(state, json_dir) + self.frequencies = parse_frequencies(state, json_dir) + if ( + self.samples is not None + and self.frequencies is not None + and len(self.samples) != len(self.frequencies) + ): + raise ValueError( + f"sample count ({len(self.samples)}) does not match " + f"frequency count ({len(self.frequencies)})" + ) self.bw = parse_bw(state) self.point = {} - for axis in state["axis_values"]: + for axis in state["axis_values"] or []: axis_name = axis["name"] name = axes_names[axis_name] value = axes_values[axis_name][axis["value"]] @@ -139,27 +216,36 @@ class SubBenchState: return str(self.__dict__) def name(self) -> str: + if not self.point: + return self.state_name return " ".join(f"{k}={v}" for k, v in self.point.items()) - def center( - self, estimator: Callable[[array.array], SupportsFloat] - ) -> SupportsFloat: + def center(self, estimator: Callable[[array.array], ResultT]) -> ResultT | None: + if self.samples is None: + return None return estimator(self.samples) + def center_with_frequencies( + self, estimator: Callable[[array.array, array.array], ResultT] + ) -> ResultT | None: + if self.samples is None or self.frequencies is None: + return None + return estimator(self.samples, self.frequencies) + class SubBenchResult: def __init__(self, bench: dict, json_dir: str): axes_names = {} axes_values = {} - for axis in bench["axes"]: + for axis in bench["axes"] or []: short_name = axis["name"] full_name = get_axis_name(axis) this_axis_values = {} for value in axis["values"]: + input_string = value["input_string"] + this_axis_values[input_string] = input_string if "value" in value: - this_axis_values[str(value["value"])] = value["input_string"] - else: - this_axis_values[value["input_string"]] = value["input_string"] + this_axis_values[str(value["value"])] = input_string axes_names[short_name] = full_name axes_values[short_name] = this_axis_values @@ -173,37 +259,73 @@ class SubBenchResult: def __repr__(self) -> str: return str(self.__dict__) + def __len__(self) -> int: + return len(self.states) + + def __getitem__( + self, state_index: int | slice + ) -> SubBenchState | list[SubBenchState]: + return self.states[state_index] + + def __iter__(self) -> Iterator[SubBenchState]: + return iter(self.states) + def centers( - self, estimator: Callable[[array.array], SupportsFloat] - ) -> dict[str, SupportsFloat]: + self, estimator: Callable[[array.array], ResultT] + ) -> dict[str, ResultT | None]: result = {} for state in self.states: result[state.name()] = state.center(estimator) return result + def centers_with_frequencies( + self, estimator: Callable[[array.array, array.array], ResultT] + ) -> dict[str, ResultT | None]: + result = {} + for state in self.states: + result[state.name()] = state.center_with_frequencies(estimator) + return result + class BenchResult: """Parsed result data from an NVBench JSON output file.""" - def __init__(self, json_fn: str, *, code: int = 0, elapsed: float = 0.0): - self.code = code - self.elapsed = elapsed + def __init__( + self, + json_fn: str | None = None, + *, + metadata: Any = None, + parse: bool = True, + ): + self.metadata = metadata self.subbenches: dict[str, SubBenchResult] = {} - if json_fn: - if code == 0: - json_dir = os.path.dirname(os.path.abspath(json_fn)) - for bench in read_json(json_fn)["benchmarks"]: - bench_name: str = bench["name"] - self.subbenches[bench_name] = SubBenchResult(bench, json_dir) + if json_fn and parse: + json_dir = os.path.dirname(os.path.abspath(json_fn)) + for bench in read_json(json_fn)["benchmarks"]: + bench_name: str = bench["name"] + self.subbenches[bench_name] = SubBenchResult(bench, json_dir) def __repr__(self) -> str: return str(self.__dict__) + def __getitem__(self, subbench_name: str) -> SubBenchResult: + return self.subbenches[subbench_name] + def centers( - self, estimator: Callable[[array.array], SupportsFloat] - ) -> dict[str, dict[str, SupportsFloat]]: + self, estimator: Callable[[array.array], ResultT] + ) -> dict[str, dict[str, ResultT | None]]: result = {} for subbench in self.subbenches: result[subbench] = self.subbenches[subbench].centers(estimator) return result + + def centers_with_frequencies( + self, estimator: Callable[[array.array, array.array], ResultT] + ) -> dict[str, dict[str, ResultT | None]]: + result = {} + for subbench in self.subbenches: + result[subbench] = self.subbenches[subbench].centers_with_frequencies( + estimator + ) + return result diff --git a/python/test/test_bench_result.py b/python/test/test_bench_result.py index ee03101..7113d08 100644 --- a/python/test/test_bench_result.py +++ b/python/test/test_bench_result.py @@ -1,5 +1,6 @@ import json import struct +from dataclasses import dataclass import cuda.bench as bench import pytest @@ -9,6 +10,9 @@ def test_bench_result_reads_jsonbin_relative_to_json_path(tmp_path): bin_dir = tmp_path / "result.json-bin" bin_dir.mkdir() (bin_dir / "0.bin").write_bytes(struct.pack("<3f", 1.0, 2.0, 4.0)) + freq_bin_dir = tmp_path / "result.json-freqs-bin" + freq_bin_dir.mkdir() + (freq_bin_dir / "0.bin").write_bytes(struct.pack("<3f", 100.0, 200.0, 400.0)) json_fn = tmp_path / "result.json" json_fn.write_text( @@ -67,6 +71,21 @@ def test_bench_result_reads_jsonbin_relative_to_json_path(tmp_path): } ], }, + { + "tag": "nv/json/freqs-bin:nv/cold/sample_freqs", + "data": [ + { + "name": "filename", + "type": "string", + "value": "result.json-freqs-bin/0.bin", + }, + { + "name": "size", + "type": "int64", + "value": "3", + }, + ], + }, ], "is_skipped": False, } @@ -78,24 +97,452 @@ def test_bench_result_reads_jsonbin_relative_to_json_path(tmp_path): encoding="utf-8", ) + metadata = {"returncode": 0, "elapsed_seconds": 0.25} default_result = bench.BenchResult(str(json_fn)) - result = bench.BenchResult(str(json_fn), elapsed=0.25) + result = bench.BenchResult(str(json_fn), metadata=metadata) assert bench.BenchResult.__module__ == bench.__name__ - assert default_result.code == 0 - assert default_result.elapsed == 0.0 - assert result.code == 0 - assert result.elapsed == 0.25 - state = result.subbenches["copy"].states[0] + assert default_result.metadata is None + assert result.metadata is metadata + subbench = result["copy"] + state = subbench[0] + assert len(subbench) == 1 + assert subbench[-1] is state + assert subbench[:] == subbench.states + assert list(subbench) == subbench.states + with pytest.raises(IndexError): + subbench[1] assert state.name() == "BlockSize[pow2]=8" assert state.bw == 0.75 + assert state.summaries["nv/cold/bw/global/utilization"] == pytest.approx(0.75) + assert state.summaries["nv/json/bin:nv/cold/sample_times"] == { + "filename": "result.json-bin/0.bin", + "size": 3, + } + assert state.summaries["nv/json/freqs-bin:nv/cold/sample_freqs"] == { + "filename": "result.json-freqs-bin/0.bin", + "size": 3, + } + assert state.samples is not None assert list(state.samples) == pytest.approx([1.0, 2.0, 4.0]) + assert state.frequencies is not None + assert list(state.frequencies) == pytest.approx([100.0, 200.0, 400.0]) centers = result.centers(lambda samples: sum(samples) / len(samples)) assert set(centers) == {"copy"} assert set(centers["copy"]) == {"BlockSize[pow2]=8"} assert centers["copy"]["BlockSize[pow2]=8"] == pytest.approx(7.0 / 3.0) + def weighted_mean(samples, frequencies): + return sum( + sample * frequency for sample, frequency in zip(samples, frequencies) + ) / sum(frequencies) -def test_bench_result_code_and_elapsed_are_keyword_only(): + weighted_centers = result.centers_with_frequencies(weighted_mean) + assert set(weighted_centers) == {"copy"} + assert set(weighted_centers["copy"]) == {"BlockSize[pow2]=8"} + assert weighted_centers["copy"]["BlockSize[pow2]=8"] == pytest.approx(3.0) + assert subbench is result.subbenches["copy"] + assert subbench.centers_with_frequencies(weighted_mean) == weighted_centers["copy"] + with pytest.raises(KeyError): + result["missing"] + + +def test_bench_result_metadata_and_parse_are_keyword_only(): with pytest.raises(TypeError): - bench.BenchResult("", 0, 0.0) + bench.BenchResult("", None) + with pytest.raises(TypeError): + bench.BenchResult("", None, False) + + +def test_bench_result_parse_false_does_not_read_json(tmp_path): + @dataclass + class RunMetadata: + returncode: int + elapsed_seconds: float + + metadata = RunMetadata(returncode=1, elapsed_seconds=0.25) + missing_json = tmp_path / "missing.json" + + result = bench.BenchResult(str(missing_json), metadata=metadata, parse=False) + + assert result.metadata is metadata + assert result.subbenches == {} + + with pytest.raises(FileNotFoundError): + bench.BenchResult(str(missing_json), metadata=metadata) + + +def test_bench_result_accepts_no_axis_benchmark_with_recorded_binary_path( + tmp_path, monkeypatch +): + data_dir = tmp_path / "temp_data" + data_dir.mkdir() + bin_dir = data_dir / "axes_run1.json-bin" + bin_dir.mkdir() + (bin_dir / "0.bin").write_bytes(struct.pack("<2f", 1.0, 4.0)) + freq_bin_dir = data_dir / "axes_run1.json-freqs-bin" + freq_bin_dir.mkdir() + (freq_bin_dir / "0.bin").write_bytes(struct.pack("<2f", 100.0, 400.0)) + + json_fn = data_dir / "axes_run1.json" + json_fn.write_text( + json.dumps( + { + "benchmarks": [ + { + "name": "simple", + "axes": None, + "states": [ + { + "name": "Device=0", + "axis_values": None, + "summaries": [ + { + "tag": "nv/json/bin:nv/cold/sample_times", + "data": [ + { + "name": "filename", + "type": "string", + "value": "temp_data/axes_run1.json-bin/0.bin", + }, + { + "name": "size", + "type": "int64", + "value": "2", + }, + ], + }, + { + "tag": "nv/json/freqs-bin:nv/cold/sample_freqs", + "data": [ + { + "name": "filename", + "type": "string", + "value": "temp_data/axes_run1.json-freqs-bin/0.bin", + }, + { + "name": "size", + "type": "int64", + "value": "2", + }, + ], + }, + ], + "is_skipped": False, + } + ], + } + ] + } + ), + encoding="utf-8", + ) + + monkeypatch.chdir(tmp_path) + + result = bench.BenchResult("temp_data/axes_run1.json") + + state = result.subbenches["simple"].states[0] + assert state.name() == "Device=0" + assert state.point == {} + assert state.samples is not None + assert list(state.samples) == pytest.approx([1.0, 4.0]) + assert state.frequencies is not None + assert list(state.frequencies) == pytest.approx([100.0, 400.0]) + + +def test_bench_result_accepts_axis_value_input_string(): + result = bench.SubBenchResult( + { + "name": "single_float64_axis", + "axes": [ + { + "name": "Duration", + "type": "float64", + "flags": "", + "values": [ + { + "input_string": "0", + "description": "", + "value": 0.0, + } + ], + } + ], + "states": [ + { + "name": "Device=0 Duration=0", + "axis_values": [ + { + "name": "Duration", + "type": "float64", + "value": "0", + } + ], + "summaries": [], + "is_skipped": False, + } + ], + }, + "", + ) + + state = result.states[0] + assert state.name() == "Duration=0" + assert state.point == {"Duration": "0"} + + +def test_bench_result_ignores_skipped_state_with_no_summaries(): + result = bench.SubBenchResult( + { + "name": "copy_sweep_grid_shape", + "axes": [ + { + "name": "BlockSize", + "type": "int64", + "flags": "pow2", + "values": [ + { + "input_string": "6", + "description": "2^6 = 64", + "value": 64, + }, + { + "input_string": "8", + "description": "2^8 = 256", + "value": 256, + }, + ], + } + ], + "states": [ + { + "name": "Device=0 BlockSize=2^8", + "axis_values": [ + { + "name": "BlockSize", + "type": "int64", + "value": "256", + } + ], + "summaries": None, + "is_skipped": True, + }, + { + "name": "Device=0 BlockSize=2^6", + "axis_values": [ + { + "name": "BlockSize", + "type": "int64", + "value": "64", + } + ], + "summaries": [], + "is_skipped": False, + }, + ], + }, + "", + ) + + assert len(result.states) == 1 + assert result.states[0].name() == "BlockSize[pow2]=6" + + +def test_bench_result_uses_none_for_unavailable_samples(tmp_path): + json_fn = tmp_path / "result.json" + json_fn.write_text( + json.dumps( + { + "benchmarks": [ + { + "name": "copy", + "axes": [ + { + "name": "BlockSize", + "type": "int64", + "flags": "pow2", + "values": [ + { + "input_string": "8", + "description": "2^8 = 256", + "value": 256, + }, + { + "input_string": "9", + "description": "2^9 = 512", + "value": 512, + }, + ], + } + ], + "states": [ + { + "name": "Device=0 BlockSize=2^8", + "axis_values": [ + { + "name": "BlockSize", + "type": "int64", + "value": "256", + } + ], + "summaries": [], + "is_skipped": False, + }, + { + "name": "Device=0 BlockSize=2^9", + "axis_values": [ + { + "name": "BlockSize", + "type": "int64", + "value": "512", + } + ], + "summaries": [ + { + "tag": "nv/json/bin:nv/cold/sample_times", + "data": [ + { + "name": "filename", + "type": "string", + "value": "result.json-bin/missing.bin", + }, + { + "name": "size", + "type": "int64", + "value": "3", + }, + ], + }, + { + "tag": "nv/json/freqs-bin:nv/cold/sample_freqs", + "data": [ + { + "name": "filename", + "type": "string", + "value": "result.json-freqs-bin/missing.bin", + }, + { + "name": "size", + "type": "int64", + "value": "3", + }, + ], + }, + ], + "is_skipped": False, + }, + ], + } + ] + } + ), + encoding="utf-8", + ) + + result = bench.BenchResult(str(json_fn)) + + states = result.subbenches["copy"].states + assert states[0].samples is None + assert states[1].samples is None + assert states[0].frequencies is None + assert states[1].frequencies is None + assert result.centers(lambda samples: pytest.fail("estimator should not run")) == { + "copy": { + "BlockSize[pow2]=8": None, + "BlockSize[pow2]=9": None, + } + } + assert result.centers_with_frequencies( + lambda samples, frequencies: pytest.fail("estimator should not run") + ) == { + "copy": { + "BlockSize[pow2]=8": None, + "BlockSize[pow2]=9": None, + } + } + + +def test_bench_result_rejects_mismatched_sample_and_frequency_counts(tmp_path): + bin_dir = tmp_path / "result.json-bin" + bin_dir.mkdir() + (bin_dir / "0.bin").write_bytes(struct.pack("<3f", 1.0, 2.0, 4.0)) + freq_bin_dir = tmp_path / "result.json-freqs-bin" + freq_bin_dir.mkdir() + (freq_bin_dir / "0.bin").write_bytes(struct.pack("<2f", 100.0, 200.0)) + + json_fn = tmp_path / "result.json" + json_fn.write_text( + json.dumps( + { + "benchmarks": [ + { + "name": "copy", + "axes": [ + { + "name": "BlockSize", + "type": "int64", + "flags": "pow2", + "values": [ + { + "input_string": "8", + "description": "2^8 = 256", + "value": 256, + } + ], + } + ], + "states": [ + { + "name": "Device=0 BlockSize=2^8", + "axis_values": [ + { + "name": "BlockSize", + "type": "int64", + "value": "256", + } + ], + "summaries": [ + { + "tag": "nv/json/bin:nv/cold/sample_times", + "data": [ + { + "name": "filename", + "type": "string", + "value": "result.json-bin/0.bin", + }, + { + "name": "size", + "type": "int64", + "value": "3", + }, + ], + }, + { + "tag": "nv/json/freqs-bin:nv/cold/sample_freqs", + "data": [ + { + "name": "filename", + "type": "string", + "value": "result.json-freqs-bin/0.bin", + }, + { + "name": "size", + "type": "int64", + "value": "2", + }, + ], + }, + ], + "is_skipped": False, + } + ], + } + ] + } + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="sample count .* frequency count"): + bench.BenchResult(str(json_fn))