From cb3ed117c0217f49157ebc356e122f1ba9eb4c7a Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk <21087696+oleksandr-pavlyk@users.noreply.github.com> Date: Thu, 2 Jul 2026 09:25:42 -0500 Subject: [PATCH] Resolve benchmark-relative jsonbin sidecar paths in nvbench-compare Thread the originating JSON path through bulk timing extraction so sidecar filenames can be resolved with knowledge of the JSON basename. This lets nvbench-compare handle --jsonbin output where the filename field may store the benchmark-process-relative path, such as data/result.json-bin/0.bin, instead of a path relative to the JSON file directory. Keep resolution anchored to the JSON location and avoid falling back to the caller working directory. Add regression coverage for this sidecar layout. This change helps nvbench_compare properly resolve location of sidecar binary files in the following CCCL-based workflow: ``` ./bin/cub.bench.bitonic_sort.warp_keys.base -d 1 --cold-warmup-runs 16 --min-samples 100 --stopping-criterion entropy --min-r2 0.88 --jsonbin ../../../perf_data/warp-bitonic-sort-run1.json ./bin/cub.bench.bitonic_sort.warp_keys.base -d 1 --cold-warmup-runs 16 --min-samples 100 --stopping-criterion entropy --min-r2 0.88 --jsonbin ../../../perf_data/warp-bitonic-sort-run2.json PYTHONPATH=./_deps/nvbench-src/python/scripts python ./_deps/nvbench-src/python/scripts/nvbench_compare.py ../../../perf_data/warp-bitonic-sort-run1.json ../../../perf_data/warp-bitonic-sort-run2.json ``` Before this change, script generated numerous RuntimeWarnigns alerting that absolute path can not be resolved and bulk data will be treated as unavailable --- python/scripts/nvbench_compare.py | 85 +++++++++++++++++++++++------ python/test/test_nvbench_compare.py | 29 ++++++++++ 2 files changed, 96 insertions(+), 18 deletions(-) diff --git a/python/scripts/nvbench_compare.py b/python/scripts/nvbench_compare.py index 054a564..39639f2 100644 --- a/python/scripts/nvbench_compare.py +++ b/python/scripts/nvbench_compare.py @@ -9,6 +9,7 @@ import argparse import math import os import pprint +import re import sys import warnings from collections import Counter @@ -467,12 +468,18 @@ class Float32BinarySource: filename: str json_dir: str description: str + json_path: str | None = None reader: Float32Reader = read_float32_file @cached_property def values(self) -> Float32Array | None: return read_float32_binary( - self.count, self.filename, self.json_dir, self.description, self.reader + self.count, + self.filename, + self.json_dir, + self.description, + self.reader, + self.json_path, ) def has_material_payload(self) -> bool: @@ -482,7 +489,7 @@ class Float32BinarySource: if self.reader is not read_float32_file: return True - filename = resolve_binary_filename(self.json_dir, self.filename) + filename = resolve_binary_filename(self.json_dir, self.filename, self.json_path) try: return os.path.getsize(filename) > 0 except OSError: @@ -954,7 +961,23 @@ def extract_binary_meta(summaries, tag): return extract_binary_size(summary), extract_binary_filename(summary) -def resolve_binary_filename(json_dir, binary_filename): +def resolve_legacy_jsonbin_filename(json_dir, binary_filename, json_path): + if json_path is None: + return None + + json_name = os.path.basename(json_path) + sidecar_names = {f"{json_name}-bin", f"{json_name}-freqs-bin"} + path_parts = re.split(r"[\\/]+", os.path.normpath(binary_filename)) + for index, part in enumerate(path_parts): + if part in sidecar_names: + candidate = os.path.join(json_dir, *path_parts[index:]) + if os.path.exists(candidate): + return candidate + + return None + + +def resolve_binary_filename(json_dir, binary_filename, json_path=None): if os.path.isabs(binary_filename): return binary_filename @@ -962,6 +985,12 @@ def resolve_binary_filename(json_dir, binary_filename): if os.path.exists(json_relative_filename): return json_relative_filename + legacy_jsonbin_filename = resolve_legacy_jsonbin_filename( + json_dir, binary_filename, json_path + ) + if legacy_jsonbin_filename is not None: + return legacy_jsonbin_filename + parent_relative_filename = os.path.join(os.path.dirname(json_dir), binary_filename) if os.path.exists(parent_relative_filename): return parent_relative_filename @@ -977,8 +1006,8 @@ def warn_unavailable_bulk_data(description, message): ) -def read_float32_binary(count, filename, json_dir, description, reader): - filename = resolve_binary_filename(json_dir, filename) +def read_float32_binary(count, filename, json_dir, description, reader, json_path=None): + filename = resolve_binary_filename(json_dir, filename, json_path) try: values = np.frombuffer(reader(filename), dtype=" str | None: if source is None: return None - return resolve_binary_filename(source.json_dir, source.filename) + return resolve_binary_filename(source.json_dir, source.filename, source.json_path) def get_bulk_source_count(source: Float32BinarySource | None) -> int | None: @@ -3025,19 +3066,27 @@ def compare_benches( ref_state, cmp_state ) if missing_summaries_decision is None: - cmp_gpu_time = extract_gpu_timing_data(cmp_summaries, cmp_json_dir) - ref_gpu_time = extract_gpu_timing_data(ref_summaries, ref_json_dir) + cmp_gpu_time = extract_gpu_timing_data( + cmp_summaries, cmp_json_dir, json_path=cmp_json_path + ) + ref_gpu_time = extract_gpu_timing_data( + ref_summaries, ref_json_dir, json_path=ref_json_path + ) comparison = compare_gpu_timings( ref_gpu_time, cmp_gpu_time, comparison_thresholds ) else: cmp_gpu_time = ( - extract_gpu_timing_data(cmp_summaries, cmp_json_dir) + extract_gpu_timing_data( + cmp_summaries, cmp_json_dir, json_path=cmp_json_path + ) if cmp_summaries else make_empty_gpu_timing_data() ) ref_gpu_time = ( - extract_gpu_timing_data(ref_summaries, ref_json_dir) + extract_gpu_timing_data( + ref_summaries, ref_json_dir, json_path=ref_json_path + ) if ref_summaries else make_empty_gpu_timing_data() ) diff --git a/python/test/test_nvbench_compare.py b/python/test/test_nvbench_compare.py index b47f53d..c3e8c58 100644 --- a/python/test/test_nvbench_compare.py +++ b/python/test/test_nvbench_compare.py @@ -931,6 +931,35 @@ def test_bulk_file_resolution_does_not_fall_back_to_cwd( assert timing.samples is None +def test_bulk_file_resolution_accepts_legacy_jsonbin_path(tmp_path, nvbench_compare): + json_dir = tmp_path / "data" + json_dir.mkdir() + json_path = json_dir / "result.json" + samples_dir = json_dir / "result.json-bin" + samples_dir.mkdir() + samples_file = samples_dir / "0.bin" + np.array([123.0], dtype="