NVIDIA/nvbench
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvbench.git synced 2026-07-12 18:17:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Store jsonbin sidecar paths relative to JSON output file location (#405)

Browse Source 
* Store jsonbin sidecar paths relative to JSON output

Keep --jsonbin sidecar files written next to the JSON output file, but record
their filenames relative to the JSON file directory. This makes generated JSON
and its sidecar directories relocatable as a unit and avoids embedding the
benchmark process working-directory relationship into the JSON payload.

Add a focused json_printer test that verifies sample-time and sample-frequency
sidecars are written beside the JSON file while summary filename fields contain
JSON-relative paths.

* Factor out jsonbin sidecar writing helper

Extract the duplicated sample-times and sample-frequencies jsonbin
sidecar write paths into a shared helper. This keeps directory creation,
file writing, warning logging, summary metadata, and write-duration
logging in one place for both sidecar types.

* Test jsonbin sidecar write failure handling

Cover the case where the expected sidecar directory path collides with a
regular file. Verify that json_printer keeps descriptive summary metadata
but omits filename and size when the sidecar file was not written.

Register printer and verify output.

* Clean up jsonbin sidecar logging

Factor repeated optional benchmark-printer logging in write_jsonbin_sidecar
into a local helper used by both warning and success paths.

Also tidy the jsonbin summary names for sample time and frequency sidecar
files.
This commit is contained in:
Oleksandr Pavlyk
2026-07-07 18:11:30 -05:00
committed by GitHub
parent d483a5bc73
commit c4acabdfaf
3 changed files with 313 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
nvbench/json_printer.cu
View File

@@ -42,6 +42,7 @@
#include <ostream>
#include <stdexcept>
#include <string>
#include <system_error>
#include <type_traits>
#include <utility>
#include <vector>
@@ -115,6 +116,66 @@ void write_named_values(JsonNode &node, const nvbench::named_values &values)
// see: https://github.com/NVIDIA/nvbench/issues/255
static constexpr std::size_t preferred_buffer_nbytes = 4096;
struct jsonbin_directory_paths
{
// Files are written next to the JSON output, but summaries record paths
// relative to the JSON file's directory so the output remains relocatable.
fs::path write_path;
fs::path summary_path;
};
struct jsonbin_sidecar_metadata
{
const char *directory_suffix;
const char *summary_tag_prefix;
const char *summary_name;
const char *summary_hint;
const char *summary_description;
};
jsonbin_directory_paths make_jsonbin_directory_paths(const std::string &stream_name,
const std::string &suffix)
{
const fs::path stream_path{stream_name};
const fs::path sidecar_dir{stream_path.filename().string() + suffix};
const auto stream_parent = stream_path.parent_path();
return {stream_parent.empty() ? sidecar_dir : stream_parent / sidecar_dir, sidecar_dir};
}
void ensure_output_directory(const fs::path &dir)
{
std::error_code create_ec;
fs::create_directories(dir, create_ec);
std::error_code status_ec;
const auto status = fs::status(dir, status_ec);
if (status_ec)
{
NVBENCH_THROW(std::runtime_error,
"Failed to inspect result directory '{}': {}.",
dir.string(),
status_ec.message());
}
if (fs::is_directory(status))
{
return;
}
if (fs::exists(status))
{
NVBENCH_THROW(std::runtime_error, "'{}' exists and is not a directory.", dir.string());
}
if (create_ec)
{
NVBENCH_THROW(std::runtime_error,
"Failed to create result directory '{}': {}.",
dir.string(),
create_ec.message());
}
NVBENCH_THROW(std::runtime_error, "Failed to create result directory '{}'.", dir.string());
}
template <std::size_t N, std::size_t... Is>
void swap_bytes_impl(char *p, std::index_sequence<Is...>)
{
@@ -186,6 +247,81 @@ void write_out_values_as_float64(std::ofstream &out, const std::vector<nvbench::
write_out_values_as<nvbench::float64_t, buffer_nbytes>(out, data);
}
void write_jsonbin_sidecar(nvbench::state &state,
const std::string &stream_name,
std::size_t &file_count,
const std::string &tag,
const std::string &hint,
const std::vector<nvbench::float64_t> &data,
const jsonbin_sidecar_metadata &metadata)
{
nvbench::cpu_timer timer;
timer.start();
auto paths = make_jsonbin_directory_paths(stream_name, metadata.directory_suffix);
const auto file_id = file_count++;
const fs::path filename{fmt::format("{:d}.bin", file_id)};
paths.write_path /= filename;
paths.summary_path /= filename;
bool write_succeeded = false;
auto log_if_present = [&state](nvbench::log_level level, const std::string &msg) {
if (auto printer_ptr = state.get_benchmark().get_printer())
{
printer_ptr->log(level, msg);
}
};
try
{
ensure_output_directory(paths.write_path.parent_path());
std::ofstream out;
out.exceptions(out.exceptions() | std::ios::failbit | std::ios::badbit);
out.open(paths.write_path, std::ios::binary | std::ios::out);
write_out_values_as_float32(out, data);
out.close();
write_succeeded = true;
}
catch (std::exception &e)
{
{
// Best-effort cleanup of any partial output. Preserve the original
// write failure as the diagnostic if removal also fails.
std::error_code remove_ec;
fs::remove(paths.write_path, remove_ec);
}
log_if_present(nvbench::log_level::warn,
fmt::format("Error writing {} ({}) to {}: {}",
tag,
hint,
paths.write_path.string(),
e.what()));
} // end catch
auto &summ = state.add_summary(fmt::format("{}:{}", metadata.summary_tag_prefix, tag));
summ.set_string("name", metadata.summary_name);
summ.set_string("hint", metadata.summary_hint);
summ.set_string("description", metadata.summary_description);
if (write_succeeded)
{
summ.set_string("filename", paths.summary_path.string());
summ.set_int64("size", static_cast<nvbench::int64_t>(data.size()));
}
summ.set_string("hide", "Not needed in table.");
timer.stop();
if (write_succeeded)
{
log_if_present(nvbench::log_level::info,
fmt::format("Wrote '{}' in {:>6.3f}ms",
paths.write_path.string(),
timer.get_duration() * 1000));
}
}
} // end namespace
namespace nvbench
@@ -222,122 +358,26 @@ void json_printer::do_process_bulk_data_float64(state &state,
if (hint == "sample_times")
{
nvbench::cpu_timer timer;
timer.start();
constexpr jsonbin_sidecar_metadata metadata{"-bin",
"nv/json/bin",
"Sample Times File",
"file/sample_times",
"Binary file containing sample times as "
"little-endian float32."};
fs::path result_path{m_stream_name + "-bin/"};
try
{
if (!fs::exists(result_path))
{
if (!fs::create_directory(result_path))
{
NVBENCH_THROW(std::runtime_error, "{}", "Failed to create result directory '{}'.");
}
}
else if (!fs::is_directory(result_path))
{
NVBENCH_THROW(std::runtime_error, "{}", "'{}' exists and is not a directory.");
}
const auto file_id = m_num_jsonbin_files++;
result_path /= fmt::format("{:d}.bin", file_id);
std::ofstream out;
out.exceptions(out.exceptions() | std::ios::failbit | std::ios::badbit);
out.open(result_path, std::ios::binary | std::ios::out);
write_out_values_as_float32(out, data);
}
catch (std::exception &e)
{
if (auto printer_ptr = state.get_benchmark().get_printer())
{
auto &printer = *printer_ptr;
printer.log(
nvbench::log_level::warn,
fmt::format("Error writing {} ({}) to {}: {}", tag, hint, result_path.string(), e.what()));
}
} // end catch
auto &summ = state.add_summary(fmt::format("nv/json/bin:{}", tag));
summ.set_string("name", "Samples Times File");
summ.set_string("hint", "file/sample_times");
summ.set_string("description",
"Binary file containing sample times as little-endian "
"float32.");
summ.set_string("filename", result_path.string());
summ.set_int64("size", static_cast<nvbench::int64_t>(data.size()));
summ.set_string("hide", "Not needed in table.");
timer.stop();
if (auto printer_ptr = state.get_benchmark().get_printer())
{
auto &printer = *printer_ptr;
printer.log(
nvbench::log_level::info,
fmt::format("Wrote '{}' in {:>6.3f}ms", result_path.string(), timer.get_duration() * 1000));
}
write_jsonbin_sidecar(state, m_stream_name, m_num_jsonbin_files, tag, hint, data, metadata);
} // end hint == sample_times
if (hint == "sample_freqs")
{
nvbench::cpu_timer timer;
timer.start();
constexpr jsonbin_sidecar_metadata metadata{"-freqs-bin",
"nv/json/freqs-bin",
"Sample Frequencies File",
"file/sample_freqs",
"Binary file containing sample frequencies as "
"little-endian float32."};
fs::path result_path{m_stream_name + "-freqs-bin/"};
try
{
if (!fs::exists(result_path))
{
if (!fs::create_directory(result_path))
{
NVBENCH_THROW(std::runtime_error, "{}", "Failed to create result directory '{}'.");
}
}
else if (!fs::is_directory(result_path))
{
NVBENCH_THROW(std::runtime_error, "{}", "'{}' exists and is not a directory.");
}
const auto file_id = m_num_jsonbin_freq_files++;
result_path /= fmt::format("{:d}.bin", file_id);
std::ofstream out;
out.exceptions(out.exceptions() | std::ios::failbit | std::ios::badbit);
out.open(result_path, std::ios::binary | std::ios::out);
write_out_values_as_float32(out, data);
}
catch (std::exception &e)
{
if (auto printer_ptr = state.get_benchmark().get_printer())
{
auto &printer = *printer_ptr;
printer.log(
nvbench::log_level::warn,
fmt::format("Error writing {} ({}) to {}: {}", tag, hint, result_path.string(), e.what()));
}
} // end catch
auto &summ = state.add_summary(fmt::format("nv/json/freqs-bin:{}", tag));
summ.set_string("name", "Samples Frequencies File");
summ.set_string("hint", "file/sample_freqs");
summ.set_string("description",
"Binary file containing sample frequencies as little-endian "
"float32.");
summ.set_string("filename", result_path.string());
summ.set_int64("size", static_cast<nvbench::int64_t>(data.size()));
summ.set_string("hide", "Not needed in table.");
timer.stop();
if (auto printer_ptr = state.get_benchmark().get_printer())
{
auto &printer = *printer_ptr;
printer.log(
nvbench::log_level::info,
fmt::format("Wrote '{}' in {:>6.3f}ms", result_path.string(), timer.get_duration() * 1000));
}
write_jsonbin_sidecar(state, m_stream_name, m_num_jsonbin_freq_files, tag, hint, data, metadata);
} // end hint == sample_freqs
}

1
testing/CMakeLists.txt
View File

@@ -16,6 +16,7 @@ set(test_srcs
exception_safety.cu
float64_axis.cu
int64_axis.cu
json_printer.cu
measure_timeout_warnings.cu
named_values.cu
option_parser.cu

162
testing/json_printer.cu Normal file
View File

@@ -0,0 +1,162 @@
/*
* Copyright 2026 NVIDIA Corporation
*
* Licensed under the Apache License, Version 2.0 with the LLVM exception
* (the "License"); you may not use this file except in compliance with
* the License.
*
* You may obtain a copy of the License at
*
* http://llvm.org/foundation/relicensing/LICENSE.txt
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <nvbench/benchmark.cuh>
#include <nvbench/callable.cuh>
#include <nvbench/json_printer.cuh>
#include <nvbench/printer_base.cuh>
#include <nvbench/state.cuh>
#include <nvbench/summary.cuh>
#include <nvbench/types.cuh>
#include <chrono>
#include <fstream>
#include <sstream>
#include <string>
#include <system_error>
#include <utility>
#include <vector>
#include "test_asserts.cuh"
#if __has_include(<filesystem>)
#include <filesystem>
namespace fs = std::filesystem;
#elif __has_include(<experimental/filesystem>)
#include <experimental/filesystem>
namespace fs = std::experimental::filesystem;
#else
#error "No <filesystem> or <experimental/filesystem> found."
#endif
void dummy_generator(nvbench::state &) {}
NVBENCH_DEFINE_CALLABLE(dummy_generator, dummy_callable);
using dummy_bench = nvbench::benchmark<dummy_callable>;
struct recording_printer : nvbench::printer_base
{
explicit recording_printer(std::ostream &stream)
: nvbench::printer_base{stream}
{}
std::vector<std::pair<nvbench::log_level, std::string>> logs;
protected:
void do_log(nvbench::log_level level, const std::string &msg) override
{
logs.emplace_back(level, msg);
}
};
struct temp_directory
{
temp_directory()
: path{fs::temp_directory_path() /
("nvbench-json-printer-test-" +
std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()))}
{
fs::create_directories(path);
}
~temp_directory()
{
std::error_code ec;
fs::remove_all(path, ec);
}
fs::path path;
};
namespace nvbench::detail
{
// nvbench::state construction is private to the state generator and this test
// hook; use the friend type to build a minimal state for printer testing.
struct state_tester : public nvbench::state
{
state_tester(const nvbench::benchmark_base &bench)
: nvbench::state{bench}
{}
};
} // namespace nvbench::detail
void test_jsonbin_filenames_are_json_relative()
{
const temp_directory tmp_dir;
const auto json_path = tmp_dir.path / "result.json";
std::ofstream json_stream{json_path};
nvbench::json_printer printer{json_stream, json_path.string(), true};
dummy_bench bench;
nvbench::detail::state_tester state{bench};
printer.process_bulk_data(state, "nv/cold/sample_times", "sample_times", {1.0, 2.0});
printer.process_bulk_data(state, "nv/cold/sample_freqs", "sample_freqs", {3.0, 4.0});
const auto samples_filename =
state.get_summary("nv/json/bin:nv/cold/sample_times").get_string("filename");
const auto freqs_filename =
state.get_summary("nv/json/freqs-bin:nv/cold/sample_freqs").get_string("filename");
ASSERT(fs::path{samples_filename} == fs::path{"result.json-bin"} / "0.bin");
ASSERT(fs::path{freqs_filename} == fs::path{"result.json-freqs-bin"} / "0.bin");
ASSERT(fs::exists(tmp_dir.path / samples_filename));
ASSERT(fs::exists(tmp_dir.path / freqs_filename));
}
void test_jsonbin_write_failure_omits_file_metadata()
{
const temp_directory tmp_dir;
const auto json_path = tmp_dir.path / "result.json";
const auto collision_path = tmp_dir.path / "result.json-bin";
{
std::ofstream collision{collision_path};
collision << "not a directory";
}
std::ofstream json_stream{json_path};
nvbench::json_printer printer{json_stream, json_path.string(), true};
std::ostringstream log_stream;
recording_printer log_printer{log_stream};
dummy_bench bench;
bench.set_printer(log_printer);
nvbench::detail::state_tester state{bench};
printer.process_bulk_data(state, "nv/cold/sample_times", "sample_times", {1.0, 2.0});
const auto &summary = state.get_summary("nv/json/bin:nv/cold/sample_times");
ASSERT(summary.has_value("name"));
ASSERT(summary.has_value("hint"));
ASSERT(summary.has_value("description"));
ASSERT(!summary.has_value("filename"));
ASSERT(!summary.has_value("size"));
ASSERT(fs::is_regular_file(collision_path));
ASSERT(log_printer.logs.size() == 1);
ASSERT(log_printer.logs[0].first == nvbench::log_level::warn);
ASSERT(log_printer.logs[0].second.find("Error writing nv/cold/sample_times (sample_times)") !=
std::string::npos);
}
int main()
{
test_jsonbin_filenames_are_json_relative();
test_jsonbin_write_failure_omits_file_metadata();
}