From 6bb22b952ca36b47d7d207fcb6a6f4ce2fdddba4 Mon Sep 17 00:00:00 2001 From: Allison Vacanti Date: Thu, 4 Feb 2021 13:15:21 -0500 Subject: [PATCH] Add option parsing to NVBENCH_MAIN. - Convert benchmark_manager into a read-only structure. - Mutable benchmarks will be provided by `option_parser::get_benchmarks()` or `benchmark_manager::clone_benchmarks()`. --- nvbench/benchmark_manager.cu | 20 +++++--------- nvbench/benchmark_manager.cuh | 18 ++++++++++--- nvbench/detail/markdown_format.cu | 6 ++--- nvbench/detail/markdown_format.cuh | 9 ++++++- nvbench/main.cuh | 15 ++++++----- nvbench/option_parser.cu | 7 +++++ testing/create.cu | 42 +++++++++++++++--------------- testing/option_parser.cu | 17 +++++++----- 8 files changed, 79 insertions(+), 55 deletions(-) diff --git a/nvbench/benchmark_manager.cu b/nvbench/benchmark_manager.cu index 4a0e096..8b73b88 100644 --- a/nvbench/benchmark_manager.cu +++ b/nvbench/benchmark_manager.cu @@ -20,20 +20,14 @@ benchmark_base &benchmark_manager::add(std::unique_ptr bench) return *m_benchmarks.back(); } -benchmark_base &benchmark_manager::get_benchmark(const std::string &name) +benchmark_manager::benchmark_vector benchmark_manager::clone_benchmarks() const { - auto iter = std::find_if(m_benchmarks.begin(), - m_benchmarks.end(), - [&name](auto &bench_ptr) { - return bench_ptr->get_name() == name; - }); - if (iter == m_benchmarks.end()) - { - throw std::runtime_error( - fmt::format("{}:{}: No benchmark named '{}'.", name)); - } - - return **iter; + benchmark_vector result(m_benchmarks.size()); + std::transform(m_benchmarks.cbegin(), + m_benchmarks.cend(), + result.begin(), + [](const auto &bench) { return bench->clone(); }); + return result; } const benchmark_base & diff --git a/nvbench/benchmark_manager.cuh b/nvbench/benchmark_manager.cuh index 575d2c3..ad9bc39 100644 --- a/nvbench/benchmark_manager.cuh +++ b/nvbench/benchmark_manager.cuh @@ -9,22 +9,34 @@ namespace nvbench { /** - * Singleton class that owns all benchmarks. + * Singleton class that owns reference copies of all known benchmarks. */ struct benchmark_manager { using benchmark_vector = std::vector>; + /** + * @return The singleton benchmark_manager instance. + */ [[nodiscard]] static benchmark_manager &get(); + /** + * Register a new benchmark. + */ benchmark_base &add(std::unique_ptr bench); - [[nodiscard]] benchmark_base &get_benchmark(const std::string &name); + /** + * Clone all benchmarks in the manager into the returned vector. + */ + [[nodiscard]] benchmark_vector clone_benchmarks() const; + + /** + * Get a non-mutable reference to benchmark with the specified name. + */ [[nodiscard]] const benchmark_base & get_benchmark(const std::string &name) const; - [[nodiscard]] benchmark_vector &get_benchmarks() { return m_benchmarks; }; [[nodiscard]] const benchmark_vector &get_benchmarks() const { return m_benchmarks; diff --git a/nvbench/detail/markdown_format.cu b/nvbench/detail/markdown_format.cu index ff3e16b..5c9f6cf 100644 --- a/nvbench/detail/markdown_format.cu +++ b/nvbench/detail/markdown_format.cu @@ -1,7 +1,6 @@ #include #include -#include #include #include @@ -116,7 +115,7 @@ namespace nvbench namespace detail { -void markdown_format::print() +void markdown_format::print(const benchmark_vector &benchmarks) { auto format_visitor = [](const auto &v) { using T = std::decay_t; @@ -193,8 +192,7 @@ void markdown_format::print() return fmt::format("{:.2f}%", percentage); }; - auto &mgr = nvbench::benchmark_manager::get(); - for (const auto &bench_ptr : mgr.get_benchmarks()) + for (const auto &bench_ptr : benchmarks) { const benchmark_base &bench = *bench_ptr; const axes_metadata &axes = bench.get_axes(); diff --git a/nvbench/detail/markdown_format.cuh b/nvbench/detail/markdown_format.cuh index 7462633..0b9fe40 100644 --- a/nvbench/detail/markdown_format.cuh +++ b/nvbench/detail/markdown_format.cuh @@ -1,5 +1,10 @@ #pragma once +#include + +#include +#include + namespace nvbench { @@ -8,9 +13,11 @@ namespace detail struct markdown_format { + using benchmark_vector = std::vector>; + // Hacked in to just print a basic summary table to stdout. There's lots of // room for improvement here. - void print(); + static void print(const benchmark_vector &benchmarks); }; } // namespace detail diff --git a/nvbench/main.cuh b/nvbench/main.cuh index 5743fde..bf9517c 100644 --- a/nvbench/main.cuh +++ b/nvbench/main.cuh @@ -3,22 +3,25 @@ #include #include #include +#include #define NVBENCH_MAIN \ - int main() \ + int main(int argc, char const *const *argv) \ { \ - NVBENCH_MAIN_BODY; \ + NVBENCH_MAIN_BODY(argc, argv); \ return 0; \ } -#define NVBENCH_MAIN_BODY \ +#define NVBENCH_MAIN_BODY(argc, argv) \ do \ { \ - auto &mgr = nvbench::benchmark_manager::get(); \ - for (auto &bench_ptr : mgr.get_benchmarks()) \ + nvbench::option_parser parser; \ + parser.parse(argc, argv); \ + auto &benchmarks = parser.get_benchmarks(); \ + for (auto &bench_ptr : benchmarks) \ { \ bench_ptr->run(); \ } \ nvbench::detail::markdown_format printer; \ - printer.print(); \ + printer.print(benchmarks); \ } while (false) diff --git a/nvbench/option_parser.cu b/nvbench/option_parser.cu index dc02973..338ae8d 100644 --- a/nvbench/option_parser.cu +++ b/nvbench/option_parser.cu @@ -293,6 +293,13 @@ void option_parser::parse_impl() arg)); } } + + if (m_benchmarks.empty()) + { + // If no benchmarks were specified, run all: + const auto &mgr = nvbench::benchmark_manager::get(); + m_benchmarks = mgr.clone_benchmarks(); + } } void option_parser::add_benchmark(const std::string &name) diff --git a/testing/create.cu b/testing/create.cu index f4ce027..5d6ab4e 100644 --- a/testing/create.cu +++ b/testing/create.cu @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -80,10 +79,11 @@ NVBENCH_CREATE_TEMPLATE(template_no_op_generator, type_axes) // Checks that the specified number of states exist and that each has been // skipped. Concatenates the skip reasons and returns the resulting string. -std::string check_states_and_get_string(const nvbench::benchmark_base &bench, - std::size_t num_type_configs, - std::size_t states_per_type_config) +std::string run_and_get_state_string(nvbench::benchmark_base &bench, + std::size_t num_type_configs, + std::size_t states_per_type_config) { + bench.run(); fmt::memory_buffer buffer; const auto &states = bench.get_states(); ASSERT(states.size() == num_type_configs); @@ -101,30 +101,30 @@ std::string check_states_and_get_string(const nvbench::benchmark_base &bench, void validate_default_name() { - nvbench::benchmark_base &bench = - nvbench::benchmark_manager::get().get_benchmark("no_op_generator"); + auto bench = + nvbench::benchmark_manager::get().get_benchmark("no_op_generator").clone(); const std::string ref = "Params:\n"; - const auto test = check_states_and_get_string(bench, 1, 1); + const auto test = run_and_get_state_string(*bench, 1, 1); ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test); } void validate_custom_name() { - nvbench::benchmark_base &bench = - nvbench::benchmark_manager::get().get_benchmark("Custom Name"); + auto bench = + nvbench::benchmark_manager::get().get_benchmark("Custom Name").clone(); const std::string ref = "Params:\n"; - const auto test = check_states_and_get_string(bench, 1, 1); + const auto test = run_and_get_state_string(*bench, 1, 1); ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test); } void validate_no_types() { - nvbench::benchmark_base &bench = - nvbench::benchmark_manager::get().get_benchmark("No Types"); + auto bench = + nvbench::benchmark_manager::get().get_benchmark("No Types").clone(); const std::string ref = R"expected(Params: Float: 11 Int: 1 String: One Params: Float: 11 Int: 2 String: One @@ -155,14 +155,14 @@ Params: Float: 13 Int: 2 String: Three Params: Float: 13 Int: 3 String: Three )expected"; - const auto test = check_states_and_get_string(bench, 1, 27); + const auto test = run_and_get_state_string(*bench, 1, 27); ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test); } void validate_only_types() { - nvbench::benchmark_base &bench = - nvbench::benchmark_manager::get().get_benchmark("Oops, All Types!"); + auto bench = + nvbench::benchmark_manager::get().get_benchmark("Oops, All Types!").clone(); const std::string ref = R"expected(Params: FloatT: F32 IntT: I32 MiscT: bool Params: FloatT: F32 IntT: I32 MiscT: void @@ -174,16 +174,17 @@ Params: FloatT: F64 IntT: I64 MiscT: bool Params: FloatT: F64 IntT: I64 MiscT: void )expected"; - const auto test = check_states_and_get_string(bench, 8, 1); + const auto test = run_and_get_state_string(*bench, 8, 1); ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test); } void validate_all_axes() { - nvbench::benchmark_base &bench = - nvbench::benchmark_manager::get().get_benchmark("All The Axes"); + auto bench = + nvbench::benchmark_manager::get().get_benchmark("All The Axes").clone(); - const std::string ref = R"expected(Params: Float: 11 FloatT: F32 Int: 1 IntT: I32 MiscT: bool String: One + const std::string ref = + R"expected(Params: Float: 11 FloatT: F32 Int: 1 IntT: I32 MiscT: bool String: One Params: Float: 11 FloatT: F32 Int: 2 IntT: I32 MiscT: bool String: One Params: Float: 11 FloatT: F32 Int: 3 IntT: I32 MiscT: bool String: One Params: Float: 12 FloatT: F32 Int: 1 IntT: I32 MiscT: bool String: One @@ -401,13 +402,12 @@ Params: Float: 13 FloatT: F64 Int: 2 IntT: I64 MiscT: void String: Three Params: Float: 13 FloatT: F64 Int: 3 IntT: I64 MiscT: void String: Three )expected"; - const auto test = check_states_and_get_string(bench, 8, 27); + const auto test = run_and_get_state_string(*bench, 8, 27); ASSERT_MSG(test == ref, "Expected:\n\"{}\"\n\nActual:\n\"{}\"", ref, test); } int main() { - NVBENCH_MAIN_BODY; validate_default_name(); validate_custom_name(); validate_no_types(); diff --git a/testing/option_parser.cu b/testing/option_parser.cu index 2102646..45abc5c 100644 --- a/testing/option_parser.cu +++ b/testing/option_parser.cu @@ -8,14 +8,17 @@ #include //============================================================================== -// Declare a benchmark for testing: +// Declare a couple benchmarks for testing: +void DummyBench(nvbench::state &state) { state.skip("Test"); } +NVBENCH_CREATE(DummyBench); + using Ts = nvbench::type_list; using Us = nvbench::type_list; template void TestBench(nvbench::state &state, nvbench::type_list) { - state.skip("Test"); + DummyBench(state); } NVBENCH_CREATE_TEMPLATE(TestBench, NVBENCH_TYPE_AXES(Ts, Us)) .set_type_axes_names({"T", "U"}) @@ -90,14 +93,14 @@ void test_empty() { nvbench::option_parser parser; parser.parse({}); - ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_benchmarks().size() == 2); ASSERT(parser.get_args().empty()); } { nvbench::option_parser parser; parser.parse(0, nullptr); - ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_benchmarks().size() == 2); ASSERT(parser.get_args().empty()); } } @@ -106,7 +109,7 @@ void test_exec_name_tolerance() { nvbench::option_parser parser; parser.parse({"TestExec"}); - ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_benchmarks().size() == 2); ASSERT(parser.get_args() == std::vector{"TestExec"}); } @@ -116,14 +119,14 @@ void test_argc_argv_parse() { nvbench::option_parser parser; parser.parse(1, argv); - ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_benchmarks().size() == 2); ASSERT(parser.get_args() == std::vector{"TestExec"}); } { nvbench::option_parser parser; parser.parse(0, nullptr); - ASSERT(parser.get_benchmarks().empty()); + ASSERT(parser.get_benchmarks().size() == 2); ASSERT(parser.get_args().empty()); } }