From 6dd4bebe6e81e1b6d0c2184ccbc65409a2724cd4 Mon Sep 17 00:00:00 2001 From: Allison Vacanti Date: Thu, 14 Jan 2021 17:51:13 -0500 Subject: [PATCH] Clonable benchmarks. Adds clone() methods to benchmark_base and axis_base, along with the scaffolding to support them. --- nvbench/axes_metadata.cu | 20 ++++++++ nvbench/axes_metadata.cuh | 9 +++- nvbench/axis_base.cu | 5 ++ nvbench/axis_base.cuh | 4 ++ nvbench/benchmark.cuh | 9 +++- nvbench/benchmark_base.cu | 12 +++++ nvbench/benchmark_base.cuh | 11 +++++ nvbench/detail/state_generator.cu | 5 +- nvbench/float64_axis.cuh | 4 ++ nvbench/int64_axis.cuh | 4 ++ nvbench/string_axis.cuh | 4 ++ nvbench/type_axis.cuh | 4 ++ testing/benchmark.cu | 40 ++++++++++++++++ testing/float64_axis.cu | 32 ++++++++++++- testing/int64_axis.cu | 80 +++++++++++++++++++++++++++---- testing/string_axis.cu | 37 ++++++++++++-- testing/type_axis.cu | 55 +++++++++++++++++++-- 17 files changed, 309 insertions(+), 26 deletions(-) diff --git a/nvbench/axes_metadata.cu b/nvbench/axes_metadata.cu index 2a963fa..3be5301 100644 --- a/nvbench/axes_metadata.cu +++ b/nvbench/axes_metadata.cu @@ -8,6 +8,26 @@ namespace nvbench { +axes_metadata::axes_metadata(const axes_metadata &other) +{ + m_axes.reserve(other.get_axes().size()); + for (const auto &axis : other.get_axes()) + { + m_axes.push_back(axis->clone()); + } +} + +axes_metadata &axes_metadata::operator=(const axes_metadata &other) +{ + m_axes.clear(); + m_axes.reserve(other.get_axes().size()); + for (const auto &axis : other.get_axes()) + { + m_axes.push_back(axis->clone()); + } + return *this; +} + void axes_metadata::add_float64_axis(std::string name, std::vector data) { diff --git a/nvbench/axes_metadata.cuh b/nvbench/axes_metadata.cuh index e50f55a..9880cb9 100644 --- a/nvbench/axes_metadata.cuh +++ b/nvbench/axes_metadata.cuh @@ -18,7 +18,14 @@ namespace nvbench // Holds dynamic axes information. struct axes_metadata { - using axes_type = std::vector>; + using axes_type = std::vector>; + + axes_metadata() = default; + axes_metadata(axes_metadata&&) = default; + axes_metadata& operator=(axes_metadata&&) = default; + + axes_metadata(const axes_metadata&); + axes_metadata& operator=(const axes_metadata&); template void set_type_axes_names(std::vector names); diff --git a/nvbench/axis_base.cu b/nvbench/axis_base.cu index 8bedec8..6ab2efe 100644 --- a/nvbench/axis_base.cu +++ b/nvbench/axis_base.cu @@ -5,4 +5,9 @@ namespace nvbench axis_base::~axis_base() = default; +std::unique_ptr axis_base::clone() const +{ + return this->do_clone(); +} + } // namespace nvbench diff --git a/nvbench/axis_base.cuh b/nvbench/axis_base.cuh index 53e288d..01772e2 100644 --- a/nvbench/axis_base.cuh +++ b/nvbench/axis_base.cuh @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -18,6 +19,8 @@ struct axis_base { virtual ~axis_base(); + [[nodiscard]] std::unique_ptr clone() const; + [[nodiscard]] const std::string &get_name() const { return m_name; } [[nodiscard]] axis_type get_type() const { return m_type; } @@ -41,6 +44,7 @@ protected: {} private: + virtual std::unique_ptr do_clone() const = 0; virtual std::size_t do_get_size() const = 0; virtual std::string do_get_input_string(std::size_t i) const = 0; virtual std::string do_get_description(std::size_t i) const = 0; diff --git a/nvbench/benchmark.cuh b/nvbench/benchmark.cuh index 9ba9542..b22b743 100644 --- a/nvbench/benchmark.cuh +++ b/nvbench/benchmark.cuh @@ -50,12 +50,17 @@ struct benchmark final : public benchmark_base ~benchmark() override = default; private: - void do_set_type_axes_names(std::vector names) override + std::unique_ptr do_clone() const final + { + return std::make_unique(); + } + + void do_set_type_axes_names(std::vector names) final { m_axes.template set_type_axes_names(std::move(names)); } - void do_run() override + void do_run() final { nvbench::runner runner{*this}; runner.generate_states(); diff --git a/nvbench/benchmark_base.cu b/nvbench/benchmark_base.cu index 40f2ceb..e5d1e64 100644 --- a/nvbench/benchmark_base.cu +++ b/nvbench/benchmark_base.cu @@ -5,4 +5,16 @@ namespace nvbench benchmark_base::~benchmark_base() = default; +std::unique_ptr benchmark_base::clone() const +{ + auto result = this->do_clone(); + + // Do not copy states. + result->m_name = m_name; + result->m_axes = m_axes; + + return std::move(result); +} + + } // namespace nvbench diff --git a/nvbench/benchmark_base.cuh b/nvbench/benchmark_base.cuh index afe1fe4..fdba1dc 100644 --- a/nvbench/benchmark_base.cuh +++ b/nvbench/benchmark_base.cuh @@ -3,6 +3,7 @@ #include #include +#include #include namespace nvbench @@ -22,6 +23,15 @@ struct benchmark_base { virtual ~benchmark_base(); + /** + * Returns a pointer to a new instance of the concrete benchmark<...> + * subclass. + * + * The result will have the same name as the source benchmark, and the + * axes will be shallow-copied. The result will have an empty `get_state()`. + */ + [[nodiscard]] std::unique_ptr clone() const; + benchmark_base &set_name(std::string name) { m_name = std::move(name); @@ -94,6 +104,7 @@ protected: private: // route these through virtuals so the templated subclass can inject type info + virtual std::unique_ptr do_clone() const = 0; virtual void do_set_type_axes_names(std::vector names) = 0; virtual void do_run() = 0; }; diff --git a/nvbench/detail/state_generator.cu b/nvbench/detail/state_generator.cu index b9dcc2f..b955c7d 100644 --- a/nvbench/detail/state_generator.cu +++ b/nvbench/detail/state_generator.cu @@ -28,9 +28,10 @@ state_generator::create(const benchmark_base &bench) // matching up states to kernel_generator instantiations much easier during // dispatch. - const axes_metadata& axes = bench.get_axes(); + const axes_metadata &axes = bench.get_axes(); // vector of all axes: - const std::vector> &axes_vec = axes.get_axes(); + const std::vector> &axes_vec = + axes.get_axes(); // Construct two state_generators: // - Only type_axis objects, diff --git a/nvbench/float64_axis.cuh b/nvbench/float64_axis.cuh index 76dccbd..93351d5 100644 --- a/nvbench/float64_axis.cuh +++ b/nvbench/float64_axis.cuh @@ -28,6 +28,10 @@ struct float64_axis final : public axis_base } private: + std::unique_ptr do_clone() const + { + return std::make_unique(*this); + } std::size_t do_get_size() const final { return m_values.size(); } std::string do_get_input_string(std::size_t i) const final; std::string do_get_description(std::size_t i) const final; diff --git a/nvbench/int64_axis.cuh b/nvbench/int64_axis.cuh index 6fd9746..d727e67 100644 --- a/nvbench/int64_axis.cuh +++ b/nvbench/int64_axis.cuh @@ -75,6 +75,10 @@ struct int64_axis final : public axis_base }; private: + std::unique_ptr do_clone() const + { + return std::make_unique(*this); + } std::size_t do_get_size() const final { return m_inputs.size(); } std::string do_get_input_string(std::size_t) const final; std::string do_get_description(std::size_t) const final; diff --git a/nvbench/string_axis.cuh b/nvbench/string_axis.cuh index 895c509..092d733 100644 --- a/nvbench/string_axis.cuh +++ b/nvbench/string_axis.cuh @@ -28,6 +28,10 @@ struct string_axis final : public axis_base } private: + std::unique_ptr do_clone() const + { + return std::make_unique(*this); + } std::size_t do_get_size() const final { return m_values.size(); } std::string do_get_input_string(std::size_t i) const final { diff --git a/nvbench/type_axis.cuh b/nvbench/type_axis.cuh index e24dec2..7d13af2 100644 --- a/nvbench/type_axis.cuh +++ b/nvbench/type_axis.cuh @@ -37,6 +37,10 @@ struct type_axis final : public axis_base get_type_index(const std::string &input_string) const; private: + std::unique_ptr do_clone() const + { + return std::make_unique(*this); + } std::size_t do_get_size() const final { return m_input_strings.size(); } std::string do_get_input_string(std::size_t i) const final { diff --git a/testing/benchmark.cu b/testing/benchmark.cu index 2a6c04f..fafbb97 100644 --- a/testing/benchmark.cu +++ b/testing/benchmark.cu @@ -229,6 +229,45 @@ void test_run() ASSERT(bench.get_states().size() == 1); } +void test_clone() +{ + lots_of_types_bench bench; + bench.set_type_axes_names({"Integer", "Float", "Other"}); + bench.add_string_axis("Strings", {"string a", "string b", "string c"}); + bench.add_int64_power_of_two_axis("I64 POT Axis", {10, 20}); + bench.add_int64_axis("I64 Axis", {10, 20}); + bench.add_float64_axis("F64 Axis", {0., .1, .25, .5, 1.}); + + std::unique_ptr clone_base = bench.clone(); + ASSERT(clone_base.get() != nullptr); + + auto *clone = dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(bench.get_name() == clone->get_name()); + + const auto &ref_axes = bench.get_axes().get_axes(); + const auto &clone_axes = clone->get_axes().get_axes(); + ASSERT(ref_axes.size() == clone_axes.size()); + for (std::size_t i = 0; i < ref_axes.size(); ++i) + { + const nvbench::axis_base *ref_axis = ref_axes[i].get(); + const nvbench::axis_base *clone_axis = clone_axes[i].get(); + ASSERT(ref_axis != nullptr); + ASSERT(clone_axis != nullptr); + ASSERT(ref_axis->get_name() == clone_axis->get_name()); + ASSERT(ref_axis->get_type() == clone_axis->get_type()); + ASSERT(ref_axis->get_size() == clone_axis->get_size()); + for (std::size_t j = 0; j < ref_axis->get_size(); ++j) + { + ASSERT(ref_axis->get_input_string(j) == clone_axis->get_input_string(j)); + ASSERT(ref_axis->get_description(j) == clone_axis->get_description(j)); + } + } + + ASSERT(clone->get_states().empty()); +} + int main() { test_type_axes(); @@ -238,4 +277,5 @@ int main() test_int64_power_of_two_axes(); test_string_axes(); test_run(); + test_clone(); } diff --git a/testing/float64_axis.cu b/testing/float64_axis.cu index 3f58095..1c5a4ed 100644 --- a/testing/float64_axis.cu +++ b/testing/float64_axis.cu @@ -5,6 +5,7 @@ void test_empty() { nvbench::float64_axis axis("Empty"); + ASSERT(axis.get_name() == "Empty"); ASSERT(axis.get_type() == nvbench::axis_type::float64); ASSERT(axis.get_size() == 0); @@ -12,15 +13,24 @@ void test_empty() axis.set_inputs({}); ASSERT(axis.get_size() == 0); + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "Empty"); + ASSERT(clone->get_type() == nvbench::axis_type::float64); + ASSERT(clone->get_size() == 0); } void test_basic() { nvbench::float64_axis axis("Basic"); - ASSERT(axis.get_name() == "Basic"); - axis.set_inputs({-100.3, 0., 2064.15}); + ASSERT(axis.get_name() == "Basic"); ASSERT(axis.get_size() == 3); ASSERT(axis.get_value(0) == -100.3); ASSERT(axis.get_input_string(0) == "-100.3"); @@ -31,6 +41,24 @@ void test_basic() ASSERT(axis.get_value(2) == 2064.15); ASSERT(axis.get_input_string(2) == "2064.15"); ASSERT(axis.get_description(2) == ""); + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "Basic"); + ASSERT(clone->get_size() == 3); + ASSERT(clone->get_value(0) == -100.3); + ASSERT(clone->get_input_string(0) == "-100.3"); + ASSERT(clone->get_description(0) == ""); + ASSERT(clone->get_value(1) == 0.); + ASSERT(clone->get_input_string(1) == "0"); + ASSERT(clone->get_description(1) == ""); + ASSERT(clone->get_value(2) == 2064.15); + ASSERT(clone->get_input_string(2) == "2064.15"); + ASSERT(clone->get_description(2) == ""); } int main() diff --git a/testing/int64_axis.cu b/testing/int64_axis.cu index 0644689..5106205 100644 --- a/testing/int64_axis.cu +++ b/testing/int64_axis.cu @@ -4,53 +4,113 @@ #include +void test_empty() +{ + nvbench::int64_axis axis("Empty"); + + ASSERT(axis.get_name() == "Empty"); + ASSERT(axis.get_type() == nvbench::axis_type::int64); + ASSERT(axis.get_size() == 0); + + axis.set_inputs({}); + + ASSERT(axis.get_size() == 0); + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "Empty"); + ASSERT(clone->get_type() == nvbench::axis_type::int64); + ASSERT(clone->get_size() == 0); +} + void test_basic() { nvbench::int64_axis axis{"BasicAxis"}; + axis.set_inputs({0, 1, 2, 3, 7, 6, 5, 4}); + const std::vector ref{0, 1, 2, 3, 7, 6, 5, 4}; + ASSERT(axis.get_name() == "BasicAxis"); ASSERT(axis.get_type() == nvbench::axis_type::int64); ASSERT(!axis.is_power_of_two()); - - axis.set_inputs({0, 1, 2, 3, 7, 6, 5, 4}); ASSERT(axis.get_size() == 8); - std::vector ref{0, 1, 2, 3, 7, 6, 5, 4}; ASSERT(axis.get_inputs() == ref); ASSERT(axis.get_values() == ref); - for (size_t i = 0; i < 8; ++i) { ASSERT(axis.get_input_string(i) == fmt::to_string(ref[i])); ASSERT(axis.get_description(i).empty()); } + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "BasicAxis"); + ASSERT(clone->get_type() == nvbench::axis_type::int64); + ASSERT(!clone->is_power_of_two()); + ASSERT(clone->get_size() == 8); + + ASSERT(clone->get_inputs() == ref); + ASSERT(clone->get_values() == ref); + for (size_t i = 0; i < 8; ++i) + { + ASSERT(clone->get_input_string(i) == fmt::to_string(ref[i])); + ASSERT(clone->get_description(i).empty()); + } } void test_power_of_two() { nvbench::int64_axis axis{"POTAxis", nvbench::int64_axis_flags::power_of_two}; + axis.set_inputs({0, 1, 2, 3, 7, 6, 5, 4}); + const std::vector ref_inputs{0, 1, 2, 3, 7, 6, 5, 4}; + const std::vector ref_values{1, 2, 4, 8, 128, 64, 32, 16}; + ASSERT(axis.get_name() == "POTAxis"); ASSERT(axis.get_type() == nvbench::axis_type::int64); ASSERT(axis.is_power_of_two()); - - axis.set_inputs({0, 1, 2, 3, 7, 6, 5, 4}); ASSERT(axis.get_size() == 8); - std::vector ref_inputs{0, 1, 2, 3, 7, 6, 5, 4}; - std::vector ref_values{1, 2, 4, 8, 128, 64, 32, 16}; ASSERT(axis.get_inputs() == ref_inputs); ASSERT(axis.get_values() == ref_values); - for (size_t i = 0; i < 8; ++i) { -// fmt::print("{}: {}\n", i, axis.get_description(i)); ASSERT(axis.get_input_string(i) == fmt::to_string(ref_inputs[i])); ASSERT(axis.get_description(i) == fmt::format("2^{} = {}", ref_inputs[i], ref_values[i])); } + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "POTAxis"); + ASSERT(clone->get_type() == nvbench::axis_type::int64); + ASSERT(clone->is_power_of_two()); + ASSERT(clone->get_size() == 8); + + ASSERT(clone->get_inputs() == ref_inputs); + ASSERT(clone->get_values() == ref_values); + for (size_t i = 0; i < 8; ++i) + { + ASSERT(clone->get_input_string(i) == fmt::to_string(ref_inputs[i])); + ASSERT(clone->get_description(i) == + fmt::format("2^{} = {}", ref_inputs[i], ref_values[i])); + } } int main() { + test_empty(); test_basic(); test_power_of_two(); diff --git a/testing/string_axis.cu b/testing/string_axis.cu index 61eb212..bcda5dd 100644 --- a/testing/string_axis.cu +++ b/testing/string_axis.cu @@ -5,22 +5,31 @@ void test_empty() { nvbench::string_axis axis("Empty"); + axis.set_inputs({}); + ASSERT(axis.get_name() == "Empty"); ASSERT(axis.get_type() == nvbench::axis_type::string); ASSERT(axis.get_size() == 0); - - axis.set_inputs({}); - ASSERT(axis.get_size() == 0); + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "Empty"); + ASSERT(clone->get_type() == nvbench::axis_type::string); + ASSERT(clone->get_size() == 0); + ASSERT(clone->get_size() == 0); } void test_basic() { nvbench::string_axis axis("Basic"); - ASSERT(axis.get_name() == "Basic"); - axis.set_inputs({"String 1", "String 2", "String 3"}); + ASSERT(axis.get_name() == "Basic"); ASSERT(axis.get_size() == 3); ASSERT(axis.get_value(0) == "String 1"); ASSERT(axis.get_input_string(0) == "String 1"); @@ -31,6 +40,24 @@ void test_basic() ASSERT(axis.get_value(2) == "String 3"); ASSERT(axis.get_input_string(2) == "String 3"); ASSERT(axis.get_description(2) == ""); + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "Basic"); + ASSERT(clone->get_size() == 3); + ASSERT(clone->get_value(0) == "String 1"); + ASSERT(clone->get_input_string(0) == "String 1"); + ASSERT(clone->get_description(0) == ""); + ASSERT(clone->get_value(1) == "String 2"); + ASSERT(clone->get_input_string(1) == "String 2"); + ASSERT(clone->get_description(1) == ""); + ASSERT(clone->get_value(2) == "String 3"); + ASSERT(clone->get_input_string(2) == "String 3"); + ASSERT(clone->get_description(2) == ""); } int main() diff --git a/testing/type_axis.cu b/testing/type_axis.cu index cdf3373..4d1aeee 100644 --- a/testing/type_axis.cu +++ b/testing/type_axis.cu @@ -9,6 +9,7 @@ void test_empty() { nvbench::type_axis axis("Basic", 0); + ASSERT(axis.get_name() == "Basic"); ASSERT(axis.get_axis_index() == 0); ASSERT(axis.get_type() == nvbench::axis_type::type); @@ -17,28 +18,48 @@ void test_empty() axis.set_inputs>(); ASSERT(axis.get_size() == 0); + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "Basic"); + ASSERT(clone->get_axis_index() == 0); + ASSERT(clone->get_type() == nvbench::axis_type::type); + ASSERT(clone->get_size() == 0); } void test_single() { nvbench::type_axis axis("Single", 0); - ASSERT(axis.get_name() == "Single"); - axis.set_inputs>(); + ASSERT(axis.get_name() == "Single"); ASSERT(axis.get_size() == 1); ASSERT(axis.get_input_string(0) == "I32"); ASSERT(axis.get_description(0) == "int32_t"); + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "Single"); + ASSERT(clone->get_size() == 1); + ASSERT(clone->get_input_string(0) == "I32"); + ASSERT(clone->get_description(0) == "int32_t"); } void test_several() { nvbench::type_axis axis("Several", 0); - ASSERT(axis.get_name() == "Several"); - axis.set_inputs< nvbench::type_list>(); + ASSERT(axis.get_name() == "Several"); ASSERT(axis.get_size() == 3); ASSERT(axis.get_input_string(0) == "I32"); ASSERT(axis.get_description(0) == "int32_t"); @@ -46,6 +67,21 @@ void test_several() ASSERT(axis.get_description(1) == "double"); ASSERT(axis.get_input_string(2) == "bool"); ASSERT(axis.get_description(2) == ""); + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_name() == "Several"); + ASSERT(clone->get_size() == 3); + ASSERT(clone->get_input_string(0) == "I32"); + ASSERT(clone->get_description(0) == "int32_t"); + ASSERT(clone->get_input_string(1) == "F64"); + ASSERT(clone->get_description(1) == "double"); + ASSERT(clone->get_input_string(2) == "bool"); + ASSERT(clone->get_description(2) == ""); } void test_get_type_index() @@ -59,6 +95,17 @@ void test_get_type_index() ASSERT(axis.get_type_index("U16") == 1); ASSERT(axis.get_type_index("F32") == 2); ASSERT(axis.get_type_index("bool") == 3); + + const auto clone_base = axis.clone(); + ASSERT(clone_base.get() != nullptr); + const auto *clone = + dynamic_cast(clone_base.get()); + ASSERT(clone != nullptr); + + ASSERT(clone->get_type_index("I8") == 0); + ASSERT(clone->get_type_index("U16") == 1); + ASSERT(clone->get_type_index("F32") == 2); + ASSERT(clone->get_type_index("bool") == 3); } int main()