diff --git a/docs/cli_help.md b/docs/cli_help.md index 6021e0e..1479ebd 100644 --- a/docs/cli_help.md +++ b/docs/cli_help.md @@ -83,36 +83,6 @@ * Applies to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. -* `--min-samples ` - * Gather at least `` samples per measurement. - * Default is 10 samples. - * Applies to the most recent `--benchmark`, or all benchmarks if specified - before any `--benchmark` arguments. - -* `--stopping-criterion ` - * After `--min-samples` is satisfied, use `` to detect if enough - samples were collected. - * Only applies to Cold measurements. - * Default is stdrel (`--stopping-criterion stdrel`) - -* `--min-time ` - * Accumulate at least `` of execution time per measurement. - * Only applies to `stdrel` stopping criterion. - * Default is 0.5 seconds. - * If both GPU and CPU times are gathered, this applies to GPU time only. - * Applies to the most recent `--benchmark`, or all benchmarks if specified - before any `--benchmark` arguments. - -* `--max-noise ` - * Gather samples until the error in the measurement drops below ``. - * Noise is specified as the percent relative standard deviation. - * Default is 0.5% (`--max-noise 0.5`) - * Only applies to `stdrel` stopping criterion. - * Only applies to Cold measurements. - * If both GPU and CPU times are gathered, this applies to GPU noise only. - * Applies to the most recent `--benchmark`, or all benchmarks if specified - before any `--benchmark` arguments. - * `--skip-time ` * Skip a measurement when a warmup run executes in less than ``. * Default is -1 seconds (disabled). @@ -123,16 +93,6 @@ * Applies to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. -* `--timeout ` - * Measurements will timeout after `` have elapsed. - * Default is 15 seconds. - * `` is walltime, not accumulated sample time. - * If a measurement times out, the default markdown log will print a warning to - report any outstanding termination criteria (min samples, min time, max - noise). - * Applies to the most recent `--benchmark`, or all benchmarks if specified - before any `--benchmark` arguments. - * `--throttle-threshold ` * Set the GPU throttle threshold as percentage of the device's default clock rate. * Default is 75. @@ -166,3 +126,68 @@ * Intended for use with external profiling tools. * Applies to the most recent `--benchmark`, or all benchmarks if specified before any `--benchmark` arguments. + +## Stopping Criteria + +* `--timeout ` + * Measurements will timeout after `` have elapsed. + * Default is 15 seconds. + * `` is walltime, not accumulated sample time. + * If a measurement times out, the default markdown log will print a warning to + report any outstanding termination criteria (min samples, min time, max + noise). + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. + +* `--min-samples ` + * Gather at least `` samples per measurement before checking any + other stopping criterion besides the timeout. + * Default is 10 samples. + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. + +* `--stopping-criterion ` + * After `--min-samples` is satisfied, use `` to detect if enough + samples were collected. + * Only applies to Cold and CPU-only measurements. + * If both GPU and CPU times are gathered, GPU time is used for stopping + analysis. + * Stopping criteria provided by NVBench are: + * "stdrel": (default) Converges to a minimal relative standard deviation, + stdev / mean + * "entropy": Converges based on the cumulative entropy of all samples. + * Each stopping criterion may provide additional parameters to customize + behavior, as detailed below: + +### "stdrel" Stopping Criterion Parameters + +* `--min-time ` + * Accumulate at least `` of execution time per measurement. + * Only applies to `stdrel` stopping criterion. + * Default is 0.5 seconds. + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. + +* `--max-noise ` + * Gather samples until the error in the measurement drops below ``. + * Noise is specified as the percent relative standard deviation (stdev/mean). + * Default is 0.5% (`--max-noise 0.5`) + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. + +### "entropy" Stopping Criterion Parameters + +* `--max-angle ` + * Maximum linear regression angle of cumulative entropy. + * Smaller values give more accurate results. + * Default is 0.048. + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. + +* `--min-r2 ` + * Minimum coefficient of determination for linear regression of cumulative + entropy. + * Larger values give more accurate results. + * Default is 0.36. + * Applies to the most recent `--benchmark`, or all benchmarks if specified + before any `--benchmark` arguments. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 8a89b31..061f8eb 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -16,7 +16,6 @@ set(example_srcs add_custom_target(nvbench.example.all) add_dependencies(nvbench.all nvbench.example.all) - function (nvbench_add_examples_target target_prefix cuda_std) add_custom_target(${target_prefix}.all) add_dependencies(nvbench.example.all ${target_prefix}.all) @@ -29,9 +28,15 @@ function (nvbench_add_examples_target target_prefix cuda_std) target_include_directories(${example_name} PRIVATE "${CMAKE_CURRENT_LIST_DIR}") target_link_libraries(${example_name} PRIVATE nvbench::main) set_target_properties(${example_name} PROPERTIES COMPILE_FEATURES cuda_std_${cuda_std}) + + set(example_args --timeout 0.1) + # The custom_criterion example doesn't support the --min-time argument: + if (NOT "${example_src}" STREQUAL "custom_criterion.cu") + list(APPEND example_args --min-time 1e-5) + endif() + add_test(NAME ${example_name} - COMMAND "$" --timeout 0.1 --min-time 1e-5 - ) + COMMAND "$" ${example_args}) # These should not deadlock. If they do, it may be that the CUDA context was created before # setting CUDA_MODULE_LOAD=EAGER in main, see NVIDIA/nvbench#136. diff --git a/nvbench/benchmark_base.cuh b/nvbench/benchmark_base.cuh index ce2f19f..829cccd 100644 --- a/nvbench/benchmark_base.cuh +++ b/nvbench/benchmark_base.cuh @@ -266,22 +266,53 @@ struct benchmark_base return *this; } + /// Control the stopping criterion for the measurement loop. + /// @{ + [[nodiscard]] const std::string &get_stopping_criterion() const { return m_stopping_criterion; } + benchmark_base &set_stopping_criterion(std::string criterion); + /// @} + + [[nodiscard]] bool has_criterion_param(const std::string &name) const + { + return m_criterion_params.has_value(name); + } + + [[nodiscard]] nvbench::int64_t get_criterion_param_int64(const std::string &name) const + { + return m_criterion_params.get_int64(name); + } + benchmark_base &set_criterion_param_int64(const std::string &name, nvbench::int64_t value) + { + m_criterion_params.set_int64(name, value); + return *this; + } + + [[nodiscard]] nvbench::float64_t get_criterion_param_float64(const std::string &name) const + { + return m_criterion_params.get_float64(name); + } + benchmark_base &set_criterion_param_float64(const std::string &name, nvbench::float64_t value) + { + m_criterion_params.set_float64(name, value); + return *this; + } + + [[nodiscard]] std::string get_criterion_param_string(const std::string &name) const + { + return m_criterion_params.get_string(name); + } + benchmark_base &set_criterion_param_string(const std::string &name, std::string value) + { + m_criterion_params.set_string(name, std::move(value)); + return *this; + } + [[nodiscard]] nvbench::criterion_params &get_criterion_params() { return m_criterion_params; } [[nodiscard]] const nvbench::criterion_params &get_criterion_params() const { return m_criterion_params; } - /// Control the stopping criterion for the measurement loop. - /// @{ - [[nodiscard]] const std::string &get_stopping_criterion() const { return m_stopping_criterion; } - benchmark_base &set_stopping_criterion(std::string criterion) - { - m_stopping_criterion = std::move(criterion); - return *this; - } - /// @} - protected: friend struct nvbench::runner_base; diff --git a/nvbench/benchmark_base.cxx b/nvbench/benchmark_base.cxx index 592796d..06b3a3b 100644 --- a/nvbench/benchmark_base.cxx +++ b/nvbench/benchmark_base.cxx @@ -17,6 +17,7 @@ */ #include +#include #include namespace nvbench @@ -88,4 +89,11 @@ std::size_t benchmark_base::get_config_count() const return per_device_count * m_devices.size(); } +benchmark_base &benchmark_base::set_stopping_criterion(std::string criterion) +{ + m_stopping_criterion = std::move(criterion); + m_criterion_params = criterion_manager::get().get_criterion(m_stopping_criterion).get_params(); + return *this; +} + } // namespace nvbench diff --git a/nvbench/criterion_manager.cuh b/nvbench/criterion_manager.cuh index 65678b8..bf7d0a3 100644 --- a/nvbench/criterion_manager.cuh +++ b/nvbench/criterion_manager.cuh @@ -50,6 +50,9 @@ public: using params_description = std::vector>; params_description get_params_description() const; + + using params_map = std::unordered_map; + params_map get_params_description_map() const; }; /** diff --git a/nvbench/criterion_manager.cxx b/nvbench/criterion_manager.cxx index d2b480c..60c8bad 100644 --- a/nvbench/criterion_manager.cxx +++ b/nvbench/criterion_manager.cxx @@ -104,4 +104,23 @@ nvbench::criterion_manager::params_description criterion_manager::get_params_des return desc; } +criterion_manager::params_map criterion_manager::get_params_description_map() const +{ + params_map result; + + for (auto &[criterion_name, criterion] : m_map) + { + params_description &desc = result[criterion_name]; + nvbench::criterion_params params = criterion->get_params(); + + for (auto param : params.get_names()) + { + nvbench::named_values::type type = params.get_type(param); + desc.emplace_back(param, type); + } + } + + return result; +} + } // namespace nvbench diff --git a/nvbench/detail/measure_cold.cu b/nvbench/detail/measure_cold.cu index 92807e6..622220a 100644 --- a/nvbench/detail/measure_cold.cu +++ b/nvbench/detail/measure_cold.cu @@ -30,6 +30,7 @@ #include #include #include +#include #include namespace nvbench::detail @@ -387,11 +388,22 @@ void measure_cold_base::generate_summaries() if (m_max_time_exceeded) { - const auto timeout = m_walltime_timer.get_duration(); - const auto max_noise = m_criterion_params.get_float64("max-noise"); - const auto min_time = m_criterion_params.get_float64("min-time"); + const auto timeout = m_walltime_timer.get_duration(); - if (cuda_noise > max_noise) + auto get_param = [this](std::optional ¶m, const std::string &name) { + if (m_criterion_params.has_value(name)) + { + param = m_criterion_params.get_float64(name); + } + }; + + std::optional max_noise; + get_param(max_noise, "max-noise"); + + std::optional min_time; + get_param(max_noise, "min-time"); + + if (max_noise && cuda_noise > *max_noise) { printer.log(nvbench::log_level::warn, fmt::format("Current measurement timed out ({:0.2f}s) " @@ -399,7 +411,7 @@ void measure_cold_base::generate_summaries() "{:0.2f}%)", timeout, cuda_noise * 100, - max_noise * 100)); + *max_noise * 100)); } if (m_total_samples < m_min_samples) { @@ -410,7 +422,7 @@ void measure_cold_base::generate_summaries() m_total_samples, m_min_samples)); } - if (m_total_cuda_time < min_time) + if (min_time && m_total_cuda_time < *min_time) { printer.log(nvbench::log_level::warn, fmt::format("Current measurement timed out ({:0.2f}s) " @@ -418,7 +430,7 @@ void measure_cold_base::generate_summaries() "{:0.2f}s)", timeout, m_total_cuda_time, - min_time)); + *min_time)); } } diff --git a/nvbench/option_parser.cu b/nvbench/option_parser.cu index 0341075..58d3166 100644 --- a/nvbench/option_parser.cu +++ b/nvbench/option_parser.cu @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -82,11 +83,35 @@ std::string_view submatch_to_sv(const sv_submatch &in) // // So we're stuck with materializing a std::string and calling std::stoX(). Ah // well. At least it's not istream. -void parse(std::string_view input, nvbench::int32_t &val) { val = std::stoi(std::string(input)); } +void parse(std::string_view input, nvbench::int32_t &val) +try +{ + val = std::stoi(std::string(input)); +} +catch (const std::exception &) +{ // The default exception messages are not very useful on gcc...it's just "stoi". + NVBENCH_THROW(std::invalid_argument, "Failed to parse int32 value from string '{}'", input); +} -void parse(std::string_view input, nvbench::int64_t &val) { val = std::stoll(std::string(input)); } +void parse(std::string_view input, nvbench::int64_t &val) +try +{ + val = std::stoll(std::string(input)); +} +catch (const std::exception &) +{ + NVBENCH_THROW(std::invalid_argument, "Failed to parse int64 value from string '{}'", input); +} -void parse(std::string_view input, nvbench::float64_t &val) { val = std::stod(std::string(input)); } +void parse(std::string_view input, nvbench::float64_t &val) +try +{ + val = std::stod(std::string(input)); +} +catch (const std::exception &) +{ + NVBENCH_THROW(std::invalid_argument, "Failed to parse float64 value from string '{}'", input); +} void parse(std::string_view input, std::string &val) { val = input; } @@ -727,6 +752,7 @@ void option_parser::enable_run_once() } void option_parser::set_stopping_criterion(const std::string &criterion) +try { // If no active benchmark, save args as global. if (m_benchmarks.empty()) @@ -739,6 +765,13 @@ void option_parser::set_stopping_criterion(const std::string &criterion) benchmark_base &bench = *m_benchmarks.back(); bench.set_stopping_criterion(criterion); } +catch (std::exception &e) +{ + NVBENCH_THROW(std::runtime_error, + "Error handling option `--stopping-criterion {}`:\n{}", + criterion, + e.what()); +} void option_parser::disable_blocking_kernel() { @@ -983,17 +1016,39 @@ void option_parser::update_criterion_prop(const std::string &prop_arg, const nvbench::named_values::type type) try { + const std::string name(prop_arg.begin() + 2, prop_arg.end()); + // If no active benchmark, save args as global. if (m_benchmarks.empty()) { + // Any global params must either belong to the default criterion or follow a + // `--stopping-criterion` arg: + nvbench::criterion_params params; + if (!params.has_value(name) && + std::find(m_global_benchmark_args.cbegin(), + m_global_benchmark_args.cend(), + "--stopping-criterion") == m_global_benchmark_args.cend()) + { + NVBENCH_THROW(std::runtime_error, + "Unrecognized stopping criterion parameter: `{}` for default criterion.", + name); + } + m_global_benchmark_args.push_back(prop_arg); m_global_benchmark_args.push_back(prop_val); return; } - benchmark_base &bench = *m_benchmarks.back(); - nvbench::criterion_params &criterion_params = bench.get_criterion_params(); - std::string name(prop_arg.begin() + 2, prop_arg.end()); + benchmark_base &bench = *m_benchmarks.back(); + + if (!bench.has_criterion_param(name)) + { + NVBENCH_THROW(std::runtime_error, + "Unrecognized stopping criterion parameter: `{}` for `{}`.", + name, + bench.get_stopping_criterion()); + } + if (type == nvbench::named_values::type::float64) { nvbench::float64_t value{}; @@ -1003,21 +1058,21 @@ try { // Specified as percentage, stored as ratio: value /= 100.0; } - criterion_params.set_float64(name, value); + bench.set_criterion_param_float64(name, value); } else if (type == nvbench::named_values::type::int64) { nvbench::int64_t value{}; ::parse(prop_val, value); - criterion_params.set_int64(name, value); + bench.set_criterion_param_int64(name, value); } else if (type == nvbench::named_values::type::string) { - criterion_params.set_string(name, prop_val); + bench.set_criterion_param_string(name, prop_val); } else { - NVBENCH_THROW(std::runtime_error, "Unrecognized property: `{}`", prop_arg); + NVBENCH_THROW(std::runtime_error, "Unrecognized type for property: `{}`", name); } } catch (std::exception &e) diff --git a/testing/option_parser.cu b/testing/option_parser.cu index 4b01a9a..36b8570 100644 --- a/testing/option_parser.cu +++ b/testing/option_parser.cu @@ -1197,26 +1197,262 @@ void test_timeout() void test_stopping_criterion() { - nvbench::option_parser parser; - parser.parse({"--benchmark", - "DummyBench", - "--stopping-criterion", - "entropy", - "--max-angle", - "0.42", - "--min-r2", - "0.6"}); - const auto &states = parser_to_states(parser); + { // Per benchmark criterion + nvbench::option_parser parser; + parser.parse({ + "--benchmark", + "DummyBench", + "--stopping-criterion", + "entropy", + "--max-angle", + "0.42", + "--min-r2", + "0.6", + }); + const auto &states = parser_to_states(parser); - ASSERT(states.size() == 1); - ASSERT(states[0].get_stopping_criterion() == "entropy"); + ASSERT(states.size() == 1); + ASSERT(states[0].get_stopping_criterion() == "entropy"); - const nvbench::criterion_params &criterion_params = states[0].get_criterion_params(); - ASSERT(criterion_params.has_value("max-angle")); - ASSERT(criterion_params.has_value("min-r2")); + const nvbench::criterion_params &criterion_params = states[0].get_criterion_params(); + ASSERT(criterion_params.has_value("max-angle")); + ASSERT(criterion_params.has_value("min-r2")); - ASSERT(criterion_params.get_float64("max-angle") == 0.42); - ASSERT(criterion_params.get_float64("min-r2") == 0.6); + ASSERT(criterion_params.get_float64("max-angle") == 0.42); + ASSERT(criterion_params.get_float64("min-r2") == 0.6); + } + { // Global criterion + nvbench::option_parser parser; + parser.parse({ + "--stopping-criterion", + "entropy", + "--max-angle", + "0.42", + "--min-r2", + "0.6", + "--benchmark", + "DummyBench", + }); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_stopping_criterion() == "entropy"); + + const nvbench::criterion_params &criterion_params = states[0].get_criterion_params(); + ASSERT(criterion_params.has_value("max-angle")); + ASSERT(criterion_params.has_value("min-r2")); + + ASSERT(criterion_params.get_float64("max-angle") == 0.42); + ASSERT(criterion_params.get_float64("min-r2") == 0.6); + } + { // Global criterion, per-benchmark params + nvbench::option_parser parser; + parser.parse({ + "--stopping-criterion", + "entropy", + "--benchmark", + "DummyBench", + "--max-angle", + "0.42", + "--min-r2", + "0.6", + }); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_stopping_criterion() == "entropy"); + + const nvbench::criterion_params &criterion_params = states[0].get_criterion_params(); + ASSERT(criterion_params.has_value("max-angle")); + ASSERT(criterion_params.has_value("min-r2")); + + ASSERT(criterion_params.get_float64("max-angle") == 0.42); + ASSERT(criterion_params.get_float64("min-r2") == 0.6); + } + { // Global params to default criterion should work: + nvbench::option_parser parser; + parser.parse({ + "--max-noise", + "0.5", + "--min-time", + "0.1", + "--benchmark", + "DummyBench", + "--stopping-criterion", + "entropy", + "--max-angle", + "0.42", + "--min-r2", + "0.6", + }); + const auto &states = parser_to_states(parser); + + ASSERT(states.size() == 1); + ASSERT(states[0].get_stopping_criterion() == "entropy"); + + const nvbench::criterion_params &criterion_params = states[0].get_criterion_params(); + ASSERT(criterion_params.has_value("max-angle")); + ASSERT(criterion_params.has_value("min-r2")); + + ASSERT(criterion_params.get_float64("max-angle") == 0.42); + ASSERT(criterion_params.get_float64("min-r2") == 0.6); + } + { // Unknown stopping criterion should throw + bool exception_thrown = false; + try + { + nvbench::option_parser parser; + parser.parse({ + "--benchmark", + "DummyBench", + "--stopping-criterion", + "I_do_not_exist", + }); + } + catch (const std::runtime_error &) + { + exception_thrown = true; + } + ASSERT(exception_thrown); + } + { // Global criterion to non-default params without global --stopping-criterion should throw + bool exception_thrown = false; + try + { + nvbench::option_parser parser; + parser.parse({ + "--max-angle", + "0.42", + "--min-r2", + "0.6", + "--benchmark", + "DummyBench", + "--stopping-criterion", + "entropy", + }); + } + catch (const std::runtime_error &) + { + exception_thrown = true; + } + ASSERT(exception_thrown); + } + { // Invalid global param throws exception: + bool exception_thrown = false; + try + { + nvbench::option_parser parser; + parser.parse({ + "--max-angle", + "0.42", + "--benchmark", + "DummyBench", + "--stopping-criterion", + "entropy", + "--min-r2", + "0.6", + "--max-angle", + "0.42", + "--benchmark", + "TestBench", + "--stopping-criterion", + "stdrel", + }); + } + catch (const std::runtime_error & /*ex*/) + { + exception_thrown = true; + } + ASSERT(exception_thrown); + } + { // Invalid per-bench param throws exception: + bool exception_thrown = false; + try + { + nvbench::option_parser parser; + parser.parse({ + "--benchmark", + "DummyBench", + "--stopping-criterion", + "entropy", + "--min-r2", + "0.6", + "--max-angle", + "0.42", + "--benchmark", + "TestBench", + "--stopping-criterion", + "stdrel", + "--max-angle", + "0.42", + }); + } + catch (const std::runtime_error & /*ex*/) + { + exception_thrown = true; + } + ASSERT(exception_thrown); + } + { // global param-before-criterion throws exception: + bool exception_thrown = false; + try + { + nvbench::option_parser parser; + parser.parse({ + "--min-r2", // + "0.6", + "--stopping-criterion", + "entropy", + "--benchmark", + "DummyBench", + }); + } + catch (const std::runtime_error & /*ex*/) + { + exception_thrown = true; + } + ASSERT(exception_thrown); + } + { // per-benchmark param-before-criterion throws exception: + bool exception_thrown = false; + try + { + nvbench::option_parser parser; + parser.parse({ + "--benchmark", // + "DummyBench", + "--min-r2", + "0.6", + "--stopping-criterion", + "entropy", + }); + } + catch (const std::runtime_error & /*ex*/) + { + exception_thrown = true; + } + ASSERT(exception_thrown); + } + { // Invalid param type throws exception: + bool exception_thrown = false; + try + { + nvbench::option_parser parser; + parser.parse({ + "--benchmark", // + "DummyBench", + "--stopping-criterion", + "entropy", + "--min-r2", + "\"foo\"", + }); + } + catch (const std::runtime_error &) + { + exception_thrown = true; + } + ASSERT(exception_thrown); + } } int main() @@ -1261,6 +1497,6 @@ try } catch (std::exception &err) { - fmt::print(stderr, "{}", err.what()); + fmt::print(stderr, "Unexpected exception:\n{}\n", err.what()); return 1; }